Limit total number of operation retries

app/src/main/java/eu/faircode/email/AdapterOperation.java

@@ -95,7 +95,10 @@ public class AdapterOperation extends RecyclerView.Adapter<AdapterOperation.View
             view.setAlpha(operation.synchronize ? 1.0f : Helper.LOW_LIGHT);
 
             StringBuilder sb = new StringBuilder();
-            sb.append(operation.name).append(':').append(operation.priority);
+            sb
+                    .append(operation.name).append(':')
+                    .append(operation.priority).append("/")
+                    .append(operation.tries);
             try {
                 JSONArray jarray = new JSONArray(operation.args);
                 if (jarray.length() > 0)

app/src/main/java/eu/faircode/email/Core.java

@@ -128,8 +128,9 @@ class Core {
     private static final int DOWNLOAD_BATCH_SIZE = 20;
     private static final long YIELD_DURATION = 200L; // milliseconds
     private static final long FUTURE_RECEIVED = 30 * 24 * 3600 * 1000L; // milliseconds
-    private static final int OPERATION_RETRY_MAX = 3;
-    private static final long OPERATION_RETRY_DELAY = 10 * 1000L; // milliseconds
+    private static final int LOCAL_RETRY_MAX = 3;
+    private static final long LOCAL_RETRY_DELAY = 10 * 1000L; // milliseconds
+    private static final int TOTAL_RETRY_MAX = LOCAL_RETRY_MAX * 10;
 
     static void processOperations(
             Context context,
@@ -145,7 +146,7 @@ class Core {
             int retry = 0;
             boolean group = true;
             Log.i(folder.name + " executing operations=" + ops.size());
-            while (retry < OPERATION_RETRY_MAX && ops.size() > 0 &&
+            while (retry < LOCAL_RETRY_MAX && ops.size() > 0 &&
                     state.batchCanRun(folder.id, priority, sequence) &&
                     state.isRunning() && state.isRecoverable()) {
                 TupleOperationEx op = ops.get(0);
@@ -246,6 +247,7 @@ class Core {
                         try {
                             db.beginTransaction();
 
+                            db.operation().setOperationTries(op.id, ++op.tries);
                             db.operation().setOperationError(op.id, null);
 
                             if (message != null)
@@ -387,7 +389,10 @@ class Core {
                             ops.remove(s);
                     } catch (Throwable ex) {
                         Log.e(folder.name, ex);
-                        EntityLog.log(context, folder.name + " " + Log.formatThrowable(ex, false));
+                        EntityLog.log(context, folder.name +
+                                " op=" + op.name + " " +
+                                " try=" + op.tries + " " +
+                                Log.formatThrowable(ex, false));
 
                         if (similar.size() > 0) {
                             // Retry individually
@@ -396,6 +401,13 @@ class Core {
                             continue;
                         }
 
+                        if (op.tries >= TOTAL_RETRY_MAX) {
+                            // Giving up
+                            db.operation().deleteOperation(op.id);
+                            ops.remove(op);
+                            continue;
+                        }
+
                         try {
                             db.beginTransaction();
 
@@ -457,9 +469,9 @@ class Core {
                             ops.remove(op);
                         } else {
                             retry++;
-                            if (retry < OPERATION_RETRY_MAX)
+                            if (retry < LOCAL_RETRY_MAX)
                                 try {
-                                    Thread.sleep(OPERATION_RETRY_DELAY);
+                                    Thread.sleep(LOCAL_RETRY_DELAY);
                                 } catch (InterruptedException ex1) {
                                     Log.w(ex1);
                                 }

app/src/main/java/eu/faircode/email/DB.java

@@ -60,7 +60,7 @@ import io.requery.android.database.sqlite.SQLiteDatabase;
 // https://developer.android.com/topic/libraries/architecture/room.html
 
 @Database(
-        version = 142,
+        version = 143,
         entities = {
                 EntityIdentity.class,
                 EntityAccount.class,
@@ -1372,6 +1372,13 @@ public abstract class DB extends RoomDatabase {
                         db.execSQL("ALTER TABLE `account` ADD COLUMN `certificate_alias` TEXT");
                         db.execSQL("ALTER TABLE `identity` ADD COLUMN `certificate_alias` TEXT");
                     }
+                })
+                .addMigrations(new Migration(142, 143) {
+                    @Override
+                    public void migrate(@NonNull SupportSQLiteDatabase db) {
+                        Log.i("DB migration from version " + startVersion + " to " + endVersion);
+                        db.execSQL("ALTER TABLE `operation` ADD COLUMN `tries` INTEGER NOT NULL DEFAULT 0");
+                    }
                 });
     }
 

app/src/main/java/eu/faircode/email/DaoOperation.java

@@ -121,6 +121,9 @@ public interface DaoOperation {
             " AND name = :name")
     int getOperationCount(long folder, long message, String name);
 
+    @Query("UPDATE operation SET tries = :tries WHERE id = :id")
+    int setOperationTries(long id, int tries);
+
     @Query("UPDATE operation SET state = :state WHERE id = :id")
     int setOperationState(long id, String state);
 

app/src/main/java/eu/faircode/email/EntityOperation.java

@@ -75,6 +75,8 @@ public class EntityOperation {
     public String args;
     @NonNull
     public Long created;
+    @NonNull
+    public int tries = 0;
     public String state;
     public String error;
 

app/src/main/java/eu/faircode/email/Log.java

@@ -1015,10 +1015,11 @@ public class Log {
             DateFormat TF = Helper.getTimeInstance(context);
 
             for (EntityOperation op : db.operation().getOperations())
-                size += write(os, String.format("%s %d %s %s %s\r\n",
+                size += write(os, String.format("%s %d %s/%d %s %s\r\n",
                         TF.format(op.created),
                         op.message == null ? -1 : op.message,
                         op.name,
+                        op.tries,
                         op.args,
                         op.error));
         }