Added FTS index worker

app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java

@@ -65,6 +65,8 @@ import javax.mail.search.RecipientStringTerm;
 import javax.mail.search.SearchTerm;
 import javax.mail.search.SubjectTerm;
 
+import io.requery.android.database.sqlite.SQLiteDatabase;
+
 public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMessageEx> {
     private Context context;
     private Long folder;
@@ -189,7 +191,9 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
                 (find == null || !find.startsWith(context.getString(R.string.title_search_special_prefix) + ":"))) {
             if (state.ids == null) {
                 FtsDbHelper ftsDb = new FtsDbHelper(context);
-                state.ids = ftsDb.match(query);
+                try (SQLiteDatabase sdb = ftsDb.getReadableDatabase()) {
+                    state.ids = ftsDb.match(sdb, folder, query);
+                }
             }
 
             try {

app/src/main/java/eu/faircode/email/Core.java

@@ -959,6 +959,8 @@ class Core {
             int count = ifolder.getMessageCount();
             db.folder().setFolderTotal(folder.id, count < 0 ? null : count);
         }
+
+        WorkerFts.init(context);
     }
 
     private static void onDelete(Context context, JSONArray jargs, EntityFolder folder, EntityMessage message, IMAPFolder ifolder) throws MessagingException {
@@ -1986,6 +1988,8 @@ class Core {
             Log.i(folder.name + " end sync state=" + state);
             db.folder().setFolderSyncState(folder.id, null);
         }
+
+        WorkerFts.init(context);
     }
 
     static EntityMessage synchronizeMessage(
@@ -2240,12 +2244,6 @@ class Core {
                     Log.i(folder.name + " inline downloaded message id=" + message.id +
                             " size=" + message.size + "/" + (body == null ? null : body.length()));
 
-                    boolean fts = prefs.getBoolean("fts", false);
-                    if (fts) {
-                        FtsDbHelper ftsDb = new FtsDbHelper(context);
-                        ftsDb.insert(message, HtmlHelper.getText(body));
-                    }
-
                     Long size = parts.getBodySize();
                     if (TextUtils.isEmpty(body) && size != null && size > 0)
                         reportEmptyMessage(context, account, istore);
@@ -2573,12 +2571,6 @@ class Core {
                     Log.i(folder.name + " downloaded message id=" + message.id +
                             " size=" + message.size + "/" + (body == null ? null : body.length()));
 
-                    boolean fts = prefs.getBoolean("fts", false);
-                    if (fts) {
-                        FtsDbHelper ftsDb = new FtsDbHelper(context);
-                        ftsDb.insert(message, HtmlHelper.getText(body));
-                    }
-
                     Long size = parts.getBodySize();
                     if (TextUtils.isEmpty(body) && size != null && size > 0)
                         reportEmptyMessage(context, account, istore);

app/src/main/java/eu/faircode/email/DB.java

@@ -56,7 +56,7 @@ import io.requery.android.database.sqlite.RequerySQLiteOpenHelperFactory;
 // https://developer.android.com/topic/libraries/architecture/room.html
 
 @Database(
-        version = 129,
+        version = 130,
         entities = {
                 EntityIdentity.class,
                 EntityAccount.class,
@@ -1247,6 +1247,13 @@ public abstract class DB extends RoomDatabase {
                         db.execSQL("ALTER TABLE `account` ADD COLUMN `poll_exempted` INTEGER NOT NULL DEFAULT 0");
                     }
                 })
+                .addMigrations(new Migration(129, 130) {
+                    @Override
+                    public void migrate(@NonNull SupportSQLiteDatabase db) {
+                        Log.i("DB migration from version " + startVersion + " to " + endVersion);
+                        db.execSQL("ALTER TABLE `message` ADD COLUMN `fts` INTEGER NOT NULL DEFAULT 0");
+                    }
+                })
                 .build();
     }
 

app/src/main/java/eu/faircode/email/DaoMessage.java

@@ -227,6 +227,11 @@ public interface DaoMessage {
             " ORDER BY message.received DESC")
     List<Long> getMessageIdsByFolder(Long folder);
 
+    @Query("SELECT id FROM message" +
+            " WHERE content AND NOT fts" +
+            " ORDER BY message.received DESC")
+    Cursor getMessageFts();
+
     @Query("SELECT id, account, thread, (:find IS NULL" +
             " OR `from` LIKE :find COLLATE NOCASE" +
             " OR `to` LIKE :find COLLATE NOCASE" +
@@ -437,6 +442,9 @@ public interface DaoMessage {
     @Query("UPDATE message SET notifying = :notifying WHERE id = :id")
     int setMessageNotifying(long id, int notifying);
 
+    @Query("UPDATE message SET fts = :fts WHERE id = :id")
+    int setMessageFts(long id, boolean fts);
+
     @Query("UPDATE message SET received = :received WHERE id = :id")
     int setMessageReceived(long id, long received);
 

app/src/main/java/eu/faircode/email/EntityMessage.java

@@ -154,6 +154,8 @@ public class EntityMessage implements Serializable {
     @NonNull
     public Integer notifying = 0;
     @NonNull
+    public Boolean fts = false;
+    @NonNull
     public Boolean ui_seen = false;
     @NonNull
     public Boolean ui_answered = false;

app/src/main/java/eu/faircode/email/FragmentOptionsMisc.java

@@ -132,6 +132,7 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
             @Override
             public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
                 prefs.edit().putBoolean("fts", checked).apply();
+                WorkerFts.init(getContext());
             }
         });
 

app/src/main/java/eu/faircode/email/FtsDbHelper.java

@@ -45,7 +45,7 @@ public class FtsDbHelper extends SQLiteOpenHelper {
     @Override
     public void onCreate(SQLiteDatabase db) {
         Log.i("FTS create");
-        db.execSQL("CREATE VIRTUAL TABLE `message` USING fts4(`time`, `address`, `subject`, `keyword`, `text`)");
+        db.execSQL("CREATE VIRTUAL TABLE `message` USING fts4(`folder`, `time`, `address`, `subject`, `keyword`, `text`)");
     }
 
     @Override
@@ -53,7 +53,7 @@ public class FtsDbHelper extends SQLiteOpenHelper {
         // Do nothing
     }
 
-    void insert(EntityMessage message, String text) {
+    void insert(SQLiteDatabase db, EntityMessage message, String text) {
         Log.i("FTS insert id=" + message.id + " subject=" + message.subject + " text=" + text);
         List<Address> address = new ArrayList<>();
         if (message.from != null)
@@ -63,7 +63,6 @@ public class FtsDbHelper extends SQLiteOpenHelper {
         if (message.cc != null)
             address.addAll(Arrays.asList(message.cc));
 
-        try (SQLiteDatabase db = getWritableDatabase()) {
         try {
             db.beginTransaction();
 
@@ -71,6 +70,7 @@ public class FtsDbHelper extends SQLiteOpenHelper {
 
             ContentValues cv = new ContentValues();
             cv.put("docid", message.id);
+            cv.put("folder", message.folder);
             cv.put("time", message.received);
             cv.put("address", MessageHelper.formatAddresses(address.toArray(new Address[0]), true, false));
             cv.put("subject", message.subject == null ? "" : message.subject);
@@ -79,24 +79,24 @@ public class FtsDbHelper extends SQLiteOpenHelper {
             db.insert("message", SQLiteDatabase.CONFLICT_FAIL, cv);
 
             db.setTransactionSuccessful();
+        } catch (Throwable ex) {
+            Log.e(ex);
         } finally {
             db.endTransaction();
         }
     }
-    }
 
-    List<Long> match(String search) {
-        Log.i("FTS search=" + search);
+    List<Long> match(SQLiteDatabase db, Long folder, String search) {
+        Log.i("FTS folder=" + folder + " search=" + search);
         List<Long> result = new ArrayList<>();
-        try (SQLiteDatabase db = getReadableDatabase()) {
         try (Cursor cursor = db.query(
                 "message", new String[]{"docid"},
-                    "message MATCH ?", new Object[]{search},
+                folder == null ? "message MATCH ?" : "folder = ? AND message MATCH ?",
+                folder == null ? new Object[]{search} : new Object[]{folder, search},
                 null, null, "time DESC", null)) {
             while (cursor != null && cursor.moveToNext())
                 result.add(cursor.getLong(0));
         }
-        }
         Log.i("FTS result=" + result.size());
         return result;
     }

app/src/main/java/eu/faircode/email/WorkerFts.java

@@ -0,0 +1,111 @@
+package eu.faircode.email;
+
+/*
+    This file is part of FairEmail.
+
+    FairEmail is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    FairEmail is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with FairEmail.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2018-2020 by Marcel Bokhorst (M66B)
+*/
+
+import android.content.Context;
+import android.content.SharedPreferences;
+import android.database.Cursor;
+
+import androidx.annotation.NonNull;
+import androidx.preference.PreferenceManager;
+import androidx.work.ExistingWorkPolicy;
+import androidx.work.OneTimeWorkRequest;
+import androidx.work.WorkManager;
+import androidx.work.Worker;
+import androidx.work.WorkerParameters;
+
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+
+import io.requery.android.database.sqlite.SQLiteDatabase;
+
+public class WorkerFts extends Worker {
+    public WorkerFts(@NonNull Context context, @NonNull WorkerParameters workerParams) {
+        super(context, workerParams);
+        Log.i("Instance " + getName());
+    }
+
+    @NonNull
+    @Override
+    public Result doWork() {
+        try {
+            Log.i("FTS index");
+
+            int indexed = 0;
+            FtsDbHelper ftsDb = new FtsDbHelper(getApplicationContext());
+            try (SQLiteDatabase sdb = ftsDb.getWritableDatabase()) {
+                DB db = DB.getInstance(getApplicationContext());
+                try (Cursor cursor = db.message().getMessageFts()) {
+                    while (cursor.moveToNext()) {
+                        EntityMessage message = db.message().getMessage(cursor.getLong(0));
+                        if (message != null)
+                            try {
+                                Log.i("FTS index=" + message.id);
+                                File file = message.getFile(getApplicationContext());
+                                String html = Helper.readText(file);
+                                ftsDb.insert(sdb, message, HtmlHelper.getText(html));
+                                db.message().setMessageFts(message.id, true);
+                                indexed++;
+                            } catch (Throwable ex) {
+                                Log.e(ex);
+                            }
+                    }
+                }
+            }
+
+            Log.i("FTS indexed=" + indexed);
+            return Result.success();
+        } catch (Throwable ex) {
+            Log.e(ex);
+            return Result.failure();
+        }
+    }
+
+    static void init(Context context) {
+        try {
+            SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
+            boolean fts = prefs.getBoolean("fts", true);
+            if (fts) {
+                Log.i("Queuing " + getName());
+
+                OneTimeWorkRequest workRequest =
+                        new OneTimeWorkRequest.Builder(WorkerFts.class)
+                                .setInitialDelay(30, TimeUnit.SECONDS)
+                                .build();
+
+                WorkManager.getInstance(context)
+                        .enqueueUniqueWork(getName(), ExistingWorkPolicy.REPLACE, workRequest);
+
+                Log.i("Queued " + getName());
+            } else {
+                Log.i("Cancelling " + getName());
+                WorkManager.getInstance(context).cancelUniqueWork(getName());
+                Log.i("Cancelled " + getName());
+            }
+        } catch (IllegalStateException ex) {
+            // https://issuetracker.google.com/issues/138465476
+            Log.w(ex);
+        }
+    }
+
+    private static String getName() {
+        return WorkerFts.class.getSimpleName();
+    }
+}