FTS improvements

app/src/main/java/eu/faircode/email/Core.java

@@ -960,7 +960,7 @@ class Core {
             db.folder().setFolderTotal(folder.id, count < 0 ? null : count);
         }
 
-        WorkerFts.init(context);
+        WorkerFts.init(context, false);
     }
 
     private static void onDelete(Context context, JSONArray jargs, EntityFolder folder, EntityMessage message, IMAPFolder ifolder) throws MessagingException {
@@ -1989,7 +1989,7 @@ class Core {
             db.folder().setFolderSyncState(folder.id, null);
         }
 
-        WorkerFts.init(context);
+        WorkerFts.init(context, false);
     }
 
     static EntityMessage synchronizeMessage(

app/src/main/java/eu/faircode/email/DaoMessage.java

@@ -202,6 +202,12 @@ public interface DaoMessage {
             " AND ui_hide")
     LiveData<List<Long>> liveHiddenThread(long account, String thread);
 
+    @Query("SELECT SUM(fts) AS fts, COUNT(*) AS total FROM message" +
+            " JOIN folder ON folder.id = message.folder" +
+            " WHERE content" +
+            " AND folder.type <> '" + EntityFolder.OUTBOX + "'")
+    LiveData<TupleFtsStats> liveFts();
+
     @Query("SELECT *" +
             " FROM message" +
             " WHERE id = :id")
@@ -227,8 +233,11 @@ public interface DaoMessage {
             " ORDER BY message.received DESC")
     List<Long> getMessageIdsByFolder(Long folder);
 
-    @Query("SELECT id FROM message" +
+    @Query("SELECT message.id FROM message" +
-            " WHERE content AND NOT fts" +
+            " JOIN folder ON folder.id = message.folder" +
+            " WHERE content" +
+            " AND NOT fts" +
+            " AND folder.type <> '" + EntityFolder.OUTBOX + "'" +
             " ORDER BY message.received DESC")
     Cursor getMessageFts();
 
@@ -541,6 +550,9 @@ public interface DaoMessage {
     @Query("UPDATE message SET headers = NULL WHERE headers IS NOT NULL")
     int clearMessageHeaders();
 
+    @Query("UPDATE message SET fts = 0")
+    int resetFts();
+
     @Query("DELETE FROM message WHERE id = :id")
     int deleteMessage(long id);
 

app/src/main/java/eu/faircode/email/EntityOperation.java

@@ -100,12 +100,16 @@ public class EntityOperation {
             for (Object value : values)
                 jargs.put(value);
 
-            if (ADD.equals(name) &&
+            if (ADD.equals(name)) {
-                    (EntityMessage.PGP_SIGNENCRYPT.equals(message.encrypt) ||
+                db.message().setMessageFts(message.id, false);
-                            EntityMessage.SMIME_SIGNENCRYPT.equals(message.encrypt))) {
+                WorkerFts.init(context, false);
-                EntityFolder folder = db.folder().getFolder(message.folder);
+
-                if (folder != null && EntityFolder.DRAFTS.equals(folder.type))
+                if (EntityMessage.PGP_SIGNENCRYPT.equals(message.encrypt) ||
-                    return;
+                        EntityMessage.SMIME_SIGNENCRYPT.equals(message.encrypt)) {
+                    EntityFolder folder = db.folder().getFolder(message.folder);
+                    if (folder != null && EntityFolder.DRAFTS.equals(folder.type))
+                        return;
+                }
             }
 
             if (MOVE.equals(name) &&
@@ -199,6 +203,7 @@ public class EntityOperation {
                     Long identity = message.identity;
                     long uid = message.uid;
                     int notifying = message.notifying;
+                    boolean fts = message.fts;
                     boolean seen = message.seen;
                     boolean ui_seen = message.ui_seen;
                     Boolean ui_hide = message.ui_hide;
@@ -211,6 +216,7 @@ public class EntityOperation {
                     message.identity = null;
                     message.uid = null;
                     message.notifying = 0;
+                    message.fts = false;
                     if (autoread) {
                         message.seen = true;
                         message.ui_seen = true;
@@ -229,6 +235,7 @@ public class EntityOperation {
                     message.identity = identity;
                     message.uid = uid;
                     message.notifying = notifying;
+                    message.fts = fts;
                     message.seen = seen;
                     message.ui_seen = ui_seen;
                     message.ui_hide = ui_hide;
@@ -245,6 +252,8 @@ public class EntityOperation {
                         }
 
                     EntityAttachment.copy(context, message.id, tmpid);
+
+                    WorkerFts.init(context, false);
                 }
 
                 // Cross account move

app/src/main/java/eu/faircode/email/FragmentMessages.java

@@ -4545,12 +4545,14 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
                                         // Write decrypted body
                                         Helper.copy(plain, message.getFile(context));
                                         db.message().setMessageStored(message.id, new Date().getTime());
+                                        db.message().setMessageFts(message.id, false);
 
                                         db.setTransactionSuccessful();
                                     } finally {
                                         db.endTransaction();
                                     }
 
+                                    WorkerFts.init(context, false);
                                 } else {
                                     // Decode message
                                     MessageHelper.MessageParts parts;
@@ -4588,11 +4590,14 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
 
                                         db.message().setMessageEncrypt(message.id, parts.getEncryption());
                                         db.message().setMessageStored(message.id, new Date().getTime());
+                                        db.message().setMessageFts(message.id, false);
 
                                         db.setTransactionSuccessful();
                                     } finally {
                                         db.endTransaction();
                                     }
+
+                                    WorkerFts.init(context, false);
                                 }
 
                             // Check signature status
@@ -4857,6 +4862,7 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
 
                         db.message().setMessageEncrypt(message.id, parts.getEncryption());
                         db.message().setMessageStored(message.id, new Date().getTime());
+                        db.message().setMessageFts(message.id, false);
 
                         if (alias != null && message.identity != null)
                             db.identity().setIdentitySignKeyAlias(message.identity, alias);
@@ -4865,6 +4871,8 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
                     } finally {
                         db.endTransaction();
                     }
+
+                    WorkerFts.init(context, false);
                 }
 
                 return result;

app/src/main/java/eu/faircode/email/FragmentOptionsMisc.java

@@ -43,11 +43,14 @@ import androidx.annotation.Nullable;
 import androidx.appcompat.widget.SwitchCompat;
 import androidx.constraintlayout.widget.Group;
 import androidx.lifecycle.Lifecycle;
+import androidx.lifecycle.Observer;
 import androidx.preference.PreferenceManager;
 
 public class FragmentOptionsMisc extends FragmentBase implements SharedPreferences.OnSharedPreferenceChangeListener {
     private SwitchCompat swExternalSearch;
     private SwitchCompat swFts;
+    private Button btnFtsReset;
+    private TextView tvFtsIndexed;
     private SwitchCompat swEnglish;
     private SwitchCompat swWatchdog;
     private SwitchCompat swUpdates;
@@ -90,6 +93,8 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
 
         swExternalSearch = view.findViewById(R.id.swExternalSearch);
         swFts = view.findViewById(R.id.swFts);
+        btnFtsReset = view.findViewById(R.id.btnFtsReset);
+        tvFtsIndexed = view.findViewById(R.id.tvFtsIndexed);
         swEnglish = view.findViewById(R.id.swEnglish);
         swWatchdog = view.findViewById(R.id.swWatchdog);
         swUpdates = view.findViewById(R.id.swUpdates);
@@ -132,7 +137,33 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
             @Override
             public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
                 prefs.edit().putBoolean("fts", checked).apply();
-                WorkerFts.init(getContext());
+                WorkerFts.init(getContext(), true);
+            }
+        });
+
+        btnFtsReset.setOnClickListener(new View.OnClickListener() {
+            @Override
+            public void onClick(View v) {
+                Bundle args = new Bundle();
+
+                new SimpleTask<Void>() {
+                    @Override
+                    protected Void onExecute(Context context, Bundle args) throws Throwable {
+                        DB db = DB.getInstance(context);
+                        db.message().resetFts();
+                        return null;
+                    }
+
+                    @Override
+                    protected void onExecuted(Bundle args, Void data) {
+                        WorkerFts.init(getContext(), true);
+                    }
+
+                    @Override
+                    protected void onException(Bundle args, Throwable ex) {
+                        Log.unexpectedError(getParentFragmentManager(), ex);
+                    }
+                }.execute(FragmentOptionsMisc.this, args, "fts:reset");
             }
         });
 
@@ -211,6 +242,19 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
             }
         });
 
+        tvFtsIndexed.setText(null);
+
+        DB db = DB.getInstance(getContext());
+        db.message().liveFts().observe(getViewLifecycleOwner(), new Observer<TupleFtsStats>() {
+            @Override
+            public void onChanged(TupleFtsStats stats) {
+                if (stats == null)
+                    tvFtsIndexed.setText(null);
+                else
+                    tvFtsIndexed.setText(getString(R.string.title_advanced_fts_indexed, stats.fts, stats.total));
+            }
+        });
+
         setLastCleanup(prefs.getLong("last_cleanup", -1));
 
         PreferenceManager.getDefaultSharedPreferences(getContext()).registerOnSharedPreferenceChangeListener(this);

app/src/main/java/eu/faircode/email/FtsDbHelper.java

@@ -38,14 +38,15 @@ public class FtsDbHelper extends SQLiteOpenHelper {
     private static final int DATABASE_VERSION = 1;
     private static final String DATABASE_NAME = "fts.db";
 
-    public FtsDbHelper(Context context) {
+    FtsDbHelper(Context context) {
         super(context, DATABASE_NAME, null, DATABASE_VERSION);
     }
 
     @Override
     public void onCreate(SQLiteDatabase db) {
         Log.i("FTS create");
-        db.execSQL("CREATE VIRTUAL TABLE `message` USING fts4(`folder`, `time`, `address`, `subject`, `keyword`, `text`)");
+        db.execSQL("CREATE VIRTUAL TABLE `message`" +
+                " USING fts4(`folder`, `time`, `address`, `subject`, `keyword`, `text`)");
     }
 
     @Override
@@ -66,7 +67,7 @@ public class FtsDbHelper extends SQLiteOpenHelper {
         try {
             db.beginTransaction();
 
-            db.delete("message", "docid = ?", new Object[]{message.id});
+            delete(db, message.id);
 
             ContentValues cv = new ContentValues();
             cv.put("docid", message.id);
@@ -86,6 +87,10 @@ public class FtsDbHelper extends SQLiteOpenHelper {
         }
     }
 
+    void delete(SQLiteDatabase db, long id) {
+        db.delete("message", "docid = ?", new Object[]{id});
+    }
+
     List<Long> match(SQLiteDatabase db, Long folder, String search) {
         Log.i("FTS folder=" + folder + " search=" + search);
         List<Long> result = new ArrayList<>();
@@ -100,4 +105,11 @@ public class FtsDbHelper extends SQLiteOpenHelper {
         Log.i("FTS result=" + result.size());
         return result;
     }
+
+    Cursor getIds(SQLiteDatabase db) {
+        return db.query(
+                "message", new String[]{"docid"},
+                null, null,
+                null, null, "time");
+    }
 }

app/src/main/java/eu/faircode/email/TupleFtsStats.java

@@ -0,0 +1,25 @@
+package eu.faircode.email;
+
+/*
+    This file is part of FairEmail.
+
+    FairEmail is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    FairEmail is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with FairEmail.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2018-2020 by Marcel Bokhorst (M66B)
+*/
+
+public class TupleFtsStats {
+    public long fts;
+    public long total;
+}

app/src/main/java/eu/faircode/email/WorkerCleanup.java

@@ -21,6 +21,7 @@ package eu.faircode.email;
 
 import android.content.Context;
 import android.content.SharedPreferences;
+import android.database.Cursor;
 
 import androidx.annotation.NonNull;
 import androidx.preference.PreferenceManager;
@@ -39,6 +40,8 @@ import java.util.Date;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import io.requery.android.database.sqlite.SQLiteDatabase;
+
 import static android.os.Process.THREAD_PRIORITY_BACKGROUND;
 
 public class WorkerCleanup extends Worker {
@@ -190,6 +193,24 @@ public class WorkerCleanup extends Worker {
                         }
                     }
 
+            Log.i("Cleanup FTS");
+            int fts = 0;
+            FtsDbHelper ftsDb = new FtsDbHelper(context);
+            try (SQLiteDatabase sdb = ftsDb.getWritableDatabase()) {
+                try (Cursor cursor = ftsDb.getIds(sdb)) {
+                    while (cursor.moveToNext()) {
+                        long docid = cursor.getLong(0);
+                        EntityMessage message = db.message().getMessage(docid);
+                        if (message == null) {
+                            Log.i("Deleting docid" + docid);
+                            ftsDb.delete(sdb, docid);
+                            fts++;
+                        }
+                    }
+                }
+            }
+            Log.i("Cleanup FTS=" + fts);
+
             Log.i("Cleanup contacts");
             int contacts = db.contact().deleteContacts(now - KEEP_CONTACTS_DURATION);
             Log.i("Deleted contacts=" + contacts);

app/src/main/java/eu/faircode/email/WorkerFts.java

@@ -37,6 +37,8 @@ import java.util.concurrent.TimeUnit;
 import io.requery.android.database.sqlite.SQLiteDatabase;
 
 public class WorkerFts extends Worker {
+    private static final int INDEX_DELAY = 30; // seconds
+
     public WorkerFts(@NonNull Context context, @NonNull WorkerParameters workerParams) {
         super(context, workerParams);
         Log.i("Instance " + getName());
@@ -78,23 +80,23 @@ public class WorkerFts extends Worker {
         }
     }
 
-    static void init(Context context) {
+    static void init(Context context, boolean immediately) {
         try {
             SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
             boolean fts = prefs.getBoolean("fts", true);
             if (fts) {
                 Log.i("Queuing " + getName());
 
-                OneTimeWorkRequest workRequest =
+                OneTimeWorkRequest.Builder builder = new OneTimeWorkRequest.Builder(WorkerFts.class);
-                        new OneTimeWorkRequest.Builder(WorkerFts.class)
+                if (!immediately)
-                                .setInitialDelay(30, TimeUnit.SECONDS)
+                    builder.setInitialDelay(INDEX_DELAY, TimeUnit.SECONDS);
-                                .build();
+                OneTimeWorkRequest workRequest = builder.build();
 
                 WorkManager.getInstance(context)
                         .enqueueUniqueWork(getName(), ExistingWorkPolicy.REPLACE, workRequest);
 
                 Log.i("Queued " + getName());
-            } else {
+            } else if (immediately) {
                 Log.i("Cancelling " + getName());
                 WorkManager.getInstance(context).cancelUniqueWork(getName());
                 Log.i("Cancelled " + getName());

app/src/main/res/layout/fragment_options_misc.xml

@@ -47,6 +47,30 @@
             app:layout_constraintStart_toStartOf="parent"
             app:layout_constraintTop_toBottomOf="@id/swFts" />
 
+        <Button
+            android:id="@+id/btnFtsReset"
+            style="?android:attr/buttonStyleSmall"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:layout_marginTop="12dp"
+            android:minWidth="0dp"
+            android:minHeight="0dp"
+            android:text="@string/title_reset"
+            app:layout_constraintStart_toStartOf="parent"
+            app:layout_constraintTop_toBottomOf="@id/tvFtsHint" />
+
+        <TextView
+            android:id="@+id/tvFtsIndexed"
+            android:layout_width="0dp"
+            android:layout_height="wrap_content"
+            android:layout_marginEnd="48dp"
+            android:text="@string/title_advanced_fts_indexed"
+            android:textAppearance="@style/TextAppearance.AppCompat.Small"
+            android:textStyle="italic"
+            app:layout_constraintEnd_toEndOf="parent"
+            app:layout_constraintStart_toStartOf="parent"
+            app:layout_constraintTop_toBottomOf="@id/btnFtsReset" />
+
         <androidx.appcompat.widget.SwitchCompat
             android:id="@+id/swEnglish"
             android:layout_width="0dp"
@@ -55,7 +79,7 @@
             android:text="@string/title_advanced_english"
             app:layout_constraintEnd_toEndOf="parent"
             app:layout_constraintStart_toStartOf="parent"
-            app:layout_constraintTop_toBottomOf="@id/tvFtsHint"
+            app:layout_constraintTop_toBottomOf="@id/tvFtsIndexed"
             app:switchPadding="12dp" />
 
         <TextView

app/src/main/res/values/strings.xml

@@ -383,6 +383,7 @@
 
     <string name="title_advanced_external_search">Allow other apps to search in messages</string>
     <string name="title_advanced_fts">Full text search</string>
+    <string name="title_advanced_fts_indexed">%1$d/%2$d indexed</string>
     <string name="title_advanced_english">Force English language</string>
     <string name="title_advanced_watchdog">Periodically check if FairEmail is still active</string>
     <string name="title_advanced_updates">Check for updates</string>