Switched to FTS5

FAQ.md

@@ -691,6 +691,11 @@ Searching through a large number of messages is not very fast because of two lim
 This means that searching for a message text requires that files containing the message texts need to be opened one by one
 to check if the searched text is contained in the file, which is a relative expensive process.
 
+In the *miscellaneous settings* you can enable *Full text search* to significantly increase the speed of searching on the device,
+but be aware that this will increase battery usage and significantly increase storage space usage too.
+An advantage is that a number of advanced query options will become available,
+see [here](https://www.sqlite.org/fts5.html#full_text_query_syntax).
+
 Searching messages on the device is a free feature, searching messages on the server is a pro feature.
 
 <br />

app/src/main/java/eu/faircode/email/DaoMessage.java

@@ -238,7 +238,7 @@ public interface DaoMessage {
             " WHERE content" +
             " AND NOT fts" +
             " AND folder.type <> '" + EntityFolder.OUTBOX + "'" +
-            " ORDER BY message.received DESC")
+            " ORDER BY message.received")
     Cursor getMessageFts();
 
     @Query("SELECT id, account, thread, (:find IS NULL" +

app/src/main/java/eu/faircode/email/FtsDbHelper.java

@@ -33,8 +33,8 @@ import javax.mail.Address;
 import io.requery.android.database.sqlite.SQLiteDatabase;
 import io.requery.android.database.sqlite.SQLiteOpenHelper;
 
+// https://www.sqlite.org/fts5.html
 public class FtsDbHelper extends SQLiteOpenHelper {
-    // https://www.sqlite.org/fts3.html
     private static final int DATABASE_VERSION = 1;
     private static final String DATABASE_NAME = "fts.db";
 
@@ -46,7 +46,7 @@ public class FtsDbHelper extends SQLiteOpenHelper {
     public void onCreate(SQLiteDatabase db) {
         Log.i("FTS create");
         db.execSQL("CREATE VIRTUAL TABLE `message`" +
-                " USING fts4(`folder`, `time`, `address`, `subject`, `keyword`, `text`)");
+                " USING fts5 (`folder` UNINDEXED, `time` UNINDEXED, `address`, `subject`, `keyword`, `text`)");
     }
 
     @Override
@@ -70,7 +70,7 @@ public class FtsDbHelper extends SQLiteOpenHelper {
             delete(db, message.id);
 
             ContentValues cv = new ContentValues();
-            cv.put("docid", message.id);
+            cv.put("rowid", message.id);
             cv.put("folder", message.folder);
             cv.put("time", message.received);
             cv.put("address", MessageHelper.formatAddresses(address.toArray(new Address[0]), true, false));
@@ -88,14 +88,14 @@ public class FtsDbHelper extends SQLiteOpenHelper {
     }
 
     void delete(SQLiteDatabase db, long id) {
-        db.delete("message", "docid = ?", new Object[]{id});
+        db.delete("message", "rowid = ?", new Object[]{id});
     }
 
     List<Long> match(SQLiteDatabase db, Long folder, String search) {
         Log.i("FTS folder=" + folder + " search=" + search);
         List<Long> result = new ArrayList<>();
         try (Cursor cursor = db.query(
-                "message", new String[]{"docid"},
+                "message", new String[]{"rowid"},
                 folder == null ? "message MATCH ?" : "folder = ? AND message MATCH ?",
                 folder == null ? new Object[]{search} : new Object[]{folder, search},
                 null, null, "time DESC", null)) {
@@ -108,7 +108,7 @@ public class FtsDbHelper extends SQLiteOpenHelper {
 
     Cursor getIds(SQLiteDatabase db) {
         return db.query(
-                "message", new String[]{"docid"},
+                "message", new String[]{"rowid"},
                 null, null,
                 null, null, "time");
     }

app/src/main/java/eu/faircode/email/WorkerCleanup.java

@@ -199,11 +199,11 @@ public class WorkerCleanup extends Worker {
             try (SQLiteDatabase sdb = ftsDb.getWritableDatabase()) {
                 try (Cursor cursor = ftsDb.getIds(sdb)) {
                     while (cursor.moveToNext()) {
-                        long docid = cursor.getLong(0);
+                        long rowid = cursor.getLong(0);
-                        EntityMessage message = db.message().getMessage(docid);
+                        EntityMessage message = db.message().getMessage(rowid);
                         if (message == null) {
-                            Log.i("Deleting docid" + docid);
+                            Log.i("Deleting rowid" + rowid);
-                            ftsDb.delete(sdb, docid);
+                            ftsDb.delete(sdb, rowid);
                             fts++;
                         }
                     }

app/src/main/java/eu/faircode/email/WorkerFts.java

@@ -36,6 +36,8 @@ import java.util.concurrent.TimeUnit;
 
 import io.requery.android.database.sqlite.SQLiteDatabase;
 
+import static android.os.Process.THREAD_PRIORITY_BACKGROUND;
+
 public class WorkerFts extends Worker {
     private static final int INDEX_DELAY = 30; // seconds
 
@@ -47,6 +49,8 @@ public class WorkerFts extends Worker {
     @NonNull
     @Override
     public Result doWork() {
+        Thread.currentThread().setPriority(THREAD_PRIORITY_BACKGROUND);
+
         try {
             Log.i("FTS index");
 

app/src/main/res/values/strings.xml

@@ -447,7 +447,7 @@
     <string name="title_advanced_notify_no_grouping">This Android version does not support notification grouping</string>
     <string name="title_advanced_notify_no_channels">This Android version does not support notification channels</string>
 
-    <string name="title_advanced_fts_hint">Enabling this improves search performance, but also increases battery and storage usage</string>
+    <string name="title_advanced_fts_hint">Enabling this improves search performance, but also increases battery and storage space usage</string>
     <string name="title_advanced_english_hint">This will restart the app</string>
     <string name="title_advanced_experiments_hint">List of current experimental features</string>
     <string name="title_advanced_debug_hint">Enable extra logging and show debug information at various places</string>