FTS proof of concept

app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java

@@ -182,6 +182,32 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
         }
 
         int found = 0;
+
+        SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
+        boolean fts = prefs.getBoolean("fts", false);
+        if (fts &&
+                (find == null || !find.startsWith(context.getString(R.string.title_search_special_prefix) + ":"))) {
+            if (state.ids == null) {
+                FtsDbHelper ftsDb = new FtsDbHelper(context);
+                state.ids = ftsDb.match(query);
+            }
+
+            try {
+                db.beginTransaction();
+
+                for (; state.index < state.ids.size() && found < pageSize && !state.destroyed; state.index++) {
+                    found++;
+                    db.message().setMessageFound(state.ids.get(state.index));
+                }
+                db.setTransactionSuccessful();
+
+            } finally {
+                db.endTransaction();
+            }
+
+            return found;
+        }
+
         try {
             db.beginTransaction();
 
@@ -546,6 +572,7 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
         boolean error = false;
         int index = 0;
         int offset = 0;
+        List<Long> ids = null;
         List<TupleMatch> matches = null;
 
         MailService iservice = null;

app/src/main/java/eu/faircode/email/Core.java

@@ -2240,6 +2240,12 @@ class Core {
                     Log.i(folder.name + " inline downloaded message id=" + message.id +
                             " size=" + message.size + "/" + (body == null ? null : body.length()));
 
+                    boolean fts = prefs.getBoolean("fts", false);
+                    if (fts) {
+                        FtsDbHelper ftsDb = new FtsDbHelper(context);
+                        ftsDb.insert(message, HtmlHelper.getText(body));
+                    }
+
                     Long size = parts.getBodySize();
                     if (TextUtils.isEmpty(body) && size != null && size > 0)
                         reportEmptyMessage(context, account, istore);
@@ -2567,6 +2573,12 @@ class Core {
                     Log.i(folder.name + " downloaded message id=" + message.id +
                             " size=" + message.size + "/" + (body == null ? null : body.length()));
 
+                    boolean fts = prefs.getBoolean("fts", false);
+                    if (fts) {
+                        FtsDbHelper ftsDb = new FtsDbHelper(context);
+                        ftsDb.insert(message, HtmlHelper.getText(body));
+                    }
+
                     Long size = parts.getBodySize();
                     if (TextUtils.isEmpty(body) && size != null && size > 0)
                         reportEmptyMessage(context, account, istore);

app/src/main/java/eu/faircode/email/FragmentOptionsMisc.java

@@ -47,6 +47,7 @@ import androidx.preference.PreferenceManager;
 
 public class FragmentOptionsMisc extends FragmentBase implements SharedPreferences.OnSharedPreferenceChangeListener {
     private SwitchCompat swExternalSearch;
+    private SwitchCompat swFts;
     private SwitchCompat swEnglish;
     private SwitchCompat swWatchdog;
     private SwitchCompat swUpdates;
@@ -67,7 +68,7 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
     private Group grpDebug;
 
     private final static String[] RESET_OPTIONS = new String[]{
-            "english", "watchdog", "updates", "experiments", "crash_reports", "debug"
+            "fts", "english", "watchdog", "updates", "experiments", "crash_reports", "debug"
     };
 
     private final static String[] RESET_QUESTIONS = new String[]{
@@ -88,6 +89,7 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
         // Get controls
 
         swExternalSearch = view.findViewById(R.id.swExternalSearch);
+        swFts = view.findViewById(R.id.swFts);
         swEnglish = view.findViewById(R.id.swEnglish);
         swWatchdog = view.findViewById(R.id.swWatchdog);
         swUpdates = view.findViewById(R.id.swUpdates);
@@ -126,6 +128,13 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
             }
         });
 
+        swFts.setOnCheckedChangeListener(new CompoundButton.OnCheckedChangeListener() {
+            @Override
+            public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
+                prefs.edit().putBoolean("fts", checked).apply();
+            }
+        });
+
         swEnglish.setOnCheckedChangeListener(new CompoundButton.OnCheckedChangeListener() {
             @Override
             public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
@@ -298,6 +307,7 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
         int state = pm.getComponentEnabledSetting(new ComponentName(getContext(), ActivitySearch.class));
 
         swExternalSearch.setChecked(state != PackageManager.COMPONENT_ENABLED_STATE_DISABLED);
+        swFts.setChecked(prefs.getBoolean("fts", false));
         swEnglish.setChecked(prefs.getBoolean("english", false));
         swWatchdog.setChecked(prefs.getBoolean("watchdog", true));
         swUpdates.setChecked(prefs.getBoolean("updates", true));

app/src/main/java/eu/faircode/email/FtsDbHelper.java

@@ -0,0 +1,103 @@
+package eu.faircode.email;
+
+/*
+    This file is part of FairEmail.
+
+    FairEmail is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    FairEmail is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with FairEmail.  If not, see <http://www.gnu.org/licenses/>.
+
+    Copyright 2018-2020 by Marcel Bokhorst (M66B)
+*/
+
+import android.content.ContentValues;
+import android.content.Context;
+import android.database.Cursor;
+import android.text.TextUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import javax.mail.Address;
+
+import io.requery.android.database.sqlite.SQLiteDatabase;
+import io.requery.android.database.sqlite.SQLiteOpenHelper;
+
+public class FtsDbHelper extends SQLiteOpenHelper {
+    // https://www.sqlite.org/fts3.html
+    private static final int DATABASE_VERSION = 1;
+    private static final String DATABASE_NAME = "fts.db";
+
+    public FtsDbHelper(Context context) {
+        super(context, DATABASE_NAME, null, DATABASE_VERSION);
+    }
+
+    @Override
+    public void onCreate(SQLiteDatabase db) {
+        Log.i("FTS create");
+        db.execSQL("CREATE VIRTUAL TABLE `message` USING fts4(`time`, `address`, `subject`, `keyword`, `text`)");
+    }
+
+    @Override
+    public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
+        // Do nothing
+    }
+
+    void insert(EntityMessage message, String text) {
+        Log.i("FTS insert id=" + message.id + " subject=" + message.subject + " text=" + text);
+        List<Address> address = new ArrayList<>();
+        if (message.from != null)
+            address.addAll(Arrays.asList(message.from));
+        if (message.to != null)
+            address.addAll(Arrays.asList(message.to));
+        if (message.cc != null)
+            address.addAll(Arrays.asList(message.cc));
+
+        try (SQLiteDatabase db = getWritableDatabase()) {
+            try {
+                db.beginTransaction();
+
+                db.delete("message", "docid = ?", new Object[]{message.id});
+
+                ContentValues cv = new ContentValues();
+                cv.put("docid", message.id);
+                cv.put("time", message.received);
+                cv.put("address", MessageHelper.formatAddresses(address.toArray(new Address[0]), true, false));
+                cv.put("subject", message.subject == null ? "" : message.subject);
+                cv.put("keyword", TextUtils.join(", ", message.keywords));
+                cv.put("text", text);
+                db.insert("message", SQLiteDatabase.CONFLICT_FAIL, cv);
+
+                db.setTransactionSuccessful();
+            } finally {
+                db.endTransaction();
+            }
+        }
+    }
+
+    List<Long> match(String search) {
+        Log.i("FTS search=" + search);
+        List<Long> result = new ArrayList<>();
+        try (SQLiteDatabase db = getReadableDatabase()) {
+            try (Cursor cursor = db.query(
+                    "message", new String[]{"docid"},
+                    "message MATCH ?", new Object[]{search},
+                    null, null, "time DESC", null)) {
+                while (cursor != null && cursor.moveToNext())
+                    result.add(cursor.getLong(0));
+            }
+        }
+        Log.i("FTS result=" + result.size());
+        return result;
+    }
+}

app/src/main/res/layout/fragment_options_misc.xml

@@ -24,6 +24,29 @@
             app:layout_constraintTop_toTopOf="parent"
             app:switchPadding="12dp" />
 
+        <androidx.appcompat.widget.SwitchCompat
+            android:id="@+id/swFts"
+            android:layout_width="0dp"
+            android:layout_height="wrap_content"
+            android:layout_marginTop="12dp"
+            android:text="@string/title_advanced_fts"
+            app:layout_constraintEnd_toEndOf="parent"
+            app:layout_constraintStart_toStartOf="parent"
+            app:layout_constraintTop_toBottomOf="@id/swExternalSearch"
+            app:switchPadding="12dp" />
+
+        <TextView
+            android:id="@+id/tvFtsHint"
+            android:layout_width="0dp"
+            android:layout_height="wrap_content"
+            android:layout_marginEnd="48dp"
+            android:text="@string/title_advanced_fts_hint"
+            android:textAppearance="@style/TextAppearance.AppCompat.Small"
+            android:textStyle="italic"
+            app:layout_constraintEnd_toEndOf="parent"
+            app:layout_constraintStart_toStartOf="parent"
+            app:layout_constraintTop_toBottomOf="@id/swFts" />
+
         <androidx.appcompat.widget.SwitchCompat
             android:id="@+id/swEnglish"
             android:layout_width="0dp"
@@ -32,7 +55,7 @@
             android:text="@string/title_advanced_english"
             app:layout_constraintEnd_toEndOf="parent"
             app:layout_constraintStart_toStartOf="parent"
-            app:layout_constraintTop_toBottomOf="@id/swExternalSearch"
+            app:layout_constraintTop_toBottomOf="@id/tvFtsHint"
             app:switchPadding="12dp" />
 
         <TextView

app/src/main/res/values/strings.xml

@@ -382,6 +382,7 @@
     <string name="title_advanced_aes_key_size" translatable="false">Max AES key size: %1$d</string>
 
     <string name="title_advanced_external_search">Allow other apps to search in messages</string>
+    <string name="title_advanced_fts">Full text search</string>
     <string name="title_advanced_english">Force English language</string>
     <string name="title_advanced_watchdog">Periodically check if FairEmail is still active</string>
     <string name="title_advanced_updates">Check for updates</string>
@@ -403,6 +404,7 @@
     <string name="title_advanced_unseen_hint">Some providers don\'t support this properly, which may cause synchronizing none or all messages</string>
     <string name="title_advanced_sync_kept_hint">This will transfer extra data and consume extra battery power, especially if a lot of messages are stored on the device</string>
     <string name="title_advanced_sync_folders_hint">Disabling this will reduce data and battery usage somewhat, but will disable updating the list of folders too</string>
+    <string name="title_advanced_subscribed_only_hint">Enabling this will delete all local folders without subscription</string>
     <string name="title_advanced_sync_delay_hint">This will slow down synchronizing messages</string>
 
     <string name="title_advanced_suggest_local_hint">In addition to contacts provided by Android. Contact data will be stored for newly sent or received messages only when enabled.</string>
@@ -443,7 +445,7 @@
     <string name="title_advanced_notify_no_grouping">This Android version does not support notification grouping</string>
     <string name="title_advanced_notify_no_channels">This Android version does not support notification channels</string>
 
-    <string name="title_advanced_subscribed_only_hint">Enabling this will delete all local folders without subscription</string>
+    <string name="title_advanced_fts_hint">Enabling this will increase search performance, but also increase battery usage and uses more storage space</string>
     <string name="title_advanced_english_hint">This will restart the app</string>
     <string name="title_advanced_experiments_hint">List of current experimental features</string>
     <string name="title_advanced_debug_hint">Enable extra logging and show debug information at various places</string>