diff --git a/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java b/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java
index b979a67e27..853dfc7c4b 100644
--- a/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java
+++ b/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java
@@ -282,8 +282,8 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback.
+
+ Copyright 2018-2022 by Marcel Bokhorst (M66B)
+*/
+
+import android.annotation.SuppressLint;
+import android.content.ContentValues;
+import android.content.Context;
+import android.database.Cursor;
+import android.text.TextUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import javax.mail.Address;
+
+import io.requery.android.database.sqlite.SQLiteDatabase;
+import io.requery.android.database.sqlite.SQLiteOpenHelper;
+
+// https://www.sqlite.org/fts3.html
+// fts4 requires sqlite version 3.7.4, API 21 Android
+public class Fts4DbHelper extends SQLiteOpenHelper {
+ private Context context;
+
+ @SuppressLint("StaticFieldLeak")
+ private static Fts4DbHelper instance = null;
+
+ private static final int DATABASE_VERSION = 1;
+ private static final String DATABASE_NAME = "fts4.db";
+
+ private Fts4DbHelper(Context context) {
+ super(context.getApplicationContext(), DATABASE_NAME, null, DATABASE_VERSION);
+ this.context = context.getApplicationContext();
+ }
+
+ static SQLiteDatabase getInstance(Context context) {
+ if (instance == null) {
+ if (!context.getDatabasePath(DATABASE_NAME).exists()) {
+ Fts5DbHelper.delete(context);
+ DB.getInstance(context).message().resetFts();
+ }
+ instance = new Fts4DbHelper(context);
+ }
+ return instance.getWritableDatabase();
+ }
+
+ @Override
+ public void onCreate(SQLiteDatabase db) {
+ Log.i("FTS create");
+ db.execSQL("CREATE VIRTUAL TABLE `message`" +
+ " USING fts4" +
+ " (`account`" +
+ ", `folder`" +
+ ", `time`" +
+ ", `address`" +
+ ", `subject`" +
+ ", `keyword`" +
+ ", `text`" +
+ ", `notes`" +
+ ", notindexed=`account`" +
+ ", notindexed=`folder`" +
+ ", notindexed=`time`" +
+ ", tokenize=unicode61 \"remove_diacritics=2\")");
+ // https://www.sqlite.org/fts3.html#tokenizer
+ // https://unicode.org/reports/tr29/
+
+ // https://www.sqlite.org/fts3.html#fts4aux
+ db.execSQL("CREATE VIRTUAL TABLE message_terms USING fts4aux('message');");
+ }
+
+ @Override
+ public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
+ Log.i("FTS upgrade from " + oldVersion + " to " + newVersion);
+
+ db.execSQL("DROP TABLE IF EXISTS `message`");
+ db.execSQL("DROP TABLE IF EXISTS `message_terms`");
+
+ onCreate(db);
+
+ DB.getInstance(context).message().resetFts();
+ }
+
+ static void insert(SQLiteDatabase db, EntityMessage message, String text) {
+ Log.i("FTS insert id=" + message.id);
+ List address = new ArrayList<>();
+ if (message.from != null)
+ address.addAll(Arrays.asList(message.from));
+ if (message.to != null)
+ address.addAll(Arrays.asList(message.to));
+ if (message.cc != null)
+ address.addAll(Arrays.asList(message.cc));
+ if (message.bcc != null)
+ address.addAll(Arrays.asList(message.bcc));
+
+ delete(db, message.id);
+
+ ContentValues cv = new ContentValues();
+ cv.put("rowid", message.id);
+ cv.put("account", message.account);
+ cv.put("folder", message.folder);
+ cv.put("time", message.received);
+ cv.put("address", MessageHelper.formatAddresses(address.toArray(new Address[0]), true, false));
+ cv.put("subject", message.subject == null ? "" : message.subject);
+ cv.put("keyword", TextUtils.join(", ", message.keywords));
+ cv.put("text", text);
+ cv.put("notes", message.notes);
+ db.insert("message", SQLiteDatabase.CONFLICT_FAIL, cv);
+ }
+
+ static void delete(SQLiteDatabase db) {
+ db.delete("message", null, null);
+ }
+
+ static void delete(SQLiteDatabase db, long id) {
+ db.delete("message", "rowid = ?", new Object[]{id});
+ }
+
+ static List getSuggestions(SQLiteDatabase db, String query, int max) {
+ List result = new ArrayList<>();
+
+ try (Cursor cursor = db.query(
+ "SELECT term FROM message_terms" +
+ " WHERE term LIKE ?" +
+ " ORDER BY occurrences DESC" +
+ " LIMIT " + max,
+ new Object[]{query})) {
+ while (cursor != null && cursor.moveToNext())
+ result.add(cursor.getString(0));
+ }
+
+ return result;
+ }
+
+ static List match(
+ SQLiteDatabase db,
+ Long account, Long folder, long[] exclude,
+ BoundaryCallbackMessages.SearchCriteria criteria) {
+
+ List word = new ArrayList<>();
+ List plus = new ArrayList<>();
+ List minus = new ArrayList<>();
+ List opt = new ArrayList<>();
+ StringBuilder all = new StringBuilder();
+ for (String w : criteria.query.trim().split("\\s+")) {
+ if (all.length() > 0)
+ all.append(' ');
+
+ if (w.length() > 1 && w.startsWith("+")) {
+ plus.add(w.substring(1));
+ all.append(w.substring(1));
+ } else if (w.length() > 1 && w.startsWith("-")) {
+ minus.add(w.substring(1));
+ all.append(w.substring(1));
+ } else if (w.length() > 1 && w.startsWith("?")) {
+ opt.add(w.substring(1));
+ all.append(w.substring(1));
+ } else {
+ word.add(w);
+ all.append(w);
+ }
+ }
+
+ StringBuilder sb = new StringBuilder();
+ if (plus.size() + minus.size() + opt.size() > 0) {
+ if (word.size() > 0)
+ sb.append(escape(TextUtils.join(" ", word)));
+
+ for (String p : plus) {
+ if (sb.length() > 0)
+ sb.append(" AND ");
+ sb.append(escape(p));
+ }
+
+ for (String m : minus) {
+ if (sb.length() > 0)
+ sb.append(" NOT ");
+ sb.append(escape(m));
+ }
+
+ if (sb.length() > 0) {
+ sb.insert(0, '(');
+ sb.append(')');
+ }
+
+ for (String o : opt) {
+ if (sb.length() > 0)
+ sb.append(" OR ");
+ sb.append(escape(o));
+ }
+ }
+
+ String search = (sb.length() > 0 ? sb.toString() : escape(criteria.query));
+
+ String select = "";
+ if (account != null)
+ select += "account = " + account + " AND ";
+ if (folder != null)
+ select += "folder = " + folder + " AND ";
+ if (exclude.length > 0) {
+ select += "NOT folder IN (";
+ for (int i = 0; i < exclude.length; i++) {
+ if (i > 0)
+ select += ", ";
+ select += exclude[i];
+ }
+ select += ") AND ";
+ }
+ if (criteria.after != null)
+ select += "time > " + criteria.after + " AND ";
+ if (criteria.before != null)
+ select += "time < " + criteria.before + " AND ";
+
+ Log.i("FTS select=" + select + " search=" + search);
+ List result = new ArrayList<>();
+ try (Cursor cursor = db.query(
+ "message", new String[]{"rowid"},
+ select + "message MATCH ?",
+ new Object[]{search},
+ null, null, "time DESC", null)) {
+ while (cursor != null && cursor.moveToNext())
+ result.add(cursor.getLong(0));
+ }
+ Log.i("FTS result=" + result.size());
+ return result;
+ }
+
+ private static String escape(String word) {
+ return "\"" + word.replaceAll("\"", "\"\"") + "\"";
+ }
+
+ static Cursor getIds(SQLiteDatabase db) {
+ return db.query(
+ "message", new String[]{"rowid"},
+ null, null,
+ null, null, "time");
+ }
+
+ static long size(Context context) {
+ return context.getDatabasePath(DATABASE_NAME).length();
+ }
+
+ static void optimize(SQLiteDatabase db) {
+ Log.i("FTS optimize");
+ db.execSQL("INSERT INTO message (message) VALUES ('optimize')");
+ }
+
+ static void delete(Context context) {
+ context.getDatabasePath(DATABASE_NAME).delete();
+ }
+}
diff --git a/app/src/main/java/eu/faircode/email/FtsDbHelper.java b/app/src/main/java/eu/faircode/email/Fts5DbHelper.java
similarity index 97%
rename from app/src/main/java/eu/faircode/email/FtsDbHelper.java
rename to app/src/main/java/eu/faircode/email/Fts5DbHelper.java
index 9417ff15b3..16f8b76c50 100644
--- a/app/src/main/java/eu/faircode/email/FtsDbHelper.java
+++ b/app/src/main/java/eu/faircode/email/Fts5DbHelper.java
@@ -35,23 +35,23 @@ import io.requery.android.database.sqlite.SQLiteDatabase;
import io.requery.android.database.sqlite.SQLiteOpenHelper;
// https://www.sqlite.org/fts5.html
-public class FtsDbHelper extends SQLiteOpenHelper {
+public class Fts5DbHelper extends SQLiteOpenHelper {
private Context context;
@SuppressLint("StaticFieldLeak")
- private static FtsDbHelper instance = null;
+ private static Fts5DbHelper instance = null;
private static final int DATABASE_VERSION = 5;
private static final String DATABASE_NAME = "fts.db";
- private FtsDbHelper(Context context) {
+ private Fts5DbHelper(Context context) {
super(context.getApplicationContext(), DATABASE_NAME, null, DATABASE_VERSION);
this.context = context.getApplicationContext();
}
static SQLiteDatabase getInstance(Context context) {
if (instance == null)
- instance = new FtsDbHelper(context);
+ instance = new Fts5DbHelper(context);
return instance.getWritableDatabase();
}
diff --git a/app/src/main/java/eu/faircode/email/WorkerCleanup.java b/app/src/main/java/eu/faircode/email/WorkerCleanup.java
index bcb6c0bc7f..4deb2b5f9f 100644
--- a/app/src/main/java/eu/faircode/email/WorkerCleanup.java
+++ b/app/src/main/java/eu/faircode/email/WorkerCleanup.java
@@ -334,21 +334,21 @@ public class WorkerCleanup extends Worker {
Log.i("Cleanup FTS=" + fts);
if (fts) {
int deleted = 0;
- SQLiteDatabase sdb = FtsDbHelper.getInstance(context);
- try (Cursor cursor = FtsDbHelper.getIds(sdb)) {
+ SQLiteDatabase sdb = Fts4DbHelper.getInstance(context);
+ try (Cursor cursor = Fts4DbHelper.getIds(sdb)) {
while (cursor.moveToNext()) {
long rowid = cursor.getLong(0);
EntityMessage message = db.message().getMessage(rowid);
if (message == null || !message.fts) {
Log.i("Deleting FTS rowid=" + rowid);
- FtsDbHelper.delete(sdb, rowid);
+ Fts4DbHelper.delete(sdb, rowid);
deleted++;
}
}
}
Log.i("Cleanup FTS=" + deleted);
if (manual)
- FtsDbHelper.optimize(sdb);
+ Fts4DbHelper.optimize(sdb);
}
Log.i("Cleanup contacts");
diff --git a/app/src/main/java/eu/faircode/email/WorkerFts.java b/app/src/main/java/eu/faircode/email/WorkerFts.java
index d2e6c193b0..3020841c89 100644
--- a/app/src/main/java/eu/faircode/email/WorkerFts.java
+++ b/app/src/main/java/eu/faircode/email/WorkerFts.java
@@ -65,7 +65,7 @@ public class WorkerFts extends Worker {
List ids = new ArrayList<>(INDEX_BATCH_SIZE);
DB db = DB.getInstance(context);
- SQLiteDatabase sdb = FtsDbHelper.getInstance(context);
+ SQLiteDatabase sdb = Fts4DbHelper.getInstance(context);
try (Cursor cursor = db.message().getMessageFts()) {
while (cursor != null && cursor.moveToNext())
@@ -92,7 +92,7 @@ public class WorkerFts extends Worker {
try {
sdb.beginTransaction();
- FtsDbHelper.insert(sdb, message, text);
+ Fts4DbHelper.insert(sdb, message, text);
sdb.setTransactionSuccessful();
} finally {
sdb.endTransaction();