From 781738089fa6aa64143ea9330067b1db077872a4 Mon Sep 17 00:00:00 2001 From: M66B Date: Tue, 11 Oct 2022 21:55:37 +0200 Subject: [PATCH] Added filtering for on-device searching --- FAQ.md | 22 ++--- .../email/BoundaryCallbackMessages.java | 82 ++++++++++++----- .../java/eu/faircode/email/Fts4DbHelper.java | 5 +- .../java/eu/faircode/email/HtmlHelper.java | 92 +++++++++++-------- 4 files changed, 128 insertions(+), 73 deletions(-) diff --git a/FAQ.md b/FAQ.md index d1c86616a5..7a19f57e83 100644 --- a/FAQ.md +++ b/FAQ.md @@ -149,13 +149,13 @@ For the record the stack trace: ``` android.database.sqlite.SQLiteDiskIOException: disk I/O error (code 778) - at io.requery.android.database.sqlite.SQLiteConnection.nativeExecute(SourceFile:-2) - at io.requery.android.database.sqlite.SQLiteConnection.execute(SQLiteConnection:595) - at io.requery.android.database.sqlite.SQLiteSession.endTransactionUnchecked(SQLiteSession:447) - at io.requery.android.database.sqlite.SQLiteSession.endTransaction(SQLiteSession:411) - at io.requery.android.database.sqlite.SQLiteDatabase.endTransaction(SQLiteDatabase:551) - at androidx.room.RoomDatabase.internalEndTransaction(RoomDatabase:594) - at androidx.room.RoomDatabase.endTransaction(RoomDatabase:584) + at io.requery.android.database.sqlite.SQLiteConnection.nativeExecute(SourceFile:-2) + at io.requery.android.database.sqlite.SQLiteConnection.execute(SQLiteConnection:595) + at io.requery.android.database.sqlite.SQLiteSession.endTransactionUnchecked(SQLiteSession:447) + at io.requery.android.database.sqlite.SQLiteSession.endTransaction(SQLiteSession:411) + at io.requery.android.database.sqlite.SQLiteDatabase.endTransaction(SQLiteDatabase:551) + at androidx.room.RoomDatabase.internalEndTransaction(RoomDatabase:594) + at androidx.room.RoomDatabase.endTransaction(RoomDatabase:584) ``` The cause might be [changes in Android 7 Nougat](https://ericsink.com/entries/sqlite_android_n.html), which is why sqlite isn't bundled anymore since version 1.1970. @@ -1158,16 +1158,16 @@ keyword: There should be no space between the prefix and the search term, which will be applied as an AND-condition. -Search expressions can be used for searching on the email server only, and not for searching on the device. +Only AND conditions (+) and NOT conditions (-) can be used for on-device searching (since version 1.1981). +If you try to use other search expressions, you get the error *Select a folder for a complex search*, +which means that a folder in an account's folder list must be selected in order to perform the search on the server. Since version 1.1733 it is possible to save searches, which means that a named entry in the navigation menu will be created to repeat the same search later. You can save a search after searching by tapping on the save button in the top action bar. After repeating a search there will be a delete button at the same place to delete a saved search again. A saved search might be useful to quickly search for starred messages, or for messages from a specific email address, etc. -Searching on the device is a free feature, using the search index and searching on the server is a pro feature. -Note that you can download as many messages to your device as you like. -The easiest way is to use the menu item *Fetch more messages* in the three-dots menu of the start screen. +Using the search index is a pro feature.
diff --git a/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java b/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java index 6cea669543..db71e9686a 100644 --- a/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java +++ b/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java @@ -60,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ExecutorService; +import java.util.regex.Pattern; import javax.mail.Address; import javax.mail.FetchProfile; @@ -269,10 +270,25 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback word = new ArrayList<>(); + List plus = new ArrayList<>(); + List minus = new ArrayList<>(); + if (criteria.query != null) { + for (String w : criteria.query.trim().split("\\s+")) + if (w.length() > 1 && w.startsWith("+")) + plus.add(w.substring(1)); + else if (w.length() > 1 && w.startsWith("-")) + minus.add(w.substring(1)); + else + word.add(w); + if (word.size() == 0 && plus.size() > 0) + word.add(plus.get(0)); + } + + if (criteria.fts && word.size() > 0) { if (state.ids == null) { SQLiteDatabase sdb = Fts4DbHelper.getInstance(context); - state.ids = Fts4DbHelper.match(sdb, account, folder, exclude, criteria); + state.ids = Fts4DbHelper.match(sdb, account, folder, exclude, criteria, TextUtils.join(" ", word)); EntityLog.log(context, "Boundary FTS " + " account=" + account + " folder=" + folder + @@ -335,21 +351,21 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback 0 && state.index >= state.matches.size())) { + String query = (word.size() == 0 ? null : '%' + TextUtils.join("%", word) + '%'); state.matches = db.message().matchMessages( account, folder, exclude, - criteria.query == null ? null : "%" + criteria.query + "%", + query, criteria.in_senders, criteria.in_recipients, criteria.in_subject, @@ -412,6 +429,7 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback 1 && (w.startsWith("+") || w.startsWith("-"))) { + if (w.startsWith("+") && !text.contains(w.substring(1))) + return false; + if (w.startsWith("-") && text.contains(w.substring(1))) + return false; + } else + word.add(w); + + if (word.size() == 0) + return true; + + Pattern pat = Pattern.compile(".*" + TextUtils.join("\\s+", word) + ".*", Pattern.DOTALL); + return pat.matcher(text).matches(); } State getState() { @@ -910,11 +954,7 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback 1 && w.startsWith("+")) - return true; - else if (w.length() > 1 && w.startsWith("-")) - return true; - else if (w.length() > 1 && w.startsWith("?")) + if (w.length() > 1 && w.startsWith("?")) return true; else if (w.length() > FROM.length() && w.startsWith(FROM)) return true; diff --git a/app/src/main/java/eu/faircode/email/Fts4DbHelper.java b/app/src/main/java/eu/faircode/email/Fts4DbHelper.java index 2d9f4c9cca..f72a146793 100644 --- a/app/src/main/java/eu/faircode/email/Fts4DbHelper.java +++ b/app/src/main/java/eu/faircode/email/Fts4DbHelper.java @@ -187,9 +187,8 @@ public class Fts4DbHelper extends SQLiteOpenHelper { static List match( SQLiteDatabase db, Long account, Long folder, long[] exclude, - BoundaryCallbackMessages.SearchCriteria criteria) { - - String search = escape(breakText(criteria.query)); + BoundaryCallbackMessages.SearchCriteria criteria, String query) { + String search = escape(breakText(query)); String select = ""; if (account != null) diff --git a/app/src/main/java/eu/faircode/email/HtmlHelper.java b/app/src/main/java/eu/faircode/email/HtmlHelper.java index e4727bf4d0..d82d21ca92 100644 --- a/app/src/main/java/eu/faircode/email/HtmlHelper.java +++ b/app/src/main/java/eu/faircode/email/HtmlHelper.java @@ -122,6 +122,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.regex.MatchResult; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -2934,52 +2935,67 @@ public class HtmlHelper { return ssb; } - static Document highlightSearched(Context context, Document document, String searched) { - String find = searched.toLowerCase(); + static Document highlightSearched(Context context, Document document, String query) { int color = Helper.resolveColor(context, R.attr.colorHighlight); - NodeTraversor.traverse(new NodeVisitor() { - @Override - public void head(Node node, int depth) { - if (node instanceof TextNode) { - TextNode tnode = (TextNode) node; - String text = tnode.getWholeText(); - - int start = text.toLowerCase().indexOf(find); - if (start < 0) - return; - - int prev = 0; - Element holder = document.createElement("span"); + List word = new ArrayList<>(); + List plus = new ArrayList<>(); + for (String w : query.trim().split("\\s+")) + if (w.length() > 1 && (w.startsWith("+") || w.startsWith("-"))) { + if (w.startsWith("+")) + plus.add(w.substring(1)); + } else + word.add(w); - while (start >= 0) { - if (start > prev) - holder.appendText(text.substring(prev, start)); + int flags = Pattern.DOTALL | Pattern.CASE_INSENSITIVE; + List pat = new ArrayList<>(); + pat.add(Pattern.compile(".*(" + TextUtils.join("\\s+", word) + ").*", flags)); + for (String w : plus) + pat.add(Pattern.compile(".*(" + w + ").*", flags)); - Element span = document.createElement("span"); - span.attr("style", mergeStyles( - span.attr("style"), - "font-size:larger; background-color:" + encodeWebColor(color) - )); - span.text(text.substring(start, start + find.length())); - holder.appendChild(span); - - prev = start + find.length(); - start = text.toLowerCase().indexOf(find, prev); - } + for (Pattern p : pat) + NodeTraversor.traverse(new NodeVisitor() { + @Override + public void head(Node node, int depth) { + if (node instanceof TextNode) + try { + TextNode tnode = (TextNode) node; + String text = tnode.getWholeText(); + + Matcher result = p.matcher(text); + if (!result.matches()) + return; + + int prev = 0; + Element holder = document.createElement("span"); + for (int i = 1; i <= result.groupCount(); i++) { + holder.appendText(text.substring(prev, result.start(i))); + + Element span = document.createElement("span"); + span.attr("style", mergeStyles( + span.attr("style"), + "font-size:larger; background-color:" + encodeWebColor(color) + )); + span.text(text.substring(result.start(i), result.end(i))); + holder.appendChild(span); + + prev = result.end(i); + } - if (prev < text.length()) - holder.appendText(text.substring(prev)); + if (prev < text.length()) + holder.appendText(text.substring(prev)); - tnode.before(holder); - tnode.text(""); + tnode.before(holder); + tnode.text(""); + } catch (Throwable ex) { + Log.e(ex); + } } - } - @Override - public void tail(Node node, int depth) { - } - }, document); + @Override + public void tail(Node node, int depth) { + } + }, document); return document; }