Added filtering for on-device searching

FAQ.md

@@ -1158,16 +1158,16 @@ keyword:<keyword>
 
 There should be no space between the prefix and the search term, which will be applied as an AND-condition.
 
-Search expressions can be used for searching on the email server only, and not for searching on the device.
+Only AND conditions (+) and NOT conditions (-) can be used for on-device searching (since version 1.1981).
+If you try to use other search expressions, you get the error *Select a folder for a complex search*,
+which means that a folder in an account's folder list must be selected in order to perform the search on the server.
 
 Since version 1.1733 it is possible to save searches, which means that a named entry in the navigation menu will be created to repeat the same search later.
 You can save a search after searching by tapping on the save button in the top action bar.
 After repeating a search there will be a delete button at the same place to delete a saved search again.
 A saved search might be useful to quickly search for starred messages, or for messages from a specific email address, etc.
 
-Searching on the device is a free feature, using the search index and searching on the server is a pro feature.
+Using the search index is a pro feature.
-Note that you can download as many messages to your device as you like.
-The easiest way is to use the menu item *Fetch more messages* in the three-dots menu of the start screen.
 
 <br />
 

app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java

@@ -60,6 +60,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ExecutorService;
+import java.util.regex.Pattern;
 
 import javax.mail.Address;
 import javax.mail.FetchProfile;
@@ -269,10 +270,25 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
 
         int found = 0;
 
-        if (criteria.fts && criteria.query != null) {
+        List<String> word = new ArrayList<>();
+        List<String> plus = new ArrayList<>();
+        List<String> minus = new ArrayList<>();
+        if (criteria.query != null) {
+            for (String w : criteria.query.trim().split("\\s+"))
+                if (w.length() > 1 && w.startsWith("+"))
+                    plus.add(w.substring(1));
+                else if (w.length() > 1 && w.startsWith("-"))
+                    minus.add(w.substring(1));
+                else
+                    word.add(w);
+            if (word.size() == 0 && plus.size() > 0)
+                word.add(plus.get(0));
+        }
+
+        if (criteria.fts && word.size() > 0) {
             if (state.ids == null) {
                 SQLiteDatabase sdb = Fts4DbHelper.getInstance(context);
-                state.ids = Fts4DbHelper.match(sdb, account, folder, exclude, criteria);
+                state.ids = Fts4DbHelper.match(sdb, account, folder, exclude, criteria, TextUtils.join(" ", word));
                 EntityLog.log(context, "Boundary FTS " +
                         " account=" + account +
                         " folder=" + folder +
@@ -335,21 +351,21 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
                     }
 
                     if (!matched && criteria.in_subject) {
-                        if (contains(message.subject, criteria.query))
+                        if (contains(message.subject, criteria.query, false))
                             matched = true;
                     }
 
                     if (!matched && criteria.in_keywords) {
                         if (message.keywords != null)
                             for (String keyword : message.keywords)
-                                if (contains(keyword, criteria.query)) {
+                                if (contains(keyword, criteria.query, false)) {
                                     matched = true;
                                     break;
                                 }
                     }
 
                     if (!matched && criteria.in_notes) {
-                        if (contains(message.notes, criteria.query))
+                        if (contains(message.notes, criteria.query, false))
                             matched = true;
                     }
 
@@ -358,9 +374,9 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
                             File file = EntityMessage.getFile(context, id);
                             if (file.exists()) {
                                 String html = Helper.readText(file);
-                                if (contains(html, criteria.query)) {
+                                if (contains(html, criteria.query, true)) {
                                     String text = HtmlHelper.getFullText(html);
-                                    if (contains(text, criteria.query))
+                                    if (contains(text, criteria.query, false))
                                         matched = true;
                                 }
                             }
@@ -386,9 +402,10 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
         while (found < pageSize && !state.destroyed) {
             if (state.matches == null ||
                     (state.matches.size() > 0 && state.index >= state.matches.size())) {
+                String query = (word.size() == 0 ? null : '%' + TextUtils.join("%", word) + '%');
                 state.matches = db.message().matchMessages(
                         account, folder, exclude,
-                        criteria.query == null ? null : "%" + criteria.query + "%",
+                        query,
                         criteria.in_senders,
                         criteria.in_recipients,
                         criteria.in_subject,
@@ -412,6 +429,7 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
                         " account=" + account +
                         " folder=" + folder +
                         " criteria=" + criteria +
+                        " query=" + query +
                         " offset=" + state.offset +
                         " size=" + state.matches.size());
                 state.offset += Math.min(state.matches.size(), SEARCH_LIMIT_DEVICE);
@@ -425,19 +443,28 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
                 state.index = i + 1;
 
                 TupleMatch match = state.matches.get(i);
-                boolean matched = (match.matched != null && match.matched);
+                boolean matched = (match.matched != null && match.matched &&
+                        plus.size() + minus.size() == 0);
 
-                if (!matched && criteria.query != null && (criteria.in_message || criteria.in_html))
+                if (!matched && criteria.query != null &&
+                        criteria.in_subject && plus.size() + minus.size() > 0) {
+                    EntityMessage message = db.message().getMessage(match.id);
+                    if (message != null)
+                        matched = contains(message.subject, criteria.query, false);
+                }
+
+                if (!matched && criteria.query != null &&
+                        (criteria.in_message || criteria.in_html))
                     try {
                         File file = EntityMessage.getFile(context, match.id);
                         if (file.exists()) {
                             String html = Helper.readText(file);
-                            if (contains(html, criteria.query)) {
+                            if (contains(html, criteria.query, true)) {
                                 if (criteria.in_html)
                                     matched = true;
                                 else {
                                     String text = HtmlHelper.getFullText(html);
-                                    if (contains(text, criteria.query))
+                                    if (contains(text, criteria.query, false))
                                         matched = true;
                                 }
                             }
@@ -795,16 +822,33 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
         if (addresses == null)
             return false;
         for (Address address : addresses)
-            if (contains(address.toString(), query))
+            if (contains(address.toString(), query, false))
                 return true;
         return false;
     }
 
-    private static boolean contains(String text, String query) {
+    private static boolean contains(String text, String query, boolean html) {
         if (TextUtils.isEmpty(text))
             return false;
-        return Fts4DbHelper.preprocessText(text)
+
-                .contains(Fts4DbHelper.preprocessText(query));
+        text = Fts4DbHelper.preprocessText(text);
+        query = Fts4DbHelper.preprocessText(query);
+
+        List<String> word = new ArrayList<>();
+        for (String w : query.trim().split("\\s+"))
+            if (w.length() > 1 && (w.startsWith("+") || w.startsWith("-"))) {
+                if (w.startsWith("+") && !text.contains(w.substring(1)))
+                    return false;
+                if (w.startsWith("-") && text.contains(w.substring(1)))
+                    return false;
+            } else
+                word.add(w);
+
+        if (word.size() == 0)
+            return true;
+
+        Pattern pat = Pattern.compile(".*" + TextUtils.join("\\s+", word) + ".*", Pattern.DOTALL);
+        return pat.matcher(text).matches();
     }
 
     State getState() {
@@ -910,11 +954,7 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
                 return false;
 
             for (String w : query.trim().split("\\s+"))
-                if (w.length() > 1 && w.startsWith("+"))
+                if (w.length() > 1 && w.startsWith("?"))
-                    return true;
-                else if (w.length() > 1 && w.startsWith("-"))
-                    return true;
-                else if (w.length() > 1 && w.startsWith("?"))
                     return true;
                 else if (w.length() > FROM.length() && w.startsWith(FROM))
                     return true;

app/src/main/java/eu/faircode/email/Fts4DbHelper.java

@@ -187,9 +187,8 @@ public class Fts4DbHelper extends SQLiteOpenHelper {
     static List<Long> match(
             SQLiteDatabase db,
             Long account, Long folder, long[] exclude,
-            BoundaryCallbackMessages.SearchCriteria criteria) {
+            BoundaryCallbackMessages.SearchCriteria criteria, String query) {
-
+        String search = escape(breakText(query));
-        String search = escape(breakText(criteria.query));
 
         String select = "";
         if (account != null)

app/src/main/java/eu/faircode/email/HtmlHelper.java

@@ -122,6 +122,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.regex.MatchResult;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -2934,38 +2935,51 @@ public class HtmlHelper {
         return ssb;
     }
 
-    static Document highlightSearched(Context context, Document document, String searched) {
+    static Document highlightSearched(Context context, Document document, String query) {
-        String find = searched.toLowerCase();
         int color = Helper.resolveColor(context, R.attr.colorHighlight);
 
+        List<String> word = new ArrayList<>();
+        List<String> plus = new ArrayList<>();
+        for (String w : query.trim().split("\\s+"))
+            if (w.length() > 1 && (w.startsWith("+") || w.startsWith("-"))) {
+                if (w.startsWith("+"))
+                    plus.add(w.substring(1));
+            } else
+                word.add(w);
+
+        int flags = Pattern.DOTALL | Pattern.CASE_INSENSITIVE;
+        List<Pattern> pat = new ArrayList<>();
+        pat.add(Pattern.compile(".*(" + TextUtils.join("\\s+", word) + ").*", flags));
+        for (String w : plus)
+            pat.add(Pattern.compile(".*(" + w + ").*", flags));
+
+        for (Pattern p : pat)
             NodeTraversor.traverse(new NodeVisitor() {
                 @Override
                 public void head(Node node, int depth) {
-                if (node instanceof TextNode) {
+                    if (node instanceof TextNode)
+                        try {
                             TextNode tnode = (TextNode) node;
                             String text = tnode.getWholeText();
 
-                    int start = text.toLowerCase().indexOf(find);
+                            Matcher result = p.matcher(text);
-                    if (start < 0)
+                            if (!result.matches())
                                 return;
 
                             int prev = 0;
                             Element holder = document.createElement("span");
-
+                            for (int i = 1; i <= result.groupCount(); i++) {
-                    while (start >= 0) {
+                                holder.appendText(text.substring(prev, result.start(i)));
-                        if (start > prev)
-                            holder.appendText(text.substring(prev, start));
 
                                 Element span = document.createElement("span");
                                 span.attr("style", mergeStyles(
                                         span.attr("style"),
                                         "font-size:larger; background-color:" + encodeWebColor(color)
                                 ));
-                        span.text(text.substring(start, start + find.length()));
+                                span.text(text.substring(result.start(i), result.end(i)));
                                 holder.appendChild(span);
 
-                        prev = start + find.length();
+                                prev = result.end(i);
-                        start = text.toLowerCase().indexOf(find, prev);
                             }
 
                             if (prev < text.length())
@@ -2973,6 +2987,8 @@ public class HtmlHelper {
 
                             tnode.before(holder);
                             tnode.text("");
+                        } catch (Throwable ex) {
+                            Log.e(ex);
                         }
                 }