From e64a43ff6446f7f5d7b7696045fe434695d3fec6 Mon Sep 17 00:00:00 2001 From: M66B Date: Sun, 2 Sep 2018 06:59:49 +0000 Subject: [PATCH] Progressive search improvements --- .../email/BoundaryCallbackMessages.java | 172 +++++++++++++ .../java/eu/faircode/email/DaoMessage.java | 7 +- .../eu/faircode/email/FragmentMessages.java | 225 ++++++------------ 3 files changed, 250 insertions(+), 154 deletions(-) create mode 100644 app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java diff --git a/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java b/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java new file mode 100644 index 0000000000..07c7756299 --- /dev/null +++ b/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java @@ -0,0 +1,172 @@ +package eu.faircode.email; + +import android.content.Context; +import android.os.Handler; +import android.util.Log; + +import com.sun.mail.imap.IMAPFolder; +import com.sun.mail.imap.IMAPMessage; +import com.sun.mail.imap.IMAPStore; + +import java.util.Date; +import java.util.Properties; + +import javax.mail.Folder; +import javax.mail.Message; +import javax.mail.Session; +import javax.mail.search.AndTerm; +import javax.mail.search.BodyTerm; +import javax.mail.search.ComparisonTerm; +import javax.mail.search.FromStringTerm; +import javax.mail.search.OrTerm; +import javax.mail.search.ReceivedDateTerm; +import javax.mail.search.SubjectTerm; + +import androidx.lifecycle.GenericLifecycleObserver; +import androidx.lifecycle.Lifecycle; +import androidx.lifecycle.LifecycleOwner; +import androidx.paging.PagedList; + +public class BoundaryCallbackMessages extends PagedList.BoundaryCallback { + private Context context; + private long fid; + private String search; + private Handler mainHandler; + private IBoundaryCallbackMessages intf; + + private boolean enabled = false; + private IMAPStore istore = null; + private IMAPFolder ifolder = null; + private Message[] imessages = null; + + interface IBoundaryCallbackMessages { + void onLoading(); + + void onLoaded(); + + void onError(Throwable ex); + } + + BoundaryCallbackMessages(Context context, LifecycleOwner owner, long folder, String search, IBoundaryCallbackMessages intf) { + this.context = context; + this.fid = folder; + this.search = search; + this.mainHandler = new Handler(context.getMainLooper()); + this.intf = intf; + + owner.getLifecycle().addObserver(new GenericLifecycleObserver() { + @Override + public void onStateChanged(LifecycleOwner source, Lifecycle.Event event) { + if (event == Lifecycle.Event.ON_DESTROY) + new Thread(new Runnable() { + @Override + public void run() { + Log.i(Helper.TAG, "Boundary close"); + try { + if (istore != null) + istore.close(); + } catch (Throwable ex) { + Log.e(Helper.TAG, "Boundary " + ex + "\n" + Log.getStackTraceString(ex)); + } finally { + istore = null; + ifolder = null; + imessages = null; + } + } + }).start(); + } + }); + } + + void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + @Override + public void onItemAtEndLoaded(final TupleMessageEx itemAtEnd) { + Log.i(Helper.TAG, "onItemAtEndLoaded enabled=" + enabled); + if (!enabled) + return; + load(itemAtEnd.received); + } + + void load(final long before) { + new Thread(new Runnable() { + @Override + public void run() { + synchronized (context.getApplicationContext()) { + try { + mainHandler.post(new Runnable() { + @Override + public void run() { + intf.onLoading(); + } + }); + + DB db = DB.getInstance(context); + EntityFolder folder = db.folder().getFolder(fid); + EntityAccount account = db.account().getAccount(folder.account); + + if (imessages == null) { + // Refresh token + //if (account.auth_type == Helper.AUTH_TYPE_GMAIL) { + // account.password = Helper.refreshToken(context, "com.google", account.user, account.password); + // db.account().setAccountPassword(account.id, account.password); + //} + + Properties props = MessageHelper.getSessionProperties(context, account.auth_type); + props.setProperty("mail.imap.throwsearchexception", "true"); + Session isession = Session.getInstance(props, null); + + Log.i(Helper.TAG, "Boundary connecting account=" + account.name); + istore = (IMAPStore) isession.getStore("imaps"); + istore.connect(account.host, account.port, account.user, account.password); + + Log.i(Helper.TAG, "Boundary opening folder=" + folder.name); + ifolder = (IMAPFolder) istore.getFolder(folder.name); + ifolder.open(Folder.READ_WRITE); + + Log.i(Helper.TAG, "Boundary searching=" + search + " before=" + new Date(before)); + imessages = ifolder.search( + new AndTerm( + new ReceivedDateTerm(ComparisonTerm.LT, new Date(before)), + new OrTerm( + new FromStringTerm(search), + new OrTerm( + new SubjectTerm(search), + new BodyTerm(search))))); + Log.i(Helper.TAG, "Boundary found messages=" + imessages.length); + } + + int index = imessages.length - 1; + while (index >= 0) { + if (imessages[index].getReceivedDate().getTime() < before) { + Log.i(Helper.TAG, "Boundary sync uid=" + ifolder.getUID(imessages[index])); + ServiceSynchronize.synchronizeMessage(context, folder, ifolder, (IMAPMessage) imessages[index], true); + break; + } + index--; + } + + Log.i(Helper.TAG, "Boundary done"); + } catch (final Throwable ex) { + Log.e(Helper.TAG, "Boundary " + ex + "\n" + Log.getStackTraceString(ex)); + mainHandler.post(new Runnable() { + @Override + public void run() { + intf.onError(ex); + } + }); + } finally { + mainHandler.post(new Runnable() { + @Override + public void run() { + intf.onLoaded(); + } + }); + } + } + } + }).start(); + } +} \ No newline at end of file diff --git a/app/src/main/java/eu/faircode/email/DaoMessage.java b/app/src/main/java/eu/faircode/email/DaoMessage.java index 48df35bf7e..bae0637d60 100644 --- a/app/src/main/java/eu/faircode/email/DaoMessage.java +++ b/app/src/main/java/eu/faircode/email/DaoMessage.java @@ -45,7 +45,6 @@ public interface DaoMessage { " JOIN folder ON folder.id = message.folder" + " WHERE account.`synchronize`" + " AND (NOT message.ui_hide OR :debug)" + - " AND NOT ui_found" + " GROUP BY CASE WHEN message.thread IS NULL THEN message.id ELSE message.thread END" + " HAVING SUM(CASE WHEN folder.type = '" + EntityFolder.INBOX + "' THEN 1 ELSE 0 END) > 0" + " ORDER BY message.received DESC") @@ -61,7 +60,7 @@ public interface DaoMessage { " JOIN folder ON folder.id = message.folder" + " LEFT JOIN folder f ON f.id = :folder" + " WHERE (NOT message.ui_hide OR :debug)" + - " AND ui_found = :found" + + " AND (NOT :found OR ui_found = :found)" + " GROUP BY CASE WHEN message.thread IS NULL THEN message.id ELSE message.thread END" + " HAVING SUM(CASE WHEN folder.id = :folder THEN 1 ELSE 0 END) > 0" + " ORDER BY message.received DESC, message.sent DESC") @@ -75,7 +74,6 @@ public interface DaoMessage { " LEFT JOIN account ON account.id = message.account" + " JOIN folder ON folder.id = message.folder" + " WHERE (NOT message.ui_hide OR :debug)" + - " AND NOT ui_found" + " AND message.account = (SELECT m1.account FROM message m1 WHERE m1.id = :msgid)" + " AND message.thread = (SELECT m2.thread FROM message m2 WHERE m2.id = :msgid)" + " ORDER BY message.received DESC, message.sent DESC") @@ -131,7 +129,6 @@ public interface DaoMessage { " WHERE account.`synchronize`" + " AND folder.type = '" + EntityFolder.INBOX + "'" + " AND NOT message.ui_seen AND NOT message.ui_hide" + - " AND NOT ui_found" + " AND (account.seen_until IS NULL OR message.stored > account.seen_until)" + " ORDER BY message.received") LiveData> liveUnseenUnified(); @@ -140,7 +137,7 @@ public interface DaoMessage { " WHERE folder = :folder" + " AND received >= :received" + " AND NOT uid IS NULL" + - " AND NOT ui_found") + " AND NOT ui_found" /* keep found messages */) List getUids(long folder, long received); @Insert diff --git a/app/src/main/java/eu/faircode/email/FragmentMessages.java b/app/src/main/java/eu/faircode/email/FragmentMessages.java index 9212830931..2c9d8b4881 100644 --- a/app/src/main/java/eu/faircode/email/FragmentMessages.java +++ b/app/src/main/java/eu/faircode/email/FragmentMessages.java @@ -39,33 +39,15 @@ import android.widget.Toast; import com.google.android.material.floatingactionbutton.FloatingActionButton; import com.google.android.material.snackbar.Snackbar; -import com.sun.mail.imap.IMAPFolder; -import com.sun.mail.imap.IMAPMessage; -import com.sun.mail.imap.IMAPStore; import java.util.Date; import java.util.List; -import java.util.Properties; - -import javax.mail.Folder; -import javax.mail.Message; -import javax.mail.Session; -import javax.mail.search.AndTerm; -import javax.mail.search.BodyTerm; -import javax.mail.search.ComparisonTerm; -import javax.mail.search.FromStringTerm; -import javax.mail.search.OrTerm; -import javax.mail.search.ReceivedDateTerm; -import javax.mail.search.SubjectTerm; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.appcompat.widget.SearchView; import androidx.constraintlayout.widget.Group; import androidx.fragment.app.FragmentTransaction; -import androidx.lifecycle.GenericLifecycleObserver; -import androidx.lifecycle.Lifecycle; -import androidx.lifecycle.LifecycleOwner; import androidx.lifecycle.LiveData; import androidx.lifecycle.Observer; import androidx.paging.LivePagedListBuilder; @@ -91,9 +73,14 @@ public class FragmentMessages extends FragmentEx { private long primary = -1; private AdapterMessage adapter; + private SearchState searchState = SearchState.Reset; + private BoundaryCallbackMessages searchCallback = null; + private static final int MESSAGES_PAGE_SIZE = 50; private static final int SEARCH_PAGE_SIZE = 10; + private enum SearchState {Reset, Database, Boundary} + @Override public void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -184,7 +171,7 @@ public class FragmentMessages extends FragmentEx { args.putInt("direction", direction); new SimpleTask() { @Override - protected String onLoad(Context context, Bundle args) throws Throwable { + protected String onLoad(Context context, Bundle args) { long id = args.getLong("id"); int direction = args.getInt("direction"); EntityFolder target = null; @@ -341,158 +328,98 @@ public class FragmentMessages extends FragmentEx { } }); } else { + Log.i(Helper.TAG, "Search state=" + searchState); setSubtitle(getString(R.string.title_searching, search)); + if (searchCallback == null) + searchCallback = new BoundaryCallbackMessages( + getContext(), FragmentMessages.this, + folder, search, + new BoundaryCallbackMessages.IBoundaryCallbackMessages() { + @Override + public void onLoading() { + pbWait.setVisibility(View.VISIBLE); + } + + @Override + public void onLoaded() { + pbWait.setVisibility(View.GONE); + } + + @Override + public void onError(Throwable ex) { + Toast.makeText(getContext(), ex.toString(), Toast.LENGTH_LONG).show(); + } + }); + Bundle args = new Bundle(); args.putLong("folder", folder); args.putString("search", search); new SimpleTask() { @Override - protected Void onLoad(Context context, Bundle args) throws Throwable { - long folder = args.getLong("folder"); - String search = args.getString("search").toLowerCase(); - - db.message().resetFound(folder); - - for (long id : db.message().getMessageIDs(folder)) { - EntityMessage message = db.message().getMessage(id); - String from = MessageHelper.getFormattedAddresses(message.from, true); - if (from.toLowerCase().contains(search) || - message.subject.toLowerCase().contains(search) || - message.read(context).toLowerCase().contains(search)) { - Log.i(Helper.TAG, "SDS found id=" + id); - db.message().setMessageFound(message.id, true); - } + protected Void onLoad(Context context, Bundle args) { + if (searchState == SearchState.Reset) { + long folder = args.getLong("folder"); + DB.getInstance(context).message().resetFound(folder); + searchState = SearchState.Database; + Log.i(Helper.TAG, "Search reset done"); } - return null; } @Override protected void onLoaded(final Bundle args, Void data) { - LiveData> messages = new LivePagedListBuilder<>(db.message().pagedFolder(folder, true, false), SEARCH_PAGE_SIZE) - .setBoundaryCallback(new PagedList.BoundaryCallback() { - private IMAPStore istore = null; - private IMAPFolder ifolder = null; - private Message[] imessages = null; - private boolean observing = false; - - @Override - public void onItemAtEndLoaded(final TupleMessageEx itemAtEnd) { - if (!observing) { - observing = true; - getLifecycle().addObserver(new GenericLifecycleObserver() { - @Override - public void onStateChanged(LifecycleOwner source, Lifecycle.Event event) { - if (event == Lifecycle.Event.ON_DESTROY) - new Thread(new Runnable() { - @Override - public void run() { - Log.i(Helper.TAG, "SDS close"); - try { - if (istore != null) - istore.close(); - } catch (Throwable ex) { - Log.i(Helper.TAG, ex + "\n" + Log.getStackTraceString(ex)); - } - } - }).start(); - } - }); - } - - Log.i(Helper.TAG, "SDS more"); - - // Hold on to context - final Context context = getContext(); - - new Thread(new Runnable() { - @Override - public void run() { - try { - long folder = args.getLong("folder"); - String search = args.getString("search"); - - EntityFolder _folder = db.folder().getFolder(folder); - EntityAccount account = db.account().getAccount(_folder.account); - - // Refresh token - //if (account.auth_type == Helper.AUTH_TYPE_GMAIL) { - // account.password = Helper.refreshToken(context, "com.google", account.user, account.password); - // db.account().setAccountPassword(account.id, account.password); - //} - - if (imessages == null) { - Properties props = MessageHelper.getSessionProperties(context, account.auth_type); - props.setProperty("mail.imap.throwsearchexception", "true"); - Session isession = Session.getInstance(props, null); - - Log.i(Helper.TAG, "SDS connecting account=" + account.name); - istore = (IMAPStore) isession.getStore("imaps"); - istore.connect(account.host, account.port, account.user, account.password); - - Log.i(Helper.TAG, "SDS opening folder=" + _folder.name); - ifolder = (IMAPFolder) istore.getFolder(_folder.name); - ifolder.open(Folder.READ_WRITE); - - Log.i(Helper.TAG, "SDS searching=" + search + " before=" + new Date(itemAtEnd.received)); - imessages = ifolder.search( - new AndTerm( - new ReceivedDateTerm(ComparisonTerm.LT, new Date(itemAtEnd.received)), - new OrTerm( - new FromStringTerm(search), - new OrTerm( - new SubjectTerm(search), - new BodyTerm(search))))); - Log.i(Helper.TAG, "SDS found messages=" + imessages.length); - } - - int index = imessages.length - 1; - while (index >= 0) { - if (imessages[index].getReceivedDate().getTime() < itemAtEnd.received) { - Log.i(Helper.TAG, "Search sync uid=" + ifolder.getUID(imessages[index])); - ServiceSynchronize.synchronizeMessage(context, _folder, ifolder, (IMAPMessage) imessages[index], true); - break; - } - index--; - } - - Log.i(Helper.TAG, "SDS done"); - } catch (Throwable ex) { - Log.i(Helper.TAG, ex + "\n" + Log.getStackTraceString(ex)); - } - } - }).start(); - } - }) - .build(); - + LivePagedListBuilder builder = new LivePagedListBuilder<>(db.message().pagedFolder(folder, true, false), SEARCH_PAGE_SIZE); + builder.setBoundaryCallback(searchCallback); + LiveData> messages = builder.build(); messages.observe(getViewLifecycleOwner(), new Observer>() { @Override - public void onChanged(@Nullable PagedList messages) { - if (messages == null) { - finish(); - return; - } - - Log.i(Helper.TAG, "Submit messages=" + messages.size()); + public void onChanged(PagedList messages) { + Log.i(Helper.TAG, "Submit found messages=" + messages.size()); adapter.submitList(messages); - - pbWait.setVisibility(View.GONE); grpReady.setVisibility(View.VISIBLE); + } + }); - if (messages.size() == 0) { - tvNoEmail.setVisibility(View.VISIBLE); - rvMessage.setVisibility(View.GONE); - } else { - tvNoEmail.setVisibility(View.GONE); - rvMessage.setVisibility(View.VISIBLE); + new SimpleTask() { + @Override + protected Long onLoad(Context context, Bundle args) throws Throwable { + long last = 0; + if (searchState == SearchState.Database) { + last = new Date().getTime(); + long folder = args.getLong("folder"); + String search = args.getString("search").toLowerCase(); + DB db = DB.getInstance(context); + for (long id : db.message().getMessageIDs(folder)) { + EntityMessage message = db.message().getMessage(id); + if (message != null) { // Message could be removed in the meantime + String from = MessageHelper.getFormattedAddresses(message.from, true); + if (from.toLowerCase().contains(search) || + message.subject.toLowerCase().contains(search) || + message.read(context).toLowerCase().contains(search)) { + Log.i(Helper.TAG, "Search found id=" + id); + db.message().setMessageFound(message.id, true); + last = message.received; + } + } + } + searchState = SearchState.Boundary; + Log.i(Helper.TAG, "Search database done"); } + return last; } - }); + + @Override + protected void onLoaded(Bundle args, Long last) { + pbWait.setVisibility(View.GONE); + searchCallback.setEnabled(true); + if (last > 0) + searchCallback.load(last); + } + }.load(FragmentMessages.this, args); } - }.load(FragmentMessages.this, args); + }.load(this, args); } Bundle args = new Bundle();