Optimized sync/donwload, improvements

pull/125/head
M66B 6 years ago
parent 5198802bc0
commit 26fdc79ad8

@ -103,6 +103,8 @@ public class AdapterFolder extends RecyclerView.Adapter<AdapterFolder.ViewHolder
ivState.setImageResource(R.drawable.baseline_close_24);
else if ("syncing".equals(folder.state))
ivState.setImageResource(R.drawable.baseline_compare_arrows_24);
else if ("downloading".equals(folder.state))
ivState.setImageResource(R.drawable.baseline_get_app_24);
else
ivState.setImageResource(R.drawable.baseline_cloud_off_24);
ivState.setVisibility(folder.synchronize || folder.state != null ? View.VISIBLE : View.INVISIBLE);
@ -115,7 +117,7 @@ public class AdapterFolder extends RecyclerView.Adapter<AdapterFolder.ViewHolder
tvName.setTypeface(null, folder.unseen > 0 ? Typeface.BOLD : Typeface.NORMAL);
tvName.setTextColor(Helper.resolveColor(context, folder.unseen > 0 ? R.attr.colorUnread : android.R.attr.textColorSecondary));
tvMessages.setText(Integer.toString(folder.messages));
tvMessages.setText(String.format("%d/%d", folder.content, folder.messages));
ivUnified.setVisibility(folder.unified ? View.VISIBLE : View.INVISIBLE);

@ -32,9 +32,11 @@ import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.mail.FetchProfile;
import javax.mail.Folder;
import javax.mail.Message;
import javax.mail.Session;
import javax.mail.UIDFolder;
import javax.mail.search.AndTerm;
import javax.mail.search.BodyTerm;
import javax.mail.search.ComparisonTerm;
@ -153,6 +155,16 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
new SubjectTerm(search),
new BodyTerm(search)))));
Log.i(Helper.TAG, "Boundary found messages=" + imessages.length);
FetchProfile fp = new FetchProfile();
fp.add(FetchProfile.Item.ENVELOPE);
fp.add(FetchProfile.Item.FLAGS);
fp.add(FetchProfile.Item.CONTENT_INFO); // body structure
fp.add(UIDFolder.FetchProfileItem.UID);
fp.add(IMAPFolder.FetchProfileItem.HEADERS);
fp.add(FetchProfile.Item.SIZE);
fp.add(IMAPFolder.FetchProfileItem.INTERNALDATE);
ifolder.fetch(imessages, fp);
}
int count = 0;

@ -39,6 +39,12 @@ public interface DaoAttachment {
" WHERE message = :message")
int getAttachmentCount(long message);
@Query("SELECT COUNT(id)" +
" FROM attachment" +
" WHERE message = :message" +
" AND NOT available")
int getAttachmentDownloadCount(long message);
@Query("SELECT COUNT(id)" +
" FROM attachment" +
" WHERE id = :id")

@ -46,6 +46,7 @@ public interface DaoFolder {
@Query("SELECT folder.*, account.name AS accountName" +
", COUNT(message.id) AS messages" +
", SUM(CASE WHEN message.content = 1 THEN 1 ELSE 0 END) AS content" +
", SUM(CASE WHEN message.ui_seen = 0 THEN 1 ELSE 0 END) AS unseen" +
" FROM folder" +
" LEFT JOIN account ON account.id = folder.account" +
@ -56,6 +57,7 @@ public interface DaoFolder {
@Query("SELECT folder.*, account.name AS accountName" +
", COUNT(message.id) AS messages" +
", SUM(CASE WHEN message.content = 1 THEN 1 ELSE 0 END) AS content" +
", SUM(CASE WHEN message.ui_seen = 0 THEN 1 ELSE 0 END) AS unseen" +
" FROM folder" +
" JOIN account ON account.id = folder.account" +
@ -70,6 +72,7 @@ public interface DaoFolder {
@Query("SELECT folder.*, account.name AS accountName" +
", COUNT(message.id) AS messages" +
", SUM(CASE WHEN message.content = 1 THEN 1 ELSE 0 END) AS content" +
", SUM(CASE WHEN message.ui_seen = 0 THEN 1 ELSE 0 END) AS unseen" +
" FROM folder" +
" LEFT JOIN account ON account.id = folder.account" +

@ -62,6 +62,7 @@ import java.net.UnknownHostException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.Enumeration;
@ -123,6 +124,8 @@ public class ServiceSynchronize extends LifecycleService {
private static final int CONNECT_BACKOFF_START = 8; // seconds
private static final int CONNECT_BACKOFF_MAX = 1024; // seconds (1024 sec ~ 17 min)
private static final long STORE_NOOP_INTERVAL = 9 * 60 * 1000L; // ms
private static final int SYNC_BATCH_SIZE = 20;
private static final int DOWNLOAD_BATCH_SIZE = 20;
private static final int MESSAGE_AUTO_DOWNLOAD_SIZE = 32 * 1024; // bytes
private static final int ATTACHMENT_AUTO_DOWNLOAD_SIZE = 32 * 1024; // bytes
@ -421,7 +424,7 @@ public class ServiceSynchronize extends LifecycleService {
// Create session
Properties props = MessageHelper.getSessionProperties(this, account.auth_type);
final Session isession = Session.getInstance(props, null);
isession.setDebug(debug);
isession.setDebug(debug || BuildConfig.DEBUG);
// adb -t 1 logcat | grep "fairemail\|System.out"
final IMAPStore istore = (IMAPStore) isession.getStore("imaps");
@ -542,6 +545,18 @@ public class ServiceSynchronize extends LifecycleService {
synchronized (lock) {
try {
Log.i(Helper.TAG, folder.name + " messages added");
FetchProfile fp = new FetchProfile();
fp.add(FetchProfile.Item.ENVELOPE);
fp.add(FetchProfile.Item.FLAGS);
fp.add(FetchProfile.Item.CONTENT_INFO); // body structure
fp.add(UIDFolder.FetchProfileItem.UID);
fp.add(IMAPFolder.FetchProfileItem.HEADERS);
fp.add(IMAPFolder.FetchProfileItem.MESSAGE);
fp.add(FetchProfile.Item.SIZE);
fp.add(IMAPFolder.FetchProfileItem.INTERNALDATE);
ifolder.fetch(e.getMessages(), fp);
for (Message imessage : e.getMessages())
try {
long id = synchronizeMessage(ServiceSynchronize.this, folder, ifolder, (IMAPMessage) imessage, false);
@ -611,6 +626,12 @@ public class ServiceSynchronize extends LifecycleService {
try {
try {
Log.i(Helper.TAG, folder.name + " message changed");
FetchProfile fp = new FetchProfile();
fp.add(UIDFolder.FetchProfileItem.UID);
fp.add(IMAPFolder.FetchProfileItem.FLAGS);
ifolder.fetch(new Message[]{e.getMessage()}, fp);
long id = synchronizeMessage(ServiceSynchronize.this, folder, ifolder, (IMAPMessage) e.getMessage(), false);
downloadMessage(ServiceSynchronize.this, folder, id, (IMAPMessage) e.getMessage());
} catch (MessageRemovedException ex) {
@ -1291,7 +1312,6 @@ public class ServiceSynchronize extends LifecycleService {
FetchProfile fp = new FetchProfile();
fp.add(UIDFolder.FetchProfileItem.UID);
fp.add(IMAPFolder.FetchProfileItem.FLAGS);
ifolder.fetch(imessages, fp);
long fetch = SystemClock.elapsedRealtime();
@ -1320,12 +1340,28 @@ public class ServiceSynchronize extends LifecycleService {
Log.i(Helper.TAG, folder.name + " delete local uid=" + uid + " count=" + count);
}
fp.add(FetchProfile.Item.ENVELOPE);
fp.add(FetchProfile.Item.FLAGS);
fp.add(FetchProfile.Item.CONTENT_INFO); // body structure
// fp.add(UIDFolder.FetchProfileItem.UID);
fp.add(IMAPFolder.FetchProfileItem.HEADERS);
// fp.add(IMAPFolder.FetchProfileItem.MESSAGE);
fp.add(FetchProfile.Item.SIZE);
fp.add(IMAPFolder.FetchProfileItem.INTERNALDATE);
// Add/update local messages
Long[] ids = new Long[imessages.length];
Log.i(Helper.TAG, folder.name + " add=" + imessages.length);
for (int i = imessages.length - 1; i >= 0; i--)
for (int i = imessages.length - 1; i >= 0; i -= SYNC_BATCH_SIZE) {
int from = Math.max(0, i - SYNC_BATCH_SIZE) + 1;
Log.i(Helper.TAG, folder.name + " update " + from + " .. " + i);
Message[] isub = Arrays.copyOfRange(imessages, from, i + 1);
ifolder.fetch(isub, fp);
for (int j = isub.length - 1; j >= 0; j--)
try {
ids[i] = synchronizeMessage(this, folder, ifolder, (IMAPMessage) imessages[i], false);
ids[from + j] = synchronizeMessage(this, folder, ifolder, (IMAPMessage) isub[j], false);
} catch (MessageRemovedException ex) {
Log.w(Helper.TAG, folder.name + " " + ex + "\n" + Log.getStackTraceString(ex));
} catch (FolderClosedException ex) {
@ -1334,20 +1370,39 @@ public class ServiceSynchronize extends LifecycleService {
throw ex;
} catch (Throwable ex) {
Log.e(Helper.TAG, folder.name + " " + ex + "\n" + Log.getStackTraceString(ex));
} finally {
((IMAPMessage) isub[j]).invalidateHeaders();
}
}
db.folder().setFolderState(folder.id, "downloading");
//fp.add(IMAPFolder.FetchProfileItem.MESSAGE);
// Download messages/attachments
Log.i(Helper.TAG, folder.name + " download=" + imessages.length);
for (int i = imessages.length - 1; i >= 0; i--)
if (ids[i] != null)
for (int i = imessages.length - 1; i >= 0; i -= DOWNLOAD_BATCH_SIZE) {
int from = Math.max(0, i - DOWNLOAD_BATCH_SIZE) + 1;
Log.i(Helper.TAG, folder.name + " download " + from + " .. " + i);
Message[] isub = Arrays.copyOfRange(imessages, from, i + 1);
ifolder.fetch(isub, fp);
for (int j = isub.length - 1; j >= 0; j--)
try {
downloadMessage(this, folder, ids[i], (IMAPMessage) imessages[i]);
Log.i(Helper.TAG, folder.name + " download index=" + (from + j) + " id=" + ids[from + j]);
if (ids[i - j] != null)
downloadMessage(this, folder, ids[i - j], (IMAPMessage) isub[j]);
} catch (FolderClosedException ex) {
throw ex;
} catch (FolderClosedIOException ex) {
throw ex;
} catch (Throwable ex) {
Log.e(Helper.TAG, folder.name + " " + ex + "\n" + Log.getStackTraceString(ex));
} finally {
// Free memory
((IMAPMessage) isub[j]).invalidateHeaders();
}
}
} finally {
@ -1357,15 +1412,7 @@ public class ServiceSynchronize extends LifecycleService {
}
static Long synchronizeMessage(Context context, EntityFolder folder, IMAPFolder ifolder, IMAPMessage imessage, boolean found) throws MessagingException, IOException {
long uid;
try {
FetchProfile fp = new FetchProfile();
fp.add(UIDFolder.FetchProfileItem.UID);
fp.add(IMAPFolder.FetchProfileItem.FLAGS);
ifolder.fetch(new Message[]{imessage}, fp);
uid = ifolder.getUID(imessage);
//Log.v(Helper.TAG, folder.name + " start sync uid=" + uid);
long uid = ifolder.getUID(imessage);
if (imessage.isExpunged()) {
Log.i(Helper.TAG, folder.name + " expunged uid=" + uid);
@ -1415,12 +1462,6 @@ public class ServiceSynchronize extends LifecycleService {
}
if (message == null) {
FetchProfile fp1 = new FetchProfile();
fp1.add(FetchProfile.Item.ENVELOPE);
fp1.add(FetchProfile.Item.CONTENT_INFO);
fp1.add(IMAPFolder.FetchProfileItem.HEADERS);
ifolder.fetch(new Message[]{imessage}, fp1);
message = new EntityMessage();
message.account = folder.account;
message.folder = folder.id;
@ -1497,12 +1538,6 @@ public class ServiceSynchronize extends LifecycleService {
} finally {
db.endTransaction();
}
} finally {
//Log.v(Helper.TAG, folder.name + " end sync uid=" + uid);
// Free memory
imessage.invalidateHeaders();
}
}
private static void downloadMessage(Context context, EntityFolder folder, long id, IMAPMessage imessage) throws MessagingException, IOException {
@ -1520,17 +1555,19 @@ public class ServiceSynchronize extends LifecycleService {
Log.i(Helper.TAG, folder.name + " downloaded message id=" + message.id + " size=" + message.size);
}
if (db.attachment().getAttachmentDownloadCount(id) > 0) {
int sequence = 1;
for (EntityAttachment a : helper.getAttachments()) {
EntityAttachment attachment = db.attachment().getAttachment(id, sequence++);
if (!attachment.available)
if (!metered || (attachment.size != null && attachment.size < ATTACHMENT_AUTO_DOWNLOAD_SIZE)) {
if (attachment.size != null && attachment.size < ATTACHMENT_AUTO_DOWNLOAD_SIZE) {
attachment.part = a.part;
attachment.download(context, db);
Log.i(Helper.TAG, folder.name + " downloaded message id=" + message.id + " attachment=" + attachment.name + " size=" + message.size);
}
}
}
}
private class ServiceManager extends ConnectivityManager.NetworkCallback {
private ServiceState state;

@ -22,6 +22,7 @@ package eu.faircode.email;
public class TupleFolderEx extends EntityFolder {
public String accountName;
public int messages;
public int content;
public int unseen;
@Override
@ -31,6 +32,7 @@ public class TupleFolderEx extends EntityFolder {
return (super.equals(obj) &&
(this.accountName == null ? other.accountName == null : accountName.equals(other.accountName)) &&
this.messages == other.messages &&
this.content == other.content &&
this.unseen == other.unseen);
} else
return false;

@ -248,6 +248,26 @@
app:layout_constraintStart_toEndOf="@id/ivSynchronizing"
app:layout_constraintTop_toTopOf="@id/ivSynchronizing" />
<ImageView
android:id="@+id/ivDownloading"
android:layout_width="24dp"
android:layout_height="24dp"
android:layout_marginTop="12dp"
android:src="@drawable/baseline_get_app_24"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/tvSynchronizing" />
<TextView
android:id="@+id/tvDownloading"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="18dp"
android:text="@string/title_legend_downloading"
android:textAppearance="@style/TextAppearance.AppCompat.Medium"
app:layout_constraintBottom_toBottomOf="@id/ivDownloading"
app:layout_constraintStart_toEndOf="@id/ivDownloading"
app:layout_constraintTop_toTopOf="@id/ivDownloading" />
<ImageView
android:id="@+id/ivClosing"
android:layout_width="24dp"
@ -255,7 +275,7 @@
android:layout_marginTop="12dp"
android:src="@drawable/baseline_close_24"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/tvSynchronizing" />
app:layout_constraintTop_toBottomOf="@id/tvDownloading" />
<TextView
android:id="@+id/tvClosing"

@ -231,6 +231,7 @@
<string name="title_legend_connecting">Connecting</string>
<string name="title_legend_connected">Connected</string>
<string name="title_legend_synchronizing">Synchronizing</string>
<string name="title_legend_downloading">Downloading</string>
<string name="title_legend_closing">Closing</string>
<string name="title_hint_folder_actions">Long press for options</string>

Loading…
Cancel
Save