msb-public
/
FairEmail
mirror of https://github.com/M66B/FairEmail.git
1
0
Code Issues Projects Releases Wiki Activity

Cloud sync: refactoring

Browse Source
pull/212/head
M66B 2 years ago
parent 617d2d964c
commit f27110acaa
2 changed files with 251 additions and 62 deletions
Show Stats Download Patch File Download Diff File

103
app/src/main/java/eu/faircode/email/CloudSync.java
Unescape View File

@ -35,7 +35,11 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.security.spec.InvalidKeySpecException; import java.security.spec.InvalidKeySpecException;
import java.security.spec.KeySpec; import java.security.spec.KeySpec;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.crypto.Cipher; import javax.crypto.Cipher;
import javax.crypto.SecretKey; import javax.crypto.SecretKey;
@ -47,15 +51,57 @@ import javax.net.ssl.HttpsURLConnection;
public class CloudSync { public class CloudSync {
private static final int CLOUD_TIMEOUT = 10 * 1000; // timeout private static final int CLOUD_TIMEOUT = 10 * 1000; // timeout
private static final int BATCH_SIZE = 25;
private static final Map<String, Pair<byte[], byte[]>> keyCache = new HashMap<>();
public static JSONObject perform(Context context, String user, String password, JSONObject jrequest) public static JSONObject perform(Context context, String user, String password, JSONObject jrequest)
throws GeneralSecurityException, JSONException, IOException { throws GeneralSecurityException, JSONException, IOException {
List<JSONObject> responses = new ArrayList<>();
for (JSONArray batch : partition(jrequest.getJSONArray("items"))) {
jrequest.put("items", batch);
responses.add(_perform(context, user, password, jrequest));
}
if (responses.size() == 1)
return responses.get(0);
else {
int count = 0;
JSONArray jall = new JSONArray();
for (JSONObject response : responses) {
JSONArray jitems = response.getJSONArray("items");
for (int i = 0; i < jitems.length(); i++)
jall.put(jitems.getJSONObject(i));
count += response.optInt("count", 0);
}
JSONObject jresponse = responses.get(0);
jresponse.put("items", jall);
jresponse.put("count", count);
return jresponse;
}
}
private static JSONObject _perform(Context context, String user, String password, JSONObject jrequest)
throws GeneralSecurityException, JSONException, IOException {
byte[] salt = MessageDigest.getInstance("SHA256").digest(user.getBytes()); byte[] salt = MessageDigest.getInstance("SHA256").digest(user.getBytes());
byte[] huser = MessageDigest.getInstance("SHA256").digest(salt); byte[] huser = MessageDigest.getInstance("SHA256").digest(salt);
byte[] userid = Arrays.copyOfRange(huser, 0, 8); byte[] userid = Arrays.copyOfRange(huser, 0, 8);
String cloudUser = Base64.encodeToString(userid, Base64.NO_PADDING | Base64.NO_WRAP); String cloudUser = Base64.encodeToString(userid, Base64.NO_PADDING | Base64.NO_WRAP);
Pair<byte[], byte[]> key = getKeyPair(salt, password); Pair<byte[], byte[]> key;
String lookup = Helper.hex(salt) + ":" + password;
synchronized (keyCache) {
key = keyCache.get(lookup);
}
if (key == null) {
Log.i("Cloud generating key");
key = getKeyPair(salt, password);
synchronized (keyCache) {
keyCache.put(lookup, key);
}
} else {
Log.i("Cloud using cached key");
}
String cloudPassword = Base64.encodeToString(key.first, Base64.NO_PADDING | Base64.NO_WRAP); String cloudPassword = Base64.encodeToString(key.first, Base64.NO_PADDING | Base64.NO_WRAP);
jrequest.put("version", 1); jrequest.put("version", 1);
@ -67,20 +113,24 @@ public class CloudSync {
JSONArray jitems = jrequest.getJSONArray("items"); JSONArray jitems = jrequest.getJSONArray("items");
for (int i = 0; i < jitems.length(); i++) { for (int i = 0; i < jitems.length(); i++) {
JSONObject jitem = jitems.getJSONObject(i); JSONObject jitem = jitems.getJSONObject(i);
int revision = jitem.getInt("revision"); int revision = jitem.getInt("rev");
String k = jitem.getString("key"); String k = jitem.getString("key");
jitem.put("key", transform(k, key.second, null, true)); jitem.put("key", transform(k, key.second, null, true));
if (jitem.has("value") && !jitem.isNull("value")) { String v = null;
String v = jitem.getString("value"); if (jitem.has("val") && !jitem.isNull("val")) {
jitem.put("value", transform(v, key.second, revision, true)); v = jitem.getString("val");
jitem.put("val", transform(v, key.second, revision, true));
} }
v = (v == null ? null : "#" + v.length());
Log.i("Cloud > " + k + "=" + v + " @" + revision);
} }
} }
String request = jrequest.toString(); String request = jrequest.toString();
Log.i("Cloud request=" + request); Log.i("Cloud request length=" + request.length());
URL url = new URL(BuildConfig.CLOUD_URI); URL url = new URL(BuildConfig.CLOUD_URI);
HttpsURLConnection connection = (HttpsURLConnection) url.openConnection(); HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
@ -111,22 +161,28 @@ public class CloudSync {
} }
String response = Helper.readStream(connection.getInputStream()); String response = Helper.readStream(connection.getInputStream());
Log.i("Cloud response=" + response); Log.i("Cloud response length=" + response.length());
JSONObject jresponse = new JSONObject(response); JSONObject jresponse = new JSONObject(response);
if (jresponse.has("items")) { if (jresponse.has("items")) {
JSONArray jitems = jresponse.getJSONArray("items"); JSONArray jitems = jresponse.getJSONArray("items");
for (int i = 0; i < jitems.length(); i++) { for (int i = 0; i < jitems.length(); i++) {
JSONObject jitem = jitems.getJSONObject(i); JSONObject jitem = jitems.getJSONObject(i);
int revision = jitem.getInt("revision"); int revision = jitem.getInt("rev");
String ekey = jitem.getString("key"); String ekey = jitem.getString("key");
jitem.put("key", transform(ekey, key.second, null, false)); String k = transform(ekey, key.second, null, false);
jitem.put("key", k);
if (jitem.has("value") && !jitem.isNull("value")) {
String evalue = jitem.getString("value"); String v = null;
jitem.put("value", transform(evalue, key.second, revision, false)); if (jitem.has("val") && !jitem.isNull("val")) {
String evalue = jitem.getString("val");
v = transform(evalue, key.second, revision, false);
jitem.put("val", v);
} }
v = (v == null ? null : "#" + v.length());
Log.i("Cloud < " + k + "=" + v + " @" + revision);
} }
} }
@ -166,4 +222,25 @@ public class CloudSync {
return new String(decrypted); return new String(decrypted);
} }
} }
private static List<JSONArray> partition(JSONArray jarray) throws JSONException {
if (jarray.length() <= BATCH_SIZE)
return Arrays.asList(jarray);
int count = 0;
List<JSONArray> jpartitions = new ArrayList<>();
for (int i = 0; i < jarray.length(); i += BATCH_SIZE) {
JSONArray jpartition = new JSONArray();
for (int j = 0; j < BATCH_SIZE && i + j < jarray.length(); j++) {
count++;
jpartition.put(jarray.get(i + j));
}
jpartitions.add(jpartition);
}
if (count != jarray.length())
throw new IllegalArgumentException("Partition error size=" + count + "/" + jarray.length());
return jpartitions;
}
} }

210
app/src/main/java/eu/faircode/email/FragmentOptionsBackup.java
Unescape View File

@ -1473,11 +1473,6 @@ public class FragmentOptionsBackup extends FragmentBase implements SharedPrefere
return intent; return intent;
} }
private void onCloudSync() {
Bundle args = new Bundle();
cloud(args);
}
private void onCloudLogin() { private void onCloudLogin() {
String username = etUser.getText().toString().trim(); String username = etUser.getText().toString().trim();
String password = tilPassword.getEditText().getText().toString(); String password = tilPassword.getEditText().getText().toString();
@ -1499,13 +1494,20 @@ public class FragmentOptionsBackup extends FragmentBase implements SharedPrefere
.apply(); .apply();
Bundle args = new Bundle(); Bundle args = new Bundle();
args.putString("command", "login");
cloud(args);
}
private void onCloudSync() {
Bundle args = new Bundle();
args.putString("command", "sync");
cloud(args); cloud(args);
} }
private void onCloudLogout() { private void onCloudLogout() {
boolean wipe = cbDelete.isChecked();
Bundle args = new Bundle(); Bundle args = new Bundle();
args.putBoolean("logout", true); args.putString("command", wipe ? "wipe" : "logout");
args.putBoolean("wipe", cbDelete.isChecked());
cloud(args); cloud(args);
} }
@ -1515,7 +1517,7 @@ public class FragmentOptionsBackup extends FragmentBase implements SharedPrefere
args.putString("user", prefs.getString("cloud_user", null)); args.putString("user", prefs.getString("cloud_user", null));
args.putString("password", prefs.getString("cloud_password", null)); args.putString("password", prefs.getString("cloud_password", null));
new SimpleTask<String>() { new SimpleTask<Void>() {
@Override @Override
protected void onPreExecute(Bundle args) { protected void onPreExecute(Bundle args) {
Helper.setViewsEnabled(cardCloud, false); Helper.setViewsEnabled(cardCloud, false);
@ -1527,65 +1529,175 @@ public class FragmentOptionsBackup extends FragmentBase implements SharedPrefere
} }
@Override @Override
protected String onExecute(Context context, Bundle args) throws Throwable { protected Void onExecute(Context context, Bundle args) throws Throwable {
String user = args.getString("user"); String user = args.getString("user");
String password = args.getString("password"); String password = args.getString("password");
boolean wipe = args.getBoolean("wipe"); String command = args.getString("command");
JSONObject jrequest = new JSONObject(); JSONObject jrequest = new JSONObject();
jrequest.put("command", wipe ? "wipe" : "login"); jrequest.put("command", command);
if (true) { if ("sync".equals(command)) {
boolean sync = true; DB db = DB.getInstance(context);
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
int sync_status = prefs.getInt("sync_status", 0);
JSONObject jstatus = new JSONObject();
jstatus.put("key", "sync.status");
jstatus.put("rev", 0);
JSONArray jitems = new JSONArray(); JSONArray jitems = new JSONArray();
jitems.put(jstatus);
JSONObject jkv0 = new JSONObject();
jkv0.put("key", "key0");
jkv0.put("revision", 1000);
if (!sync)
jkv0.put("value", "value0");
jitems.put(jkv0);
JSONObject jkv1 = new JSONObject();
jkv1.put("key", "key1");
jkv1.put("revision", 1001);
if (!sync)
jkv1.put("value", "value1");
jitems.put(jkv1);
jrequest.put("command", sync ? "read" : "write");
if (sync)
jrequest.put("compare", true);
jrequest.put("items", jitems); jrequest.put("items", jitems);
}
JSONObject jresponse = CloudSync.perform(context, user, password, jrequest); JSONObject jresponse = CloudSync.perform(context, user, password, jrequest);
jitems = jresponse.getJSONArray("items");
int count = jresponse.getInt("count");
if (count == 0) {
Log.i("Cloud server is empty");
List<EntityAccount> accounts = db.account().getSynchronizingAccounts(null);
Log.i("Cloud accounts=" + (accounts == null ? null : accounts.size()));
if (accounts != null && accounts.size() != 0) {
JSONArray jupload = new JSONArray();
JSONArray juuids = new JSONArray();
for (EntityAccount account : accounts)
if (!TextUtils.isEmpty(account.uuid)) {
juuids.put(account.uuid);
JSONArray jidentities = new JSONArray();
List<EntityIdentity> identities = db.identity().getIdentities(account.id);
if (identities != null)
for (EntityIdentity identity : identities)
if (!TextUtils.isEmpty(identity.uuid)) {
jidentities.put(identity.uuid);
JSONObject jitem = new JSONObject();
jitem.put("key", "identity." + identity.uuid);
jitem.put("val", identity.toJSON().toString());
jitem.put("rev", 1);
jupload.put(jitem);
}
JSONObject jaccountdata = new JSONObject();
jaccountdata.put("account", account.toJSON());
jaccountdata.put("identities", jidentities);
if (jresponse.has("items")) { JSONObject jitem = new JSONObject();
JSONArray jitems = jresponse.getJSONArray("items"); jitem.put("key", "account." + account.uuid);
for (int i = 0; i < jitems.length(); i++) { jitem.put("val", jaccountdata.toString());
JSONObject jitem = jitems.getJSONObject(i); jitem.put("rev", 1);
String key = jitem.getString("key"); jupload.put(jitem);
long revision = jitem.getLong("revision"); }
String value = (jitem.has("value") ? jitem.getString("value") : null);
Log.i("Cloud item " + key + "=" + value + " @" + revision); JSONObject jstatusdata = new JSONObject();
} jstatusdata.put("accounts", juuids);
}
return jresponse.optString("status"); jstatus.put("key", "sync.status");
jstatus.put("val", jstatusdata.toString());
jstatus.put("rev", 1);
jupload.put(jstatus);
jrequest.put("command", "write");
jrequest.put("items", jupload);
CloudSync.perform(context, user, password, jrequest);
prefs.edit().putInt("sync_status", 1).apply();
}
return null;
} else if (count == 1) {
if (jitems.length() == 1) {
// New revision
JSONObject jitem = jitems.getJSONObject(0);
Log.i("Cloud status revision=" + jitem.getInt("rev") + "/" + sync_status);
JSONArray jdownload = new JSONArray();
// Get accounts
JSONObject jstatusdata = new JSONObject(jitem.getString("val"));
JSONArray juuids = jstatusdata.getJSONArray("accounts");
for (int i = 0; i < juuids.length(); i++) {
String uuid = juuids.getString(i);
JSONObject jaccount = new JSONObject();
jaccount.put("key", "account." + uuid);
jaccount.put("rev", 0);
jdownload.put(jaccount);
Log.i("Cloud account " + uuid);
}
jrequest.put("items", jdownload);
jresponse = CloudSync.perform(context, user, password, jrequest);
// Process accounts
jitems = jresponse.getJSONArray("items");
jdownload = new JSONArray();
for (int i = 0; i < jitems.length(); i++) {
JSONObject jaccount = jitems.getJSONObject(i);
String key = jaccount.getString("key");
String value = jaccount.getString("val");
int revision = jaccount.getInt("rev");
String uuid = key.split("\\.")[1];
EntityAccount account = db.account().getAccountByUUID(uuid);
JSONObject jaccountdata = new JSONObject(value);
JSONArray jidentities = jaccountdata.getJSONArray("identities");
Log.i("Cloud account " + uuid + "=" + (account != null) +
" rev=" + revision +
" identities=" + jidentities +
" size=" + value.length());
for (int j = 0; j < jidentities.length(); j++) {
JSONObject jidentity = new JSONObject();
jidentity.put("key", "identity." + jidentities.getString(j));
jidentity.put("rev", 0);
jdownload.put(jidentity);
}
}
// Get identities
jrequest.put("items", jdownload);
jresponse = CloudSync.perform(context, user, password, jrequest);
// Process identities
jitems = jresponse.getJSONArray("items");
for (int i = 0; i < jitems.length(); i++) {
JSONObject jaccount = jitems.getJSONObject(i);
String key = jaccount.getString("key");
String value = jaccount.getString("val");
int revision = jaccount.getInt("rev");
String uuid = key.split("\\.")[1];
EntityIdentity identity = db.identity().getIdentityByUUID(uuid);
Log.i("Cloud identity " + uuid + "=" + (identity != null) +
" rev=" + revision +
" size=" + value.length());
}
} else {
// No changes
}
} else
throw new IllegalArgumentException("Expected one status item");
} else
CloudSync.perform(context, user, password, jrequest);
return null;
} }
@Override @Override
protected void onExecuted(Bundle args, String status) { protected void onExecuted(Bundle args, Void data) {
if ("ok".equals(status) && !args.getBoolean("logout")) String command = args.getString("command");
if ("logout".equals(command))
prefs.edit() prefs.edit()
.putString("cloud_user", args.getString("user")) .remove("cloud_user")
.putString("cloud_password", args.getString("password")) .remove("cloud_password")
.apply(); .apply();
else else
prefs.edit() prefs.edit()
.remove("cloud_user") .putString("cloud_user", args.getString("user"))
.remove("cloud_password") .putString("cloud_password", args.getString("password"))
.apply(); .apply();
view.post(new Runnable() { view.post(new Runnable() {

Write Preview
Loading…
Cancel
Save