Use message headers hash to identify messages

app/src/main/java/eu/faircode/email/AdapterMessage.java

@@ -1051,6 +1051,7 @@ public class AdapterMessage extends RecyclerView.Adapter<AdapterMessage.ViewHold
                         " unseen=" + message.unseen +
                         " ignored=" + message.ui_ignored +
                         " found=" + message.ui_found +
+                        "\nhash=" + message.hash +
                         "\nmsgid=" + message.msgid +
                         "\nthread=" + message.thread +
                         "\nsender=" + message.sender;

app/src/main/java/eu/faircode/email/Core.java

@@ -78,6 +78,7 @@ import java.net.InetAddress;
 import java.net.SocketException;
 import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
+import java.security.NoSuchAlgorithmException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -2133,6 +2134,12 @@ class Core {
             if (TextUtils.isEmpty(message.msgid))
                 Log.w("No Message-ID id=" + message.id + " uid=" + message.uid);
 
+            try {
+                message.hash = Helper.sha1(helper.getHeaders().getBytes());
+            } catch (NoSuchAlgorithmException ex) {
+                Log.e(ex);
+            }
+
             message.references = TextUtils.join(" ", helper.getReferences());
             message.inreplyto = helper.getInReplyTo();
             // Local address contains control or whitespace in string ``mailing list someone@example.org''

app/src/main/java/eu/faircode/email/DB.java

@@ -60,7 +60,7 @@ import io.requery.android.database.sqlite.SQLiteDatabase;
 // https://developer.android.com/topic/libraries/architecture/room.html
 
 @Database(
-        version = 151,
+        version = 152,
         entities = {
                 EntityIdentity.class,
                 EntityAccount.class,
@@ -1436,6 +1436,13 @@ public abstract class DB extends RoomDatabase {
                         Log.i("DB migration from version " + startVersion + " to " + endVersion);
                         db.execSQL("ALTER TABLE `account` ADD COLUMN `keep_alive_succeeded` INTEGER NOT NULL DEFAULT 0");
                     }
+                })
+                .addMigrations(new Migration(151, 152) {
+                    @Override
+                    public void migrate(@NonNull SupportSQLiteDatabase db) {
+                        Log.i("DB migration from version " + startVersion + " to " + endVersion);
+                        db.execSQL("ALTER TABLE `message` ADD COLUMN `hash` TEXT");
+                    }
                 });
     }
 

app/src/main/java/eu/faircode/email/DaoMessage.java

@@ -55,7 +55,10 @@ public interface DaoMessage {
             ", SUM(folder.type = '" + EntityFolder.DRAFTS + "') AS drafts" +
             ", (message.ui_encrypt IN (2, 4)) AS signed" +
             ", (message.ui_encrypt IN (1, 3)) AS encrypted" +
-            ", COUNT(DISTINCT CASE WHEN message.msgid IS NULL THEN message.id ELSE message.msgid END) AS visible" +
+            ", COUNT(DISTINCT" +
+            "   CASE WHEN NOT message.message.hash IS NULL THEN message.hash" +
+            "   WHEN NOT message.msgid IS NULL THEN message.msgid" +
+            "   ELSE message.id END) AS visible" +
             ", SUM(message.total) AS totalSize" +
             ", MAX(message.priority) AS ui_priority" +
             ", MAX(message.importance) AS ui_importance" +
@@ -115,7 +118,10 @@ public interface DaoMessage {
             ", SUM(folder.type = '" + EntityFolder.DRAFTS + "') AS drafts" +
             ", (message.ui_encrypt IN (2, 4)) AS signed" +
             ", (message.ui_encrypt IN (1, 3)) AS encrypted" +
-            ", COUNT(DISTINCT CASE WHEN message.msgid IS NULL THEN message.id ELSE message.msgid END) AS visible" +
+            ", COUNT(DISTINCT" +
+            "   CASE WHEN NOT message.message.hash IS NULL THEN message.hash" +
+            "   WHEN NOT message.msgid IS NULL THEN message.msgid" +
+            "   ELSE message.id END) AS visible" +
             ", SUM(message.total) AS totalSize" +
             ", MAX(message.priority) AS ui_priority" +
             ", MAX(message.importance) AS ui_importance" +
@@ -182,7 +188,9 @@ public interface DaoMessage {
             " AND message.thread = :thread" +
             " AND (:id IS NULL OR message.id = :id)" +
             " AND (NOT :filter_archive OR folder.type <> '" + EntityFolder.ARCHIVE +
-            "' OR (SELECT COUNT(m.id) FROM message m WHERE m.account = message.account AND m.msgid = message.msgid) = 1)" +
+            "' OR (SELECT COUNT(m.id) FROM message m" +
+            "   WHERE m.account = message.account" +
+            "   AND (m.hash = message.hash OR m.msgid = message.msgid)) = 1)" +
             " AND (NOT message.ui_hide OR :debug)" +
             " ORDER BY CASE WHEN :ascending THEN message.received ELSE -message.received END" +
             ", CASE" +

app/src/main/java/eu/faircode/email/EntityMessage.java

@@ -102,6 +102,7 @@ public class EntityMessage implements Serializable {
     public Long forwarding; // obsolete
     public Long uid; // compose/moved = null
     public String msgid;
+    public String hash; // headers hash
     public String references;
     public String deliveredto;
     public String inreplyto;