Decode smtp.mailfrom

app/src/main/java/eu/faircode/email/AdapterMessage.java

@@ -1108,6 +1108,7 @@ public class AdapterMessage extends RecyclerView.Adapter<AdapterMessage.ViewHold
                     !((Boolean.FALSE.equals(message.dkim) && check_authentication) ||
                             (Boolean.FALSE.equals(message.spf) && check_authentication) ||
                             (Boolean.FALSE.equals(message.dmarc) && check_authentication) ||
+                            (Boolean.FALSE.equals(message.from_domain) && BuildConfig.DEBUG) ||
                             (Boolean.FALSE.equals(message.reply_domain) && check_reply_domain) ||
                             (Boolean.FALSE.equals(message.mx) && check_mx) ||
                             (Boolean.TRUE.equals(message.blocklist) && check_blocklist));
@@ -3836,6 +3837,13 @@ public class AdapterMessage extends RecyclerView.Adapter<AdapterMessage.ViewHold
                 sb.append(context.getString(R.string.title_on_blocklist));
             }
 
+            if (Boolean.FALSE.equals(message.from_domain) && BuildConfig.DEBUG) {
+                if (sb.length() > 0)
+                    sb.append('\n');
+                for (String domain : message.checkFromDomain(context))
+                    sb.append(domain).append(' ');
+            }
+
             if (Boolean.FALSE.equals(message.reply_domain)) {
                 String[] warning = message.checkReplyDomain(context);
                 if (warning != null) {
@@ -6203,7 +6211,10 @@ public class AdapterMessage extends RecyclerView.Adapter<AdapterMessage.ViewHold
                     log("msgid changed", next.id);
                 }
                 // references
-                // deliveredto
+                if (!Objects.equals(prev.deliveredto, next.deliveredto)) {
+                    same = false;
+                    log("deliveredto changed", next.id);
+                }
                 // inreplyto
                 if (!Objects.equals(prev.thread, next.thread)) {
                     same = false;
@@ -6249,6 +6260,10 @@ public class AdapterMessage extends RecyclerView.Adapter<AdapterMessage.ViewHold
                     same = false;
                     log("blocklist changed", next.id);
                 }
+                if (!Objects.equals(prev.from_domain, next.from_domain)) {
+                    same = false;
+                    log("from_domain changed", next.id);
+                }
                 if (!Objects.equals(prev.reply_domain, next.reply_domain)) {
                     same = false;
                     log("reply_domain changed", next.id);
@@ -6261,6 +6276,12 @@ public class AdapterMessage extends RecyclerView.Adapter<AdapterMessage.ViewHold
                     same = false;
                     log("sender changed", next.id);
                 }
+                // return_path
+                // smtp_from
+                if (!MessageHelper.equal(prev.submitter, next.submitter)) {
+                    same = false;
+                    log("submitter changed", next.id);
+                }
                 if (!MessageHelper.equal(prev.from, next.from)) {
                     same = false;
                     log("from changed", next.id);

app/src/main/java/eu/faircode/email/Core.java

@@ -98,7 +98,6 @@ import java.util.Calendar;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
-import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -2740,6 +2739,7 @@ class Core {
                         if (message.spf == null && helper.getSPF())
                             message.spf = true;
                         message.dmarc = MessageHelper.getAuthentication("dmarc", authentication);
+                        message.smtp_from = helper.getMailFrom(authentication);
                         message.return_path = helper.getReturnPath();
                         message.submitter = helper.getSender();
                         message.from = helper.getFrom();
@@ -2796,7 +2796,11 @@ class Core {
                         if (message.avatar == null && notify_known && pro)
                             message.ui_ignored = true;
 
+                        message.from_domain = (message.checkFromDomain(context) == null);
+
+                        // No reply_domain
                         // No MX check
+                        // No blocklist
 
                         boolean needsHeaders = EntityRule.needsHeaders(rules);
                         List<Header> headers = (needsHeaders ? helper.getAllHeaders() : null);
@@ -3705,6 +3709,7 @@ class Core {
             if (message.spf == null && helper.getSPF())
                 message.spf = true;
             message.dmarc = MessageHelper.getAuthentication("dmarc", authentication);
+            message.smtp_from = helper.getMailFrom(authentication);
             message.return_path = helper.getReturnPath();
             message.submitter = helper.getSender();
             message.from = helper.getFrom();
@@ -3778,6 +3783,8 @@ class Core {
             if (message.avatar == null && notify_known && pro)
                 message.ui_ignored = true;
 
+            message.from_domain = (message.checkFromDomain(context) == null);
+
             // For contact forms
             boolean self = false;
             if (identity != null && message.from != null)

app/src/main/java/eu/faircode/email/DB.java

@@ -68,7 +68,7 @@ import io.requery.android.database.sqlite.SQLiteDatabase;
 // https://developer.android.com/topic/libraries/architecture/room.html
 
 @Database(
-        version = 217,
+        version = 218,
         entities = {
                 EntityIdentity.class,
                 EntityAccount.class,
@@ -2207,6 +2207,13 @@ public abstract class DB extends RoomDatabase {
                         Log.i("DB migration from version " + startVersion + " to " + endVersion);
                         db.execSQL("ALTER TABLE `folder` ADD COLUMN `last_sync_foreground` INTEGER");
                     }
+                }).addMigrations(new Migration(217, 218) {
+                    @Override
+                    public void migrate(@NonNull SupportSQLiteDatabase db) {
+                        Log.i("DB migration from version " + startVersion + " to " + endVersion);
+                        db.execSQL("ALTER TABLE `message` ADD COLUMN `smtp_from` TEXT");
+                        db.execSQL("ALTER TABLE `message` ADD COLUMN `from_domain` INTEGER");
+                    }
                 }).addMigrations(new Migration(998, 999) {
                     @Override
                     public void migrate(@NonNull SupportSQLiteDatabase db) {

app/src/main/java/eu/faircode/email/EntityMessage.java

@@ -147,10 +147,12 @@ public class EntityMessage implements Serializable {
     public Boolean dmarc;
     public Boolean mx;
     public Boolean blocklist;
-    public Boolean reply_domain; // differs from 'from'
+    public Boolean from_domain; // spf/smtp.mailfrom <> from
+    public Boolean reply_domain; // reply-to <> from
     public String avatar; // lookup URI from sender
     public String sender; // sort key: from email address
     public Address[] return_path;
+    public Address[] smtp_from;
     public Address[] submitter; // sent on behalf of
     public Address[] from;
     public Address[] to;
@@ -307,32 +309,12 @@ public class EntityMessage implements Serializable {
         return hasKeyword(MessageHelper.FLAG_FORWARDED);
     }
 
-    String[] checkReplyDomain(Context context) {
-        if (from == null || from.length == 0)
-            return null;
-        if (reply == null || reply.length == 0)
-            return null;
-
-        for (Address _reply : reply) {
-            String r = ((InternetAddress) _reply).getAddress();
-            int rat = (r == null ? -1 : r.indexOf('@'));
-            if (rat < 0)
-                continue;
-            String rdomain = UriHelper.getParentDomain(context, r.substring(rat + 1));
-
-            for (Address _from : from) {
-                String f = ((InternetAddress) _from).getAddress();
-                int fat = (f == null ? -1 : f.indexOf('@'));
-                if (fat < 0)
-                    continue;
-                String fdomain = UriHelper.getParentDomain(context, f.substring(fat + 1));
-
-                if (!rdomain.equalsIgnoreCase(fdomain))
-                    return new String[]{fdomain, rdomain};
-            }
-        }
+    String[] checkFromDomain(Context context) {
+        return MessageHelper.equalDomain(context, from, smtp_from);
+    }
 
-        return null;
+    String[] checkReplyDomain(Context context) {
+        return MessageHelper.equalDomain(context, reply, from);
     }
 
     static String collapsePrefixes(Context context, String language, String subject, boolean forward) {
@@ -534,10 +516,12 @@ public class EntityMessage implements Serializable {
                     Objects.equals(this.dmarc, other.dmarc) &&
                     Objects.equals(this.mx, other.mx) &&
                     Objects.equals(this.blocklist, other.blocklist) &&
+                    Objects.equals(this.from_domain, other.from_domain) &&
                     Objects.equals(this.reply_domain, other.reply_domain) &&
                     Objects.equals(this.avatar, other.avatar) &&
                     Objects.equals(this.sender, other.sender) &&
                     MessageHelper.equal(this.return_path, other.return_path) &&
+                    MessageHelper.equal(this.smtp_from, other.smtp_from) &&
                     MessageHelper.equal(this.submitter, other.submitter) &&
                     MessageHelper.equal(this.from, other.from) &&
                     MessageHelper.equal(this.to, other.to) &&

app/src/main/java/eu/faircode/email/MessageHelper.java

@@ -139,6 +139,7 @@ public class MessageHelper {
     private static final String DOCTYPE = "<!DOCTYPE";
     private static final String HTML_START = "<html>";
     private static final String HTML_END = "</html>";
+    private static final String SMTP_MAILFORM = "smtp.mailfrom";
 
     private static final List<Charset> CHARSET16 = Collections.unmodifiableList(Arrays.asList(
             StandardCharsets.UTF_16,
@@ -1477,6 +1478,35 @@ public class MessageHelper {
         return (spf.trim().toLowerCase(Locale.ROOT).startsWith("pass"));
     }
 
+    Address[] getMailFrom(String[] headers) {
+        if (headers == null)
+            return null;
+
+        Address[] mailfrom = null;
+        for (String header : headers) {
+            String spf = getKeyValues(header).get("spf");
+            if (spf == null)
+                continue;
+
+            int i = spf.indexOf(SMTP_MAILFORM + "=");
+            if (i < 0)
+                continue;
+
+            String v = spf.substring(i + SMTP_MAILFORM.length() + 1);
+            int s = v.indexOf(' ');
+            if (s > 0)
+                v = v.substring(0, s);
+
+            try {
+                mailfrom = InternetAddress.parseHeader(v, false);
+            } catch (Throwable ex) {
+                Log.w(ex);
+            }
+        }
+
+        return mailfrom;
+    }
+
     private String fixEncoding(String name, String header) {
         if (header.trim().startsWith("=?"))
             return header;
@@ -1565,6 +1595,7 @@ public class MessageHelper {
         Address[] sender = getAddressHeader("X-Google-Original-From");
         if (sender == null)
             sender = getAddressHeader("Sender");
+
         return sender;
     }
 
@@ -3510,6 +3541,32 @@ public class MessageHelper {
         return true;
     }
 
+    static String[] equalDomain(Context context, Address[] a1, Address[] a2) {
+        if (a1 == null || a1.length == 0)
+            return null;
+        if (a2 == null || a2.length == 0)
+            return null;
+
+        for (Address _a1 : a1) {
+            String r = UriHelper.getEmailDomain(((InternetAddress) _a1).getAddress());
+            if (r == null)
+                continue;
+            String d1 = UriHelper.getParentDomain(context, r);
+
+            for (Address _a2 : a2) {
+                String f = UriHelper.getEmailDomain(((InternetAddress) _a2).getAddress());
+                if (f == null)
+                    continue;
+                String d2 = UriHelper.getParentDomain(context, f);
+
+                if (!d1.equalsIgnoreCase(d2))
+                    return new String[]{d2, d1};
+            }
+        }
+
+        return null;
+    }
+
     static boolean equal(Address[] a1, Address[] a2) {
         if (a1 == null && a2 == null)
             return true;