Added option to scan full web page for favicons

app/src/main/java/eu/faircode/email/ActivityView.java

@@ -708,7 +708,7 @@ public class ActivityView extends ActivityBilling implements FragmentManager.OnB
                 if (BuildConfig.DEBUG)
                     try {
                         DnsBlockList.clearCache();
-                        ContactInfo.clearCache(ActivityView.this, true);
+                        ContactInfo.clearCache(ActivityView.this);
                         ToastEx.makeText(ActivityView.this, R.string.title_completed, Toast.LENGTH_LONG).show();
                     } catch (Throwable ex) {
                         Log.unexpectedError(getSupportFragmentManager(), ex);

app/src/main/java/eu/faircode/email/ContactInfo.java

@@ -97,6 +97,8 @@ public class ContactInfo {
     private boolean known;
     private long time;
 
+    static final int FAVICON_READ_BYTES = 5000;
+
     private static Map<String, Lookup> emailLookup = new ConcurrentHashMap<>();
     private static final Map<String, ContactInfo> emailContactInfo = new HashMap<>();
 
@@ -111,7 +113,6 @@ public class ContactInfo {
     private static final int GRAVATAR_TIMEOUT = 5 * 1000; // milliseconds
     private static final int FAVICON_CONNECT_TIMEOUT = 5 * 1000; // milliseconds
     private static final int FAVICON_READ_TIMEOUT = 10 * 1000; // milliseconds
-    private static final int FAVICON_READ_BYTES = 4096;
     private static final long CACHE_CONTACT_DURATION = 2 * 60 * 1000L; // milliseconds
     private static final long CACHE_FAVICON_DURATION = 2 * 7 * 24 * 60 * 60 * 1000L; // milliseconds
 
@@ -539,6 +540,9 @@ public class ContactInfo {
     }
 
     private static Favicon parseFavicon(URL base, int scaleToPixels, Context context) throws IOException {
+        SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
+        boolean favicons_partial = prefs.getBoolean("favicons_partial", true);
+
         Log.i("PARSE favicon " + base);
         HttpsURLConnection connection = (HttpsURLConnection) base.openConnection();
         connection.setRequestMethod("GET");
@@ -554,26 +558,28 @@ public class ContactInfo {
         connection.setRequestProperty("User-Agent", WebViewEx.getUserAgent(context));
         connection.connect();
 
-        String response;
+        Document doc;
         try {
-            byte[] buffer = new byte[FAVICON_READ_BYTES];
+            Log.i("Favicon partial=" + favicons_partial);
-            int len = 0;
+            if (favicons_partial) {
-            while (len < buffer.length) {
+                byte[] buffer = new byte[FAVICON_READ_BYTES];
-                int read = connection.getInputStream().read(buffer, len, buffer.length - len);
+                int len = 0;
-                if (read < 0)
+                while (len < buffer.length) {
-                    break;
+                    int read = connection.getInputStream().read(buffer, len, buffer.length - len);
-                else
+                    if (read < 0)
-                    len += read;
+                        break;
-            }
+                    else
-            if (len < 0)
+                        len += read;
-                throw new IOException("length");
+                }
-            response = new String(buffer, 0, len, StandardCharsets.UTF_8.name());
+                if (len < 0)
+                    throw new IOException("length");
+                doc = JsoupEx.parse(new String(buffer, 0, len, StandardCharsets.UTF_8.name()));
+            } else
+                doc = JsoupEx.parse(connection.getInputStream(), StandardCharsets.UTF_8.name(), base.toString());
         } finally {
             connection.disconnect();
         }
 
-        Document doc = JsoupEx.parse(response);
-
         // Use canonical address
         Element canonical = doc.head().select("link[rel=canonical]").first();
         if (canonical != null) {
@@ -663,6 +669,7 @@ public class ContactInfo {
             }
         });
 
+        Log.i("Favicons " + base + "=" + imgs.size());
         for (int i = 0; i < imgs.size(); i++)
             Log.i("Favicon " + i + "=" + imgs.get(i) + " @" + base);
 

app/src/main/java/eu/faircode/email/FragmentDialogTheme.java

@@ -247,14 +247,14 @@ public class FragmentDialogTheme extends FragmentDialogBase {
         if (grpDebug != null)
             grpDebug.setVisibility(debug || BuildConfig.DEBUG ? View.VISIBLE : View.GONE);
 
-        return new AlertDialog.Builder(getContext())
+        return new AlertDialog.Builder(context)
                 .setView(dview)
                 .setPositiveButton(android.R.string.ok, new DialogInterface.OnClickListener() {
                     @Override
                     public void onClick(DialogInterface dialog, int which) {
                         getActivity().getIntent().putExtra("tab", "display");
 
-                        ContactInfo.clearCache(getContext());
+                        ContactInfo.clearCache(context);
 
                         int optionId = rgThemeOptions.getCheckedRadioButtonId();
                         boolean reverse = (swReverse.isEnabled() && swReverse.isChecked());

app/src/main/java/eu/faircode/email/FragmentOptionsDisplay.java

@@ -106,6 +106,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
     private SwitchCompat swGravatars;
     private TextView tvGravatarsHint;
     private SwitchCompat swFavicons;
+    private SwitchCompat swFaviconsPartial;
     private TextView tvFaviconsHint;
     private SwitchCompat swGeneratedIcons;
     private SwitchCompat swIdenticons;
@@ -182,7 +183,8 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
             "nav_options", "nav_count", "nav_unseen_drafts", "navbar_colorize",
             "threading", "threading_unread", "indentation", "seekbar", "actionbar", "actionbar_color",
             "highlight_unread", "highlight_color", "color_stripe", "color_stripe_wide",
-            "avatars", "bimi", "gravatars", "favicons", "generated_icons", "identicons", "circular", "saturation", "brightness", "threshold",
+            "avatars", "bimi", "gravatars", "favicons", "favicons_partial", "generated_icons", "identicons",
+            "circular", "saturation", "brightness", "threshold",
             "email_format", "prefer_contact", "only_contact", "distinguish_contacts", "show_recipients",
             "font_size_sender", "sender_ellipsize",
             "subject_top", "subject_italic", "highlight_subject", "font_size_subject", "subject_ellipsize",
@@ -251,6 +253,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
         swGravatars = view.findViewById(R.id.swGravatars);
         tvGravatarsHint = view.findViewById(R.id.tvGravatarsHint);
         swFavicons = view.findViewById(R.id.swFavicons);
+        swFaviconsPartial = view.findViewById(R.id.swFaviconsPartial);
         tvFaviconsHint = view.findViewById(R.id.tvFaviconsHint);
         swGeneratedIcons = view.findViewById(R.id.swGeneratedIcons);
         swIdenticons = view.findViewById(R.id.swIdenticons);
@@ -657,7 +660,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
             @Override
             public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
                 prefs.edit().putBoolean("avatars", checked).apply();
-                ContactInfo.clearCache(getContext());
+                ContactInfo.clearCache(compoundButton.getContext());
             }
         });
 
@@ -665,7 +668,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
             @Override
             public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
                 prefs.edit().putBoolean("bimi", checked).apply();
-                ContactInfo.clearCache(getContext());
+                ContactInfo.clearCache(compoundButton.getContext());
             }
         });
 
@@ -688,7 +691,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
             @Override
             public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
                 prefs.edit().putBoolean("gravatars", checked).apply();
-                ContactInfo.clearCache(getContext());
+                ContactInfo.clearCache(compoundButton.getContext());
             }
         });
 
@@ -704,7 +707,16 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
             @Override
             public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
                 prefs.edit().putBoolean("favicons", checked).apply();
-                ContactInfo.clearCache(getContext());
+                swFaviconsPartial.setEnabled(checked);
+                ContactInfo.clearCache(compoundButton.getContext());
+            }
+        });
+
+        swFaviconsPartial.setOnCheckedChangeListener(new CompoundButton.OnCheckedChangeListener() {
+            @Override
+            public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
+                prefs.edit().putBoolean("favicons_partial", checked).apply();
+                ContactInfo.clearCache(compoundButton.getContext());
             }
         });
 
@@ -724,7 +736,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
                 sbSaturation.setEnabled(checked);
                 sbBrightness.setEnabled(checked);
                 sbThreshold.setEnabled(checked);
-                ContactInfo.clearCache(getContext());
+                ContactInfo.clearCache(compoundButton.getContext());
             }
         });
 
@@ -732,7 +744,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
             @Override
             public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
                 prefs.edit().putBoolean("identicons", checked).apply();
-                ContactInfo.clearCache(getContext());
+                ContactInfo.clearCache(compoundButton.getContext());
             }
         });
 
@@ -741,7 +753,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
             public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
                 prefs.edit().putBoolean("circular", checked).apply();
                 updateColor();
-                ContactInfo.clearCache(getContext());
+                ContactInfo.clearCache(compoundButton.getContext());
             }
         });
 
@@ -750,7 +762,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
             public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) {
                 prefs.edit().putInt("saturation", progress).apply();
                 updateColor();
-                ContactInfo.clearCache(getContext());
+                ContactInfo.clearCache(seekBar.getContext());
             }
 
             @Override
@@ -769,7 +781,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
             public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) {
                 prefs.edit().putInt("brightness", progress).apply();
                 updateColor();
-                ContactInfo.clearCache(getContext());
+                ContactInfo.clearCache(seekBar.getContext());
             }
 
             @Override
@@ -788,7 +800,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
             public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) {
                 prefs.edit().putInt("threshold", progress).apply();
                 updateColor();
-                ContactInfo.clearCache(getContext());
+                ContactInfo.clearCache(seekBar.getContext());
             }
 
             @Override
@@ -1170,6 +1182,8 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
 
         // Initialize
         FragmentDialogTheme.setBackground(getContext(), view, false);
+        swFaviconsPartial.setText(getString(R.string.title_advanced_favicons_partial,
+                Helper.humanReadableByteCount(ContactInfo.FAVICON_READ_BYTES)));
         grpGravatars.setVisibility(ContactInfo.canGravatars() ? View.VISIBLE : View.GONE);
         tvBimiUnverified.setVisibility(BuildConfig.DEBUG ? View.VISIBLE : View.GONE);
 
@@ -1279,6 +1293,8 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer
         swBimi.setChecked(prefs.getBoolean("bimi", false));
         swGravatars.setChecked(prefs.getBoolean("gravatars", false));
         swFavicons.setChecked(prefs.getBoolean("favicons", false));
+        swFaviconsPartial.setChecked(prefs.getBoolean("favicons_partial", true));
+        swFaviconsPartial.setEnabled(swFavicons.isChecked());
         swGeneratedIcons.setChecked(prefs.getBoolean("generated_icons", true));
         swIdenticons.setChecked(prefs.getBoolean("identicons", false));
         swIdenticons.setEnabled(swGeneratedIcons.isChecked());

app/src/main/java/eu/faircode/email/JsoupEx.java

@@ -54,6 +54,15 @@ public class JsoupEx {
         }
     }
 
+    static Document parse(InputStream stream, String charset, String baseUri) throws IOException {
+        try {
+            return Jsoup.parse(stream, charset, baseUri);
+        } catch (OutOfMemoryError ex) {
+            Log.e(ex);
+            return Document.createShell("");
+        }
+    }
+
     static Document parse(File in) throws IOException {
         try (InputStream is = new FileInputStream(in)) {
             return Jsoup.parse(new FilteredStream(is), StandardCharsets.UTF_8.name(), "");

app/src/main/java/eu/faircode/email/WorkerCleanup.java

@@ -305,7 +305,7 @@ public class WorkerCleanup extends Worker {
 
             // Cleanup contact info
             if (manual)
-                ContactInfo.clearCache(context, true);
+                ContactInfo.clearCache(context);
             else
                 ContactInfo.cleanup(context);
 

app/src/main/res/layout/fragment_options_display.xml

@@ -857,6 +857,19 @@
                     app:layout_constraintStart_toStartOf="parent"
                     app:layout_constraintTop_toBottomOf="@id/swFavicons" />
 
+                <androidx.appcompat.widget.SwitchCompat
+                    android:id="@+id/swFaviconsPartial"
+                    android:layout_width="0dp"
+                    android:layout_height="wrap_content"
+                    android:layout_marginStart="12dp"
+                    android:layout_marginTop="12dp"
+                    android:checked="true"
+                    android:text="@string/title_advanced_favicons_partial"
+                    app:layout_constraintEnd_toEndOf="parent"
+                    app:layout_constraintStart_toStartOf="parent"
+                    app:layout_constraintTop_toBottomOf="@id/tvFaviconsHint"
+                    app:switchPadding="12dp" />
+
                 <androidx.appcompat.widget.SwitchCompat
                     android:id="@+id/swGeneratedIcons"
                     android:layout_width="0dp"
@@ -866,7 +879,7 @@
                     android:text="@string/title_advanced_generated_icons"
                     app:layout_constraintEnd_toEndOf="parent"
                     app:layout_constraintStart_toStartOf="parent"
-                    app:layout_constraintTop_toBottomOf="@id/tvFaviconsHint"
+                    app:layout_constraintTop_toBottomOf="@id/swFaviconsPartial"
                     app:switchPadding="12dp" />
 
                 <androidx.appcompat.widget.SwitchCompat

app/src/main/res/values/strings.xml

@@ -469,6 +469,7 @@
     <string name="title_advanced_bimi_unverified">Unverified sender</string>
     <string name="title_advanced_bimi_verified">Verified sender</string>
     <string name="title_advanced_favicons">Show favicons</string>
+    <string name="title_advanced_favicons_partial">Scan only the first %1$s of the web page</string>
     <string name="title_advanced_generated_icons">Show generated icons</string>
     <string name="title_advanced_identicons">Show identicons</string>
     <string name="title_advanced_circular">Show round icons</string>