diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md
index f60a3a7c0f..beded60b13 100644
--- a/ATTRIBUTION.md
+++ b/ATTRIBUTION.md
@@ -28,4 +28,5 @@ FairEmail uses:
* [Disconnect's tracker protection lists](https://github.com/disconnectme/disconnect-tracking-protection). Copyright 2010-2020 Disconnect, Inc. [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license](https://github.com/disconnectme/disconnect-tracking-protection/blob/master/LICENSE).
* [Over-Scroll Support For Android's RecyclerView, ListView, GridView, ScrollView ...](https://github.com/EverythingMe/overscroll-decor). Copyright (c) 2015, DoAT Media Ltd. [BSD-2-Clause License](https://github.com/EverythingMe/overscroll-decor/blob/master/LICENSE)
* [Compact Encoding Detection](https://github.com/google/compact_enc_det). Copyright 2016 Google Inc. [Apache License 2.0](https://github.com/google/compact_enc_det/blob/master/LICENSE).
+* [Compact Language Detector v3](https://github.com/google/cld3). Copyright 2016 Google Inc. All rights reserved. [Apache License 2.0](https://github.com/google/cld3/blob/master/LICENSE).
* [POI-HMEF](https://poi.apache.org/components/hmef/index.html). Copyright © 2001-2020 The Apache Software Foundation. [Apache Software License v2](https://poi.apache.org/devel/guidelines.html#The+Licensing).
diff --git a/README.md b/README.md
index 85e8466e10..ed08a98aeb 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,7 @@ FairEmail might be for you if you value your privacy.
* Offline storage and operations
* Battery friendly
* Low data usage
-* Small (~ 15 MB)
+* Small (< 20 MB)
* Material design (including dark/black theme)
* Maintained and supported
diff --git a/app/src/main/assets/ATTRIBUTION.md b/app/src/main/assets/ATTRIBUTION.md
index f60a3a7c0f..beded60b13 100644
--- a/app/src/main/assets/ATTRIBUTION.md
+++ b/app/src/main/assets/ATTRIBUTION.md
@@ -28,4 +28,5 @@ FairEmail uses:
* [Disconnect's tracker protection lists](https://github.com/disconnectme/disconnect-tracking-protection). Copyright 2010-2020 Disconnect, Inc. [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license](https://github.com/disconnectme/disconnect-tracking-protection/blob/master/LICENSE).
* [Over-Scroll Support For Android's RecyclerView, ListView, GridView, ScrollView ...](https://github.com/EverythingMe/overscroll-decor). Copyright (c) 2015, DoAT Media Ltd. [BSD-2-Clause License](https://github.com/EverythingMe/overscroll-decor/blob/master/LICENSE)
* [Compact Encoding Detection](https://github.com/google/compact_enc_det). Copyright 2016 Google Inc. [Apache License 2.0](https://github.com/google/compact_enc_det/blob/master/LICENSE).
+* [Compact Language Detector v3](https://github.com/google/cld3). Copyright 2016 Google Inc. All rights reserved. [Apache License 2.0](https://github.com/google/cld3/blob/master/LICENSE).
* [POI-HMEF](https://poi.apache.org/components/hmef/index.html). Copyright © 2001-2020 The Apache Software Foundation. [Apache Software License v2](https://poi.apache.org/devel/guidelines.html#The+Licensing).
diff --git a/app/src/main/java/eu/faircode/email/FragmentMessages.java b/app/src/main/java/eu/faircode/email/FragmentMessages.java
index b6c564235a..3ee8f57895 100644
--- a/app/src/main/java/eu/faircode/email/FragmentMessages.java
+++ b/app/src/main/java/eu/faircode/email/FragmentMessages.java
@@ -3983,8 +3983,7 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
(viewType == AdapterMessage.ViewType.UNIFIED ||
(viewType == AdapterMessage.ViewType.FOLDER && !outbox));
- String filter_language = prefs.getString("filter_language", null);
- boolean filter_active = (filter_seen || filter_unflagged || filter_unknown || !TextUtils.isEmpty(filter_language));
+ boolean filter_active = (filter_seen || filter_unflagged || filter_unknown);
MenuItem menuFilter = menu.findItem(R.id.menu_filter);
menuFilter.setShowAsAction(folder && filter_active
? MenuItem.SHOW_AS_ACTION_ALWAYS
@@ -4055,8 +4054,7 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
menu.findItem(R.id.menu_compact).setChecked(compact);
- menu.findItem(R.id.menu_select_language).setVisible(
- language_detection && folder && Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q);
+ menu.findItem(R.id.menu_select_language).setVisible(language_detection && folder);
menu.findItem(R.id.menu_select_all).setVisible(folder);
menu.findItem(R.id.menu_select_found).setVisible(viewType == AdapterMessage.ViewType.SEARCH);
menu.findItem(R.id.menu_mark_all_read).setVisible(folder);
@@ -4421,8 +4419,8 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
boolean filter_unflagged = prefs.getBoolean("filter_unflagged", false);
boolean filter_unknown = prefs.getBoolean("filter_unknown", false);
boolean filter_snoozed = prefs.getBoolean("filter_snoozed", true);
- String filter_language = prefs.getString("filter_language", null);
boolean language_detection = prefs.getBoolean("language_detection", false);
+ String filter_language = prefs.getString("filter_language", null);
DB db = DB.getInstance(context);
try {
@@ -4683,8 +4681,10 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
boolean filter_seen = prefs.getBoolean("filter_seen", false);
boolean filter_unflagged = prefs.getBoolean("filter_unflagged", false);
boolean filter_unknown = prefs.getBoolean("filter_unknown", false);
+ boolean language_detection = prefs.getBoolean("language_detection", false);
String filter_language = prefs.getString("filter_language", null);
- boolean filter_active = (filter_seen || filter_unflagged || filter_unknown || !TextUtils.isEmpty(filter_language));
+ boolean filter_active = (filter_seen || filter_unflagged || filter_unknown ||
+ (language_detection && !TextUtils.isEmpty(filter_language)));
boolean none = (items == 0 && !loading && tasks == 0 && initialized);
boolean filtered = (filter_active && viewType != AdapterMessage.ViewType.SEARCH);
diff --git a/app/src/main/java/eu/faircode/email/FragmentOptionsBehavior.java b/app/src/main/java/eu/faircode/email/FragmentOptionsBehavior.java
index e29d5353ed..f0d28aefaa 100644
--- a/app/src/main/java/eu/faircode/email/FragmentOptionsBehavior.java
+++ b/app/src/main/java/eu/faircode/email/FragmentOptionsBehavior.java
@@ -170,7 +170,6 @@ public class FragmentOptionsBehavior extends FragmentBase implements SharedPrefe
}
});
- swLanguageDetection.setEnabled(Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q);
swLanguageDetection.setOnCheckedChangeListener(new CompoundButton.OnCheckedChangeListener() {
@Override
public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
diff --git a/app/src/main/java/eu/faircode/email/HtmlHelper.java b/app/src/main/java/eu/faircode/email/HtmlHelper.java
index 925392abc3..5b5f925542 100644
--- a/app/src/main/java/eu/faircode/email/HtmlHelper.java
+++ b/app/src/main/java/eu/faircode/email/HtmlHelper.java
@@ -1716,7 +1716,7 @@ public class HtmlHelper {
if (!language_detection)
return null;
- String text = getPreview(body);
+ String text = getFullText(body);
Locale locale = TextHelper.detectLanguage(context, text);
return (locale == null ? null : locale.getLanguage());
} catch (Throwable ex) {
diff --git a/app/src/main/java/eu/faircode/email/TextHelper.java b/app/src/main/java/eu/faircode/email/TextHelper.java
index 8a28ce27c2..e135187be1 100644
--- a/app/src/main/java/eu/faircode/email/TextHelper.java
+++ b/app/src/main/java/eu/faircode/email/TextHelper.java
@@ -27,7 +27,6 @@ import android.view.textclassifier.ConversationAction;
import android.view.textclassifier.ConversationActions;
import android.view.textclassifier.TextClassificationManager;
import android.view.textclassifier.TextClassifier;
-import android.view.textclassifier.TextLanguage;
import java.time.ZoneId;
import java.time.ZonedDateTime;
@@ -41,6 +40,8 @@ import java.util.Locale;
import java.util.Set;
public class TextHelper {
+ private static final int MAX_SAMPLE_SIZE = 8192;
+
static {
System.loadLibrary("fairemail");
}
@@ -48,33 +49,35 @@ public class TextHelper {
private static native DetectResult jni_detect_language(byte[] octets);
static Locale detectLanguage(Context context, String text) {
- // Why not ML kit? https://developers.google.com/ml-kit/terms
+ // Why not ML kit?
+ // https://developers.google.com/ml-kit/terms
+
if (TextUtils.isEmpty(text))
return null;
- if (BuildConfig.DEBUG) {
- // https://github.com/google/cld3
- DetectResult result = jni_detect_language(text.getBytes());
- Log.i("Language=" + result);
- return Locale.forLanguageTag(result.language);
+ byte[] octets = text.getBytes();
+ byte[] sample;
+ if (octets.length < MAX_SAMPLE_SIZE)
+ sample = octets;
+ else {
+ sample = new byte[MAX_SAMPLE_SIZE];
+ System.arraycopy(octets, 0, sample, 0, MAX_SAMPLE_SIZE);
}
- if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q)
- return null;
+ // https://github.com/google/cld3
+ Log.i("cld3 sample=" + sample.length);
+ DetectResult result = jni_detect_language(sample);
+ Log.i("Language=" + result);
- // https://issuetracker.google.com/issues/173337263
- TextClassificationManager tcm =
- (TextClassificationManager) context.getSystemService(Context.TEXT_CLASSIFICATION_SERVICE);
- if (tcm == null)
+ if (result.probability < 0.5)
return null;
- TextLanguage.Request request = new TextLanguage.Request.Builder(text).build();
- TextClassifier tc = tcm.getTextClassifier();
- TextLanguage tlanguage = tc.detectLanguage(request);
- if (tlanguage.getLocaleHypothesisCount() > 0)
- return tlanguage.getLocale(0).toLocale();
-
- return null;
+ try {
+ return Locale.forLanguageTag(result.language);
+ } catch (Throwable ex) {
+ Log.w(ex);
+ return null;
+ }
}
static ConversationActions getConversationActions(
diff --git a/app/src/main/java/eu/faircode/email/ViewModelMessages.java b/app/src/main/java/eu/faircode/email/ViewModelMessages.java
index 5a6f190e34..f1d6b720c3 100644
--- a/app/src/main/java/eu/faircode/email/ViewModelMessages.java
+++ b/app/src/main/java/eu/faircode/email/ViewModelMessages.java
@@ -21,7 +21,6 @@ package eu.faircode.email;
import android.content.Context;
import android.content.SharedPreferences;
-import android.os.Build;
import android.os.Bundle;
import android.text.TextUtils;
import android.util.Pair;
@@ -449,10 +448,8 @@ public class ViewModelMessages extends ViewModel {
this.filter_snoozed = prefs.getBoolean("filter_snoozed", true);
boolean language_detection = prefs.getBoolean("language_detection", false);
- if (!language_detection || Build.VERSION.SDK_INT < Build.VERSION_CODES.Q)
- this.filter_language = null;
- else
- this.filter_language = prefs.getString("filter_language", null);
+ String filter_language = prefs.getString("filter_language", null);
+ this.filter_language = (language_detection ? filter_language : null);
this.debug = prefs.getBoolean("debug", false);
}
diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml
index 0ef000d83c..1135222748 100644
--- a/app/src/main/res/values/strings.xml
+++ b/app/src/main/res/values/strings.xml
@@ -517,7 +517,7 @@
Show frequently used contacts in Android share menu
Suggest actions (Android 10+)
Suggest reply texts (Android 10+)
- Detect message text language (Android 10+)
+ Detect message text language
Build search index
%1$d / %2$d messages indexed (%3$s)
Classify messages
diff --git a/metadata/en-US/full_description.txt b/metadata/en-US/full_description.txt
index 4dc718acd1..5fd7d781e5 100644
--- a/metadata/en-US/full_description.txt
+++ b/metadata/en-US/full_description.txt
@@ -21,7 +21,7 @@ FairEmail might be for you if you value your privacy.
* Offline storage and operations
* Battery friendly
* Low data usage
-* Small (~ 15 MB)
+* Small (< 20 MB)
* Material design (including dark/black theme)
* Maintained and supported