From 1eb8d7fa3f7e9330a91b6e89197efc182293af3f Mon Sep 17 00:00:00 2001 From: M66B Date: Sat, 2 Jan 2021 15:57:24 +0100 Subject: [PATCH] Integrate experimental classifier --- app/src/main/java/eu/faircode/email/Core.java | 11 +++++++--- .../eu/faircode/email/EntityOperation.java | 3 +++ .../eu/faircode/email/MessageClassifier.java | 20 +++++++++++++------ .../java/eu/faircode/email/WorkerFts.java | 5 +---- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/app/src/main/java/eu/faircode/email/Core.java b/app/src/main/java/eu/faircode/email/Core.java index 878a431734..9b6c7d4dc8 100644 --- a/app/src/main/java/eu/faircode/email/Core.java +++ b/app/src/main/java/eu/faircode/email/Core.java @@ -1499,6 +1499,7 @@ class Core { parts.isPlainOnly(), HtmlHelper.getPreview(body), parts.getWarnings(message.warning)); + MessageClassifier.classify(message, true, context); if (body != null) EntityLog.log(context, "Operation body size=" + body.length()); @@ -3002,6 +3003,8 @@ class Core { parts.isPlainOnly(), HtmlHelper.getPreview(body), parts.getWarnings(message.warning)); + MessageClassifier.classify(message, true, context); + if (stats != null && body != null) stats.content += body.length(); Log.i(folder.name + " inline downloaded message id=" + message.id + @@ -3126,10 +3129,10 @@ class Core { db.endTransaction(); } - if (process) + if (process) { updateContactInfo(context, folder, message); - - else + MessageClassifier.classify(message, true, context); + } else Log.d(folder.name + " unchanged uid=" + uid); } @@ -3430,6 +3433,8 @@ class Core { parts.isPlainOnly(), HtmlHelper.getPreview(body), parts.getWarnings(message.warning)); + MessageClassifier.classify(message, true, context); + if (stats != null && body != null) stats.content += body.length(); Log.i(folder.name + " downloaded message id=" + message.id + diff --git a/app/src/main/java/eu/faircode/email/EntityOperation.java b/app/src/main/java/eu/faircode/email/EntityOperation.java index 01c6282318..a295ad6e7c 100644 --- a/app/src/main/java/eu/faircode/email/EntityOperation.java +++ b/app/src/main/java/eu/faircode/email/EntityOperation.java @@ -229,6 +229,9 @@ public class EntityOperation { db.rule().deleteRule(rule.id); } + if (message.content) + MessageClassifier.classify(message, false, context); + // Create copy without uid in target folder // Message with same msgid can be in archive if (message.uid != null && diff --git a/app/src/main/java/eu/faircode/email/MessageClassifier.java b/app/src/main/java/eu/faircode/email/MessageClassifier.java index 2756189b15..b718b99a64 100644 --- a/app/src/main/java/eu/faircode/email/MessageClassifier.java +++ b/app/src/main/java/eu/faircode/email/MessageClassifier.java @@ -20,8 +20,11 @@ package eu.faircode.email; */ import android.content.Context; +import android.content.SharedPreferences; import android.text.TextUtils; +import androidx.preference.PreferenceManager; + import org.jetbrains.annotations.NotNull; import java.io.File; @@ -42,10 +45,11 @@ public class MessageClassifier { private static final double CHANCE_THRESHOLD = 2.0; static String classify(EntityMessage message, boolean added, Context context) { - DB db = DB.getInstance(context); + SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); + if (!prefs.getBoolean("classify", BuildConfig.DEBUG)) + return null; - if (!message.content) - throw new IllegalArgumentException("Message without content"); + DB db = DB.getInstance(context); EntityFolder folder = db.folder().getFolder(message.folder); if (folder == null) @@ -80,14 +84,15 @@ public class MessageClassifier { m = (m == null ? 1 : m + 1); classMessages.put(folder.name, m); } else { - if (m != null) + if (m != null && m > 0) classMessages.put(folder.name, m - 1); } + Log.i("Classifier classify=" + folder.name + " messages=" + classMessages.get(folder.name)); return classified; } - static String classify(String classify, String text, boolean added) { + private static String classify(String classify, String text, boolean added) { int maxFrequency = 0; int maxMatchedWords = 0; List words = new ArrayList<>(); @@ -107,7 +112,10 @@ public class MessageClassifier { if (!added) { Integer c = (classFrequency == null ? null : classFrequency.get(classify)); if (c != null) - classFrequency.put(classify, c - 1); + if (c > 0) + classFrequency.put(classify, c - 1); + else + classFrequency.remove(classify); continue; } diff --git a/app/src/main/java/eu/faircode/email/WorkerFts.java b/app/src/main/java/eu/faircode/email/WorkerFts.java index b3eb2c5240..9535670510 100644 --- a/app/src/main/java/eu/faircode/email/WorkerFts.java +++ b/app/src/main/java/eu/faircode/email/WorkerFts.java @@ -42,7 +42,7 @@ import io.requery.android.database.sqlite.SQLiteDatabase; import static android.os.Process.THREAD_PRIORITY_BACKGROUND; public class WorkerFts extends Worker { - private static final int INDEX_DELAY = BuildConfig.DEBUG ? 3 : 30; // seconds + private static final int INDEX_DELAY = 30; // seconds private static final int INDEX_BATCH_SIZE = 100; public WorkerFts(@NonNull Context context, @NonNull WorkerParameters workerParams) { @@ -78,9 +78,6 @@ public class WorkerFts extends Worker { continue; } - if (BuildConfig.DEBUG) - MessageClassifier.classify(message, true, context); - File file = message.getFile(context); String text = HtmlHelper.getFullText(file); if (TextUtils.isEmpty(text)) {