msb-public
/
FairEmail
mirror of https://github.com/M66B/FairEmail.git
1
0
Code Issues Projects Releases Wiki Activity

FTS improvements

Browse Source
pull/172/head
M66B 5 years ago
parent 08098c2285
commit f93bce328b
11 changed files with 179 additions and 21 deletions
Show Stats Download Patch File Download Diff File

4
app/src/main/java/eu/faircode/email/Core.java
Unescape View File

@ -960,7 +960,7 @@ class Core {
db.folder().setFolderTotal(folder.id, count < 0 ? null : count);
}
WorkerFts.init(context);
WorkerFts.init(context, false);
}
private static void onDelete(Context context, JSONArray jargs, EntityFolder folder, EntityMessage message, IMAPFolder ifolder) throws MessagingException {
@ -1989,7 +1989,7 @@ class Core {
db.folder().setFolderSyncState(folder.id, null);
}
WorkerFts.init(context);
WorkerFts.init(context, false);
}
static EntityMessage synchronizeMessage(

16
app/src/main/java/eu/faircode/email/DaoMessage.java
Unescape View File

@ -202,6 +202,12 @@ public interface DaoMessage {
" AND ui_hide")
LiveData<List<Long>> liveHiddenThread(long account, String thread);
@Query("SELECT SUM(fts) AS fts, COUNT(*) AS total FROM message" +
" JOIN folder ON folder.id = message.folder" +
" WHERE content" +
" AND folder.type <> '" + EntityFolder.OUTBOX + "'")
LiveData<TupleFtsStats> liveFts();
@Query("SELECT *" +
" FROM message" +
" WHERE id = :id")
@ -227,8 +233,11 @@ public interface DaoMessage {
" ORDER BY message.received DESC")
List<Long> getMessageIdsByFolder(Long folder);
@Query("SELECT id FROM message" +
" WHERE content AND NOT fts" +
@Query("SELECT message.id FROM message" +
" JOIN folder ON folder.id = message.folder" +
" WHERE content" +
" AND NOT fts" +
" AND folder.type <> '" + EntityFolder.OUTBOX + "'" +
" ORDER BY message.received DESC")
Cursor getMessageFts();
@ -541,6 +550,9 @@ public interface DaoMessage {
@Query("UPDATE message SET headers = NULL WHERE headers IS NOT NULL")
int clearMessageHeaders();
@Query("UPDATE message SET fts = 0")
int resetFts();
@Query("DELETE FROM message WHERE id = :id")
int deleteMessage(long id);

21
app/src/main/java/eu/faircode/email/EntityOperation.java
Unescape View File

@ -100,12 +100,16 @@ public class EntityOperation {
for (Object value : values)
jargs.put(value);
if (ADD.equals(name) &&
(EntityMessage.PGP_SIGNENCRYPT.equals(message.encrypt) ||
EntityMessage.SMIME_SIGNENCRYPT.equals(message.encrypt))) {
EntityFolder folder = db.folder().getFolder(message.folder);
if (folder != null && EntityFolder.DRAFTS.equals(folder.type))
return;
if (ADD.equals(name)) {
db.message().setMessageFts(message.id, false);
WorkerFts.init(context, false);
if (EntityMessage.PGP_SIGNENCRYPT.equals(message.encrypt) ||
EntityMessage.SMIME_SIGNENCRYPT.equals(message.encrypt)) {
EntityFolder folder = db.folder().getFolder(message.folder);
if (folder != null && EntityFolder.DRAFTS.equals(folder.type))
return;
}
}
if (MOVE.equals(name) &&
@ -199,6 +203,7 @@ public class EntityOperation {
Long identity = message.identity;
long uid = message.uid;
int notifying = message.notifying;
boolean fts = message.fts;
boolean seen = message.seen;
boolean ui_seen = message.ui_seen;
Boolean ui_hide = message.ui_hide;
@ -211,6 +216,7 @@ public class EntityOperation {
message.identity = null;
message.uid = null;
message.notifying = 0;
message.fts = false;
if (autoread) {
message.seen = true;
message.ui_seen = true;
@ -229,6 +235,7 @@ public class EntityOperation {
message.identity = identity;
message.uid = uid;
message.notifying = notifying;
message.fts = fts;
message.seen = seen;
message.ui_seen = ui_seen;
message.ui_hide = ui_hide;
@ -245,6 +252,8 @@ public class EntityOperation {
}
EntityAttachment.copy(context, message.id, tmpid);
WorkerFts.init(context, false);
}
// Cross account move

8
app/src/main/java/eu/faircode/email/FragmentMessages.java
Unescape View File

@ -4545,12 +4545,14 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
// Write decrypted body
Helper.copy(plain, message.getFile(context));
db.message().setMessageStored(message.id, new Date().getTime());
db.message().setMessageFts(message.id, false);
db.setTransactionSuccessful();
} finally {
db.endTransaction();
}
WorkerFts.init(context, false);
} else {
// Decode message
MessageHelper.MessageParts parts;
@ -4588,11 +4590,14 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
db.message().setMessageEncrypt(message.id, parts.getEncryption());
db.message().setMessageStored(message.id, new Date().getTime());
db.message().setMessageFts(message.id, false);
db.setTransactionSuccessful();
} finally {
db.endTransaction();
}
WorkerFts.init(context, false);
}
// Check signature status
@ -4857,6 +4862,7 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
db.message().setMessageEncrypt(message.id, parts.getEncryption());
db.message().setMessageStored(message.id, new Date().getTime());
db.message().setMessageFts(message.id, false);
if (alias != null && message.identity != null)
db.identity().setIdentitySignKeyAlias(message.identity, alias);
@ -4865,6 +4871,8 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences.
} finally {
db.endTransaction();
}
WorkerFts.init(context, false);
}
return result;

46
app/src/main/java/eu/faircode/email/FragmentOptionsMisc.java
Unescape View File

@ -43,11 +43,14 @@ import androidx.annotation.Nullable;
import androidx.appcompat.widget.SwitchCompat;
import androidx.constraintlayout.widget.Group;
import androidx.lifecycle.Lifecycle;
import androidx.lifecycle.Observer;
import androidx.preference.PreferenceManager;
public class FragmentOptionsMisc extends FragmentBase implements SharedPreferences.OnSharedPreferenceChangeListener {
private SwitchCompat swExternalSearch;
private SwitchCompat swFts;
private Button btnFtsReset;
private TextView tvFtsIndexed;
private SwitchCompat swEnglish;
private SwitchCompat swWatchdog;
private SwitchCompat swUpdates;
@ -90,6 +93,8 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
swExternalSearch = view.findViewById(R.id.swExternalSearch);
swFts = view.findViewById(R.id.swFts);
btnFtsReset = view.findViewById(R.id.btnFtsReset);
tvFtsIndexed = view.findViewById(R.id.tvFtsIndexed);
swEnglish = view.findViewById(R.id.swEnglish);
swWatchdog = view.findViewById(R.id.swWatchdog);
swUpdates = view.findViewById(R.id.swUpdates);
@ -132,7 +137,33 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
@Override
public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
prefs.edit().putBoolean("fts", checked).apply();
WorkerFts.init(getContext());
WorkerFts.init(getContext(), true);
}
});
btnFtsReset.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Bundle args = new Bundle();
new SimpleTask<Void>() {
@Override
protected Void onExecute(Context context, Bundle args) throws Throwable {
DB db = DB.getInstance(context);
db.message().resetFts();
return null;
}
@Override
protected void onExecuted(Bundle args, Void data) {
WorkerFts.init(getContext(), true);
}
@Override
protected void onException(Bundle args, Throwable ex) {
Log.unexpectedError(getParentFragmentManager(), ex);
}
}.execute(FragmentOptionsMisc.this, args, "fts:reset");
}
});
@ -211,6 +242,19 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
}
});
tvFtsIndexed.setText(null);
DB db = DB.getInstance(getContext());
db.message().liveFts().observe(getViewLifecycleOwner(), new Observer<TupleFtsStats>() {
@Override
public void onChanged(TupleFtsStats stats) {
if (stats == null)
tvFtsIndexed.setText(null);
else
tvFtsIndexed.setText(getString(R.string.title_advanced_fts_indexed, stats.fts, stats.total));
}
});
setLastCleanup(prefs.getLong("last_cleanup", -1));
PreferenceManager.getDefaultSharedPreferences(getContext()).registerOnSharedPreferenceChangeListener(this);

18
app/src/main/java/eu/faircode/email/FtsDbHelper.java
Unescape View File

@ -38,14 +38,15 @@ public class FtsDbHelper extends SQLiteOpenHelper {
private static final int DATABASE_VERSION = 1;
private static final String DATABASE_NAME = "fts.db";
public FtsDbHelper(Context context) {
FtsDbHelper(Context context) {
super(context, DATABASE_NAME, null, DATABASE_VERSION);
}
@Override
public void onCreate(SQLiteDatabase db) {
Log.i("FTS create");
db.execSQL("CREATE VIRTUAL TABLE `message` USING fts4(`folder`, `time`, `address`, `subject`, `keyword`, `text`)");
db.execSQL("CREATE VIRTUAL TABLE `message`" +
" USING fts4(`folder`, `time`, `address`, `subject`, `keyword`, `text`)");
}
@Override
@ -66,7 +67,7 @@ public class FtsDbHelper extends SQLiteOpenHelper {
try {
db.beginTransaction();
db.delete("message", "docid = ?", new Object[]{message.id});
delete(db, message.id);
ContentValues cv = new ContentValues();
cv.put("docid", message.id);
@ -86,6 +87,10 @@ public class FtsDbHelper extends SQLiteOpenHelper {
}
}
void delete(SQLiteDatabase db, long id) {
db.delete("message", "docid = ?", new Object[]{id});
}
List<Long> match(SQLiteDatabase db, Long folder, String search) {
Log.i("FTS folder=" + folder + " search=" + search);
List<Long> result = new ArrayList<>();
@ -100,4 +105,11 @@ public class FtsDbHelper extends SQLiteOpenHelper {
Log.i("FTS result=" + result.size());
return result;
}
Cursor getIds(SQLiteDatabase db) {
return db.query(
"message", new String[]{"docid"},
null, null,
null, null, "time");
}
}

25
app/src/main/java/eu/faircode/email/TupleFtsStats.java
Unescape View File

@ -0,0 +1,25 @@
package eu.faircode.email;
/*
This file is part of FairEmail.
FairEmail is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
FairEmail is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with FairEmail. If not, see <http://www.gnu.org/licenses/>.
Copyright 2018-2020 by Marcel Bokhorst (M66B)
*/
public class TupleFtsStats {
public long fts;
public long total;
}

21
app/src/main/java/eu/faircode/email/WorkerCleanup.java
Unescape View File

@ -21,6 +21,7 @@ package eu.faircode.email;
import android.content.Context;
import android.content.SharedPreferences;
import android.database.Cursor;
import androidx.annotation.NonNull;
import androidx.preference.PreferenceManager;
@ -39,6 +40,8 @@ import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
import io.requery.android.database.sqlite.SQLiteDatabase;
import static android.os.Process.THREAD_PRIORITY_BACKGROUND;
public class WorkerCleanup extends Worker {
@ -190,6 +193,24 @@ public class WorkerCleanup extends Worker {
}
}
Log.i("Cleanup FTS");
int fts = 0;
FtsDbHelper ftsDb = new FtsDbHelper(context);
try (SQLiteDatabase sdb = ftsDb.getWritableDatabase()) {
try (Cursor cursor = ftsDb.getIds(sdb)) {
while (cursor.moveToNext()) {
long docid = cursor.getLong(0);
EntityMessage message = db.message().getMessage(docid);
if (message == null) {
Log.i("Deleting docid" + docid);
ftsDb.delete(sdb, docid);
fts++;
}
}
}
}
Log.i("Cleanup FTS=" + fts);
Log.i("Cleanup contacts");
int contacts = db.contact().deleteContacts(now - KEEP_CONTACTS_DURATION);
Log.i("Deleted contacts=" + contacts);

14
app/src/main/java/eu/faircode/email/WorkerFts.java
Unescape View File

@ -37,6 +37,8 @@ import java.util.concurrent.TimeUnit;
import io.requery.android.database.sqlite.SQLiteDatabase;
public class WorkerFts extends Worker {
private static final int INDEX_DELAY = 30; // seconds
public WorkerFts(@NonNull Context context, @NonNull WorkerParameters workerParams) {
super(context, workerParams);
Log.i("Instance " + getName());
@ -78,23 +80,23 @@ public class WorkerFts extends Worker {
}
}
static void init(Context context) {
static void init(Context context, boolean immediately) {
try {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
boolean fts = prefs.getBoolean("fts", true);
if (fts) {
Log.i("Queuing " + getName());
OneTimeWorkRequest workRequest =
new OneTimeWorkRequest.Builder(WorkerFts.class)
.setInitialDelay(30, TimeUnit.SECONDS)
.build();
OneTimeWorkRequest.Builder builder = new OneTimeWorkRequest.Builder(WorkerFts.class);
if (!immediately)
builder.setInitialDelay(INDEX_DELAY, TimeUnit.SECONDS);
OneTimeWorkRequest workRequest = builder.build();
WorkManager.getInstance(context)
.enqueueUniqueWork(getName(), ExistingWorkPolicy.REPLACE, workRequest);
Log.i("Queued " + getName());
} else {
} else if (immediately) {
Log.i("Cancelling " + getName());
WorkManager.getInstance(context).cancelUniqueWork(getName());
Log.i("Cancelled " + getName());

26
app/src/main/res/layout/fragment_options_misc.xml
Unescape View File

@ -47,6 +47,30 @@
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/swFts" />
<Button
android:id="@+id/btnFtsReset"
style="?android:attr/buttonStyleSmall"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="12dp"
android:minWidth="0dp"
android:minHeight="0dp"
android:text="@string/title_reset"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/tvFtsHint" />
<TextView
android:id="@+id/tvFtsIndexed"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_marginEnd="48dp"
android:text="@string/title_advanced_fts_indexed"
android:textAppearance="@style/TextAppearance.AppCompat.Small"
android:textStyle="italic"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/btnFtsReset" />
<androidx.appcompat.widget.SwitchCompat
android:id="@+id/swEnglish"
android:layout_width="0dp"
@ -55,7 +79,7 @@
android:text="@string/title_advanced_english"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/tvFtsHint"
app:layout_constraintTop_toBottomOf="@id/tvFtsIndexed"
app:switchPadding="12dp" />
<TextView

1
app/src/main/res/values/strings.xml
Unescape View File

@ -383,6 +383,7 @@
<string name="title_advanced_external_search">Allow other apps to search in messages</string>
<string name="title_advanced_fts">Full text search</string>
<string name="title_advanced_fts_indexed">%1$d/%2$d indexed</string>
<string name="title_advanced_english">Force English language</string>
<string name="title_advanced_watchdog">Periodically check if FairEmail is still active</string>
<string name="title_advanced_updates">Check for updates</string>

Write Preview
Loading…
Cancel
Save