Bring back external attachment storage

FAQ.md

@@ -3347,15 +3347,16 @@ Messages, attachments, etc stored on external storage media, like an sdcard, can
 See [here](https://developer.android.com/training/data-storage) for the details.
 Instead, consider to use [adoptable storage](https://source.android.com/devices/storage/adoptable).
 
-~~Since version 1.1829 is it possible to store attachments to external storage space private to the app (except for file managers) via an option in the debug panel.~~
+Since version 1.1829 is it possible to store attachments to external storage space private to the app (except for file managers) via an option in the debug panel.
-~~You can enable the debug panel by enabling debug mode in the miscellaneous settings (last option).~~
+You can enable the debug panel by enabling debug mode in the miscellaneous settings (last option).
-~~To prevent ongoing operations from storing attachments at the old location~~
+To prevent ongoing operations from storing attachments at the old location
-~~you should disable receiving messages in the receive settings and wait until all operations have been completed before changing this option.~~
+you should disable receiving messages in the receive settings and wait until all operations have been completed before changing this option.
-~~Please be aware that removing the storage space will inevitably result in problems, which is one of the reasons why this option is hidden.~~
+Please be aware that removing the storage space will inevitably result in problems, which is one of the reasons why this option is hidden.
 
-~~Moving messages to an sdcard is not an option because this would significantly reduce the response times of the app.~~
+Please note that more often than not the external storage space is emulated on recent Android versions and recent devices.
+and that there is no longer a viable way to get permission to write to other locations.
 
-The option to move attachments to external storage space was removed in version 1.2137 because Play Store policies considers it unsafe.
+Moving messages to an sdcard is not an option because this would significantly reduce the response times of the app.
 
 When needed you can save (raw) messages via the three-dots menu just above the message text
 and save attachments by tapping on the floppy icon.

app/src/main/java/eu/faircode/email/DB.java

@@ -2877,11 +2877,11 @@ public abstract class DB extends RoomDatabase {
                     @Override
                     public void migrate(@NonNull SupportSQLiteDatabase db) {
                         logMigration(startVersion, endVersion);
-                        SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
+                        //SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
-                        boolean external_storage = prefs.getBoolean("external_storage", false);
+                        //boolean external_storage = prefs.getBoolean("external_storage", false);
-                        if (external_storage || BuildConfig.DEBUG)
+                        //if (external_storage || BuildConfig.DEBUG)
-                            db.execSQL("UPDATE `attachment` SET available = 0");
+                        //    db.execSQL("UPDATE `attachment` SET available = 0");
-                        prefs.edit().remove("external_storage").apply();
+                        //prefs.edit().remove("external_storage").apply();
                     }
                 }).addMigrations(new Migration(286, 287) {
                     @Override

app/src/main/java/eu/faircode/email/DebugHelper.java

@@ -57,6 +57,7 @@ import android.net.Uri;
 import android.os.Build;
 import android.os.Bundle;
 import android.os.Debug;
+import android.os.Environment;
 import android.os.IBinder;
 import android.os.LocaleList;
 import android.os.PowerManager;
@@ -1647,6 +1648,10 @@ public class DebugHelper {
                             ai.sourceDir, ai.publicSourceDir));
                 size += write(os, String.format("Files: %s\r\n", context.getFilesDir()));
 
+                File external = Helper.getExternalFilesDir(context);
+                boolean emulated = (external != null && Environment.isExternalStorageEmulated(external));
+                size += write(os, String.format("External: %s emulated: %b\r\n", external, emulated));
+
                 size += write(os, String.format("Cache: %s\r\n  external: %s\n",
                         context.getCacheDir(), context.getExternalCacheDir()));
                 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N)

app/src/main/java/eu/faircode/email/EntityAttachment.java

@@ -22,10 +22,12 @@ package eu.faircode.email;
 import static androidx.room.ForeignKey.CASCADE;
 
 import android.content.Context;
+import android.content.SharedPreferences;
 import android.net.Uri;
 import android.text.TextUtils;
 
 import androidx.annotation.NonNull;
+import androidx.preference.PreferenceManager;
 import androidx.room.Entity;
 import androidx.room.ForeignKey;
 import androidx.room.Ignore;
@@ -170,7 +172,7 @@ public class EntityAttachment {
     }
 
     static File getFile(Context context, long id, String name) {
-        File dir = Helper.ensureExists(context, "attachments");
+        File dir = getRoot(context);
         String filename = Long.toString(id);
         if (!TextUtils.isEmpty(name))
             filename += "." + Helper.sanitizeFilename(name);
@@ -179,6 +181,19 @@ public class EntityAttachment {
         return new File(dir, filename);
     }
 
+    static File getRoot(Context context) {
+        SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
+        boolean external_storage = prefs.getBoolean("external_storage", false);
+
+        if (external_storage) {
+            File dir = Helper.getExternalFilesDir(context);
+            dir.mkdirs();
+            return dir;
+        }
+
+        return Helper.ensureExists(context, "attachments");
+    }
+
     static void copy(Context context, long oldid, long newid) {
         DB db = DB.getInstance(context);
 

app/src/main/java/eu/faircode/email/FragmentOptionsBackup.java

@@ -1186,6 +1186,9 @@ public class FragmentOptionsBackup extends FragmentBase implements SharedPrefere
                                 continue;
                             }
 
+                            if ("external_storage".equals(key))
+                                continue;
+
                             if ("reformatted_hint".equals(key))
                                 continue;
 

app/src/main/java/eu/faircode/email/FragmentOptionsMisc.java

@@ -43,6 +43,7 @@ import android.net.Uri;
 import android.os.Build;
 import android.os.Bundle;
 import android.os.Debug;
+import android.os.Environment;
 import android.provider.Settings;
 import android.text.Editable;
 import android.text.SpannableStringBuilder;
@@ -82,6 +83,7 @@ import androidx.work.WorkManager;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.lang.reflect.Field;
@@ -165,6 +167,8 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
     private SwitchCompat swAutostart;
     private SwitchCompat swEmergency;
     private SwitchCompat swWorkManager;
+    private SwitchCompat swExternalStorage;
+    private TextView tvExternalStorageFolder;
     private SwitchCompat swIntegrity;
     private SwitchCompat swWal;
     private SwitchCompat swCheckpoints;
@@ -267,7 +271,7 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
             "crash_reports", "cleanup_attachments",
             "watchdog", "experiments", "main_log", "main_log_memory", "protocol", "log_level", "debug", "leak_canary",
             "test1", "test2", "test3", "test4", "test5",
-            "emergency_file", "work_manager",
+            "emergency_file", "work_manager", // "external_storage",
             "sqlite_integrity_check", "wal", "sqlite_checkpoints", "sqlite_analyze", "sqlite_auto_vacuum", "sqlite_sync_extra", "sqlite_cache",
             "chunk_size", "thread_range",
             "autoscroll_editor", "undo_manager",
@@ -400,6 +404,8 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
         swAutostart = view.findViewById(R.id.swAutostart);
         swEmergency = view.findViewById(R.id.swEmergency);
         swWorkManager = view.findViewById(R.id.swWorkManager);
+        swExternalStorage = view.findViewById(R.id.swExternalStorage);
+        tvExternalStorageFolder = view.findViewById(R.id.tvExternalStorageFolder);
         swIntegrity = view.findViewById(R.id.swIntegrity);
         swWal = view.findViewById(R.id.swWal);
         swCheckpoints = view.findViewById(R.id.swCheckpoints);
@@ -1080,6 +1086,61 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
             }
         });
 
+        swExternalStorage.setEnabled(Helper.getExternalFilesDir(getContext()) != null);
+        swExternalStorage.setOnCheckedChangeListener(new CompoundButton.OnCheckedChangeListener() {
+            @Override
+            public void onCheckedChanged(CompoundButton buttonView, boolean isChecked) {
+                SharedPreferences.Editor editor = prefs.edit();
+                editor.putBoolean("external_storage", isChecked);
+                editor.apply();
+
+                Bundle args = new Bundle();
+                args.putBoolean("external_storage", isChecked);
+
+                new SimpleTask<Integer>() {
+                    @Override
+                    protected Integer onExecute(Context context, Bundle args) throws IOException {
+                        boolean external_storage = args.getBoolean("external_storage");
+
+                        File sourceRoot = (!external_storage
+                                ? Helper.getExternalFilesDir(context)
+                                : context.getFilesDir());
+
+                        File targetRoot = (external_storage
+                                ? Helper.getExternalFilesDir(context)
+                                : context.getFilesDir());
+
+                        File source = new File(sourceRoot, "attachments");
+                        File target = new File(targetRoot, "attachments");
+                        source.mkdirs();
+                        target.mkdirs();
+
+                        File[] attachments = source.listFiles();
+                        if (attachments != null)
+                            for (File attachment : attachments) {
+                                File dest = new File(target, attachment.getName());
+                                Log.w("Move " + attachment + " to " + dest);
+                                Helper.copy(attachment, dest);
+                                Helper.secureDelete(attachment);
+                            }
+
+                        return (attachments == null ? -1 : attachments.length);
+                    }
+
+                    @Override
+                    protected void onExecuted(Bundle args, Integer count) {
+                        String msg = String.format("Moved %d attachments", count);
+                        ToastEx.makeText(getContext(), msg, Toast.LENGTH_LONG).show();
+                    }
+
+                    @Override
+                    protected void onException(Bundle args, Throwable ex) {
+                        Log.unexpectedError(getParentFragmentManager(), ex);
+                    }
+                }.execute(FragmentOptionsMisc.this, args, "external");
+            }
+        });
+
         swIntegrity.setOnCheckedChangeListener(new CompoundButton.OnCheckedChangeListener() {
             @Override
             public void onCheckedChanged(CompoundButton v, boolean checked) {
@@ -1919,6 +1980,11 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
         else
             tvLastDaily.setText(("-"));
 
+        File external = Helper.getExternalFilesDir(getContext());
+        boolean emulated = (external != null && Environment.isExternalStorageEmulated(external));
+        tvExternalStorageFolder.setText(
+                (external == null ? null : external.getAbsolutePath()) + (emulated ? " emulated" : ""));
+
         swExactAlarms.setEnabled(AlarmManagerCompatEx.canScheduleExactAlarms(getContext()));
         swTestIab.setVisibility(BuildConfig.DEBUG && BuildConfig.TEST_RELEASE ? View.VISIBLE : View.GONE);
 
@@ -2206,6 +2272,7 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
             swAutostart.setChecked(Helper.isComponentEnabled(getContext(), ReceiverAutoStart.class));
             swEmergency.setChecked(prefs.getBoolean("emergency_file", true));
             swWorkManager.setChecked(prefs.getBoolean("work_manager", true));
+            swExternalStorage.setChecked(prefs.getBoolean("external_storage", false));
 
             swIntegrity.setChecked(prefs.getBoolean("sqlite_integrity_check", true));
             swWal.setChecked(prefs.getBoolean("wal", true));

app/src/main/java/eu/faircode/email/Helper.java

@@ -2659,6 +2659,18 @@ public class Helper {
         return dir;
     }
 
+    static File getExternalFilesDir(Context context) {
+        return getExternalFilesDir(context, null);
+    }
+
+    static File getExternalFilesDir(Context context, String type) {
+        File[] dirs = ContextCompat.getExternalFilesDirs(context, type);
+        if (dirs == null || dirs.length == 0)
+            return context.getExternalFilesDir(type);
+        else
+            return dirs[0];
+    }
+
     static String sanitizeFilename(String name) {
         if (name == null)
             return null;

app/src/main/java/eu/faircode/email/WorkerCleanup.java

@@ -258,7 +258,7 @@ public class WorkerCleanup extends Worker {
             // Cleanup attachment files
             {
                 Log.breadcrumb("worker", "cleanup", "attachment files");
-                File[] attachments = new File(context.getFilesDir(), "attachments").listFiles();
+                File[] attachments = new File(EntityAttachment.getRoot(context), "attachments").listFiles();
                 if (attachments != null)
                     for (File file : attachments)
                         if (manual || file.lastModified() + KEEP_FILES_DURATION < now)

app/src/main/res/layout/fragment_options_misc.xml

@@ -997,6 +997,32 @@
                     app:layout_constraintTop_toBottomOf="@id/swEmergency"
                     app:switchPadding="12dp" />
 
+                <androidx.appcompat.widget.SwitchCompat
+                    android:id="@+id/swExternalStorage"
+                    android:layout_width="0dp"
+                    android:layout_height="wrap_content"
+                    android:layout_marginTop="12dp"
+                    android:drawableStart="@drawable/twotone_warning_24"
+                    android:drawablePadding="6dp"
+                    android:text="@string/title_advanced_external_storage"
+                    app:drawableTint="?attr/colorWarning"
+                    app:layout_constraintEnd_toEndOf="parent"
+                    app:layout_constraintStart_toStartOf="parent"
+                    app:layout_constraintTop_toBottomOf="@id/swWorkManager"
+                    app:switchPadding="12dp" />
+
+                <TextView
+                    android:id="@+id/tvExternalStorageFolder"
+                    android:layout_width="0dp"
+                    android:layout_height="wrap_content"
+                    android:fontFamily="monospace"
+                    android:text="/data/data/..."
+                    android:textAppearance="@style/TextAppearance.AppCompat.Small"
+                    android:textSize="12sp"
+                    app:layout_constraintEnd_toEndOf="parent"
+                    app:layout_constraintStart_toStartOf="parent"
+                    app:layout_constraintTop_toBottomOf="@id/swExternalStorage" />
+
                 <androidx.appcompat.widget.SwitchCompat
                     android:id="@+id/swIntegrity"
                     android:layout_width="0dp"
@@ -1006,7 +1032,7 @@
                     android:text="@string/title_advanced_integrity"
                     app:layout_constraintEnd_toEndOf="parent"
                     app:layout_constraintStart_toStartOf="parent"
-                    app:layout_constraintTop_toBottomOf="@id/swWorkManager"
+                    app:layout_constraintTop_toBottomOf="@id/tvExternalStorageFolder"
                     app:switchPadding="12dp" />
 
                 <androidx.appcompat.widget.SwitchCompat

app/src/main/res/values/strings.xml

@@ -884,6 +884,7 @@
     <string name="title_advanced_autostart" translatable="false">Autostart app</string>
     <string name="title_advanced_emergency" translatable="false">Write emergency file</string>
     <string name="title_advanced_work_manager" translatable="false">Initialize work manager</string>
+    <string name="title_advanced_external_storage" translatable="false">Use external storage</string>
     <string name="title_advanced_integrity" translatable="false">sqlite integrity check</string>
     <string name="title_advanced_wal" translatable="false">sqlite WAL</string>
     <string name="title_advanced_checkpoints" translatable="false">sqlite checkpoints</string>