From 19d59a220324d1adfcd7e55f91a6cbcac737df81 Mon Sep 17 00:00:00 2001 From: M66B Date: Fri, 11 Mar 2022 12:20:39 +0100 Subject: [PATCH] Switched to Apache commons compress --- ATTRIBUTION.md | 1 + app/build.gradle | 5 + app/src/main/assets/ATTRIBUTION.md | 1 + .../eu/faircode/email/EntityAttachment.java | 33 +++ .../email/FragmentOptionsDisplay.java | 4 + .../java/eu/faircode/email/MessageHelper.java | 189 ++++++++++++------ .../res/layout/fragment_options_display.xml | 14 +- app/src/main/res/values/strings.xml | 1 + 8 files changed, 190 insertions(+), 58 deletions(-) diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 2e356fae11..59a6bd3681 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -41,3 +41,4 @@ FairEmail uses: * [Cousine font](https://fonts.google.com/specimen/Cousine). By Steve Matteson. [Apache License 2.0](https://fonts.google.com/specimen/Cousine#license). * [Lato font](https://fonts.google.com/specimen/Lato). By Łukasz Dziedzic. [Apache License 2.0](https://fonts.google.com/specimen/Lato#license). * [Caladea font](https://fonts.google.com/specimen/Caladea). By Andrés Torresi, Carolina Giovanolli. [Apache License 2.0](https://fonts.google.com/specimen/Caladea#license). +* [Apache Commons Compress](https://commons.apache.org/proper/commons-compress/). Copyright © 2002-2021 The Apache Software Foundation. All Rights Reserved. [Apache License 2.0](https://www.apache.org/licenses/). diff --git a/app/build.gradle b/app/build.gradle index 7055df13db..29d8c3b64d 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -376,6 +376,7 @@ dependencies { def reactivestreams_version = "1.0.3" def rxjava2_version = "2.2.21" def svg_version = "1.4" + def compress_version = "1.21" // https://developer.android.com/jetpack/androidx/releases/startup implementation "androidx.startup:startup-runtime:$startup_version" @@ -572,4 +573,8 @@ dependencies { // http://bigbadaboom.github.io/androidsvg/ implementation "com.caverock:androidsvg:$svg_version" + + // https://commons.apache.org/proper/commons-compress/ + // https://mvnrepository.com/artifact/org.apache.commons/commons-compress + implementation "org.apache.commons:commons-compress:$compress_version" } diff --git a/app/src/main/assets/ATTRIBUTION.md b/app/src/main/assets/ATTRIBUTION.md index 2e356fae11..59a6bd3681 100644 --- a/app/src/main/assets/ATTRIBUTION.md +++ b/app/src/main/assets/ATTRIBUTION.md @@ -41,3 +41,4 @@ FairEmail uses: * [Cousine font](https://fonts.google.com/specimen/Cousine). By Steve Matteson. [Apache License 2.0](https://fonts.google.com/specimen/Cousine#license). * [Lato font](https://fonts.google.com/specimen/Lato). By Łukasz Dziedzic. [Apache License 2.0](https://fonts.google.com/specimen/Lato#license). * [Caladea font](https://fonts.google.com/specimen/Caladea). By Andrés Torresi, Carolina Giovanolli. [Apache License 2.0](https://fonts.google.com/specimen/Caladea#license). +* [Apache Commons Compress](https://commons.apache.org/proper/commons-compress/). Copyright © 2002-2021 The Apache Software Foundation. All Rights Reserved. [Apache License 2.0](https://www.apache.org/licenses/). diff --git a/app/src/main/java/eu/faircode/email/EntityAttachment.java b/app/src/main/java/eu/faircode/email/EntityAttachment.java index 65d8d28a33..63b6c296ba 100644 --- a/app/src/main/java/eu/faircode/email/EntityAttachment.java +++ b/app/src/main/java/eu/faircode/email/EntityAttachment.java @@ -100,6 +100,39 @@ public class EntityAttachment { return ImageHelper.isImage(getMimeType()); } + boolean isCompressed() { + if ("application/zip".equals(type)) + return true; + if ("application/gzip".equals(type) && !BuildConfig.PLAY_STORE_RELEASE) + return true; + + String extension = Helper.getExtension(name); + if ("zip".equals(extension)) + return true; + if ("gz".equals(extension) && !BuildConfig.PLAY_STORE_RELEASE) + return true; + + return false; + } + + boolean isGzip() { + if (BuildConfig.PLAY_STORE_RELEASE) + return false; + + if ("application/gzip".equals(type)) + return true; + + String extension = Helper.getExtension(name); + if ("gz".equals(extension)) + return true; + + return false; + } + + boolean isTarGzip() { + return (name != null && name.endsWith(".tar.gz")); + } + boolean isEncryption() { if ("application/pkcs7-mime".equals(type)) return true; diff --git a/app/src/main/java/eu/faircode/email/FragmentOptionsDisplay.java b/app/src/main/java/eu/faircode/email/FragmentOptionsDisplay.java index c30c887971..b7b16fd45e 100644 --- a/app/src/main/java/eu/faircode/email/FragmentOptionsDisplay.java +++ b/app/src/main/java/eu/faircode/email/FragmentOptionsDisplay.java @@ -167,6 +167,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer private SwitchCompat swImagesInline; private SwitchCompat swButtonExtra; private SwitchCompat swUnzip; + private TextView tvUnzipHint; private SwitchCompat swAttachmentsAlt; private SwitchCompat swThumbnails; @@ -317,6 +318,7 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer swImagesInline = view.findViewById(R.id.swImagesInline); swButtonExtra = view.findViewById(R.id.swButtonExtra); swUnzip = view.findViewById(R.id.swUnzip); + tvUnzipHint = view.findViewById(R.id.tvUnzipHint); swAttachmentsAlt = view.findViewById(R.id.swAttachmentsAlt); swThumbnails = view.findViewById(R.id.swThumbnails); swBundledFonts = view.findViewById(R.id.swBundledFonts); @@ -1175,6 +1177,8 @@ public class FragmentOptionsDisplay extends FragmentBase implements SharedPrefer } }); + tvUnzipHint.setText(getString(R.string.title_advanced_unzip_hint, MessageHelper.MAX_UNZIP)); + swAttachmentsAlt.setOnCheckedChangeListener(new CompoundButton.OnCheckedChangeListener() { @Override public void onCheckedChanged(CompoundButton compoundButton, boolean checked) { diff --git a/app/src/main/java/eu/faircode/email/MessageHelper.java b/app/src/main/java/eu/faircode/email/MessageHelper.java index 19d38bdad4..0f5a1a1896 100644 --- a/app/src/main/java/eu/faircode/email/MessageHelper.java +++ b/app/src/main/java/eu/faircode/email/MessageHelper.java @@ -49,6 +49,11 @@ import com.sun.mail.util.BASE64DecoderStream; import com.sun.mail.util.FolderClosedIOException; import com.sun.mail.util.MessageRemovedIOException; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.nodes.Node; @@ -144,6 +149,7 @@ public class MessageHelper { static final String HEADER_CORRELATION_ID = "X-Correlation-ID"; static final int MAX_SUBJECT_AGE = 48; // hours static final int DEFAULT_THREAD_RANGE = 7; // 2^7 = 128 days + static final int MAX_UNZIP = 10; static final List RECEIVED_WORDS = Collections.unmodifiableList(Arrays.asList( "from", "by", "via", "with", "id", "for" @@ -3373,77 +3379,146 @@ public class MessageHelper { } } catch (Throwable ex) { Log.e(ex); + db.attachment().setWarning(local.id, Log.formatThrowable(ex)); } - else if ("application/zip".equals(local.type)) { + else if (local.isCompressed()) { SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); boolean unzip = prefs.getBoolean("unzip", false); - if (unzip) { - // https://developer.android.com/reference/java/util/zip/ZipInputStream - try (ZipInputStream zis = new ZipInputStream( - new BufferedInputStream(new FileInputStream(local.getFile(context))))) { - - int subsequence = 1; - - ZipEntry ze; - while ((ze = zis.getNextEntry()) != null) - try { - String name = ze.getName(); - long total = ze.getSize(); - - // isDirectory: - // A directory entry is defined to be one whose name ends with a '/'. - if (ze.isDirectory() || - (name != null && name.endsWith("\\"))) { - Log.i("Zipped folder=" + name); - continue; + if (unzip) + if (local.isGzip() && !local.isTarGzip()) + try (GzipCompressorInputStream gzip = new GzipCompressorInputStream( + new BufferedInputStream(new FileInputStream(local.getFile(context))))) { + String name = gzip.getMetaData().getFilename(); + long total = gzip.getUncompressedCount(); + + Log.i("Gzipped attachment seq=" + local.sequence + " " + name + ":" + total); + + if (name == null && + local.name != null && local.name.endsWith(".gz")) + name = local.name.substring(0, local.name.length() - 3); + + EntityAttachment attachment = new EntityAttachment(); + attachment.message = local.message; + attachment.sequence = local.sequence; + attachment.subsequence = 1; + attachment.name = name; + attachment.type = Helper.guessMimeType(name); + if (total >= 0) + attachment.size = total; + attachment.id = db.attachment().insertAttachment(attachment); + + File efile = attachment.getFile(context); + Log.i("Gunzipping to " + efile); + + int last = 0; + long size = 0; + try (OutputStream os = new FileOutputStream(efile)) { + byte[] buffer = new byte[Helper.BUFFER_SIZE]; + for (int len = gzip.read(buffer); len != -1; len = gzip.read(buffer)) { + size += len; + os.write(buffer, 0, len); + + if (total > 0) { + int progress = (int) (size * 100 / total); + if (progress / 20 > last / 20) { + last = progress; + db.attachment().setProgress(attachment.id, progress); + } + } } + } catch (Throwable ex) { + Log.e(ex); + db.attachment().setError(attachment.id, Log.formatThrowable(ex)); + db.attachment().setAvailable(attachment.id, true); // unrecoverable + } - Log.i("Zipped attachment seq=" + local.sequence + ":" + subsequence + - " " + name + ":" + total); - - EntityAttachment entry = new EntityAttachment(); - entry.message = local.message; - entry.sequence = local.sequence; - entry.subsequence = subsequence++; - entry.name = name; - entry.type = Helper.guessMimeType(entry.name); - entry.size = total; - entry.id = db.attachment().insertAttachment(entry); + db.attachment().setDownloaded(attachment.id, efile.length()); + } catch (Throwable ex) { + Log.e(ex); + db.attachment().setWarning(local.id, Log.formatThrowable(ex)); + } + else + try (FileInputStream fis = new FileInputStream(local.getFile(context))) { + ArchiveInputStream ais = new ArchiveStreamFactory().createArchiveInputStream( + new BufferedInputStream(local.isTarGzip() ? new GzipCompressorInputStream(fis) : fis)); + + int count = 0; + ArchiveEntry entry; + while ((entry = ais.getNextEntry()) != null) + if (ais.canReadEntryData(entry) && !entry.isDirectory()) + if (++count > MAX_UNZIP) + break; + + Log.i("Zip entries=" + count); + if (count <= MAX_UNZIP) { + fis.getChannel().position(0); + + ais = new ArchiveStreamFactory().createArchiveInputStream( + new BufferedInputStream(local.isTarGzip() ? new GzipCompressorInputStream(fis) : fis)); + + int subsequence = 1; + while ((entry = ais.getNextEntry()) != null) { + if (!ais.canReadEntryData(entry)) { + Log.w("Zip invalid=" + entry); + continue; + } - File efile = entry.getFile(context); - Log.i("Unzipping to " + efile); + String name = entry.getName(); + long total = entry.getSize(); - int last = 0; - long size = 0; - try (OutputStream os = new FileOutputStream(efile)) { - byte[] buffer = new byte[Helper.BUFFER_SIZE]; - for (int len = zis.read(buffer); len != -1; len = zis.read(buffer)) { - size += len; - os.write(buffer, 0, len); + if (entry.isDirectory() || + (name != null && name.endsWith("\\"))) { + Log.i("Zipped folder=" + name); + continue; + } - int progress = (int) (size * 100 / total); - if (progress / 20 > last / 20) { - last = progress; - db.attachment().setProgress(entry.id, progress); + Log.i("Zipped attachment seq=" + local.sequence + ":" + subsequence + + " " + name + ":" + total); + + EntityAttachment attachment = new EntityAttachment(); + attachment.message = local.message; + attachment.sequence = local.sequence; + attachment.subsequence = subsequence++; + attachment.name = name; + attachment.type = Helper.guessMimeType(name); + if (total >= 0) + attachment.size = total; + attachment.id = db.attachment().insertAttachment(attachment); + + File efile = attachment.getFile(context); + Log.i("Unzipping to " + efile); + + int last = 0; + long size = 0; + try (OutputStream os = new FileOutputStream(efile)) { + byte[] buffer = new byte[Helper.BUFFER_SIZE]; + for (int len = ais.read(buffer); len != -1; len = ais.read(buffer)) { + size += len; + os.write(buffer, 0, len); + + if (total > 0) { + int progress = (int) (size * 100 / total); + if (progress / 20 > last / 20) { + last = progress; + db.attachment().setProgress(attachment.id, progress); + } + } } + } catch (Throwable ex) { + Log.e(ex); + db.attachment().setError(attachment.id, Log.formatThrowable(ex)); + db.attachment().setAvailable(attachment.id, true); // unrecoverable } - } catch (Throwable ex) { - Log.e(ex); - db.attachment().setError(entry.id, Log.formatThrowable(ex)); - db.attachment().setAvailable(entry.id, true); // unrecoverable - } - db.attachment().setDownloaded(entry.id, efile.length()); - } finally { - zis.closeEntry(); + db.attachment().setDownloaded(attachment.id, efile.length()); + } } - } catch (Throwable ex) { - Log.e(ex); - db.attachment().setWarning(local.id, Log.formatThrowable(ex)); - } - } + } catch (Throwable ex) { + Log.e(ex); + db.attachment().setWarning(local.id, Log.formatThrowable(ex)); + } } } } diff --git a/app/src/main/res/layout/fragment_options_display.xml b/app/src/main/res/layout/fragment_options_display.xml index 768ee049fc..5fa5ebcd3b 100644 --- a/app/src/main/res/layout/fragment_options_display.xml +++ b/app/src/main/res/layout/fragment_options_display.xml @@ -1761,6 +1761,18 @@ app:layout_constraintTop_toBottomOf="@id/swButtonExtra" app:switchPadding="12dp" /> + + Plain text only messages will be considered as preformatted This applies to reformatted messages only Inline images are images included in the message + The contents of zip files with up to %1$d files will automatically be shown This will more accurately display messages, but possibly with a delay Language detection support depends on the device manufacturer