msb-public
/
FairEmail
mirror of https://github.com/M66B/FairEmail.git
1
0
Code Issues Projects Releases Wiki Activity

Added charset override

Browse Source
pull/208/head
M66B 4 years ago
parent abfd93f531
commit b55db9a876
6 changed files with 203 additions and 93 deletions
Show Stats Download Patch File Download Diff File

77
app/src/main/java/eu/faircode/email/AdapterMessage.java
Unescape View File

@ -161,6 +161,7 @@ import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.text.Collator; import java.text.Collator;
@ -177,6 +178,7 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Properties; import java.util.Properties;
import java.util.SortedMap;
import java.util.concurrent.Executor; import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
@ -5574,6 +5576,9 @@ public class AdapterMessage extends RecyclerView.Adapter<AdapterMessage.ViewHold
} else if (itemId == R.id.menu_resync) { } else if (itemId == R.id.menu_resync) {
onMenuResync(message); onMenuResync(message);
return true; return true;
} else if (itemId == R.id.menu_charset) {
onMenuCharset(message);
return true;
} else if (itemId == R.id.menu_alternative) { } else if (itemId == R.id.menu_alternative) {
onMenuAlt(message); onMenuAlt(message);
return true; return true;
@ -6062,6 +6067,78 @@ public class AdapterMessage extends RecyclerView.Adapter<AdapterMessage.ViewHold
}.execute(context, owner, args, "message:resync"); }.execute(context, owner, args, "message:resync");
} }
private void onMenuCharset(TupleMessageEx message) {
Bundle args = new Bundle();
args.putLong("id", message.id);
new SimpleTask<SortedMap<String, Charset>>() {
@Override
protected SortedMap<String, Charset> onExecute(Context context, Bundle args) {
return Charset.availableCharsets();
}
@Override
protected void onExecuted(Bundle args, SortedMap<String, Charset> charsets) {
PopupMenuLifecycle popupMenu = new PopupMenuLifecycle(context, powner, ibMore);
int order = 0;
for (String name : charsets.keySet()) {
order++;
popupMenu.getMenu().add(Menu.NONE, order, order, name)
.setIntent(new Intent().putExtra("charset", name));
}
popupMenu.setOnMenuItemClickListener(new PopupMenu.OnMenuItemClickListener() {
@Override
public boolean onMenuItemClick(MenuItem item) {
args.putString("charset", item.getIntent().getStringExtra("charset"));
new SimpleTask<Void>() {
@Override
protected Void onExecute(Context context, Bundle args) {
long id = args.getLong("id");
String charset = args.getString("charset");
DB db = DB.getInstance(context);
try {
db.beginTransaction();
EntityMessage message = db.message().getMessage(id);
if (message == null)
return null;
db.message().resetMessageContent(id);
EntityOperation.queue(context, message, EntityOperation.BODY, null, charset);
db.setTransactionSuccessful();
} finally {
db.endTransaction();
}
return null;
}
@Override
protected void onException(Bundle args, Throwable ex) {
Log.unexpectedError(parentFragment.getParentFragmentManager(), ex);
}
}.execute(context, owner, args, "body:charset");
return true;
}
});
popupMenu.show();
}
@Override
protected void onException(Bundle args, Throwable ex) {
Log.unexpectedError(parentFragment.getParentFragmentManager(), ex);
}
}.execute(context, owner, args, "message:charset");
}
private void onMenuAlt(TupleMessageEx message) { private void onMenuAlt(TupleMessageEx message) {
properties.setSize(message.id, null); properties.setSize(message.id, null);
properties.setHeight(message.id, null); properties.setHeight(message.id, null);

5
app/src/main/java/eu/faircode/email/Core.java
Unescape View File

@ -1990,11 +1990,12 @@ class Core {
private static void onBody(Context context, JSONArray jargs, EntityFolder folder, EntityMessage message, IMAPFolder ifolder) throws MessagingException, IOException { private static void onBody(Context context, JSONArray jargs, EntityFolder folder, EntityMessage message, IMAPFolder ifolder) throws MessagingException, IOException {
boolean plain_text = jargs.optBoolean(0); boolean plain_text = jargs.optBoolean(0);
String charset = jargs.optString(1, null);
// Download message body // Download message body
DB db = DB.getInstance(context); DB db = DB.getInstance(context);
if (message.content && message.isPlainOnly() == plain_text) if (message.content && message.isPlainOnly() == plain_text && charset == null)
return; return;
// Get message // Get message
@ -2004,7 +2005,7 @@ class Core {
MessageHelper helper = new MessageHelper((MimeMessage) imessage, context); MessageHelper helper = new MessageHelper((MimeMessage) imessage, context);
MessageHelper.MessageParts parts = helper.getMessageParts(); MessageHelper.MessageParts parts = helper.getMessageParts();
String body = parts.getHtml(context, plain_text); String body = parts.getHtml(context, plain_text, charset);
File file = message.getFile(context); File file = message.getFile(context);
Helper.writeText(file, body); Helper.writeText(file, body);
String text = HtmlHelper.getFullText(body); String text = HtmlHelper.getFullText(body);

198
app/src/main/java/eu/faircode/email/MessageHelper.java
Unescape View File

@ -3068,6 +3068,10 @@ public class MessageHelper {
} }
String getHtml(Context context, boolean plain_text) throws MessagingException, IOException { String getHtml(Context context, boolean plain_text) throws MessagingException, IOException {
return getHtml(context, plain_text, null);
}
String getHtml(Context context, boolean plain_text, String override) throws MessagingException, IOException {
if (text.size() == 0) { if (text.size() == 0) {
Log.i("No body part"); Log.i("No body part");
return null; return null;
@ -3167,24 +3171,30 @@ public class MessageHelper {
} }
if (h.isPlainText()) { if (h.isPlainText()) {
if (charset == null || StandardCharsets.ISO_8859_1.equals(cs)) { if (override == null) {
if (StandardCharsets.ISO_8859_1.equals(cs) && CharsetHelper.isUTF8(result)) { if (charset == null || StandardCharsets.ISO_8859_1.equals(cs)) {
Log.i("Charset upgrade=UTF8"); if (StandardCharsets.ISO_8859_1.equals(cs) && CharsetHelper.isUTF8(result)) {
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8); Log.i("Charset upgrade=UTF8");
} else { result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
Charset detected = CharsetHelper.detect(result, StandardCharsets.ISO_8859_1);
if (detected == null) {
if (CharsetHelper.isUTF8(result)) {
Log.i("Charset plain=UTF8");
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
}
} else { } else {
Log.i("Charset plain=" + detected.name()); Charset detected = CharsetHelper.detect(result, StandardCharsets.ISO_8859_1);
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), detected); if (detected == null) {
if (CharsetHelper.isUTF8(result)) {
Log.i("Charset plain=UTF8");
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
}
} else {
Log.i("Charset plain=" + detected.name());
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), detected);
}
} }
} } else if (StandardCharsets.UTF_8.equals(cs))
} else if (StandardCharsets.UTF_8.equals(cs)) result = CharsetHelper.utf8toW1252(result);
result = CharsetHelper.utf8toW1252(result); } else {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
Helper.copy(h.part.getDataHandler().getInputStream(), bos);
result = bos.toString(override);
}
// https://datatracker.ietf.org/doc/html/rfc3676 // https://datatracker.ietf.org/doc/html/rfc3676
if ("flowed".equalsIgnoreCase(h.contentType.getParameter("format"))) if ("flowed".equalsIgnoreCase(h.contentType.getParameter("format")))
@ -3215,94 +3225,100 @@ public class MessageHelper {
result = "<div x-plain=\"true\">" + HtmlHelper.formatPlainText(result) + "</div>"; result = "<div x-plain=\"true\">" + HtmlHelper.formatPlainText(result) + "</div>";
} else if (h.isHtml()) { } else if (h.isHtml()) {
// Conditionally upgrade to UTF8 if (override == null) {
if ((cs == null || // Conditionally upgrade to UTF8
StandardCharsets.US_ASCII.equals(cs) || if ((cs == null ||
StandardCharsets.ISO_8859_1.equals(cs)) && StandardCharsets.US_ASCII.equals(cs) ||
CharsetHelper.isUTF8(result)) StandardCharsets.ISO_8859_1.equals(cs)) &&
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8); CharsetHelper.isUTF8(result))
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
//if (StandardCharsets.UTF_8.equals(cs)) //if (StandardCharsets.UTF_8.equals(cs))
// result = CharsetHelper.utf8w1252(result); // result = CharsetHelper.utf8w1252(result);
// Fix incorrect UTF16 // Fix incorrect UTF16
try { try {
if (CHARSET16.contains(cs)) { if (CHARSET16.contains(cs)) {
Charset detected = CharsetHelper.detect(result, cs); Charset detected = CharsetHelper.detect(result, cs);
// UTF-16 can be detected as US-ASCII // UTF-16 can be detected as US-ASCII
if (!CHARSET16.contains(detected)) if (!CHARSET16.contains(detected))
Log.w(new Throwable("Charset=" + cs + " detected=" + detected)); Log.w(new Throwable("Charset=" + cs + " detected=" + detected));
if (StandardCharsets.UTF_8.equals(detected)) { if (StandardCharsets.UTF_8.equals(detected)) {
charset = null; charset = null;
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), detected); result = new String(result.getBytes(StandardCharsets.ISO_8859_1), detected);
}
} }
} catch (Throwable ex) {
Log.w(ex);
} }
} catch (Throwable ex) {
Log.w(ex);
}
if (charset == null) { if (charset == null) {
// <meta charset="utf-8" /> // <meta charset="utf-8" />
// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> // <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
String excerpt = result.substring(0, Math.min(MAX_META_EXCERPT, result.length())); String excerpt = result.substring(0, Math.min(MAX_META_EXCERPT, result.length()));
Document d = JsoupEx.parse(excerpt); Document d = JsoupEx.parse(excerpt);
for (Element meta : d.select("meta")) { for (Element meta : d.select("meta")) {
if ("Content-Type".equalsIgnoreCase(meta.attr("http-equiv"))) { if ("Content-Type".equalsIgnoreCase(meta.attr("http-equiv"))) {
try { try {
ContentType ct = new ContentType(meta.attr("content")); ContentType ct = new ContentType(meta.attr("content"));
charset = ct.getParameter("charset"); charset = ct.getParameter("charset");
} catch (ParseException ex) { } catch (ParseException ex) {
Log.w(ex); Log.w(ex);
} }
} else } else
charset = meta.attr("charset"); charset = meta.attr("charset");
if (!TextUtils.isEmpty(charset)) if (!TextUtils.isEmpty(charset))
try { try {
Log.i("Charset meta=" + meta); Log.i("Charset meta=" + meta);
Charset c = Charset.forName(charset); Charset c = Charset.forName(charset);
// US-ASCII is a subset of ISO8859-1 // US-ASCII is a subset of ISO8859-1
if (StandardCharsets.US_ASCII.equals(c)) if (StandardCharsets.US_ASCII.equals(c))
break; break;
// Check if really UTF-8 // Check if really UTF-8
if (StandardCharsets.UTF_8.equals(c) && !CharsetHelper.isUTF8(result)) { if (StandardCharsets.UTF_8.equals(c) && !CharsetHelper.isUTF8(result)) {
Log.w("Charset meta=" + meta + " !isUTF8"); Log.w("Charset meta=" + meta + " !isUTF8");
break; break;
} }
// 16 bits charsets cannot be converted to 8 bits // 16 bits charsets cannot be converted to 8 bits
if (CHARSET16.contains(c)) { if (CHARSET16.contains(c)) {
Log.w("Charset meta=" + meta); Log.w("Charset meta=" + meta);
break; break;
} }
Charset detected = CharsetHelper.detect(result, c); Charset detected = CharsetHelper.detect(result, c);
if (c.equals(detected)) if (c.equals(detected))
break; break;
// Common detected/meta // Common detected/meta
// - windows-1250, windows-1257 / ISO-8859-1 // - windows-1250, windows-1257 / ISO-8859-1
// - ISO-8859-1 / windows-1252 // - ISO-8859-1 / windows-1252
// - US-ASCII / windows-1250, windows-1252, ISO-8859-1, ISO-8859-15, UTF-8 // - US-ASCII / windows-1250, windows-1252, ISO-8859-1, ISO-8859-15, UTF-8
if (StandardCharsets.US_ASCII.equals(detected) && if (StandardCharsets.US_ASCII.equals(detected) &&
("ISO-8859-15".equals(c.name()) || ("ISO-8859-15".equals(c.name()) ||
"windows-1250".equals(c.name()) || "windows-1250".equals(c.name()) ||
"windows-1252".equals(c.name()) || "windows-1252".equals(c.name()) ||
StandardCharsets.UTF_8.equals(c) || StandardCharsets.UTF_8.equals(c) ||
StandardCharsets.ISO_8859_1.equals(c))) StandardCharsets.ISO_8859_1.equals(c)))
break; break;
// Convert // Convert
Log.w("Converting detected=" + detected + " meta=" + c); Log.w("Converting detected=" + detected + " meta=" + c);
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), c); result = new String(result.getBytes(StandardCharsets.ISO_8859_1), c);
break; break;
} catch (Throwable ex) { } catch (Throwable ex) {
Log.e(ex); Log.e(ex);
} }
}
} }
} else {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
Helper.copy(h.part.getDataHandler().getInputStream(), bos);
result = bos.toString(override);
} }
} else if (h.isReport()) { } else if (h.isReport()) {
Report report = new Report(h.contentType.getBaseType(), result); Report report = new Report(h.contentType.getBaseType(), result);

10
app/src/main/res/drawable/twotone_abc_24.xml
Unescape View File

@ -0,0 +1,10 @@
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="24dp"
android:height="24dp"
android:viewportWidth="24"
android:viewportHeight="24"
android:tint="?attr/colorControlNormal">
<path
android:fillColor="@android:color/white"
android:pathData="M21,11h-1.5v-0.5h-2v3h2V13H21v1c0,0.55 -0.45,1 -1,1h-3c-0.55,0 -1,-0.45 -1,-1v-4c0,-0.55 0.45,-1 1,-1h3c0.55,0 1,0.45 1,1V11zM8,10v5H6.5v-1.5h-2V15H3v-5c0,-0.55 0.45,-1 1,-1h3C7.55,9 8,9.45 8,10zM6.5,10.5h-2V12h2V10.5zM13.5,12c0.55,0 1,0.45 1,1v1c0,0.55 -0.45,1 -1,1h-4V9h4c0.55,0 1,0.45 1,1v1C14.5,11.55 14.05,12 13.5,12zM11,10.5v0.75h2V10.5H11zM13,12.75h-2v0.75h2V12.75z"/>
</vector>

5
app/src/main/res/menu/popup_message_more.xml
Unescape View File

@ -160,6 +160,11 @@
android:icon="@drawable/twotone_sync_24" android:icon="@drawable/twotone_sync_24"
android:title="@string/title_resync" /> android:title="@string/title_resync" />
<item
android:id="@+id/menu_charset"
android:icon="@drawable/twotone_abc_24"
android:title="@string/title_charset" />
<item <item
android:id="@+id/menu_alternative" android:id="@+id/menu_alternative"
android:icon="@drawable/twotone_sync_alt_24" android:icon="@drawable/twotone_sync_alt_24"

1
app/src/main/res/values/strings.xml
Unescape View File

@ -1472,6 +1472,7 @@
<string name="title_decrypt">Decrypt</string> <string name="title_decrypt">Decrypt</string>
<string name="title_thread_info" translatable="false">Thread info</string> <string name="title_thread_info" translatable="false">Thread info</string>
<string name="title_resync">Resync</string> <string name="title_resync">Resync</string>
<string name="title_charset">Encoding</string>
<string name="title_alternative_text">Show plain text</string> <string name="title_alternative_text">Show plain text</string>
<string name="title_alternative_html">Show HTML</string> <string name="title_alternative_html">Show HTML</string>
<string name="title_no_openpgp">OpenKeychain not found</string> <string name="title_no_openpgp">OpenKeychain not found</string>

Write Preview
Loading…
Cancel
Save