msb-public
/
FairEmail
mirror of https://github.com/M66B/FairEmail.git
1
0
Code Issues Projects Releases Wiki Activity

Added charset override

Browse Source
pull/208/head
M66B 3 years ago
parent abfd93f531
commit b55db9a876
6 changed files with 203 additions and 93 deletions

77
app/src/main/java/eu/faircode/email/AdapterMessage.java
Unescape View File

@ -161,6 +161,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Field;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.security.NoSuchAlgorithmException;
import java.text.Collator;
@ -177,6 +178,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.SortedMap;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
@ -5574,6 +5576,9 @@ public class AdapterMessage extends RecyclerView.Adapter<AdapterMessage.ViewHold
} else if (itemId == R.id.menu_resync) {
onMenuResync(message);
return true;
} else if (itemId == R.id.menu_charset) {
onMenuCharset(message);
return true;
} else if (itemId == R.id.menu_alternative) {
onMenuAlt(message);
return true;
@ -6062,6 +6067,78 @@ public class AdapterMessage extends RecyclerView.Adapter<AdapterMessage.ViewHold
}.execute(context, owner, args, "message:resync");
}
private void onMenuCharset(TupleMessageEx message) {
Bundle args = new Bundle();
args.putLong("id", message.id);
new SimpleTask<SortedMap<String, Charset>>() {
@Override
protected SortedMap<String, Charset> onExecute(Context context, Bundle args) {
return Charset.availableCharsets();
}
@Override
protected void onExecuted(Bundle args, SortedMap<String, Charset> charsets) {
PopupMenuLifecycle popupMenu = new PopupMenuLifecycle(context, powner, ibMore);
int order = 0;
for (String name : charsets.keySet()) {
order++;
popupMenu.getMenu().add(Menu.NONE, order, order, name)
.setIntent(new Intent().putExtra("charset", name));
}
popupMenu.setOnMenuItemClickListener(new PopupMenu.OnMenuItemClickListener() {
@Override
public boolean onMenuItemClick(MenuItem item) {
args.putString("charset", item.getIntent().getStringExtra("charset"));
new SimpleTask<Void>() {
@Override
protected Void onExecute(Context context, Bundle args) {
long id = args.getLong("id");
String charset = args.getString("charset");
DB db = DB.getInstance(context);
try {
db.beginTransaction();
EntityMessage message = db.message().getMessage(id);
if (message == null)
return null;
db.message().resetMessageContent(id);
EntityOperation.queue(context, message, EntityOperation.BODY, null, charset);
db.setTransactionSuccessful();
} finally {
db.endTransaction();
}
return null;
}
@Override
protected void onException(Bundle args, Throwable ex) {
Log.unexpectedError(parentFragment.getParentFragmentManager(), ex);
}
}.execute(context, owner, args, "body:charset");
return true;
}
});
popupMenu.show();
}
@Override
protected void onException(Bundle args, Throwable ex) {
Log.unexpectedError(parentFragment.getParentFragmentManager(), ex);
}
}.execute(context, owner, args, "message:charset");
}
private void onMenuAlt(TupleMessageEx message) {
properties.setSize(message.id, null);
properties.setHeight(message.id, null);

5
app/src/main/java/eu/faircode/email/Core.java
Unescape View File

@ -1990,11 +1990,12 @@ class Core {
private static void onBody(Context context, JSONArray jargs, EntityFolder folder, EntityMessage message, IMAPFolder ifolder) throws MessagingException, IOException {
boolean plain_text = jargs.optBoolean(0);
String charset = jargs.optString(1, null);
// Download message body
DB db = DB.getInstance(context);
if (message.content && message.isPlainOnly() == plain_text)
if (message.content && message.isPlainOnly() == plain_text && charset == null)
return;
// Get message
@ -2004,7 +2005,7 @@ class Core {
MessageHelper helper = new MessageHelper((MimeMessage) imessage, context);
MessageHelper.MessageParts parts = helper.getMessageParts();
String body = parts.getHtml(context, plain_text);
String body = parts.getHtml(context, plain_text, charset);
File file = message.getFile(context);
Helper.writeText(file, body);
String text = HtmlHelper.getFullText(body);

198
app/src/main/java/eu/faircode/email/MessageHelper.java
Unescape View File

@ -3068,6 +3068,10 @@ public class MessageHelper {
}
String getHtml(Context context, boolean plain_text) throws MessagingException, IOException {
return getHtml(context, plain_text, null);
}
String getHtml(Context context, boolean plain_text, String override) throws MessagingException, IOException {
if (text.size() == 0) {
Log.i("No body part");
return null;
@ -3167,24 +3171,30 @@ public class MessageHelper {
}
if (h.isPlainText()) {
if (charset == null || StandardCharsets.ISO_8859_1.equals(cs)) {
if (StandardCharsets.ISO_8859_1.equals(cs) && CharsetHelper.isUTF8(result)) {
Log.i("Charset upgrade=UTF8");
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
} else {
Charset detected = CharsetHelper.detect(result, StandardCharsets.ISO_8859_1);
if (detected == null) {
if (CharsetHelper.isUTF8(result)) {
Log.i("Charset plain=UTF8");
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
}
if (override == null) {
if (charset == null || StandardCharsets.ISO_8859_1.equals(cs)) {
if (StandardCharsets.ISO_8859_1.equals(cs) && CharsetHelper.isUTF8(result)) {
Log.i("Charset upgrade=UTF8");
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
} else {
Log.i("Charset plain=" + detected.name());
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), detected);
Charset detected = CharsetHelper.detect(result, StandardCharsets.ISO_8859_1);
if (detected == null) {
if (CharsetHelper.isUTF8(result)) {
Log.i("Charset plain=UTF8");
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
}
} else {
Log.i("Charset plain=" + detected.name());
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), detected);
}
}
}
} else if (StandardCharsets.UTF_8.equals(cs))
result = CharsetHelper.utf8toW1252(result);
} else if (StandardCharsets.UTF_8.equals(cs))
result = CharsetHelper.utf8toW1252(result);
} else {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
Helper.copy(h.part.getDataHandler().getInputStream(), bos);
result = bos.toString(override);
}
// https://datatracker.ietf.org/doc/html/rfc3676
if ("flowed".equalsIgnoreCase(h.contentType.getParameter("format")))
@ -3215,94 +3225,100 @@ public class MessageHelper {
result = "<div x-plain=\"true\">" + HtmlHelper.formatPlainText(result) + "</div>";
} else if (h.isHtml()) {
// Conditionally upgrade to UTF8
if ((cs == null ||
StandardCharsets.US_ASCII.equals(cs) ||
StandardCharsets.ISO_8859_1.equals(cs)) &&
CharsetHelper.isUTF8(result))
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
if (override == null) {
// Conditionally upgrade to UTF8
if ((cs == null ||
StandardCharsets.US_ASCII.equals(cs) ||
StandardCharsets.ISO_8859_1.equals(cs)) &&
CharsetHelper.isUTF8(result))
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
//if (StandardCharsets.UTF_8.equals(cs))
// result = CharsetHelper.utf8w1252(result);
//if (StandardCharsets.UTF_8.equals(cs))
// result = CharsetHelper.utf8w1252(result);
// Fix incorrect UTF16
try {
if (CHARSET16.contains(cs)) {
Charset detected = CharsetHelper.detect(result, cs);
// UTF-16 can be detected as US-ASCII
if (!CHARSET16.contains(detected))
Log.w(new Throwable("Charset=" + cs + " detected=" + detected));
if (StandardCharsets.UTF_8.equals(detected)) {
charset = null;
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), detected);
// Fix incorrect UTF16
try {
if (CHARSET16.contains(cs)) {
Charset detected = CharsetHelper.detect(result, cs);
// UTF-16 can be detected as US-ASCII
if (!CHARSET16.contains(detected))
Log.w(new Throwable("Charset=" + cs + " detected=" + detected));
if (StandardCharsets.UTF_8.equals(detected)) {
charset = null;
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), detected);
}
}
} catch (Throwable ex) {
Log.w(ex);
}
} catch (Throwable ex) {
Log.w(ex);
}
if (charset == null) {
// <meta charset="utf-8" />
// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
String excerpt = result.substring(0, Math.min(MAX_META_EXCERPT, result.length()));
Document d = JsoupEx.parse(excerpt);
for (Element meta : d.select("meta")) {
if ("Content-Type".equalsIgnoreCase(meta.attr("http-equiv"))) {
try {
ContentType ct = new ContentType(meta.attr("content"));
charset = ct.getParameter("charset");
} catch (ParseException ex) {
Log.w(ex);
}
} else
charset = meta.attr("charset");
if (charset == null) {
// <meta charset="utf-8" />
// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
String excerpt = result.substring(0, Math.min(MAX_META_EXCERPT, result.length()));
Document d = JsoupEx.parse(excerpt);
for (Element meta : d.select("meta")) {
if ("Content-Type".equalsIgnoreCase(meta.attr("http-equiv"))) {
try {
ContentType ct = new ContentType(meta.attr("content"));
charset = ct.getParameter("charset");
} catch (ParseException ex) {
Log.w(ex);
}
} else
charset = meta.attr("charset");
if (!TextUtils.isEmpty(charset))
try {
Log.i("Charset meta=" + meta);
Charset c = Charset.forName(charset);
if (!TextUtils.isEmpty(charset))
try {
Log.i("Charset meta=" + meta);
Charset c = Charset.forName(charset);
// US-ASCII is a subset of ISO8859-1
if (StandardCharsets.US_ASCII.equals(c))
break;
// US-ASCII is a subset of ISO8859-1
if (StandardCharsets.US_ASCII.equals(c))
break;
// Check if really UTF-8
if (StandardCharsets.UTF_8.equals(c) && !CharsetHelper.isUTF8(result)) {
Log.w("Charset meta=" + meta + " !isUTF8");
break;
}
// Check if really UTF-8
if (StandardCharsets.UTF_8.equals(c) && !CharsetHelper.isUTF8(result)) {
Log.w("Charset meta=" + meta + " !isUTF8");
break;
}
// 16 bits charsets cannot be converted to 8 bits
if (CHARSET16.contains(c)) {
Log.w("Charset meta=" + meta);
break;
}
// 16 bits charsets cannot be converted to 8 bits
if (CHARSET16.contains(c)) {
Log.w("Charset meta=" + meta);
break;
}
Charset detected = CharsetHelper.detect(result, c);
if (c.equals(detected))
break;
Charset detected = CharsetHelper.detect(result, c);
if (c.equals(detected))
break;
// Common detected/meta
// - windows-1250, windows-1257 / ISO-8859-1
// - ISO-8859-1 / windows-1252
// - US-ASCII / windows-1250, windows-1252, ISO-8859-1, ISO-8859-15, UTF-8
if (StandardCharsets.US_ASCII.equals(detected) &&
("ISO-8859-15".equals(c.name()) ||
"windows-1250".equals(c.name()) ||
"windows-1252".equals(c.name()) ||
StandardCharsets.UTF_8.equals(c) ||
StandardCharsets.ISO_8859_1.equals(c)))
break;
// Common detected/meta
// - windows-1250, windows-1257 / ISO-8859-1
// - ISO-8859-1 / windows-1252
// - US-ASCII / windows-1250, windows-1252, ISO-8859-1, ISO-8859-15, UTF-8
if (StandardCharsets.US_ASCII.equals(detected) &&
("ISO-8859-15".equals(c.name()) ||
"windows-1250".equals(c.name()) ||
"windows-1252".equals(c.name()) ||
StandardCharsets.UTF_8.equals(c) ||
StandardCharsets.ISO_8859_1.equals(c)))
break;
// Convert
Log.w("Converting detected=" + detected + " meta=" + c);
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), c);
break;
} catch (Throwable ex) {
Log.e(ex);
}
// Convert
Log.w("Converting detected=" + detected + " meta=" + c);
result = new String(result.getBytes(StandardCharsets.ISO_8859_1), c);
break;
} catch (Throwable ex) {
Log.e(ex);
}
}
}
} else {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
Helper.copy(h.part.getDataHandler().getInputStream(), bos);
result = bos.toString(override);
}
} else if (h.isReport()) {
Report report = new Report(h.contentType.getBaseType(), result);

10
app/src/main/res/drawable/twotone_abc_24.xml
Unescape View File

@ -0,0 +1,10 @@
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="24dp"
android:height="24dp"
android:viewportWidth="24"
android:viewportHeight="24"
android:tint="?attr/colorControlNormal">
<path
android:fillColor="@android:color/white"
android:pathData="M21,11h-1.5v-0.5h-2v3h2V13H21v1c0,0.55 -0.45,1 -1,1h-3c-0.55,0 -1,-0.45 -1,-1v-4c0,-0.55 0.45,-1 1,-1h3c0.55,0 1,0.45 1,1V11zM8,10v5H6.5v-1.5h-2V15H3v-5c0,-0.55 0.45,-1 1,-1h3C7.55,9 8,9.45 8,10zM6.5,10.5h-2V12h2V10.5zM13.5,12c0.55,0 1,0.45 1,1v1c0,0.55 -0.45,1 -1,1h-4V9h4c0.55,0 1,0.45 1,1v1C14.5,11.55 14.05,12 13.5,12zM11,10.5v0.75h2V10.5H11zM13,12.75h-2v0.75h2V12.75z"/>
</vector>

5
app/src/main/res/menu/popup_message_more.xml
Unescape View File

@ -160,6 +160,11 @@
android:icon="@drawable/twotone_sync_24"
android:title="@string/title_resync" />
<item
android:id="@+id/menu_charset"
android:icon="@drawable/twotone_abc_24"
android:title="@string/title_charset" />
<item
android:id="@+id/menu_alternative"
android:icon="@drawable/twotone_sync_alt_24"

1
app/src/main/res/values/strings.xml
Unescape View File

@ -1472,6 +1472,7 @@
<string name="title_decrypt">Decrypt</string>
<string name="title_thread_info" translatable="false">Thread info</string>
<string name="title_resync">Resync</string>
<string name="title_charset">Encoding</string>
<string name="title_alternative_text">Show plain text</string>
<string name="title_alternative_html">Show HTML</string>
<string name="title_no_openpgp">OpenKeychain not found</string>

Write Preview
Loading…
Cancel
Save