From 6abb6dd47d6f3e2c1180052067d57cab40711b3a Mon Sep 17 00:00:00 2001 From: M66B Date: Mon, 20 May 2024 15:39:14 +0200 Subject: [PATCH] AI improvements --- app/src/main/java/eu/faircode/email/AI.java | 27 +++++++---- .../main/java/eu/faircode/email/OpenAI.java | 46 +++++++++---------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/app/src/main/java/eu/faircode/email/AI.java b/app/src/main/java/eu/faircode/email/AI.java index 78be92c7fb..e22c1a9b01 100644 --- a/app/src/main/java/eu/faircode/email/AI.java +++ b/app/src/main/java/eu/faircode/email/AI.java @@ -53,9 +53,9 @@ public class AI { boolean multimodal = prefs.getBoolean("openai_multimodal", false); OpenAI.Message message; - if (body instanceof Spannable) + if (body instanceof Spannable && multimodal) message = new OpenAI.Message(OpenAI.USER, - OpenAI.Content.get((Spannable) body, id, multimodal, context)); + OpenAI.Content.get((Spannable) body, id, context)); else message = new OpenAI.Message(OpenAI.USER, new OpenAI.Content[]{ new OpenAI.Content(OpenAI.CONTENT_TEXT, body.toString())}); @@ -139,9 +139,13 @@ public class AI { input.add(new OpenAI.Message(OpenAI.USER, new OpenAI.Content[]{new OpenAI.Content(OpenAI.CONTENT_TEXT, message.subject)})); - SpannableStringBuilder ssb = HtmlHelper.fromDocument(context, d, null, null); - input.add(new OpenAI.Message(OpenAI.USER, - OpenAI.Content.get(ssb, message.id, multimodal, context))); + if (multimodal) { + SpannableStringBuilder ssb = HtmlHelper.fromDocument(context, d, null, null); + input.add(new OpenAI.Message(OpenAI.USER, + OpenAI.Content.get(ssb, message.id, context))); + } else + input.add(new OpenAI.Message(OpenAI.USER, new OpenAI.Content[]{ + new OpenAI.Content(OpenAI.CONTENT_TEXT, d.text())})); OpenAI.Message[] completions = OpenAI.completeChat(context, model, input.toArray(new OpenAI.Message[0]), temperature, 1); @@ -160,10 +164,15 @@ public class AI { float temperature = prefs.getFloat("gemini_temperature", Gemini.DEFAULT_TEMPERATURE); String prompt = prefs.getString("gemini_summarize", Gemini.DEFAULT_SUMMARY_PROMPT); - String text = d.text(); - if (TextUtils.isEmpty(text)) - return null; - Gemini.Message content = new Gemini.Message(Gemini.USER, new String[]{prompt, text}); + String body = d.text(); + + List texts = new ArrayList<>(); + texts.add(prompt); + if (!TextUtils.isEmpty(message.subject)) + texts.add(message.subject); + if (!TextUtils.isEmpty(body)) + texts.add(body); + Gemini.Message content = new Gemini.Message(Gemini.USER, texts.toArray(new String[0])); Gemini.Message[] completions = Gemini.generate(context, model, new Gemini.Message[]{content}, temperature, 1); diff --git a/app/src/main/java/eu/faircode/email/OpenAI.java b/app/src/main/java/eu/faircode/email/OpenAI.java index 0e06ddc696..5995bf875d 100644 --- a/app/src/main/java/eu/faircode/email/OpenAI.java +++ b/app/src/main/java/eu/faircode/email/OpenAI.java @@ -277,7 +277,7 @@ public class OpenAI { return this.content; } - static Content[] get(Spannable ssb, long id, boolean multimodal, Context context) { + static Content[] get(Spannable ssb, long id, Context context) { DB db = DB.getInstance(context); List contents = new ArrayList<>(); int start = 0; @@ -298,29 +298,29 @@ public class OpenAI { Log.i("OpenAI images=" + (spans == null ? null : spans.length)); if (spans != null && spans.length == 1) { int e = ssb.getSpanEnd(spans[0]); - if (multimodal) { - String url = null; - String src = spans[0].getSource(); - Log.i("OpenAI image url=" + src); - if (src != null && src.startsWith("cid:")) { - String cid = '<' + src.substring(4) + '>'; - EntityAttachment attachment = db.attachment().getAttachment(id, cid); - if (attachment != null && attachment.available) { - File file = attachment.getFile(context); - try (InputStream is = new FileInputStream(file)) { - Bitmap bm = ImageHelper.getScaledBitmap(is, null, null, SCALE2PIXELS); - Helper.ByteArrayInOutStream bos = new Helper.ByteArrayInOutStream(); - bm.compress(Bitmap.CompressFormat.PNG, 90, bos); - url = ImageHelper.getDataUri(bos.getInputStream(), "image/png"); - } catch (Throwable ex) { - Log.w(ex); - } + + String url = null; + String src = spans[0].getSource(); + Log.i("OpenAI image url=" + src); + if (src != null && src.startsWith("cid:")) { + String cid = '<' + src.substring(4) + '>'; + EntityAttachment attachment = db.attachment().getAttachment(id, cid); + if (attachment != null && attachment.available) { + File file = attachment.getFile(context); + try (InputStream is = new FileInputStream(file)) { + Bitmap bm = ImageHelper.getScaledBitmap(is, null, null, SCALE2PIXELS); + Helper.ByteArrayInOutStream bos = new Helper.ByteArrayInOutStream(); + bm.compress(Bitmap.CompressFormat.PNG, 90, bos); + url = ImageHelper.getDataUri(bos.getInputStream(), "image/png"); + } catch (Throwable ex) { + Log.w(ex); } - } else - url = src; - if (url != null) - contents.add(new OpenAI.Content(OpenAI.CONTENT_IMAGE, url)); - } + } + } else + url = src; + if (url != null) + contents.add(new OpenAI.Content(OpenAI.CONTENT_IMAGE, url)); + end = e; } }