msb-public
/
FairEmail
mirror of https://github.com/M66B/FairEmail.git
1
0
Code Issues Projects Releases Wiki Activity

OpenAI: prepare multi-modal support

Browse Source
pull/215/head
M66B 3 months ago
parent 4a52109212
commit 1cbebf7870
3 changed files with 77 additions and 21 deletions
Show Stats Download Patch File Download Diff File

27
app/src/main/java/eu/faircode/email/FragmentCompose.java
Unescape View File

@ -2690,11 +2690,13 @@ public class FragmentCompose extends FragmentBase {
Document parsed = JsoupEx.parse(inreplyto.get(0).getFile(context)); Document parsed = JsoupEx.parse(inreplyto.get(0).getFile(context));
Document document = HtmlHelper.sanitizeView(context, parsed, false); Document document = HtmlHelper.sanitizeView(context, parsed, false);
Spanned spanned = HtmlHelper.fromDocument(context, document, null, null); Spanned spanned = HtmlHelper.fromDocument(context, document, null, null);
result.add(new OpenAI.Message(role, OpenAI.truncateParagraphs(spanned.toString()))); result.add(new OpenAI.Message(role, new OpenAI.Content[]{
new OpenAI.Content(OpenAI.CONTENT_TEXT, OpenAI.truncateParagraphs(spanned.toString()))}));
} }
if (!TextUtils.isEmpty(body)) if (!TextUtils.isEmpty(body))
result.add(new OpenAI.Message(OpenAI.ASSISTANT, OpenAI.truncateParagraphs(body))); result.add(new OpenAI.Message(OpenAI.ASSISTANT, new OpenAI.Content[]{
new OpenAI.Content(OpenAI.CONTENT_TEXT, OpenAI.truncateParagraphs(body))}));
if (result.size() == 0) if (result.size() == 0)
return null; return null;
@ -2706,7 +2708,9 @@ public class FragmentCompose extends FragmentBase {
if (moderation) if (moderation)
for (OpenAI.Message message : result) for (OpenAI.Message message : result)
OpenAI.checkModeration(context, message.getContent()); for (OpenAI.Content content : message.getContent())
if (OpenAI.CONTENT_TEXT.equals(content.getContent()))
OpenAI.checkModeration(context, content.getContent());
OpenAI.Message[] completions = OpenAI.Message[] completions =
OpenAI.completeChat(context, model, result.toArray(new OpenAI.Message[0]), temperature, 1); OpenAI.completeChat(context, model, result.toArray(new OpenAI.Message[0]), temperature, 1);
@ -2719,8 +2723,15 @@ public class FragmentCompose extends FragmentBase {
if (messages == null || messages.length == 0) if (messages == null || messages.length == 0)
return; return;
String text = messages[0].getContent() StringBuilder sb = new StringBuilder();
.replaceAll("^\\n+", "").replaceAll("\\n+$", ""); for (OpenAI.Message message : messages)
for (OpenAI.Content content : message.getContent())
if (OpenAI.CONTENT_TEXT.equals(content.getType())) {
if (sb.length() > 0)
sb.append('\n');
sb.append(content.getContent().replaceAll("^\\n+", "").replaceAll("\\n+$", ""));
}
Editable edit = etBody.getText(); Editable edit = etBody.getText();
int start = etBody.getSelectionStart(); int start = etBody.getSelectionStart();
@ -2738,10 +2749,10 @@ public class FragmentCompose extends FragmentBase {
if (index > 0 && edit.charAt(index - 1) != '\n') if (index > 0 && edit.charAt(index - 1) != '\n')
edit.insert(index++, "\n"); edit.insert(index++, "\n");
edit.insert(index, text + "\n"); edit.insert(index, sb + "\n");
etBody.setSelection(index + text.length() + 1); etBody.setSelection(index + sb.length() + 1);
StyleHelper.markAsInserted(edit, index, index + text.length() + 1); StyleHelper.markAsInserted(edit, index, index + sb.length() + 1);
if (args.containsKey("used") && args.containsKey("granted")) { if (args.containsKey("used") && args.containsKey("granted")) {
double used = args.getDouble("used"); double used = args.getDouble("used");

18
app/src/main/java/eu/faircode/email/FragmentDialogSummarize.java
Unescape View File

@ -108,16 +108,20 @@ public class FragmentDialogSummarize extends FragmentDialogBase {
String prompt = prefs.getString("openai_summarize", OpenAI.SUMMARY_PROMPT); String prompt = prefs.getString("openai_summarize", OpenAI.SUMMARY_PROMPT);
List<OpenAI.Message> result = new ArrayList<>(); List<OpenAI.Message> result = new ArrayList<>();
result.add(new OpenAI.Message(OpenAI.ASSISTANT, prompt)); result.add(new OpenAI.Message(OpenAI.ASSISTANT,
result.add(new OpenAI.Message(OpenAI.USER, text)); new OpenAI.Content[]{new OpenAI.Content(OpenAI.CONTENT_TEXT, prompt)}));
result.add(new OpenAI.Message(OpenAI.USER,
new OpenAI.Content[]{new OpenAI.Content(OpenAI.CONTENT_TEXT, text)}));
OpenAI.Message[] completions = OpenAI.Message[] completions =
OpenAI.completeChat(context, model, result.toArray(new OpenAI.Message[0]), temperature, 1); OpenAI.completeChat(context, model, result.toArray(new OpenAI.Message[0]), temperature, 1);
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
for (OpenAI.Message completion : completions) { for (OpenAI.Message completion : completions)
if (sb.length() != 0) for (OpenAI.Content content : completion.getContent())
sb.append('\n'); if (OpenAI.CONTENT_TEXT.equals(content.getType())) {
sb.append(completion.getContent()); if (sb.length() != 0)
} sb.append('\n');
sb.append(content.getContent());
}
return sb.toString(); return sb.toString();
} else if (Gemini.isAvailable(context)) { } else if (Gemini.isAvailable(context)) {
String model = prefs.getString("gemini_model", "gemini-pro"); String model = prefs.getString("gemini_model", "gemini-pro");

53
app/src/main/java/eu/faircode/email/OpenAI.java
Unescape View File

@ -46,6 +46,10 @@ public class OpenAI {
static final String USER = "user"; static final String USER = "user";
static final String SUMMARY_PROMPT = "Summarize the following text:"; static final String SUMMARY_PROMPT = "Summarize the following text:";
// https://cookbook.openai.com/examples/gpt4o/introduction_to_gpt4o
static final String CONTENT_TEXT = "text";
static final String CONTENT_IMAGE = "image_url";
private static final int MAX_OPENAI_LEN = 1000; // characters private static final int MAX_OPENAI_LEN = 1000; // characters
private static final int TIMEOUT = 45; // seconds private static final int TIMEOUT = 45; // seconds
@ -108,7 +112,18 @@ public class OpenAI {
for (Message message : messages) { for (Message message : messages) {
JSONObject jmessage = new JSONObject(); JSONObject jmessage = new JSONObject();
jmessage.put("role", message.role); jmessage.put("role", message.role);
jmessage.put("content", message.content);
JSONArray jcontents = new JSONArray();
for (Content content : message.content) {
JSONObject jcontent = new JSONObject();
jcontent.put("type", content.type);
jcontent.put(content.type, content.content);
jcontents.put(jcontent);
}
jmessage.put("content", jcontents);
jmessages.put(jmessage); jmessages.put(jmessage);
} }
@ -125,7 +140,8 @@ public class OpenAI {
for (int i = 0; i < jchoices.length(); i++) { for (int i = 0; i < jchoices.length(); i++) {
JSONObject jchoice = jchoices.getJSONObject(i); JSONObject jchoice = jchoices.getJSONObject(i);
JSONObject jmessage = jchoice.getJSONObject("message"); JSONObject jmessage = jchoice.getJSONObject("message");
choices[i] = new Message(jmessage.getString("role"), jmessage.getString("content")); choices[i] = new Message(jmessage.getString("role"),
new Content[]{new Content(CONTENT_TEXT, jmessage.getString("content"))});
} }
return choices; return choices;
@ -232,11 +248,29 @@ public class OpenAI {
return sb.toString(); return sb.toString();
} }
static class Content {
private String type;
private String content;
public Content(String type, String content) {
this.type = type;
this.content = content;
}
public String getType() {
return this.type;
}
public String getContent() {
return this.content;
}
}
static class Message { static class Message {
private final String role; // system, user, assistant private final String role; // system, user, assistant
private final String content; private final Content[] content;
public Message(String role, String content) { public Message(String role, Content[] content) {
this.role = role; this.role = role;
this.content = content; this.content = content;
} }
@ -245,14 +279,21 @@ public class OpenAI {
return this.role; return this.role;
} }
public String getContent() { public Content[] getContent() {
return this.content; return this.content;
} }
@NonNull @NonNull
@Override @Override
public String toString() { public String toString() {
return this.role + ": " + this.content; StringBuilder sb = new StringBuilder();
if (this.content != null)
for (Content c : this.content) {
if (sb.length() > 0)
sb.append(", ");
sb.append(c.type).append(':').append(c.content);
}
return this.role + ": " + sb;
} }
} }

Write Preview
Loading…
Cancel
Save