support message detail interface

src/chatbox.tsx

@@ -13,6 +13,7 @@ import ChatGPT, {
   calculate_token_length,
   ChunkMessage,
   FetchResponse,
+  MessageDetail,
 } from "./chatgpt";
 import Message from "./message";
 import models from "./models";
@@ -258,6 +259,7 @@ export default function ChatBOX(props: {
     );
     _setTemplateAPIs(templateAPIs);
   };
+  const [images, setImages] = useState<MessageDetail[]>([]);
 
   return (
     <div className="grow flex flex-col p-2 dark:text-black">
@@ -633,7 +635,13 @@ export default function ChatBOX(props: {
                     chatStore.history
                       .filter(({ hide }) => !hide)
                       .slice(chatStore.postBeginIndex)
-                      .map(({ content }) => content)
+                      .map(({ content }) => {
+                        if (typeof content === "string") {
+                          return content;
+                        } else {
+                          return content.map((c) => c?.text).join(" ");
+                        }
+                      })
                   )
                   .concat([inputMsg])
                   .join(" ");

src/chatgpt.ts

@@ -1,8 +1,22 @@
+export interface MessageDetail {
+  type: "text" | "image_url";
+  text?: string;
+  image_url?: string;
+}
 export interface Message {
   role: "system" | "user" | "assistant" | "function";
-  content: string;
+  content: string | MessageDetail[];
   name?: "example_user" | "example_assistant";
 }
+export const getMessageText = (message: Message): string => {
+  if (typeof message.content === "string") {
+    return message.content;
+  }
+  return message.content
+    .filter((c) => c.type === "text")
+    .map((c) => c?.text)
+    .join("\n");
+};
 
 export interface ChunkMessage {
   model: string;
@@ -29,9 +43,15 @@ export interface FetchResponse {
   }[];
 }
 // https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them
-export function calculate_token_length(content: string): number {
-  const totalCount = content.length;
-  const chineseCount = content.match(/[\u00ff-\uffff]|\S+/g)?.length ?? 0;
+export function calculate_token_length(
+  content: string | MessageDetail[]
+): number {
+  const text =
+    typeof content === "string"
+      ? content
+      : content.map((c) => c?.text).join(" ");
+  const totalCount = text.length;
+  const chineseCount = text.match(/[\u00ff-\uffff]|\S+/g)?.length ?? 0;
   const englishCount = totalCount - chineseCount;
   const tokenLength = englishCount / 4 + (chineseCount * 4) / 3;
   return ~~tokenLength;
@@ -151,12 +171,6 @@ class Chat {
     return j;
   }
 
-  async say(content: string): Promise<string> {
-    this.messages.push({ role: "user", content });
-    await this.complete();
-    return this.messages.slice(-1)[0].content;
-  }
-
   async *processStreamResponse(resp: Response) {
     const reader = resp?.body?.pipeThrough(new TextDecoderStream()).getReader();
     if (reader === undefined) {
@@ -210,7 +224,8 @@ class Chat {
     }
 
     return (
-      resp?.choices[0]?.message?.content ?? `Error: ${JSON.stringify(resp)}`
+      (resp?.choices[0]?.message?.content as string) ??
+      `Error: ${JSON.stringify(resp)}`
     );
   }
 
@@ -221,7 +236,7 @@ class Chat {
 
   completeWithSteam() {
     this.total_tokens = this.messages
-      .map((msg) => this.calculate_token_length(msg.content) + 20)
+      .map((msg) => this.calculate_token_length(msg.content as string) + 20)
       .reduce((a, v) => a + v);
     return this._fetch(true);
   }

src/message.tsx

@@ -1,7 +1,7 @@
 import { Tr, langCodeContext, LANG_OPTIONS } from "./translate";
 import { useState, useEffect, StateUpdater } from "preact/hooks";
 import { ChatStore, ChatStoreMessage } from "./app";
-import { calculate_token_length } from "./chatgpt";
+import { calculate_token_length, getMessageText } from "./chatgpt";
 import Markdown from "preact-markdown";
 import TTSButton from "./tts";
 
@@ -14,20 +14,6 @@ interface EditMessageProps {
 
 function EditMessage(props: EditMessageProps) {
   const { setShowEdit, chat, setChatStore, chatStore } = props;
-  useEffect(() => {
-    const handleKeyPress = (event: any) => {
-      if (event.keyCode === 27) {
-        // keyCode for ESC key is 27
-        setShowEdit(false);
-      }
-    };
-
-    document.addEventListener("keydown", handleKeyPress);
-
-    return () => {
-      document.removeEventListener("keydown", handleKeyPress);
-    };
-  }, []); // The empty dependency array ensures that the effect runs only once
 
   return (
     <div
@@ -37,18 +23,25 @@ function EditMessage(props: EditMessageProps) {
       onClick={() => setShowEdit(false)}
     >
       <div className="w-full h-full z-20">
-        <textarea
-          className={"w-full h-full"}
-          value={chat.content}
-          onClick={(event: any) => {
-            event.stopPropagation();
-          }}
-          onChange={(event: any) => {
-            chat.content = event.target.value;
-            chat.token = calculate_token_length(chat.content);
-            setChatStore({ ...chatStore });
-          }}
-        ></textarea>
+        {typeof chat.content === "string" && (
+          <textarea
+            className={"w-full h-full"}
+            value={chat.content}
+            onClick={(event: any) => {
+              event.stopPropagation();
+            }}
+            onChange={(event: any) => {
+              chat.content = event.target.value;
+              chat.token = calculate_token_length(chat.content);
+              setChatStore({ ...chatStore });
+            }}
+            onKeyPress={(event: any) => {
+              if (event.keyCode == 27) {
+                setShowEdit(false);
+              }
+            }}
+          ></textarea>
+        )}
         <div className={"w-full flex justify-center"}>
           <button
             className={"m-2 p-1 rounded bg-green-500"}
@@ -112,7 +105,7 @@ export default function Message(props: Props) {
       <>
         <button
           onClick={() => {
-            navigator.clipboard.writeText(chat.content);
+            navigator.clipboard.writeText(getMessageText(chat));
             setShowCopiedHint(true);
             setTimeout(() => setShowCopiedHint(false), 1000);
           }}
@@ -152,7 +145,8 @@ export default function Message(props: Props) {
           >
             <p className={renderMarkdown ? "" : "message-content"}>
               {chat.hide ? (
-                chat.content.split("\n")[0].slice(0, 16) + "... (deleted)"
+                getMessageText(chat).split("\n")[0].slice(0, 16) +
+                "... (deleted)"
               ) : renderMarkdown ? (
                 // @ts-ignore
                 <Markdown markdown={chat.content} />
@@ -167,7 +161,7 @@ export default function Message(props: Props) {
               {chatStore.tts_api && chatStore.tts_key && (
                 <TTSButton
                   chatStore={chatStore}
-                  text={chat.content}
+                  text={getMessageText(chat)}
                   setChatStore={setChatStore}
                 />
               )}