Added image inputs; Version 1.0.0

package.json

@@ -1,6 +1,6 @@
 {
     "name": "aslobot-matrix",
-    "version": "0.9.0",
+    "version": "1.0.0",
     "description": "",
     "license": "ISC",
     "author": "",

src/modules/ai/ai.ts

@@ -34,6 +34,7 @@ const onAI = async (
     sender: string,
     repliedMessage?: string,
     repliedSender?: string,
+    image?: Buffer<ArrayBuffer>,
 ) => {
     if (text.startsWith(`${config.app.triggerPrefix}aileaderboard`)) {
         return;
@@ -87,6 +88,7 @@ const onAI = async (
         instructions,
         `${username}: ${textMod}`,
         `${repliedUsername}: ${repliedMessage}`,
+        image,
     );
 
     user.aiCost += responseAI.tokens * prices.text;

src/modules/module.ts

@@ -1,6 +1,7 @@
 import {
     MatrixClient,
     MatrixEvent,
+    MsgType,
     RoomEvent,
     type IContent,
 } from "matrix-js-sdk";
@@ -83,6 +84,30 @@ const registerModules = (client: MatrixClient) => {
             return;
         }
 
+        let image: Buffer<ArrayBuffer> | undefined;
+        if (content.msgtype === MsgType.Image) {
+            if (typeof content.url === "string") {
+                const httpUrl = client.mxcUrlToHttp(
+                    content.url,
+                    undefined,
+                    undefined,
+                    undefined,
+                    undefined,
+                    true,
+                    true,
+                );
+                if (httpUrl) {
+                    const imageResponse = await fetch(httpUrl, {
+                        headers: {
+                            Authorization: `Bearer ${client.getAccessToken()}`,
+                        },
+                    });
+                    const arrayBuffer = await imageResponse.arrayBuffer();
+                    image = Buffer.from(new Uint8Array(arrayBuffer));
+                }
+            }
+        }
+
         const relatesTo = event.getContent()["m.relates_to"];
         const replyToId = relatesTo?.["m.in_reply_to"]?.event_id;
         let repliedMessage: string | undefined;
@@ -122,6 +147,7 @@ const registerModules = (client: MatrixClient) => {
                     sender,
                     repliedMessage,
                     repliedSender,
+                    image,
                 );
             }
         });

src/modules/types.ts

@@ -15,6 +15,7 @@ interface ICallback {
         sender: string,
         repliedMessage?: string,
         repliedSender?: string,
+        image?: Buffer<ArrayBuffer>,
     ) => void;
 }
 

src/services/ai/ai.ts

@@ -20,15 +20,31 @@ const getTextGemini = async (
     instructions: IAIInstructions,
     input: string,
     oldInput?: string,
+    inputImage?: Buffer<ArrayBuffer>,
 ): Promise<AIResponseText> => {
-    const inputContent: Content = {
-        role: "user",
-        parts: [
-            {
-                text: input,
-            },
-        ],
-    };
+    const inputContent: Content = inputImage
+        ? {
+              role: "user",
+              parts: [
+                  {
+                      text: input,
+                  },
+                  {
+                      inlineData: {
+                          mimeType: "image/png",
+                          data: Buffer.from(inputImage).toString("base64"),
+                      },
+                  },
+              ],
+          }
+        : {
+              role: "user",
+              parts: [
+                  {
+                      text: input,
+                  },
+              ],
+          };
     const oldInputContent: Content = {
         role: "user",
         parts: [