diff --git a/package.json b/package.json index 9216279..4c92f5f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "aslobot-matrix", - "version": "0.9.0", + "version": "1.0.0", "description": "", "license": "ISC", "author": "", diff --git a/src/modules/ai/ai.ts b/src/modules/ai/ai.ts index 3f2b7f0..ede9900 100644 --- a/src/modules/ai/ai.ts +++ b/src/modules/ai/ai.ts @@ -34,6 +34,7 @@ const onAI = async ( sender: string, repliedMessage?: string, repliedSender?: string, + image?: Buffer, ) => { if (text.startsWith(`${config.app.triggerPrefix}aileaderboard`)) { return; @@ -87,6 +88,7 @@ const onAI = async ( instructions, `${username}: ${textMod}`, `${repliedUsername}: ${repliedMessage}`, + image, ); user.aiCost += responseAI.tokens * prices.text; diff --git a/src/modules/module.ts b/src/modules/module.ts index acb08e5..0514b93 100644 --- a/src/modules/module.ts +++ b/src/modules/module.ts @@ -1,6 +1,7 @@ import { MatrixClient, MatrixEvent, + MsgType, RoomEvent, type IContent, } from "matrix-js-sdk"; @@ -83,6 +84,30 @@ const registerModules = (client: MatrixClient) => { return; } + let image: Buffer | undefined; + if (content.msgtype === MsgType.Image) { + if (typeof content.url === "string") { + const httpUrl = client.mxcUrlToHttp( + content.url, + undefined, + undefined, + undefined, + undefined, + true, + true, + ); + if (httpUrl) { + const imageResponse = await fetch(httpUrl, { + headers: { + Authorization: `Bearer ${client.getAccessToken()}`, + }, + }); + const arrayBuffer = await imageResponse.arrayBuffer(); + image = Buffer.from(new Uint8Array(arrayBuffer)); + } + } + } + const relatesTo = event.getContent()["m.relates_to"]; const replyToId = relatesTo?.["m.in_reply_to"]?.event_id; let repliedMessage: string | undefined; @@ -122,6 +147,7 @@ const registerModules = (client: MatrixClient) => { sender, repliedMessage, repliedSender, + image, ); } }); diff --git a/src/modules/types.ts b/src/modules/types.ts index 00752a3..1a6b04a 100644 --- a/src/modules/types.ts +++ b/src/modules/types.ts @@ -15,6 +15,7 @@ interface ICallback { sender: string, repliedMessage?: string, repliedSender?: string, + image?: Buffer, ) => void; } diff --git a/src/services/ai/ai.ts b/src/services/ai/ai.ts index 31664c7..5fc0019 100644 --- a/src/services/ai/ai.ts +++ b/src/services/ai/ai.ts @@ -20,15 +20,31 @@ const getTextGemini = async ( instructions: IAIInstructions, input: string, oldInput?: string, + inputImage?: Buffer, ): Promise => { - const inputContent: Content = { - role: "user", - parts: [ - { - text: input, - }, - ], - }; + const inputContent: Content = inputImage + ? { + role: "user", + parts: [ + { + text: input, + }, + { + inlineData: { + mimeType: "image/png", + data: Buffer.from(inputImage).toString("base64"), + }, + }, + ], + } + : { + role: "user", + parts: [ + { + text: input, + }, + ], + }; const oldInputContent: Content = { role: "user", parts: [