dev: gemini function calling

Author: KernelDeimos

src/backend/src/modules/puterai/GeminiService.js

@@ -3,6 +3,7 @@ const { GoogleGenerativeAI } = require('@google/generative-ai');
 const GeminiSquareHole = require("./lib/GeminiSquareHole");
 const { TypedValue } = require("../../services/drivers/meta/Runtime");
 const putility = require("@heyputer/putility");
+const FunctionCalling = require("./lib/FunctionCalling");
 
 class GeminiService extends BaseService {
     async _init () {
@@ -31,17 +32,22 @@ class GeminiService extends BaseService {
             },
 
             async complete ({ messages, stream, model, tools }) {
+                tools = FunctionCalling.make_gemini_tools(tools);
+
                 const genAI = new GoogleGenerativeAI(this.config.apiKey);
                 const genModel = genAI.getGenerativeModel({
                     model: model ?? 'gemini-2.0-flash',
+                    tools,
                 });
 
                 messages = await GeminiSquareHole.process_input_messages(messages);
 
                 // History is separate, so the last message gets special treatment.
                 const last_message = messages.pop();
                 const last_message_parts = last_message.parts.map(
-                    part => typeof part === 'string' ? part : part.text
+                    part => typeof part === 'string' ? part :
+                            typeof part.text === 'string' ? part.text :
+                            part
                 );
 
                 const chat = genModel.startChat({

src/backend/src/modules/puterai/lib/FunctionCalling.js

@@ -90,4 +90,16 @@ module.exports = class FunctionCalling {
             };
         });
     }
+
+    static make_gemini_tools (tools) {
+        return [
+            {
+                function_declarations: tools.map(t => {
+                    const tool = t.function;
+                    delete tool.parameters.additionalProperties;
+                    return tool;
+                })
+            }
+        ];
+    }
 }

src/backend/src/modules/puterai/lib/GeminiSquareHole.js

@@ -3,6 +3,8 @@
  * but Google's AI API defies all the established conventions
  * so it made sense to defy them here as well.
  */
+
+const crypto = require('crypto');
 module.exports = class GeminiSquareHole {
     static process_input_messages = async (messages) => {
         messages = messages.slice();
@@ -14,6 +16,35 @@ module.exports = class GeminiSquareHole {
             if ( msg.role === 'assistant' ) {
                 msg.role = 'model';
             }
+
+            for ( let i=0 ; i < msg.parts.length ; i++ ) {
+                const part = msg.parts[i];
+                console.log('what the part is', part);
+                if ( part.type === 'tool_use' ) {
+                    msg.parts[i] = {
+                        functionCall: {
+                            name: part.id,
+                            args: part.input,
+                        },
+                    };
+                }
+                if ( part.type === 'tool_result' ) {
+                    msg.parts[i] = {
+                        functionResponse: {
+                            name: part.tool_use_id,
+                            response: {
+                                name: part.tool_use_id,
+                                content: part.content,
+                            },
+                        },
+                    };
+                }
+                if ( part.type === 'text' ) {
+                    msg.parts[i] = {
+                        text: part.text,
+                    };
+                }
+            }
         }
 
         return messages;
@@ -46,7 +77,12 @@ module.exports = class GeminiSquareHole {
         usage_promise,
     }) => async ({ chatStream }) => {
         const message = chatStream.message();
+        
         let textblock = message.contentBlock({ type: 'text' });
+        let toolblock = null;
+        let mode = 'text';
+
+        
         let last_usage = null;
         for await ( const chunk of stream ) {
             // This is spread across several lines so that the stack trace
@@ -56,6 +92,31 @@ module.exports = class GeminiSquareHole {
             const content = candidate.content;
             const parts = content.parts;
             for ( const part of parts ) {
+                if ( part.functionCall ) {
+                    if ( mode === 'text' ) {
+                        mode = 'tool';
+                        textblock.end();
+                    }
+
+                    toolblock = message.contentBlock({
+                        type: 'tool_use',
+                        id: part.functionCall.name,
+                        name: part.functionCall.name,
+                    });
+                    toolblock.addPartialJSON(JSON.stringify(
+                        part.functionCall.args,
+                    ));
+
+                    continue;
+                }
+
+                if ( mode === 'tool' ) {
+                    mode = 'text';
+                    toolblock.end();
+                    textblock = message.contentBlock({ type: 'text' });
+                }
+
+                // assume text as default
                 const text = part.text;
                 textblock.addText(text);
             }
@@ -65,7 +126,8 @@ module.exports = class GeminiSquareHole {
 
         usage_promise.resolve(last_usage);
 
-        textblock.end();
+        if ( mode === 'text' ) textblock.end();
+        if ( mode === 'tool' ) toolblock.end();
         message.end();
         chatStream.end();
     }