diff --git a/package-lock.json b/package-lock.json
index 5674e8574b..054db68a4d 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -2679,6 +2679,14 @@
"@hapi/topo": "^5.0.0"
}
},
+ "node_modules/@heyputer/airouter": {
+ "resolved": "src/airouter.js",
+ "link": true
+ },
+ "node_modules/@heyputer/airouter.js": {
+ "resolved": "src/airouter.js",
+ "link": true
+ },
"node_modules/@heyputer/backend": {
"resolved": "src/backend",
"link": true
@@ -17579,6 +17587,11 @@
"url": "https://github.com/sponsors/colinhacks"
}
},
+ "src/airouter.js": {
+ "name": "@heyputer/airouter.js",
+ "version": "0.0.0",
+ "license": "UNLICENSED"
+ },
"src/backend": {
"name": "@heyputer/backend",
"version": "2.5.1",
@@ -17588,6 +17601,7 @@
"@aws-sdk/client-polly": "^3.622.0",
"@aws-sdk/client-textract": "^3.621.0",
"@google/generative-ai": "^0.21.0",
+ "@heyputer/airouter": "^0.0.0",
"@heyputer/kv.js": "^0.1.9",
"@heyputer/multest": "^0.0.2",
"@heyputer/putility": "^1.0.0",
diff --git a/src/airouter.js/airouter.js b/src/airouter.js/airouter.js
new file mode 100644
index 0000000000..6e4a52a8ed
--- /dev/null
+++ b/src/airouter.js/airouter.js
@@ -0,0 +1,49 @@
+import { Registry } from './core/Registry.js';
+
+
+const registry = new Registry();
+const define = registry.getDefineAPI();
+
+import convenienceRegistrants from './common/convenience.js';
+convenienceRegistrants(define);
+
+import commonRegistrants from './common/index.js';
+commonRegistrants(define);
+
+import anthropicRegistrants from './anthropic/index.js';
+anthropicRegistrants(define);
+
+import openaiRegistrants from './openai/index.js';
+openaiRegistrants(define);
+
+export const obtain = registry.getObtainAPI();
+
+export * from './common/types.js';
+
+// Streaming Utilities
+export { CompletionWriter } from './common/stream/CompletionWriter.js';
+export { MessageWriter } from './common/stream/MessageWriter.js';
+export { ToolUseWriter } from './common/stream/ToolUseWriter.js';
+export { TextWriter } from './common/stream/TextWriter.js';
+export { BaseWriter } from './common/stream/BaseWriter.js';
+
+// Common prompt processing
+export { NormalizedPromptUtil } from './common/prompt/NormalizedPromptUtil.js';
+export { UniversalToolsNormalizer } from './common/prompt/UniversalToolsNormalizer.js';
+
+// Conventional Processing
+export { OpenAIStyleMessagesAdapter } from './convention/openai/OpenAIStyleMessagesAdapter.js';
+export { OpenAIStyleStreamAdapter } from './convention/openai/OpenAIStyleStreamAdapter.js';
+
+// Model-Specific Processing
+export { OpenAIToolsAdapter } from './openai/OpenAIToolsAdapter.js';
+export { GeminiToolsAdapter } from './gemini/GeminiToolsAdapter.js';
+
+// API Keys
+export { ANTHROPIC_API_KEY } from './anthropic/index.js';
+export { OPENAI_CLIENT } from './openai/index.js';
+
+import openai_models from './models/openai.json' with { type: 'json' };
+export const models = {
+ openai: openai_models,
+};
diff --git a/src/airouter.js/anthropic/consts.js b/src/airouter.js/anthropic/consts.js
new file mode 100644
index 0000000000..28d5647630
--- /dev/null
+++ b/src/airouter.js/anthropic/consts.js
@@ -0,0 +1 @@
+export const betas = ['files-api-2025-04-14'];
diff --git a/src/airouter.js/anthropic/handle_files.js b/src/airouter.js/anthropic/handle_files.js
new file mode 100644
index 0000000000..f7e6e1cfc0
--- /dev/null
+++ b/src/airouter.js/anthropic/handle_files.js
@@ -0,0 +1,59 @@
+import { toFile } from "@anthropic-ai/sdk";
+import { betas } from "./consts.js";
+
+export default async ({ client, cleanups, messages }) => {
+ const file_input_tasks = [];
+ for ( const message of messages ) {
+ // We can assume `message.content` is not undefined because
+ // UniversalPromptNormalizer ensures this.
+ for ( const contentPart of message.content ) {
+ if ( contentPart.type !== 'data' ) continue;
+ const { data } = contentPart;
+ delete contentPart.data;
+ file_input_tasks.push({
+ data,
+ contentPart,
+ });
+ }
+ }
+
+ if ( file_input_tasks.length === 0 ) return false;
+
+ const promises = [];
+ for ( const task of file_input_tasks ) promises.push((async () => {
+ const stream = await task.data.getStream();
+ const mimeType = await task.data.getMimeType();
+
+ const fileUpload = await client.files.upload({
+ file: await toFile(stream, undefined, { type: mimeType })
+ }, { betas });
+
+ cleanups.push(() => client.files.delete( fileUpload.id, { betas }));
+
+ // We have to copy a table from the documentation here:
+ // https://docs.anthropic.com/en/docs/build-with-claude/files
+ const contentBlockTypeForFileBasedOnMime = (() => {
+ if ( mimeType.startsWith('image/') ) {
+ return 'image';
+ }
+ if ( mimeType.startsWith('text/') ) {
+ return 'document';
+ }
+ if ( mimeType === 'application/pdf' || mimeType === 'application/x-pdf' ) {
+ return 'document';
+ }
+ return 'container_upload';
+ })();
+
+ delete task.contentPart.data,
+ task.contentPart.type = contentBlockTypeForFileBasedOnMime;
+ task.contentPart.source = {
+ type: 'file',
+ file_id: fileUpload.id,
+ };
+ })());
+
+ await Promise.all(promises);
+
+ return true;
+}
\ No newline at end of file
diff --git a/src/airouter.js/anthropic/index.js b/src/airouter.js/anthropic/index.js
new file mode 100644
index 0000000000..9d3aba9d93
--- /dev/null
+++ b/src/airouter.js/anthropic/index.js
@@ -0,0 +1,110 @@
+import { ASYNC_RESPONSE, COERCED_PARAMS, COERCED_TOOLS, COMPLETION_WRITER, NORMALIZED_LLM_PARAMS, NORMALIZED_LLM_TOOLS, PROVIDER_NAME, STREAM, SYNC_RESPONSE, USAGE_WRITER } from "../common/types.js";
+
+import { NormalizedPromptUtil } from '../common/prompt/NormalizedPromptUtil.js';
+import Anthropic from "@anthropic-ai/sdk";
+
+import { betas } from "./consts.js";
+import handle_files from "./handle_files.js";
+import write_to_stream from "./write_to_stream.js";
+
+export const ANTHROPIC_API_KEY = Symbol('ANTHROPIC_API_KEY');
+export const ANTHROPIC_CLIENT = Symbol('ANTHROPIC_CLIENT');
+
+export default define => {
+ // Define how to get parameters for the Anthropic client
+ define.howToGet(COERCED_PARAMS).from(NORMALIZED_LLM_PARAMS)
+ .provided(x => x.get(PROVIDER_NAME) == 'anthropic')
+ .as(async x => {
+ const params = x.get(NORMALIZED_LLM_PARAMS);
+ params.tools = await x.obtain(COERCED_TOOLS);
+
+ let system_prompts;
+ [system_prompts, params.messages] = NormalizedPromptUtil.extract_and_remove_system_messages(params.messages);
+
+ if ( ! x.memo.cleanups ) x.memo.cleanups = [];
+ await handle_files({
+ client: await x.obtain(ANTHROPIC_CLIENT),
+ cleanups: x.memo.cleanups,
+ messages: params.messages
+ });
+
+ return {
+ model: params.model,
+ max_tokens: Math.floor(params.max_tokens) ||
+ ((
+ params.model === 'claude-3-5-sonnet-20241022'
+ || params.model === 'claude-3-5-sonnet-20240620'
+ ) ? 8192 : 4096), //required
+ temperature: params.temperature || 0, // required
+ ...(system_prompts ? {
+ system: system_prompts.length > 1
+ ? JSON.stringify(system_prompts)
+ : JSON.stringify(system_prompts[0])
+ } : {}),
+ messages: params.messages,
+ ...(params.tools ? { tools: params.tools } : {}),
+ betas,
+ };
+ });
+
+ // Define how to get tools in the format expected by Anthropic
+ define.howToGet(COERCED_TOOLS).from(NORMALIZED_LLM_TOOLS)
+ .provided(x => x.get(PROVIDER_NAME) == 'anthropic')
+ .as(async x => {
+ const tools = x.get(NORMALIZED_LLM_TOOLS);
+ if ( ! tools ) return undefined;
+ return tools.map(tool => {
+ const { name, description, parameters } = tool.function;
+ return {
+ name,
+ description,
+ input_schema: parameters,
+ };
+ });
+ });
+
+ define.howToGet(ANTHROPIC_CLIENT).from(ANTHROPIC_API_KEY).as(async x => {
+ let client = new Anthropic({
+ apiKey: await x.obtain(ANTHROPIC_API_KEY),
+ });
+ return client.beta;
+ });
+
+ define.howToGet(ASYNC_RESPONSE).from(NORMALIZED_LLM_PARAMS)
+ .provided(x => x.get(PROVIDER_NAME) == 'anthropic')
+ .as(async x => {
+ const anthropic_params = await x.obtain(COERCED_PARAMS, {
+ [NORMALIZED_LLM_PARAMS]: x.get(NORMALIZED_LLM_PARAMS),
+ });
+ let client = await x.obtain(ANTHROPIC_CLIENT);
+
+ const anthropicStream = await client.messages.stream(anthropic_params);
+
+ const completionWriter = x.get(COMPLETION_WRITER);
+ await write_to_stream({
+ input: anthropicStream,
+ completionWriter,
+ usageWriter: x.get(USAGE_WRITER) ?? { resolve: () => {} },
+ });
+ if ( x.memo?.cleanups ) await Promise.all(x.memo.cleanups);
+ });
+
+ define.howToGet(SYNC_RESPONSE).from(NORMALIZED_LLM_PARAMS)
+ .provided(x => x.get(PROVIDER_NAME) == 'anthropic')
+ .as(async x => {
+ const anthropic_params = await x.obtain(COERCED_PARAMS, {
+ [NORMALIZED_LLM_PARAMS]: x.get(NORMALIZED_LLM_PARAMS),
+ });
+ let client = await x.obtain(ANTHROPIC_CLIENT);
+
+ const msg = await client.messages.create(anthropic_params);
+
+ if ( x.memo?.cleanups ) await Promise.all(x.memo.cleanups);
+
+ return {
+ message: msg,
+ usage: msg.usage,
+ finish_reason: 'stop',
+ };
+ })
+};
diff --git a/src/airouter.js/anthropic/write_to_stream.js b/src/airouter.js/anthropic/write_to_stream.js
new file mode 100644
index 0000000000..d8b0c31900
--- /dev/null
+++ b/src/airouter.js/anthropic/write_to_stream.js
@@ -0,0 +1,57 @@
+export default async ({ input, completionWriter, usageWriter }) => {
+ let message, contentBlock;
+ let counts = { input_tokens: 0, output_tokens: 0 };
+ for await ( const event of input ) {
+ const input_tokens =
+ (event?.usage ?? event?.message?.usage)?.input_tokens;
+ const output_tokens =
+ (event?.usage ?? event?.message?.usage)?.output_tokens;
+
+ if ( input_tokens ) counts.input_tokens += input_tokens;
+ if ( output_tokens ) counts.output_tokens += output_tokens;
+
+ if ( event.type === 'message_start' ) {
+ message = completionWriter.message();
+ continue;
+ }
+ if ( event.type === 'message_stop' ) {
+ message.end();
+ message = null;
+ continue;
+ }
+
+ if ( event.type === 'content_block_start' ) {
+ if ( event.content_block.type === 'tool_use' ) {
+ contentBlock = message.contentBlock({
+ type: event.content_block.type,
+ id: event.content_block.id,
+ name: event.content_block.name,
+ });
+ continue;
+ }
+ contentBlock = message.contentBlock({
+ type: event.content_block.type,
+ });
+ continue;
+ }
+
+ if ( event.type === 'content_block_stop' ) {
+ contentBlock.end();
+ contentBlock = null;
+ continue;
+ }
+
+ if ( event.type === 'content_block_delta' ) {
+ if ( event.delta.type === 'input_json_delta' ) {
+ contentBlock.addPartialJSON(event.delta.partial_json);
+ continue;
+ }
+ if ( event.delta.type === 'text_delta' ) {
+ contentBlock.addText(event.delta.text);
+ continue;
+ }
+ }
+ }
+ completionWriter.end();
+ usageWriter.resolve(counts);
+}
diff --git a/src/airouter.js/common/convenience.js b/src/airouter.js/common/convenience.js
new file mode 100644
index 0000000000..774eb48e23
--- /dev/null
+++ b/src/airouter.js/common/convenience.js
@@ -0,0 +1,14 @@
+import { NORMALIZED_LLM_MESSAGES, NORMALIZED_LLM_PARAMS, NORMALIZED_LLM_TOOLS, SDK_STYLE, USAGE_SDK_STYLE } from "./types.js"
+
+export default define => {
+ define.howToGet(NORMALIZED_LLM_TOOLS).from(NORMALIZED_LLM_PARAMS).as(x => {
+ return x.get(NORMALIZED_LLM_PARAMS).tools;
+ })
+ define.howToGet(NORMALIZED_LLM_MESSAGES).from(NORMALIZED_LLM_PARAMS).as(x => {
+ return x.get(NORMALIZED_LLM_PARAMS).messages;
+ })
+
+ define.howToGet(USAGE_SDK_STYLE).from(SDK_STYLE).as(x => {
+ return x.get(SDK_STYLE);
+ });
+}
diff --git a/src/backend/src/modules/puterai/lib/Messages.js b/src/airouter.js/common/index.js
similarity index 68%
rename from src/backend/src/modules/puterai/lib/Messages.js
rename to src/airouter.js/common/index.js
index bee8cc37ae..f37c34b681 100644
--- a/src/backend/src/modules/puterai/lib/Messages.js
+++ b/src/airouter.js/common/index.js
@@ -1,10 +1,24 @@
-const { whatis } = require("../../../util/langutil");
+import { NORMALIZED_LLM_MESSAGES, NORMALIZED_LLM_PARAMS, NORMALIZED_SINGLE_MESSAGE, UNIVERSAL_LLM_MESSAGES, UNIVERSAL_LLM_PARAMS, UNIVERSAL_SINGLE_MESSAGE } from "./types.js"
+import { whatis } from "./util/lang.js";
-module.exports = class Messages {
- static normalize_single_message (message, params = {}) {
- params = Object.assign({
- role: 'user',
- }, params);
+export default define => {
+ define.howToGet(NORMALIZED_LLM_PARAMS).from(UNIVERSAL_LLM_PARAMS)
+ .as(async x => {
+ const universal_params = x.get(UNIVERSAL_LLM_PARAMS);
+ const normalized_params = {
+ ...universal_params,
+ };
+
+ normalized_params.messages = await x.obtain(NORMALIZED_LLM_MESSAGES);
+
+ return normalized_params;
+ });
+
+ define.howToGet(NORMALIZED_SINGLE_MESSAGE).from(UNIVERSAL_SINGLE_MESSAGE)
+ .as(async x => {
+ let message = x.get(UNIVERSAL_SINGLE_MESSAGE);
+
+ const params = { role: 'user' };
if ( typeof message === 'string' ) {
message = {
@@ -64,10 +78,16 @@ module.exports = class Messages {
}
return message;
- }
- static normalize_messages (messages, params = {}) {
+ });
+
+ define.howToGet(NORMALIZED_LLM_MESSAGES).from(UNIVERSAL_LLM_MESSAGES)
+ .as(async x => {
+ let messages = [...x.get(UNIVERSAL_LLM_MESSAGES)];
+
for ( let i=0 ; i < messages.length ; i++ ) {
- messages[i] = this.normalize_single_message(messages[i], params);
+ messages[i] = await x.obtain(NORMALIZED_SINGLE_MESSAGE, {
+ [UNIVERSAL_SINGLE_MESSAGE]: messages[i],
+ })
}
// Split messages with tool_use content into separate messages
@@ -105,45 +125,5 @@ module.exports = class Messages {
}
return merged_messages;
- }
-
- static extract_and_remove_system_messages (messages) {
- let system_messages = [];
- let new_messages = [];
- for ( let i=0 ; i < messages.length ; i++ ) {
- if ( messages[i].role === 'system' ) {
- system_messages.push(messages[i]);
- } else {
- new_messages.push(messages[i]);
- }
- }
- return [system_messages, new_messages];
- }
-
- static extract_text (messages) {
- return messages.map(m => {
- if ( whatis(m) === 'string' ) {
- return m;
- }
- if ( whatis(m) !== 'object' ) {
- return '';
- }
- if ( whatis(m.content) === 'array' ) {
- return m.content.map(c => c.text).join(' ');
- }
- if ( whatis(m.content) === 'string' ) {
- return m.content;
- } else {
- const is_text_type = m.content.type === 'text' ||
- ! m.content.hasOwnProperty('type');
- if ( is_text_type ) {
- if ( whatis(m.content.text) !== 'string' ) {
- throw new Error('text content must be a string');
- }
- return m.content.text;
- }
- return '';
- }
- }).join(' ');
- }
+ });
}
\ No newline at end of file
diff --git a/src/airouter.js/common/prompt/NormalizedPromptUtil.js b/src/airouter.js/common/prompt/NormalizedPromptUtil.js
new file mode 100644
index 0000000000..53589399e6
--- /dev/null
+++ b/src/airouter.js/common/prompt/NormalizedPromptUtil.js
@@ -0,0 +1,47 @@
+import { whatis } from "../util/lang.js";
+
+/**
+ * NormalizedPromptUtil provides utility functions that can be called on
+ * normalized arrays of "chat" messages.
+ */
+export class NormalizedPromptUtil {
+ static extract_text (messages) {
+ return messages.map(m => {
+ if ( whatis(m) === 'string' ) {
+ return m;
+ }
+ if ( whatis(m) !== 'object' ) {
+ return '';
+ }
+ if ( whatis(m.content) === 'array' ) {
+ return m.content.map(c => c.text).join(' ');
+ }
+ if ( whatis(m.content) === 'string' ) {
+ return m.content;
+ } else {
+ const is_text_type = m.content.type === 'text' ||
+ ! m.content.hasOwnProperty('type');
+ if ( is_text_type ) {
+ if ( whatis(m.content.text) !== 'string' ) {
+ throw new Error('text content must be a string');
+ }
+ return m.content.text;
+ }
+ return '';
+ }
+ }).join(' ');
+ }
+
+ static extract_and_remove_system_messages (messages) {
+ let system_messages = [];
+ let new_messages = [];
+ for ( let i=0 ; i < messages.length ; i++ ) {
+ if ( messages[i].role === 'system' ) {
+ system_messages.push(messages[i]);
+ } else {
+ new_messages.push(messages[i]);
+ }
+ }
+ return [system_messages, new_messages];
+ }
+}
\ No newline at end of file
diff --git a/src/airouter.js/common/prompt/NormalizedPromptUtil.test.js b/src/airouter.js/common/prompt/NormalizedPromptUtil.test.js
new file mode 100644
index 0000000000..e13cfb3720
--- /dev/null
+++ b/src/airouter.js/common/prompt/NormalizedPromptUtil.test.js
@@ -0,0 +1,50 @@
+import { describe, it, expect } from 'vitest';
+const { NormalizedPromptUtil } = require('./NormalizedPromptUtil.js');
+
+describe('NormalizedPromptUtil', () => {
+ describe('extract_text', () => {
+ const cases = [
+ {
+ name: 'string message',
+ input: ['Hello, world!'],
+ output: 'Hello, world!',
+ },
+ {
+ name: 'object message',
+ input: [{
+ content: [
+ {
+ type: 'text',
+ text: 'Hello, world!',
+ }
+ ]
+ }],
+ output: 'Hello, world!',
+ },
+ {
+ name: 'irregular message',
+ input: [
+ 'First Part',
+ {
+ content: [
+ {
+ type: 'text',
+ text: 'Second Part',
+ }
+ ]
+ },
+ {
+ content: 'Third Part',
+ }
+ ],
+ output: 'First Part Second Part Third Part',
+ }
+ ];
+ for ( const tc of cases ) {
+ it(`should extract text from ${tc.name}`, () => {
+ const output = NormalizedPromptUtil.extract_text(tc.input);
+ expect(output).toBe(tc.output);
+ });
+ }
+ });
+});
diff --git a/src/backend/src/modules/puterai/lib/FunctionCalling.js b/src/airouter.js/common/prompt/UniversalToolsNormalizer.js
similarity index 98%
rename from src/backend/src/modules/puterai/lib/FunctionCalling.js
rename to src/airouter.js/common/prompt/UniversalToolsNormalizer.js
index 00f438aa71..24a525bbdc 100644
--- a/src/backend/src/modules/puterai/lib/FunctionCalling.js
+++ b/src/airouter.js/common/prompt/UniversalToolsNormalizer.js
@@ -1,4 +1,4 @@
-module.exports = class FunctionCalling {
+export class UniversalToolsNormalizer {
/**
* Normalizes the 'tools' object in-place.
*
diff --git a/src/airouter.js/common/stream/BaseWriter.js b/src/airouter.js/common/stream/BaseWriter.js
new file mode 100644
index 0000000000..640225941a
--- /dev/null
+++ b/src/airouter.js/common/stream/BaseWriter.js
@@ -0,0 +1,9 @@
+export class BaseWriter {
+ constructor (chatStream, params) {
+ this.chatStream = chatStream;
+ if ( this._start ) this._start(params);
+ }
+ end () {
+ if ( this._end ) this._end();
+ }
+}
diff --git a/src/airouter.js/common/stream/CompletionWriter.js b/src/airouter.js/common/stream/CompletionWriter.js
new file mode 100644
index 0000000000..d4e1b46169
--- /dev/null
+++ b/src/airouter.js/common/stream/CompletionWriter.js
@@ -0,0 +1,15 @@
+import { MessageWriter } from "./MessageWriter.js";
+
+export class CompletionWriter {
+ constructor ({ stream }) {
+ this.stream = stream;
+ }
+
+ end () {
+ this.stream.end();
+ }
+
+ message () {
+ return new MessageWriter(this);
+ }
+}
diff --git a/src/airouter.js/common/stream/MessageWriter.js b/src/airouter.js/common/stream/MessageWriter.js
new file mode 100644
index 0000000000..091b9f61cc
--- /dev/null
+++ b/src/airouter.js/common/stream/MessageWriter.js
@@ -0,0 +1,15 @@
+import { BaseWriter } from "./BaseWriter.js";
+import { TextWriter } from "./TextWriter.js";
+import { ToolUseWriter } from "./ToolUseWriter.js";
+
+export class MessageWriter extends BaseWriter {
+ contentBlock ({ type, ...params }) {
+ if ( type === 'tool_use' ) {
+ return new ToolUseWriter(this.chatStream, params);
+ }
+ if ( type === 'text' ) {
+ return new TextWriter(this.chatStream, params);
+ }
+ throw new Error(`Unknown content block type: ${type}`);
+ }
+}
diff --git a/src/airouter.js/common/stream/TextWriter.js b/src/airouter.js/common/stream/TextWriter.js
new file mode 100644
index 0000000000..6da2bdc548
--- /dev/null
+++ b/src/airouter.js/common/stream/TextWriter.js
@@ -0,0 +1,10 @@
+import { BaseWriter } from "./BaseWriter.js";
+
+export class TextWriter extends BaseWriter {
+ addText (text) {
+ const json = JSON.stringify({
+ type: 'text', text,
+ });
+ this.chatStream.stream.write(json + '\n');
+ }
+}
diff --git a/src/airouter.js/common/stream/ToolUseWriter.js b/src/airouter.js/common/stream/ToolUseWriter.js
new file mode 100644
index 0000000000..c5bbc8e6c9
--- /dev/null
+++ b/src/airouter.js/common/stream/ToolUseWriter.js
@@ -0,0 +1,45 @@
+import { BaseWriter } from "./BaseWriter.js";
+
+/**
+ * Assign the properties of the override object to the original object,
+ * like Object.assign, except properties are ordered so override properties
+ * are enumerated first.
+ *
+ * @param {*} original
+ * @param {*} override
+ */
+const objectAssignTop = (original, override) => {
+ let o = {
+ ...original,
+ ...override,
+ };
+ o = {
+ ...override,
+ ...original,
+ };
+ return o;
+}
+
+export class ToolUseWriter extends BaseWriter {
+ _start (params) {
+ this.contentBlock = params;
+ this.buffer = '';
+ }
+ addPartialJSON (partial_json) {
+ this.buffer += partial_json;
+ }
+ _end () {
+ if ( this.buffer.trim() === '' ) {
+ this.buffer = '{}';
+ }
+ if ( process.env.DEBUG ) console.log('BUFFER BEING PARSED', this.buffer);
+ const str = JSON.stringify(objectAssignTop({
+ ...this.contentBlock,
+ input: JSON.parse(this.buffer),
+ ...( ! this.contentBlock.text ? { text: "" } : {}),
+ }, {
+ type: 'tool_use',
+ }));
+ this.chatStream.stream.write(str + '\n');
+ }
+}
diff --git a/src/airouter.js/common/types.js b/src/airouter.js/common/types.js
new file mode 100644
index 0000000000..f6c46b46c2
--- /dev/null
+++ b/src/airouter.js/common/types.js
@@ -0,0 +1,32 @@
+export const UNIVERSAL_LLM_MESSAGES = Symbol('UNIVERSAL_LLM_PARAMS');
+export const UNIVERSAL_LLM_PARAMS = Symbol('UNIVERSAL_LLM_PARAMS');
+
+export const NORMALIZED_LLM_MESSAGES = Symbol('NORMALIZED_LLM_MESSAGES');
+export const NORMALIZED_LLM_TOOLS = Symbol('NORMALIZED_LLM_TOOLS');
+export const NORMALIZED_LLM_PARAMS = Symbol('NORMALIZED_LLM_PARAMS');
+export const NORMALIZED_LLM_STREAM = Symbol('NORMALIZED_LLM_STREAM');
+
+export const UNIVERSAL_SINGLE_MESSAGE = Symbol('UNIVERSAL_SINGLE_MESSAGE');
+export const NORMALIZED_SINGLE_MESSAGE = Symbol('NORMALIZED_SINGLE_MESSAGE');
+
+export const USAGE_WRITER = Symbol('USAGE_WRITER');
+export const ASYNC_RESPONSE = Symbol('ASYNC_RESPONSE');
+export const STREAM = Symbol('STREAM');
+export const SYNC_RESPONSE = Symbol('SYNC_RESPONSE');
+export const PROVIDER_NAME = Symbol('PROVIDER_NAME');
+
+export const COERCED_PARAMS = Symbol('COERCED_TOOLS');
+export const COERCED_TOOLS = Symbol('COERCED_TOOLS');
+export const COERCED_MESSAGES = Symbol('COERCED_MESSAGES');
+export const COERCED_USAGE = Symbol('COERCED_USAGE');
+export const MODEL_DETAILS = Symbol('COERCED_USAGE');
+
+// SDK Styles
+export const SDK_STYLE = Symbol('SDK_STYLE');
+export const USAGE_SDK_STYLE = Symbol('USAGE_SDK_STYLE');
+
+// Puter Migration Intermediaries
+export const COMPLETION_WRITER = Symbol('COMPLETION_WRITER');
+
+// Awkward
+export const STREAM_WRITTEN_TO_COMPLETION_WRITER = Symbol('STREAM_WRITTEN_TO_COMPLETION_WRITER');
diff --git a/src/airouter.js/common/usage/TransformUsageWriter.js b/src/airouter.js/common/usage/TransformUsageWriter.js
new file mode 100644
index 0000000000..4f174986d8
--- /dev/null
+++ b/src/airouter.js/common/usage/TransformUsageWriter.js
@@ -0,0 +1,11 @@
+export class TransformUsageWriter {
+ constructor (fn, delegate) {
+ this.fn = fn;
+ this.delegate = delegate;
+ }
+
+ async resolve (v) {
+ const v_or_p = this.fn.call(null, v);
+ return this.delegate.resolve(await v_or_p);
+ }
+}
diff --git a/src/airouter.js/common/util/lang.js b/src/airouter.js/common/util/lang.js
new file mode 100644
index 0000000000..9e623f85c4
--- /dev/null
+++ b/src/airouter.js/common/util/lang.js
@@ -0,0 +1,24 @@
+// Utilities that cover language builtin shortcomings, and make
+// writing javascript code a little more convenient.
+
+/**
+ * whatis exists because checking types such as 'object' and 'array'
+ * can be done incorrectly very easily. This give sthe correct
+ * implementation a single source of truth.
+ * @param {*} thing
+ * @returns {string}
+ */
+export const whatis = thing => {
+ if ( Array.isArray(thing) ) return 'array';
+ if ( thing === null ) return 'null';
+ return typeof thing;
+};
+
+/**
+ * nou makes a null or undefined check the path of least resistance,
+ * encouraging developers to treat both as the same which encourages
+ * more predictable branching behavior.
+ * @param {*} v
+ * @returns {boolean}
+ */
+export const nou = v => v === null || v === undefined;
diff --git a/src/airouter.js/common/util/streamutil.js b/src/airouter.js/common/util/streamutil.js
new file mode 100644
index 0000000000..025a9962da
--- /dev/null
+++ b/src/airouter.js/common/util/streamutil.js
@@ -0,0 +1,7 @@
+export const stream_to_buffer = async (stream) => {
+ const chunks = [];
+ for await (const chunk of stream) {
+ chunks.push(chunk);
+ }
+ return Buffer.concat(chunks);
+};
diff --git a/src/airouter.js/convention/openai/OpenAIStyleMessagesAdapter.js b/src/airouter.js/convention/openai/OpenAIStyleMessagesAdapter.js
new file mode 100644
index 0000000000..5cea81d0a0
--- /dev/null
+++ b/src/airouter.js/convention/openai/OpenAIStyleMessagesAdapter.js
@@ -0,0 +1,52 @@
+export class OpenAIStyleMessagesAdapter {
+ async adapt_messages (messages) {
+ for ( const msg of messages ) {
+ if ( ! msg.content ) continue;
+ if ( typeof msg.content !== 'object' ) continue;
+
+ const content = msg.content;
+
+ for ( const o of content ) {
+ if ( ! o.hasOwnProperty('image_url') ) continue;
+ if ( o.type ) continue;
+ o.type = 'image_url';
+ }
+
+ // coerce tool calls
+ let is_tool_call = false;
+ for ( let i = content.length - 1 ; i >= 0 ; i-- ) {
+ const content_block = content[i];
+
+ if ( content_block.type === 'tool_use' ) {
+ if ( ! msg.hasOwnProperty('tool_calls') ) {
+ msg.tool_calls = [];
+ is_tool_call = true;
+ }
+ msg.tool_calls.push({
+ id: content_block.id,
+ type: 'function',
+ function: {
+ name: content_block.name,
+ arguments: JSON.stringify(content_block.input),
+ }
+ });
+ content.splice(i, 1);
+ }
+ }
+
+ if ( is_tool_call ) msg.content = null;
+
+ // coerce tool results
+ // (we assume multiple tool results were already split into separate messages)
+ for ( let i = content.length - 1 ; i >= 0 ; i-- ) {
+ const content_block = content[i];
+ if ( content_block.type !== 'tool_result' ) continue;
+ msg.role = 'tool';
+ msg.tool_call_id = content_block.tool_use_id;
+ msg.content = content_block.content;
+ }
+ }
+
+ return messages;
+ }
+}
diff --git a/src/airouter.js/convention/openai/OpenAIStyleStreamAdapter.js b/src/airouter.js/convention/openai/OpenAIStyleStreamAdapter.js
new file mode 100644
index 0000000000..ea93494b9b
--- /dev/null
+++ b/src/airouter.js/convention/openai/OpenAIStyleStreamAdapter.js
@@ -0,0 +1,81 @@
+import { nou } from "../../common/util/lang.js";
+
+export class OpenAIStyleStreamAdapter {
+ static async write_to_stream ({ input, completionWriter, usageWriter }) {
+ const message = completionWriter.message();
+ let textblock = message.contentBlock({ type: 'text' });
+ let toolblock = null;
+ let mode = 'text';
+ const tool_call_blocks = [];
+
+ let last_usage = null;
+ for await ( let chunk of input ) {
+ chunk = this.chunk_but_like_actually(chunk);
+ if ( process.env.DEBUG ) {
+ const delta = chunk?.choices?.[0]?.delta;
+ console.log(
+ `AI CHUNK`,
+ chunk,
+ delta && JSON.stringify(delta)
+ );
+ }
+ const chunk_usage = this.index_usage_from_stream_chunk(chunk);
+ if ( chunk_usage ) last_usage = chunk_usage;
+ if ( chunk.choices.length < 1 ) continue;
+
+ const choice = chunk.choices[0];
+
+ if ( ! nou(choice.delta.content) ) {
+ if ( mode === 'tool' ) {
+ toolblock.end();
+ mode = 'text';
+ textblock = message.contentBlock({ type: 'text' });
+ }
+ textblock.addText(choice.delta.content);
+ continue;
+ }
+
+ const tool_calls = this.index_tool_calls_from_stream_choice(choice);
+ if ( ! nou(tool_calls) ) {
+ if ( mode === 'text' ) {
+ mode = 'tool';
+ textblock.end();
+ }
+ for ( const tool_call of tool_calls ) {
+ if ( ! tool_call_blocks[tool_call.index] ) {
+ toolblock = message.contentBlock({
+ type: 'tool_use',
+ id: tool_call.id,
+ name: tool_call.function.name,
+ });
+ tool_call_blocks[tool_call.index] = toolblock;
+ } else {
+ toolblock = tool_call_blocks[tool_call.index];
+ }
+ toolblock.addPartialJSON(tool_call.function.arguments);
+ }
+ }
+ }
+ usageWriter.resolve(last_usage);
+
+ if ( mode === 'text' ) textblock.end();
+ if ( mode === 'tool' ) toolblock.end();
+ message.end();
+ completionWriter.end();
+ }
+
+ /**
+ *
+ * @param {*} chunk
+ * @returns
+ */
+ static index_usage_from_stream_chunk (chunk) {
+ return chunk.usage;
+ }
+ static chunk_but_like_actually (chunk) {
+ return chunk;
+ }
+ static index_tool_calls_from_stream_choice (choice) {
+ return choice.delta.tool_calls;
+ }
+}
diff --git a/src/airouter.js/core/Registry.js b/src/airouter.js/core/Registry.js
new file mode 100644
index 0000000000..8ad27d3e69
--- /dev/null
+++ b/src/airouter.js/core/Registry.js
@@ -0,0 +1,192 @@
+class ProviderContext {
+ constructor ({ inputs, evaluatingSet, memo } = {}) {
+ this.inputs = inputs ?? {};
+ this.evaluatingSet = evaluatingSet ?? new Set();
+ this.memo = memo ?? {};
+ }
+
+ sub (inputs) {
+ const allInputs = {};
+ Object.assign(allInputs, this.inputs);
+ Object.assign(allInputs, inputs);
+
+ return new ProviderContext({
+ inputs: allInputs,
+ evaluatingSet: this.evaluatingSet,
+ memo: this.memo,
+ });
+ }
+
+ get (key) {
+ return this.inputs[key];
+ }
+
+ getAvailableInputsSet () {
+ return new Set([
+ ...Object.getOwnPropertySymbols(this.inputs),
+ ...Object.keys(this.inputs),
+ ]);;
+ }
+
+ startEvaluating (outputType) {
+ if ( this.evaluatingSet.has(outputType) ) {
+ // TODO: diagnostic information in error
+ throw new Error('cyclic evaluation');
+ }
+
+ this.evaluatingSet.add(outputType);
+ }
+
+ stopEvaluating (outputType) {
+ if ( ! this.evaluatingSet.has(outputType) ) {
+ // TODO: diagnostic information in error
+ throw new Error('internal error: evaluation hasn\'t started');
+ }
+
+ this.evaluatingSet.delete(outputType);
+ }
+}
+
+export class Registry {
+ constructor () {
+ this.singleValueProviders_ = {};
+ }
+
+ getDefineAPI () {
+ const registry = this;
+
+ const define = {
+ howToGet (outputType) {
+ const provider = { outputType, inputTypes: [] };
+
+ if ( ! registry.singleValueProviders_[outputType] ) {
+ registry.singleValueProviders_[outputType] = [];
+ }
+
+ registry.singleValueProviders_[outputType].push(provider);
+
+ const defineProviderAPI = {
+ from (...inputTypes) {
+ provider.inputTypes = inputTypes;
+ return this;
+ },
+ provided (predicateFn) {
+ provider.predicate = predicateFn;
+ return this;
+ },
+ as (fn) {
+ provider.fn = fn;
+ return this;
+ },
+ };
+
+ return defineProviderAPI;
+ }
+ };
+ return define;
+ }
+
+ getObtainAPI (parentContext) {
+ const registry = this;
+
+ if ( ! parentContext ) parentContext = new ProviderContext();
+
+ return async (outputType, inputs = {}) => {
+ const context = parentContext.sub(inputs);
+
+ // We might already have this value
+ if ( context.get(outputType) ) {
+ return context.get(outputType);
+ }
+
+ const providers = this.singleValueProviders_[outputType];
+ if ( !providers || providers.length === 0 ) {
+ throw new Error(`No providers found for output type: ${outputType.toString()}`);
+ }
+
+ const availableInputs = context.getAvailableInputsSet();
+
+ const applicableProviders = [];
+ for ( const provider of providers ) {
+ if ( ! provider.fn || ! provider.inputTypes ) {
+ console.warn(`Incomplete provider for ${outputType.toString()}`);
+ continue;
+ }
+
+ let canSatisfyRequiredInputs = true;
+ for ( const neededInputType of provider.inputTypes ) {
+ if ( ! availableInputs.has(neededInputType) ) {
+ canSatisfyRequiredInputs = false;
+ break;
+ }
+ }
+
+ if ( canSatisfyRequiredInputs ) {
+ applicableProviders.push(provider);
+ }
+ }
+
+ // If no providers are applicable, try to obtain missing inputs
+ // Recursion can be disabled by commenting out this block
+ if (applicableProviders.length === 0) {
+ for (const provider of providers) {
+ if ( ! provider.fn || ! provider.inputTypes ) {
+ console.warn(`Incomplete provider for ${outputType.toString()}`);
+ continue;
+ }
+
+ const newInputs = {};
+ let canSatisfyWithRecursion = true;
+
+ // Try to obtain each missing input
+ for (const neededInputType of provider.inputTypes) {
+ if (!availableInputs.has(neededInputType)) {
+ try {
+ // Recursively obtain the missing input
+ const inputValue = await this.getObtainAPI(context)(neededInputType, {});
+ newInputs[neededInputType] = inputValue;
+ context.inputs[neededInputType] = inputValue; // Update context
+ availableInputs.add(neededInputType); // Update available inputs
+ } catch (e) {
+ canSatisfyWithRecursion = false;
+ break; // Cannot satisfy this provider
+ }
+ }
+ }
+
+ if (canSatisfyWithRecursion) {
+ applicableProviders.push(provider);
+ }
+ }
+ }
+
+ if ( applicableProviders.length === 0 ) {
+ // TODO: diagnostic information in error message
+ console.log('???', outputType, inputs, registry.singleValueProviders_);
+ throw new Error(`no applicable providers: ` + outputType.description);
+ }
+
+ // Randomly order providers to prevent reliance on order
+ const shuffledProviders = [...applicableProviders].sort(() => Math.random() - 0.5);
+
+ const providerAPI = {
+ get: valueType => context.get(valueType),
+ obtain: registry.getObtainAPI(context),
+ memo: context.memo,
+ };
+
+ for ( const provider of shuffledProviders ) {
+ if ( provider.predicate ) {
+ const predicateResult = await provider.predicate(providerAPI);
+ if ( ! predicateResult ) continue;
+ }
+
+ return await provider.fn(providerAPI);
+ }
+
+ // TODO: diagnostic information in error message
+ console.error('no applicable providers', outputType);
+ throw new Error(`no applicable providers (2)`);
+ }
+ }
+}
diff --git a/src/airouter.js/core/Registry.test.js b/src/airouter.js/core/Registry.test.js
new file mode 100644
index 0000000000..632e69dcbd
--- /dev/null
+++ b/src/airouter.js/core/Registry.test.js
@@ -0,0 +1,217 @@
+import { describe, it, expect } from 'vitest';
+import { Registry } from './Registry.js';
+
+describe('Registry', () => {
+ it('should define and execute a simple provider', async () => {
+ const registry = new Registry();
+ const define = registry.getDefineAPI();
+
+ const PAPER_PULP = Symbol('PAPER_PULP');
+ const PAPER = Symbol('PAPER');
+
+ define.howToGet(PAPER).from(PAPER_PULP).as(async x => {
+ const paperPulp = x.get(PAPER_PULP);
+ return `paper made from ${paperPulp}`;
+ });
+
+ const obtain = registry.getObtainAPI();
+
+ const result = await obtain(PAPER, {
+ [PAPER_PULP]: 'high-quality wood pulp',
+ });
+
+ expect(result).toBe('paper made from high-quality wood pulp');
+ });
+
+ it('should handle nested obtain calls', async () => {
+ const registry = new Registry();
+ const define = registry.getDefineAPI();
+
+ const WOOD = Symbol('WOOD');
+ const PAPER_PULP = Symbol('PAPER_PULP');
+ const PAPER = Symbol('PAPER');
+
+ // Define how to get paper pulp from wood
+ define.howToGet(PAPER_PULP).from(WOOD).as(async x => {
+ const wood = x.get(WOOD);
+ return `pulp processed from ${wood}`;
+ });
+
+ // Define how to get paper from paper pulp
+ define.howToGet(PAPER).as(async x => {
+ const paperPulp = await x.obtain(PAPER_PULP, {
+ [WOOD]: x.get(WOOD) || 'default wood'
+ });
+ return `paper made from ${paperPulp}`;
+ });
+
+ const obtain = registry.getObtainAPI();
+
+ const result = await obtain(PAPER, {
+ [WOOD]: 'oak trees',
+ });
+
+ expect(result).toBe('paper made from pulp processed from oak trees');
+ });
+
+ it('should throw error for undefined provider', async () => {
+ const registry = new Registry();
+ const obtain = registry.getObtainAPI();
+
+ const UNKNOWN_TYPE = Symbol('UNKNOWN_TYPE');
+
+ await expect(obtain(UNKNOWN_TYPE)).rejects.toThrow('No providers found for output type');
+ });
+
+ it('should throw error for incomplete provider definitions', async () => {
+ const registry = new Registry();
+ const define = registry.getDefineAPI();
+ const obtain = registry.getObtainAPI();
+
+ const INCOMPLETE_TYPE = Symbol('INCOMPLETE_TYPE');
+
+ // Provider with no .from() or .as()
+ define.howToGet(INCOMPLETE_TYPE);
+
+ await expect(obtain(INCOMPLETE_TYPE)).rejects.toThrow('no applicable providers');
+ });
+
+ it('should support multiple providers for the same output type', async () => {
+ const registry = new Registry();
+ const define = registry.getDefineAPI();
+
+ const INPUT_A = Symbol('INPUT_A');
+ const INPUT_B = Symbol('INPUT_B');
+ const OUTPUT = Symbol('OUTPUT');
+
+ // Provider 1: uses INPUT_A
+ define.howToGet(OUTPUT).from(INPUT_A).as(async x => {
+ return `result from A: ${x.get(INPUT_A)}`;
+ });
+
+ // Provider 2: uses INPUT_B
+ define.howToGet(OUTPUT).from(INPUT_B).as(async x => {
+ return `result from B: ${x.get(INPUT_B)}`;
+ });
+
+ const obtain = registry.getObtainAPI();
+
+ // Should work with INPUT_A
+ const resultA = await obtain(OUTPUT, { [INPUT_A]: 'value A' });
+ expect(resultA).toBe('result from A: value A');
+
+ // Should work with INPUT_B
+ const resultB = await obtain(OUTPUT, { [INPUT_B]: 'value B' });
+ expect(resultB).toBe('result from B: value B');
+
+ // Should work with both (will pick one randomly)
+ const resultBoth = await obtain(OUTPUT, { [INPUT_A]: 'value A', [INPUT_B]: 'value B' });
+ expect(resultBoth).toMatch(/result from [AB]/);
+ });
+
+ it('should support predicates with .provided()', async () => {
+ const registry = new Registry();
+ const define = registry.getDefineAPI();
+
+ const QUALITY = Symbol('QUALITY');
+ const PAPER = Symbol('PAPER');
+
+ // Provider 1: for high quality
+ define.howToGet(PAPER).from(QUALITY)
+ .provided(x => x.get(QUALITY) === 'high')
+ .as(async x => 'premium paper');
+
+ // Provider 2: for low quality
+ define.howToGet(PAPER).from(QUALITY)
+ .provided(x => x.get(QUALITY) === 'low')
+ .as(async x => 'standard paper');
+
+ const obtain = registry.getObtainAPI();
+
+ const highQuality = await obtain(PAPER, { [QUALITY]: 'high' });
+ expect(highQuality).toBe('premium paper');
+
+ const lowQuality = await obtain(PAPER, { [QUALITY]: 'low' });
+ expect(lowQuality).toBe('standard paper');
+
+ // Should fail when no predicate matches
+ await expect(obtain(PAPER, { [QUALITY]: 'medium' }))
+ .rejects.toThrow('no applicable');
+ });
+
+ it('should handle context merging in nested calls', async () => {
+ const registry = new Registry();
+ const define = registry.getDefineAPI();
+
+ const WATER = Symbol('WATER');
+ const CHEMICALS = Symbol('CHEMICALS');
+ const TREATED_PULP = Symbol('TREATED_PULP');
+
+ define.howToGet(TREATED_PULP).from(WATER, CHEMICALS).as(async x => {
+ const water = x.get(WATER);
+ const chemicals = x.get(CHEMICALS);
+ return `treated pulp using ${water} and ${chemicals}`;
+ });
+
+ const obtain = registry.getObtainAPI();
+
+ const result = await obtain(TREATED_PULP, {
+ [WATER]: 'filtered water',
+ [CHEMICALS]: 'bleaching agents',
+ });
+
+ expect(result).toBe('treated pulp using filtered water and bleaching agents');
+ });
+
+ it('should pass context through calls to .obtain()', async () => {
+ const registry = new Registry();
+ const define = registry.getDefineAPI();
+
+ const WATER = Symbol('WATER');
+ const CHEMICALS = Symbol('CHEMICALS');
+ const TREATED_PULP = Symbol('TREATED_PULP');
+
+ define.howToGet(WATER).from(CHEMICALS).as(async x => {
+ return `suspicious water`;
+ })
+
+ define.howToGet(TREATED_PULP).from(CHEMICALS).as(async x => {
+ const water = await x.obtain(WATER);
+ const chemicals = x.get(CHEMICALS);
+ return `treated pulp using ${water} and ${chemicals}`;
+ });
+
+ const obtain = registry.getObtainAPI();
+
+ const result = await obtain(TREATED_PULP, {
+ [CHEMICALS]: 'bleaching agents',
+ });
+
+ expect(result).toBe('treated pulp using suspicious water and bleaching agents');
+ });
+
+ it('should allow obtaining or getting non-specified values', async () => {
+ const registry = new Registry();
+ const define = registry.getDefineAPI();
+
+ const BURGER = Symbol('BURGER');
+ const BURGER_STUFF = Symbol('BURGER_STUFF');
+ const BURGER_BUNS = Symbol('BURGER_BUNS');
+
+ define.howToGet(BURGER).as(async x => {
+ const stuff = x.get(BURGER_STUFF);
+ const buns = await x.obtain(BURGER_BUNS);
+
+ return `burger with ${stuff} between two ${buns}`
+ })
+
+ const obtain = registry.getObtainAPI();
+
+ const result = await obtain(BURGER, {
+ [BURGER_BUNS]: 'multigrain buns',
+ [BURGER_STUFF]: 'the works',
+ });
+
+ expect(result).toBe('burger with the works between two multigrain buns');
+ });
+});
\ No newline at end of file
diff --git a/src/airouter.js/docgen/ANTHROPIC_DOCS.md b/src/airouter.js/docgen/ANTHROPIC_DOCS.md
new file mode 100644
index 0000000000..74acd91cc6
--- /dev/null
+++ b/src/airouter.js/docgen/ANTHROPIC_DOCS.md
@@ -0,0 +1,109 @@
+# Provider Registry Documentation
+
+This document describes all the providers available in the registry and how to obtain different types of values.
+
+## Available Types
+
+- [COERCED_TOOLS](#coerced-tools)
+- [ANTHROPIC_CLIENT](#anthropic-client)
+- [ASYNC_RESPONSE](#async-response)
+- [SYNC_RESPONSE](#sync-response)
+
+## COERCED_TOOLS
+
+There are 2 ways to obtain **COERCED_TOOLS**:
+
+### Option 1
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+### Option 2
+
+**Requires:** `NORMALIZED_LLM_TOOLS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+## ANTHROPIC_CLIENT
+
+**Requires:** `ANTHROPIC_API_KEY`
+
+**Produces:** Custom provider function
+
+
+## ASYNC_RESPONSE
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+## SYNC_RESPONSE
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+
+---
+
+# Dependency Tree
+
+- COERCED_TOOLS
+ - NORMALIZED_LLM_PARAMS
+ - NORMALIZED_LLM_TOOLS
+- ANTHROPIC_CLIENT
+ - ANTHROPIC_API_KEY
+- ASYNC_RESPONSE
+ - NORMALIZED_LLM_PARAMS
+- SYNC_RESPONSE
+ - NORMALIZED_LLM_PARAMS
+
+---
+
+# Usage Examples
+
+## Basic Usage
+
+```javascript
+
+const registry = new Registry();
+
+const obtain = registry.getObtainAPI();
+
+
+// Obtain a value with required inputs
+
+const result = await obtain(OUTPUT_TYPE, {
+
+ [INPUT_TYPE]: "input value"
+
+});
+
+```
+
+
+## Available Providers
+
+The following types can be obtained:
+
+
+- **COERCED_TOOLS**
+
+- **ANTHROPIC_CLIENT**
+
+- **ASYNC_RESPONSE**
+
+- **SYNC_RESPONSE**
diff --git a/src/airouter.js/docgen/DocumentationGenerator.js b/src/airouter.js/docgen/DocumentationGenerator.js
new file mode 100644
index 0000000000..612c480e3a
--- /dev/null
+++ b/src/airouter.js/docgen/DocumentationGenerator.js
@@ -0,0 +1,291 @@
+export class DocumentationGenerator {
+ constructor() {
+ this.providers = [];
+ }
+
+ /**
+ * Apply definition functions to generate documentation
+ * @param {...Function} definitionFns - Functions that take a define API and add definitions
+ */
+ applyDefinitions(...definitionFns) {
+ const define = this.getDefineAPI();
+
+ for (const definitionFn of definitionFns) {
+ definitionFn(define);
+ }
+
+ return this;
+ }
+
+ /**
+ * Get a human-readable name for a symbol
+ */
+ getSymbolName(symbol) {
+ // Use symbol description if available
+ const desc = symbol.description;
+ if (desc) {
+ return desc;
+ }
+
+ // Last resort: use toString
+ return symbol.toString();
+ }
+
+ /**
+ * Get the documentation API that mirrors Registry's getDefineAPI
+ */
+ getDefineAPI() {
+ const docGen = this;
+
+ const define = {
+ howToGet(outputType) {
+ const providerDoc = {
+ outputType,
+ inputTypes: [],
+ predicate: null,
+ description: null,
+ example: null
+ };
+
+ docGen.providers.push(providerDoc);
+
+ const defineProviderAPI = {
+ from(...inputTypes) {
+ providerDoc.inputTypes = inputTypes;
+ return this;
+ },
+ provided(predicateDescription) {
+ // For documentation, we expect a string description instead of a function
+ if (typeof predicateDescription === 'string') {
+ providerDoc.predicate = predicateDescription;
+ } else {
+ providerDoc.predicate = 'Custom predicate function';
+ }
+ return this;
+ },
+ as(description) {
+ // For documentation, we expect a string description instead of a function
+ if (typeof description === 'string') {
+ providerDoc.description = description;
+ } else {
+ providerDoc.description = 'Custom provider function';
+ }
+ return this;
+ },
+ withExample(example) {
+ // Additional method for documentation
+ providerDoc.example = example;
+ return this;
+ }
+ };
+
+ return defineProviderAPI;
+ }
+ };
+
+ return define;
+ }
+
+ /**
+ * Generate markdown documentation
+ */
+ generateMarkdown() {
+ const sections = [];
+
+ // Group providers by output type
+ const providersByOutput = new Map();
+ for (const provider of this.providers) {
+ const outputName = this.getSymbolName(provider.outputType);
+ if (!providersByOutput.has(outputName)) {
+ providersByOutput.set(outputName, []);
+ }
+ providersByOutput.get(outputName).push(provider);
+ }
+
+ // Generate header
+ sections.push('# Provider Registry Documentation\n');
+ sections.push('This document describes all the providers available in the registry and how to obtain different types of values.\n');
+
+ // Generate table of contents
+ sections.push('## Available Types\n');
+ for (const outputName of providersByOutput.keys()) {
+ sections.push(`- [${outputName}](#${outputName.toLowerCase().replace(/[^a-z0-9]/g, '-')})`);
+ }
+ sections.push('');
+
+ // Generate sections for each output type
+ for (const [outputName, providers] of providersByOutput.entries()) {
+ sections.push(`## ${outputName}\n`);
+
+ if (providers.length === 1) {
+ const provider = providers[0];
+ sections.push(this.generateProviderDoc(provider));
+ } else {
+ sections.push(`There are ${providers.length} ways to obtain **${outputName}**:\n`);
+ providers.forEach((provider, index) => {
+ sections.push(`### Option ${index + 1}\n`);
+ sections.push(this.generateProviderDoc(provider));
+ });
+ }
+ }
+
+ return sections.join('\n');
+ }
+
+ /**
+ * Generate documentation for a single provider
+ */
+ generateProviderDoc(provider) {
+ const parts = [];
+
+ // Requirements
+ if (provider.inputTypes.length > 0) {
+ const inputNames = provider.inputTypes.map(type =>
+ `\`${this.getSymbolName(type)}\``
+ ).join(', ');
+ parts.push(`**Requires:** ${inputNames}\n`);
+ } else {
+ parts.push(`**Requires:** No inputs\n`);
+ }
+
+ // Predicate condition
+ if (provider.predicate) {
+ parts.push(`**When:** ${provider.predicate}\n`);
+ }
+
+ // Description
+ if (provider.description) {
+ parts.push(`**Produces:** ${provider.description}\n`);
+ }
+
+ // Example
+ if (provider.example) {
+ parts.push(`**Example:**\n\`\`\`javascript\n${provider.example}\n\`\`\`\n`);
+ }
+
+ return parts.join('\n') + '\n';
+ }
+
+ /**
+ * Generate a simple tree view showing dependencies
+ */
+ generateDependencyTree() {
+ const sections = [];
+ sections.push('# Dependency Tree\n');
+
+ const dependencies = new Map();
+
+ // Build dependency map
+ for (const provider of this.providers) {
+ const outputName = this.getSymbolName(provider.outputType);
+ const inputs = provider.inputTypes.map(type => this.getSymbolName(type));
+
+ if (!dependencies.has(outputName)) {
+ dependencies.set(outputName, new Set());
+ }
+
+ for (const input of inputs) {
+ dependencies.get(outputName).add(input);
+ }
+ }
+
+ // Find root nodes (types that don't depend on anything produced by other providers)
+ const allOutputs = new Set(dependencies.keys());
+ const allInputs = new Set();
+ for (const inputSet of dependencies.values()) {
+ for (const input of inputSet) {
+ allInputs.add(input);
+ }
+ }
+
+ const roots = [];
+ for (const output of allOutputs) {
+ if (!allInputs.has(output) || dependencies.get(output).size === 0) {
+ roots.push(output);
+ }
+ }
+
+ // Generate tree for each root
+ const visited = new Set();
+ for (const root of roots) {
+ sections.push(this.generateTreeNode(root, dependencies, visited, 0));
+ }
+
+ return sections.join('\n');
+ }
+
+ generateTreeNode(nodeName, dependencies, visited, depth) {
+ const indent = ' '.repeat(depth);
+ const parts = [`${indent}- ${nodeName}`];
+
+ if (visited.has(nodeName)) {
+ parts[0] += ' (circular reference)';
+ return parts.join('\n');
+ }
+
+ visited.add(nodeName);
+
+ const deps = dependencies.get(nodeName);
+ if (deps && deps.size > 0) {
+ for (const dep of deps) {
+ parts.push(this.generateTreeNode(dep, dependencies, new Set(visited), depth + 1));
+ }
+ }
+
+ visited.delete(nodeName);
+ return parts.join('\n');
+ }
+
+ /**
+ * Generate comprehensive documentation including examples
+ */
+ generateFullDocumentation() {
+ const sections = [];
+
+ sections.push(this.generateMarkdown());
+ sections.push('\n---\n');
+ sections.push(this.generateDependencyTree());
+
+ // Add usage examples
+ sections.push('\n---\n');
+ sections.push('# Usage Examples\n');
+ sections.push(this.generateUsageExamples());
+
+ return sections.join('\n');
+ }
+
+ generateUsageExamples() {
+ const sections = [];
+
+ sections.push('## Basic Usage\n');
+ sections.push('```javascript\n');
+ sections.push('const registry = new Registry();\n');
+ sections.push('const obtain = registry.getObtainAPI();\n\n');
+ sections.push('// Obtain a value with required inputs\n');
+ sections.push('const result = await obtain(OUTPUT_TYPE, {\n');
+ sections.push(' [INPUT_TYPE]: "input value"\n');
+ sections.push('});\n');
+ sections.push('```\n\n');
+
+ sections.push('## Available Providers\n');
+ sections.push('The following types can be obtained:\n\n');
+
+ // List all output types
+ const outputTypes = [...new Set(this.providers.map(p => this.getSymbolName(p.outputType)))];
+ for (const outputType of outputTypes) {
+ sections.push(`- **${outputType}**\n`);
+ }
+
+ return sections.join('\n');
+ }
+}
+
+// Example usage with your Registry
+export function generateDocsForRegistry(...definitionFns) {
+ const docGen = new DocumentationGenerator();
+
+ // Apply all the definition functions
+ docGen.applyDefinitions(...definitionFns);
+
+ return docGen.generateFullDocumentation();
+}
\ No newline at end of file
diff --git a/src/airouter.js/docgen/OPENAI_DOCS.md b/src/airouter.js/docgen/OPENAI_DOCS.md
new file mode 100644
index 0000000000..411f94eab7
--- /dev/null
+++ b/src/airouter.js/docgen/OPENAI_DOCS.md
@@ -0,0 +1,123 @@
+# Provider Registry Documentation
+
+This document describes all the providers available in the registry and how to obtain different types of values.
+
+## Available Types
+
+- [COERCED_TOOLS](#coerced-tools)
+- [COERCED_USAGE](#coerced-usage)
+- [ASYNC_RESPONSE](#async-response)
+- [SYNC_RESPONSE](#sync-response)
+- [COERCED_MESSAGES](#coerced-messages)
+
+## COERCED_TOOLS
+
+There are 2 ways to obtain **COERCED_TOOLS**:
+
+### Option 1
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+### Option 2
+
+**Requires:** `NORMALIZED_LLM_TOOLS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+## COERCED_USAGE
+
+**Requires:** `OPENAI_USAGE`, `COERCED_USAGE`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+## ASYNC_RESPONSE
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+## SYNC_RESPONSE
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+## COERCED_MESSAGES
+
+**Requires:** `NORMALIZED_LLM_MESSAGES`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+
+---
+
+# Dependency Tree
+
+- COERCED_TOOLS
+ - NORMALIZED_LLM_PARAMS
+ - NORMALIZED_LLM_TOOLS
+- ASYNC_RESPONSE
+ - NORMALIZED_LLM_PARAMS
+- SYNC_RESPONSE
+ - NORMALIZED_LLM_PARAMS
+- COERCED_MESSAGES
+ - NORMALIZED_LLM_MESSAGES
+
+---
+
+# Usage Examples
+
+## Basic Usage
+
+```javascript
+
+const registry = new Registry();
+
+const obtain = registry.getObtainAPI();
+
+
+// Obtain a value with required inputs
+
+const result = await obtain(OUTPUT_TYPE, {
+
+ [INPUT_TYPE]: "input value"
+
+});
+
+```
+
+
+## Available Providers
+
+The following types can be obtained:
+
+
+- **COERCED_TOOLS**
+
+- **COERCED_USAGE**
+
+- **ASYNC_RESPONSE**
+
+- **SYNC_RESPONSE**
+
+- **COERCED_MESSAGES**
diff --git a/src/airouter.js/docgen/REGISTRY_DOCS.md b/src/airouter.js/docgen/REGISTRY_DOCS.md
new file mode 100644
index 0000000000..8d4d3fe9b0
--- /dev/null
+++ b/src/airouter.js/docgen/REGISTRY_DOCS.md
@@ -0,0 +1,213 @@
+# Provider Registry Documentation
+
+This document describes all the providers available in the registry and how to obtain different types of values.
+
+## Available Types
+
+- [NORMALIZED_LLM_TOOLS](#normalized-llm-tools)
+- [NORMALIZED_LLM_MESSAGES](#normalized-llm-messages)
+- [USAGE_SDK_STYLE](#usage-sdk-style)
+- [COERCED_TOOLS](#coerced-tools)
+- [ANTHROPIC_CLIENT](#anthropic-client)
+- [ASYNC_RESPONSE](#async-response)
+- [SYNC_RESPONSE](#sync-response)
+- [COERCED_USAGE](#coerced-usage)
+- [COERCED_MESSAGES](#coerced-messages)
+
+## NORMALIZED_LLM_TOOLS
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**Produces:** Custom provider function
+
+
+## NORMALIZED_LLM_MESSAGES
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**Produces:** Custom provider function
+
+
+## USAGE_SDK_STYLE
+
+**Requires:** `SDK_STYLE`
+
+**Produces:** Custom provider function
+
+
+## COERCED_TOOLS
+
+There are 4 ways to obtain **COERCED_TOOLS**:
+
+### Option 1
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+### Option 2
+
+**Requires:** `NORMALIZED_LLM_TOOLS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+### Option 3
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+### Option 4
+
+**Requires:** `NORMALIZED_LLM_TOOLS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+## ANTHROPIC_CLIENT
+
+**Requires:** `ANTHROPIC_API_KEY`
+
+**Produces:** Custom provider function
+
+
+## ASYNC_RESPONSE
+
+There are 2 ways to obtain **ASYNC_RESPONSE**:
+
+### Option 1
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+### Option 2
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+## SYNC_RESPONSE
+
+There are 2 ways to obtain **SYNC_RESPONSE**:
+
+### Option 1
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+### Option 2
+
+**Requires:** `NORMALIZED_LLM_PARAMS`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+## COERCED_USAGE
+
+**Requires:** `OPENAI_USAGE`, `COERCED_USAGE`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+## COERCED_MESSAGES
+
+**Requires:** `NORMALIZED_LLM_MESSAGES`
+
+**When:** Custom predicate function
+
+**Produces:** Custom provider function
+
+
+
+---
+
+# Dependency Tree
+
+- USAGE_SDK_STYLE
+ - SDK_STYLE
+- COERCED_TOOLS
+ - NORMALIZED_LLM_PARAMS
+ - NORMALIZED_LLM_TOOLS
+ - NORMALIZED_LLM_PARAMS
+- ANTHROPIC_CLIENT
+ - ANTHROPIC_API_KEY
+- ASYNC_RESPONSE
+ - NORMALIZED_LLM_PARAMS
+- SYNC_RESPONSE
+ - NORMALIZED_LLM_PARAMS
+- COERCED_MESSAGES
+ - NORMALIZED_LLM_MESSAGES
+ - NORMALIZED_LLM_PARAMS
+
+---
+
+# Usage Examples
+
+## Basic Usage
+
+```javascript
+
+const registry = new Registry();
+
+const obtain = registry.getObtainAPI();
+
+
+// Obtain a value with required inputs
+
+const result = await obtain(OUTPUT_TYPE, {
+
+ [INPUT_TYPE]: "input value"
+
+});
+
+```
+
+
+## Available Providers
+
+The following types can be obtained:
+
+
+- **NORMALIZED_LLM_TOOLS**
+
+- **NORMALIZED_LLM_MESSAGES**
+
+- **USAGE_SDK_STYLE**
+
+- **COERCED_TOOLS**
+
+- **ANTHROPIC_CLIENT**
+
+- **ASYNC_RESPONSE**
+
+- **SYNC_RESPONSE**
+
+- **COERCED_USAGE**
+
+- **COERCED_MESSAGES**
diff --git a/src/airouter.js/docgen/generate-docs.js b/src/airouter.js/docgen/generate-docs.js
new file mode 100644
index 0000000000..07560e49e2
--- /dev/null
+++ b/src/airouter.js/docgen/generate-docs.js
@@ -0,0 +1,26 @@
+// docgen/generate-docs.js
+import { generateDocsForRegistry } from './DocumentationGenerator.js';
+
+import commonRegistrants from '../common/definitions.js';
+import anthropicRegistrants from '../anthropic/index.js';
+import openaiRegistrants from '../openai/index.js';
+
+// Generate documentation for all your registrants
+const docs = generateDocsForRegistry(
+ commonRegistrants,
+ anthropicRegistrants,
+ openaiRegistrants
+);
+
+console.log(docs);
+
+// Or write to file
+import { writeFileSync } from 'fs';
+writeFileSync('REGISTRY_DOCS.md', docs);
+
+// You could also generate docs for specific subsets
+const anthropicDocs = generateDocsForRegistry(anthropicRegistrants);
+writeFileSync('ANTHROPIC_DOCS.md', anthropicDocs);
+
+const openaiDocs = generateDocsForRegistry(openaiRegistrants);
+writeFileSync('OPENAI_DOCS.md', openaiDocs);
\ No newline at end of file
diff --git a/src/airouter.js/gemini/GeminiToolsAdapter.js b/src/airouter.js/gemini/GeminiToolsAdapter.js
new file mode 100644
index 0000000000..53f0728964
--- /dev/null
+++ b/src/airouter.js/gemini/GeminiToolsAdapter.js
@@ -0,0 +1,13 @@
+export class GeminiToolsAdapter {
+ static adapt_tools (tools) {
+ return [
+ {
+ function_declarations: tools.map(t => {
+ const tool = t.function;
+ delete tool.parameters.additionalProperties;
+ return tool;
+ })
+ }
+ ];
+ }
+}
diff --git a/src/airouter.js/models/deepseek.json b/src/airouter.js/models/deepseek.json
new file mode 100644
index 0000000000..c8d38e3041
--- /dev/null
+++ b/src/airouter.js/models/deepseek.json
@@ -0,0 +1,26 @@
+[
+ {
+ "id": "deepseek-chat",
+ "name": "DeepSeek Chat",
+ "context": 64000,
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 14,
+ "output": 28
+ },
+ "max_tokens": 8000
+ },
+ {
+ "id": "deepseek-reasoner",
+ "name": "DeepSeek Reasoner",
+ "context": 64000,
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 55,
+ "output": 219
+ },
+ "max_tokens": 64000
+ }
+]
diff --git a/src/airouter.js/models/openai.json b/src/airouter.js/models/openai.json
new file mode 100644
index 0000000000..1c6b54540b
--- /dev/null
+++ b/src/airouter.js/models/openai.json
@@ -0,0 +1,121 @@
+[
+ {
+ "id": "gpt-4o",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 250,
+ "output": 1000
+ },
+ "max_tokens": 16384
+ },
+ {
+ "id": "gpt-4o-mini",
+ "max_tokens": 16384,
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 15,
+ "output": 60
+ }
+ },
+ {
+ "id": "o1",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 1500,
+ "output": 6000
+ },
+ "max_tokens": 100000
+ },
+ {
+ "id": "o1-mini",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 300,
+ "output": 1200
+ },
+ "max_tokens": 65536
+ },
+ {
+ "id": "o1-pro",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 15000,
+ "output": 60000
+ },
+ "max_tokens": 100000
+ },
+ {
+ "id": "o3",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 1000,
+ "output": 4000
+ },
+ "max_tokens": 100000
+ },
+ {
+ "id": "o3-mini",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 110,
+ "output": 440
+ },
+ "max_tokens": 100000
+ },
+ {
+ "id": "o4-mini",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 110,
+ "output": 440
+ },
+ "max_tokens": 100000
+ },
+ {
+ "id": "gpt-4.1",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 200,
+ "output": 800
+ },
+ "max_tokens": 32768
+ },
+ {
+ "id": "gpt-4.1-mini",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 40,
+ "output": 160
+ },
+ "max_tokens": 32768
+ },
+ {
+ "id": "gpt-4.1-nano",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 10,
+ "output": 40
+ },
+ "max_tokens": 32768
+ },
+ {
+ "id": "gpt-4.5-preview",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 7500,
+ "output": 15000
+ }
+ }
+]
diff --git a/src/airouter.js/myprovider/index.js b/src/airouter.js/myprovider/index.js
new file mode 100644
index 0000000000..477f86d456
--- /dev/null
+++ b/src/airouter.js/myprovider/index.js
@@ -0,0 +1,9 @@
+import { NORMALIZED_LLM_PARAMS, PROVIDER_NAME, SYNC_RESPONSE } from "../airouter";
+
+export default define => {
+ define.howToGet(SYNC_RESPONSE).from(NORMALIZED_LLM_PARAMS)
+ .provided(x => x.get(PROVIDER_NAME) === 'myprovider')
+ .as(async x => {
+ //
+ })
+};
diff --git a/src/airouter.js/normalize.test.js b/src/airouter.js/normalize.test.js
new file mode 100644
index 0000000000..b1842c2128
--- /dev/null
+++ b/src/airouter.js/normalize.test.js
@@ -0,0 +1,136 @@
+import { describe, it, expect } from 'vitest';
+
+import commonRegistrants from './common/index.js';
+import { NORMALIZED_LLM_MESSAGES, NORMALIZED_SINGLE_MESSAGE, UNIVERSAL_LLM_MESSAGES, UNIVERSAL_SINGLE_MESSAGE } from './airouter';
+import { Registry } from './core/Registry';
+
+describe('normalize', () => {
+ const registry = new Registry();
+
+ const define = registry.getDefineAPI();
+ commonRegistrants(define);
+
+ const obtain = registry.getObtainAPI();
+
+ it('converts strings into message with content parts', async () => {
+ const universal_messages = [
+ 'fox of quick brown, jump over the lazy dogs',
+ 'the black quartz sphinx judges over the funny vow',
+ ];
+
+ const output = await obtain(NORMALIZED_LLM_MESSAGES, {
+ [UNIVERSAL_LLM_MESSAGES]: universal_messages,
+ });
+
+ expect(output.length).toBe(1);
+
+ const message = output[0];
+ expect(typeof message).toBe('object');
+ expect(message?.content?.length).toBe(universal_messages.length);
+ expect(message?.content?.length).not.toBe(0);
+
+ for ( let i=0 ; i < output.length ; i++ ) {
+ expect(message?.content?.[0]?.text).toBe(universal_messages[i])
+ }
+ });
+
+ const cases = [
+ {
+ name: 'string message',
+ input: 'Hello, world!',
+ output: {
+ role: 'user',
+ content: [
+ {
+ type: 'text',
+ text: 'Hello, world!',
+ }
+ ]
+ }
+ }
+ ];
+ for ( const tc of cases ) {
+ it(`should normalize ${tc.name}`, async () => {
+ const output = await obtain(NORMALIZED_SINGLE_MESSAGE, {
+ [UNIVERSAL_SINGLE_MESSAGE]: tc.input,
+ });
+ expect(output).toEqual(tc.output);
+ });
+ }
+ describe('normalize OpenAI tool calls', () => {
+ const cases = [
+ {
+ name: 'string message',
+ input: {
+ role: 'assistant',
+ tool_calls: [
+ {
+ id: 'tool-1',
+ type: 'function',
+ function: {
+ name: 'tool-1-function',
+ arguments: {},
+ }
+ }
+ ]
+ },
+ output: {
+ role: 'assistant',
+ content: [
+ {
+ type: 'tool_use',
+ id: 'tool-1',
+ name: 'tool-1-function',
+ input: {},
+ }
+ ]
+ }
+ }
+ ];
+ for ( const tc of cases ) {
+ it(`should normalize ${tc.name}`, async () => {
+ const output = await obtain(NORMALIZED_SINGLE_MESSAGE, {
+ [UNIVERSAL_SINGLE_MESSAGE]: tc.input,
+ });
+ expect(output).toEqual(tc.output);
+ });
+ }
+ });
+ describe('normalize Claude tool calls', () => {
+ const cases = [
+ {
+ name: 'string message',
+ input: {
+ role: 'assistant',
+ content: [
+ {
+ type: 'tool_use',
+ id: 'tool-1',
+ name: 'tool-1-function',
+ input: "{}",
+ }
+ ]
+ },
+ output: {
+ role: 'assistant',
+ content: [
+ {
+ type: 'tool_use',
+ id: 'tool-1',
+ name: 'tool-1-function',
+ input: "{}",
+ }
+ ]
+ }
+ }
+ ];
+ for ( const tc of cases ) {
+ it(`should normalize ${tc.name}`, async () => {
+ const output = await obtain(NORMALIZED_SINGLE_MESSAGE, {
+ [UNIVERSAL_SINGLE_MESSAGE]: tc.input,
+ });
+ expect(output).toEqual(tc.output);
+ });
+ }
+ });
+});
\ No newline at end of file
diff --git a/src/airouter.js/openai/OpenAIStreamAdapter.js b/src/airouter.js/openai/OpenAIStreamAdapter.js
new file mode 100644
index 0000000000..b22c7d177f
--- /dev/null
+++ b/src/airouter.js/openai/OpenAIStreamAdapter.js
@@ -0,0 +1,10 @@
+import { OpenAIStyleStreamAdapter } from '../convention/openai/OpenAIStyleStreamAdapter.js';
+
+/**
+ * OpenAIStreamAdapter extends OpenAIStyleStreamAdapter without overriding
+ * any methods instead. It's redundant in terms of functionality, as
+ * OpenAIStreamAdapter could be used directly. However, this makes the
+ * intended architecture clearer and more consistent with other integrations,
+ * where each provider has its own adapter class.
+ */
+export class OpenAIStreamAdapter extends OpenAIStyleStreamAdapter {}
diff --git a/src/airouter.js/openai/OpenAIToolsAdapter.js b/src/airouter.js/openai/OpenAIToolsAdapter.js
new file mode 100644
index 0000000000..b97787d1ec
--- /dev/null
+++ b/src/airouter.js/openai/OpenAIToolsAdapter.js
@@ -0,0 +1,5 @@
+export class OpenAIToolsAdapter {
+ static adapt_tools (tools) {
+ return tools;
+ }
+}
diff --git a/src/airouter.js/openai/handle_files.js b/src/airouter.js/openai/handle_files.js
new file mode 100644
index 0000000000..29467b4332
--- /dev/null
+++ b/src/airouter.js/openai/handle_files.js
@@ -0,0 +1,56 @@
+import { stream_to_buffer } from "../common/util/streamutil.js";
+
+const MAX_FILE_SIZE = 5 * 1_000_000;
+
+export default async ({ messages }) => {
+ const file_input_tasks = [];
+ for ( const message of messages ) {
+ // We can assume `message.content` is not undefined because
+ // UniversalPromptNormalizer ensures this.
+ for ( const contentPart of message.content ) {
+ if ( contentPart.type !== 'data' ) continue;
+ const { data } = contentPart;
+ delete contentPart.data;
+ file_input_tasks.push({
+ data,
+ contentPart,
+ });
+ }
+ }
+
+ const promises = [];
+ for ( const task of file_input_tasks ) promises.push((async () => {
+ if ( await task.data.getSize() > MAX_FILE_SIZE ) {
+ delete task.contentPart.puter_path;
+ task.contentPart.type = 'text';
+ task.contentPart.text = `{error: input file exceeded maximum of ${MAX_FILE_SIZE} bytes; ` +
+ `the user did not write this message}`; // "poor man's system prompt"
+ return; // "continue"
+ }
+
+ const stream = await task.data.getStream();
+ const mimeType = await task.data.getMimeType();
+
+ const buffer = await stream_to_buffer(stream);
+ const base64 = buffer.toString('base64');
+
+ delete task.contentPart.puter_path;
+ if ( mimeType.startsWith('image/') ) {
+ task.contentPart.type = 'image_url',
+ task.contentPart.image_url = {
+ url: `data:${mimeType};base64,${base64}`,
+ };
+ } else if ( mimeType.startsWith('audio/') ) {
+ task.contentPart.type = 'input_audio',
+ task.contentPart.input_audio = {
+ data: `data:${mimeType};base64,${base64}`,
+ format: mimeType.split('/')[1],
+ }
+ } else {
+ task.contentPart.type = 'text';
+ task.contentPart.text = `{error: input file has unsupported MIME type; ` +
+ `the user did not write this message}`; // "poor man's system prompt"
+ }
+ })());
+ await Promise.all(promises);
+}
diff --git a/src/airouter.js/openai/index.js b/src/airouter.js/openai/index.js
new file mode 100644
index 0000000000..df9a2c0723
--- /dev/null
+++ b/src/airouter.js/openai/index.js
@@ -0,0 +1,164 @@
+import openai_models from '../models/openai.json' with { type: 'json' };
+import { ASYNC_RESPONSE, COERCED_MESSAGES, COERCED_PARAMS, COERCED_TOOLS, COERCED_USAGE, COMPLETION_WRITER, MODEL_DETAILS, NORMALIZED_LLM_MESSAGES, NORMALIZED_LLM_PARAMS, NORMALIZED_LLM_TOOLS, PROVIDER_NAME, STREAM_WRITTEN_TO_COMPLETION_WRITER, SYNC_RESPONSE, USAGE_SDK_STYLE, USAGE_WRITER } from "../common/types.js";
+import handle_files from './handle_files.js';
+import { TransformUsageWriter } from '../common/usage/TransformUsageWriter.js';
+import { OpenAIStreamAdapter } from './OpenAIStreamAdapter.js';
+
+export const OPENAI_CLIENT = Symbol('OPENAI_CLIENT');
+export const OPENAI_USAGE = Symbol('OPENAI_USAGE');
+
+export default define => {
+ define.howToGet(COERCED_PARAMS).from(NORMALIZED_LLM_PARAMS)
+ .provided(x => x.get(PROVIDER_NAME) == 'openai')
+ .as(async x => {
+ const params = x.get(NORMALIZED_LLM_PARAMS);
+ params.tools = await x.obtain(COERCED_TOOLS);
+ params.messages = await x.obtain(COERCED_MESSAGES);
+
+ return {
+ user: params.user_id,
+ messages: params.messages,
+ model: params.model,
+ ...(params.tools ? { tools: params.tools } : {}),
+ ...(params.max_tokens ? { max_completion_tokens: params.max_tokens } : {}),
+ ...(params.temperature ? { temperature: params.temperature } : {}),
+ // TODO: move as assign on stream getter
+ // stream: is_stream,
+ // ...(is_stream ? {
+ // stream_options: { include_usage: true },
+ // } : {}),
+ };
+ });
+
+ define.howToGet(COERCED_TOOLS).from(NORMALIZED_LLM_TOOLS)
+ .provided(x => x.get(PROVIDER_NAME) == 'openai')
+ .as(async x => {
+ // Normalized tools follow OpenAI's format, so no coercion is required
+ return x.get(NORMALIZED_LLM_TOOLS);
+ })
+
+ define.howToGet(COERCED_USAGE).from(OPENAI_USAGE, MODEL_DETAILS)
+ .provided(async x => await x.obtain(USAGE_SDK_STYLE) === 'openai')
+ .as(async x => {
+ const openai_usage = x.get(OPENAI_USAGE);
+ const model_details = x.get(MODEL_DETAILS);
+ const standard_usage = [];
+
+ standard_usage.push({
+ type: 'prompt',
+ model: model_details.id,
+ amount: openai_usage.prompt_tokens,
+ cost: model_details.cost.input * openai_usage.prompt_tokens,
+ });
+ standard_usage.push({
+ type: 'completion',
+ model: model_details.id,
+ amount: openai_usage.completion_tokens,
+ cost: model_details.cost.output * openai_usage.completion_tokens,
+ });
+
+ return standard_usage;
+ })
+
+ define.howToGet(ASYNC_RESPONSE).from(NORMALIZED_LLM_PARAMS)
+ .provided(x => x.get(PROVIDER_NAME) == 'openai')
+ .as(async x => {
+ const params = await x.obtain(COERCED_PARAMS);
+ params.stream = true;
+ params.stream_options = { include_usage: true };
+
+ const client = await x.obtain(OPENAI_CLIENT);
+ const model_details = openai_models.find(entry => entry.id === params.model);
+
+ const stream = await client.chat.completions.create({
+ ...params,
+ });
+
+ await OpenAIStreamAdapter.write_to_stream({
+ input: stream,
+ completionWriter: x.get(COMPLETION_WRITER),
+ usageWriter: new TransformUsageWriter(async usage => {
+ return await x.obtain(COERCED_USAGE, {
+ [OPENAI_USAGE]: usage,
+ [MODEL_DETAILS]: model_details,
+ });
+ }, x.get(USAGE_WRITER)),
+ });
+ });
+
+ define.howToGet(SYNC_RESPONSE).from(NORMALIZED_LLM_PARAMS)
+ .provided(x => x.get(PROVIDER_NAME) == 'openai')
+ .as(async x => {
+ const params = await x.obtain(COERCED_PARAMS);
+
+ const client = await x.obtain(OPENAI_CLIENT);
+ const model_details = openai_models.find(entry => entry.id === params.model);
+ const completion = await client.chat.completions.create(params);
+
+ const ret = completion.choices[0];
+
+ ret.usage = await x.obtain(COERCED_USAGE, {
+ [OPENAI_USAGE]: completion.usage,
+ [MODEL_DETAILS]: model_details,
+ });
+
+ return ret;
+ });
+
+ define.howToGet(COERCED_MESSAGES).from(NORMALIZED_LLM_MESSAGES)
+ .provided(x => x.get(PROVIDER_NAME) == 'openai')
+ .as(async x => {
+ let messages = x.get(NORMALIZED_LLM_MESSAGES);
+
+ await handle_files({ messages });
+
+ for ( const msg of messages ) {
+ if ( ! msg.content ) continue;
+ if ( typeof msg.content !== 'object' ) continue;
+
+ const content = msg.content;
+
+ for ( const o of content ) {
+ if ( ! o.hasOwnProperty('image_url') ) continue;
+ if ( o.type ) continue;
+ o.type = 'image_url';
+ }
+
+ // coerce tool calls
+ let is_tool_call = false;
+ for ( let i = content.length - 1 ; i >= 0 ; i-- ) {
+ const content_block = content[i];
+
+ if ( content_block.type === 'tool_use' ) {
+ if ( ! msg.hasOwnProperty('tool_calls') ) {
+ msg.tool_calls = [];
+ is_tool_call = true;
+ }
+ msg.tool_calls.push({
+ id: content_block.id,
+ type: 'function',
+ function: {
+ name: content_block.name,
+ arguments: JSON.stringify(content_block.input),
+ }
+ });
+ content.splice(i, 1);
+ }
+ }
+
+ if ( is_tool_call ) msg.content = null;
+
+ // coerce tool results
+ // (we assume multiple tool results were already split into separate messages)
+ for ( let i = content.length - 1 ; i >= 0 ; i-- ) {
+ const content_block = content[i];
+ if ( content_block.type !== 'tool_result' ) continue;
+ msg.role = 'tool';
+ msg.tool_call_id = content_block.tool_use_id;
+ msg.content = content_block.content;
+ }
+ }
+
+ return messages;
+ })
+};
diff --git a/src/airouter.js/openai/models.json b/src/airouter.js/openai/models.json
new file mode 100644
index 0000000000..1c6b54540b
--- /dev/null
+++ b/src/airouter.js/openai/models.json
@@ -0,0 +1,121 @@
+[
+ {
+ "id": "gpt-4o",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 250,
+ "output": 1000
+ },
+ "max_tokens": 16384
+ },
+ {
+ "id": "gpt-4o-mini",
+ "max_tokens": 16384,
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 15,
+ "output": 60
+ }
+ },
+ {
+ "id": "o1",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 1500,
+ "output": 6000
+ },
+ "max_tokens": 100000
+ },
+ {
+ "id": "o1-mini",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 300,
+ "output": 1200
+ },
+ "max_tokens": 65536
+ },
+ {
+ "id": "o1-pro",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 15000,
+ "output": 60000
+ },
+ "max_tokens": 100000
+ },
+ {
+ "id": "o3",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 1000,
+ "output": 4000
+ },
+ "max_tokens": 100000
+ },
+ {
+ "id": "o3-mini",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 110,
+ "output": 440
+ },
+ "max_tokens": 100000
+ },
+ {
+ "id": "o4-mini",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 110,
+ "output": 440
+ },
+ "max_tokens": 100000
+ },
+ {
+ "id": "gpt-4.1",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 200,
+ "output": 800
+ },
+ "max_tokens": 32768
+ },
+ {
+ "id": "gpt-4.1-mini",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 40,
+ "output": 160
+ },
+ "max_tokens": 32768
+ },
+ {
+ "id": "gpt-4.1-nano",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 10,
+ "output": 40
+ },
+ "max_tokens": 32768
+ },
+ {
+ "id": "gpt-4.5-preview",
+ "cost": {
+ "currency": "usd-cents",
+ "tokens": 1000000,
+ "input": 7500,
+ "output": 15000
+ }
+ }
+]
diff --git a/src/airouter.js/package.json b/src/airouter.js/package.json
new file mode 100644
index 0000000000..e10088c9fd
--- /dev/null
+++ b/src/airouter.js/package.json
@@ -0,0 +1,16 @@
+{
+ "name": "@heyputer/airouter.js",
+ "version": "0.0.0",
+ "main": "airouter.js",
+ "type": "module",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "UNLICENSED",
+ "description": "",
+ "dependencies": {
+ "@anthropic-ai/sdk": "^0.56.0"
+ }
+}
diff --git a/src/airouter.js/router/LLMRegistry.js b/src/airouter.js/router/LLMRegistry.js
new file mode 100644
index 0000000000..ffed67fa1e
--- /dev/null
+++ b/src/airouter.js/router/LLMRegistry.js
@@ -0,0 +1,22 @@
+export class LLMRegistry {
+ apiTypes = {}
+
+ /**
+ * Add configuration for an LLM provider
+ * @param {Object} params
+ * @param {string} apiType determienes SDK and coercions used
+ * @param {string} [id] identifier, defaults to random uuid
+ */
+ link ({
+ id,
+ apiType,
+ config,
+ }) {}
+
+ /**
+ * Add a type of LLM provider (an API format)
+ */
+ registerApiType (name, apiType) {
+ this.apiTypes[name] = apiType;
+ }
+}
diff --git a/src/backend/exports.js b/src/backend/exports.js
index d4a836b0cd..143ad6a781 100644
--- a/src/backend/exports.js
+++ b/src/backend/exports.js
@@ -39,6 +39,7 @@ const { InternetModule } = require("./src/modules/internet/InternetModule.js");
const { CaptchaModule } = require("./src/modules/captcha/CaptchaModule.js");
const { EntityStoreModule } = require("./src/modules/entitystore/EntityStoreModule.js");
const { KVStoreModule } = require("./src/modules/kvstore/KVStoreModule.js");
+const { AIRouterModule } = require("./src/modules/airouter/AIRouterModule.js");
module.exports = {
helloworld: () => {
@@ -73,6 +74,7 @@ module.exports = {
LocalDiskStorageModule,
SelfHostedModule,
TestDriversModule,
+ AIRouterModule,
PuterAIModule,
BroadcastModule,
InternetModule,
diff --git a/src/backend/package.json b/src/backend/package.json
index 07d44101a2..bc0bce52f2 100644
--- a/src/backend/package.json
+++ b/src/backend/package.json
@@ -15,6 +15,7 @@
"@heyputer/kv.js": "^0.1.9",
"@heyputer/multest": "^0.0.2",
"@heyputer/putility": "^1.0.0",
+ "@heyputer/airouter": "^0.0.0",
"@mistralai/mistralai": "^1.3.4",
"@opentelemetry/api": "^1.4.1",
"@opentelemetry/auto-instrumentations-node": "^0.43.0",
diff --git a/src/backend/src/modules/puterai/AIChatService.js b/src/backend/src/modules/airouter/AIChatService.js
similarity index 94%
rename from src/backend/src/modules/puterai/AIChatService.js
rename to src/backend/src/modules/airouter/AIChatService.js
index 6698caeea5..e1b8c296a6 100644
--- a/src/backend/src/modules/puterai/AIChatService.js
+++ b/src/backend/src/modules/airouter/AIChatService.js
@@ -28,13 +28,16 @@ const { TypeSpec } = require("../../services/drivers/meta/Construct");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { Context } = require("../../util/context");
const { AsModeration } = require("./lib/AsModeration");
-const FunctionCalling = require("./lib/FunctionCalling");
-const Messages = require("./lib/Messages");
-const Streaming = require("./lib/Streaming");
// Maximum number of fallback attempts when a model fails, including the first attempt
const MAX_FALLBACKS = 3 + 1; // includes first attempt
+// Imported in _construct bleow.
+let
+ obtain,
+ NORMALIZED_LLM_MESSAGES, UNIVERSAL_LLM_MESSAGES,
+ NORMALIZED_SINGLE_MESSAGE, UNIVERSAL_SINGLE_MESSAGE,
+ NormalizedPromptUtil, CompletionWriter;
/**
* AIChatService class extends BaseService to provide AI chat completion functionality.
@@ -50,6 +53,41 @@ class AIChatService extends BaseService {
cuid2: require('@paralleldrive/cuid2').createId,
}
+ async ['__on_driver.register.interfaces'] () {
+ const svc_registry = this.services.get('registry');
+ const col_interfaces = svc_registry.get('interfaces');
+
+ col_interfaces.set('puter-chat-completion', {
+ description: 'Chatbot.',
+ methods: {
+ models: {
+ description: 'List supported models and their details.',
+ result: { type: 'json' },
+ parameters: {},
+ },
+ list: {
+ description: 'List supported models',
+ result: { type: 'json' },
+ parameters: {},
+ },
+ complete: {
+ description: 'Get completions for a chat log.',
+ parameters: {
+ messages: { type: 'json' },
+ tools: { type: 'json' },
+ vision: { type: 'flag' },
+ stream: { type: 'flag' },
+ response: { type: 'json' },
+ model: { type: 'string' },
+ temperature: { type: 'number' },
+ max_tokens: { type: 'number' },
+ },
+ result: { type: 'json' },
+ }
+ }
+ });
+ }
+
/**
* Initializes the service by setting up core properties.
@@ -58,12 +96,20 @@ class AIChatService extends BaseService {
* Called during service instantiation.
* @private
*/
- _construct () {
+ async _construct () {
this.providers = [];
this.simple_model_list = [];
this.detail_model_list = [];
this.detail_model_map = {};
+
+
+ ({
+ obtain,
+ NORMALIZED_LLM_MESSAGES, UNIVERSAL_LLM_MESSAGES,
+ NORMALIZED_SINGLE_MESSAGE, UNIVERSAL_SINGLE_MESSAGE,
+ NormalizedPromptUtil, CompletionWriter,
+ } = await import("@heyputer/airouter.js"));
}
get_model_details (model_name, context) {
@@ -396,8 +442,9 @@ class AIChatService extends BaseService {
}
if ( parameters.messages ) {
- parameters.messages =
- Messages.normalize_messages(parameters.messages);
+ parameters.messages = await obtain(NORMALIZED_LLM_MESSAGES, {
+ [UNIVERSAL_LLM_MESSAGES]: parameters.messages,
+ });
}
if ( ! test_mode && ! await this.moderate(parameters) ) {
@@ -416,7 +463,7 @@ class AIChatService extends BaseService {
}
if ( parameters.tools ) {
- FunctionCalling.normalize_tools_object(parameters.tools);
+ UniversalToolsNormalizer.normalize_tools_object(parameters.tools);
}
if ( intended_service === this.service_name ) {
@@ -452,7 +499,7 @@ class AIChatService extends BaseService {
const model_input_cost = model_details.cost.input;
const model_output_cost = model_details.cost.output;
const model_max_tokens = model_details.max_tokens;
- const text = Messages.extract_text(parameters.messages);
+ const text = NormalizedPromptUtil.extract_text(parameters.messages);
const approximate_input_cost = text.length / 4 * model_input_cost;
const usageAllowed = await svc_cost.get_funding_allowed({
available,
@@ -668,7 +715,7 @@ class AIChatService extends BaseService {
chunked: true,
}, stream);
- const chatStream = new Streaming.AIChatStream({
+ const chatStream = new CompletionWriter({
stream,
});
@@ -718,8 +765,9 @@ class AIChatService extends BaseService {
if ( parameters.response?.normalize ) {
- ret.result.message =
- Messages.normalize_single_message(ret.result.message);
+ ret.result.message = await obtain(NORMALIZED_SINGLE_MESSAGE, {
+ [UNIVERSAL_SINGLE_MESSAGE]: ret.result.message,
+ });
ret.result = {
message: ret.result.message,
via_ai_chat_service: true,
diff --git a/src/backend/src/modules/airouter/AIRouterModule.js b/src/backend/src/modules/airouter/AIRouterModule.js
new file mode 100644
index 0000000000..13c7bd541f
--- /dev/null
+++ b/src/backend/src/modules/airouter/AIRouterModule.js
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+// METADATA // {"ai-commented":{"service":"claude"}}
+const { AdvancedBase } = require("@heyputer/putility");
+const config = require("../../config");
+
+
+/**
+* PuterAIModule class extends AdvancedBase to manage and register various AI services.
+* This module handles the initialization and registration of multiple AI-related services
+* including text processing, speech synthesis, chat completion, and image generation.
+* Services are conditionally registered based on configuration settings, allowing for
+* flexible deployment with different AI providers like AWS, OpenAI, Claude, Together AI,
+* Mistral, Groq, and XAI.
+* @extends AdvancedBase
+*/
+class AIRouterModule extends AdvancedBase {
+ /**
+ * Module for managing AI-related services in the Puter platform
+ * Extends AdvancedBase to provide core functionality
+ * Handles registration and configuration of various AI services like OpenAI, Claude, AWS services etc.
+ */
+ async install (context) {
+ const services = context.get('services');
+
+ if ( !! config?.openai ) {
+ const { OpenAICompletionService } = require('./OpenAICompletionService');
+ services.registerService('openai-completion', OpenAICompletionService);
+ }
+
+ if ( !! config?.services?.claude ) {
+ const { ClaudeService } = require('./ClaudeService');
+ services.registerService('claude', ClaudeService);
+ }
+
+ if ( !! config?.services?.['together-ai'] ) {
+ const { TogetherAIService } = require('./TogetherAIService');
+ services.registerService('together-ai', TogetherAIService);
+ }
+
+ if ( !! config?.services?.['mistral'] ) {
+ const { MistralAIService } = require('./MistralAIService');
+ services.registerService('mistral', MistralAIService);
+ }
+
+ if ( !! config?.services?.['groq'] ) {
+ const { GroqAIService } = require('./GroqAIService');
+ services.registerService('groq', GroqAIService);
+ }
+
+ if ( !! config?.services?.['xai'] ) {
+ const { XAIService } = require('./XAIService');
+ services.registerService('xai', XAIService);
+ }
+
+ if ( !! config?.services?.['deepseek'] ) {
+ const { DeepSeekService } = require('./DeepSeekService');
+ services.registerService('deepseek', DeepSeekService);
+ }
+ if ( !! config?.services?.['gemini'] ) {
+ const { GeminiService } = require('./GeminiService');
+ services.registerService('gemini', GeminiService);
+ }
+ if ( !! config?.services?.['openrouter'] ) {
+ const { OpenRouterService } = require('./OpenRouterService');
+ services.registerService('openrouter', OpenRouterService);
+ }
+
+ const { AIChatService } = require('./AIChatService');
+ services.registerService('ai-chat', AIChatService);
+
+ const { FakeChatService } = require('./FakeChatService');
+ services.registerService('fake-chat', FakeChatService);
+
+ const{ AITestModeService } = require('./AITestModeService');
+ services.registerService('ai-test-mode', AITestModeService);
+
+ const { UsageLimitedChatService } = require('./UsageLimitedChatService');
+ services.registerService('usage-limited-chat', UsageLimitedChatService);
+ }
+}
+
+module.exports = {
+ AIRouterModule,
+};
diff --git a/src/backend/src/modules/puterai/AITestModeService.js b/src/backend/src/modules/airouter/AITestModeService.js
similarity index 100%
rename from src/backend/src/modules/puterai/AITestModeService.js
rename to src/backend/src/modules/airouter/AITestModeService.js
diff --git a/src/backend/src/modules/airouter/ClaudeService.js b/src/backend/src/modules/airouter/ClaudeService.js
new file mode 100644
index 0000000000..020bbc1b8d
--- /dev/null
+++ b/src/backend/src/modules/airouter/ClaudeService.js
@@ -0,0 +1,311 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+// METADATA // {"ai-commented":{"service":"claude"}}
+const { default: Anthropic, toFile } = require("@anthropic-ai/sdk");
+const BaseService = require("../../services/BaseService");
+const { TypedValue } = require("../../services/drivers/meta/Runtime");
+const FSNodeParam = require("../../api/filesystem/FSNodeParam");
+const { LLRead } = require("../../filesystem/ll_operations/ll_read");
+const { Context } = require("../../util/context");
+const { TeePromise } = require('@heyputer/putility').libs.promise;
+
+
+let
+ obtain,
+ ANTHROPIC_API_KEY,
+ NORMALIZED_LLM_PARAMS, COMPLETION_WRITER, PROVIDER_NAME,
+ USAGE_WRITER,
+ ASYNC_RESPONSE, SYNC_RESPONSE
+;
+
+/**
+* ClaudeService class extends BaseService to provide integration with Anthropic's Claude AI models.
+* Implements the puter-chat-completion interface for handling AI chat interactions.
+* Manages message streaming, token limits, model selection, and API communication with Claude.
+* Supports system prompts, message adaptation, and usage tracking.
+* @extends BaseService
+*/
+class ClaudeService extends BaseService {
+ static MODULES = {
+ Anthropic: require('@anthropic-ai/sdk'),
+ }
+
+ /**
+ * @type {import('@anthropic-ai/sdk').Anthropic}
+ */
+ anthropic;
+
+ async _construct () {
+ const airouter = await import('@heyputer/airouter.js');
+ ({
+ obtain,
+ ANTHROPIC_API_KEY,
+ NORMALIZED_LLM_PARAMS, COMPLETION_WRITER, PROVIDER_NAME,
+ ASYNC_RESPONSE, SYNC_RESPONSE
+ } = airouter);
+
+ }
+
+
+ /**
+ * Initializes the Claude service by creating an Anthropic client instance
+ * and registering this service as a provider with the AI chat service.
+ * @private
+ * @returns {Promise}
+ */
+ async _init () {
+ this.anthropic = new Anthropic({
+ apiKey: this.config.apiKey
+ });
+
+ const svc_aiChat = this.services.get('ai-chat');
+ svc_aiChat.register_provider({
+ service_name: this.service_name,
+ alias: true,
+ });
+ }
+
+
+ /**
+ * Returns the default model identifier for Claude API interactions
+ * @returns {string} The default model ID 'claude-3-5-sonnet-latest'
+ */
+ get_default_model () {
+ return 'claude-3-5-sonnet-latest';
+ }
+
+ static IMPLEMENTS = {
+ ['puter-chat-completion']: {
+ /**
+ * Returns a list of available models and their details.
+ * See AIChatService for more information.
+ *
+ * @returns Promise> Array of model details
+ */
+ async models () {
+ return await this.models_();
+ },
+
+ /**
+ * Returns a list of available model names including their aliases
+ * @returns {Promise} Array of model identifiers and their aliases
+ * @description Retrieves all available model IDs and their aliases,
+ * flattening them into a single array of strings that can be used for model selection
+ */
+ async list () {
+ const models = await this.models_();
+ const model_names = [];
+ for ( const model of models ) {
+ model_names.push(model.id);
+ if ( model.aliases ) {
+ model_names.push(...model.aliases);
+ }
+ }
+ return model_names;
+ },
+
+ /**
+ * Completes a chat interaction with the Claude AI model
+ * @param {Object} options - The completion options
+ * @param {Array} options.messages - Array of chat messages to process
+ * @param {boolean} options.stream - Whether to stream the response
+ * @param {string} [options.model] - The Claude model to use, defaults to service default
+ * @returns {TypedValue|Object} Returns either a TypedValue with streaming response or a completion object
+ * @this {ClaudeService}
+ */
+ async complete ({ messages, stream, model, tools, max_tokens, temperature}) {
+ await this.handle_puter_paths_(messages);
+
+ if ( stream ) {
+ let usage_promise = new TeePromise();
+
+ let streamOperation;
+ const init_chat_stream = async ({ chatStream: completionWriter }) => {
+ console.log('the completion writer?', completionWriter);
+ await obtain(ASYNC_RESPONSE, {
+ [PROVIDER_NAME]: 'anthropic',
+ [NORMALIZED_LLM_PARAMS]: {
+ messages, model, tools, max_tokens, temperature,
+ },
+ [COMPLETION_WRITER]: completionWriter,
+ [ANTHROPIC_API_KEY]: this.config.apiKey,
+ [USAGE_WRITER]: usage_promise,
+ });
+ // streamOperation = await this.anthropicApiType.stream(this.anthropic, completionWriter, {
+ // messages, model, tools, max_tokens, temperature,
+ // })
+ // await streamOperation.run();
+ };
+
+ return new TypedValue({ $: 'ai-chat-intermediate' }, {
+ init_chat_stream,
+ stream: true,
+ usage_promise: usage_promise,
+ finally_fn: async () => {
+ await streamOperation.cleanup();
+ },
+ });
+ } else {
+ return await obtain(SYNC_RESPONSE, {
+ [PROVIDER_NAME]: 'anthropic',
+ [NORMALIZED_LLM_PARAMS]: {
+ messages, model, tools, max_tokens, temperature,
+ },
+ [ANTHROPIC_API_KEY]: this.config.apiKey,
+ });
+ }
+ }
+ }
+ }
+
+ async handle_puter_paths_(messages) {
+ const require = this.require;
+
+ const actor = Context.get('actor');
+ const { user } = actor.type;
+ for ( const message of messages ) {
+ for ( const contentPart of message.content ) {
+ if ( ! contentPart.puter_path ) continue;
+
+ const node = await (new FSNodeParam(contentPart.puter_path)).consolidate({
+ req: { user },
+ getParam: () => contentPart.puter_path,
+ });
+
+ delete contentPart.puter_path;
+ contentPart.type = 'data';
+ contentPart.data = {
+ async getStream () {
+ const ll_read = new LLRead();
+ return await ll_read.run({
+ actor: Context.get('actor'),
+ fsNode: node,
+ });
+ },
+ async getMimeType () {
+ const mime = require('mime-types');
+ return mime.contentType(await node.get('name'));
+ },
+ };
+ }
+ }
+ }
+
+
+ /**
+ * Retrieves available Claude AI models and their specifications
+ * @returns {Promise} Array of model objects containing:
+ * - id: Model identifier
+ * - name: Display name
+ * - aliases: Alternative names for the model
+ * - context: Maximum context window size
+ * - cost: Pricing details (currency, token counts, input/output costs)
+ * - qualitative_speed: Relative speed rating
+ * - max_output: Maximum output tokens
+ * - training_cutoff: Training data cutoff date
+ */
+ async models_ () {
+ return [
+ {
+ id: 'claude-opus-4-20250514',
+ aliases: ['claude-opus-4', 'claude-opus-4-latest'],
+ name: 'Claude Opus 4',
+ context: 200000,
+ cost: {
+ currency: 'usd-cents',
+ tokens: 1_000_000,
+ input: 1500,
+ output: 7500,
+ },
+ max_tokens: 32000,
+ },
+ {
+ id: 'claude-sonnet-4-20250514',
+ aliases: ['claude-sonnet-4', 'claude-sonnet-4-latest'],
+ name: 'Claude Sonnet 4',
+ context: 200000,
+ cost: {
+ currency: 'usd-cents',
+ tokens: 1_000_000,
+ input: 300,
+ output: 1500,
+ },
+ max_tokens: 64000,
+ },
+ {
+ id: 'claude-3-7-sonnet-20250219',
+ aliases: ['claude-3-7-sonnet-latest'],
+ succeeded_by: 'claude-sonnet-4-20250514',
+ context: 200000,
+ cost: {
+ currency: 'usd-cents',
+ tokens: 1_000_000,
+ input: 300,
+ output: 1500,
+ },
+ max_tokens: 8192,
+ },
+ {
+ id: 'claude-3-5-sonnet-20241022',
+ name: 'Claude 3.5 Sonnet',
+ aliases: ['claude-3-5-sonnet-latest'],
+ context: 200000,
+ cost: {
+ currency: 'usd-cents',
+ tokens: 1_000_000,
+ input: 300,
+ output: 1500,
+ },
+ qualitative_speed: 'fast',
+ training_cutoff: '2024-04',
+ max_tokens: 8192,
+ },
+ {
+ id: 'claude-3-5-sonnet-20240620',
+ succeeded_by: 'claude-3-5-sonnet-20241022',
+ context: 200000, // might be wrong
+ cost: {
+ currency: 'usd-cents',
+ tokens: 1_000_000,
+ input: 300,
+ output: 1500,
+ },
+ max_tokens: 8192,
+ },
+ {
+ id: 'claude-3-haiku-20240307',
+ // aliases: ['claude-3-haiku-latest'],
+ context: 200000,
+ cost: {
+ currency: 'usd-cents',
+ tokens: 1_000_000,
+ input: 25,
+ output: 125,
+ },
+ qualitative_speed: 'fastest',
+ max_tokens: 4096,
+ },
+ ];
+ }
+}
+
+module.exports = {
+ ClaudeService,
+};
diff --git a/src/backend/src/modules/puterai/DeepSeekService.js b/src/backend/src/modules/airouter/DeepSeekService.js
similarity index 60%
rename from src/backend/src/modules/puterai/DeepSeekService.js
rename to src/backend/src/modules/airouter/DeepSeekService.js
index 2cec41b48d..a3eebac4b1 100644
--- a/src/backend/src/modules/puterai/DeepSeekService.js
+++ b/src/backend/src/modules/airouter/DeepSeekService.js
@@ -18,9 +18,13 @@
*/
// METADATA // {"ai-commented":{"service":"claude"}}
+const { TeePromise } = require("@heyputer/putility/src/libs/promise");
const BaseService = require("../../services/BaseService");
-const OpenAIUtil = require("./lib/OpenAIUtil");
-const dedent = require('dedent');
+const { TypedValue } = require("../../services/drivers/meta/Runtime");
+let
+ obtain,
+ OPENAI_CLIENT, SYNC_RESPONSE, PROVIDER_NAME, NORMALIZED_LLM_PARAMS,
+ ASYNC_RESPONSE, USAGE_WRITER, COMPLETION_WRITER;
/**
* DeepSeekService class - Provides integration with X.AI's API for chat completions
@@ -43,6 +47,13 @@ class DeepSeekService extends BaseService {
return model;
}
+ async _construct () {
+ ({
+ obtain,
+ OPENAI_CLIENT, SYNC_RESPONSE, PROVIDER_NAME, NORMALIZED_LLM_PARAMS,
+ ASYNC_RESPONSE, USAGE_WRITER, COMPLETION_WRITER
+ } = require('@heyputer/airouter.js'));
+ }
/**
* Initializes the XAI service by setting up the OpenAI client and registering with the AI chat provider
@@ -106,60 +117,77 @@ class DeepSeekService extends BaseService {
*/
async complete ({ messages, stream, model, tools, max_tokens, temperature }) {
model = this.adapt_model(model);
-
- messages = await OpenAIUtil.process_input_messages(messages);
- for ( const message of messages ) {
- // DeepSeek doesn't appreciate arrays here
- if ( message.tool_calls && Array.isArray(message.content) ) {
- message.content = "";
- }
- }
- // Function calling is just broken on DeepSeek - it never awknowledges
- // the tool results and instead keeps calling the function over and over.
- // (see https://github.com/deepseek-ai/DeepSeek-V3/issues/15)
- // To fix this, we inject a message that tells DeepSeek what happened.
- const TOOL_TEXT = message => dedent(`
- Hi DeepSeek V3, your tool calling is broken and you are not able to
- obtain tool results in the expected way. That's okay, we can work
- around this.
-
- Please do not repeat this tool call.
-
- We have provided the tool call results below:
-
- Tool call ${message.tool_call_id} returned: ${message.content}.
- `);
- for ( let i=messages.length-1; i >= 0 ; i-- ) {
- const message = messages[i];
- if ( message.role === 'tool' ) {
- messages.splice(i+1, 0, {
- role: 'system',
- content: [
- {
- type: 'text',
- text: TOOL_TEXT(message),
- }
- ]
- });
- }
+
+ if ( stream ) {
+ let usage_promise = new TeePromise();
+
+ let streamOperation;
+ const init_chat_stream = async ({ chatStream: completionWriter }) => {
+ await obtain(ASYNC_RESPONSE, {
+ [PROVIDER_NAME]: 'openai',
+ [NORMALIZED_LLM_PARAMS]: {
+ messages, model, tools, max_tokens, temperature,
+ },
+ [COMPLETION_WRITER]: completionWriter,
+ [OPENAI_CLIENT]: this.openai,
+ [USAGE_WRITER]: usage_promise,
+ })
+ };
+
+ return new TypedValue({ $: 'ai-chat-intermediate' }, {
+ init_chat_stream,
+ stream: true,
+ usage_promise: usage_promise,
+ finally_fn: async () => {
+ await streamOperation.cleanup();
+ },
+ });
+ } else {
+ return await obtain(SYNC_RESPONSE, {
+ [PROVIDER_NAME]: 'openai',
+ [NORMALIZED_LLM_PARAMS]: {
+ messages, model, tools, max_tokens, temperature,
+ },
+ [OPENAI_CLIENT]: this.openai,
+ });
}
+ }
+ }
+ }
- const completion = await this.openai.chat.completions.create({
- messages,
- model: model ?? this.get_default_model(),
- ...(tools ? { tools } : {}),
- max_tokens: max_tokens || 1000,
- temperature, // the default temperature is 1.0. suggested 0 for math/coding and 1.5 for creative poetry
- stream,
- ...(stream ? {
- stream_options: { include_usage: true },
- } : {}),
- });
+ async handle_puter_paths_(messages) {
+ const require = this.require;
- return OpenAIUtil.handle_completion_output({
- stream, completion,
+ const actor = Context.get('actor');
+ const { user } = actor.type;
+ for ( const message of messages ) {
+ for ( const contentPart of message.content ) {
+ if ( ! contentPart.puter_path ) continue;
+
+ const node = await (new FSNodeParam(contentPart.puter_path)).consolidate({
+ req: { user },
+ getParam: () => contentPart.puter_path,
});
+
+ delete contentPart.puter_path;
+ contentPart.type = 'data';
+ contentPart.data = {
+ async getSize () {
+ return await node.get('size')
+ },
+ async getStream () {
+ const ll_read = new LLRead();
+ return await ll_read.run({
+ actor: Context.get('actor'),
+ fsNode: node,
+ });
+ },
+ async getMimeType () {
+ const mime = require('mime-types');
+ return mime.contentType(await node.get('name'));
+ },
+ };
}
}
}
diff --git a/src/backend/src/modules/puterai/FakeChatService.js b/src/backend/src/modules/airouter/FakeChatService.js
similarity index 100%
rename from src/backend/src/modules/puterai/FakeChatService.js
rename to src/backend/src/modules/airouter/FakeChatService.js
diff --git a/src/backend/src/modules/puterai/GeminiService.js b/src/backend/src/modules/airouter/GeminiService.js
similarity index 95%
rename from src/backend/src/modules/puterai/GeminiService.js
rename to src/backend/src/modules/airouter/GeminiService.js
index b9f065e463..6c6a23ea41 100644
--- a/src/backend/src/modules/puterai/GeminiService.js
+++ b/src/backend/src/modules/airouter/GeminiService.js
@@ -3,9 +3,13 @@ const { GoogleGenerativeAI } = require('@google/generative-ai');
const GeminiSquareHole = require("./lib/GeminiSquareHole");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const putility = require("@heyputer/putility");
-const FunctionCalling = require("./lib/FunctionCalling");
+
+let OpenAIToolsAdapter;
class GeminiService extends BaseService {
+ async _construct () {
+ ({ OpenAIToolsAdapter } = await import("@heyputer/airouter.js"));
+ }
async _init () {
const svc_aiChat = this.services.get('ai-chat');
svc_aiChat.register_provider({
@@ -32,7 +36,7 @@ class GeminiService extends BaseService {
},
async complete ({ messages, stream, model, tools, max_tokens, temperature }) {
- tools = FunctionCalling.make_gemini_tools(tools);
+ tools = OpenAIToolsAdapter.adapt_tools(tools);
const genAI = new GoogleGenerativeAI(this.config.apiKey);
const genModel = genAI.getGenerativeModel({
diff --git a/src/backend/src/modules/puterai/GroqAIService.js b/src/backend/src/modules/airouter/GroqAIService.js
similarity index 100%
rename from src/backend/src/modules/puterai/GroqAIService.js
rename to src/backend/src/modules/airouter/GroqAIService.js
diff --git a/src/backend/src/modules/puterai/MistralAIService.js b/src/backend/src/modules/airouter/MistralAIService.js
similarity index 100%
rename from src/backend/src/modules/puterai/MistralAIService.js
rename to src/backend/src/modules/airouter/MistralAIService.js
diff --git a/src/backend/src/modules/puterai/OpenAICompletionService.js b/src/backend/src/modules/airouter/OpenAICompletionService.js
similarity index 51%
rename from src/backend/src/modules/puterai/OpenAICompletionService.js
rename to src/backend/src/modules/airouter/OpenAICompletionService.js
index a35a93e594..c982e58894 100644
--- a/src/backend/src/modules/puterai/OpenAICompletionService.js
+++ b/src/backend/src/modules/airouter/OpenAICompletionService.js
@@ -18,12 +18,16 @@
*/
// METADATA // {"ai-commented":{"service":"claude"}}
+const { TeePromise } = require('@heyputer/putility/src/libs/promise');
const FSNodeParam = require('../../api/filesystem/FSNodeParam');
const { LLRead } = require('../../filesystem/ll_operations/ll_read');
const BaseService = require('../../services/BaseService');
const { Context } = require('../../util/context');
-const { stream_to_buffer } = require('../../util/streamutil');
-const OpenAIUtil = require('./lib/OpenAIUtil');
+const { TypedValue } = require('../../services/drivers/meta/Runtime');
+let
+ obtain,
+ OPENAI_CLIENT, SYNC_RESPONSE, PROVIDER_NAME, NORMALIZED_LLM_PARAMS,
+ ASYNC_RESPONSE, USAGE_WRITER, COMPLETION_WRITER;
// We're capping at 5MB, which sucks, but Chat Completions doesn't suuport
// file inputs.
@@ -46,6 +50,14 @@ class OpenAICompletionService extends BaseService {
* @type {import('openai').OpenAI}
*/
openai;
+
+ async _construct () {
+ ({
+ obtain,
+ OPENAI_CLIENT, SYNC_RESPONSE, PROVIDER_NAME, NORMALIZED_LLM_PARAMS,
+ ASYNC_RESPONSE, USAGE_WRITER, COMPLETION_WRITER
+ } = require('@heyputer/airouter.js'));
+ }
/**
* Initializes the OpenAI service by setting up the API client with credentials
@@ -104,128 +116,8 @@ class OpenAICompletionService extends BaseService {
* @returns {Promise}
*/
async models_ () {
- return [
- {
- id: 'gpt-4o',
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 250,
- output: 1000, // https://platform.openai.com/docs/pricing
- },
- max_tokens: 16384,
- },
- {
- id: 'gpt-4o-mini',
- max_tokens: 16384,
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 15,
- output: 60,
- },
- max_tokens: 16384,
- },
- {
- id: 'o1',
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 1500,
- output: 6000,
- },
- max_tokens: 100000,
- },
- {
- id: 'o1-mini',
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 300,
- output: 1200,
- },
- max_tokens: 65536,
- },
- {
- id: 'o1-pro',
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 15000,
- output: 60000,
- },
- max_tokens: 100000,
- },
- {
- id: 'o3',
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 1000,
- output: 4000,
- },
- max_tokens: 100000,
- },
- {
- id: 'o3-mini',
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 110,
- output: 440,
- },
- max_tokens: 100000,
- },
- {
- id: 'o4-mini',
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 110,
- output: 440,
- },
- max_tokens: 100000,
- },
- {
- id: 'gpt-4.1',
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 200,
- output: 800,
- },
- max_tokens: 32768,
- },
- {
- id: 'gpt-4.1-mini',
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 40,
- output: 160,
- },
- max_tokens: 32768,
- },
- {
- id: 'gpt-4.1-nano',
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 10,
- output: 40,
- },
- max_tokens: 32768,
- },
- {
- id: 'gpt-4.5-preview',
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 7500,
- output: 15000,
- }
- }
- ];
+ const { models } = await import('@heyputer/airouter.js');
+ return models.openai;
}
static IMPLEMENTS = {
@@ -338,95 +230,77 @@ class OpenAICompletionService extends BaseService {
this.log.info('PRIVATE UID FOR USER ' + user_private_uid)
- // Perform file uploads
- {
- const actor = Context.get('actor');
- const { user } = actor.type;
-
- const file_input_tasks = [];
- for ( const message of messages ) {
- // We can assume `message.content` is not undefined because
- // Messages.normalize_single_message ensures this.
- for ( const contentPart of message.content ) {
- if ( ! contentPart.puter_path ) continue;
- file_input_tasks.push({
- node: await (new FSNodeParam(contentPart.puter_path)).consolidate({
- req: { user },
- getParam: () => contentPart.puter_path,
- }),
- contentPart,
- });
- }
- }
-
- const promises = [];
- for ( const task of file_input_tasks ) promises.push((async () => {
- if ( await task.node.get('size') > MAX_FILE_SIZE ) {
- delete task.contentPart.puter_path;
- task.contentPart.type = 'text';
- task.contentPart.text = `{error: input file exceeded maximum of ${MAX_FILE_SIZE} bytes; ` +
- `the user did not write this message}`; // "poor man's system prompt"
- return; // "continue"
- }
-
- const ll_read = new LLRead();
- const stream = await ll_read.run({
- actor: Context.get('actor'),
- fsNode: task.node,
- });
- const require = this.require;
- const mime = require('mime-types');
- const mimeType = mime.contentType(await task.node.get('name'));
-
- const buffer = await stream_to_buffer(stream);
- const base64 = buffer.toString('base64');
-
- delete task.contentPart.puter_path;
- if ( mimeType.startsWith('image/') ) {
- task.contentPart.type = 'image_url',
- task.contentPart.image_url = {
- url: `data:${mimeType};base64,${base64}`,
- };
- } else if ( mimeType.startsWith('audio/') ) {
- task.contentPart.type = 'input_audio',
- task.contentPart.input_audio = {
- data: `data:${mimeType};base64,${base64}`,
- format: mimeType.split('/')[1],
- }
- } else {
- task.contentPart.type = 'text';
- task.contentPart.text = `{error: input file has unsupported MIME type; ` +
- `the user did not write this message}`; // "poor man's system prompt"
- }
- })());
- await Promise.all(promises);
- }
+ await this.handle_puter_paths_(messages);
- // Here's something fun; the documentation shows `type: 'image_url'` in
- // objects that contain an image url, but everything still works if
- // that's missing. We normalise it here so the token count code works.
- messages = await OpenAIUtil.process_input_messages(messages);
+ if ( stream ) {
+ let usage_promise = new TeePromise();
- const completion = await this.openai.chat.completions.create({
- user: user_private_uid,
- messages: messages,
- model: model,
- ...(tools ? { tools } : {}),
- ...(max_tokens ? { max_completion_tokens: max_tokens } : {}),
- ...(temperature ? { temperature } : {}),
- stream,
- ...(stream ? {
- stream_options: { include_usage: true },
- } : {}),
- });
+ let streamOperation;
+ const init_chat_stream = async ({ chatStream: completionWriter }) => {
+ await obtain(ASYNC_RESPONSE, {
+ [PROVIDER_NAME]: 'openai',
+ [NORMALIZED_LLM_PARAMS]: {
+ messages, model, tools, max_tokens, temperature,
+ },
+ [COMPLETION_WRITER]: completionWriter,
+ [OPENAI_CLIENT]: this.openai,
+ [USAGE_WRITER]: usage_promise,
+ })
+ };
- return OpenAIUtil.handle_completion_output({
- usage_calculator: OpenAIUtil.create_usage_calculator({
- model_details: (await this.models_()).find(m => m.id === model),
- }),
- stream, completion,
- moderate: moderation && this.check_moderation.bind(this),
- });
+ return new TypedValue({ $: 'ai-chat-intermediate' }, {
+ init_chat_stream,
+ stream: true,
+ usage_promise: usage_promise,
+ finally_fn: async () => {
+ await streamOperation.cleanup();
+ },
+ });
+ } else {
+ return await obtain(SYNC_RESPONSE, {
+ [PROVIDER_NAME]: 'openai',
+ [NORMALIZED_LLM_PARAMS]: {
+ messages, model, tools, max_tokens, temperature,
+ },
+ [OPENAI_CLIENT]: this.openai,
+ });
+ }
+ }
+
+ async handle_puter_paths_(messages) {
+ const require = this.require;
+
+ const actor = Context.get('actor');
+ const { user } = actor.type;
+ for ( const message of messages ) {
+ for ( const contentPart of message.content ) {
+ if ( ! contentPart.puter_path ) continue;
+
+ const node = await (new FSNodeParam(contentPart.puter_path)).consolidate({
+ req: { user },
+ getParam: () => contentPart.puter_path,
+ });
+
+ delete contentPart.puter_path;
+ contentPart.type = 'data';
+ contentPart.data = {
+ async getSize () {
+ return await node.get('size')
+ },
+ async getStream () {
+ const ll_read = new LLRead();
+ return await ll_read.run({
+ actor: Context.get('actor'),
+ fsNode: node,
+ });
+ },
+ async getMimeType () {
+ const mime = require('mime-types');
+ return mime.contentType(await node.get('name'));
+ },
+ };
+ }
+ }
}
}
diff --git a/src/backend/src/modules/puterai/OpenRouterService.js b/src/backend/src/modules/airouter/OpenRouterService.js
similarity index 100%
rename from src/backend/src/modules/puterai/OpenRouterService.js
rename to src/backend/src/modules/airouter/OpenRouterService.js
diff --git a/src/backend/src/modules/puterai/TogetherAIService.js b/src/backend/src/modules/airouter/TogetherAIService.js
similarity index 100%
rename from src/backend/src/modules/puterai/TogetherAIService.js
rename to src/backend/src/modules/airouter/TogetherAIService.js
diff --git a/src/backend/src/modules/puterai/UsageLimitedChatService.js b/src/backend/src/modules/airouter/UsageLimitedChatService.js
similarity index 96%
rename from src/backend/src/modules/puterai/UsageLimitedChatService.js
rename to src/backend/src/modules/airouter/UsageLimitedChatService.js
index 7774709413..68867a0f12 100644
--- a/src/backend/src/modules/puterai/UsageLimitedChatService.js
+++ b/src/backend/src/modules/airouter/UsageLimitedChatService.js
@@ -22,7 +22,9 @@ const { default: dedent } = require("dedent");
const BaseService = require("../../services/BaseService");
const { PassThrough } = require("stream");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
-const Streaming = require("./lib/Streaming");
+
+// Imported in _construct below
+let CompletionWriter;
/**
* UsageLimitedChatService - A specialized chat service that returns resource exhaustion messages.
@@ -31,6 +33,9 @@ const Streaming = require("./lib/Streaming");
* Can handle both streaming and non-streaming requests consistently.
*/
class UsageLimitedChatService extends BaseService {
+ async _construct () {
+ ({ CompletionWriter } = await import('@heyputer/airouter'));
+ }
get_default_model () {
return 'usage-limited';
}
@@ -85,7 +90,7 @@ class UsageLimitedChatService extends BaseService {
chunked: true,
}, streamObj);
- const chatStream = new Streaming.AIChatStream({
+ const chatStream = new CompletionWriter({
stream: streamObj,
});
diff --git a/src/backend/src/modules/puterai/XAIService.js b/src/backend/src/modules/airouter/XAIService.js
similarity index 100%
rename from src/backend/src/modules/puterai/XAIService.js
rename to src/backend/src/modules/airouter/XAIService.js
diff --git a/src/backend/src/modules/puterai/experiment/stream_claude.js b/src/backend/src/modules/airouter/experiment/stream_claude.js
similarity index 100%
rename from src/backend/src/modules/puterai/experiment/stream_claude.js
rename to src/backend/src/modules/airouter/experiment/stream_claude.js
diff --git a/src/backend/src/modules/puterai/experiment/stream_openai.js b/src/backend/src/modules/airouter/experiment/stream_openai.js
similarity index 100%
rename from src/backend/src/modules/puterai/experiment/stream_openai.js
rename to src/backend/src/modules/airouter/experiment/stream_openai.js
diff --git a/src/backend/src/modules/puterai/lib/AsModeration.js b/src/backend/src/modules/airouter/lib/AsModeration.js
similarity index 100%
rename from src/backend/src/modules/puterai/lib/AsModeration.js
rename to src/backend/src/modules/airouter/lib/AsModeration.js
diff --git a/src/backend/src/modules/puterai/lib/GeminiSquareHole.js b/src/backend/src/modules/airouter/lib/GeminiSquareHole.js
similarity index 100%
rename from src/backend/src/modules/puterai/lib/GeminiSquareHole.js
rename to src/backend/src/modules/airouter/lib/GeminiSquareHole.js
diff --git a/src/backend/src/modules/airouter/lib/OpenAIUtil.js b/src/backend/src/modules/airouter/lib/OpenAIUtil.js
new file mode 100644
index 0000000000..72acdaa5e9
--- /dev/null
+++ b/src/backend/src/modules/airouter/lib/OpenAIUtil.js
@@ -0,0 +1,127 @@
+const putility = require("@heyputer/putility");
+const { TypedValue } = require("../../../services/drivers/meta/Runtime");
+const { nou } = require("../../../util/langutil");
+
+module.exports = class OpenAIUtil {
+ /**
+ * Process input messages from Puter's normalized format to OpenAI's format
+ * May make changes in-place.
+ *
+ * @param {Array} messages - array of normalized messages
+ * @returns {Array} - array of messages in OpenAI format
+ */
+ static process_input_messages = async (messages) => {
+ const { obtain, NORMALIZED_LLM_MESSAGES, UNIVERSAL_LLM_MESSAGES } = await import("@heyputer/airouter.js");
+ return await obtain(NORMALIZED_LLM_MESSAGES, {
+ [UNIVERSAL_LLM_MESSAGES]: messages,
+ });
+ }
+
+ static create_usage_calculator = ({ model_details }) => {
+ return ({ usage }) => {
+ const tokens = [];
+
+ tokens.push({
+ type: 'prompt',
+ model: model_details.id,
+ amount: usage.prompt_tokens,
+ cost: model_details.cost.input * usage.prompt_tokens,
+ });
+
+ tokens.push({
+ type: 'completion',
+ model: model_details.id,
+ amount: usage.completion_tokens,
+ cost: model_details.cost.output * usage.completion_tokens,
+ });
+
+ return tokens;
+ };
+ };
+
+ static create_chat_stream_handler = ({
+ deviations,
+ completion, usage_promise,
+ }) => async ({ chatStream }) => {
+ const { OpenAIStyleStreamAdapter } = await import("@heyputer/airouter.js");
+ const StreamAdapter = class extends OpenAIStyleStreamAdapter {};
+ for ( const key in deviations ) {
+ StreamAdapter[key] = deviations[key];
+ }
+
+ await StreamAdapter.write_to_stream({
+ input: completion,
+ completionWriter: chatStream,
+ usageWriter: usage_promise,
+ });
+ };
+
+ static async handle_completion_output ({
+ deviations,
+ stream, completion, moderate,
+ usage_calculator,
+ finally_fn,
+ }) {
+ deviations = Object.assign({
+ // affected by: Mistral
+ coerce_completion_usage: completion => completion.usage,
+ }, deviations);
+
+ if ( stream ) {
+ let usage_promise = new putility.libs.promise.TeePromise();
+
+ const init_chat_stream =
+ OpenAIUtil.create_chat_stream_handler({
+ deviations,
+ completion,
+ usage_promise,
+ usage_calculator,
+ });
+
+ return new TypedValue({ $: 'ai-chat-intermediate' }, {
+ stream: true,
+ init_chat_stream,
+ finally_fn,
+ usage_promise: usage_promise.then(usage => {
+ return usage_calculator ? usage_calculator({ usage }) : {
+ input_tokens: usage.prompt_tokens,
+ output_tokens: usage.completion_tokens,
+ };
+ }),
+ });
+ }
+
+ if ( finally_fn ) await finally_fn();
+
+ const is_empty = completion.choices?.[0]?.message?.content?.trim() === '';
+ if ( is_empty && ! completion.choices?.[0]?.message?.tool_calls ) {
+ // GPT refuses to generate an empty response if you ask it to,
+ // so this will probably only happen on an error condition.
+ throw new Error('an empty response was generated');
+ }
+
+ // We need to moderate the completion too
+ const mod_text = completion.choices[0].message.content;
+ if ( moderate && mod_text !== null ) {
+ const moderation_result = await moderate(mod_text);
+ if ( moderation_result.flagged ) {
+ throw new Error('message is not allowed');
+ }
+ }
+
+ const ret = completion.choices[0];
+ const completion_usage = deviations.coerce_completion_usage(completion);
+ ret.usage = usage_calculator ? usage_calculator({
+ ...completion,
+ usage: completion_usage,
+ }) : {
+ input_tokens: completion_usage.prompt_tokens,
+ output_tokens: completion_usage.completion_tokens,
+ };
+ // TODO: turn these into toggle logs
+ // console.log('ORIGINAL COMPLETION', completion);
+ // console.log('COMPLETION USAGE', completion_usage);
+ // console.log('RETURN VALUE', ret);
+ return ret;
+ }
+};
\ No newline at end of file
diff --git a/src/backend/src/modules/puterai/samples/claude-1.js b/src/backend/src/modules/airouter/samples/claude-1.js
similarity index 100%
rename from src/backend/src/modules/puterai/samples/claude-1.js
rename to src/backend/src/modules/airouter/samples/claude-1.js
diff --git a/src/backend/src/modules/puterai/samples/claude-tools-1.js b/src/backend/src/modules/airouter/samples/claude-tools-1.js
similarity index 100%
rename from src/backend/src/modules/puterai/samples/claude-tools-1.js
rename to src/backend/src/modules/airouter/samples/claude-tools-1.js
diff --git a/src/backend/src/modules/puterai/samples/openai-1.js b/src/backend/src/modules/airouter/samples/openai-1.js
similarity index 100%
rename from src/backend/src/modules/puterai/samples/openai-1.js
rename to src/backend/src/modules/airouter/samples/openai-1.js
diff --git a/src/backend/src/modules/puterai/samples/openai-tools-1.js b/src/backend/src/modules/airouter/samples/openai-tools-1.js
similarity index 100%
rename from src/backend/src/modules/puterai/samples/openai-tools-1.js
rename to src/backend/src/modules/airouter/samples/openai-tools-1.js
diff --git a/src/backend/src/modules/puterai/AIInterfaceService.js b/src/backend/src/modules/puterai/AIInterfaceService.js
index 6f97e592a1..ebe77543c4 100644
--- a/src/backend/src/modules/puterai/AIInterfaceService.js
+++ b/src/backend/src/modules/puterai/AIInterfaceService.js
@@ -58,36 +58,6 @@ class AIInterfaceService extends BaseService {
}
});
- col_interfaces.set('puter-chat-completion', {
- description: 'Chatbot.',
- methods: {
- models: {
- description: 'List supported models and their details.',
- result: { type: 'json' },
- parameters: {},
- },
- list: {
- description: 'List supported models',
- result: { type: 'json' },
- parameters: {},
- },
- complete: {
- description: 'Get completions for a chat log.',
- parameters: {
- messages: { type: 'json' },
- tools: { type: 'json' },
- vision: { type: 'flag' },
- stream: { type: 'flag' },
- response: { type: 'json' },
- model: { type: 'string' },
- temperature: { type: 'number' },
- max_tokens: { type: 'number' },
- },
- result: { type: 'json' },
- }
- }
- });
-
col_interfaces.set('puter-image-generation', {
description: 'AI Image Generation.',
methods: {
diff --git a/src/backend/src/modules/puterai/ClaudeService.js b/src/backend/src/modules/puterai/ClaudeService.js
deleted file mode 100644
index 2e04764dad..0000000000
--- a/src/backend/src/modules/puterai/ClaudeService.js
+++ /dev/null
@@ -1,425 +0,0 @@
-/*
- * Copyright (C) 2024-present Puter Technologies Inc.
- *
- * This file is part of Puter.
- *
- * Puter is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published
- * by the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-
-// METADATA // {"ai-commented":{"service":"claude"}}
-const { default: Anthropic, toFile } = require("@anthropic-ai/sdk");
-const BaseService = require("../../services/BaseService");
-const { TypedValue } = require("../../services/drivers/meta/Runtime");
-const FunctionCalling = require("./lib/FunctionCalling");
-const Messages = require("./lib/Messages");
-const { NodePathSelector } = require("../../filesystem/node/selectors");
-const FSNodeParam = require("../../api/filesystem/FSNodeParam");
-const { LLRead } = require("../../filesystem/ll_operations/ll_read");
-const { Context } = require("../../util/context");
-const { TeePromise } = require('@heyputer/putility').libs.promise;
-
-/**
-* ClaudeService class extends BaseService to provide integration with Anthropic's Claude AI models.
-* Implements the puter-chat-completion interface for handling AI chat interactions.
-* Manages message streaming, token limits, model selection, and API communication with Claude.
-* Supports system prompts, message adaptation, and usage tracking.
-* @extends BaseService
-*/
-class ClaudeService extends BaseService {
- static MODULES = {
- Anthropic: require('@anthropic-ai/sdk'),
- }
-
- /**
- * @type {import('@anthropic-ai/sdk').Anthropic}
- */
- anthropic;
-
-
- /**
- * Initializes the Claude service by creating an Anthropic client instance
- * and registering this service as a provider with the AI chat service.
- * @private
- * @returns {Promise}
- */
- async _init () {
- this.anthropic = new Anthropic({
- apiKey: this.config.apiKey
- });
-
- const svc_aiChat = this.services.get('ai-chat');
- svc_aiChat.register_provider({
- service_name: this.service_name,
- alias: true,
- });
- }
-
-
- /**
- * Returns the default model identifier for Claude API interactions
- * @returns {string} The default model ID 'claude-3-5-sonnet-latest'
- */
- get_default_model () {
- return 'claude-3-5-sonnet-latest';
- }
-
- static IMPLEMENTS = {
- ['puter-chat-completion']: {
- /**
- * Returns a list of available models and their details.
- * See AIChatService for more information.
- *
- * @returns Promise> Array of model details
- */
- async models () {
- return await this.models_();
- },
-
- /**
- * Returns a list of available model names including their aliases
- * @returns {Promise} Array of model identifiers and their aliases
- * @description Retrieves all available model IDs and their aliases,
- * flattening them into a single array of strings that can be used for model selection
- */
- async list () {
- const models = await this.models_();
- const model_names = [];
- for ( const model of models ) {
- model_names.push(model.id);
- if ( model.aliases ) {
- model_names.push(...model.aliases);
- }
- }
- return model_names;
- },
-
- /**
- * Completes a chat interaction with the Claude AI model
- * @param {Object} options - The completion options
- * @param {Array} options.messages - Array of chat messages to process
- * @param {boolean} options.stream - Whether to stream the response
- * @param {string} [options.model] - The Claude model to use, defaults to service default
- * @returns {TypedValue|Object} Returns either a TypedValue with streaming response or a completion object
- * @this {ClaudeService}
- */
- async complete ({ messages, stream, model, tools, max_tokens, temperature}) {
- tools = FunctionCalling.make_claude_tools(tools);
-
- let system_prompts;
- [system_prompts, messages] = Messages.extract_and_remove_system_messages(messages);
-
- const sdk_params = {
- model: model ?? this.get_default_model(),
- max_tokens: Math.floor(max_tokens) ||
- ((
- model === 'claude-3-5-sonnet-20241022'
- || model === 'claude-3-5-sonnet-20240620'
- ) ? 8192 : 4096), //required
- temperature: temperature || 0, // required
- ...(system_prompts ? {
- system: system_prompts.length > 1
- ? JSON.stringify(system_prompts)
- : JSON.stringify(system_prompts[0])
- } : {}),
- messages,
- ...(tools ? { tools } : {}),
- };
-
- console.log('\x1B[26;1m ===== SDK PARAMETERS', require('util').inspect(sdk_params, undefined, Infinity));
-
- let beta_mode = false;
-
- // Perform file uploads
- const file_delete_tasks = [];
- {
- const actor = Context.get('actor');
- const { user } = actor.type;
-
- const file_input_tasks = [];
- for ( const message of messages ) {
- // We can assume `message.content` is not undefined because
- // Messages.normalize_single_message ensures this.
- for ( const contentPart of message.content ) {
- if ( ! contentPart.puter_path ) continue;
- file_input_tasks.push({
- node: await (new FSNodeParam(contentPart.puter_path)).consolidate({
- req: { user },
- getParam: () => contentPart.puter_path,
- }),
- contentPart,
- });
- }
- }
-
- const promises = [];
- for ( const task of file_input_tasks ) promises.push((async () => {
- const ll_read = new LLRead();
- const stream = await ll_read.run({
- actor: Context.get('actor'),
- fsNode: task.node,
- });
-
- const require = this.require;
- const mime = require('mime-types');
- const mimeType = mime.contentType(await task.node.get('name'));
-
- beta_mode = true;
- const fileUpload = await this.anthropic.beta.files.upload({
- file: await toFile(stream, undefined, { type: mimeType })
- }, {
- betas: ['files-api-2025-04-14']
- });
-
- file_delete_tasks.push({ file_id: fileUpload.id });
- // We have to copy a table from the documentation here:
- // https://docs.anthropic.com/en/docs/build-with-claude/files
- const contentBlockTypeForFileBasedOnMime = (() => {
- if ( mimeType.startsWith('image/') ) {
- return 'image';
- }
- if ( mimeType.startsWith('text/') ) {
- return 'document';
- }
- if ( mimeType === 'application/pdf' || mimeType === 'application/x-pdf' ) {
- return 'document';
- }
- return 'container_upload';
- })();
-
- // {
- // 'application/pdf': 'document',
- // 'text/plain': 'document',
- // 'image/': 'image'
- // }[mimeType];
-
-
- delete task.contentPart.puter_path,
- task.contentPart.type = contentBlockTypeForFileBasedOnMime;
- task.contentPart.source = {
- type: 'file',
- file_id: fileUpload.id,
- };
- })());
- await Promise.all(promises);
- }
- const cleanup_files = async () => {
- const promises = [];
- for ( const task of file_delete_tasks ) promises.push((async () => {
- try {
- await this.anthropic.beta.files.delete(
- task.file_id,
- { betas: ['files-api-2025-04-14'] }
- );
- } catch (e) {
- this.errors.report('claude:file-delete-task', {
- source: e,
- trace: true,
- alarm: true,
- extra: { file_id: task.file_id },
- });
- }
- })());
- await Promise.all(promises);
- };
-
-
- if ( beta_mode ) {
- Object.assign(sdk_params, { betas: ['files-api-2025-04-14'] });
- }
- const anthropic = (c => beta_mode ? c.beta : c)(this.anthropic);
-
- if ( stream ) {
- let usage_promise = new TeePromise();
-
- const init_chat_stream = async ({ chatStream }) => {
- const completion = await anthropic.messages.stream(sdk_params);
- const counts = { input_tokens: 0, output_tokens: 0 };
-
- let message, contentBlock;
- for await ( const event of completion ) {
- const input_tokens =
- (event?.usage ?? event?.message?.usage)?.input_tokens;
- const output_tokens =
- (event?.usage ?? event?.message?.usage)?.output_tokens;
-
- if ( input_tokens ) counts.input_tokens += input_tokens;
- if ( output_tokens ) counts.output_tokens += output_tokens;
-
- if ( event.type === 'message_start' ) {
- message = chatStream.message();
- continue;
- }
- if ( event.type === 'message_stop' ) {
- message.end();
- message = null;
- continue;
- }
-
- if ( event.type === 'content_block_start' ) {
- if ( event.content_block.type === 'tool_use' ) {
- contentBlock = message.contentBlock({
- type: event.content_block.type,
- id: event.content_block.id,
- name: event.content_block.name,
- });
- continue;
- }
- contentBlock = message.contentBlock({
- type: event.content_block.type,
- });
- continue;
- }
-
- if ( event.type === 'content_block_stop' ) {
- contentBlock.end();
- contentBlock = null;
- continue;
- }
-
- if ( event.type === 'content_block_delta' ) {
- if ( event.delta.type === 'input_json_delta' ) {
- contentBlock.addPartialJSON(event.delta.partial_json);
- continue;
- }
- if ( event.delta.type === 'text_delta' ) {
- contentBlock.addText(event.delta.text);
- continue;
- }
- }
- }
- chatStream.end();
- usage_promise.resolve(counts);
- };
-
- return new TypedValue({ $: 'ai-chat-intermediate' }, {
- init_chat_stream,
- stream: true,
- usage_promise: usage_promise,
- finally_fn: cleanup_files,
- });
- }
-
- const msg = await anthropic.messages.create(sdk_params);
- await cleanup_files();
-
- return {
- message: msg,
- usage: msg.usage,
- finish_reason: 'stop'
- };
- }
- }
- }
-
-
- /**
- * Retrieves available Claude AI models and their specifications
- * @returns {Promise} Array of model objects containing:
- * - id: Model identifier
- * - name: Display name
- * - aliases: Alternative names for the model
- * - context: Maximum context window size
- * - cost: Pricing details (currency, token counts, input/output costs)
- * - qualitative_speed: Relative speed rating
- * - max_output: Maximum output tokens
- * - training_cutoff: Training data cutoff date
- */
- async models_ () {
- return [
- {
- id: 'claude-opus-4-20250514',
- aliases: ['claude-opus-4', 'claude-opus-4-latest'],
- name: 'Claude Opus 4',
- context: 200000,
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 1500,
- output: 7500,
- },
- max_tokens: 32000,
- },
- {
- id: 'claude-sonnet-4-20250514',
- aliases: ['claude-sonnet-4', 'claude-sonnet-4-latest'],
- name: 'Claude Sonnet 4',
- context: 200000,
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 300,
- output: 1500,
- },
- max_tokens: 64000,
- },
- {
- id: 'claude-3-7-sonnet-20250219',
- aliases: ['claude-3-7-sonnet-latest'],
- succeeded_by: 'claude-sonnet-4-20250514',
- context: 200000,
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 300,
- output: 1500,
- },
- max_tokens: 8192,
- },
- {
- id: 'claude-3-5-sonnet-20241022',
- name: 'Claude 3.5 Sonnet',
- aliases: ['claude-3-5-sonnet-latest'],
- context: 200000,
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 300,
- output: 1500,
- },
- qualitative_speed: 'fast',
- training_cutoff: '2024-04',
- max_tokens: 8192,
- },
- {
- id: 'claude-3-5-sonnet-20240620',
- succeeded_by: 'claude-3-5-sonnet-20241022',
- context: 200000, // might be wrong
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 300,
- output: 1500,
- },
- max_tokens: 8192,
- },
- {
- id: 'claude-3-haiku-20240307',
- // aliases: ['claude-3-haiku-latest'],
- context: 200000,
- cost: {
- currency: 'usd-cents',
- tokens: 1_000_000,
- input: 25,
- output: 125,
- },
- qualitative_speed: 'fastest',
- max_tokens: 4096,
- },
- ];
- }
-}
-
-module.exports = {
- ClaudeService,
-};
diff --git a/src/backend/src/modules/puterai/PuterAIModule.js b/src/backend/src/modules/puterai/PuterAIModule.js
index 67e3d79a83..9629b9eb60 100644
--- a/src/backend/src/modules/puterai/PuterAIModule.js
+++ b/src/backend/src/modules/puterai/PuterAIModule.js
@@ -57,62 +57,9 @@ class PuterAIModule extends AdvancedBase {
}
if ( !! config?.openai ) {
- const { OpenAICompletionService } = require('./OpenAICompletionService');
- services.registerService('openai-completion', OpenAICompletionService);
-
const { OpenAIImageGenerationService } = require('./OpenAIImageGenerationService');
services.registerService('openai-image-generation', OpenAIImageGenerationService);
}
-
- if ( !! config?.services?.claude ) {
- const { ClaudeService } = require('./ClaudeService');
- services.registerService('claude', ClaudeService);
- }
-
- if ( !! config?.services?.['together-ai'] ) {
- const { TogetherAIService } = require('./TogetherAIService');
- services.registerService('together-ai', TogetherAIService);
- }
-
- if ( !! config?.services?.['mistral'] ) {
- const { MistralAIService } = require('./MistralAIService');
- services.registerService('mistral', MistralAIService);
- }
-
- if ( !! config?.services?.['groq'] ) {
- const { GroqAIService } = require('./GroqAIService');
- services.registerService('groq', GroqAIService);
- }
-
- if ( !! config?.services?.['xai'] ) {
- const { XAIService } = require('./XAIService');
- services.registerService('xai', XAIService);
- }
-
- if ( !! config?.services?.['deepseek'] ) {
- const { DeepSeekService } = require('./DeepSeekService');
- services.registerService('deepseek', DeepSeekService);
- }
- if ( !! config?.services?.['gemini'] ) {
- const { GeminiService } = require('./GeminiService');
- services.registerService('gemini', GeminiService);
- }
- if ( !! config?.services?.['openrouter'] ) {
- const { OpenRouterService } = require('./OpenRouterService');
- services.registerService('openrouter', OpenRouterService);
- }
-
- const { AIChatService } = require('./AIChatService');
- services.registerService('ai-chat', AIChatService);
-
- const { FakeChatService } = require('./FakeChatService');
- services.registerService('fake-chat', FakeChatService);
-
- const{ AITestModeService } = require('./AITestModeService');
- services.registerService('ai-test-mode', AITestModeService);
-
- const { UsageLimitedChatService } = require('./UsageLimitedChatService');
- services.registerService('usage-limited-chat', UsageLimitedChatService);
}
}
diff --git a/src/backend/src/modules/puterai/lib/OpenAIUtil.js b/src/backend/src/modules/puterai/lib/OpenAIUtil.js
deleted file mode 100644
index e7b80367a2..0000000000
--- a/src/backend/src/modules/puterai/lib/OpenAIUtil.js
+++ /dev/null
@@ -1,228 +0,0 @@
-const putility = require("@heyputer/putility");
-const { TypedValue } = require("../../../services/drivers/meta/Runtime");
-const { nou } = require("../../../util/langutil");
-
-module.exports = class OpenAIUtil {
- /**
- * Process input messages from Puter's normalized format to OpenAI's format
- * May make changes in-place.
- *
- * @param {Array} messages - array of normalized messages
- * @returns {Array} - array of messages in OpenAI format
- */
- static process_input_messages = async (messages) => {
- for ( const msg of messages ) {
- if ( ! msg.content ) continue;
- if ( typeof msg.content !== 'object' ) continue;
-
- const content = msg.content;
-
- for ( const o of content ) {
- if ( ! o.hasOwnProperty('image_url') ) continue;
- if ( o.type ) continue;
- o.type = 'image_url';
- }
-
- // coerce tool calls
- let is_tool_call = false;
- for ( let i = content.length - 1 ; i >= 0 ; i-- ) {
- const content_block = content[i];
-
- if ( content_block.type === 'tool_use' ) {
- if ( ! msg.hasOwnProperty('tool_calls') ) {
- msg.tool_calls = [];
- is_tool_call = true;
- }
- msg.tool_calls.push({
- id: content_block.id,
- type: 'function',
- function: {
- name: content_block.name,
- arguments: JSON.stringify(content_block.input),
- }
- });
- content.splice(i, 1);
- }
- }
-
- if ( is_tool_call ) msg.content = null;
-
- // coerce tool results
- // (we assume multiple tool results were already split into separate messages)
- for ( let i = content.length - 1 ; i >= 0 ; i-- ) {
- const content_block = content[i];
- if ( content_block.type !== 'tool_result' ) continue;
- msg.role = 'tool';
- msg.tool_call_id = content_block.tool_use_id;
- msg.content = content_block.content;
- }
- }
-
- return messages;
- }
-
- static create_usage_calculator = ({ model_details }) => {
- return ({ usage }) => {
- const tokens = [];
-
- tokens.push({
- type: 'prompt',
- model: model_details.id,
- amount: usage.prompt_tokens,
- cost: model_details.cost.input * usage.prompt_tokens,
- });
-
- tokens.push({
- type: 'completion',
- model: model_details.id,
- amount: usage.completion_tokens,
- cost: model_details.cost.output * usage.completion_tokens,
- });
-
- return tokens;
- };
- };
-
- static create_chat_stream_handler = ({
- deviations,
- completion, usage_promise,
- }) => async ({ chatStream }) => {
- deviations = Object.assign({
- // affected by: Groq
- index_usage_from_stream_chunk: chunk => chunk.usage,
- // affected by: Mistral
- chunk_but_like_actually: chunk => chunk,
- index_tool_calls_from_stream_choice: choice => choice.delta.tool_calls,
- }, deviations);
-
- const message = chatStream.message();
- let textblock = message.contentBlock({ type: 'text' });
- let toolblock = null;
- let mode = 'text';
- const tool_call_blocks = [];
-
- let last_usage = null;
- for await ( let chunk of completion ) {
- chunk = deviations.chunk_but_like_actually(chunk);
- if ( process.env.DEBUG ) {
- const delta = chunk?.choices?.[0]?.delta;
- console.log(
- `AI CHUNK`,
- chunk,
- delta && JSON.stringify(delta)
- );
- }
- const chunk_usage = deviations.index_usage_from_stream_chunk(chunk);
- if ( chunk_usage ) last_usage = chunk_usage;
- if ( chunk.choices.length < 1 ) continue;
-
- const choice = chunk.choices[0];
-
- if ( ! nou(choice.delta.content) ) {
- if ( mode === 'tool' ) {
- toolblock.end();
- mode = 'text';
- textblock = message.contentBlock({ type: 'text' });
- }
- textblock.addText(choice.delta.content);
- continue;
- }
-
- const tool_calls = deviations.index_tool_calls_from_stream_choice(choice);
- if ( ! nou(tool_calls) ) {
- if ( mode === 'text' ) {
- mode = 'tool';
- textblock.end();
- }
- for ( const tool_call of tool_calls ) {
- if ( ! tool_call_blocks[tool_call.index] ) {
- toolblock = message.contentBlock({
- type: 'tool_use',
- id: tool_call.id,
- name: tool_call.function.name,
- });
- tool_call_blocks[tool_call.index] = toolblock;
- } else {
- toolblock = tool_call_blocks[tool_call.index];
- }
- toolblock.addPartialJSON(tool_call.function.arguments);
- }
- }
- }
- usage_promise.resolve(last_usage);
-
- if ( mode === 'text' ) textblock.end();
- if ( mode === 'tool' ) toolblock.end();
- message.end();
- chatStream.end();
- };
-
- static async handle_completion_output ({
- deviations,
- stream, completion, moderate,
- usage_calculator,
- finally_fn,
- }) {
- deviations = Object.assign({
- // affected by: Mistral
- coerce_completion_usage: completion => completion.usage,
- }, deviations);
-
- if ( stream ) {
- let usage_promise = new putility.libs.promise.TeePromise();
-
- const init_chat_stream =
- OpenAIUtil.create_chat_stream_handler({
- deviations,
- completion,
- usage_promise,
- usage_calculator,
- });
-
- return new TypedValue({ $: 'ai-chat-intermediate' }, {
- stream: true,
- init_chat_stream,
- finally_fn,
- usage_promise: usage_promise.then(usage => {
- return usage_calculator ? usage_calculator({ usage }) : {
- input_tokens: usage.prompt_tokens,
- output_tokens: usage.completion_tokens,
- };
- }),
- });
- }
-
- if ( finally_fn ) await finally_fn();
-
- const is_empty = completion.choices?.[0]?.message?.content?.trim() === '';
- if ( is_empty && ! completion.choices?.[0]?.message?.tool_calls ) {
- // GPT refuses to generate an empty response if you ask it to,
- // so this will probably only happen on an error condition.
- throw new Error('an empty response was generated');
- }
-
- // We need to moderate the completion too
- const mod_text = completion.choices[0].message.content;
- if ( moderate && mod_text !== null ) {
- const moderation_result = await moderate(mod_text);
- if ( moderation_result.flagged ) {
- throw new Error('message is not allowed');
- }
- }
-
- const ret = completion.choices[0];
- const completion_usage = deviations.coerce_completion_usage(completion);
- ret.usage = usage_calculator ? usage_calculator({
- ...completion,
- usage: completion_usage,
- }) : {
- input_tokens: completion_usage.prompt_tokens,
- output_tokens: completion_usage.completion_tokens,
- };
- // TODO: turn these into toggle logs
- // console.log('ORIGINAL COMPLETION', completion);
- // console.log('COMPLETION USAGE', completion_usage);
- // console.log('RETURN VALUE', ret);
- return ret;
- }
-};
\ No newline at end of file
diff --git a/src/backend/src/modules/puterai/lib/Streaming.js b/src/backend/src/modules/puterai/lib/Streaming.js
deleted file mode 100644
index 5bbfb8bff5..0000000000
--- a/src/backend/src/modules/puterai/lib/Streaming.js
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Assign the properties of the override object to the original object,
- * like Object.assign, except properties are ordered so override properties
- * are enumerated first.
- *
- * @param {*} original
- * @param {*} override
- */
-const objectAssignTop = (original, override) => {
- let o = {
- ...original,
- ...override,
- };
- o = {
- ...override,
- ...original,
- };
- return o;
-}
-
-class AIChatConstructStream {
- constructor (chatStream, params) {
- this.chatStream = chatStream;
- if ( this._start ) this._start(params);
- }
- end () {
- if ( this._end ) this._end();
- }
-}
-
-class AIChatTextStream extends AIChatConstructStream {
- addText (text) {
- const json = JSON.stringify({
- type: 'text', text,
- });
- this.chatStream.stream.write(json + '\n');
- }
-}
-
-class AIChatToolUseStream extends AIChatConstructStream {
- _start (params) {
- this.contentBlock = params;
- this.buffer = '';
- }
- addPartialJSON (partial_json) {
- this.buffer += partial_json;
- }
- _end () {
- if ( this.buffer.trim() === '' ) {
- this.buffer = '{}';
- }
- if ( process.env.DEBUG ) console.log('BUFFER BEING PARSED', this.buffer);
- const str = JSON.stringify(objectAssignTop({
- ...this.contentBlock,
- input: JSON.parse(this.buffer),
- ...( ! this.contentBlock.text ? { text: "" } : {}),
- }, {
- type: 'tool_use',
- }));
- this.chatStream.stream.write(str + '\n');
- }
-}
-
-class AIChatMessageStream extends AIChatConstructStream {
- contentBlock ({ type, ...params }) {
- if ( type === 'tool_use' ) {
- return new AIChatToolUseStream(this.chatStream, params);
- }
- if ( type === 'text' ) {
- return new AIChatTextStream(this.chatStream, params);
- }
- throw new Error(`Unknown content block type: ${type}`);
- }
-}
-
-class AIChatStream {
- constructor ({ stream }) {
- this.stream = stream;
- }
-
- end () {
- this.stream.end();
- }
-
- message () {
- return new AIChatMessageStream(this);
- }
-}
-
-module.exports = class Streaming {
- static AIChatStream = AIChatStream;
-}
diff --git a/src/backend/src/modules/puterai/lib/messages.test.js b/src/backend/src/modules/puterai/lib/messages.test.js
deleted file mode 100644
index 3a4c7b4e3e..0000000000
--- a/src/backend/src/modules/puterai/lib/messages.test.js
+++ /dev/null
@@ -1,184 +0,0 @@
-import { describe, it, expect } from 'vitest';
-const Messages = require('./Messages.js');
-const OpenAIUtil = require('./OpenAIUtil.js');
-
-describe('Messages', () => {
- describe('normalize_single_message', () => {
- const cases = [
- {
- name: 'string message',
- input: 'Hello, world!',
- output: {
- role: 'user',
- content: [
- {
- type: 'text',
- text: 'Hello, world!',
- }
- ]
- }
- }
- ];
- for ( const tc of cases ) {
- it(`should normalize ${tc.name}`, () => {
- const output = Messages.normalize_single_message(tc.input);
- expect(output).toEqual(tc.output);
- });
- }
- });
- describe('extract_text', () => {
- const cases = [
- {
- name: 'string message',
- input: ['Hello, world!'],
- output: 'Hello, world!',
- },
- {
- name: 'object message',
- input: [{
- content: [
- {
- type: 'text',
- text: 'Hello, world!',
- }
- ]
- }],
- output: 'Hello, world!',
- },
- {
- name: 'irregular messages',
- input: [
- 'First Part',
- {
- content: [
- {
- type: 'text',
- text: 'Second Part',
- }
- ]
- },
- {
- content: 'Third Part',
- }
- ],
- output: 'First Part Second Part Third Part',
- }
- ];
- for ( const tc of cases ) {
- it(`should extract text from ${tc.name}`, () => {
- const output = Messages.extract_text(tc.input);
- expect(output).toBe(tc.output);
- });
- }
- });
- describe('normalize OpenAI tool calls', () => {
- const cases = [
- {
- name: 'string message',
- input: {
- role: 'assistant',
- tool_calls: [
- {
- id: 'tool-1',
- type: 'function',
- function: {
- name: 'tool-1-function',
- arguments: {},
- }
- }
- ]
- },
- output: {
- role: 'assistant',
- content: [
- {
- type: 'tool_use',
- id: 'tool-1',
- name: 'tool-1-function',
- input: {},
- }
- ]
- }
- }
- ];
- for ( const tc of cases ) {
- it(`should normalize ${tc.name}`, () => {
- const output = Messages.normalize_single_message(tc.input);
- expect(output).toEqual(tc.output);
- });
- }
- });
- describe('normalize Claude tool calls', () => {
- const cases = [
- {
- name: 'string message',
- input: {
- role: 'assistant',
- content: [
- {
- type: 'tool_use',
- id: 'tool-1',
- name: 'tool-1-function',
- input: "{}",
- }
- ]
- },
- output: {
- role: 'assistant',
- content: [
- {
- type: 'tool_use',
- id: 'tool-1',
- name: 'tool-1-function',
- input: "{}",
- }
- ]
- }
- }
- ];
- for ( const tc of cases ) {
- it(`should normalize ${tc.name}`, () => {
- const output = Messages.normalize_single_message(tc.input);
- expect(output).toEqual(tc.output);
- });
- }
- });
- describe('OpenAI-ify normalized tool calls', () => {
- const cases = [
- {
- name: 'string message',
- input: [{
- role: 'assistant',
- content: [
- {
- type: 'tool_use',
- id: 'tool-1',
- name: 'tool-1-function',
- input: {},
- }
- ]
- }],
- output: [{
- role: 'assistant',
- content: null,
- tool_calls: [
- {
- id: 'tool-1',
- type: 'function',
- function: {
- name: 'tool-1-function',
- arguments: '{}',
- }
- }
- ]
- }]
- }
- ];
- for ( const tc of cases ) {
- it(`should normalize ${tc.name}`, async () => {
- const output = await OpenAIUtil.process_input_messages(tc.input);
- expect(output).toEqual(tc.output);
- });
- }
- });
-});
\ No newline at end of file
diff --git a/tools/run-selfhosted.js b/tools/run-selfhosted.js
index 9c0f085145..dde83f2d7f 100644
--- a/tools/run-selfhosted.js
+++ b/tools/run-selfhosted.js
@@ -87,6 +87,7 @@ const main = async () => {
BroadcastModule,
TestDriversModule,
PuterAIModule,
+ AIRouterModule,
InternetModule,
DevelopmentModule,
} = (await import('@heyputer/backend')).default;
@@ -103,6 +104,7 @@ const main = async () => {
k.add_module(new BroadcastModule());
k.add_module(new TestDriversModule());
k.add_module(new PuterAIModule());
+ k.add_module(new AIRouterModule());
k.add_module(new InternetModule());
if ( process.env.UNSAFE_PUTER_DEV ) {
k.add_module(new DevelopmentModule());