From 8006795897c54180823323357ad5a74c8660fe25 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Wed, 2 Oct 2024 01:00:48 +0300
Subject: [PATCH 01/50] New tool calling framework

---
 public/global.d.ts                            |  41 --
 public/script.js                              |  35 +-
 .../scripts/extensions/expressions/index.js   |  52 +--
 public/scripts/openai.js                      | 149 +------
 public/scripts/tool-calling.js                | 381 ++++++++++++++++++
 src/endpoints/backends/chat-completions.js    |  22 +-
 6 files changed, 427 insertions(+), 253 deletions(-)
 create mode 100644 public/scripts/tool-calling.js

diff --git a/public/global.d.ts b/public/global.d.ts
index c8bfe14c2..39ba4ea0c 100644
--- a/public/global.d.ts
+++ b/public/global.d.ts
@@ -1365,44 +1365,3 @@ declare namespace moment {
 declare global {
   const moment: typeof moment;
 }
-
-/**
- * Callback data for the `LLM_FUNCTION_TOOL_REGISTER` event type that is triggered when a function tool can be registered.
- */
-interface FunctionToolRegister {
-    /**
-     * The type of generation that is being used
-     */
-    type?: string;
-    /**
-     * Generation data, including messages and sampling parameters
-     */
-    data: Record<string, object>;
-    /**
-     * Callback to register an LLM function tool.
-     */
-    registerFunctionTool: typeof registerFunctionTool;
-}
-
-/**
- * Callback data for the `LLM_FUNCTION_TOOL_REGISTER` event type that is triggered when a function tool is registered.
- * @param name Name of the function tool to register
- * @param description Description of the function tool
- * @param params JSON schema for the parameters of the function tool
- * @param required Whether the function tool should be forced to be used
- */
-declare function registerFunctionTool(name: string, description: string, params: object, required: boolean): Promise<void>;
-
-/**
- * Callback data for the `LLM_FUNCTION_TOOL_CALL` event type that is triggered when a function tool is called.
- */
-interface FunctionToolCall {
-    /**
-     * Name of the function tool to call
-     */
-    name: string;
-    /**
-     * JSON object with the parameters to pass to the function tool
-     */
-    arguments: string;
-}
diff --git a/public/script.js b/public/script.js
index 35bfcbab0..fac11b8f4 100644
--- a/public/script.js
+++ b/public/script.js
@@ -246,6 +246,7 @@ import { initInputMarkdown } from './scripts/input-md-formatting.js';
 import { AbortReason } from './scripts/util/AbortReason.js';
 import { initSystemPrompts } from './scripts/sysprompt.js';
 import { registerExtensionSlashCommands as initExtensionSlashCommands } from './scripts/extensions-slashcommands.js';
+import { ToolManager } from './scripts/tool-calling.js';
 
 //exporting functions and vars for mods
 export {
@@ -463,8 +464,6 @@ export const event_types = {
     FILE_ATTACHMENT_DELETED: 'file_attachment_deleted',
     WORLDINFO_FORCE_ACTIVATE: 'worldinfo_force_activate',
     OPEN_CHARACTER_LIBRARY: 'open_character_library',
-    LLM_FUNCTION_TOOL_REGISTER: 'llm_function_tool_register',
-    LLM_FUNCTION_TOOL_CALL: 'llm_function_tool_call',
     ONLINE_STATUS_CHANGED: 'online_status_changed',
     IMAGE_SWIPED: 'image_swiped',
     CONNECTION_PROFILE_LOADED: 'connection_profile_loaded',
@@ -2921,6 +2920,7 @@ class StreamingProcessor {
         this.swipes = [];
         /** @type {import('./scripts/logprobs.js').TokenLogprobs[]} */
         this.messageLogprobs = [];
+        this.toolCalls = [];
     }
 
     #checkDomElements(messageId) {
@@ -3139,7 +3139,7 @@ class StreamingProcessor {
     }
 
     /**
-     * @returns {Generator<{ text: string, swipes: string[], logprobs: import('./scripts/logprobs.js').TokenLogprobs }, void, void>}
+     * @returns {Generator<{ text: string, swipes: string[], logprobs: import('./scripts/logprobs.js').TokenLogprobs, toolCalls: any[] }, void, void>}
      */
     *nullStreamingGeneration() {
         throw new Error('Generation function for streaming is not hooked up');
@@ -3161,12 +3161,13 @@ class StreamingProcessor {
         try {
             const sw = new Stopwatch(1000 / power_user.streaming_fps);
             const timestamps = [];
-            for await (const { text, swipes, logprobs } of this.generator()) {
+            for await (const { text, swipes, logprobs, toolCalls } of this.generator()) {
                 timestamps.push(Date.now());
                 if (this.isStopped) {
                     return;
                 }
 
+                this.toolCalls = toolCalls;
                 this.result = text;
                 this.swipes = Array.from(swipes ?? []);
                 if (logprobs) {
@@ -4405,6 +4406,20 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
                 getMessage = continue_mag + getMessage;
             }
 
+            if (ToolManager.isFunctionCallingSupported() && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length) {
+                const invocations = await ToolManager.checkFunctionToolCalls(streamingProcessor.toolCalls);
+                if (invocations.length) {
+                    const lastMessage = chat[chat.length - 1];
+                    const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor.result);
+                    if (shouldDeleteMessage) {
+                        await deleteLastMessage();
+                        streamingProcessor = null;
+                    }
+                    ToolManager.saveFunctionToolInvocations(invocations);
+                    return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
+                }
+            }
+
             if (streamingProcessor && !streamingProcessor.isStopped && streamingProcessor.isFinished) {
                 await streamingProcessor.onFinishStreaming(streamingProcessor.messageId, getMessage);
                 streamingProcessor = null;
@@ -4440,6 +4455,14 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
             throw new Error(data?.response);
         }
 
+        if (ToolManager.isFunctionCallingSupported()) {
+            const invocations = await ToolManager.checkFunctionToolCalls(data);
+            if (invocations.length) {
+                ToolManager.saveFunctionToolInvocations(invocations);
+                return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
+            }
+        }
+
         //const getData = await response.json();
         let getMessage = extractMessageFromData(data);
         let title = extractTitleFromData(data);
@@ -7853,7 +7876,7 @@ function openAlternateGreetings() {
             if (menu_type !== 'create') {
                 await createOrEditCharacter();
             }
-        }
+        },
     });
 
     for (let index = 0; index < getArray().length; index++) {
@@ -8130,6 +8153,8 @@ window['SillyTavern'].getContext = function () {
         registerHelper: () => { },
         registerMacro: MacrosParser.registerMacro.bind(MacrosParser),
         unregisterMacro: MacrosParser.unregisterMacro.bind(MacrosParser),
+        registerFunctionTool: ToolManager.registerFunctionTool.bind(ToolManager),
+        unregisterFunctionTool: ToolManager.unregisterFunctionTool.bind(ToolManager),
         registerDebugFunction: registerDebugFunction,
         /** @deprecated Use renderExtensionTemplateAsync instead. */
         renderExtensionTemplate: renderExtensionTemplate,
diff --git a/public/scripts/extensions/expressions/index.js b/public/scripts/extensions/expressions/index.js
index 4a09e3aeb..078375cf1 100644
--- a/public/scripts/extensions/expressions/index.js
+++ b/public/scripts/extensions/expressions/index.js
@@ -9,7 +9,6 @@ import { debounce_timeout } from '../../constants.js';
 import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
 import { SlashCommand } from '../../slash-commands/SlashCommand.js';
 import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
-import { isFunctionCallingSupported } from '../../openai.js';
 import { SlashCommandEnumValue, enumTypes } from '../../slash-commands/SlashCommandEnumValue.js';
 import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
 import { slashCommandReturnHelper } from '../../slash-commands/SlashCommandReturnHelper.js';
@@ -21,7 +20,6 @@ const UPDATE_INTERVAL = 2000;
 const STREAMING_UPDATE_INTERVAL = 10000;
 const TALKINGCHECK_UPDATE_INTERVAL = 500;
 const DEFAULT_FALLBACK_EXPRESSION = 'joy';
-const FUNCTION_NAME = 'set_emotion';
 const DEFAULT_LLM_PROMPT = 'Ignore previous instructions. Classify the emotion of the last message. Output just one word, e.g. "joy" or "anger". Choose only one of the following labels: {{labels}}';
 const DEFAULT_EXPRESSIONS = [
     'talkinghead',
@@ -1017,10 +1015,6 @@ async function getLlmPrompt(labels) {
         return '';
     }
 
-    if (isFunctionCallingSupported()) {
-        return '';
-    }
-
     const labelsString = labels.map(x => `"${x}"`).join(', ');
     const prompt = substituteParamsExtended(String(extension_settings.expressions.llmPrompt), { labels: labelsString });
     return prompt;
@@ -1056,41 +1050,6 @@ function parseLlmResponse(emotionResponse, labels) {
     throw new Error('Could not parse emotion response ' + emotionResponse);
 }
 
-/**
- * Registers the function tool for the LLM API.
- * @param {FunctionToolRegister} args Function tool register arguments.
- */
-function onFunctionToolRegister(args) {
-    if (inApiCall && extension_settings.expressions.api === EXPRESSION_API.llm && isFunctionCallingSupported()) {
-        // Only trigger on quiet mode
-        if (args.type !== 'quiet') {
-            return;
-        }
-
-        const emotions = DEFAULT_EXPRESSIONS.filter((e) => e != 'talkinghead');
-        const jsonSchema = {
-            $schema: 'http://json-schema.org/draft-04/schema#',
-            type: 'object',
-            properties: {
-                emotion: {
-                    type: 'string',
-                    enum: emotions,
-                    description: `One of the following: ${JSON.stringify(emotions)}`,
-                },
-            },
-            required: [
-                'emotion',
-            ],
-        };
-        args.registerFunctionTool(
-            FUNCTION_NAME,
-            substituteParams('Sets the label that best describes the current emotional state of {{char}}. Only select one of the enumerated values.'),
-            jsonSchema,
-            true,
-        );
-    }
-}
-
 function onTextGenSettingsReady(args) {
     // Only call if inside an API call
     if (inApiCall && extension_settings.expressions.api === EXPRESSION_API.llm && isJsonSchemaSupported()) {
@@ -1164,18 +1123,9 @@ export async function getExpressionLabel(text, expressionsApi = extension_settin
 
                 const expressionsList = await getExpressionsList();
                 const prompt = substituteParamsExtended(customPrompt, { labels: expressionsList }) || await getLlmPrompt(expressionsList);
-                let functionResult = null;
                 eventSource.once(event_types.TEXT_COMPLETION_SETTINGS_READY, onTextGenSettingsReady);
-                eventSource.once(event_types.LLM_FUNCTION_TOOL_REGISTER, onFunctionToolRegister);
-                eventSource.once(event_types.LLM_FUNCTION_TOOL_CALL, (/** @type {FunctionToolCall} */ args) => {
-                    if (args.name !== FUNCTION_NAME) {
-                        return;
-                    }
-
-                    functionResult = args?.arguments;
-                });
                 const emotionResponse = await generateRaw(text, main_api, false, false, prompt);
-                return parseLlmResponse(functionResult || emotionResponse, expressionsList);
+                return parseLlmResponse(emotionResponse, expressionsList);
             }
             // Extras
             default: {
diff --git a/public/scripts/openai.js b/public/scripts/openai.js
index 20c141316..a6048dce5 100644
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -70,6 +70,7 @@ import { renderTemplateAsync } from './templates.js';
 import { SlashCommandEnumValue } from './slash-commands/SlashCommandEnumValue.js';
 import { Popup, POPUP_RESULT } from './popup.js';
 import { t } from './i18n.js';
+import { ToolManager } from './tool-calling.js';
 
 export {
     openai_messages_count,
@@ -1863,8 +1864,8 @@ async function sendOpenAIRequest(type, messages, signal) {
         generate_data['seed'] = oai_settings.seed;
     }
 
-    if (isFunctionCallingSupported() && !stream) {
-        await registerFunctionTools(type, generate_data);
+    if (!canMultiSwipe && ToolManager.isFunctionCallingSupported()) {
+        await ToolManager.registerFunctionToolsOpenAI(generate_data);
     }
 
     if (isOAI && oai_settings.openai_model.startsWith('o1-')) {
@@ -1911,6 +1912,7 @@ async function sendOpenAIRequest(type, messages, signal) {
         return async function* streamData() {
             let text = '';
             const swipes = [];
+            const toolCalls = [];
             while (true) {
                 const { done, value } = await reader.read();
                 if (done) return;
@@ -1926,7 +1928,9 @@ async function sendOpenAIRequest(type, messages, signal) {
                     text += getStreamingReply(parsed);
                 }
 
-                yield { text, swipes: swipes, logprobs: parseChatCompletionLogprobs(parsed) };
+                ToolManager.parseToolCalls(toolCalls, parsed);
+
+                yield { text, swipes: swipes, logprobs: parseChatCompletionLogprobs(parsed), toolCalls: toolCalls };
             }
         };
     }
@@ -1948,147 +1952,10 @@ async function sendOpenAIRequest(type, messages, signal) {
             delay(1).then(() => saveLogprobsForActiveMessage(logprobs, null));
         }
 
-        if (isFunctionCallingSupported()) {
-            await checkFunctionToolCalls(data);
-        }
-
         return data;
     }
 }
 
-/**
- * Register function tools for the next chat completion request.
- * @param {string} type Generation type
- * @param {object} data Generation data
- */
-async function registerFunctionTools(type, data) {
-    let toolChoice = 'auto';
-    const tools = [];
-
-    /**
-     * @type {registerFunctionTool}
-     */
-    const registerFunctionTool = (name, description, parameters, required) => {
-        tools.push({
-            type: 'function',
-            function: {
-                name,
-                description,
-                parameters,
-            },
-        });
-
-        if (required) {
-            toolChoice = 'required';
-        }
-    };
-
-    /**
-     * @type {FunctionToolRegister}
-     */
-    const args = {
-        type,
-        data,
-        registerFunctionTool,
-    };
-
-    await eventSource.emit(event_types.LLM_FUNCTION_TOOL_REGISTER, args);
-
-    if (tools.length) {
-        console.log('Registered function tools:', tools);
-
-        data['tools'] = tools;
-        data['tool_choice'] = toolChoice;
-    }
-}
-
-async function checkFunctionToolCalls(data) {
-    const oaiCompat = [
-        chat_completion_sources.OPENAI,
-        chat_completion_sources.CUSTOM,
-        chat_completion_sources.MISTRALAI,
-        chat_completion_sources.OPENROUTER,
-        chat_completion_sources.GROQ,
-    ];
-    if (oaiCompat.includes(oai_settings.chat_completion_source)) {
-        if (!Array.isArray(data?.choices)) {
-            return;
-        }
-
-        // Find a choice with 0-index
-        const choice = data.choices.find(choice => choice.index === 0);
-
-        if (!choice) {
-            return;
-        }
-
-        const toolCalls = choice.message.tool_calls;
-
-        if (!Array.isArray(toolCalls)) {
-            return;
-        }
-
-        for (const toolCall of toolCalls) {
-            if (typeof toolCall.function !== 'object') {
-                continue;
-            }
-
-            /** @type {FunctionToolCall} */
-            const args = toolCall.function;
-            console.log('Function tool call:', toolCall);
-            await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
-        }
-    }
-
-    if ([chat_completion_sources.CLAUDE].includes(oai_settings.chat_completion_source)) {
-        if (!Array.isArray(data?.content)) {
-            return;
-        }
-
-        for (const content of data.content) {
-            if (content.type === 'tool_use') {
-                /** @type {FunctionToolCall} */
-                const args = { name: content.name, arguments: JSON.stringify(content.input) };
-                await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
-            }
-        }
-    }
-
-    if ([chat_completion_sources.COHERE].includes(oai_settings.chat_completion_source)) {
-        if (!Array.isArray(data?.tool_calls)) {
-            return;
-        }
-
-        for (const toolCall of data.tool_calls) {
-            /** @type {FunctionToolCall} */
-            const args = { name: toolCall.name, arguments: JSON.stringify(toolCall.parameters) };
-            console.log('Function tool call:', toolCall);
-            await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args);
-        }
-    }
-}
-
-export function isFunctionCallingSupported() {
-    if (main_api !== 'openai') {
-        return false;
-    }
-
-    if (!oai_settings.function_calling) {
-        return false;
-    }
-
-    const supportedSources = [
-        chat_completion_sources.OPENAI,
-        chat_completion_sources.COHERE,
-        chat_completion_sources.CUSTOM,
-        chat_completion_sources.MISTRALAI,
-        chat_completion_sources.CLAUDE,
-        chat_completion_sources.OPENROUTER,
-        chat_completion_sources.GROQ,
-    ];
-    return supportedSources.includes(oai_settings.chat_completion_source);
-}
-
 function getStreamingReply(data) {
     if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {
         return data?.delta?.text || '';
@@ -4019,7 +3886,7 @@ async function onModelChange() {
             $('#openai_max_context').attr('max', max_32k);
         } else if (value === 'text-bison-001') {
             $('#openai_max_context').attr('max', max_8k);
-        // The ultra endpoints are possibly dead:
+            // The ultra endpoints are possibly dead:
         } else if (value.includes('gemini-1.0-ultra') || value === 'gemini-ultra') {
             $('#openai_max_context').attr('max', max_32k);
         } else {
diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
new file mode 100644
index 000000000..c2f6c209f
--- /dev/null
+++ b/public/scripts/tool-calling.js
@@ -0,0 +1,381 @@
+import { chat, main_api } from '../script.js';
+import { chat_completion_sources, oai_settings } from './openai.js';
+
+/**
+ * @typedef {object} ToolInvocation
+ * @property {string} id - A unique identifier for the tool invocation.
+ * @property {string} name - The name of the tool.
+ * @property {string} parameters - The parameters for the tool invocation.
+ * @property {string} result - The result of the tool invocation.
+ */
+
+/**
+ * A class that represents a tool definition.
+ */
+class ToolDefinition {
+    /**
+     * A unique name for the tool.
+     * @type {string}
+     */
+    #name;
+
+    /**
+     * A description of what the tool does.
+     * @type {string}
+     */
+    #description;
+
+    /**
+     * A JSON schema for the parameters that the tool accepts.
+     * @type {object}
+     */
+    #parameters;
+
+    /**
+     * A function that will be called when the tool is executed.
+     * @type {function}
+     */
+    #action;
+
+    /**
+     * Creates a new ToolDefinition.
+     * @param {string} name A unique name for the tool.
+     * @param {string} description A description of what the tool does.
+     * @param {object} parameters A JSON schema for the parameters that the tool accepts.
+     * @param {function} action A function that will be called when the tool is executed.
+     */
+    constructor(name, description, parameters, action) {
+        this.#name = name;
+        this.#description = description;
+        this.#parameters = parameters;
+        this.#action = action;
+    }
+
+    /**
+     * Converts the ToolDefinition to an OpenAI API representation
+     * @returns {object} OpenAI API representation of the tool.
+     */
+    toFunctionOpenAI() {
+        return {
+            type: 'function',
+            function: {
+                name: this.#name,
+                description: this.#description,
+                parameters: this.#parameters,
+            },
+        };
+    }
+
+    /**
+     * Invokes the tool with the given parameters.
+     * @param {object} parameters The parameters to pass to the tool.
+     * @returns {Promise<any>} The result of the tool's action function.
+     */
+    async invoke(parameters) {
+        return await this.#action(parameters);
+    }
+}
+
+/**
+ * A class that manages the registration and invocation of tools.
+ */
+export class ToolManager {
+    /**
+     * A map of tool names to tool definitions.
+     * @type {Map<string, ToolDefinition>}
+     */
+    static #tools = new Map();
+
+    /**
+     * Returns an Array of all tools that have been registered.
+     * @type {ToolDefinition[]}
+     */
+    static get tools() {
+        return Array.from(this.#tools.values());
+    }
+
+    /**
+     * Registers a new tool with the tool registry.
+     * @param {string} name The name of the tool.
+     * @param {string} description A description of what the tool does.
+     * @param {object} parameters A JSON schema for the parameters that the tool accepts.
+     * @param {function} action A function that will be called when the tool is executed.
+     */
+    static registerFunctionTool(name, description, parameters, action) {
+        if (this.#tools.has(name)) {
+            console.warn(`A tool with the name "${name}" has already been registered. The definition will be overwritten.`);
+        }
+
+        const definition = new ToolDefinition(name, description, parameters, action);
+        this.#tools.set(name, definition);
+    }
+
+    /**
+     * Removes a tool from the tool registry.
+     * @param {string} name The name of the tool to unregister.
+     */
+    static unregisterFunctionTool(name) {
+        if (!this.#tools.has(name)) {
+            console.warn(`No tool with the name "${name}" has been registered.`);
+            return;
+        }
+
+        this.#tools.delete(name);
+    }
+
+    /**
+     * Invokes a tool by name. Returns the result of the tool's action function.
+     * @param {string} name The name of the tool to invoke.
+     * @param {object} parameters Function parameters. For example, if the tool requires a "name" parameter, you would pass {name: "value"}.
+     * @returns {Promise<string|null>} The result of the tool's action function. If an error occurs, null is returned. Non-string results are JSON-stringified.
+     */
+    static async invokeFunctionTool(name, parameters) {
+        try {
+            if (!this.#tools.has(name)) {
+                throw new Error(`No tool with the name "${name}" has been registered.`);
+            }
+
+            const invokeParameters = typeof parameters === 'string' ? JSON.parse(parameters) : parameters;
+            const tool = this.#tools.get(name);
+            const result = await tool.invoke(invokeParameters);
+            return typeof result === 'string' ? result : JSON.stringify(result);
+        } catch (error) {
+            console.error(`An error occurred while invoking the tool "${name}":`, error);
+            return null;
+        }
+    }
+
+    /**
+     * Register function tools for the next chat completion request.
+     * @param {object} data Generation data
+     */
+    static async registerFunctionToolsOpenAI(data) {
+        const tools = [];
+
+        for (const tool of ToolManager.tools) {
+            tools.push(tool.toFunctionOpenAI());
+        }
+
+        if (tools.length) {
+            console.log('Registered function tools:', tools);
+
+            data['tools'] = tools;
+            data['tool_choice'] = 'auto';
+        }
+    }
+
+    /**
+     * Utility function to parse tool calls from a parsed response.
+     * @param {any[]} toolCalls The tool calls to update.
+     * @param {any} parsed The parsed response from the OpenAI API.
+     * @returns {void}
+     */
+    static parseToolCalls(toolCalls, parsed) {
+        if (!Array.isArray(parsed?.choices)) {
+            return;
+        }
+        for (const choice of parsed.choices) {
+            const choiceIndex = (typeof choice.index === 'number') ? choice.index : null;
+            const choiceDelta = choice.delta;
+
+            if (choiceIndex === null || !choiceDelta) {
+                continue;
+            }
+
+            const toolCallDeltas = choiceDelta?.tool_calls;
+
+            if (!Array.isArray(toolCallDeltas)) {
+                continue;
+            }
+
+            if (!Array.isArray(toolCalls[choiceIndex])) {
+                toolCalls[choiceIndex] = [];
+            }
+
+            for (const toolCallDelta of toolCallDeltas) {
+                const toolCallIndex = (typeof toolCallDelta?.index === 'number') ? toolCallDelta.index : null;
+
+                if (toolCallIndex === null) {
+                    continue;
+                }
+
+                if (toolCalls[choiceIndex][toolCallIndex] === undefined) {
+                    toolCalls[choiceIndex][toolCallIndex] = {};
+                }
+
+                const targetToolCall = toolCalls[choiceIndex][toolCallIndex];
+
+                ToolManager.#applyToolCallDelta(targetToolCall, toolCallDelta);
+            }
+        }
+    }
+
+    static #applyToolCallDelta(target, delta) {
+        for (const key in delta) {
+            if (!delta.hasOwnProperty(key)) continue;
+
+            const deltaValue = delta[key];
+            const targetValue = target[key];
+
+            if (deltaValue === null || deltaValue === undefined) {
+                target[key] = deltaValue;
+                continue;
+            }
+
+            if (typeof deltaValue === 'string') {
+                if (typeof targetValue === 'string') {
+                    // Concatenate strings
+                    target[key] = targetValue + deltaValue;
+                } else {
+                    target[key] = deltaValue;
+                }
+            } else if (typeof deltaValue === 'object' && !Array.isArray(deltaValue)) {
+                if (typeof targetValue !== 'object' || targetValue === null || Array.isArray(targetValue)) {
+                    target[key] = {};
+                }
+                // Recursively apply deltas to nested objects
+                ToolManager.#applyToolCallDelta(target[key], deltaValue);
+            } else {
+                // Assign other types directly
+                target[key] = deltaValue;
+            }
+        }
+    }
+
+    static isFunctionCallingSupported() {
+        if (main_api !== 'openai') {
+            return false;
+        }
+
+        if (!oai_settings.function_calling) {
+            return false;
+        }
+
+        const supportedSources = [
+            chat_completion_sources.OPENAI,
+            //chat_completion_sources.COHERE,
+            chat_completion_sources.CUSTOM,
+            chat_completion_sources.MISTRALAI,
+            //chat_completion_sources.CLAUDE,
+            chat_completion_sources.OPENROUTER,
+            chat_completion_sources.GROQ,
+        ];
+        return supportedSources.includes(oai_settings.chat_completion_source);
+    }
+
+    static #getToolCallsFromData(data) {
+        // Parsed tool calls from streaming data
+        if (Array.isArray(data) && data.length > 0) {
+            return data[0];
+        }
+
+        // Parsed tool calls from non-streaming data
+        if (!Array.isArray(data?.choices)) {
+            return;
+        }
+
+        // Find a choice with 0-index
+        const choice = data.choices.find(choice => choice.index === 0);
+
+        if (!choice) {
+            return;
+        }
+
+        return choice.message.tool_calls;
+    }
+
+    /**
+     * Check for function tool calls in the response data and invoke them.
+     * @param {any} data Reply data
+     * @returns {Promise<ToolInvocation[]>} Successful tool invocations
+     */
+    static async checkFunctionToolCalls(data) {
+        if (!ToolManager.isFunctionCallingSupported()) {
+            return [];
+        }
+
+        /** @type {ToolInvocation[]} */
+        const invocations = [];
+        const toolCalls = ToolManager.#getToolCallsFromData(data);
+        const oaiCompat = [
+            chat_completion_sources.OPENAI,
+            chat_completion_sources.CUSTOM,
+            chat_completion_sources.MISTRALAI,
+            chat_completion_sources.OPENROUTER,
+            chat_completion_sources.GROQ,
+        ];
+
+        if (oaiCompat.includes(oai_settings.chat_completion_source)) {
+            if (!Array.isArray(toolCalls)) {
+                return;
+            }
+
+            for (const toolCall of toolCalls) {
+                if (typeof toolCall.function !== 'object') {
+                    continue;
+                }
+
+                console.log('Function tool call:', toolCall);
+                const id = toolCall.id;
+                const parameters = toolCall.function.arguments;
+                const name = toolCall.function.name;
+
+                toastr.info('Invoking function tool: ' + name);
+                const result = await ToolManager.invokeFunctionTool(name, parameters);
+                toastr.info('Function tool result: ' + result);
+
+                // Save a successful invocation
+                if (result) {
+                    invocations.push({ id, name, result, parameters });
+                }
+            }
+        }
+
+        /*
+        if ([chat_completion_sources.CLAUDE].includes(oai_settings.chat_completion_source)) {
+            if (!Array.isArray(data?.content)) {
+                return;
+            }
+
+            for (const content of data.content) {
+                if (content.type === 'tool_use') {
+                    const args = { name: content.name, arguments: JSON.stringify(content.input) };
+                }
+            }
+        }
+        */
+
+        /*
+        if ([chat_completion_sources.COHERE].includes(oai_settings.chat_completion_source)) {
+            if (!Array.isArray(data?.tool_calls)) {
+                return;
+            }
+
+            for (const toolCall of data.tool_calls) {
+                const args = { name: toolCall.name, arguments: JSON.stringify(toolCall.parameters) };
+                console.log('Function tool call:', toolCall);
+            }
+        }
+        */
+
+        return invocations;
+    }
+
+    /**
+     * Saves function tool invocations to the last user chat message extra metadata.
+     * @param {ToolInvocation[]} invocations
+     */
+    static saveFunctionToolInvocations(invocations) {
+        for (let index = chat.length - 1; index >= 0; index--) {
+            const message = chat[index];
+            if (message.is_user) {
+                if (!message.extra || typeof message.extra !== 'object') {
+                    message.extra = {};
+                }
+                message.extra.tool_invocations = invocations;
+                debugger;
+                break;
+            }
+        }
+    }
+}
diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js
index ca86db051..095a65ceb 100644
--- a/src/endpoints/backends/chat-completions.js
+++ b/src/endpoints/backends/chat-completions.js
@@ -121,18 +121,20 @@ async function sendClaudeRequest(request, response) {
                 ? [{ type: 'text', text: convertedPrompt.systemPrompt, cache_control: { type: 'ephemeral' } }]
                 : convertedPrompt.systemPrompt;
         }
+        /*
         if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
             // Claude doesn't do prefills on function calls, and doesn't allow empty messages
             if (convertedPrompt.messages.length && convertedPrompt.messages[convertedPrompt.messages.length - 1].role === 'assistant') {
                 convertedPrompt.messages.push({ role: 'user', content: '.' });
             }
             additionalHeaders['anthropic-beta'] = 'tools-2024-05-16';
-            requestBody.tool_choice = { type: request.body.tool_choice === 'required' ? 'any' : 'auto' };
+            requestBody.tool_choice = { type: request.body.tool_choice };
             requestBody.tools = request.body.tools
                 .filter(tool => tool.type === 'function')
                 .map(tool => tool.function)
                 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
         }
+        */
         if (enableSystemPromptCache) {
             additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';
         }
@@ -479,7 +481,7 @@ async function sendMistralAIRequest(request, response) {
 
         if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
             requestBody['tools'] = request.body.tools;
-            requestBody['tool_choice'] = request.body.tool_choice === 'required' ? 'any' : 'auto';
+            requestBody['tool_choice'] = request.body.tool_choice;
         }
 
         const config = {
@@ -549,11 +551,13 @@ async function sendCohereRequest(request, response) {
             });
         }
 
+        /*
         if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
             tools.push(...convertCohereTools(request.body.tools));
             // Can't have both connectors and tools in the same request
             connectors.splice(0, connectors.length);
         }
+        */
 
         // https://docs.cohere.com/reference/chat
         const requestBody = {
@@ -910,18 +914,6 @@ router.post('/generate', jsonParser, function (request, response) {
         apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
         headers = {};
         bodyParams = {};
-
-        // 'required' tool choice is not supported by Groq
-        if (request.body.tool_choice === 'required') {
-            if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
-                request.body.tool_choice = request.body.tools.length > 1
-                    ? 'auto' :
-                    { type: 'function', function: { name: request.body.tools[0]?.function?.name } };
-
-            } else {
-                request.body.tool_choice = 'none';
-            }
-        }
     } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ZEROONEAI) {
         apiUrl = API_01AI;
         apiKey = readSecret(request.user.directories, SECRET_KEYS.ZEROONEAI);
@@ -958,7 +950,7 @@ router.post('/generate', jsonParser, function (request, response) {
         controller.abort();
     });
 
-    if (!isTextCompletion) {
+    if (!isTextCompletion && Array.isArray(request.body.tools) && request.body.tools.length > 0) {
         bodyParams['tools'] = request.body.tools;
         bodyParams['tool_choice'] = request.body.tool_choice;
     }

From c94c06ed4dac3c4895e4afb5b0e6c1524829e49b Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Wed, 2 Oct 2024 01:45:57 +0300
Subject: [PATCH 02/50] Implement function tool calling for OpenAI

---
 public/script.js               |  4 +--
 public/scripts/openai.js       | 52 +++++++++++++++++++++++++++++++---
 public/scripts/tool-calling.js |  5 ++--
 3 files changed, 52 insertions(+), 9 deletions(-)

diff --git a/public/script.js b/public/script.js
index fac11b8f4..3312c556f 100644
--- a/public/script.js
+++ b/public/script.js
@@ -4408,7 +4408,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
 
             if (ToolManager.isFunctionCallingSupported() && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length) {
                 const invocations = await ToolManager.checkFunctionToolCalls(streamingProcessor.toolCalls);
-                if (invocations.length) {
+                if (Array.isArray(invocations) && invocations.length) {
                     const lastMessage = chat[chat.length - 1];
                     const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor.result);
                     if (shouldDeleteMessage) {
@@ -4457,7 +4457,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
 
         if (ToolManager.isFunctionCallingSupported()) {
             const invocations = await ToolManager.checkFunctionToolCalls(data);
-            if (invocations.length) {
+            if (Array.isArray(invocations) && invocations.length) {
                 ToolManager.saveFunctionToolInvocations(invocations);
                 return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
             }
diff --git a/public/scripts/openai.js b/public/scripts/openai.js
index a6048dce5..8b9c06048 100644
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -454,7 +454,8 @@ function setOpenAIMessages(chat) {
         if (role == 'user' && oai_settings.wrap_in_quotes) content = `"${content}"`;
         const name = chat[j]['name'];
         const image = chat[j]?.extra?.image;
-        messages[i] = { 'role': role, 'content': content, name: name, 'image': image };
+        const invocations = chat[j]?.extra?.tool_invocations;
+        messages[i] = { 'role': role, 'content': content, name: name, 'image': image, 'invocations': invocations };
         j++;
     }
 
@@ -702,6 +703,7 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul
     }
 
     const imageInlining = isImageInliningSupported();
+    const toolCalling = ToolManager.isFunctionCallingSupported();
 
     // Insert chat messages as long as there is budget available
     const chatPool = [...messages].reverse();
@@ -723,6 +725,24 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul
             await chatMessage.addImage(chatPrompt.image);
         }
 
+        if (toolCalling && Array.isArray(chatPrompt.invocations)) {
+            /** @type {import('./tool-calling.js').ToolInvocation[]} */
+            const invocations = chatPrompt.invocations.slice().reverse();
+            const toolCallMessage = new Message('assistant', undefined, 'toolCall-' + chatMessage.identifier);
+            toolCallMessage.setToolCalls(invocations);
+            if (chatCompletion.canAfford(toolCallMessage)) {
+                for (const invocation of invocations) {
+                    const toolResultMessage = new Message('tool', invocation.result, invocation.id);
+                    const canAfford = chatCompletion.canAfford(toolResultMessage) && chatCompletion.canAfford(toolCallMessage);
+                    if (!canAfford) {
+                        break;
+                    }
+                    chatCompletion.insertAtStart(toolResultMessage, 'chatHistory');
+                }
+                chatCompletion.insertAtStart(toolCallMessage, 'chatHistory');
+            }
+        }
+
         if (chatCompletion.canAfford(chatMessage)) {
             if (type === 'continue' && oai_settings.continue_prefill && chatPrompt === firstNonInjected) {
                 // in case we are using continue_prefill and the latest message is an assistant message, we want to prepend the users assistant prefill on the message
@@ -2193,6 +2213,8 @@ class Message {
     content;
     /** @type {string} */
     name;
+    /** @type {object} */
+    tool_call = null;
 
     /**
      * @constructor
@@ -2217,6 +2239,22 @@ class Message {
         }
     }
 
+    /**
+     * Reconstruct the message from a tool invocation.
+     * @param {import('./tool-calling.js').ToolInvocation[]} invocations
+     */
+    setToolCalls(invocations) {
+        this.tool_calls = invocations.map(i => ({
+            id: i.id,
+            type: 'function',
+            function: {
+                arguments: i.parameters,
+                name: i.name,
+            },
+        }));
+        this.tokens = tokenHandler.count({ role: this.role, tool_calls: JSON.stringify(this.tool_calls) });
+    }
+
     setName(name) {
         this.name = name;
         this.tokens = tokenHandler.count({ role: this.role, content: this.content, name: this.name });
@@ -2564,7 +2602,7 @@ export class ChatCompletion {
         this.checkTokenBudget(message, message.identifier);
 
         const index = this.findMessageIndex(identifier);
-        if (message.content) {
+        if (message.content || message.tool_calls) {
             if ('start' === position) this.messages.collection[index].collection.unshift(message);
             else if ('end' === position) this.messages.collection[index].collection.push(message);
             else if (typeof position === 'number') this.messages.collection[index].collection.splice(position, 0, message);
@@ -2633,8 +2671,14 @@ export class ChatCompletion {
         for (let item of this.messages.collection) {
             if (item instanceof MessageCollection) {
                 chat.push(...item.getChat());
-            } else if (item instanceof Message && item.content) {
-                const message = { role: item.role, content: item.content, ...(item.name ? { name: item.name } : {}) };
+            } else if (item instanceof Message && (item.content || item.tool_calls)) {
+                const message = {
+                    role: item.role,
+                    content: item.content,
+                    ...(item.name ? { name: item.name } : {}),
+                    ...(item.tool_calls ? { tool_calls: item.tool_calls } : {}),
+                    ...(item.role === 'tool' ? { tool_call_id: item.identifier } : {}),
+                };
                 chat.push(message);
             } else {
                 this.log(`Skipping invalid or empty message in collection: ${JSON.stringify(item)}`);
diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index c2f6c209f..c88528dd7 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -307,7 +307,7 @@ export class ToolManager {
 
         if (oaiCompat.includes(oai_settings.chat_completion_source)) {
             if (!Array.isArray(toolCalls)) {
-                return;
+                return [];
             }
 
             for (const toolCall of toolCalls) {
@@ -363,7 +363,7 @@ export class ToolManager {
 
     /**
      * Saves function tool invocations to the last user chat message extra metadata.
-     * @param {ToolInvocation[]} invocations
+     * @param {ToolInvocation[]} invocations Successful tool invocations
      */
     static saveFunctionToolInvocations(invocations) {
         for (let index = chat.length - 1; index >= 0; index--) {
@@ -373,7 +373,6 @@ export class ToolManager {
                     message.extra = {};
                 }
                 message.extra.tool_invocations = invocations;
-                debugger;
                 break;
             }
         }

From 68c87f7e7aca14e382d8d7976154629a3f007908 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Wed, 2 Oct 2024 01:53:03 +0300
Subject: [PATCH 03/50] Fix code scanning alert no. 231: Prototype-polluting
 function

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
---
 public/scripts/tool-calling.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index c88528dd7..c19d1ac7a 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -213,6 +213,7 @@ export class ToolManager {
     static #applyToolCallDelta(target, delta) {
         for (const key in delta) {
             if (!delta.hasOwnProperty(key)) continue;
+            if (key === "__proto__" || key === "constructor") continue;
 
             const deltaValue = delta[key];
             const targetValue = target[key];

From 63724a2b383aaa2bcaa5ca4009f65062f45c39cd Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Wed, 2 Oct 2024 01:54:47 +0300
Subject: [PATCH 04/50] eslint update

---
 public/scripts/tool-calling.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index c19d1ac7a..964093bc3 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -213,7 +213,7 @@ export class ToolManager {
     static #applyToolCallDelta(target, delta) {
         for (const key in delta) {
             if (!delta.hasOwnProperty(key)) continue;
-            if (key === "__proto__" || key === "constructor") continue;
+            if (key === '__proto__' || key === 'constructor') continue;
 
             const deltaValue = delta[key];
             const targetValue = target[key];

From e8b972042553b27fbce73cb9377067ebaeebc965 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Wed, 2 Oct 2024 01:56:27 +0300
Subject: [PATCH 05/50] Budgeting fix

---
 public/scripts/openai.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/public/scripts/openai.js b/public/scripts/openai.js
index 8b9c06048..96821eda4 100644
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -727,18 +727,20 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul
 
         if (toolCalling && Array.isArray(chatPrompt.invocations)) {
             /** @type {import('./tool-calling.js').ToolInvocation[]} */
-            const invocations = chatPrompt.invocations.slice().reverse();
+            const invocations = chatPrompt.invocations;
             const toolCallMessage = new Message('assistant', undefined, 'toolCall-' + chatMessage.identifier);
             toolCallMessage.setToolCalls(invocations);
             if (chatCompletion.canAfford(toolCallMessage)) {
-                for (const invocation of invocations) {
+                chatCompletion.reserveBudget(toolCallMessage);
+                for (const invocation of invocations.slice().reverse()) {
                     const toolResultMessage = new Message('tool', invocation.result, invocation.id);
-                    const canAfford = chatCompletion.canAfford(toolResultMessage) && chatCompletion.canAfford(toolCallMessage);
+                    const canAfford = chatCompletion.canAfford(toolResultMessage);
                     if (!canAfford) {
                         break;
                     }
                     chatCompletion.insertAtStart(toolResultMessage, 'chatHistory');
                 }
+                chatCompletion.freeBudget(toolCallMessage);
                 chatCompletion.insertAtStart(toolCallMessage, 'chatHistory');
             }
         }

From 3335dbf1a75dd0ffeaede2239c64fb5b85425f31 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Wed, 2 Oct 2024 01:59:53 +0300
Subject: [PATCH 06/50] Add empty tool calls to streaming processors

---
 public/scripts/kai-settings.js     | 2 +-
 public/scripts/nai-settings.js     | 2 +-
 public/scripts/textgen-settings.js | 3 ++-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/public/scripts/kai-settings.js b/public/scripts/kai-settings.js
index 27a204c42..6efadce87 100644
--- a/public/scripts/kai-settings.js
+++ b/public/scripts/kai-settings.js
@@ -188,7 +188,7 @@ export async function generateKoboldWithStreaming(generate_data, signal) {
             if (data?.token) {
                 text += data.token;
             }
-            yield { text, swipes: [] };
+            yield { text, swipes: [], toolCalls: [] };
         }
     };
 }
diff --git a/public/scripts/nai-settings.js b/public/scripts/nai-settings.js
index fc4a042c5..040c6f203 100644
--- a/public/scripts/nai-settings.js
+++ b/public/scripts/nai-settings.js
@@ -746,7 +746,7 @@ export async function generateNovelWithStreaming(generate_data, signal) {
                 text += data.token;
             }
 
-            yield { text, swipes: [], logprobs: parseNovelAILogprobs(data.logprobs) };
+            yield { text, swipes: [], logprobs: parseNovelAILogprobs(data.logprobs), toolCalls: [] };
         }
     };
 }
diff --git a/public/scripts/textgen-settings.js b/public/scripts/textgen-settings.js
index 9403555bd..c021c9682 100644
--- a/public/scripts/textgen-settings.js
+++ b/public/scripts/textgen-settings.js
@@ -916,6 +916,7 @@ async function generateTextGenWithStreaming(generate_data, signal) {
         /** @type {import('./logprobs.js').TokenLogprobs | null} */
         let logprobs = null;
         const swipes = [];
+        const toolCalls = [];
         while (true) {
             const { done, value } = await reader.read();
             if (done) return;
@@ -934,7 +935,7 @@ async function generateTextGenWithStreaming(generate_data, signal) {
                 logprobs = parseTextgenLogprobs(newText, data.choices?.[0]?.logprobs || data?.completion_probabilities);
             }
 
-            yield { text, swipes, logprobs };
+            yield { text, swipes, logprobs, toolCalls };
         }
     };
 }

From 0f8c1fa95d7bdfea5d69ffed3c0ee76eef444886 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Wed, 2 Oct 2024 22:17:27 +0300
Subject: [PATCH 07/50] Save tool calls to visible chats.

---
 public/script.js               | 26 ++++++++------
 public/scripts/openai.js       | 23 +++++++-----
 public/scripts/tool-calling.js | 66 +++++++++++++++++++++-------------
 3 files changed, 71 insertions(+), 44 deletions(-)

diff --git a/public/script.js b/public/script.js
index 3312c556f..1651d786e 100644
--- a/public/script.js
+++ b/public/script.js
@@ -3571,7 +3571,9 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
     }
 
     // Collect messages with usable content
-    let coreChat = chat.filter(x => !x.is_system);
+    const canUseTools = ToolManager.isToolCallingSupported();
+    const canPerformToolCalls = !dryRun && ToolManager.canPerformToolCalls(type);
+    let coreChat = chat.filter(x => !x.is_system || (canUseTools && Array.isArray(x.extra?.tool_invocations)));
     if (type === 'swipe') {
         coreChat.pop();
     }
@@ -4406,8 +4408,8 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
                 getMessage = continue_mag + getMessage;
             }
 
-            if (ToolManager.isFunctionCallingSupported() && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length) {
-                const invocations = await ToolManager.checkFunctionToolCalls(streamingProcessor.toolCalls);
+            if (canPerformToolCalls && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length) {
+                const invocations = await ToolManager.invokeFunctionTools(streamingProcessor.toolCalls);
                 if (Array.isArray(invocations) && invocations.length) {
                     const lastMessage = chat[chat.length - 1];
                     const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor.result);
@@ -4455,14 +4457,6 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
             throw new Error(data?.response);
         }
 
-        if (ToolManager.isFunctionCallingSupported()) {
-            const invocations = await ToolManager.checkFunctionToolCalls(data);
-            if (Array.isArray(invocations) && invocations.length) {
-                ToolManager.saveFunctionToolInvocations(invocations);
-                return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
-            }
-        }
-
         //const getData = await response.json();
         let getMessage = extractMessageFromData(data);
         let title = extractTitleFromData(data);
@@ -4502,6 +4496,16 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
             parseAndSaveLogprobs(data, continue_mag);
         }
 
+        if (canPerformToolCalls) {
+            const invocations = await ToolManager.invokeFunctionTools(data);
+            if (Array.isArray(invocations) && invocations.length) {
+                const shouldDeleteMessage = ['', '...'].includes(getMessage);
+                shouldDeleteMessage && await deleteLastMessage();
+                ToolManager.saveFunctionToolInvocations(invocations);
+                return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
+            }
+        }
+
         if (type !== 'quiet') {
             playMessageSound();
         }
diff --git a/public/scripts/openai.js b/public/scripts/openai.js
index 96821eda4..e9216eccf 100644
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -703,7 +703,7 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul
     }
 
     const imageInlining = isImageInliningSupported();
-    const toolCalling = ToolManager.isFunctionCallingSupported();
+    const canUseTools = ToolManager.isToolCallingSupported();
 
     // Insert chat messages as long as there is budget available
     const chatPool = [...messages].reverse();
@@ -725,10 +725,10 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul
             await chatMessage.addImage(chatPrompt.image);
         }
 
-        if (toolCalling && Array.isArray(chatPrompt.invocations)) {
+        if (canUseTools && Array.isArray(chatPrompt.invocations)) {
             /** @type {import('./tool-calling.js').ToolInvocation[]} */
             const invocations = chatPrompt.invocations;
-            const toolCallMessage = new Message('assistant', undefined, 'toolCall-' + chatMessage.identifier);
+            const toolCallMessage = new Message(chatMessage.role, undefined, 'toolCall-' + chatMessage.identifier);
             toolCallMessage.setToolCalls(invocations);
             if (chatCompletion.canAfford(toolCallMessage)) {
                 chatCompletion.reserveBudget(toolCallMessage);
@@ -1285,7 +1285,7 @@ export async function prepareOpenAIMessages({
     const eventData = { chat, dryRun };
     await eventSource.emit(event_types.CHAT_COMPLETION_PROMPT_READY, eventData);
 
-    openai_messages_count = chat.filter(x => x?.role === 'user' || x?.role === 'assistant')?.length || 0;
+    openai_messages_count = chat.filter(x => !x?.tool_calls && (x?.role === 'user' || x?.role === 'assistant'))?.length || 0;
 
     return [chat, promptManager.tokenHandler.counts];
 }
@@ -1886,7 +1886,7 @@ async function sendOpenAIRequest(type, messages, signal) {
         generate_data['seed'] = oai_settings.seed;
     }
 
-    if (!canMultiSwipe && ToolManager.isFunctionCallingSupported()) {
+    if (!canMultiSwipe && ToolManager.canPerformToolCalls(type)) {
         await ToolManager.registerFunctionToolsOpenAI(generate_data);
     }
 
@@ -2393,13 +2393,20 @@ class MessageCollection {
     }
 
     /**
-     * Get chat in the format of {role, name, content}.
+     * Get chat in the format of {role, name, content, tool_calls}.
      * @returns {Array} Array of objects with role, name, and content properties.
      */
     getChat() {
         return this.collection.reduce((acc, message) => {
-            const name = message.name;
-            if (message.content) acc.push({ role: message.role, ...(name && { name }), content: message.content });
+            if (message.content || message.tool_calls) {
+                acc.push({
+                    role: message.role,
+                    content: message.content,
+                    ...(message.name && { name: message.name }),
+                    ...(message.tool_calls && { tool_calls: message.tool_calls }),
+                    ...(message.role === 'tool' && { tool_call_id: message.identifier }),
+                });
+            }
             return acc;
         }, []);
     }
diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index 964093bc3..def0c078a 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -1,4 +1,4 @@
-import { chat, main_api } from '../script.js';
+import { addOneMessage, chat, main_api, system_avatar, systemUserName } from '../script.js';
 import { chat_completion_sources, oai_settings } from './openai.js';
 
 /**
@@ -243,12 +243,12 @@ export class ToolManager {
         }
     }
 
-    static isFunctionCallingSupported() {
-        if (main_api !== 'openai') {
-            return false;
-        }
-
-        if (!oai_settings.function_calling) {
+    /**
+     * Checks if tool calling is supported for the current settings and generation type.
+     * @returns {boolean} Whether tool calling is supported for the given type
+     */
+    static isToolCallingSupported() {
+        if (main_api !== 'openai' || !oai_settings.function_calling) {
             return false;
         }
 
@@ -264,6 +264,22 @@ export class ToolManager {
         return supportedSources.includes(oai_settings.chat_completion_source);
     }
 
+    /**
+     * Checks if tool calls can be performed for the current settings and generation type.
+     * @param {string} type Generation type
+     * @returns {boolean} Whether tool calls can be performed for the given type
+     */
+    static canPerformToolCalls(type) {
+        const noToolCallTypes = ['swipe', 'impersonate', 'quiet', 'continue'];
+        const isSupported = ToolManager.isToolCallingSupported();
+        return isSupported && !noToolCallTypes.includes(type);
+    }
+
+    /**
+     * Utility function to get tool calls from the response data.
+     * @param {any} data Response data
+     * @returns {any[]} Tool calls from the response data
+     */
     static #getToolCallsFromData(data) {
         // Parsed tool calls from streaming data
         if (Array.isArray(data) && data.length > 0) {
@@ -290,15 +306,11 @@ export class ToolManager {
      * @param {any} data Reply data
      * @returns {Promise<ToolInvocation[]>} Successful tool invocations
      */
-    static async checkFunctionToolCalls(data) {
-        if (!ToolManager.isFunctionCallingSupported()) {
-            return [];
-        }
-
+    static async invokeFunctionTools(data) {
         /** @type {ToolInvocation[]} */
         const invocations = [];
         const toolCalls = ToolManager.#getToolCallsFromData(data);
-        const oaiCompat = [
+        const oaiCompatibleSources = [
             chat_completion_sources.OPENAI,
             chat_completion_sources.CUSTOM,
             chat_completion_sources.MISTRALAI,
@@ -306,7 +318,7 @@ export class ToolManager {
             chat_completion_sources.GROQ,
         ];
 
-        if (oaiCompat.includes(oai_settings.chat_completion_source)) {
+        if (oaiCompatibleSources.includes(oai_settings.chat_completion_source)) {
             if (!Array.isArray(toolCalls)) {
                 return [];
             }
@@ -323,7 +335,7 @@ export class ToolManager {
 
                 toastr.info('Invoking function tool: ' + name);
                 const result = await ToolManager.invokeFunctionTool(name, parameters);
-                toastr.info('Function tool result: ' + result);
+                console.log('Function tool result:', result);
 
                 // Save a successful invocation
                 if (result) {
@@ -367,15 +379,19 @@ export class ToolManager {
      * @param {ToolInvocation[]} invocations Successful tool invocations
      */
     static saveFunctionToolInvocations(invocations) {
-        for (let index = chat.length - 1; index >= 0; index--) {
-            const message = chat[index];
-            if (message.is_user) {
-                if (!message.extra || typeof message.extra !== 'object') {
-                    message.extra = {};
-                }
-                message.extra.tool_invocations = invocations;
-                break;
-            }
-        }
+        const toolNames = invocations.map(i => i.name).join(', ');
+        const message = {
+            name: systemUserName,
+            force_avatar: system_avatar,
+            is_system: true,
+            is_user: false,
+            mes: `Performed tool calls: ${toolNames}`,
+            extra: {
+                isSmallSys: true,
+                tool_invocations: invocations,
+            },
+        };
+        chat.push(message);
+        addOneMessage(message);
     }
 }

From 2dad86e0769f803f674995a3b15ca48369144c36 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Wed, 2 Oct 2024 23:12:49 +0300
Subject: [PATCH 08/50] Delete empty message before tool invocations

---
 public/script.js | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/public/script.js b/public/script.js
index c3aa593e8..3a38a04a9 100644
--- a/public/script.js
+++ b/public/script.js
@@ -4409,14 +4409,12 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
             }
 
             if (canPerformToolCalls && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length) {
+                const lastMessage = chat[chat.length - 1];
+                const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor.result);
+                shouldDeleteMessage && await deleteLastMessage();
                 const invocations = await ToolManager.invokeFunctionTools(streamingProcessor.toolCalls);
                 if (Array.isArray(invocations) && invocations.length) {
-                    const lastMessage = chat[chat.length - 1];
-                    const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor.result);
-                    if (shouldDeleteMessage) {
-                        await deleteLastMessage();
-                        streamingProcessor = null;
-                    }
+                    streamingProcessor = null;
                     ToolManager.saveFunctionToolInvocations(invocations);
                     return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
                 }
@@ -4497,10 +4495,10 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
         }
 
         if (canPerformToolCalls) {
+            const shouldDeleteMessage = ['', '...'].includes(getMessage);
+            shouldDeleteMessage && await deleteLastMessage();
             const invocations = await ToolManager.invokeFunctionTools(data);
             if (Array.isArray(invocations) && invocations.length) {
-                const shouldDeleteMessage = ['', '...'].includes(getMessage);
-                shouldDeleteMessage && await deleteLastMessage();
                 ToolManager.saveFunctionToolInvocations(invocations);
                 return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
             }

From 2b7c03f3b0c9662471880b25b1ee83ea90f51860 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Wed, 2 Oct 2024 23:13:11 +0300
Subject: [PATCH 09/50] Nicely format message for tool calls

---
 public/scripts/tool-calling.js | 26 +++++++++++++++++++++++---
 public/style.css               |  9 +++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index def0c078a..b3c8ac1b1 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -108,6 +108,7 @@ export class ToolManager {
 
         const definition = new ToolDefinition(name, description, parameters, action);
         this.#tools.set(name, definition);
+        console.log('[ToolManager] Registered function tool:', definition);
     }
 
     /**
@@ -121,6 +122,7 @@ export class ToolManager {
         }
 
         this.#tools.delete(name);
+        console.log(`[ToolManager] Unregistered function tool: ${name}`);
     }
 
     /**
@@ -333,8 +335,9 @@ export class ToolManager {
                 const parameters = toolCall.function.arguments;
                 const name = toolCall.function.name;
 
-                toastr.info('Invoking function tool: ' + name);
+                const toast = toastr.info(`Invoking function tool: ${name}`);
                 const result = await ToolManager.invokeFunctionTool(name, parameters);
+                toastr.clear(toast);
                 console.log('Function tool result:', result);
 
                 // Save a successful invocation
@@ -374,18 +377,35 @@ export class ToolManager {
         return invocations;
     }
 
+    /**
+     * Formats a message with tool invocations.
+     * @param {ToolInvocation[]} invocations Tool invocations.
+     * @returns {string} Formatted message with tool invocations.
+     */
+    static #formatMessage(invocations) {
+        const toolNames = invocations.map(i => i.name).join(', ');
+        const detailsElement = document.createElement('details');
+        const summaryElement = document.createElement('summary');
+        const preElement = document.createElement('pre');
+        const codeElement = document.createElement('code');
+        codeElement.textContent = JSON.stringify(invocations, null, 2);
+        summaryElement.textContent = `Performed tool calls: ${toolNames}`;
+        preElement.append(codeElement);
+        detailsElement.append(summaryElement, preElement);
+        return detailsElement.outerHTML;
+    }
+
     /**
      * Saves function tool invocations to the last user chat message extra metadata.
      * @param {ToolInvocation[]} invocations Successful tool invocations
      */
     static saveFunctionToolInvocations(invocations) {
-        const toolNames = invocations.map(i => i.name).join(', ');
         const message = {
             name: systemUserName,
             force_avatar: system_avatar,
             is_system: true,
             is_user: false,
-            mes: `Performed tool calls: ${toolNames}`,
+            mes: ToolManager.#formatMessage(invocations),
             extra: {
                 isSmallSys: true,
                 tool_invocations: invocations,
diff --git a/public/style.css b/public/style.css
index 01a49ab9f..19a9bb738 100644
--- a/public/style.css
+++ b/public/style.css
@@ -418,6 +418,15 @@ small {
     text-align: center;
 }
 
+.mes.smallSysMes pre {
+    text-align: initial;
+    word-break: break-all;
+}
+
+.mes.smallSysMes summary {
+    cursor: pointer;
+}
+
 .mes.smallSysMes .mes_text p:last-child {
     margin: 0;
 }

From 1e076a3e4367cc228bb63cadaed3812298897308 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Wed, 2 Oct 2024 23:32:29 +0300
Subject: [PATCH 10/50] Prettify displayed message

---
 public/scripts/tool-calling.js | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index b3c8ac1b1..dd1d1d8c4 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -342,7 +342,7 @@ export class ToolManager {
 
                 // Save a successful invocation
                 if (result) {
-                    invocations.push({ id, name, result, parameters });
+                    invocations.push({ id, name, parameters, result });
                 }
             }
         }
@@ -383,12 +383,16 @@ export class ToolManager {
      * @returns {string} Formatted message with tool invocations.
      */
     static #formatMessage(invocations) {
-        const toolNames = invocations.map(i => i.name).join(', ');
+        const tryParse = (x) => { try { return JSON.parse(x); } catch { return x; } };
+        const data = structuredClone(invocations);
         const detailsElement = document.createElement('details');
         const summaryElement = document.createElement('summary');
         const preElement = document.createElement('pre');
         const codeElement = document.createElement('code');
-        codeElement.textContent = JSON.stringify(invocations, null, 2);
+        codeElement.classList.add('language-json');
+        data.forEach(i => i.parameters = tryParse(i.parameters));
+        codeElement.textContent = JSON.stringify(data, null, 2);
+        const toolNames = data.map(i => i.name).join(', ');
         summaryElement.textContent = `Performed tool calls: ${toolNames}`;
         preElement.append(codeElement);
         detailsElement.append(summaryElement, preElement);

From da9200c82ed39f790b3ff982e796f33aeaec3ae1 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Thu, 3 Oct 2024 12:59:59 +0000
Subject: [PATCH 11/50] Skip adding tool messages as regular chats

---
 public/scripts/openai.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/public/scripts/openai.js b/public/scripts/openai.js
index e9216eccf..9ab05b467 100644
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -743,6 +743,8 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul
                 chatCompletion.freeBudget(toolCallMessage);
                 chatCompletion.insertAtStart(toolCallMessage, 'chatHistory');
             }
+
+            continue;
         }
 
         if (chatCompletion.canAfford(chatMessage)) {

From 90809852c2a7e1ecc133003b536d237908e1f498 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Thu, 3 Oct 2024 13:23:53 +0000
Subject: [PATCH 12/50] Hide message on streaming tool calls

---
 public/script.js | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/public/script.js b/public/script.js
index 3a38a04a9..e3c7bf1ba 100644
--- a/public/script.js
+++ b/public/script.js
@@ -2932,6 +2932,13 @@ class StreamingProcessor {
         }
     }
 
+    #updateMessageBlockVisibility() {
+        if (this.messageDom instanceof HTMLElement && Array.isArray(this.toolCalls) && this.toolCalls.length > 0) {
+            const shouldHide = ['', '...'].includes(this.result);
+            this.messageDom.classList.toggle('displayNone', shouldHide);
+        }
+    }
+
     showMessageButtons(messageId) {
         if (messageId == -1) {
             return;
@@ -2997,6 +3004,7 @@ class StreamingProcessor {
         }
         else {
             this.#checkDomElements(messageId);
+            this.#updateMessageBlockVisibility();
             const currentTime = new Date();
             // Don't waste time calculating token count for streaming
             const currentTokenCount = isFinal && power_user.message_token_count_enabled ? getTokenCount(processedText, 0) : 0;

From 6558b106754a73faff6ca171254192ed9a1157e9 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 00:11:36 +0300
Subject: [PATCH 13/50] Show an error when all tools fail

---
 public/script.js               | 27 +++++++++++++----
 public/scripts/tool-calling.js | 54 +++++++++++++++++++++++++++-------
 public/style.css               |  1 +
 3 files changed, 66 insertions(+), 16 deletions(-)

diff --git a/public/script.js b/public/script.js
index e3c7bf1ba..4d59bde08 100644
--- a/public/script.js
+++ b/public/script.js
@@ -4420,10 +4420,18 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
                 const lastMessage = chat[chat.length - 1];
                 const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor.result);
                 shouldDeleteMessage && await deleteLastMessage();
-                const invocations = await ToolManager.invokeFunctionTools(streamingProcessor.toolCalls);
-                if (Array.isArray(invocations) && invocations.length) {
+                const invocationResult = await ToolManager.invokeFunctionTools(streamingProcessor.toolCalls);
+                if (invocationResult.hadToolCalls) {
+                    if (!invocationResult.invocations.length && shouldDeleteMessage) {
+                        ToolManager.showToolCallError(invocationResult.errors);
+                        unblockGeneration(type);
+                        generatedPromptCache = '';
+                        streamingProcessor = null;
+                        return;
+                    }
+
                     streamingProcessor = null;
-                    ToolManager.saveFunctionToolInvocations(invocations);
+                    ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
                     return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
                 }
             }
@@ -4505,9 +4513,16 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
         if (canPerformToolCalls) {
             const shouldDeleteMessage = ['', '...'].includes(getMessage);
             shouldDeleteMessage && await deleteLastMessage();
-            const invocations = await ToolManager.invokeFunctionTools(data);
-            if (Array.isArray(invocations) && invocations.length) {
-                ToolManager.saveFunctionToolInvocations(invocations);
+            const invocationResult = await ToolManager.invokeFunctionTools(data);
+            if (invocationResult.hadToolCalls) {
+                if (!invocationResult.invocations.length && shouldDeleteMessage) {
+                    ToolManager.showToolCallError(invocationResult.errors);
+                    unblockGeneration(type);
+                    generatedPromptCache = '';
+                    return;
+                }
+
+                ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
                 return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
             }
         }
diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index dd1d1d8c4..bd8c572f8 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -1,5 +1,6 @@
 import { addOneMessage, chat, main_api, system_avatar, systemUserName } from '../script.js';
 import { chat_completion_sources, oai_settings } from './openai.js';
+import { Popup } from './popup.js';
 
 /**
  * @typedef {object} ToolInvocation
@@ -9,6 +10,13 @@ import { chat_completion_sources, oai_settings } from './openai.js';
  * @property {string} result - The result of the tool invocation.
  */
 
+/**
+ * @typedef {object} ToolInvocationResult
+ * @property {ToolInvocation[]} invocations Successful tool invocations
+ * @property {boolean} hadToolCalls Whether any tool calls were found
+ * @property {Error[]} errors Errors that occurred during tool invocation
+ */
+
 /**
  * A class that represents a tool definition.
  */
@@ -129,7 +137,7 @@ export class ToolManager {
      * Invokes a tool by name. Returns the result of the tool's action function.
      * @param {string} name The name of the tool to invoke.
      * @param {object} parameters Function parameters. For example, if the tool requires a "name" parameter, you would pass {name: "value"}.
-     * @returns {Promise<string|null>} The result of the tool's action function. If an error occurs, null is returned. Non-string results are JSON-stringified.
+     * @returns {Promise<string|Error>} The result of the tool's action function. If an error occurs, null is returned. Non-string results are JSON-stringified.
      */
     static async invokeFunctionTool(name, parameters) {
         try {
@@ -143,7 +151,13 @@ export class ToolManager {
             return typeof result === 'string' ? result : JSON.stringify(result);
         } catch (error) {
             console.error(`An error occurred while invoking the tool "${name}":`, error);
-            return null;
+
+            if (error instanceof Error) {
+                error.cause = name;
+                return error;
+            }
+
+            return new Error('Unknown error occurred while invoking the tool.', { cause: name });
         }
     }
 
@@ -306,11 +320,15 @@ export class ToolManager {
     /**
      * Check for function tool calls in the response data and invoke them.
      * @param {any} data Reply data
-     * @returns {Promise<ToolInvocation[]>} Successful tool invocations
+     * @returns {Promise<ToolInvocationResult>} Successful tool invocations
      */
     static async invokeFunctionTools(data) {
-        /** @type {ToolInvocation[]} */
-        const invocations = [];
+        /** @type {ToolInvocationResult} */
+        const result = {
+            invocations: [],
+            hadToolCalls: false,
+            errors: [],
+        };
         const toolCalls = ToolManager.#getToolCallsFromData(data);
         const oaiCompatibleSources = [
             chat_completion_sources.OPENAI,
@@ -322,7 +340,7 @@ export class ToolManager {
 
         if (oaiCompatibleSources.includes(oai_settings.chat_completion_source)) {
             if (!Array.isArray(toolCalls)) {
-                return [];
+                return result;
             }
 
             for (const toolCall of toolCalls) {
@@ -334,16 +352,20 @@ export class ToolManager {
                 const id = toolCall.id;
                 const parameters = toolCall.function.arguments;
                 const name = toolCall.function.name;
+                result.hadToolCalls = true;
 
                 const toast = toastr.info(`Invoking function tool: ${name}`);
-                const result = await ToolManager.invokeFunctionTool(name, parameters);
+                const toolResult = await ToolManager.invokeFunctionTool(name, parameters);
                 toastr.clear(toast);
                 console.log('Function tool result:', result);
 
                 // Save a successful invocation
-                if (result) {
-                    invocations.push({ id, name, parameters, result });
+                if (toolResult instanceof Error) {
+                    result.errors.push(toolResult);
+                    continue;
                 }
+
+                result.invocations.push({ id, name, parameters, result: toolResult });
             }
         }
 
@@ -374,7 +396,7 @@ export class ToolManager {
         }
         */
 
-        return invocations;
+        return result;
     }
 
     /**
@@ -418,4 +440,16 @@ export class ToolManager {
         chat.push(message);
         addOneMessage(message);
     }
+
+    /**
+     * Shows an error message for tool calls.
+     * @param {Error[]} errors Errors that occurred during tool invocation
+     * @returns {void}
+     */
+    static showToolCallError(errors) {
+        toastr.error('An error occurred while invoking function tools. Click here for more details.', 'Tool Calling', {
+            onclick: () => Popup.show.text('Tool Calling Errors', DOMPurify.sanitize(errors.map(e => `${e.cause}: ${e.message}`).join('<br>'))),
+            timeOut: 5000,
+        });
+    }
 }
diff --git a/public/style.css b/public/style.css
index 19a9bb738..96ae504b7 100644
--- a/public/style.css
+++ b/public/style.css
@@ -421,6 +421,7 @@ small {
 .mes.smallSysMes pre {
     text-align: initial;
     word-break: break-all;
+    margin-top: 5px;
 }
 
 .mes.smallSysMes summary {

From 5cf64a2613252cc9ab8557fe145fab7546ea56ec Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 00:39:28 +0300
Subject: [PATCH 14/50] Update tool registration

---
 public/scripts/tool-calling.js | 100 +++++++++++++++++++++++++++++----
 1 file changed, 89 insertions(+), 11 deletions(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index bd8c572f8..0eaada477 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -5,6 +5,7 @@ import { Popup } from './popup.js';
 /**
  * @typedef {object} ToolInvocation
  * @property {string} id - A unique identifier for the tool invocation.
+ * @property {string} displayName - The display name of the tool.
  * @property {string} name - The name of the tool.
  * @property {string} parameters - The parameters for the tool invocation.
  * @property {string} result - The result of the tool invocation.
@@ -27,6 +28,12 @@ class ToolDefinition {
      */
     #name;
 
+    /**
+     * A user-friendly display name for the tool.
+     * @type {string}
+     */
+    #displayName;
+
     /**
      * A description of what the tool does.
      * @type {string}
@@ -45,18 +52,28 @@ class ToolDefinition {
      */
     #action;
 
+    /**
+     * A function that will be called to format the tool call toast.
+     * @type {function}
+     */
+    #formatMessage;
+
     /**
      * Creates a new ToolDefinition.
      * @param {string} name A unique name for the tool.
+     * @param {string} displayName A user-friendly display name for the tool.
      * @param {string} description A description of what the tool does.
      * @param {object} parameters A JSON schema for the parameters that the tool accepts.
      * @param {function} action A function that will be called when the tool is executed.
+     * @param {function} formatMessage A function that will be called to format the tool call toast.
      */
-    constructor(name, description, parameters, action) {
+    constructor(name, displayName, description, parameters, action, formatMessage) {
         this.#name = name;
+        this.#displayName = displayName;
         this.#description = description;
         this.#parameters = parameters;
         this.#action = action;
+        this.#formatMessage = formatMessage;
     }
 
     /**
@@ -82,6 +99,21 @@ class ToolDefinition {
     async invoke(parameters) {
         return await this.#action(parameters);
     }
+
+    /**
+     * Formats a message with the tool invocation.
+     * @param {object} parameters The parameters to pass to the tool.
+     * @returns {string} The formatted message.
+     */
+    formatMessage(parameters) {
+        return typeof this.#formatMessage === 'function'
+            ? this.#formatMessage(parameters)
+            : `Invoking tool: ${this.#displayName || this.#name}`;
+    }
+
+    get displayName() {
+        return this.#displayName;
+    }
 }
 
 /**
@@ -104,17 +136,25 @@ export class ToolManager {
 
     /**
      * Registers a new tool with the tool registry.
-     * @param {string} name The name of the tool.
-     * @param {string} description A description of what the tool does.
-     * @param {object} parameters A JSON schema for the parameters that the tool accepts.
-     * @param {function} action A function that will be called when the tool is executed.
+     * @param {object} tool The tool to register.
+     * @param {string} tool.name The name of the tool.
+     * @param {string} tool.displayName A user-friendly display name for the tool.
+     * @param {string} tool.description A description of what the tool does.
+     * @param {object} tool.parameters A JSON schema for the parameters that the tool accepts.
+     * @param {function} tool.action A function that will be called when the tool is executed.
+     * @param {function} tool.formatMessage A function that will be called to format the tool call toast.
      */
-    static registerFunctionTool(name, description, parameters, action) {
+    static registerFunctionTool({ name, displayName, description, parameters, action, formatMessage }) {
+        // Convert WIP arguments
+        if (typeof arguments[0] !== 'object') {
+            [name, description, parameters, action] = arguments;
+        }
+
         if (this.#tools.has(name)) {
             console.warn(`A tool with the name "${name}" has already been registered. The definition will be overwritten.`);
         }
 
-        const definition = new ToolDefinition(name, description, parameters, action);
+        const definition = new ToolDefinition(name, displayName, description, parameters, action, formatMessage);
         this.#tools.set(name, definition);
         console.log('[ToolManager] Registered function tool:', definition);
     }
@@ -161,6 +201,35 @@ export class ToolManager {
         }
     }
 
+    static formatToolCallMessage(name, parameters) {
+        if (!this.#tools.has(name)) {
+            return `Invoked unknown tool: ${name}`;
+        }
+
+        try {
+            const tool = this.#tools.get(name);
+            const formatParameters = typeof parameters === 'string' ? JSON.parse(parameters) : parameters;
+            return tool.formatMessage(formatParameters);
+        } catch (error) {
+            console.error(`An error occurred while formatting the tool call message for "${name}":`, error);
+            return `Invoking tool: ${name}`;
+        }
+    }
+
+    /**
+     * Gets the display name of a tool by name.
+     * @param {string} name
+     * @returns {string} The display name of the tool.
+     */
+    static getDisplayName(name) {
+        if (!this.#tools.has(name)) {
+            return name;
+        }
+
+        const tool = this.#tools.get(name);
+        return tool.displayName || name;
+    }
+
     /**
      * Register function tools for the next chat completion request.
      * @param {object} data Generation data
@@ -352,9 +421,11 @@ export class ToolManager {
                 const id = toolCall.id;
                 const parameters = toolCall.function.arguments;
                 const name = toolCall.function.name;
+                const displayName = ToolManager.getDisplayName(name);
                 result.hadToolCalls = true;
 
-                const toast = toastr.info(`Invoking function tool: ${name}`);
+                const message = ToolManager.formatToolCallMessage(name, parameters);
+                const toast = message && toastr.info(message, 'Tool Calling', { timeOut: 0 });
                 const toolResult = await ToolManager.invokeFunctionTool(name, parameters);
                 toastr.clear(toast);
                 console.log('Function tool result:', result);
@@ -365,7 +436,14 @@ export class ToolManager {
                     continue;
                 }
 
-                result.invocations.push({ id, name, parameters, result: toolResult });
+                const invocation = {
+                    id,
+                    displayName,
+                    name,
+                    parameters,
+                    result: toolResult,
+                };
+                result.invocations.push(invocation);
             }
         }
 
@@ -414,8 +492,8 @@ export class ToolManager {
         codeElement.classList.add('language-json');
         data.forEach(i => i.parameters = tryParse(i.parameters));
         codeElement.textContent = JSON.stringify(data, null, 2);
-        const toolNames = data.map(i => i.name).join(', ');
-        summaryElement.textContent = `Performed tool calls: ${toolNames}`;
+        const toolNames = data.map(i => i.displayName || i.name).join(', ');
+        summaryElement.textContent = `Tool calls: ${toolNames}`;
         preElement.append(codeElement);
         detailsElement.append(summaryElement, preElement);
         return detailsElement.outerHTML;

From 6cb82fc21e3fc7996b5f3c4a1e480890038e47da Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 00:39:45 +0300
Subject: [PATCH 15/50] Add json param to /transcript

---
 src/endpoints/search.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/endpoints/search.js b/src/endpoints/search.js
index 457124228..8c8ed7055 100644
--- a/src/endpoints/search.js
+++ b/src/endpoints/search.js
@@ -60,6 +60,7 @@ router.post('/transcript', jsonParser, async (request, response) => {
         const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
         const id = request.body.id;
         const lang = request.body.lang;
+        const json = request.body.json;
 
         if (!id) {
             console.log('Id is required for /transcript');
@@ -129,7 +130,9 @@ router.post('/transcript', jsonParser, async (request, response) => {
         // The text is double-encoded
         const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
 
-        return response.send(transcriptText);
+        return json
+            ? response.json({ transcript: transcriptText, html: videoPageBody })
+            : response.send(transcriptText);
     } catch (error) {
         console.log(error);
         return response.sendStatus(500);

From 777b2518bdb0153f194db79fb651c558e196fc3d Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 01:12:12 +0300
Subject: [PATCH 16/50] Allow returning page if transcript extraction failed

---
 src/endpoints/search.js | 132 +++++++++++++++++++++++-----------------
 1 file changed, 75 insertions(+), 57 deletions(-)

diff --git a/src/endpoints/search.js b/src/endpoints/search.js
index 8c8ed7055..64993589e 100644
--- a/src/endpoints/search.js
+++ b/src/endpoints/search.js
@@ -22,6 +22,72 @@ const visitHeaders = {
     'Sec-Fetch-User': '?1',
 };
 
+/**
+ * Extract the transcript of a YouTube video
+ * @param {string} videoPageBody HTML of the video page
+ * @param {string} lang Language code
+ * @returns {Promise<string>} Transcript text
+ */
+async function extractTranscript(videoPageBody, lang) {
+    const he = require('he');
+    const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
+    const splittedHTML = videoPageBody.split('"captions":');
+
+    if (splittedHTML.length <= 1) {
+        if (videoPageBody.includes('class="g-recaptcha"')) {
+            throw new Error('Too many requests');
+        }
+        if (!videoPageBody.includes('"playabilityStatus":')) {
+            throw new Error('Video is not available');
+        }
+        throw new Error('Transcript not available');
+    }
+
+    const captions = (() => {
+        try {
+            return JSON.parse(splittedHTML[1].split(',"videoDetails')[0].replace('\n', ''));
+        } catch (e) {
+            return undefined;
+        }
+    })()?.['playerCaptionsTracklistRenderer'];
+
+    if (!captions) {
+        throw new Error('Transcript disabled');
+    }
+
+    if (!('captionTracks' in captions)) {
+        throw new Error('Transcript not available');
+    }
+
+    if (lang && !captions.captionTracks.some(track => track.languageCode === lang)) {
+        throw new Error('Transcript not available in this language');
+    }
+
+    const transcriptURL = (lang ? captions.captionTracks.find(track => track.languageCode === lang) : captions.captionTracks[0]).baseUrl;
+    const transcriptResponse = await fetch(transcriptURL, {
+        headers: {
+            ...(lang && { 'Accept-Language': lang }),
+            'User-Agent': visitHeaders['User-Agent'],
+        },
+    });
+
+    if (!transcriptResponse.ok) {
+        throw new Error('Transcript request failed');
+    }
+
+    const transcriptBody = await transcriptResponse.text();
+    const results = [...transcriptBody.matchAll(RE_XML_TRANSCRIPT)];
+    const transcript = results.map((result) => ({
+        text: result[3],
+        duration: parseFloat(result[2]),
+        offset: parseFloat(result[1]),
+        lang: lang ?? captions.captionTracks[0].languageCode,
+    }));
+    // The text is double-encoded
+    const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
+    return transcriptText;
+}
+
 router.post('/serpapi', jsonParser, async (request, response) => {
     try {
         const key = readSecret(request.user.directories, SECRET_KEYS.SERPAPI);
@@ -56,8 +122,6 @@ router.post('/serpapi', jsonParser, async (request, response) => {
  */
 router.post('/transcript', jsonParser, async (request, response) => {
     try {
-        const he = require('he');
-        const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
         const id = request.body.id;
         const lang = request.body.lang;
         const json = request.body.json;
@@ -75,64 +139,18 @@ router.post('/transcript', jsonParser, async (request, response) => {
         });
 
         const videoPageBody = await videoPageResponse.text();
-        const splittedHTML = videoPageBody.split('"captions":');
 
-        if (splittedHTML.length <= 1) {
-            if (videoPageBody.includes('class="g-recaptcha"')) {
-                throw new Error('Too many requests');
+        try {
+            const transcriptText = await extractTranscript(videoPageBody, lang);
+            return json
+                ? response.json({ transcript: transcriptText, html: videoPageBody })
+                : response.send(transcriptText);
+        } catch (error) {
+            if (json) {
+                return response.json({ html: videoPageBody, transcript: '' });
             }
-            if (!videoPageBody.includes('"playabilityStatus":')) {
-                throw new Error('Video is not available');
-            }
-            throw new Error('Transcript not available');
+            throw error;
         }
-
-        const captions = (() => {
-            try {
-                return JSON.parse(splittedHTML[1].split(',"videoDetails')[0].replace('\n', ''));
-            } catch (e) {
-                return undefined;
-            }
-        })()?.['playerCaptionsTracklistRenderer'];
-
-        if (!captions) {
-            throw new Error('Transcript disabled');
-        }
-
-        if (!('captionTracks' in captions)) {
-            throw new Error('Transcript not available');
-        }
-
-        if (lang && !captions.captionTracks.some(track => track.languageCode === lang)) {
-            throw new Error('Transcript not available in this language');
-        }
-
-        const transcriptURL = (lang ? captions.captionTracks.find(track => track.languageCode === lang) : captions.captionTracks[0]).baseUrl;
-        const transcriptResponse = await fetch(transcriptURL, {
-            headers: {
-                ...(lang && { 'Accept-Language': lang }),
-                'User-Agent': visitHeaders['User-Agent'],
-            },
-        });
-
-        if (!transcriptResponse.ok) {
-            throw new Error('Transcript request failed');
-        }
-
-        const transcriptBody = await transcriptResponse.text();
-        const results = [...transcriptBody.matchAll(RE_XML_TRANSCRIPT)];
-        const transcript = results.map((result) => ({
-            text: result[3],
-            duration: parseFloat(result[2]),
-            offset: parseFloat(result[1]),
-            lang: lang ?? captions.captionTracks[0].languageCode,
-        }));
-        // The text is double-encoded
-        const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
-
-        return json
-            ? response.json({ transcript: transcriptText, html: videoPageBody })
-            : response.send(transcriptText);
     } catch (error) {
         console.log(error);
         return response.sendStatus(500);

From 447a7fba68298cdae8becd82acb3b2f97dc785fd Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 01:45:37 +0300
Subject: [PATCH 17/50] Only delete message if had successful tool calls

---
 public/script.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/public/script.js b/public/script.js
index 4d59bde08..22ce72641 100644
--- a/public/script.js
+++ b/public/script.js
@@ -4417,11 +4417,11 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
             }
 
             if (canPerformToolCalls && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length) {
-                const lastMessage = chat[chat.length - 1];
-                const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor.result);
-                shouldDeleteMessage && await deleteLastMessage();
                 const invocationResult = await ToolManager.invokeFunctionTools(streamingProcessor.toolCalls);
                 if (invocationResult.hadToolCalls) {
+                    const lastMessage = chat[chat.length - 1];
+                    const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor.result);
+                    shouldDeleteMessage && await deleteLastMessage();
                     if (!invocationResult.invocations.length && shouldDeleteMessage) {
                         ToolManager.showToolCallError(invocationResult.errors);
                         unblockGeneration(type);
@@ -4511,10 +4511,10 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
         }
 
         if (canPerformToolCalls) {
-            const shouldDeleteMessage = ['', '...'].includes(getMessage);
-            shouldDeleteMessage && await deleteLastMessage();
             const invocationResult = await ToolManager.invokeFunctionTools(data);
             if (invocationResult.hadToolCalls) {
+                const shouldDeleteMessage = ['', '...'].includes(getMessage);
+                shouldDeleteMessage && await deleteLastMessage();
                 if (!invocationResult.invocations.length && shouldDeleteMessage) {
                     ToolManager.showToolCallError(invocationResult.errors);
                     unblockGeneration(type);

From 01f03dbf508a00240e3019c517d8f61166af9eb1 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 01:51:41 +0300
Subject: [PATCH 18/50] Support MistralAI streaming tool calls

---
 public/scripts/tool-calling.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index 0eaada477..e0e3c53e0 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -278,9 +278,9 @@ export class ToolManager {
             }
 
             for (const toolCallDelta of toolCallDeltas) {
-                const toolCallIndex = (typeof toolCallDelta?.index === 'number') ? toolCallDelta.index : null;
+                const toolCallIndex = (typeof toolCallDelta?.index === 'number') ? toolCallDelta.index : toolCallDeltas.indexOf(toolCallDelta);
 
-                if (toolCallIndex === null) {
+                if (isNaN(toolCallIndex) || toolCallIndex < 0) {
                     continue;
                 }
 

From 559f1b81f79b2c4e2dc68538d29ead31ed738c36 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 02:11:46 +0300
Subject: [PATCH 19/50] Remove tool calling for Cohere v1

---
 public/scripts/tool-calling.js             | 13 ----
 src/endpoints/backends/chat-completions.js | 10 +--
 src/prompt-converters.js                   | 71 ----------------------
 3 files changed, 1 insertion(+), 93 deletions(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index e0e3c53e0..2f101d9ba 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -461,19 +461,6 @@ export class ToolManager {
         }
         */
 
-        /*
-        if ([chat_completion_sources.COHERE].includes(oai_settings.chat_completion_source)) {
-            if (!Array.isArray(data?.tool_calls)) {
-                return;
-            }
-
-            for (const toolCall of data.tool_calls) {
-                const args = { name: toolCall.name, arguments: JSON.stringify(toolCall.parameters) };
-                console.log('Function tool call:', toolCall);
-            }
-        }
-        */
-
         return result;
     }
 
diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js
index 55cb08c70..552ebefbd 100644
--- a/src/endpoints/backends/chat-completions.js
+++ b/src/endpoints/backends/chat-completions.js
@@ -4,7 +4,7 @@ const fetch = require('node-fetch').default;
 const { jsonParser } = require('../../express-common');
 const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY, OPENROUTER_HEADERS } = require('../../constants');
 const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml, color } = require('../../util');
-const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt, convertCohereMessages, convertMistralMessages, convertCohereTools, convertAI21Messages } = require('../../prompt-converters');
+const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt, convertCohereMessages, convertMistralMessages, convertAI21Messages } = require('../../prompt-converters');
 const CohereStream = require('../../cohere-stream');
 
 const { readSecret, SECRET_KEYS } = require('../secrets');
@@ -555,14 +555,6 @@ async function sendCohereRequest(request, response) {
             });
         }
 
-        /*
-        if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
-            tools.push(...convertCohereTools(request.body.tools));
-            // Can't have both connectors and tools in the same request
-            connectors.splice(0, connectors.length);
-        }
-        */
-
         // https://docs.cohere.com/reference/chat
         const requestBody = {
             stream: Boolean(request.body.stream),
diff --git a/src/prompt-converters.js b/src/prompt-converters.js
index 5d52adb6e..2f6aaf960 100644
--- a/src/prompt-converters.js
+++ b/src/prompt-converters.js
@@ -522,76 +522,6 @@ function convertTextCompletionPrompt(messages) {
     return messageStrings.join('\n') + '\nassistant:';
 }
 
-/**
- * Convert OpenAI Chat Completion tools to the format used by Cohere.
- * @param {object[]} tools OpenAI Chat Completion tool definitions
- */
-function convertCohereTools(tools) {
-    if (!Array.isArray(tools) || tools.length === 0) {
-        return [];
-    }
-
-    const jsonSchemaToPythonTypes = {
-        'string': 'str',
-        'number': 'float',
-        'integer': 'int',
-        'boolean': 'bool',
-        'array': 'list',
-        'object': 'dict',
-    };
-
-    const cohereTools = [];
-
-    for (const tool of tools) {
-        if (tool?.type !== 'function') {
-            console.log(`Unsupported tool type: ${tool.type}`);
-            continue;
-        }
-
-        const name = tool?.function?.name;
-        const description = tool?.function?.description;
-        const properties = tool?.function?.parameters?.properties;
-        const required = tool?.function?.parameters?.required;
-        const parameters = {};
-
-        if (!name) {
-            console.log('Tool name is missing');
-            continue;
-        }
-
-        if (!description) {
-            console.log('Tool description is missing');
-        }
-
-        if (!properties || typeof properties !== 'object') {
-            console.log(`No properties found for tool: ${tool?.function?.name}`);
-            continue;
-        }
-
-        for (const property in properties) {
-            const parameterDefinition = properties[property];
-            const description = parameterDefinition.description || (parameterDefinition.enum ? JSON.stringify(parameterDefinition.enum) : '');
-            const type = jsonSchemaToPythonTypes[parameterDefinition.type] || 'str';
-            const isRequired = Array.isArray(required) && required.includes(property);
-            parameters[property] = {
-                description: description,
-                type: type,
-                required: isRequired,
-            };
-        }
-
-        const cohereTool = {
-            name: tool.function.name,
-            description: tool.function.description,
-            parameter_definitions: parameters,
-        };
-
-        cohereTools.push(cohereTool);
-    }
-
-    return cohereTools;
-}
-
 module.exports = {
     convertClaudePrompt,
     convertClaudeMessages,
@@ -599,6 +529,5 @@ module.exports = {
     convertTextCompletionPrompt,
     convertCohereMessages,
     convertMistralMessages,
-    convertCohereTools,
     convertAI21Messages,
 };

From c3c10a629e000f3d7479f9a50fbb6d28316acaa0 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 03:41:25 +0300
Subject: [PATCH 20/50] Claude: new prompt converter + non-streaming tools

---
 public/scripts/tool-calling.js             | 111 +++++++++---------
 src/endpoints/backends/chat-completions.js |   2 -
 src/prompt-converters.js                   | 127 ++++++++++++++-------
 3 files changed, 141 insertions(+), 99 deletions(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index 2f101d9ba..15d3db954 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -339,10 +339,9 @@ export class ToolManager {
 
         const supportedSources = [
             chat_completion_sources.OPENAI,
-            //chat_completion_sources.COHERE,
             chat_completion_sources.CUSTOM,
             chat_completion_sources.MISTRALAI,
-            //chat_completion_sources.CLAUDE,
+            chat_completion_sources.CLAUDE,
             chat_completion_sources.OPENROUTER,
             chat_completion_sources.GROQ,
         ];
@@ -372,18 +371,29 @@ export class ToolManager {
         }
 
         // Parsed tool calls from non-streaming data
-        if (!Array.isArray(data?.choices)) {
-            return;
+        if (Array.isArray(data?.choices)) {
+            // Find a choice with 0-index
+            const choice = data.choices.find(choice => choice.index === 0);
+
+            if (choice) {
+                return choice.message.tool_calls;
+            }
         }
 
-        // Find a choice with 0-index
-        const choice = data.choices.find(choice => choice.index === 0);
+        if (Array.isArray(data?.content)) {
+            // Claude tool calls to OpenAI tool calls
+            const content = data.content.filter(c => c.type === 'tool_use').map(c => {
+                return {
+                    id: c.id,
+                    function: {
+                        name: c.name,
+                        arguments: c.input,
+                    },
+                };
+            });
 
-        if (!choice) {
-            return;
+            return content;
         }
-
-        return choice.message.tool_calls;
     }
 
     /**
@@ -407,59 +417,43 @@ export class ToolManager {
             chat_completion_sources.GROQ,
         ];
 
-        if (oaiCompatibleSources.includes(oai_settings.chat_completion_source)) {
-            if (!Array.isArray(toolCalls)) {
-                return result;
-            }
-
-            for (const toolCall of toolCalls) {
-                if (typeof toolCall.function !== 'object') {
-                    continue;
-                }
-
-                console.log('Function tool call:', toolCall);
-                const id = toolCall.id;
-                const parameters = toolCall.function.arguments;
-                const name = toolCall.function.name;
-                const displayName = ToolManager.getDisplayName(name);
-                result.hadToolCalls = true;
-
-                const message = ToolManager.formatToolCallMessage(name, parameters);
-                const toast = message && toastr.info(message, 'Tool Calling', { timeOut: 0 });
-                const toolResult = await ToolManager.invokeFunctionTool(name, parameters);
-                toastr.clear(toast);
-                console.log('Function tool result:', result);
-
-                // Save a successful invocation
-                if (toolResult instanceof Error) {
-                    result.errors.push(toolResult);
-                    continue;
-                }
-
-                const invocation = {
-                    id,
-                    displayName,
-                    name,
-                    parameters,
-                    result: toolResult,
-                };
-                result.invocations.push(invocation);
-            }
+        if (!Array.isArray(toolCalls)) {
+            return result;
         }
 
-        /*
-        if ([chat_completion_sources.CLAUDE].includes(oai_settings.chat_completion_source)) {
-            if (!Array.isArray(data?.content)) {
-                return;
+        for (const toolCall of toolCalls) {
+            if (typeof toolCall.function !== 'object') {
+                continue;
             }
 
-            for (const content of data.content) {
-                if (content.type === 'tool_use') {
-                    const args = { name: content.name, arguments: JSON.stringify(content.input) };
-                }
+            console.log('Function tool call:', toolCall);
+            const id = toolCall.id;
+            const parameters = toolCall.function.arguments;
+            const name = toolCall.function.name;
+            const displayName = ToolManager.getDisplayName(name);
+            result.hadToolCalls = true;
+
+            const message = ToolManager.formatToolCallMessage(name, parameters);
+            const toast = message && toastr.info(message, 'Tool Calling', { timeOut: 0 });
+            const toolResult = await ToolManager.invokeFunctionTool(name, parameters);
+            toastr.clear(toast);
+            console.log('Function tool result:', result);
+
+            // Save a successful invocation
+            if (toolResult instanceof Error) {
+                result.errors.push(toolResult);
+                continue;
             }
+
+            const invocation = {
+                id,
+                displayName,
+                name,
+                parameters,
+                result: toolResult,
+            };
+            result.invocations.push(invocation);
         }
-        */
 
         return result;
     }
@@ -491,6 +485,9 @@ export class ToolManager {
      * @param {ToolInvocation[]} invocations Successful tool invocations
      */
     static saveFunctionToolInvocations(invocations) {
+        if (!Array.isArray(invocations) || invocations.length === 0) {
+            return;
+        }
         const message = {
             name: systemUserName,
             force_avatar: system_avatar,
diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js
index 552ebefbd..9c29a3b44 100644
--- a/src/endpoints/backends/chat-completions.js
+++ b/src/endpoints/backends/chat-completions.js
@@ -124,7 +124,6 @@ async function sendClaudeRequest(request, response) {
         } else {
             delete requestBody.system;
         }
-        /*
         if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
             // Claude doesn't do prefills on function calls, and doesn't allow empty messages
             if (convertedPrompt.messages.length && convertedPrompt.messages[convertedPrompt.messages.length - 1].role === 'assistant') {
@@ -137,7 +136,6 @@ async function sendClaudeRequest(request, response) {
                 .map(tool => tool.function)
                 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
         }
-        */
         if (enableSystemPromptCache) {
             additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';
         }
diff --git a/src/prompt-converters.js b/src/prompt-converters.js
index 2f6aaf960..b4b3d718b 100644
--- a/src/prompt-converters.js
+++ b/src/prompt-converters.js
@@ -118,8 +118,27 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
             });
         }
     }
+
     // Now replace all further messages that have the role 'system' with the role 'user'. (or all if we're not using one)
     messages.forEach((message) => {
+        if (message.role === 'assistant' && message.tool_calls) {
+            message.content = message.tool_calls.map((tc) => ({
+                type: 'tool_use',
+                id: tc.id,
+                name: tc.function.name,
+                input: tc.function.arguments,
+            }));
+        }
+
+        if (message.role === 'tool') {
+            message.role = 'user';
+            message.content = [{
+                type: 'tool_result',
+                tool_use_id: message.tool_call_id,
+                content: message.content,
+            }];
+        }
+
         if (message.role === 'system') {
             if (userName && message.name === 'example_user') {
                 message.content = `${userName}: ${message.content}`;
@@ -128,13 +147,80 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
                 message.content = `${charName}: ${message.content}`;
             }
             message.role = 'user';
+
+            // Delete name here so it doesn't get added later
+            delete message.name;
         }
+
+        // Convert everything to an array of it would be easier to work with
+        if (typeof message.content === 'string') {
+            // Take care of name properties since claude messages don't support them
+            if (message.name) {
+                message.content = `${message.name}: ${message.content}`;
+            }
+
+            message.content = [{ type: 'text', text: message.content }];
+        } else if (Array.isArray(message.content)) {
+            message.content = message.content.map((content) => {
+                if (content.type === 'image_url') {
+                    const imageEntry = content?.image_url;
+                    const imageData = imageEntry?.url;
+                    const mimeType = imageData?.split(';')?.[0].split(':')?.[1];
+                    const base64Data = imageData?.split(',')?.[1];
+
+                    return {
+                        type: 'image',
+                        source: {
+                            type: 'base64',
+                            media_type: mimeType,
+                            data: base64Data,
+                        },
+                    };
+                }
+
+                if (content.type === 'text') {
+                    if (message.name) {
+                        content.text = `${message.name}: ${content.text}`;
+                    }
+
+                    return content;
+                }
+
+                return content;
+            });
+        }
+
+        // Remove offending properties
+        delete message.name;
+        delete message.tool_calls;
+        delete message.tool_call_id;
     });
 
+    // Images in assistant messages should be moved to the next user message
+    for (let i = 0; i < messages.length; i++) {
+        if (messages[i].role === 'assistant' && messages[i].content.some(c => c.type === 'image')) {
+            // Find the next user message
+            let j = i + 1;
+            while (j < messages.length && messages[j].role !== 'user') {
+                j++;
+            }
+
+            // Move the images
+            if (j >= messages.length) {
+                // If there is no user message after the assistant message, add a new one
+                messages.splice(i + 1, 0, { role: 'user', content: [] });
+            }
+
+            messages[j].content.push(...messages[i].content.filter(c => c.type === 'image'));
+            messages[i].content = messages[i].content.filter(c => c.type !== 'image');
+        }
+    }
+
     // Shouldn't be conditional anymore, messages api expects the last role to be user unless we're explicitly prefilling
     if (prefillString) {
         messages.push({
             role: 'assistant',
+            // Dangling whitespace are not allowed for prefilling
             content: prefillString.trimEnd(),
         });
     }
@@ -143,50 +229,11 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
     // Also handle multi-modality, holy slop.
     let mergedMessages = [];
     messages.forEach((message) => {
-        const imageEntry = message.content?.[1]?.image_url;
-        const imageData = imageEntry?.url;
-        const mimeType = imageData?.split(';')?.[0].split(':')?.[1];
-        const base64Data = imageData?.split(',')?.[1];
-
-        // Take care of name properties since claude messages don't support them
-        if (message.name) {
-            if (Array.isArray(message.content)) {
-                message.content[0].text = `${message.name}: ${message.content[0].text}`;
-            } else {
-                message.content = `${message.name}: ${message.content}`;
-            }
-            delete message.name;
-        }
-
         if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role) {
-            if (Array.isArray(message.content)) {
-                if (Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
-                    mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content[0].text;
-                } else {
-                    mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content[0].text;
-                }
-            } else {
-                if (Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
-                    mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content;
-                } else {
-                    mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content;
-                }
-            }
+            mergedMessages[mergedMessages.length - 1].content.push(...message.content);
         } else {
             mergedMessages.push(message);
         }
-        if (imageData) {
-            mergedMessages[mergedMessages.length - 1].content = [
-                { type: 'text', text: mergedMessages[mergedMessages.length - 1].content[0]?.text || mergedMessages[mergedMessages.length - 1].content },
-                {
-                    type: 'image', source: {
-                        type: 'base64',
-                        media_type: mimeType,
-                        data: base64Data,
-                    },
-                },
-            ];
-        }
     });
 
     return { messages: mergedMessages, systemPrompt: systemPrompt.trim() };

From 4a2989718caf6f8a64515035c01a43f26b38cbe3 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 10:34:17 +0000
Subject: [PATCH 21/50] ESLint and JSDoc fixes

---
 public/script.js               |  2 ++
 public/scripts/tool-calling.js | 42 ++++++++++++++++++++--------------
 2 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/public/script.js b/public/script.js
index 22ce72641..109c6cc36 100644
--- a/public/script.js
+++ b/public/script.js
@@ -8180,6 +8180,8 @@ window['SillyTavern'].getContext = function () {
         unregisterMacro: MacrosParser.unregisterMacro.bind(MacrosParser),
         registerFunctionTool: ToolManager.registerFunctionTool.bind(ToolManager),
         unregisterFunctionTool: ToolManager.unregisterFunctionTool.bind(ToolManager),
+        isToolCallingSupported: ToolManager.isToolCallingSupported.bind(ToolManager),
+        canPerformToolCalls: ToolManager.canPerformToolCalls.bind(ToolManager),
         registerDebugFunction: registerDebugFunction,
         /** @deprecated Use renderExtensionTemplateAsync instead. */
         renderExtensionTemplate: renderExtensionTemplate,
diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index 15d3db954..7457392b3 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -18,6 +18,16 @@ import { Popup } from './popup.js';
  * @property {Error[]} errors Errors that occurred during tool invocation
  */
 
+/**
+ * @typedef {object} ToolRegistration
+ * @property {string} name - The name of the tool.
+ * @property {string} displayName - The display name of the tool.
+ * @property {string} description - A description of the tool.
+ * @property {object} parameters - The parameters for the tool.
+ * @property {function} action - The action to perform when the tool is invoked.
+ * @property {function} formatMessage - A function to format the tool call message.
+ */
+
 /**
  * A class that represents a tool definition.
  */
@@ -136,13 +146,7 @@ export class ToolManager {
 
     /**
      * Registers a new tool with the tool registry.
-     * @param {object} tool The tool to register.
-     * @param {string} tool.name The name of the tool.
-     * @param {string} tool.displayName A user-friendly display name for the tool.
-     * @param {string} tool.description A description of what the tool does.
-     * @param {object} tool.parameters A JSON schema for the parameters that the tool accepts.
-     * @param {function} tool.action A function that will be called when the tool is executed.
-     * @param {function} tool.formatMessage A function that will be called to format the tool call toast.
+     * @param {ToolRegistration} tool The tool to register.
      */
     static registerFunctionTool({ name, displayName, description, parameters, action, formatMessage }) {
         // Convert WIP arguments
@@ -201,6 +205,12 @@ export class ToolManager {
         }
     }
 
+    /**
+     * Formats a message for a tool call by name.
+     * @param {string} name The name of the tool to format the message for.
+     * @param {object} parameters Function tool call parameters.
+     * @returns {string} The formatted message for the tool call.
+     */
     static formatToolCallMessage(name, parameters) {
         if (!this.#tools.has(name)) {
             return `Invoked unknown tool: ${name}`;
@@ -295,9 +305,14 @@ export class ToolManager {
         }
     }
 
+    /**
+     * Apply a tool call delta to a target object.
+     * @param {object} target The target object to apply the delta to
+     * @param {object} delta The delta object to apply
+     */
     static #applyToolCallDelta(target, delta) {
         for (const key in delta) {
-            if (!delta.hasOwnProperty(key)) continue;
+            if (!Object.prototype.hasOwnProperty.call(delta, key)) continue;
             if (key === '__proto__' || key === 'constructor') continue;
 
             const deltaValue = delta[key];
@@ -409,13 +424,6 @@ export class ToolManager {
             errors: [],
         };
         const toolCalls = ToolManager.#getToolCallsFromData(data);
-        const oaiCompatibleSources = [
-            chat_completion_sources.OPENAI,
-            chat_completion_sources.CUSTOM,
-            chat_completion_sources.MISTRALAI,
-            chat_completion_sources.OPENROUTER,
-            chat_completion_sources.GROQ,
-        ];
 
         if (!Array.isArray(toolCalls)) {
             return result;
@@ -463,7 +471,7 @@ export class ToolManager {
      * @param {ToolInvocation[]} invocations Tool invocations.
      * @returns {string} Formatted message with tool invocations.
      */
-    static #formatMessage(invocations) {
+    static #formatToolInvocationMessage(invocations) {
         const tryParse = (x) => { try { return JSON.parse(x); } catch { return x; } };
         const data = structuredClone(invocations);
         const detailsElement = document.createElement('details');
@@ -493,7 +501,7 @@ export class ToolManager {
             force_avatar: system_avatar,
             is_system: true,
             is_user: false,
-            mes: ToolManager.#formatMessage(invocations),
+            mes: ToolManager.#formatToolInvocationMessage(invocations),
             extra: {
                 isSmallSys: true,
                 tool_invocations: invocations,

From 0cab91e0f8c0074d4bfdf33c3971b3dde1029412 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 13:39:08 +0300
Subject: [PATCH 22/50] Add Claude streamed tool parser

---
 public/scripts/tool-calling.js | 69 +++++++++++++++++++++-------------
 1 file changed, 43 insertions(+), 26 deletions(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index 15d3db954..57e21b618 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -256,41 +256,58 @@ export class ToolManager {
      * @returns {void}
      */
     static parseToolCalls(toolCalls, parsed) {
-        if (!Array.isArray(parsed?.choices)) {
-            return;
-        }
-        for (const choice of parsed.choices) {
-            const choiceIndex = (typeof choice.index === 'number') ? choice.index : null;
-            const choiceDelta = choice.delta;
+        if (Array.isArray(parsed?.choices)) {
+            for (const choice of parsed.choices) {
+                const choiceIndex = (typeof choice.index === 'number') ? choice.index : null;
+                const choiceDelta = choice.delta;
 
-            if (choiceIndex === null || !choiceDelta) {
-                continue;
-            }
-
-            const toolCallDeltas = choiceDelta?.tool_calls;
-
-            if (!Array.isArray(toolCallDeltas)) {
-                continue;
-            }
-
-            if (!Array.isArray(toolCalls[choiceIndex])) {
-                toolCalls[choiceIndex] = [];
-            }
-
-            for (const toolCallDelta of toolCallDeltas) {
-                const toolCallIndex = (typeof toolCallDelta?.index === 'number') ? toolCallDelta.index : toolCallDeltas.indexOf(toolCallDelta);
-
-                if (isNaN(toolCallIndex) || toolCallIndex < 0) {
+                if (choiceIndex === null || !choiceDelta) {
                     continue;
                 }
 
+                const toolCallDeltas = choiceDelta?.tool_calls;
+
+                if (!Array.isArray(toolCallDeltas)) {
+                    continue;
+                }
+
+                if (!Array.isArray(toolCalls[choiceIndex])) {
+                    toolCalls[choiceIndex] = [];
+                }
+
+                for (const toolCallDelta of toolCallDeltas) {
+                    const toolCallIndex = (typeof toolCallDelta?.index === 'number') ? toolCallDelta.index : toolCallDeltas.indexOf(toolCallDelta);
+
+                    if (isNaN(toolCallIndex) || toolCallIndex < 0) {
+                        continue;
+                    }
+
+                    if (toolCalls[choiceIndex][toolCallIndex] === undefined) {
+                        toolCalls[choiceIndex][toolCallIndex] = {};
+                    }
+
+                    const targetToolCall = toolCalls[choiceIndex][toolCallIndex];
+
+                    ToolManager.#applyToolCallDelta(targetToolCall, toolCallDelta);
+                }
+            }
+        }
+        if (typeof parsed?.content_block === 'object') {
+            const choiceIndex = 0;
+
+            if (parsed?.content_block?.type === 'tool_use') {
+                if (!Array.isArray(toolCalls[choiceIndex])) {
+                    toolCalls[choiceIndex] = [];
+                }
+
+                const toolCallIndex = toolCalls[choiceIndex].length;
+
                 if (toolCalls[choiceIndex][toolCallIndex] === undefined) {
                     toolCalls[choiceIndex][toolCallIndex] = {};
                 }
 
                 const targetToolCall = toolCalls[choiceIndex][toolCallIndex];
-
-                ToolManager.#applyToolCallDelta(targetToolCall, toolCallDelta);
+                Object.assign(targetToolCall, parsed.content_block);
             }
         }
     }

From db04fff3df342e7c035c7dccccb88df5dd2fc75c Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 11:31:15 +0000
Subject: [PATCH 23/50] Claude: Streamed tool calls parser

---
 public/scripts/tool-calling.js | 63 ++++++++++++++++++++++++----------
 1 file changed, 45 insertions(+), 18 deletions(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index cd989cc0b..264649753 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -136,6 +136,8 @@ export class ToolManager {
      */
     static #tools = new Map();
 
+    static #INPUT_DELTA_KEY = '__input_json_delta';
+
     /**
      * Returns an Array of all tools that have been registered.
      * @type {ToolDefinition[]}
@@ -304,20 +306,48 @@ export class ToolManager {
         }
         if (typeof parsed?.content_block === 'object') {
             const choiceIndex = 0;
+            const toolCallIndex = parsed?.index ?? 0;
 
             if (parsed?.content_block?.type === 'tool_use') {
                 if (!Array.isArray(toolCalls[choiceIndex])) {
                     toolCalls[choiceIndex] = [];
                 }
-
-                const toolCallIndex = toolCalls[choiceIndex].length;
-
                 if (toolCalls[choiceIndex][toolCallIndex] === undefined) {
                     toolCalls[choiceIndex][toolCallIndex] = {};
                 }
-
                 const targetToolCall = toolCalls[choiceIndex][toolCallIndex];
-                Object.assign(targetToolCall, parsed.content_block);
+                ToolManager.#applyToolCallDelta(targetToolCall, parsed.content_block);
+            }
+        }
+        if (typeof parsed?.delta === 'object') {
+            const choiceIndex = 0;
+            const toolCallIndex = parsed?.index ?? 0;
+            const targetToolCall = toolCalls[choiceIndex]?.[toolCallIndex];
+            if (targetToolCall){
+                if (parsed?.delta?.type === 'input_json_delta') {
+                    const jsonDelta = parsed?.delta?.partial_json;
+                    if (!targetToolCall[this.#INPUT_DELTA_KEY]) {
+                        targetToolCall[this.#INPUT_DELTA_KEY] = '';
+                    }
+                    targetToolCall[this.#INPUT_DELTA_KEY] += jsonDelta;
+                }
+            }
+        }
+        if (parsed?.type === 'content_block_stop') {
+            const choiceIndex = 0;
+            const toolCallIndex = parsed?.index ?? 0;
+            const targetToolCall = toolCalls[choiceIndex]?.[toolCallIndex];
+            if (targetToolCall) {
+                const jsonDeltaString = targetToolCall[this.#INPUT_DELTA_KEY];
+                if (jsonDeltaString) {
+                    try {
+                        const jsonDelta = { input: JSON.parse(jsonDeltaString) };
+                        delete targetToolCall[this.#INPUT_DELTA_KEY];
+                        ToolManager.#applyToolCallDelta(targetToolCall, jsonDelta);
+                    } catch (error) {
+                        console.warn('Failed to apply input JSON delta:', error);
+                    }
+                }
             }
         }
     }
@@ -397,9 +427,12 @@ export class ToolManager {
      * @returns {any[]} Tool calls from the response data
      */
     static #getToolCallsFromData(data) {
+        const isClaudeToolCall = c => Array.isArray(c) ? c.filter(x => x).every(isClaudeToolCall) : c?.input && c?.name && c?.id;
+        const convertClaudeToolCall = c => ({ id: c.id, function: { name: c.name, arguments: c.input } });
+
         // Parsed tool calls from streaming data
-        if (Array.isArray(data) && data.length > 0) {
-            return data[0];
+        if (Array.isArray(data) && data.length > 0 && Array.isArray(data[0])) {
+            return isClaudeToolCall(data[0]) ? data[0].filter(x => x).map(convertClaudeToolCall) : data[0];
         }
 
         // Parsed tool calls from non-streaming data
@@ -412,19 +445,13 @@ export class ToolManager {
             }
         }
 
+        // Claude tool calls to OpenAI tool calls
         if (Array.isArray(data?.content)) {
-            // Claude tool calls to OpenAI tool calls
-            const content = data.content.filter(c => c.type === 'tool_use').map(c => {
-                return {
-                    id: c.id,
-                    function: {
-                        name: c.name,
-                        arguments: c.input,
-                    },
-                };
-            });
+            const content = data.content.filter(c => c.type === 'tool_use').map(convertClaudeToolCall);
 
-            return content;
+            if (content) {
+                return content;
+            }
         }
     }
 

From 8c095f204a014c5e5f7f784b1f203c6b5089b1d7 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 12:00:20 +0000
Subject: [PATCH 24/50] Fix error on streaming if the processor was already
 destroyed

---
 public/script.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/script.js b/public/script.js
index 109c6cc36..f34d57cf8 100644
--- a/public/script.js
+++ b/public/script.js
@@ -4420,7 +4420,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
                 const invocationResult = await ToolManager.invokeFunctionTools(streamingProcessor.toolCalls);
                 if (invocationResult.hadToolCalls) {
                     const lastMessage = chat[chat.length - 1];
-                    const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor.result);
+                    const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor?.result);
                     shouldDeleteMessage && await deleteLastMessage();
                     if (!invocationResult.invocations.length && shouldDeleteMessage) {
                         ToolManager.showToolCallError(invocationResult.errors);

From cc3cc58a06a7ecfd841e4ddad8f7190c61da5d98 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 12:24:10 +0000
Subject: [PATCH 25/50] Claude: fix token counting when tool_calls are used

---
 src/prompt-converters.js | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/prompt-converters.js b/src/prompt-converters.js
index b4b3d718b..ceb3c1189 100644
--- a/src/prompt-converters.js
+++ b/src/prompt-converters.js
@@ -19,6 +19,14 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill,
     //Prepare messages for claude.
     //When 'Exclude Human/Assistant prefixes' checked, setting messages role to the 'system'(last message is exception).
     if (messages.length > 0) {
+        messages.forEach((m) => {
+            if (!m.content) {
+                m.content = '';
+            }
+            if (m.tool_calls) {
+                m.content += JSON.stringify(m.tool_calls);
+            }
+        });
         if (excludePrefixes) {
             messages.slice(0, -1).forEach(message => message.role = 'system');
         } else {

From c853547b11a8d219fb9c3c04bd05737dc56678e7 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 13:04:19 +0000
Subject: [PATCH 26/50] Add a function tool for image generation

---
 .../extensions/stable-diffusion/index.js      | 54 +++++++++++++++++++
 .../extensions/stable-diffusion/settings.html |  4 ++
 2 files changed, 58 insertions(+)

diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js
index e96575d56..0289ff807 100644
--- a/public/scripts/extensions/stable-diffusion/index.js
+++ b/public/scripts/extensions/stable-diffusion/index.js
@@ -32,6 +32,7 @@ import { debounce_timeout } from '../../constants.js';
 import { SlashCommandEnumValue } from '../../slash-commands/SlashCommandEnumValue.js';
 import { POPUP_RESULT, POPUP_TYPE, Popup, callGenericPopup } from '../../popup.js';
 import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
+import { ToolManager } from '../../tool-calling.js';
 export { MODULE_NAME };
 
 const MODULE_NAME = 'sd';
@@ -62,6 +63,7 @@ const initiators = {
     interactive: 'interactive',
     wand: 'wand',
     swipe: 'swipe',
+    tool: 'tool',
 };
 
 const generationMode = {
@@ -226,6 +228,7 @@ const defaultSettings = {
     multimodal_captioning: false,
     snap: false,
     free_extend: false,
+    function_tool: false,
 
     prompts: promptTemplates,
 
@@ -291,6 +294,10 @@ const defaultSettings = {
 const writePromptFieldsDebounced = debounce(writePromptFields, debounce_timeout.relaxed);
 
 function processTriggers(chat, _, abort) {
+    if (extension_settings.sd.function_tool && ToolManager.isToolCallingSupported()) {
+        return;
+    }
+
     if (!extension_settings.sd.interactive_mode) {
         return;
     }
@@ -447,6 +454,7 @@ async function loadSettings() {
     $('#sd_interactive_visible').prop('checked', extension_settings.sd.interactive_visible);
     $('#sd_stability_style_preset').val(extension_settings.sd.stability_style_preset);
     $('#sd_huggingface_model_id').val(extension_settings.sd.huggingface_model_id);
+    $('#sd_function_tool').prop('checked', extension_settings.sd.function_tool);
 
     for (const style of extension_settings.sd.styles) {
         const option = document.createElement('option');
@@ -461,6 +469,7 @@ async function loadSettings() {
 
     toggleSourceControls();
     addPromptTemplates();
+    registerFunctionTool();
 
     await loadSettingOptions();
 }
@@ -910,6 +919,12 @@ async function onSourceChange() {
     await loadSettingOptions();
 }
 
+function onFunctionToolInput() {
+    extension_settings.sd.function_tool = !!$(this).prop('checked');
+    saveSettingsDebounced();
+    registerFunctionTool();
+}
+
 async function onOpenAiStyleSelect() {
     extension_settings.sd.openai_style = String($('#sd_openai_style').find(':selected').val());
     saveSettingsDebounced();
@@ -3822,6 +3837,44 @@ function applyCommandArguments(args) {
     return currentSettings;
 }
 
+function registerFunctionTool() {
+    if (!extension_settings.sd.function_tool) {
+        return ToolManager.unregisterFunctionTool('GenerateImage');
+    }
+
+    ToolManager.registerFunctionTool({
+        name: 'GenerateImage',
+        displayName: 'Generate Image',
+        description: [
+            'Generate an image from a given text prompt.',
+            'Use when a user asks for an image, a selfie, to picture a scene, etc.',
+        ].join(' '),
+        parameters: Object.freeze({
+            $schema: 'http://json-schema.org/draft-04/schema#',
+            type: 'object',
+            properties: {
+                prompt: {
+                    type: 'string',
+                    description: [
+                        'The text prompt used to generate the image.',
+                        'Must represent an exhaustive description of the desired image that will allow an artist or a photographer to perfectly recreate it.',
+                    ],
+                },
+            },
+            required: [
+                'prompt',
+            ],
+        }),
+        action: async (args) => {
+            if (!isValidState()) throw new Error('Image generation is not configured.');
+            if (!args) throw new Error('Missing arguments');
+            if (!args.prompt) throw new Error('Missing prompt');
+            return generatePicture(initiators.tool, {}, args.prompt);
+        },
+        formatMessage: () => 'Generating an image...',
+    });
+}
+
 jQuery(async () => {
     await addSDGenButtons();
 
@@ -4175,6 +4228,7 @@ jQuery(async () => {
     $('#sd_stability_key').on('click', onStabilityKeyClick);
     $('#sd_stability_style_preset').on('change', onStabilityStylePresetChange);
     $('#sd_huggingface_model_id').on('input', onHFModelInput);
+    $('#sd_function_tool').on('input', onFunctionToolInput);
 
     if (!CSS.supports('field-sizing', 'content')) {
         $('.sd_settings .inline-drawer-toggle').on('click', function () {
diff --git a/public/scripts/extensions/stable-diffusion/settings.html b/public/scripts/extensions/stable-diffusion/settings.html
index 2efa74d83..6870facd2 100644
--- a/public/scripts/extensions/stable-diffusion/settings.html
+++ b/public/scripts/extensions/stable-diffusion/settings.html
@@ -18,6 +18,10 @@
                 <input id="sd_interactive_mode" type="checkbox" />
                 <span data-i18n="sd_interactive_mode_txt">Interactive mode</span>
             </label>
+            <label for="sd_function_tool" class="checkbox_label" data-i18n="[title]sd_function_tool" title="Use the function tool to automatically detect intents to generate images.">
+                <input id="sd_function_tool" type="checkbox" />
+                <span data-i18n="sd_function_tool_txt">Enable function tool</span>
+            </label>
             <label for="sd_multimodal_captioning" class="checkbox_label" data-i18n="[title]sd_multimodal_captioning" title="Use multimodal captioning to generate prompts for user and character portraits based on their avatars.">
                 <input id="sd_multimodal_captioning" type="checkbox" />
                 <span data-i18n="sd_multimodal_captioning_txt">Use multimodal captioning for portraits</span>

From d5d8d6c876be79c390cbea3f41e4309cd41aaca7 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 13:05:03 +0000
Subject: [PATCH 27/50] Remove warning log for unregister

---
 public/scripts/tool-calling.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index 264649753..f1ce95e91 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -171,7 +171,6 @@ export class ToolManager {
      */
     static unregisterFunctionTool(name) {
         if (!this.#tools.has(name)) {
-            console.warn(`No tool with the name "${name}" has been registered.`);
             return;
         }
 

From 8b13c40e0de24a18be6825e42d45901d7e7a6826 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 21:48:35 +0300
Subject: [PATCH 28/50] Claude: fix prefills

---
 src/prompt-converters.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/prompt-converters.js b/src/prompt-converters.js
index fc58d78f7..925e8d209 100644
--- a/src/prompt-converters.js
+++ b/src/prompt-converters.js
@@ -229,7 +229,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
         messages.push({
             role: 'assistant',
             // Dangling whitespace are not allowed for prefilling
-            content: prefillString.trimEnd(),
+            content: [{ type: 'text', text: prefillString.trimEnd() }],
         });
     }
 

From bc5e7b1ee88d5d080bd487156c3ddcdd626de99a Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 21:49:24 +0300
Subject: [PATCH 29/50] Fix description in tool definition

---
 public/scripts/extensions/stable-diffusion/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js
index 0289ff807..4ac12d412 100644
--- a/public/scripts/extensions/stable-diffusion/index.js
+++ b/public/scripts/extensions/stable-diffusion/index.js
@@ -3858,7 +3858,7 @@ function registerFunctionTool() {
                     description: [
                         'The text prompt used to generate the image.',
                         'Must represent an exhaustive description of the desired image that will allow an artist or a photographer to perfectly recreate it.',
-                    ],
+                    ].join(' '),
                 },
             },
             required: [

From 8ff9ef3610f7eb1b793062528f6b548859f54431 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 21:58:55 +0300
Subject: [PATCH 30/50] Image Generation: Fix race condition in adding images
 to chat

---
 public/scripts/extensions/stable-diffusion/index.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js
index 4ac12d412..3c0d65152 100644
--- a/public/scripts/extensions/stable-diffusion/index.js
+++ b/public/scripts/extensions/stable-diffusion/index.js
@@ -2305,9 +2305,9 @@ async function generatePicture(initiator, args, trigger, message, callback) {
             eventSource.emit(event_types.FORCE_SET_BACKGROUND, { url: imgUrl, path: imagePath });
 
             if (typeof callbackOriginal === 'function') {
-                callbackOriginal(prompt, imagePath, generationType, negativePromptPrefix, initiator);
+                await callbackOriginal(prompt, imagePath, generationType, negativePromptPrefix, initiator);
             } else {
-                sendMessage(prompt, imagePath, generationType, negativePromptPrefix, initiator);
+                await sendMessage(prompt, imagePath, generationType, negativePromptPrefix, initiator);
             }
         };
     }
@@ -2636,7 +2636,9 @@ async function sendGenerationRequest(generationType, prompt, additionalNegativeP
 
     const filename = `${characterName}_${humanizedDateTime()}`;
     const base64Image = await saveBase64AsFile(result.data, characterName, filename, result.format);
-    callback ? callback(prompt, base64Image, generationType, additionalNegativePrefix, initiator) : sendMessage(prompt, base64Image, generationType, additionalNegativePrefix, initiator);
+    callback
+        ? await callback(prompt, base64Image, generationType, additionalNegativePrefix, initiator)
+        : await sendMessage(prompt, base64Image, generationType, additionalNegativePrefix, initiator);
     return base64Image;
 }
 

From 01fc5113d74447c734df4e8dd54628abfc2dcbe7 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 23:13:56 +0300
Subject: [PATCH 31/50] Fix ghost messages

---
 public/script.js               | 16 +++++++++-------
 public/scripts/tool-calling.js | 14 ++++++++++----
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/public/script.js b/public/script.js
index 545cfe020..cd57ed49b 100644
--- a/public/script.js
+++ b/public/script.js
@@ -4418,11 +4418,12 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
             }
 
             if (canPerformToolCalls && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length) {
+                const lastMessage = chat[chat.length - 1];
+                const hasToolCalls = ToolManager.hasToolCalls(streamingProcessor.toolCalls);
+                const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor?.result);
+                hasToolCalls && shouldDeleteMessage && await deleteLastMessage();
                 const invocationResult = await ToolManager.invokeFunctionTools(streamingProcessor.toolCalls);
-                if (invocationResult.hadToolCalls) {
-                    const lastMessage = chat[chat.length - 1];
-                    const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor?.result);
-                    shouldDeleteMessage && await deleteLastMessage();
+                if (hasToolCalls) {
                     if (!invocationResult.invocations.length && shouldDeleteMessage) {
                         ToolManager.showToolCallError(invocationResult.errors);
                         unblockGeneration(type);
@@ -4512,10 +4513,11 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
         }
 
         if (canPerformToolCalls) {
+            const hasToolCalls = ToolManager.hasToolCalls(data);
+            const shouldDeleteMessage = ['', '...'].includes(getMessage);
+            hasToolCalls && shouldDeleteMessage && await deleteLastMessage();
             const invocationResult = await ToolManager.invokeFunctionTools(data);
-            if (invocationResult.hadToolCalls) {
-                const shouldDeleteMessage = ['', '...'].includes(getMessage);
-                shouldDeleteMessage && await deleteLastMessage();
+            if (hasToolCalls) {
                 if (!invocationResult.invocations.length && shouldDeleteMessage) {
                     ToolManager.showToolCallError(invocationResult.errors);
                     unblockGeneration(type);
diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index f1ce95e91..bbd3d7d34 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -14,7 +14,6 @@ import { Popup } from './popup.js';
 /**
  * @typedef {object} ToolInvocationResult
  * @property {ToolInvocation[]} invocations Successful tool invocations
- * @property {boolean} hadToolCalls Whether any tool calls were found
  * @property {Error[]} errors Errors that occurred during tool invocation
  */
 
@@ -322,7 +321,7 @@ export class ToolManager {
             const choiceIndex = 0;
             const toolCallIndex = parsed?.index ?? 0;
             const targetToolCall = toolCalls[choiceIndex]?.[toolCallIndex];
-            if (targetToolCall){
+            if (targetToolCall) {
                 if (parsed?.delta?.type === 'input_json_delta') {
                     const jsonDelta = parsed?.delta?.partial_json;
                     if (!targetToolCall[this.#INPUT_DELTA_KEY]) {
@@ -454,6 +453,15 @@ export class ToolManager {
         }
     }
 
+    /**
+     * Checks if the response data contains tool calls.
+     * @param {object} data Response data
+     * @returns {boolean} Whether the response data contains tool calls
+     */
+    static hasToolCalls(data) {
+        return Array.isArray(ToolManager.#getToolCallsFromData(data));
+    }
+
     /**
      * Check for function tool calls in the response data and invoke them.
      * @param {any} data Reply data
@@ -463,7 +471,6 @@ export class ToolManager {
         /** @type {ToolInvocationResult} */
         const result = {
             invocations: [],
-            hadToolCalls: false,
             errors: [],
         };
         const toolCalls = ToolManager.#getToolCallsFromData(data);
@@ -482,7 +489,6 @@ export class ToolManager {
             const parameters = toolCall.function.arguments;
             const name = toolCall.function.name;
             const displayName = ToolManager.getDisplayName(name);
-            result.hadToolCalls = true;
 
             const message = ToolManager.formatToolCallMessage(name, parameters);
             const toast = message && toastr.info(message, 'Tool Calling', { timeOut: 0 });

From c3078a67e1616c9162da9034346f121de3e7eae0 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 4 Oct 2024 23:33:31 +0300
Subject: [PATCH 32/50] Claude: Add prefill/tool interaction warning

---
 public/css/toggle-dependent.css | 11 +++++++++++
 public/index.html               |  4 ++++
 2 files changed, 15 insertions(+)

diff --git a/public/css/toggle-dependent.css b/public/css/toggle-dependent.css
index 9ffc382cc..08bb08fe0 100644
--- a/public/css/toggle-dependent.css
+++ b/public/css/toggle-dependent.css
@@ -463,3 +463,14 @@ body.expandMessageActions .mes .mes_buttons .extraMesButtonsHint {
 label[for="trim_spaces"]:has(input:checked) i.warning {
     display: none;
 }
+
+#claude_function_prefill_warning {
+    display: none;
+}
+
+#openai_settings:has(#openai_function_calling:checked):has(#claude_assistant_prefill:not(:placeholder-shown), #claude_assistant_impersonation:not(:placeholder-shown)) #claude_function_prefill_warning {
+    display: flex;
+    align-items: center;
+    gap: 5px;
+    margin: 10px 0;
+}
diff --git a/public/index.html b/public/index.html
index 132d4d4bf..8d48af979 100644
--- a/public/index.html
+++ b/public/index.html
@@ -1873,6 +1873,10 @@
                                             </div>
                                             <textarea id="claude_assistant_impersonation" class="text_pole textarea_compact autoSetHeight" name="assistant_impersonation" rows="2" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
                                         </div>
+                                        <div id="claude_function_prefill_warning">
+                                            <i class="fa-solid fa-circle-info"></i>
+                                            <span>Prefills won't work when function calling is enabled and any tools are registered.</span>
+                                        </div>
                                         <label for="claude_use_sysprompt" class="checkbox_label widthFreeExpand">
                                             <input id="claude_use_sysprompt" type="checkbox" />
                                             <span data-i18n="Use system prompt (Claude 2.1+ only)">

From 6a304e888426439f63d76d8b34505f4317e65834 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sat, 5 Oct 2024 00:20:06 +0300
Subject: [PATCH 33/50] Fix generation looping

---
 public/scripts/tool-calling.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index bbd3d7d34..a8c026242 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -459,7 +459,8 @@ export class ToolManager {
      * @returns {boolean} Whether the response data contains tool calls
      */
     static hasToolCalls(data) {
-        return Array.isArray(ToolManager.#getToolCallsFromData(data));
+        const toolCalls = ToolManager.#getToolCallsFromData(data);
+        return Array.isArray(toolCalls) && toolCalls.length > 0;
     }
 
     /**

From 68376f7dd02a542fa1f50380d9bbc4db243c10bf Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sat, 5 Oct 2024 00:20:28 +0300
Subject: [PATCH 34/50] Indicate no content on function result

---
 public/scripts/openai.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/public/scripts/openai.js b/public/scripts/openai.js
index 60dcac83e..124af762e 100644
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -733,7 +733,7 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul
             if (chatCompletion.canAfford(toolCallMessage)) {
                 chatCompletion.reserveBudget(toolCallMessage);
                 for (const invocation of invocations.slice().reverse()) {
-                    const toolResultMessage = new Message('tool', invocation.result, invocation.id);
+                    const toolResultMessage = new Message('tool', invocation.result || '[No content]', invocation.id);
                     const canAfford = chatCompletion.canAfford(toolResultMessage);
                     if (!canAfford) {
                         break;

From 0637223bc20b3ea00c63864a152b409bf3849a98 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sat, 5 Oct 2024 02:05:18 +0300
Subject: [PATCH 35/50] Add class to tool call messages

---
 public/script.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/public/script.js b/public/script.js
index cd57ed49b..d2e96c756 100644
--- a/public/script.js
+++ b/public/script.js
@@ -2328,6 +2328,10 @@ export function addOneMessage(mes, { type = 'normal', insertAfter = null, scroll
         newMessage.addClass('smallSysMes');
     }
 
+    if (Array.isArray(mes?.extra?.tool_invocations)) {
+        newMessage.addClass('toolCall');
+    }
+
     //shows or hides the Prompt display button
     let mesIdToFind = type == 'swipe' ? params.mesId - 1 : params.mesId;  //Number(newMessage.attr('mesId'));
 

From 3b4a455ef845ca6682cbd07e6c97be199a8a1399 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sat, 5 Oct 2024 16:09:39 +0300
Subject: [PATCH 36/50] Split Custom OAI prompt post-processing modes

---
 default/config.yaml                        |  2 +
 public/index.html                          |  3 +-
 public/scripts/openai.js                   |  7 ++
 src/endpoints/backends/chat-completions.js |  9 ++-
 src/prompt-converters.js                   | 89 ++++++++++++++++++++--
 5 files changed, 101 insertions(+), 9 deletions(-)

diff --git a/default/config.yaml b/default/config.yaml
index a7e7a747e..b3057451a 100644
--- a/default/config.yaml
+++ b/default/config.yaml
@@ -121,6 +121,8 @@ extras:
   speechToTextModel: Xenova/whisper-small
   textToSpeechModel: Xenova/speecht5_tts
 # -- OPENAI CONFIGURATION --
+# A placeholder message to use in strict prompt post-processing mode when the prompt doesn't start with a user message
+promptPlaceholder: "[Start a new chat]"
 openai:
   # Will send a random user ID to OpenAI completion API
   randomizeUserId: false
diff --git a/public/index.html b/public/index.html
index 8d48af979..7e7e157d1 100644
--- a/public/index.html
+++ b/public/index.html
@@ -3102,7 +3102,8 @@
                             <h4 data-i18n="Prompt Post-Processing">Prompt Post-Processing</h4>
                             <select id="custom_prompt_post_processing" class="text_pole" title="Applies additional processing to the prompt before sending it to the API." data-i18n="[title]Applies additional processing to the prompt before sending it to the API.">
                                 <option data-i18n="prompt_post_processing_none" value="">None</option>
-                                <option value="claude">Claude</option>
+                                <option value="merge">Merge consecutive roles</option>
+                                <option value="strict">Strict (user first, alternating roles)</option>
                             </select>
                         </form>
                         <div id="01ai_form" data-source="01ai">
diff --git a/public/scripts/openai.js b/public/scripts/openai.js
index 124af762e..b4d5bb692 100644
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -199,7 +199,10 @@ const continue_postfix_types = {
 
 const custom_prompt_post_processing_types = {
     NONE: '',
+    /** @deprecated Use MERGE instead. */
     CLAUDE: 'claude',
+    MERGE: 'merge',
+    STRICT: 'strict',
 };
 
 const sensitiveFields = [
@@ -3043,6 +3046,10 @@ function loadOpenAISettings(data, settings) {
     setNamesBehaviorControls();
     setContinuePostfixControls();
 
+    if (oai_settings.custom_prompt_post_processing === custom_prompt_post_processing_types.CLAUDE) {
+        oai_settings.custom_prompt_post_processing = custom_prompt_post_processing_types.MERGE;
+    }
+
     $('#chat_completion_source').val(oai_settings.chat_completion_source).trigger('change');
     $('#oai_max_context_unlocked').prop('checked', oai_settings.max_context_unlocked);
     $('#custom_prompt_post_processing').val(oai_settings.custom_prompt_post_processing);
diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js
index 9c29a3b44..603f4e7e4 100644
--- a/src/endpoints/backends/chat-completions.js
+++ b/src/endpoints/backends/chat-completions.js
@@ -4,7 +4,7 @@ const fetch = require('node-fetch').default;
 const { jsonParser } = require('../../express-common');
 const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY, OPENROUTER_HEADERS } = require('../../constants');
 const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml, color } = require('../../util');
-const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt, convertCohereMessages, convertMistralMessages, convertAI21Messages } = require('../../prompt-converters');
+const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt, convertCohereMessages, convertMistralMessages, convertAI21Messages, mergeMessages } = require('../../prompt-converters');
 const CohereStream = require('../../cohere-stream');
 
 const { readSecret, SECRET_KEYS } = require('../secrets');
@@ -31,8 +31,11 @@ const API_AI21 = 'https://api.ai21.com/studio/v1';
  */
 function postProcessPrompt(messages, type, charName, userName) {
     switch (type) {
+        case 'merge':
         case 'claude':
-            return convertClaudeMessages(messages, '', false, '', charName, userName).messages;
+            return mergeMessages(messages, charName, userName, false);
+        case 'strict':
+            return mergeMessages(messages, charName, userName, true);
         default:
             return messages;
     }
@@ -902,7 +905,7 @@ router.post('/generate', jsonParser, function (request, response) {
         apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);
         headers = {};
         bodyParams = {};
-        request.body.messages = postProcessPrompt(request.body.messages, 'claude', request.body.char_name, request.body.user_name);
+        request.body.messages = postProcessPrompt(request.body.messages, 'strict', request.body.char_name, request.body.user_name);
     } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.GROQ) {
         apiUrl = API_GROQ;
         apiKey = readSecret(request.user.directories, SECRET_KEYS.GROQ);
diff --git a/src/prompt-converters.js b/src/prompt-converters.js
index 925e8d209..57362b8e2 100644
--- a/src/prompt-converters.js
+++ b/src/prompt-converters.js
@@ -1,6 +1,8 @@
 require('./polyfill.js');
 const { getConfigValue } = require('./util.js');
 
+const PROMPT_PLACEHOLDER = getConfigValue('promptPlaceholder', 'Let\'s get started.');
+
 /**
  * Convert a prompt from the ChatML objects to the format used by Claude.
  * Mainly deprecated. Only used for counting tokens.
@@ -122,7 +124,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
         if (messages.length === 0 || (messages.length > 0 && messages[0].role !== 'user')) {
             messages.unshift({
                 role: 'user',
-                content: humanMsgFix || '[Start a new chat]',
+                content: humanMsgFix || PROMPT_PLACEHOLDER,
             });
         }
     }
@@ -260,7 +262,6 @@ function convertCohereMessages(messages, charName = '', userName = '') {
         'user': 'USER',
         'assistant': 'CHATBOT',
     };
-    const placeholder = '[Start a new chat]';
     let systemPrompt = '';
 
     // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
@@ -288,12 +289,12 @@ function convertCohereMessages(messages, charName = '', userName = '') {
     if (messages.length === 0) {
         messages.unshift({
             role: 'user',
-            content: placeholder,
+            content: PROMPT_PLACEHOLDER,
         });
     }
 
     const lastNonSystemMessageIndex = messages.findLastIndex(msg => msg.role === 'user' || msg.role === 'assistant');
-    const userPrompt = messages.slice(lastNonSystemMessageIndex).map(msg => msg.content).join('\n\n') || placeholder;
+    const userPrompt = messages.slice(lastNonSystemMessageIndex).map(msg => msg.content).join('\n\n') || PROMPT_PLACEHOLDER;
 
     const chatHistory = messages.slice(0, lastNonSystemMessageIndex).map(msg => {
         return {
@@ -469,7 +470,7 @@ function convertAI21Messages(messages, charName = '', userName = '') {
     if (messages.length === 0) {
         messages.unshift({
             role: 'user',
-            content: '[Start a new chat]',
+            content: PROMPT_PLACEHOLDER,
         });
     }
 
@@ -553,6 +554,83 @@ function convertMistralMessages(messages, charName = '', userName = '') {
     return messages;
 }
 
+/**
+ * Merge messages with the same consecutive role, removing names if they exist.
+ * @param {any[]} messages Messages to merge
+ * @param {string} charName Character name
+ * @param {string} userName User name
+ * @param {boolean} strict Enable strict mode: only allow one system message at the start, force user first message
+ * @returns {any[]} Merged messages
+ */
+function mergeMessages(messages, charName, userName, strict) {
+    let mergedMessages = [];
+
+    // Remove names from the messages
+    messages.forEach((message) => {
+        if (!message.content) {
+            message.content = '';
+        }
+        if (message.role === 'system' && message.name === 'example_assistant') {
+            if (charName && !message.content.startsWith(`${charName}: `)) {
+                message.content = `${charName}: ${message.content}`;
+            }
+        }
+        if (message.role === 'system' && message.name === 'example_user') {
+            if (userName && !message.content.startsWith(`${userName}: `)) {
+                message.content = `${userName}: ${message.content}`;
+            }
+        }
+        if (message.name && message.role !== 'system') {
+            if (!message.content.startsWith(`${message.name}: `)) {
+                message.content = `${message.name}: ${message.content}`;
+            }
+        }
+        if (message.role === 'tool') {
+            message.role = 'user';
+        }
+        delete message.name;
+        delete message.tool_calls;
+        delete message.tool_call_id;
+    });
+
+    // Squash consecutive messages with the same role
+    messages.forEach((message) => {
+        if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role && message.content) {
+            mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content;
+        } else {
+            mergedMessages.push(message);
+        }
+    });
+
+    // Prevent erroring out if the messages array is empty.
+    if (messages.length === 0) {
+        messages.unshift({
+            role: 'user',
+            content: PROMPT_PLACEHOLDER,
+        });
+    }
+
+    if (strict) {
+        for (let i = 0; i < mergedMessages.length; i++) {
+            // Force mid-prompt system messages to be user messages
+            if (i > 0 && mergedMessages[i].role === 'system') {
+                mergedMessages[i].role = 'user';
+            }
+        }
+        if (mergedMessages.length) {
+            if (mergedMessages[0].role === 'system' && (mergedMessages.length === 1 || mergedMessages[1].role !== 'user')) {
+                mergedMessages.splice(1, 0, { role: 'user', content: PROMPT_PLACEHOLDER });
+            }
+            else if (mergedMessages[0].role !== 'system' && mergedMessages[0].role !== 'user') {
+                mergedMessages.unshift({ role: 'user', content: PROMPT_PLACEHOLDER });
+            }
+        }
+        return mergeMessages(mergedMessages, charName, userName, false);
+    }
+
+    return mergedMessages;
+}
+
 /**
  * Convert a prompt from the ChatML objects to the format used by Text Completion API.
  * @param {object[]} messages Array of messages
@@ -586,4 +664,5 @@ module.exports = {
     convertCohereMessages,
     convertMistralMessages,
     convertAI21Messages,
+    mergeMessages,
 };

From 6185974e17a77d7d5caacd3ff2853413283b5ac7 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sat, 5 Oct 2024 18:04:08 +0300
Subject: [PATCH 37/50] Claude: Use multi-part system prompt, cache tools

---
 src/endpoints/backends/chat-completions.js | 16 +++++++++++-----
 src/prompt-converters.js                   |  6 +++---
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js
index 603f4e7e4..4836a9f8b 100644
--- a/src/endpoints/backends/chat-completions.js
+++ b/src/endpoints/backends/chat-completions.js
@@ -87,7 +87,7 @@ async function sendClaudeRequest(request, response) {
     const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
     const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);
     const divider = '-'.repeat(process.stdout.columns);
-    const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false);
+    const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false) && request.body.model.startsWith('claude-3');
 
     if (!apiKey) {
         console.log(color.red(`Claude API key is missing.\n${divider}`));
@@ -110,7 +110,7 @@ async function sendClaudeRequest(request, response) {
         }
 
         const requestBody = {
-            /** @type {any} */ system: '',
+            /** @type {any} */ system: [],
             messages: convertedPrompt.messages,
             model: request.body.model,
             max_tokens: request.body.max_tokens,
@@ -121,9 +121,11 @@ async function sendClaudeRequest(request, response) {
             stream: request.body.stream,
         };
         if (useSystemPrompt) {
-            requestBody.system = enableSystemPromptCache
-                ? [{ type: 'text', text: convertedPrompt.systemPrompt, cache_control: { type: 'ephemeral' } }]
-                : convertedPrompt.systemPrompt;
+            if (enableSystemPromptCache && Array.isArray(convertedPrompt.systemPrompt) && convertedPrompt.systemPrompt.length) {
+                convertedPrompt.systemPrompt[convertedPrompt.systemPrompt.length - 1]['cache_control'] = { type: 'ephemeral' };
+            }
+
+            requestBody.system = convertedPrompt.systemPrompt;
         } else {
             delete requestBody.system;
         }
@@ -138,6 +140,10 @@ async function sendClaudeRequest(request, response) {
                 .filter(tool => tool.type === 'function')
                 .map(tool => tool.function)
                 .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
+
+            if (enableSystemPromptCache && requestBody.tools.length) {
+                requestBody.tools[requestBody.tools.length - 1]['cache_control'] = { type: 'ephemeral' };
+            }
         }
         if (enableSystemPromptCache) {
             additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';
diff --git a/src/prompt-converters.js b/src/prompt-converters.js
index 57362b8e2..c27ff8f8a 100644
--- a/src/prompt-converters.js
+++ b/src/prompt-converters.js
@@ -95,7 +95,7 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill,
  * @param {string}   userName User name
  */
 function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFix, charName = '', userName = '') {
-    let systemPrompt = '';
+    let systemPrompt = [];
     if (useSysPrompt) {
         // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
         let i;
@@ -114,7 +114,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
                     messages[i].content = `${charName}: ${messages[i].content}`;
                 }
             }
-            systemPrompt += `${messages[i].content}\n\n`;
+            systemPrompt.push({ type: 'text', text: messages[i].content });
         }
 
         messages.splice(0, i);
@@ -246,7 +246,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
         }
     });
 
-    return { messages: mergedMessages, systemPrompt: systemPrompt.trim() };
+    return { messages: mergedMessages, systemPrompt: systemPrompt };
 }
 
 /**

From 47e3cf82e09279efc498325abdab7d42db8b0bd5 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sat, 5 Oct 2024 19:08:57 +0300
Subject: [PATCH 38/50] Don't call tools on stopped streams. Emit events

---
 public/script.js               | 12 ++++++++----
 public/scripts/tool-calling.js |  7 +++++--
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/public/script.js b/public/script.js
index d2e96c756..f2176fe39 100644
--- a/public/script.js
+++ b/public/script.js
@@ -467,6 +467,8 @@ export const event_types = {
     ONLINE_STATUS_CHANGED: 'online_status_changed',
     IMAGE_SWIPED: 'image_swiped',
     CONNECTION_PROFILE_LOADED: 'connection_profile_loaded',
+    TOOL_CALLS_PERFORMED: 'tool_calls_performed',
+    TOOL_CALLS_RENDERED: 'tool_calls_rendered',
 };
 
 export const eventSource = new EventEmitter();
@@ -4421,7 +4423,9 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
                 getMessage = continue_mag + getMessage;
             }
 
-            if (canPerformToolCalls && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length) {
+            const isStreamFinished = streamingProcessor && !streamingProcessor.isStopped && streamingProcessor.isFinished;
+            const isStreamWithToolCalls = streamingProcessor && Array.isArray(streamingProcessor.toolCalls) && streamingProcessor.toolCalls.length;
+            if (canPerformToolCalls && isStreamFinished && isStreamWithToolCalls) {
                 const lastMessage = chat[chat.length - 1];
                 const hasToolCalls = ToolManager.hasToolCalls(streamingProcessor.toolCalls);
                 const shouldDeleteMessage = ['', '...'].includes(lastMessage?.mes) && ['', '...'].includes(streamingProcessor?.result);
@@ -4437,12 +4441,12 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
                     }
 
                     streamingProcessor = null;
-                    ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
+                    await ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
                     return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
                 }
             }
 
-            if (streamingProcessor && !streamingProcessor.isStopped && streamingProcessor.isFinished) {
+            if (isStreamFinished) {
                 await streamingProcessor.onFinishStreaming(streamingProcessor.messageId, getMessage);
                 streamingProcessor = null;
                 triggerAutoContinue(messageChunk, isImpersonate);
@@ -4529,7 +4533,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
                     return;
                 }
 
-                ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
+                await ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
                 return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
             }
         }
diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index a8c026242..ffef21071 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -1,4 +1,4 @@
-import { addOneMessage, chat, main_api, system_avatar, systemUserName } from '../script.js';
+import { addOneMessage, chat, event_types, eventSource, main_api, saveChatConditional, system_avatar, systemUserName } from '../script.js';
 import { chat_completion_sources, oai_settings } from './openai.js';
 import { Popup } from './popup.js';
 
@@ -542,7 +542,7 @@ export class ToolManager {
      * Saves function tool invocations to the last user chat message extra metadata.
      * @param {ToolInvocation[]} invocations Successful tool invocations
      */
-    static saveFunctionToolInvocations(invocations) {
+    static async saveFunctionToolInvocations(invocations) {
         if (!Array.isArray(invocations) || invocations.length === 0) {
             return;
         }
@@ -558,7 +558,10 @@ export class ToolManager {
             },
         };
         chat.push(message);
+        await eventSource.emit(event_types.TOOL_CALLS_PERFORMED, invocations);
         addOneMessage(message);
+        await eventSource.emit(event_types.TOOL_CALLS_RENDERED, invocations);
+        await saveChatConditional();
     }
 
     /**

From 54837680f772f94cf14c5dc379711cc60023f36b Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sat, 5 Oct 2024 20:54:25 +0300
Subject: [PATCH 39/50] Fix looping on regen tool call

---
 public/script.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/public/script.js b/public/script.js
index ec68a916f..027fcf0c6 100644
--- a/public/script.js
+++ b/public/script.js
@@ -4485,7 +4485,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
 
                     streamingProcessor = null;
                     await ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
-                    return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
+                    return Generate('normal', { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
                 }
             }
 
@@ -4577,7 +4577,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
                 }
 
                 await ToolManager.saveFunctionToolInvocations(invocationResult.invocations);
-                return Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
+                return Generate('normal', { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName }, dryRun);
             }
         }
 

From 7dea59f026721c0c6b03067abdcb1ccc07dabbb9 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sat, 5 Oct 2024 20:54:37 +0300
Subject: [PATCH 40/50] Format tool result if JSON

---
 public/scripts/tool-calling.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index ffef21071..922b9dc14 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -529,7 +529,10 @@ export class ToolManager {
         const preElement = document.createElement('pre');
         const codeElement = document.createElement('code');
         codeElement.classList.add('language-json');
-        data.forEach(i => i.parameters = tryParse(i.parameters));
+        data.forEach(i => {
+            i.parameters = tryParse(i.parameters);
+            i.result = tryParse(i.result);
+        });
         codeElement.textContent = JSON.stringify(data, null, 2);
         const toolNames = data.map(i => i.displayName || i.name).join(', ');
         summaryElement.textContent = `Tool calls: ${toolNames}`;

From 077ba8b03d38c6e61be21229d6c2157cd0def533 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sun, 6 Oct 2024 12:49:08 +0300
Subject: [PATCH 41/50] Add slash commands for tools management

---
 public/script.js               |   1 +
 public/scripts/tool-calling.js | 286 ++++++++++++++++++++++++++++++++-
 2 files changed, 285 insertions(+), 2 deletions(-)

diff --git a/public/script.js b/public/script.js
index 027fcf0c6..2acdcc02b 100644
--- a/public/script.js
+++ b/public/script.js
@@ -948,6 +948,7 @@ async function firstLoadInit() {
     initSystemPrompts();
     initExtensions();
     initExtensionSlashCommands();
+    ToolManager.initToolSlashCommands();
     await initPresetManager();
     await getSystemMessages();
     sendSystemMessage(system_message_types.WELCOME);
diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index 922b9dc14..43312d3d8 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -1,6 +1,13 @@
 import { addOneMessage, chat, event_types, eventSource, main_api, saveChatConditional, system_avatar, systemUserName } from '../script.js';
 import { chat_completion_sources, oai_settings } from './openai.js';
 import { Popup } from './popup.js';
+import { SlashCommand } from './slash-commands/SlashCommand.js';
+import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from './slash-commands/SlashCommandArgument.js';
+import { SlashCommandClosure } from './slash-commands/SlashCommandClosure.js';
+import { enumIcons } from './slash-commands/SlashCommandCommonEnumsProvider.js';
+import { enumTypes, SlashCommandEnumValue } from './slash-commands/SlashCommandEnumValue.js';
+import { SlashCommandParser } from './slash-commands/SlashCommandParser.js';
+import { slashCommandReturnHelper } from './slash-commands/SlashCommandReturnHelper.js';
 
 /**
  * @typedef {object} ToolInvocation
@@ -27,6 +34,60 @@ import { Popup } from './popup.js';
  * @property {function} formatMessage - A function to format the tool call message.
  */
 
+/**
+ * @typedef {object} ToolDefinitionOpenAI
+ * @property {string} type - The type of the tool.
+ * @property {object} function - The function definition.
+ * @property {string} function.name - The name of the function.
+ * @property {string} function.description - The description of the function.
+ * @property {object} function.parameters - The parameters of the function.
+ * @property {function} toString - A function to convert the tool to a string.
+ */
+
+/**
+ * Assigns nested variables to a scope.
+ * @param {import('./slash-commands/SlashCommandScope.js').SlashCommandScope} scope The scope to assign variables to.
+ * @param {object} arg Object to assign variables from.
+ * @param {string} prefix Prefix for the variable names.
+ */
+function assignNestedVariables(scope, arg, prefix) {
+    Object.entries(arg).forEach(([key, value]) => {
+        const newPrefix = `${prefix}.${key}`;
+        if (typeof value === 'object' && value !== null) {
+            assignNestedVariables(scope, value, newPrefix);
+        } else {
+            scope.letVariable(newPrefix, value);
+        }
+    });
+}
+
+/**
+ * Checks if a string is a valid JSON string.
+ * @param {string} str The string to check
+ * @returns {boolean} If the string is a valid JSON string
+ */
+function isJson(str) {
+    try {
+        JSON.parse(str);
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+/**
+ * Tries to parse a string as JSON, returning the original string if parsing fails.
+ * @param {string} str The string to try to parse
+ * @returns {object|string} Parsed JSON or the original string
+ */
+function tryParse(str) {
+    try {
+        return JSON.parse(str);
+    } catch {
+        return str;
+    }
+}
+
 /**
  * A class that represents a tool definition.
  */
@@ -87,7 +148,7 @@ class ToolDefinition {
 
     /**
      * Converts the ToolDefinition to an OpenAI API representation
-     * @returns {object} OpenAI API representation of the tool.
+     * @returns {ToolDefinitionOpenAI} OpenAI API representation of the tool.
      */
     toFunctionOpenAI() {
         return {
@@ -97,6 +158,9 @@ class ToolDefinition {
                 description: this.#description,
                 parameters: this.#parameters,
             },
+            toString: (/** @type {ToolDefinitionOpenAI} */ tool) => {
+                return `${tool?.function?.name} - ${tool?.function?.description}\n${JSON.stringify(tool?.function?.parameters, null, 2)}`;
+            },
         };
     }
 
@@ -522,7 +586,6 @@ export class ToolManager {
      * @returns {string} Formatted message with tool invocations.
      */
     static #formatToolInvocationMessage(invocations) {
-        const tryParse = (x) => { try { return JSON.parse(x); } catch { return x; } };
         const data = structuredClone(invocations);
         const detailsElement = document.createElement('details');
         const summaryElement = document.createElement('summary');
@@ -578,4 +641,223 @@ export class ToolManager {
             timeOut: 5000,
         });
     }
+
+    static initToolSlashCommands() {
+        const toolsEnumProvider = () => ToolManager.tools.map(tool => {
+            const toolOpenAI = tool.toFunctionOpenAI();
+            return new SlashCommandEnumValue(toolOpenAI.function.name, toolOpenAI.function.description, enumTypes.enum, enumIcons.closure);
+        });
+
+        SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+            name: 'tools-list',
+            aliases: ['tool-list'],
+            helpString: 'Gets a list of all registered tools in the OpenAI function JSON format. Use the <code>return</code> argument to specify the return value type.',
+            returns: 'A list of all registered tools.',
+            namedArgumentList: [
+                SlashCommandNamedArgument.fromProps({
+                    name: 'return',
+                    description: 'The way how you want the return value to be provided',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    defaultValue: 'none',
+                    enumList: slashCommandReturnHelper.enumList({ allowObject: true }),
+                    forceEnum: true,
+                }),
+            ],
+            callback: async (args) => {
+                /** @type {any} */
+                const returnType = String(args?.return ?? 'popup-html').trim().toLowerCase();
+                const objectToStringFunc = (tool) => tool.toString();
+                const tools = ToolManager.tools.map(tool => tool.toFunctionOpenAI());
+                return await slashCommandReturnHelper.doReturn(returnType ?? 'popup-html', tools ?? [], { objectToStringFunc });
+            },
+        }));
+
+        SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+            name: 'tools-invoke',
+            aliases: ['tool-invoke'],
+            helpString: 'Invokes a registered tool by name. The <code>parameters</code> argument MUST be a JSON-serialized object.',
+            namedArgumentList: [
+                SlashCommandNamedArgument.fromProps({
+                    name: 'parameters',
+                    description: 'The parameters to pass to the tool.',
+                    typeList: [ARGUMENT_TYPE.DICTIONARY],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+            ],
+            unnamedArgumentList: [
+                SlashCommandArgument.fromProps({
+                    description: 'The name of the tool to invoke.',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                    forceEnum: true,
+                    enumProvider: toolsEnumProvider,
+                }),
+            ],
+            callback: async (args, name) => {
+                const { parameters } = args;
+
+                const result = await ToolManager.invokeFunctionTool(String(name), parameters);
+                if (result instanceof Error) {
+                    throw result;
+                }
+
+                return result;
+            },
+        }));
+
+        SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+            name: 'tools-register',
+            aliases: ['tool-register'],
+            helpString: `<div>Registers a new tool with the tool registry.</div>
+                <ul>
+                    <li>The <code>parameters</code> argument MUST be a JSON-serialized object with a valid JSON schema.</li>
+                    <li>The unnamed argument MUST be a closure that accepts the function parameters as local script variables.</li>
+                </ul>
+                <div>See <a target="_blank" href="https://json-schema.org/learn/">json-schema.org</a> and <a target="_blank" href="https://platform.openai.com/docs/guides/function-calling">OpenAI Function Calling</a> for more information.</div>
+                <div>Example:</div>
+                <pre><code>/let key=echoSchema
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "type": "object",
+    "properties": {
+        "message": {
+            "type": "string",
+            "description": "The message to echo."
+        }
+    },
+    "required": [
+        "message"
+    ]
+}
+||
+/tools-register name=Echo description="Echoes a message. Call when the user is asking to repeat something" parameters={{var::echoSchema}} {: /echo {{var::arg.message}} :}</code></pre>`,
+            namedArgumentList: [
+                SlashCommandNamedArgument.fromProps({
+                    name: 'name',
+                    description: 'The name of the tool.',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+                SlashCommandNamedArgument.fromProps({
+                    name: 'description',
+                    description: 'A description of what the tool does.',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+                SlashCommandNamedArgument.fromProps({
+                    name: 'parameters',
+                    description: 'The parameters for the tool.',
+                    typeList: [ARGUMENT_TYPE.DICTIONARY],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+                SlashCommandNamedArgument.fromProps({
+                    name: 'displayName',
+                    description: 'The display name of the tool.',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    isRequired: false,
+                    acceptsMultiple: false,
+                }),
+                SlashCommandNamedArgument.fromProps({
+                    name: 'formatMessage',
+                    description: 'The closure to be executed to format the tool call message. Must return a string.',
+                    typeList: [ARGUMENT_TYPE.CLOSURE],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+            ],
+            unnamedArgumentList: [
+                SlashCommandArgument.fromProps({
+                    description: 'The closure to be executed when the tool is invoked.',
+                    typeList: [ARGUMENT_TYPE.CLOSURE],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                }),
+            ],
+            callback: async (args, action) => {
+                /**
+                 * Converts a slash command closure to a function.
+                 * @param {SlashCommandClosure} action Closure to convert to a function
+                 * @returns {function} Function that executes the closure
+                 */
+                function closureToFunction(action) {
+                    return async (args) => {
+                        const localClosure = action.getCopy();
+                        localClosure.onProgress = () => { };
+                        const scope = localClosure.scope;
+                        if (typeof args === 'object' && args !== null) {
+                            assignNestedVariables(scope, args, 'arg');
+                        } else if (typeof args !== 'undefined') {
+                            scope.letVariable('arg', args);
+                        }
+                        const result = await localClosure.execute();
+                        return result.pipe;
+                    };
+                }
+
+                const { name, displayName, description, parameters, formatMessage } = args;
+
+                if (!(action instanceof SlashCommandClosure)) {
+                    throw new Error('The unnamed argument must be a closure.');
+                }
+                if (typeof name !== 'string' || !name) {
+                    throw new Error('The "name" argument must be a non-empty string.');
+                }
+                if (typeof description !== 'string' || !description) {
+                    throw new Error('The "description" argument must be a non-empty string.');
+                }
+                if (typeof parameters !== 'string' || !isJson(parameters)) {
+                    throw new Error('The "parameters" argument must be a JSON-serialized object.');
+                }
+                if (displayName && typeof displayName !== 'string') {
+                    throw new Error('The "displayName" argument must be a string.');
+                }
+                if (formatMessage && !(formatMessage instanceof SlashCommandClosure)) {
+                    throw new Error('The "formatMessage" argument must be a closure.');
+                }
+
+                const actionFunc = closureToFunction(action);
+                const formatMessageFunc = formatMessage instanceof SlashCommandClosure ? closureToFunction(formatMessage) : null;
+
+                ToolManager.registerFunctionTool({
+                    name: String(name ?? ''),
+                    displayName: String(displayName ?? ''),
+                    description: String(description ?? ''),
+                    parameters: JSON.parse(parameters ?? '{}'),
+                    action: actionFunc,
+                    formatMessage: formatMessageFunc,
+                });
+
+                return '';
+            },
+        }));
+
+        SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+            name: 'tools-unregister',
+            aliases: ['tool-unregister'],
+            helpString: 'Unregisters a tool from the tool registry.',
+            unnamedArgumentList: [
+                SlashCommandArgument.fromProps({
+                    description: 'The name of the tool to unregister.',
+                    typeList: [ARGUMENT_TYPE.STRING],
+                    isRequired: true,
+                    acceptsMultiple: false,
+                    forceEnum: true,
+                    enumProvider: toolsEnumProvider,
+                }),
+            ],
+            callback: async (name) => {
+                if (typeof name !== 'string' || !name) {
+                    throw new Error('The unnamed argument must be a non-empty string.');
+                }
+
+                ToolManager.unregisterFunctionTool(name);
+                return '';
+            },
+        }));
+    }
 }

From 96862be6b02a4a0b4a9ff13ed7ca2486c8ea89f8 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sun, 6 Oct 2024 13:01:14 +0300
Subject: [PATCH 42/50] Fix tools-list

---
 public/css/st-tailwind.css                                | 4 ++++
 public/scripts/slash-commands/SlashCommandReturnHelper.js | 2 +-
 public/scripts/tool-calling.js                            | 6 +++---
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/public/css/st-tailwind.css b/public/css/st-tailwind.css
index 69f5843a9..88e4ccb10 100644
--- a/public/css/st-tailwind.css
+++ b/public/css/st-tailwind.css
@@ -612,3 +612,7 @@ ul.li-padding-bot5 li {
 ul.li-padding-bot10 li {
     padding-bottom: 10px;
 }
+
+.wordBreakAll {
+    word-break: break-all;
+}
diff --git a/public/scripts/slash-commands/SlashCommandReturnHelper.js b/public/scripts/slash-commands/SlashCommandReturnHelper.js
index 3601c30cd..947e92cd3 100644
--- a/public/scripts/slash-commands/SlashCommandReturnHelper.js
+++ b/public/scripts/slash-commands/SlashCommandReturnHelper.js
@@ -58,7 +58,7 @@ export const slashCommandReturnHelper = {
             case 'toast-html': {
                 const htmlOrNotHtml = shouldHtml ? DOMPurify.sanitize((new showdown.Converter()).makeHtml(stringValue)) : escapeHtml(stringValue);
 
-                if (type.startsWith('popup')) await callGenericPopup(htmlOrNotHtml, POPUP_TYPE.TEXT);
+                if (type.startsWith('popup')) await callGenericPopup(htmlOrNotHtml, POPUP_TYPE.TEXT, '', { allowVerticalScrolling: true, wide: true });
                 if (type.startsWith('chat')) sendSystemMessage(system_message_types.GENERIC, htmlOrNotHtml);
                 if (type.startsWith('toast')) toastr.info(htmlOrNotHtml, null, { escapeHtml: !shouldHtml });
 
diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index 43312d3d8..2e5c0a65c 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -158,8 +158,8 @@ class ToolDefinition {
                 description: this.#description,
                 parameters: this.#parameters,
             },
-            toString: (/** @type {ToolDefinitionOpenAI} */ tool) => {
-                return `${tool?.function?.name} - ${tool?.function?.description}\n${JSON.stringify(tool?.function?.parameters, null, 2)}`;
+            toString: function() {
+                return `<div><b>${this.function.name}</b></div><div><small>${this.function.description}</small></div><pre class="justifyLeft wordBreakAll"><code class="flex padding5">${JSON.stringify(this.function.parameters, null, 2)}</code></pre><hr>`;
             },
         };
     }
@@ -666,7 +666,7 @@ export class ToolManager {
             callback: async (args) => {
                 /** @type {any} */
                 const returnType = String(args?.return ?? 'popup-html').trim().toLowerCase();
-                const objectToStringFunc = (tool) => tool.toString();
+                const objectToStringFunc = (tools) => Array.isArray(tools) ? tools.map(x => x.toString()).join('\n\n') : tools.toString();
                 const tools = ToolManager.tools.map(tool => tool.toFunctionOpenAI());
                 return await slashCommandReturnHelper.doReturn(returnType ?? 'popup-html', tools ?? [], { objectToStringFunc });
             },

From 303df1087adfcc07fc259d10cdcf3c9dc3492b69 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sun, 6 Oct 2024 13:27:06 +0300
Subject: [PATCH 43/50] ImageGen: Configurable tool prompt description, encode
 response URI

---
 .../extensions/stable-diffusion/index.js      | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js
index 3c0d65152..f7e569c75 100644
--- a/public/scripts/extensions/stable-diffusion/index.js
+++ b/public/scripts/extensions/stable-diffusion/index.js
@@ -67,6 +67,7 @@ const initiators = {
 };
 
 const generationMode = {
+    TOOL: -2,
     MESSAGE: -1,
     CHARACTER: 0,
     USER: 1,
@@ -89,6 +90,7 @@ const multimodalMap = {
 };
 
 const modeLabels = {
+    [generationMode.TOOL]: 'Function Tool Prompt Description',
     [generationMode.MESSAGE]: 'Chat Message Template',
     [generationMode.CHARACTER]: 'Character ("Yourself")',
     [generationMode.FACE]: 'Portrait ("Your Face")',
@@ -126,8 +128,12 @@ const messageTrigger = {
 };
 
 const promptTemplates = {
-    // Not really a prompt template, rather an outcome message template
+    // Not really a prompt template, rather an outcome message template and function tool prompt
     [generationMode.MESSAGE]: '[{{char}} sends a picture that contains: {{prompt}}].',
+    [generationMode.TOOL]: [
+        'The text prompt used to generate the image.',
+        'Must represent an exhaustive description of the desired image that will allow an artist or a photographer to perfectly recreate it.',
+    ].join(' '),
     [generationMode.CHARACTER]: 'In the next response I want you to provide only a detailed comma-delimited list of keywords and phrases which describe {{char}}. The list must include all of the following items in this order: name, species and race, gender, age, clothing, occupation, physical features and appearances. Do not include descriptions of non-visual qualities such as personality, movements, scents, mental traits, or anything which could not be seen in a still photograph. Do not write in full sentences. Prefix your description with the phrase \'full body portrait,\'',
     //face-specific prompt
     [generationMode.FACE]: 'In the next response I want you to provide only a detailed comma-delimited list of keywords and phrases which describe {{char}}. The list must include all of the following items in this order: name, species and race, gender, age, facial features and expressions, occupation, hair and hair accessories (if any), what they are wearing on their upper body (if anything). Do not describe anything below their neck. Do not include descriptions of non-visual qualities such as personality, movements, scents, mental traits, or anything which could not be seen in a still photograph. Do not write in full sentences. Prefix your description with the phrase \'close up facial portrait,\'',
@@ -533,6 +539,9 @@ function addPromptTemplates() {
             .on('click', () => {
                 textarea.val(promptTemplates[name]);
                 extension_settings.sd.prompts[name] = promptTemplates[name];
+                if (String(name) === String(generationMode.TOOL)) {
+                    registerFunctionTool();
+                }
                 saveSettingsDebounced();
             });
         const container = $('<div></div>')
@@ -3857,10 +3866,7 @@ function registerFunctionTool() {
             properties: {
                 prompt: {
                     type: 'string',
-                    description: [
-                        'The text prompt used to generate the image.',
-                        'Must represent an exhaustive description of the desired image that will allow an artist or a photographer to perfectly recreate it.',
-                    ].join(' '),
+                    description: extension_settings.sd.prompts[generationMode.TOOL] || promptTemplates[generationMode.TOOL],
                 },
             },
             required: [
@@ -3871,7 +3877,8 @@ function registerFunctionTool() {
             if (!isValidState()) throw new Error('Image generation is not configured.');
             if (!args) throw new Error('Missing arguments');
             if (!args.prompt) throw new Error('Missing prompt');
-            return generatePicture(initiators.tool, {}, args.prompt);
+            const url = await generatePicture(initiators.tool, {}, args.prompt);
+            return encodeURI(url);
         },
         formatMessage: () => 'Generating an image...',
     });

From f66261a52cb57c092bedeca7164ae5fb468a2375 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sun, 6 Oct 2024 14:28:52 +0300
Subject: [PATCH 44/50] Mistral: Squash user messages that follow tools

---
 src/prompt-converters.js | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/prompt-converters.js b/src/prompt-converters.js
index c27ff8f8a..d3e910859 100644
--- a/src/prompt-converters.js
+++ b/src/prompt-converters.js
@@ -544,6 +544,28 @@ function convertMistralMessages(messages, charName = '', userName = '') {
         }
     });
 
+    // If user role message immediately follows a tool message, append it to the last user message
+    const fixToolMessages = () => {
+        let rerun = true;
+        while (rerun) {
+            rerun = false;
+            messages.forEach((message, i) => {
+                if (i === messages.length - 1) {
+                    return;
+                }
+                if (message.role === 'tool' && messages[i + 1].role === 'user') {
+                    const lastUserMessage = messages.slice(0, i).findLastIndex(m => m.role === 'user' && m.content);
+                    if (lastUserMessage !== -1) {
+                        messages[lastUserMessage].content += '\n\n' + messages[i + 1].content;
+                        messages.splice(i + 1, 1);
+                        rerun = true;
+                    }
+                }
+            });
+        }
+    }
+    fixToolMessages();
+
     // If system role message immediately follows an assistant message, change its role to user
     for (let i = 0; i < messages.length - 1; i++) {
         if (messages[i].role === 'assistant' && messages[i + 1].role === 'system') {

From c8ef4b02bcfba124b3fd6e92a183c44b13d11657 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sun, 6 Oct 2024 19:12:28 +0300
Subject: [PATCH 45/50] Add .git to jsconfig ignore

---
 .eslintrc.js                    | 19 +++++++++++--------
 jsconfig.json                   | 19 +++++++++----------
 public/script.js                |  2 +-
 public/scripts/instruct-mode.js |  2 --
 public/scripts/openai.js        |  1 -
 src/endpoints/novelai.js        |  2 +-
 src/prompt-converters.js        |  2 +-
 7 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/.eslintrc.js b/.eslintrc.js
index 8d9e36fe3..8d8386505 100644
--- a/.eslintrc.js
+++ b/.eslintrc.js
@@ -59,15 +59,18 @@ module.exports = {
             },
         },
     ],
-    // There are various vendored libraries that shouldn't be linted
     ignorePatterns: [
-        'public/lib/**/*',
-        '*.min.js',
-        'src/ai_horde/**/*',
-        'plugins/**/*',
-        'data/**/*',
-        'backups/**/*',
-        'node_modules/**/*',
+        '**/node_modules/**',
+        '**/dist/**',
+        '**/.git/**',
+        'public/lib/**',
+        'backups/**',
+        'data/**',
+        'cache/**',
+        'src/tokenizers/**',
+        'docker/**',
+        'plugins/**',
+        '**/*.min.js',
     ],
     rules: {
         'no-unused-vars': ['error', { args: 'none' }],
diff --git a/jsconfig.json b/jsconfig.json
index a48606f1f..0e554015b 100644
--- a/jsconfig.json
+++ b/jsconfig.json
@@ -11,15 +11,14 @@
         "resolveJsonModule": true
     },
     "exclude": [
-        "node_modules",
-        "**/node_modules/*",
-        "public/lib",
-        "backups/*",
-        "data/*",
-        "**/dist/*",
-        "dist/*",
-        "cache/*",
-        "src/tokenizers/*",
-        "docker/*",
+        "**/node_modules/**",
+        "**/dist/**",
+        "**/.git/**",
+        "public/lib/**",
+        "backups/**",
+        "data/**",
+        "cache/**",
+        "src/tokenizers/**",
+        "docker/**"
     ]
 }
diff --git a/public/script.js b/public/script.js
index 2acdcc02b..710526f3b 100644
--- a/public/script.js
+++ b/public/script.js
@@ -2029,7 +2029,7 @@ export function messageFormatting(mes, ch_name, isSystem, isUser, messageId, san
                     // Return the original match if no quotes are found
                     return match;
                 }
-            }
+            },
         );
 
         // Restore double quotes in tags
diff --git a/public/scripts/instruct-mode.js b/public/scripts/instruct-mode.js
index ba641935f..c5b70ac52 100644
--- a/public/scripts/instruct-mode.js
+++ b/public/scripts/instruct-mode.js
@@ -554,11 +554,9 @@ export function formatInstructModePrompt(name, isImpersonate, promptBias, name1,
  * @param {string} name Preset name.
  */
 function selectMatchingContextTemplate(name) {
-    let foundMatch = false;
     for (const context_preset of context_presets) {
         // If context template matches the instruct preset
         if (context_preset.name === name) {
-            foundMatch = true;
             selectContextPreset(context_preset.name, { isAuto: true });
             break;
         }
diff --git a/public/scripts/openai.js b/public/scripts/openai.js
index 7a9ea9368..3d8355216 100644
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -1715,7 +1715,6 @@ async function sendOpenAIRequest(type, messages, signal) {
     messages = messages.filter(msg => msg && typeof msg === 'object');
 
     let logit_bias = {};
-    const messageId = getNextMessageId(type);
     const isClaude = oai_settings.chat_completion_source == chat_completion_sources.CLAUDE;
     const isOpenRouter = oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER;
     const isScale = oai_settings.chat_completion_source == chat_completion_sources.SCALE;
diff --git a/src/endpoints/novelai.js b/src/endpoints/novelai.js
index 2d7e6887d..e2c6fef19 100644
--- a/src/endpoints/novelai.js
+++ b/src/endpoints/novelai.js
@@ -51,7 +51,7 @@ const eratoRepPenWhitelist = [
     6, 1, 11, 13, 25, 198, 12, 9, 8, 279, 264, 459, 323, 477, 539, 912, 374, 574, 1051, 1550, 1587, 4536, 5828, 15058,
     3287, 3250, 1461, 1077, 813, 11074, 872, 1202, 1436, 7846, 1288, 13434, 1053, 8434, 617, 9167, 1047, 19117, 706,
     12775, 649, 4250, 527, 7784, 690, 2834, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 1210, 1359, 608, 220, 596, 956,
-    3077, 44886, 4265, 3358, 2351, 2846, 311, 389, 315, 304, 520, 505, 430
+    3077, 44886, 4265, 3358, 2351, 2846, 311, 389, 315, 304, 520, 505, 430,
 ];
 
 // Ban the dinkus and asterism
diff --git a/src/prompt-converters.js b/src/prompt-converters.js
index d3e910859..38ac48f35 100644
--- a/src/prompt-converters.js
+++ b/src/prompt-converters.js
@@ -563,7 +563,7 @@ function convertMistralMessages(messages, charName = '', userName = '') {
                 }
             });
         }
-    }
+    };
     fixToolMessages();
 
     // If system role message immediately follows an assistant message, change its role to user

From 680dc806495ceed93c7e4133a8ec9afb7942b4fd Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sun, 6 Oct 2024 19:26:37 +0300
Subject: [PATCH 46/50] Extend exclude list for frontend jsconfig

---
 public/jsconfig.json | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/public/jsconfig.json b/public/jsconfig.json
index f362b9731..7905ed388 100644
--- a/public/jsconfig.json
+++ b/public/jsconfig.json
@@ -8,7 +8,11 @@
         "allowSyntheticDefaultImports": true
     },
     "exclude": [
-        "node_modules"
+        "**/node_modules/**",
+        "**/dist/**",
+        "**/.git/**",
+        "lib/**",
+        "**/*.min.js"
     ],
     "typeAcquisition": {
         "include": [

From 2e36f45e241363d21e6f05433b4570f5bfce1013 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sun, 6 Oct 2024 19:51:59 +0300
Subject: [PATCH 47/50] Fix types for dompurify and toastr

---
 package-lock.json    | 46 ++++++++++++++++++++++++++++++++------------
 package.json         |  5 +++--
 public/global.d.ts   |  1 -
 public/jsconfig.json |  4 ----
 4 files changed, 37 insertions(+), 19 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 94134d9b0..59b3cad86 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -59,9 +59,10 @@
                 "sillytavern": "server.js"
             },
             "devDependencies": {
+                "@types/dompurify": "^3.0.5",
                 "@types/jquery": "^3.5.29",
-                "eslint": "^8.57.0",
-                "jquery": "^3.6.4"
+                "@types/toastr": "^2.1.43",
+                "eslint": "^8.57.0"
             },
             "engines": {
                 "node": ">= 18"
@@ -960,6 +961,16 @@
                 "@types/responselike": "^1.0.0"
             }
         },
+        "node_modules/@types/dompurify": {
+            "version": "3.0.5",
+            "resolved": "https://registry.npmjs.org/@types/dompurify/-/dompurify-3.0.5.tgz",
+            "integrity": "sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@types/trusted-types": "*"
+            }
+        },
         "node_modules/@types/http-cache-semantics": {
             "version": "4.0.2",
             "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz",
@@ -967,10 +978,11 @@
             "license": "MIT"
         },
         "node_modules/@types/jquery": {
-            "version": "3.5.29",
-            "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.29.tgz",
-            "integrity": "sha512-oXQQC9X9MOPRrMhPHHOsXqeQDnWeCDT3PelUIg/Oy8FAbzSZtFHRjc7IpbfFVmpLtJ+UOoywpRsuO5Jxjybyeg==",
+            "version": "3.5.31",
+            "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.31.tgz",
+            "integrity": "sha512-rf/iB+cPJ/YZfMwr+FVuQbm7IaWC4y3FVYfVDxRGqmUCFjjPII0HWaP0vTPJGp6m4o13AXySCcMbWfrWtBFAKw==",
             "dev": true,
+            "license": "MIT",
             "dependencies": {
                 "@types/sizzle": "*"
             }
@@ -1021,6 +1033,23 @@
             "integrity": "sha512-0vWLNK2D5MT9dg0iOo8GlKguPAU02QjmZitPEsXRuJXU/OGIOt9vT9Fc26wtYuavLxtO45v9PGleoL9Z0k1LHg==",
             "dev": true
         },
+        "node_modules/@types/toastr": {
+            "version": "2.1.43",
+            "resolved": "https://registry.npmjs.org/@types/toastr/-/toastr-2.1.43.tgz",
+            "integrity": "sha512-sLC2fr2OXeE1iyhUixpQ64wQ2tA26awmLidn4tXTLBz4yP/VhtYUKHpmiIyDtztKkHjucdiTLH8F5uRRyhNi2Q==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@types/jquery": "*"
+            }
+        },
+        "node_modules/@types/trusted-types": {
+            "version": "2.0.7",
+            "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz",
+            "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/@ungap/structured-clone": {
             "version": "1.2.0",
             "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",
@@ -3881,13 +3910,6 @@
             "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==",
             "license": "BSD-3-Clause"
         },
-        "node_modules/jquery": {
-            "version": "3.7.0",
-            "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.7.0.tgz",
-            "integrity": "sha512-umpJ0/k8X0MvD1ds0P9SfowREz2LenHsQaxSohMZ5OMNEU2r0tf8pdeEFTHMFxWVxKNyU9rTtK3CWzUCTKJUeQ==",
-            "dev": true,
-            "license": "MIT"
-        },
         "node_modules/js-yaml": {
             "version": "4.1.0",
             "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
diff --git a/package.json b/package.json
index 382d6ab57..97ec01a9c 100644
--- a/package.json
+++ b/package.json
@@ -85,8 +85,9 @@
     },
     "main": "server.js",
     "devDependencies": {
+        "@types/dompurify": "^3.0.5",
         "@types/jquery": "^3.5.29",
-        "eslint": "^8.57.0",
-        "jquery": "^3.6.4"
+        "@types/toastr": "^2.1.43",
+        "eslint": "^8.57.0"
     }
 }
diff --git a/public/global.d.ts b/public/global.d.ts
index 39ba4ea0c..a33182d46 100644
--- a/public/global.d.ts
+++ b/public/global.d.ts
@@ -1,5 +1,4 @@
 // Global namespace modules
-declare var DOMPurify;
 declare var droll;
 declare var Handlebars;
 declare var hljs;
diff --git a/public/jsconfig.json b/public/jsconfig.json
index 7905ed388..593c2df94 100644
--- a/public/jsconfig.json
+++ b/public/jsconfig.json
@@ -16,12 +16,8 @@
     ],
     "typeAcquisition": {
         "include": [
-            "jquery",
             "@popperjs/core",
-            "toastr",
             "showdown",
-            "dompurify",
-            "moment",
             "seedrandom",
             "showdown-katex",
             "droll",

From 2a25f75fa5e6c8160dd195a7686b6220de977d3e Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sun, 6 Oct 2024 20:07:43 +0300
Subject: [PATCH 48/50] Mistral: sanitize tool call id

---
 src/prompt-converters.js | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/prompt-converters.js b/src/prompt-converters.js
index 38ac48f35..fe9f8e88f 100644
--- a/src/prompt-converters.js
+++ b/src/prompt-converters.js
@@ -522,8 +522,18 @@ function convertMistralMessages(messages, charName = '', userName = '') {
         lastMsg.prefix = true;
     }
 
+    const sanitizeToolId = (id) => id.replace(/[^a-zA-Z0-9]/g, '').slice(0, 9);
+
     // Doesn't support completion names, so prepend if not already done by the frontend (e.g. for group chats).
     messages.forEach(msg => {
+        if ('tool_calls' in msg && Array.isArray(msg.tool_calls)) {
+            msg.tool_calls.forEach(tool => {
+                tool.id = sanitizeToolId(tool.id);
+            });
+        }
+        if ('tool_call_id' in msg && msg.role === 'tool') {
+            msg.tool_call_id = sanitizeToolId(msg.tool_call_id);
+        }
         if (msg.role === 'system' && msg.name === 'example_assistant') {
             if (charName && !msg.content.startsWith(`${charName}: `)) {
                 msg.content = `${charName}: ${msg.content}`;

From 927c2418e0159cd9c10c041bd0ec534189bd7369 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sun, 6 Oct 2024 20:19:58 +0300
Subject: [PATCH 49/50] Group tool calls in the result

---
 public/scripts/tool-calling.js | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/public/scripts/tool-calling.js b/public/scripts/tool-calling.js
index 2e5c0a65c..26ae26091 100644
--- a/public/scripts/tool-calling.js
+++ b/public/scripts/tool-calling.js
@@ -158,7 +158,7 @@ class ToolDefinition {
                 description: this.#description,
                 parameters: this.#parameters,
             },
-            toString: function() {
+            toString: function () {
                 return `<div><b>${this.function.name}</b></div><div><small>${this.function.description}</small></div><pre class="justifyLeft wordBreakAll"><code class="flex padding5">${JSON.stringify(this.function.parameters, null, 2)}</code></pre><hr>`;
             },
         };
@@ -580,6 +580,19 @@ export class ToolManager {
         return result;
     }
 
+    /**
+     * Groups tool names by count.
+     * @param {string[]} toolNames Tool names
+     * @returns {string} Grouped tool names
+     */
+    static #groupToolNames(toolNames) {
+        const toolCounts = toolNames.reduce((acc, name) => {
+            acc[name] = (acc[name] || 0) + 1;
+            return acc;
+        }, {});
+        return Object.entries(toolCounts).map(([name, count]) => count > 1 ? `${name} (${count})` : name).join(', ');
+    }
+
     /**
      * Formats a message with tool invocations.
      * @param {ToolInvocation[]} invocations Tool invocations.
@@ -597,8 +610,8 @@ export class ToolManager {
             i.result = tryParse(i.result);
         });
         codeElement.textContent = JSON.stringify(data, null, 2);
-        const toolNames = data.map(i => i.displayName || i.name).join(', ');
-        summaryElement.textContent = `Tool calls: ${toolNames}`;
+        const toolNames = data.map(i => i.displayName || i.name);
+        summaryElement.textContent = `Tool calls: ${this.#groupToolNames(toolNames)}`;
         preElement.append(codeElement);
         detailsElement.append(summaryElement, preElement);
         return detailsElement.outerHTML;

From 4f46a8ff10ae68a65742415304036267ea129e15 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Sun, 6 Oct 2024 20:24:07 +0300
Subject: [PATCH 50/50] Mistral: Change tool call id algorithm

---
 src/prompt-converters.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/prompt-converters.js b/src/prompt-converters.js
index fe9f8e88f..0d1412301 100644
--- a/src/prompt-converters.js
+++ b/src/prompt-converters.js
@@ -1,5 +1,6 @@
 require('./polyfill.js');
 const { getConfigValue } = require('./util.js');
+const crypto = require('crypto');
 
 const PROMPT_PLACEHOLDER = getConfigValue('promptPlaceholder', 'Let\'s get started.');
 
@@ -522,7 +523,7 @@ function convertMistralMessages(messages, charName = '', userName = '') {
         lastMsg.prefix = true;
     }
 
-    const sanitizeToolId = (id) => id.replace(/[^a-zA-Z0-9]/g, '').slice(0, 9);
+    const sanitizeToolId = (id) => crypto.hash('sha512', id, 'base64').slice(0, 9);
 
     // Doesn't support completion names, so prepend if not already done by the frontend (e.g. for group chats).
     messages.forEach(msg => {