Claude: new prompt converter + non-streaming tools
This commit is contained in:
Cohee
parent
559f1b81f7
commit
c3c10a629e
|
|
@ -339,10 +339,9 @@ export class ToolManager {
|
|||
|
||||
const supportedSources = [
|
||||
chat_completion_sources.OPENAI,
|
||||
//chat_completion_sources.COHERE,
|
||||
chat_completion_sources.CUSTOM,
|
||||
chat_completion_sources.MISTRALAI,
|
||||
//chat_completion_sources.CLAUDE,
|
||||
chat_completion_sources.CLAUDE,
|
||||
chat_completion_sources.OPENROUTER,
|
||||
chat_completion_sources.GROQ,
|
||||
];
|
||||
|
|
@ -372,18 +371,29 @@ export class ToolManager {
|
|||
}
|
||||
|
||||
// Parsed tool calls from non-streaming data
|
||||
if (!Array.isArray(data?.choices)) {
|
||||
return;
|
||||
if (Array.isArray(data?.choices)) {
|
||||
// Find a choice with 0-index
|
||||
const choice = data.choices.find(choice => choice.index === 0);
|
||||
|
||||
if (choice) {
|
||||
return choice.message.tool_calls;
|
||||
}
|
||||
}
|
||||
|
||||
// Find a choice with 0-index
|
||||
const choice = data.choices.find(choice => choice.index === 0);
|
||||
if (Array.isArray(data?.content)) {
|
||||
// Claude tool calls to OpenAI tool calls
|
||||
const content = data.content.filter(c => c.type === 'tool_use').map(c => {
|
||||
return {
|
||||
id: c.id,
|
||||
function: {
|
||||
name: c.name,
|
||||
arguments: c.input,
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
if (!choice) {
|
||||
return;
|
||||
return content;
|
||||
}
|
||||
|
||||
return choice.message.tool_calls;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -407,59 +417,43 @@ export class ToolManager {
|
|||
chat_completion_sources.GROQ,
|
||||
];
|
||||
|
||||
if (oaiCompatibleSources.includes(oai_settings.chat_completion_source)) {
|
||||
if (!Array.isArray(toolCalls)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
for (const toolCall of toolCalls) {
|
||||
if (typeof toolCall.function !== 'object') {
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log('Function tool call:', toolCall);
|
||||
const id = toolCall.id;
|
||||
const parameters = toolCall.function.arguments;
|
||||
const name = toolCall.function.name;
|
||||
const displayName = ToolManager.getDisplayName(name);
|
||||
result.hadToolCalls = true;
|
||||
|
||||
const message = ToolManager.formatToolCallMessage(name, parameters);
|
||||
const toast = message && toastr.info(message, 'Tool Calling', { timeOut: 0 });
|
||||
const toolResult = await ToolManager.invokeFunctionTool(name, parameters);
|
||||
toastr.clear(toast);
|
||||
console.log('Function tool result:', result);
|
||||
|
||||
// Save a successful invocation
|
||||
if (toolResult instanceof Error) {
|
||||
result.errors.push(toolResult);
|
||||
continue;
|
||||
}
|
||||
|
||||
const invocation = {
|
||||
id,
|
||||
displayName,
|
||||
name,
|
||||
parameters,
|
||||
result: toolResult,
|
||||
};
|
||||
result.invocations.push(invocation);
|
||||
}
|
||||
if (!Array.isArray(toolCalls)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
if ([chat_completion_sources.CLAUDE].includes(oai_settings.chat_completion_source)) {
|
||||
if (!Array.isArray(data?.content)) {
|
||||
return;
|
||||
for (const toolCall of toolCalls) {
|
||||
if (typeof toolCall.function !== 'object') {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const content of data.content) {
|
||||
if (content.type === 'tool_use') {
|
||||
const args = { name: content.name, arguments: JSON.stringify(content.input) };
|
||||
}
|
||||
console.log('Function tool call:', toolCall);
|
||||
const id = toolCall.id;
|
||||
const parameters = toolCall.function.arguments;
|
||||
const name = toolCall.function.name;
|
||||
const displayName = ToolManager.getDisplayName(name);
|
||||
result.hadToolCalls = true;
|
||||
|
||||
const message = ToolManager.formatToolCallMessage(name, parameters);
|
||||
const toast = message && toastr.info(message, 'Tool Calling', { timeOut: 0 });
|
||||
const toolResult = await ToolManager.invokeFunctionTool(name, parameters);
|
||||
toastr.clear(toast);
|
||||
console.log('Function tool result:', result);
|
||||
|
||||
// Save a successful invocation
|
||||
if (toolResult instanceof Error) {
|
||||
result.errors.push(toolResult);
|
||||
continue;
|
||||
}
|
||||
|
||||
const invocation = {
|
||||
id,
|
||||
displayName,
|
||||
name,
|
||||
parameters,
|
||||
result: toolResult,
|
||||
};
|
||||
result.invocations.push(invocation);
|
||||
}
|
||||
*/
|
||||
|
||||
return result;
|
||||
}
|
||||
|
|
@ -491,6 +485,9 @@ export class ToolManager {
|
|||
* @param {ToolInvocation[]} invocations Successful tool invocations
|
||||
*/
|
||||
static saveFunctionToolInvocations(invocations) {
|
||||
if (!Array.isArray(invocations) || invocations.length === 0) {
|
||||
return;
|
||||
}
|
||||
const message = {
|
||||
name: systemUserName,
|
||||
force_avatar: system_avatar,
|
||||
|
|
|
|||
|
|
@ -124,7 +124,6 @@ async function sendClaudeRequest(request, response) {
|
|||
} else {
|
||||
delete requestBody.system;
|
||||
}
|
||||
/*
|
||||
if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
|
||||
// Claude doesn't do prefills on function calls, and doesn't allow empty messages
|
||||
if (convertedPrompt.messages.length && convertedPrompt.messages[convertedPrompt.messages.length - 1].role === 'assistant') {
|
||||
|
|
@ -137,7 +136,6 @@ async function sendClaudeRequest(request, response) {
|
|||
.map(tool => tool.function)
|
||||
.map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
|
||||
}
|
||||
*/
|
||||
if (enableSystemPromptCache) {
|
||||
additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -118,8 +118,27 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
|
|||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Now replace all further messages that have the role 'system' with the role 'user'. (or all if we're not using one)
|
||||
messages.forEach((message) => {
|
||||
if (message.role === 'assistant' && message.tool_calls) {
|
||||
message.content = message.tool_calls.map((tc) => ({
|
||||
type: 'tool_use',
|
||||
id: tc.id,
|
||||
name: tc.function.name,
|
||||
input: tc.function.arguments,
|
||||
}));
|
||||
}
|
||||
|
||||
if (message.role === 'tool') {
|
||||
message.role = 'user';
|
||||
message.content = [{
|
||||
type: 'tool_result',
|
||||
tool_use_id: message.tool_call_id,
|
||||
content: message.content,
|
||||
}];
|
||||
}
|
||||
|
||||
if (message.role === 'system') {
|
||||
if (userName && message.name === 'example_user') {
|
||||
message.content = `${userName}: ${message.content}`;
|
||||
|
|
@ -128,13 +147,80 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
|
|||
message.content = `${charName}: ${message.content}`;
|
||||
}
|
||||
message.role = 'user';
|
||||
|
||||
// Delete name here so it doesn't get added later
|
||||
delete message.name;
|
||||
}
|
||||
|
||||
// Convert everything to an array of it would be easier to work with
|
||||
if (typeof message.content === 'string') {
|
||||
// Take care of name properties since claude messages don't support them
|
||||
if (message.name) {
|
||||
message.content = `${message.name}: ${message.content}`;
|
||||
}
|
||||
|
||||
message.content = [{ type: 'text', text: message.content }];
|
||||
} else if (Array.isArray(message.content)) {
|
||||
message.content = message.content.map((content) => {
|
||||
if (content.type === 'image_url') {
|
||||
const imageEntry = content?.image_url;
|
||||
const imageData = imageEntry?.url;
|
||||
const mimeType = imageData?.split(';')?.[0].split(':')?.[1];
|
||||
const base64Data = imageData?.split(',')?.[1];
|
||||
|
||||
return {
|
||||
type: 'image',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: mimeType,
|
||||
data: base64Data,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (content.type === 'text') {
|
||||
if (message.name) {
|
||||
content.text = `${message.name}: ${content.text}`;
|
||||
}
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
return content;
|
||||
});
|
||||
}
|
||||
|
||||
// Remove offending properties
|
||||
delete message.name;
|
||||
delete message.tool_calls;
|
||||
delete message.tool_call_id;
|
||||
});
|
||||
|
||||
// Images in assistant messages should be moved to the next user message
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
if (messages[i].role === 'assistant' && messages[i].content.some(c => c.type === 'image')) {
|
||||
// Find the next user message
|
||||
let j = i + 1;
|
||||
while (j < messages.length && messages[j].role !== 'user') {
|
||||
j++;
|
||||
}
|
||||
|
||||
// Move the images
|
||||
if (j >= messages.length) {
|
||||
// If there is no user message after the assistant message, add a new one
|
||||
messages.splice(i + 1, 0, { role: 'user', content: [] });
|
||||
}
|
||||
|
||||
messages[j].content.push(...messages[i].content.filter(c => c.type === 'image'));
|
||||
messages[i].content = messages[i].content.filter(c => c.type !== 'image');
|
||||
}
|
||||
}
|
||||
|
||||
// Shouldn't be conditional anymore, messages api expects the last role to be user unless we're explicitly prefilling
|
||||
if (prefillString) {
|
||||
messages.push({
|
||||
role: 'assistant',
|
||||
// Dangling whitespace are not allowed for prefilling
|
||||
content: prefillString.trimEnd(),
|
||||
});
|
||||
}
|
||||
|
|
@ -143,50 +229,11 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
|
|||
// Also handle multi-modality, holy slop.
|
||||
let mergedMessages = [];
|
||||
messages.forEach((message) => {
|
||||
const imageEntry = message.content?.[1]?.image_url;
|
||||
const imageData = imageEntry?.url;
|
||||
const mimeType = imageData?.split(';')?.[0].split(':')?.[1];
|
||||
const base64Data = imageData?.split(',')?.[1];
|
||||
|
||||
// Take care of name properties since claude messages don't support them
|
||||
if (message.name) {
|
||||
if (Array.isArray(message.content)) {
|
||||
message.content[0].text = `${message.name}: ${message.content[0].text}`;
|
||||
} else {
|
||||
message.content = `${message.name}: ${message.content}`;
|
||||
}
|
||||
delete message.name;
|
||||
}
|
||||
|
||||
if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role) {
|
||||
if (Array.isArray(message.content)) {
|
||||
if (Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
|
||||
mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content[0].text;
|
||||
} else {
|
||||
mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content[0].text;
|
||||
}
|
||||
} else {
|
||||
if (Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
|
||||
mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content;
|
||||
} else {
|
||||
mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content;
|
||||
}
|
||||
}
|
||||
mergedMessages[mergedMessages.length - 1].content.push(...message.content);
|
||||
} else {
|
||||
mergedMessages.push(message);
|
||||
}
|
||||
if (imageData) {
|
||||
mergedMessages[mergedMessages.length - 1].content = [
|
||||
{ type: 'text', text: mergedMessages[mergedMessages.length - 1].content[0]?.text || mergedMessages[mergedMessages.length - 1].content },
|
||||
{
|
||||
type: 'image', source: {
|
||||
type: 'base64',
|
||||
media_type: mimeType,
|
||||
data: base64Data,
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
});
|
||||
|
||||
return { messages: mergedMessages, systemPrompt: systemPrompt.trim() };
|
||||
|
|
|
|||
Loading…
Reference in New Issue