rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Claude: Use multi-part system prompt, cache tools

This commit is contained in:
Cohee 2024-10-05 18:04:08 +03:00
parent 3b4a455ef8
commit 6185974e17
2 changed files with 14 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/endpoints/backends/chat-completions.js
View File

@ -87,7 +87,7 @@ async function sendClaudeRequest(request, response) {
const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);
const divider = '-'.repeat(process.stdout.columns);
const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false);
const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false) && request.body.model.startsWith('claude-3');
if (!apiKey) {
console.log(color.red(`Claude API key is missing.\n${divider}`));
@ -110,7 +110,7 @@ async function sendClaudeRequest(request, response) {
}
const requestBody = {
/** @type {any} */ system: '',
/** @type {any} */ system: [],
messages: convertedPrompt.messages,
model: request.body.model,
max_tokens: request.body.max_tokens,
@ -121,9 +121,11 @@ async function sendClaudeRequest(request, response) {
stream: request.body.stream,
};
if (useSystemPrompt) {
requestBody.system = enableSystemPromptCache
? [{ type: 'text', text: convertedPrompt.systemPrompt, cache_control: { type: 'ephemeral' } }]
: convertedPrompt.systemPrompt;
if (enableSystemPromptCache && Array.isArray(convertedPrompt.systemPrompt) && convertedPrompt.systemPrompt.length) {
convertedPrompt.systemPrompt[convertedPrompt.systemPrompt.length - 1]['cache_control'] = { type: 'ephemeral' };
}
requestBody.system = convertedPrompt.systemPrompt;
} else {
delete requestBody.system;
}
@ -138,6 +140,10 @@ async function sendClaudeRequest(request, response) {
.filter(tool => tool.type === 'function')
.map(tool => tool.function)
.map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
if (enableSystemPromptCache && requestBody.tools.length) {
requestBody.tools[requestBody.tools.length - 1]['cache_control'] = { type: 'ephemeral' };
}
}
if (enableSystemPromptCache) {
additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';

6
src/prompt-converters.js
View File

@ -95,7 +95,7 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill,
* @param {string} userName User name
*/
function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFix, charName = '', userName = '') {
let systemPrompt = '';
let systemPrompt = [];
if (useSysPrompt) {
// Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
let i;
@ -114,7 +114,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
messages[i].content = `${charName}: ${messages[i].content}`;
}
}
systemPrompt += `${messages[i].content}\n\n`;
systemPrompt.push({ type: 'text', text: messages[i].content });
}
messages.splice(0, i);
@ -246,7 +246,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
}
});
return { messages: mergedMessages, systemPrompt: systemPrompt.trim() };
return { messages: mergedMessages, systemPrompt: systemPrompt };
}
/**