rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Claude: Use multi-part system prompt, cache tools

This commit is contained in:
Cohee 2024-10-05 18:04:08 +03:00
parent 3b4a455ef8
commit 6185974e17
2 changed files with 14 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/endpoints/backends/chat-completions.js
View File

@ -87,7 +87,7 @@ async function sendClaudeRequest(request, response) {
const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString(); const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE); const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);
const divider = '-'.repeat(process.stdout.columns); const divider = '-'.repeat(process.stdout.columns);
const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false); const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false) && request.body.model.startsWith('claude-3');
if (!apiKey) { if (!apiKey) {
console.log(color.red(`Claude API key is missing.\n${divider}`)); console.log(color.red(`Claude API key is missing.\n${divider}`));
@ -110,7 +110,7 @@ async function sendClaudeRequest(request, response) {
} }
const requestBody = { const requestBody = {
/** @type {any} */ system: '', /** @type {any} */ system: [],
messages: convertedPrompt.messages, messages: convertedPrompt.messages,
model: request.body.model, model: request.body.model,
max_tokens: request.body.max_tokens, max_tokens: request.body.max_tokens,
@ -121,9 +121,11 @@ async function sendClaudeRequest(request, response) {
stream: request.body.stream, stream: request.body.stream,
}; };
if (useSystemPrompt) { if (useSystemPrompt) {
requestBody.system = enableSystemPromptCache if (enableSystemPromptCache && Array.isArray(convertedPrompt.systemPrompt) && convertedPrompt.systemPrompt.length) {
? [{ type: 'text', text: convertedPrompt.systemPrompt, cache_control: { type: 'ephemeral' } }] convertedPrompt.systemPrompt[convertedPrompt.systemPrompt.length - 1]['cache_control'] = { type: 'ephemeral' };
: convertedPrompt.systemPrompt; }
requestBody.system = convertedPrompt.systemPrompt;
} else { } else {
delete requestBody.system; delete requestBody.system;
} }
@ -138,6 +140,10 @@ async function sendClaudeRequest(request, response) {
.filter(tool => tool.type === 'function') .filter(tool => tool.type === 'function')
.map(tool => tool.function) .map(tool => tool.function)
.map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters })); .map(fn => ({ name: fn.name, description: fn.description, input_schema: fn.parameters }));
if (enableSystemPromptCache && requestBody.tools.length) {
requestBody.tools[requestBody.tools.length - 1]['cache_control'] = { type: 'ephemeral' };
}
} }
if (enableSystemPromptCache) { if (enableSystemPromptCache) {
additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31'; additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';

6
src/prompt-converters.js
View File

@ -95,7 +95,7 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill,
* @param {string} userName User name * @param {string} userName User name
*/ */
function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFix, charName = '', userName = '') { function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFix, charName = '', userName = '') {
let systemPrompt = ''; let systemPrompt = [];
if (useSysPrompt) { if (useSysPrompt) {
// Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array. // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
let i; let i;
@ -114,7 +114,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
messages[i].content = `${charName}: ${messages[i].content}`; messages[i].content = `${charName}: ${messages[i].content}`;
} }
} }
systemPrompt += `${messages[i].content}\n\n`; systemPrompt.push({ type: 'text', text: messages[i].content });
} }
messages.splice(0, i); messages.splice(0, i);
@ -246,7 +246,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
} }
}); });
return { messages: mergedMessages, systemPrompt: systemPrompt.trim() }; return { messages: mergedMessages, systemPrompt: systemPrompt };
} }
/** /**