rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Defaulting to -1 rather than boolean false

Browse Source
This commit is contained in:
Honey Tree
2024-11-17 14:01:22 -03:00
parent ac33e4d668
commit 5397614347
2 changed files with 9 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
default/config.yaml
View File

@ -174,6 +174,7 @@ claude:
# Use with caution. Behavior may be unpredictable and no guarantees can or will be made. # Use with caution. Behavior may be unpredictable and no guarantees can or will be made.
# Set to an integer to specify the desired depth. 0 (which does NOT include the prefill) # Set to an integer to specify the desired depth. 0 (which does NOT include the prefill)
# should be ideal for most use cases. # should be ideal for most use cases.
cachingAtDepth: false # Any value other than a non-negative integer will be ignored and caching at depth will not be enabled.
cachingAtDepth: -1
# -- SERVER PLUGIN CONFIGURATION -- # -- SERVER PLUGIN CONFIGURATION --
enableServerPlugins: false enableServerPlugins: false

10
src/endpoints/backends/chat-completions.js
View File

@ -80,7 +80,11 @@ async function sendClaudeRequest(request, response) {
const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE); const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);
const divider = '-'.repeat(process.stdout.columns); const divider = '-'.repeat(process.stdout.columns);
const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false) && request.body.model.startsWith('claude-3'); const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false) && request.body.model.startsWith('claude-3');
let cachingAtDepth = getConfigValue('claude.cachingAtDepth', false) && request.body.model.startsWith('claude-3'); let cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1) && request.body.model.startsWith('claude-3');
// Disabled if not an integer or negative
if (!Number.isInteger(cachingAtDepth) || cachingAtDepth < 0) {
cachingAtDepth = -1;
}
if (!apiKey) { if (!apiKey) {
console.log(color.red(`Claude API key is missing.\n${divider}`)); console.log(color.red(`Claude API key is missing.\n${divider}`));
@ -140,7 +144,7 @@ async function sendClaudeRequest(request, response) {
} }
} }
if (cachingAtDepth !== false) { if (cachingAtDepth !== -1) {
// There are extremely few scenarios in which caching the prefill is a good idea, it mostly just breaks everything // There are extremely few scenarios in which caching the prefill is a good idea, it mostly just breaks everything
const messageCount = convertedPrompt.messages.length; const messageCount = convertedPrompt.messages.length;
cachingAtDepth += convertedPrompt.messages[messageCount - 1].role === 'assistant' ? 1 : 0; cachingAtDepth += convertedPrompt.messages[messageCount - 1].role === 'assistant' ? 1 : 0;
@ -156,7 +160,7 @@ async function sendClaudeRequest(request, response) {
} }
} }
if (enableSystemPromptCache || cachingAtDepth !== false) { if (enableSystemPromptCache || cachingAtDepth !== -1) {
additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31'; additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31';
} }