From 53976143470dce5e1e11166c81c37e04cd3cbfbe Mon Sep 17 00:00:00 2001 From: Honey Tree Date: Sun, 17 Nov 2024 14:01:22 -0300 Subject: [PATCH] Defaulting to -1 rather than boolean false --- default/config.yaml | 3 ++- src/endpoints/backends/chat-completions.js | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/default/config.yaml b/default/config.yaml index fd6f1ea18..6f174e523 100644 --- a/default/config.yaml +++ b/default/config.yaml @@ -174,6 +174,7 @@ claude: # Use with caution. Behavior may be unpredictable and no guarantees can or will be made. # Set to an integer to specify the desired depth. 0 (which does NOT include the prefill) # should be ideal for most use cases. - cachingAtDepth: false + # Any value other than a non-negative integer will be ignored and caching at depth will not be enabled. + cachingAtDepth: -1 # -- SERVER PLUGIN CONFIGURATION -- enableServerPlugins: false diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index 580e4388f..bd5417464 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -80,7 +80,11 @@ async function sendClaudeRequest(request, response) { const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE); const divider = '-'.repeat(process.stdout.columns); const enableSystemPromptCache = getConfigValue('claude.enableSystemPromptCache', false) && request.body.model.startsWith('claude-3'); - let cachingAtDepth = getConfigValue('claude.cachingAtDepth', false) && request.body.model.startsWith('claude-3'); + let cachingAtDepth = getConfigValue('claude.cachingAtDepth', -1) && request.body.model.startsWith('claude-3'); + // Disabled if not an integer or negative + if (!Number.isInteger(cachingAtDepth) || cachingAtDepth < 0) { + cachingAtDepth = -1; + } if (!apiKey) { console.log(color.red(`Claude API key is missing.\n${divider}`)); @@ -140,7 +144,7 @@ async function sendClaudeRequest(request, response) { } } - if (cachingAtDepth !== false) { + if (cachingAtDepth !== -1) { // There are extremely few scenarios in which caching the prefill is a good idea, it mostly just breaks everything const messageCount = convertedPrompt.messages.length; cachingAtDepth += convertedPrompt.messages[messageCount - 1].role === 'assistant' ? 1 : 0; @@ -156,7 +160,7 @@ async function sendClaudeRequest(request, response) { } } - if (enableSystemPromptCache || cachingAtDepth !== false) { + if (enableSystemPromptCache || cachingAtDepth !== -1) { additionalHeaders['anthropic-beta'] = 'prompt-caching-2024-07-31'; }