From fec27820ff7b10eb5256577b8f1a28861164239c Mon Sep 17 00:00:00 2001 From: DonMoralez Date: Wed, 13 Dec 2023 21:19:26 +0200 Subject: [PATCH] (claude)reworked prefix assignment, sysprompt mode, console message display --- public/index.html | 18 ++++++-- public/scripts/openai.js | 26 +++++++++++ server.js | 20 ++++----- src/chat-completion.js | 96 +++++++++++++++++----------------------- 4 files changed, 90 insertions(+), 70 deletions(-) diff --git a/public/index.html b/public/index.html index 348a47fa3..3ad3b7eba 100644 --- a/public/index.html +++ b/public/index.html @@ -546,10 +546,6 @@ -
- Assistant Prefill - -
@@ -1523,6 +1519,20 @@
Exclude the assistant suffix from being added to the end of prompt (Requires jailbreak with 'Assistant:' in it).
+
+ Assistant Prefill + +
+ +
+ Exclude the 'Human: ' prefix from being added to the beginning of the prompt. Instead, place it between the sysprompt and the first message with the role 'assistant'(right before 'Chat History', by default). +
+
+ Human: first message + +
diff --git a/public/scripts/openai.js b/public/scripts/openai.js index a9bc5e304..42dc37c00 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -222,8 +222,10 @@ const default_settings = { show_external_models: false, proxy_password: '', assistant_prefill: '', + human_sysprompt_message: '', use_ai21_tokenizer: false, exclude_assistant: false, + claude_use_sysprompt: false, use_alt_scale: false, squash_system_messages: false, image_inlining: false, @@ -275,8 +277,10 @@ const oai_settings = { show_external_models: false, proxy_password: '', assistant_prefill: '', + human_sysprompt_message: '', use_ai21_tokenizer: false, exclude_assistant: false, + claude_use_sysprompt: false, use_alt_scale: false, squash_system_messages: false, image_inlining: false, @@ -1519,7 +1523,9 @@ async function sendOpenAIRequest(type, messages, signal) { if (isClaude) { generate_data['top_k'] = Number(oai_settings.top_k_openai); generate_data['exclude_assistant'] = oai_settings.exclude_assistant; + generate_data['claude_use_sysprompt'] = oai_settings.claude_use_sysprompt; generate_data['stop'] = getCustomStoppingStrings(); // Claude shouldn't have limits on stop strings. + generate_data['human_sysprompt_message'] = substituteParams(oai_settings.human_sysprompt_message); // Don't add a prefill on quiet gens (summarization) if (!isQuiet && !oai_settings.exclude_assistant) { generate_data['assistant_prefill'] = substituteParams(oai_settings.assistant_prefill); @@ -2295,6 +2301,7 @@ function loadOpenAISettings(data, settings) { oai_settings.show_external_models = settings.show_external_models ?? default_settings.show_external_models; oai_settings.proxy_password = settings.proxy_password ?? default_settings.proxy_password; oai_settings.assistant_prefill = settings.assistant_prefill ?? default_settings.assistant_prefill; + oai_settings.human_sysprompt_message = settings.human_sysprompt_message ?? default_settings.human_sysprompt_message; oai_settings.image_inlining = settings.image_inlining ?? default_settings.image_inlining; oai_settings.bypass_status_check = settings.bypass_status_check ?? default_settings.bypass_status_check; @@ -2312,11 +2319,13 @@ function loadOpenAISettings(data, settings) { if (settings.openai_model !== undefined) oai_settings.openai_model = settings.openai_model; if (settings.use_ai21_tokenizer !== undefined) { oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer; oai_settings.use_ai21_tokenizer ? ai21_max = 8191 : ai21_max = 9200; } if (settings.exclude_assistant !== undefined) oai_settings.exclude_assistant = !!settings.exclude_assistant; + if (settings.claude_use_sysprompt !== undefined) oai_settings.claude_use_sysprompt = !!settings.claude_use_sysprompt; if (settings.use_alt_scale !== undefined) { oai_settings.use_alt_scale = !!settings.use_alt_scale; updateScaleForm(); } $('#stream_toggle').prop('checked', oai_settings.stream_openai); $('#api_url_scale').val(oai_settings.api_url_scale); $('#openai_proxy_password').val(oai_settings.proxy_password); $('#claude_assistant_prefill').val(oai_settings.assistant_prefill); + $('#claude_human_sysprompt_message').val(oai_settings.human_sysprompt_message); $('#openai_image_inlining').prop('checked', oai_settings.image_inlining); $('#openai_bypass_status_check').prop('checked', oai_settings.bypass_status_check); @@ -2342,6 +2351,7 @@ function loadOpenAISettings(data, settings) { $('#openai_external_category').toggle(oai_settings.show_external_models); $('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer); $('#exclude_assistant').prop('checked', oai_settings.exclude_assistant); + $('#claude_use_sysprompt').prop('checked', oai_settings.claude_use_sysprompt); $('#scale-alt').prop('checked', oai_settings.use_alt_scale); $('#openrouter_use_fallback').prop('checked', oai_settings.openrouter_use_fallback); $('#openrouter_force_instruct').prop('checked', oai_settings.openrouter_force_instruct); @@ -2531,8 +2541,10 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) { api_url_scale: settings.api_url_scale, show_external_models: settings.show_external_models, assistant_prefill: settings.assistant_prefill, + human_sysprompt_message: settings.human_sysprompt_message, use_ai21_tokenizer: settings.use_ai21_tokenizer, exclude_assistant: settings.exclude_assistant, + claude_use_sysprompt: settings.claude_use_sysprompt, use_alt_scale: settings.use_alt_scale, squash_system_messages: settings.squash_system_messages, image_inlining: settings.image_inlining, @@ -2891,8 +2903,10 @@ function onSettingsPresetChange() { show_external_models: ['#openai_show_external_models', 'show_external_models', true], proxy_password: ['#openai_proxy_password', 'proxy_password', false], assistant_prefill: ['#claude_assistant_prefill', 'assistant_prefill', false], + human_sysprompt_message: ['#claude_human_sysprompt_message', 'human_sysprompt_message', false], use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', true], exclude_assistant: ['#exclude_assistant', 'exclude_assistant', true], + claude_use_sysprompt: ['#claude_use_sysprompt', 'claude_use_sysprompt', true], use_alt_scale: ['#use_alt_scale', 'use_alt_scale', true], squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true], image_inlining: ['#openai_image_inlining', 'image_inlining', true], @@ -3345,6 +3359,7 @@ function toggleChatCompletionForms() { if (chat_completion_sources.CLAUDE == oai_settings.chat_completion_source) { $('#claude_assistant_prefill_block').toggle(!oai_settings.exclude_assistant); + $('#claude_human_sysprompt_message_block').toggle(oai_settings.claude_use_sysprompt); } } @@ -3497,6 +3512,12 @@ $(document).ready(async function () { saveSettingsDebounced(); }); + $('#claude_use_sysprompt').on('change', function () { + oai_settings.claude_use_sysprompt = !!$('#claude_use_sysprompt').prop('checked'); + $('#claude_human_sysprompt_message_block').toggle(oai_settings.claude_use_sysprompt); + saveSettingsDebounced(); + }); + $('#names_in_completion').on('change', function () { oai_settings.names_in_completion = !!$('#names_in_completion').prop('checked'); saveSettingsDebounced(); @@ -3657,6 +3678,11 @@ $(document).ready(async function () { saveSettingsDebounced(); }); + $('#claude_human_sysprompt_message').on('input', function () { + oai_settings.human_sysprompt_message = String($(this).val()); + saveSettingsDebounced(); + }); + $('#openrouter_use_fallback').on('input', function () { oai_settings.openrouter_use_fallback = !!$(this).prop('checked'); saveSettingsDebounced(); diff --git a/server.js b/server.js index ec9fb03aa..a627ed3f7 100644 --- a/server.js +++ b/server.js @@ -1297,9 +1297,11 @@ async function sendClaudeRequest(request, response) { const api_url = new URL(request.body.reverse_proxy || API_CLAUDE).toString(); const api_key_claude = request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.CLAUDE); + const chalk = require('chalk'); + const divider = '-'.repeat(process.stdout.columns); if (!api_key_claude) { - console.log('Claude API key is missing.'); + console.log(chalk.red(`Claude API key is missing.\n${divider}`)); return response.status(400).send({ error: true }); } @@ -1310,14 +1312,10 @@ async function sendClaudeRequest(request, response) { controller.abort(); }); - let doSystemPrompt = request.body.model === 'claude-2' || request.body.model === 'claude-2.1'; - let requestPrompt = convertClaudePrompt(request.body.messages, true, !request.body.exclude_assistant, doSystemPrompt); + let isSyspromptSupported = request.body.model === 'claude-2' || request.body.model === 'claude-2.1'; + let requestPrompt = convertClaudePrompt(request.body.messages, !request.body.exclude_assistant, request.body.assistant_prefill, isSyspromptSupported, request.body.claude_use_sysprompt, request.body.human_sysprompt_message); - if (request.body.assistant_prefill && !request.body.exclude_assistant) { - requestPrompt += request.body.assistant_prefill; - } - - console.log('Claude request:', requestPrompt); + console.log(chalk.green(`${divider}\nClaude request\n`) + chalk.cyan(`PROMPT\n${divider}\n${requestPrompt}\n${divider}`)); const stop_sequences = ['\n\nHuman:', '\n\nSystem:', '\n\nAssistant:']; // Add custom stop sequences @@ -1351,20 +1349,20 @@ async function sendClaudeRequest(request, response) { forwardFetchResponse(generateResponse, response); } else { if (!generateResponse.ok) { - console.log(`Claude API returned error: ${generateResponse.status} ${generateResponse.statusText} ${await generateResponse.text()}`); + console.log(chalk.red(`Claude API returned error: ${generateResponse.status} ${generateResponse.statusText}\n${await generateResponse.text()}\n${divider}`)); return response.status(generateResponse.status).send({ error: true }); } const generateResponseJson = await generateResponse.json(); const responseText = generateResponseJson.completion; - console.log('Claude response:', responseText); + console.log(chalk.green(`Claude response\n${divider}\n${responseText}\n${divider}`)); // Wrap it back to OAI format const reply = { choices: [{ 'message': { 'content': responseText } }] }; return response.send(reply); } } catch (error) { - console.log('Error communicating with Claude: ', error); + console.log(chalk.red(`Error communicating with Claude: ${error}\n${divider}`)); if (!response.headersSent) { return response.status(500).send({ error: true }); } diff --git a/src/chat-completion.js b/src/chat-completion.js index 4fc21a550..d673ae37d 100644 --- a/src/chat-completion.js +++ b/src/chat-completion.js @@ -1,73 +1,59 @@ /** * Convert a prompt from the ChatML objects to the format used by Claude. * @param {object[]} messages Array of messages - * @param {boolean} addHumanPrefix Add Human prefix - * @param {boolean} addAssistantPostfix Add Assistant postfix - * @param {boolean} withSystemPrompt Build system prompt before "\n\nHuman: " + * @param {boolean} addAssistantPostfix Add Assistant postfix. + * @param {string} addAssistantPrefill Add Assistant prefill after the assistant postfix. + * @param {boolean} withSyspromptSupport Indicates if the Claude model supports the system prompt format. + * @param {boolean} useSystemPrompt Indicates if the system prompt format should be used. + * @param {string} addSysHumanMsg Add Human message between system prompt and assistant. * @returns {string} Prompt for Claude * @copyright Prompt Conversion script taken from RisuAI by kwaroran (GPLv3). */ -function convertClaudePrompt(messages, addHumanPrefix, addAssistantPostfix, withSystemPrompt) { - // Claude doesn't support message names, so we'll just add them to the message content. - for (const message of messages) { - if (message.name && message.role !== 'system') { - message.content = message.name + ': ' + message.content; - delete message.name; - } - } - let systemPrompt = ''; - if (withSystemPrompt) { - let lastSystemIdx = -1; +function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill, withSyspromptSupport, useSystemPrompt, addSysHumanMsg) { - for (let i = 0; i < messages.length - 1; i++) { - const message = messages[i]; - if (message.role === 'system' && !message.name) { - systemPrompt += message.content + '\n\n'; - } else { - lastSystemIdx = i - 1; - break; - } + // Find the index of the first message with an assistant role and check for a "'user' role/Human:" before it. + let hasUser = false; + const firstAssistantIndex = messages.findIndex((message) => { + if (message.role === 'user' || message.content.includes('Human:')) { + hasUser = true; + } + return message.role === 'assistant'; + }); + + let setHumanMsg = addSysHumanMsg ? '\n\nHuman: ' + addSysHumanMsg : '\n\nHuman: Let\'s get started.'; + let requestPrompt = messages.map((v, i) => { + // Claude doesn't support message names, so we'll just add them to the message content. + if (v.name && v.role !== 'system') { + v.content = `${v.name}: ${v.content}`; + delete v.name; } - if (lastSystemIdx >= 0) { - messages.splice(0, lastSystemIdx + 1); - } - } - let requestPrompt = messages.map((v) => { let prefix = ''; - switch (v.role) { - case 'assistant': - prefix = '\n\nAssistant: '; - break; - case 'user': - prefix = '\n\nHuman: '; - break; - case 'system': - // According to the Claude docs, H: and A: should be used for example conversations. - if (v.name === 'example_assistant') { - prefix = '\n\nA: '; - } else if (v.name === 'example_user') { - prefix = '\n\nH: '; - } else { - prefix = '\n\n'; - } - break; + // Switches to system prompt format by adding empty prefix to the first message of the assistant, when the "use system prompt" checked and the model is 2.1. + // Otherwise, use the default message format by adding "Human: " prefix to the first message(compatible with all claude models including 2.1.) + if (i === 0) { + prefix = withSyspromptSupport && useSystemPrompt ? '' : '\n\nHuman: '; + // For system prompt format. If there is no message with role "user" or prefix "Human:" change the first assistant's prefix(insert the human's message). + } else if (i === firstAssistantIndex && !hasUser && withSyspromptSupport && useSystemPrompt) { + prefix = `${setHumanMsg}\n\nAssistant: `; + //prefix = addSysHumanMsg ? '\n\nHuman: ' + addSysHumanMsg + '\n\nAssistant: ' : '\n\nHuman: Let\'s get started.\n\nAssistant: '; + // Merge two messages with "\n\nHuman: " prefixes into one before the first Assistant's message. Fix messages order for default claude format when(messages > Context Size). + } else if (i > 0 && i === firstAssistantIndex - 1 && v.role === 'user' && (!withSyspromptSupport || (withSyspromptSupport && !useSystemPrompt))) { + prefix = '\n\nFirst message: '; + //Define role prefixes(role : prefix). Set the correct prefix according to the role/name. + } else { + prefix = { + 'assistant': '\n\nAssistant: ', + 'user': '\n\nHuman: ', + 'system': v.name === 'example_assistant' ? '\n\nA: ' : v.name === 'example_user' ? '\n\nH: ' : '\n\n', + }[v.role] || '\n\n'; } return prefix + v.content; }).join(''); - if (addHumanPrefix) { - requestPrompt = '\n\nHuman: ' + requestPrompt; - } - - if (addAssistantPostfix) { - requestPrompt = requestPrompt + '\n\nAssistant: '; - } - - if (withSystemPrompt) { - requestPrompt = systemPrompt + requestPrompt; - } + //Add the assistant suffix(if the option unchecked), add a prefill after it(if filled). Also Add the first human message before the assistant suffix(when using sysprompt and there are no other messages with the role 'Assistant'). + requestPrompt += addAssistantPostfix ? `${withSyspromptSupport && useSystemPrompt && firstAssistantIndex === -1 ? setHumanMsg : ''}\n\nAssistant: ${addAssistantPrefill ? addAssistantPrefill : ''}` : ''; return requestPrompt; }