rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

(claude)reworked prefix assignment, sysprompt mode, console message display

This commit is contained in:
DonMoralez 2023-12-13 21:19:26 +02:00
parent 51d50f97cc
commit fec27820ff
4 changed files with 90 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
public/index.html
View File

@ -546,10 +546,6 @@
<textarea id="jailbreak_prompt_quick_edit_textarea" class="text_pole textarea_compact autoSetHeight" rows="6" placeholder="&mdash;" data-pm-prompt="jailbreak"></textarea>
</div>
</div>
<div id="claude_assistant_prefill_block" data-source="claude" class="range-block">
<span id="claude_assistant_prefill_text" data-i18n="Assistant Prefill">Assistant Prefill</span>
<textarea id="claude_assistant_prefill" class="text_pole textarea_compact" name="assistant_prefill autoSetHeight" rows="3" maxlength="10000" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
</div>
</div>
</div>
<div data-newbie-hidden class="inline-drawer wide100p">
@ -1523,6 +1519,20 @@
<div class="toggle-description justifyLeft">
<span data-i18n="Exclude the assistant suffix from being added to the end of prompt.">Exclude the assistant suffix from being added to the end of prompt (Requires jailbreak with 'Assistant:' in it).</span>
</div>
<div id="claude_assistant_prefill_block" data-source="claude" class="wide100p">
<span id="claude_assistant_prefill_text" data-i18n="Assistant Prefill">Assistant Prefill</span>
<textarea id="claude_assistant_prefill" class="text_pole textarea_compact" name="assistant_prefill autoSetHeight" rows="3" maxlength="10000" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
</div>
<label for="claude_use_sysprompt" title="Use system prompt(claude 2.1+ only)" class="checkbox_label widthFreeExpand">
<input id="claude_use_sysprompt" type="checkbox" /><span data-i18n="Use system prompt(claude 2.1+ only)">Use system prompt(claude 2.1+ only)</span>
</label>
<div class="toggle-description justifyLeft">
<span data-i18n="Exclude the 'Human: ' prefix from being added to the beginning of the prompt.">Exclude the 'Human: ' prefix from being added to the beginning of the prompt. Instead, place it between the sysprompt and the first message with the role 'assistant'(right before 'Chat History', by default).</span>
</div>
<div id="claude_human_sysprompt_message_block" data-source="claude" class="wide100p">
<span id="claude_human_sysprompt_message_text" data-i18n="Human: first message">Human: first message</span>
<textarea id="claude_human_sysprompt_message" class="text_pole textarea_compact" name="human_sysprompt_message autoSetHeight" rows="3" maxlength="10000" data-i18n="[placeholder]Your message, instruction, etc... WHEN EMPTY and no manually added Human's message found between sysprompt and the first assistant message(Chat history), add 'Human: Let's get started.'." placeholder="Your message, instruction, etc... WHEN EMPTY and no manually added Human's message found between sysprompt and the first assistant message(Chat history), add 'Human: Let's get started.'."></textarea>
</div>
</div>
</div>
<div data-newbie-hidden class="range-block m-t-1" data-source="openai,openrouter,scale">

26
public/scripts/openai.js
View File

@ -222,8 +222,10 @@ const default_settings = {
show_external_models: false,
proxy_password: '',
assistant_prefill: '',
human_sysprompt_message: '',
use_ai21_tokenizer: false,
exclude_assistant: false,
claude_use_sysprompt: false,
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
@ -275,8 +277,10 @@ const oai_settings = {
show_external_models: false,
proxy_password: '',
assistant_prefill: '',
human_sysprompt_message: '',
use_ai21_tokenizer: false,
exclude_assistant: false,
claude_use_sysprompt: false,
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
@ -1519,7 +1523,9 @@ async function sendOpenAIRequest(type, messages, signal) {
if (isClaude) {
generate_data['top_k'] = Number(oai_settings.top_k_openai);
generate_data['exclude_assistant'] = oai_settings.exclude_assistant;
generate_data['claude_use_sysprompt'] = oai_settings.claude_use_sysprompt;
generate_data['stop'] = getCustomStoppingStrings(); // Claude shouldn't have limits on stop strings.
generate_data['human_sysprompt_message'] = substituteParams(oai_settings.human_sysprompt_message);
// Don't add a prefill on quiet gens (summarization)
if (!isQuiet && !oai_settings.exclude_assistant) {
generate_data['assistant_prefill'] = substituteParams(oai_settings.assistant_prefill);
@ -2295,6 +2301,7 @@ function loadOpenAISettings(data, settings) {
oai_settings.show_external_models = settings.show_external_models ?? default_settings.show_external_models;
oai_settings.proxy_password = settings.proxy_password ?? default_settings.proxy_password;
oai_settings.assistant_prefill = settings.assistant_prefill ?? default_settings.assistant_prefill;
oai_settings.human_sysprompt_message = settings.human_sysprompt_message ?? default_settings.human_sysprompt_message;
oai_settings.image_inlining = settings.image_inlining ?? default_settings.image_inlining;
oai_settings.bypass_status_check = settings.bypass_status_check ?? default_settings.bypass_status_check;
@ -2312,11 +2319,13 @@ function loadOpenAISettings(data, settings) {
if (settings.openai_model !== undefined) oai_settings.openai_model = settings.openai_model;
if (settings.use_ai21_tokenizer !== undefined) { oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer; oai_settings.use_ai21_tokenizer ? ai21_max = 8191 : ai21_max = 9200; }
if (settings.exclude_assistant !== undefined) oai_settings.exclude_assistant = !!settings.exclude_assistant;
if (settings.claude_use_sysprompt !== undefined) oai_settings.claude_use_sysprompt = !!settings.claude_use_sysprompt;
if (settings.use_alt_scale !== undefined) { oai_settings.use_alt_scale = !!settings.use_alt_scale; updateScaleForm(); }
$('#stream_toggle').prop('checked', oai_settings.stream_openai);
$('#api_url_scale').val(oai_settings.api_url_scale);
$('#openai_proxy_password').val(oai_settings.proxy_password);
$('#claude_assistant_prefill').val(oai_settings.assistant_prefill);
$('#claude_human_sysprompt_message').val(oai_settings.human_sysprompt_message);
$('#openai_image_inlining').prop('checked', oai_settings.image_inlining);
$('#openai_bypass_status_check').prop('checked', oai_settings.bypass_status_check);
@ -2342,6 +2351,7 @@ function loadOpenAISettings(data, settings) {
$('#openai_external_category').toggle(oai_settings.show_external_models);
$('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer);
$('#exclude_assistant').prop('checked', oai_settings.exclude_assistant);
$('#claude_use_sysprompt').prop('checked', oai_settings.claude_use_sysprompt);
$('#scale-alt').prop('checked', oai_settings.use_alt_scale);
$('#openrouter_use_fallback').prop('checked', oai_settings.openrouter_use_fallback);
$('#openrouter_force_instruct').prop('checked', oai_settings.openrouter_force_instruct);
@ -2531,8 +2541,10 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
api_url_scale: settings.api_url_scale,
show_external_models: settings.show_external_models,
assistant_prefill: settings.assistant_prefill,
human_sysprompt_message: settings.human_sysprompt_message,
use_ai21_tokenizer: settings.use_ai21_tokenizer,
exclude_assistant: settings.exclude_assistant,
claude_use_sysprompt: settings.claude_use_sysprompt,
use_alt_scale: settings.use_alt_scale,
squash_system_messages: settings.squash_system_messages,
image_inlining: settings.image_inlining,
@ -2891,8 +2903,10 @@ function onSettingsPresetChange() {
show_external_models: ['#openai_show_external_models', 'show_external_models', true],
proxy_password: ['#openai_proxy_password', 'proxy_password', false],
assistant_prefill: ['#claude_assistant_prefill', 'assistant_prefill', false],
human_sysprompt_message: ['#claude_human_sysprompt_message', 'human_sysprompt_message', false],
use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', true],
exclude_assistant: ['#exclude_assistant', 'exclude_assistant', true],
claude_use_sysprompt: ['#claude_use_sysprompt', 'claude_use_sysprompt', true],
use_alt_scale: ['#use_alt_scale', 'use_alt_scale', true],
squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true],
image_inlining: ['#openai_image_inlining', 'image_inlining', true],
@ -3345,6 +3359,7 @@ function toggleChatCompletionForms() {
if (chat_completion_sources.CLAUDE == oai_settings.chat_completion_source) {
$('#claude_assistant_prefill_block').toggle(!oai_settings.exclude_assistant);
$('#claude_human_sysprompt_message_block').toggle(oai_settings.claude_use_sysprompt);
}
}
@ -3497,6 +3512,12 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#claude_use_sysprompt').on('change', function () {
oai_settings.claude_use_sysprompt = !!$('#claude_use_sysprompt').prop('checked');
$('#claude_human_sysprompt_message_block').toggle(oai_settings.claude_use_sysprompt);
saveSettingsDebounced();
});
$('#names_in_completion').on('change', function () {
oai_settings.names_in_completion = !!$('#names_in_completion').prop('checked');
saveSettingsDebounced();
@ -3657,6 +3678,11 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#claude_human_sysprompt_message').on('input', function () {
oai_settings.human_sysprompt_message = String($(this).val());
saveSettingsDebounced();
});
$('#openrouter_use_fallback').on('input', function () {
oai_settings.openrouter_use_fallback = !!$(this).prop('checked');
saveSettingsDebounced();

20
server.js
View File

@ -1297,9 +1297,11 @@ async function sendClaudeRequest(request, response) {
const api_url = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
const api_key_claude = request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.CLAUDE);
const chalk = require('chalk');
const divider = '-'.repeat(process.stdout.columns);
if (!api_key_claude) {
console.log('Claude API key is missing.');
console.log(chalk.red(`Claude API key is missing.\n${divider}`));
return response.status(400).send({ error: true });
}
@ -1310,14 +1312,10 @@ async function sendClaudeRequest(request, response) {
controller.abort();
});
let doSystemPrompt = request.body.model === 'claude-2' || request.body.model === 'claude-2.1';
let requestPrompt = convertClaudePrompt(request.body.messages, true, !request.body.exclude_assistant, doSystemPrompt);
let isSyspromptSupported = request.body.model === 'claude-2' || request.body.model === 'claude-2.1';
let requestPrompt = convertClaudePrompt(request.body.messages, !request.body.exclude_assistant, request.body.assistant_prefill, isSyspromptSupported, request.body.claude_use_sysprompt, request.body.human_sysprompt_message);
if (request.body.assistant_prefill && !request.body.exclude_assistant) {
requestPrompt += request.body.assistant_prefill;
}
console.log('Claude request:', requestPrompt);
console.log(chalk.green(`${divider}\nClaude request\n`) + chalk.cyan(`PROMPT\n${divider}\n${requestPrompt}\n${divider}`));
const stop_sequences = ['\n\nHuman:', '\n\nSystem:', '\n\nAssistant:'];
// Add custom stop sequences
@ -1351,20 +1349,20 @@ async function sendClaudeRequest(request, response) {
forwardFetchResponse(generateResponse, response);
} else {
if (!generateResponse.ok) {
console.log(`Claude API returned error: ${generateResponse.status} ${generateResponse.statusText} ${await generateResponse.text()}`);
console.log(chalk.red(`Claude API returned error: ${generateResponse.status} ${generateResponse.statusText}\n${await generateResponse.text()}\n${divider}`));
return response.status(generateResponse.status).send({ error: true });
}
const generateResponseJson = await generateResponse.json();
const responseText = generateResponseJson.completion;
console.log('Claude response:', responseText);
console.log(chalk.green(`Claude response\n${divider}\n${responseText}\n${divider}`));
// Wrap it back to OAI format
const reply = { choices: [{ 'message': { 'content': responseText } }] };
return response.send(reply);
}
} catch (error) {
console.log('Error communicating with Claude: ', error);
console.log(chalk.red(`Error communicating with Claude: ${error}\n${divider}`));
if (!response.headersSent) {
return response.status(500).send({ error: true });
}

96
src/chat-completion.js
View File

@ -1,73 +1,59 @@
/**
* Convert a prompt from the ChatML objects to the format used by Claude.
* @param {object[]} messages Array of messages
* @param {boolean} addHumanPrefix Add Human prefix
* @param {boolean} addAssistantPostfix Add Assistant postfix
* @param {boolean} withSystemPrompt Build system prompt before "\n\nHuman: "
* @param {boolean} addAssistantPostfix Add Assistant postfix.
* @param {string} addAssistantPrefill Add Assistant prefill after the assistant postfix.
* @param {boolean} withSyspromptSupport Indicates if the Claude model supports the system prompt format.
* @param {boolean} useSystemPrompt Indicates if the system prompt format should be used.
* @param {string} addSysHumanMsg Add Human message between system prompt and assistant.
* @returns {string} Prompt for Claude
* @copyright Prompt Conversion script taken from RisuAI by kwaroran (GPLv3).
*/
function convertClaudePrompt(messages, addHumanPrefix, addAssistantPostfix, withSystemPrompt) {
// Claude doesn't support message names, so we'll just add them to the message content.
for (const message of messages) {
if (message.name && message.role !== 'system') {
message.content = message.name + ': ' + message.content;
delete message.name;
}
}
let systemPrompt = '';
if (withSystemPrompt) {
let lastSystemIdx = -1;
function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill, withSyspromptSupport, useSystemPrompt, addSysHumanMsg) {
for (let i = 0; i < messages.length - 1; i++) {
const message = messages[i];
if (message.role === 'system' && !message.name) {
systemPrompt += message.content + '\n\n';
} else {
lastSystemIdx = i - 1;
break;
}
// Find the index of the first message with an assistant role and check for a "'user' role/Human:" before it.
let hasUser = false;
const firstAssistantIndex = messages.findIndex((message) => {
if (message.role === 'user' || message.content.includes('Human:')) {
hasUser = true;
}
return message.role === 'assistant';
});
let setHumanMsg = addSysHumanMsg ? '\n\nHuman: ' + addSysHumanMsg : '\n\nHuman: Let\'s get started.';
let requestPrompt = messages.map((v, i) => {
// Claude doesn't support message names, so we'll just add them to the message content.
if (v.name && v.role !== 'system') {
v.content = `${v.name}: ${v.content}`;
delete v.name;
}
if (lastSystemIdx >= 0) {
messages.splice(0, lastSystemIdx + 1);
}
}
let requestPrompt = messages.map((v) => {
let prefix = '';
switch (v.role) {
case 'assistant':
prefix = '\n\nAssistant: ';
break;
case 'user':
prefix = '\n\nHuman: ';
break;
case 'system':
// According to the Claude docs, H: and A: should be used for example conversations.
if (v.name === 'example_assistant') {
prefix = '\n\nA: ';
} else if (v.name === 'example_user') {
prefix = '\n\nH: ';
} else {
prefix = '\n\n';
}
break;
// Switches to system prompt format by adding empty prefix to the first message of the assistant, when the "use system prompt" checked and the model is 2.1.
// Otherwise, use the default message format by adding "Human: " prefix to the first message(compatible with all claude models including 2.1.)
if (i === 0) {
prefix = withSyspromptSupport && useSystemPrompt ? '' : '\n\nHuman: ';
// For system prompt format. If there is no message with role "user" or prefix "Human:" change the first assistant's prefix(insert the human's message).
} else if (i === firstAssistantIndex && !hasUser && withSyspromptSupport && useSystemPrompt) {
prefix = `${setHumanMsg}\n\nAssistant: `;
//prefix = addSysHumanMsg ? '\n\nHuman: ' + addSysHumanMsg + '\n\nAssistant: ' : '\n\nHuman: Let\'s get started.\n\nAssistant: ';
// Merge two messages with "\n\nHuman: " prefixes into one before the first Assistant's message. Fix messages order for default claude format when(messages > Context Size).
} else if (i > 0 && i === firstAssistantIndex - 1 && v.role === 'user' && (!withSyspromptSupport || (withSyspromptSupport && !useSystemPrompt))) {
prefix = '\n\nFirst message: ';
//Define role prefixes(role : prefix). Set the correct prefix according to the role/name.
} else {
prefix = {
'assistant': '\n\nAssistant: ',
'user': '\n\nHuman: ',
'system': v.name === 'example_assistant' ? '\n\nA: ' : v.name === 'example_user' ? '\n\nH: ' : '\n\n',
}[v.role] || '\n\n';
}
return prefix + v.content;
}).join('');
if (addHumanPrefix) {
requestPrompt = '\n\nHuman: ' + requestPrompt;
}
if (addAssistantPostfix) {
requestPrompt = requestPrompt + '\n\nAssistant: ';
}
if (withSystemPrompt) {
requestPrompt = systemPrompt + requestPrompt;
}
//Add the assistant suffix(if the option unchecked), add a prefill after it(if filled). Also Add the first human message before the assistant suffix(when using sysprompt and there are no other messages with the role 'Assistant').
requestPrompt += addAssistantPostfix ? `${withSyspromptSupport && useSystemPrompt && firstAssistantIndex === -1 ? setHumanMsg : ''}\n\nAssistant: ${addAssistantPrefill ? addAssistantPrefill : ''}` : '';
return requestPrompt;
}