rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1890 from kingbased/nuclaude 

Claude 3
This commit is contained in:
Cohee 2024-03-05 20:42:36 +02:00 committed by GitHub
parent dc32884ced 00a4a12d7d
commit d303b36023
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 219 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
default/settings.json
View File

@ -623,7 +623,6 @@
"show_external_models": false,
"proxy_password": "",
"assistant_prefill": "",
"use_ai21_tokenizer": false,
"exclude_assistant": false
"use_ai21_tokenizer": false
}
}

56
public/index.html
View File

@ -1654,12 +1654,12 @@
</span>
</div>
</div>
<div class="range-block" data-source="openai,openrouter,makersuite,custom">
<div class="range-block" data-source="openai,openrouter,makersuite,claude,custom">
<label for="openai_image_inlining" class="checkbox_label flexWrap widthFreeExpand">
<input id="openai_image_inlining" type="checkbox" />
<span data-i18n="Send inline images">Send inline images</span>
<div id="image_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
Sends images in prompts if the model supports it (e.g. GPT-4V or Llava 13B).
Sends images in prompts if the model supports it (e.g. GPT-4V, Claude 3 or Llava 13B).
Use the <code><i class="fa-solid fa-paperclip"></i></code> action on any message or the
<code><i class="fa-solid fa-wand-magic-sparkles"></i></code> menu to attach an image file to the chat.
</div>
@ -1682,26 +1682,7 @@
</div>
</div>
<div data-newbie-hidden class="range-block" data-source="claude">
<label for="claude_exclude_prefixes" title="Exclude Human/Assistant prefixes" class="checkbox_label widthFreeExpand">
<input id="claude_exclude_prefixes" type="checkbox" />
<span data-i18n="Exclude Human/Assistant prefixes">Exclude Human/Assistant prefixes</span>
</label>
<div class="toggle-description justifyLeft marginBot5">
<span data-i18n="Exclude Human/Assistant prefixes from being added to the prompt.">
Exclude Human/Assistant prefixes from being added to the prompt, except very first/last message, system prompt Human message and Assistant suffix.
Requires 'Add character names' checked.
</span>
</div>
<label for="exclude_assistant" title="Exclude Assistant suffix" class="checkbox_label widthFreeExpand">
<input id="exclude_assistant" type="checkbox" />
<span data-i18n="Exclude Assistant suffix">Exclude Assistant suffix</span>
</label>
<div class="toggle-description justifyLeft marginBot5">
<span data-i18n="Exclude the assistant suffix from being added to the end of prompt.">
Exclude the assistant suffix from being added to the end of prompt. Requires jailbreak with 'Assistant:' in it.
</span>
</div>
<div id="claude_assistant_prefill_block" class="wide100p">
<div class="wide100p">
<span id="claude_assistant_prefill_text" data-i18n="Assistant Prefill">Assistant Prefill</span>
<textarea id="claude_assistant_prefill" class="text_pole textarea_compact" name="assistant_prefill autoSetHeight" rows="3" maxlength="10000" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
</div>
@ -1712,19 +1693,18 @@
</span>
</label>
<div class="toggle-description justifyLeft marginBot5">
<span data-i18n="Exclude the 'Human: ' prefix from being added to the beginning of the prompt.">
Exclude the 'Human: ' prefix from being added to the beginning of the prompt.
Instead, place it between the system prompt and the first message with the role 'assistant' (right before 'Chat History' by default).
<span data-i18n="Send the system prompt for supported models.">
Send the system prompt for supported models. If disabled, the user message is added to the beginning of the prompt.
</span>
</div>
<div id="claude_human_sysprompt_message_block" class="wide100p">
<div class="range-block-title openai_restorable">
<span data-i18n="Human: first message">Human: first message</span>
<div id="claude_human_sysprompt_message_restore" title="Restore Human: first message" class="right_menu_button">
<span data-i18n="User first message">User first message</span>
<div id="claude_human_sysprompt_message_restore" title="Restore User first message" class="right_menu_button">
<div class="fa-solid fa-clock-rotate-left"></div>
</div>
</div>
<textarea id="claude_human_sysprompt_textarea" class="text_pole textarea_compact" rows="4" maxlength="10000" data-i18n="[placeholder]Human message" placeholder="Human message, instruction, etc.&#10;Adds nothing when empty, i.e. requires a new prompt with the role 'user' or manually adding the 'Human: ' prefix."></textarea>
<textarea id="claude_human_sysprompt_textarea" class="text_pole textarea_compact" rows="4" maxlength="10000" data-i18n="[placeholder]Human message" placeholder="Human message, instruction, etc.&#10;Adds nothing when empty, i.e. requires a new prompt with the role 'user'."></textarea>
</div>
</div>
</div>
@ -2347,24 +2327,14 @@
<div>
<h4 data-i18n="Claude Model">Claude Model</h4>
<select id="model_claude_select">
<optgroup label="Latest">
<option value="claude-2">claude-2</option>
<option value="claude-v1">claude-v1</option>
<option value="claude-v1-100k">claude-v1-100k</option>
<option value="claude-instant-v1">claude-instant-v1</option>
<option value="claude-instant-v1-100k">claude-instant-v1-100k</option>
</optgroup>
<optgroup label="Sub-versions">
<optgroup label="Versions">
<option value="claude-3-opus-20240229">claude-3-opus-20240229</option>
<option value="claude-3-sonnet-20240229">claude-3-sonnet-20240229</option>
<option value="claude-2.1">claude-2.1</option>
<option value="claude-2.0">claude-2.0</option>
<option value="claude-v1.3">claude-v1.3</option>
<option value="claude-v1.3-100k">claude-v1.3-100k</option>
<option value="claude-v1.2">claude-v1.2</option>
<option value="claude-v1.0">claude-v1.0</option>
<option value="claude-1.3">claude-1.3</option>
<option value="claude-instant-1.2">claude-instant-1.2</option>
<option value="claude-instant-v1.1">claude-instant-v1.1</option>
<option value="claude-instant-v1.1-100k">claude-instant-v1.1-100k</option>
<option value="claude-instant-v1.0">claude-instant-v1.0</option>
<option value="claude-instant-1.1">claude-instant-1.1</option>
</optgroup>
</select>
</div>

11
public/scripts/extensions/caption/index.js
View File

@ -284,6 +284,7 @@ jQuery(function () {
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openai' && (secret_state[SECRET_KEYS.OPENAI] || extension_settings.caption.allow_reverse_proxy)) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openrouter' && secret_state[SECRET_KEYS.OPENROUTER]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && secret_state[SECRET_KEYS.MAKERSUITE]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'anthropic' && secret_state[SECRET_KEYS.CLAUDE]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ollama' && textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'llamacpp' && textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ooba' && textgenerationwebui_settings.server_urls[textgen_types.OOBA]) ||
@ -316,7 +317,8 @@ jQuery(function () {
$('#caption_multimodal_model').val(extension_settings.caption.multimodal_model);
$('#caption_multimodal_block [data-type]').each(function () {
const type = $(this).data('type');
$(this).toggle(type === extension_settings.caption.multimodal_api);
const types = type.split(',');
$(this).toggle(types.includes(extension_settings.caption.multimodal_api));
});
$('#caption_multimodal_api').on('change', () => {
const api = String($('#caption_multimodal_api').val());
@ -343,7 +345,7 @@ jQuery(function () {
<label for="caption_source">Source</label>
<select id="caption_source" class="text_pole">
<option value="local">Local</option>
<option value="multimodal">Multimodal (OpenAI / llama / Google)</option>
<option value="multimodal">Multimodal (OpenAI / Anthropic / llama / Google)</option>
<option value="extras">Extras</option>
<option value="horde">Horde</option>
</select>
@ -355,6 +357,7 @@ jQuery(function () {
<option value="ooba">Text Generation WebUI (oobabooga)</option>
<option value="ollama">Ollama</option>
<option value="openai">OpenAI</option>
<option value="anthropic">Anthropic</option>
<option value="openrouter">OpenRouter</option>
<option value="google">Google MakerSuite</option>
<option value="custom">Custom (OpenAI-compatible)</option>
@ -364,6 +367,8 @@ jQuery(function () {
<label for="caption_multimodal_model">Model</label>
<select id="caption_multimodal_model" class="flex1 text_pole">
<option data-type="openai" value="gpt-4-vision-preview">gpt-4-vision-preview</option>
<option data-type="anthropic" value="claude-3-opus-20240229">claude-3-opus-20240229</option>
<option data-type="anthropic" value="claude-3-sonnet-20240229">claude-3-sonnet-20240229</option>
<option data-type="google" value="gemini-pro-vision">gemini-pro-vision</option>
<option data-type="openrouter" value="openai/gpt-4-vision-preview">openai/gpt-4-vision-preview</option>
<option data-type="openrouter" value="haotian-liu/llava-13b">haotian-liu/llava-13b</option>
@ -375,7 +380,7 @@ jQuery(function () {
<option data-type="custom" value="custom_current">[Currently selected]</option>
</select>
</div>
<label data-type="openai" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
<label data-type="openai,anthropic" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
<input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">
Allow reverse proxy
</label>

5
public/scripts/extensions/shared.js
View File

@ -23,6 +23,7 @@ export async function getMultimodalCaption(base64Img, prompt) {
// OpenRouter has a payload limit of ~2MB. Google is 4MB, but we love democracy.
// Ooba requires all images to be JPEGs.
const isGoogle = extension_settings.caption.multimodal_api === 'google';
const isClaude = extension_settings.caption.multimodal_api === 'anthropic';
const isOllama = extension_settings.caption.multimodal_api === 'ollama';
const isLlamaCpp = extension_settings.caption.multimodal_api === 'llamacpp';
const isCustom = extension_settings.caption.multimodal_api === 'custom';
@ -39,7 +40,7 @@ export async function getMultimodalCaption(base64Img, prompt) {
}
const useReverseProxy =
extension_settings.caption.multimodal_api === 'openai'
(extension_settings.caption.multimodal_api === 'openai' || extension_settings.caption.multimodal_api === 'anthropic')
&& extension_settings.caption.allow_reverse_proxy
&& oai_settings.reverse_proxy
&& isValidUrl(oai_settings.reverse_proxy);
@ -87,6 +88,8 @@ export async function getMultimodalCaption(base64Img, prompt) {
switch (extension_settings.caption.multimodal_api) {
case 'google':
return '/api/google/caption-image';
case 'anthropic':
return '/api/anthropic/caption-image';
case 'llamacpp':
return '/api/backends/text-completions/llamacpp/caption-image';
case 'ollama':

46
public/scripts/openai.js
View File

@ -213,7 +213,7 @@ const default_settings = {
scenario_format: default_scenario_format,
personality_format: default_personality_format,
openai_model: 'gpt-3.5-turbo',
claude_model: 'claude-instant-v1',
claude_model: 'claude-2.1',
google_model: 'gemini-pro',
ai21_model: 'j2-ultra',
mistralai_model: 'mistral-medium-latest',
@ -239,9 +239,7 @@ const default_settings = {
human_sysprompt_message: default_claude_human_sysprompt_message,
use_ai21_tokenizer: false,
use_google_tokenizer: false,
exclude_assistant: false,
claude_use_sysprompt: false,
claude_exclude_prefixes: false,
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
@ -282,7 +280,7 @@ const oai_settings = {
scenario_format: default_scenario_format,
personality_format: default_personality_format,
openai_model: 'gpt-3.5-turbo',
claude_model: 'claude-instant-v1',
claude_model: 'claude-2.1',
google_model: 'gemini-pro',
ai21_model: 'j2-ultra',
mistralai_model: 'mistral-medium-latest',
@ -308,9 +306,7 @@ const oai_settings = {
human_sysprompt_message: default_claude_human_sysprompt_message,
use_ai21_tokenizer: false,
use_google_tokenizer: false,
exclude_assistant: false,
claude_use_sysprompt: false,
claude_exclude_prefixes: false,
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
@ -1634,13 +1630,11 @@ async function sendOpenAIRequest(type, messages, signal) {
if (isClaude) {
generate_data['top_k'] = Number(oai_settings.top_k_openai);
generate_data['exclude_assistant'] = oai_settings.exclude_assistant;
generate_data['claude_use_sysprompt'] = oai_settings.claude_use_sysprompt;
generate_data['claude_exclude_prefixes'] = oai_settings.claude_exclude_prefixes;
generate_data['stop'] = getCustomStoppingStrings(); // Claude shouldn't have limits on stop strings.
generate_data['human_sysprompt_message'] = substituteParams(oai_settings.human_sysprompt_message);
// Don't add a prefill on quiet gens (summarization)
if (!isQuiet && !oai_settings.exclude_assistant) {
if (!isQuiet) {
generate_data['assistant_prefill'] = substituteParams(oai_settings.assistant_prefill);
}
}
@ -1751,7 +1745,7 @@ async function sendOpenAIRequest(type, messages, signal) {
function getStreamingReply(data) {
if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {
return data?.completion || '';
return data?.delta?.text || '';
} else if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
return data?.candidates?.[0]?.content?.parts?.[0]?.text || '';
} else {
@ -2564,9 +2558,7 @@ function loadOpenAISettings(data, settings) {
if (settings.openai_model !== undefined) oai_settings.openai_model = settings.openai_model;
if (settings.use_ai21_tokenizer !== undefined) { oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer; oai_settings.use_ai21_tokenizer ? ai21_max = 8191 : ai21_max = 9200; }
if (settings.use_google_tokenizer !== undefined) oai_settings.use_google_tokenizer = !!settings.use_google_tokenizer;
if (settings.exclude_assistant !== undefined) oai_settings.exclude_assistant = !!settings.exclude_assistant;
if (settings.claude_use_sysprompt !== undefined) oai_settings.claude_use_sysprompt = !!settings.claude_use_sysprompt;
if (settings.claude_exclude_prefixes !== undefined) oai_settings.claude_exclude_prefixes = !!settings.claude_exclude_prefixes;
if (settings.use_alt_scale !== undefined) { oai_settings.use_alt_scale = !!settings.use_alt_scale; updateScaleForm(); }
$('#stream_toggle').prop('checked', oai_settings.stream_openai);
$('#api_url_scale').val(oai_settings.api_url_scale);
@ -2604,9 +2596,7 @@ function loadOpenAISettings(data, settings) {
$('#openai_external_category').toggle(oai_settings.show_external_models);
$('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer);
$('#use_google_tokenizer').prop('checked', oai_settings.use_google_tokenizer);
$('#exclude_assistant').prop('checked', oai_settings.exclude_assistant);
$('#claude_use_sysprompt').prop('checked', oai_settings.claude_use_sysprompt);
$('#claude_exclude_prefixes').prop('checked', oai_settings.claude_exclude_prefixes);
$('#scale-alt').prop('checked', oai_settings.use_alt_scale);
$('#openrouter_use_fallback').prop('checked', oai_settings.openrouter_use_fallback);
$('#openrouter_force_instruct').prop('checked', oai_settings.openrouter_force_instruct);
@ -2828,9 +2818,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
human_sysprompt_message: settings.human_sysprompt_message,
use_ai21_tokenizer: settings.use_ai21_tokenizer,
use_google_tokenizer: settings.use_google_tokenizer,
exclude_assistant: settings.exclude_assistant,
claude_use_sysprompt: settings.claude_use_sysprompt,
claude_exclude_prefixes: settings.claude_exclude_prefixes,
use_alt_scale: settings.use_alt_scale,
squash_system_messages: settings.squash_system_messages,
image_inlining: settings.image_inlining,
@ -3205,9 +3193,7 @@ function onSettingsPresetChange() {
human_sysprompt_message: ['#claude_human_sysprompt_textarea', 'human_sysprompt_message', false],
use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', true],
use_google_tokenizer: ['#use_google_tokenizer', 'use_google_tokenizer', true],
exclude_assistant: ['#exclude_assistant', 'exclude_assistant', true],
claude_use_sysprompt: ['#claude_use_sysprompt', 'claude_use_sysprompt', true],
claude_exclude_prefixes: ['#claude_exclude_prefixes', 'claude_exclude_prefixes', true],
use_alt_scale: ['#use_alt_scale', 'use_alt_scale', true],
squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true],
image_inlining: ['#openai_image_inlining', 'image_inlining', true],
@ -3330,8 +3316,15 @@ async function onModelChange() {
let value = String($(this).val() || '');
if ($(this).is('#model_claude_select')) {
if (value.includes('-v')) {
value = value.replace('-v', '-');
} else if (value === '' || value === 'claude-2') {
value = default_settings.claude_model;
}
console.log('Claude model changed to', value);
oai_settings.claude_model = value;
$('#model_claude_select').val(oai_settings.claude_model);
}
if ($(this).is('#model_windowai_select')) {
@ -3439,7 +3432,7 @@ async function onModelChange() {
if (oai_settings.max_context_unlocked) {
$('#openai_max_context').attr('max', max_200k);
}
else if (value == 'claude-2.1' || value == 'claude-2') {
else if (value == 'claude-2.1' || value.startsWith('claude-3')) {
$('#openai_max_context').attr('max', max_200k);
}
else if (value.endsWith('100k') || value.startsWith('claude-2') || value === 'claude-instant-1.2') {
@ -3735,7 +3728,6 @@ function toggleChatCompletionForms() {
});
if (chat_completion_sources.CLAUDE == oai_settings.chat_completion_source) {
$('#claude_assistant_prefill_block').toggle(!oai_settings.exclude_assistant);
$('#claude_human_sysprompt_message_block').toggle(oai_settings.claude_use_sysprompt);
}
}
@ -3829,6 +3821,7 @@ export function isImageInliningSupported() {
const gpt4v = 'gpt-4-vision';
const geminiProV = 'gemini-pro-vision';
const claude = 'claude-3';
const llava = 'llava';
if (!oai_settings.image_inlining) {
@ -3840,6 +3833,8 @@ export function isImageInliningSupported() {
return oai_settings.openai_model.includes(gpt4v);
case chat_completion_sources.MAKERSUITE:
return oai_settings.google_model.includes(geminiProV);
case chat_completion_sources.CLAUDE:
return oai_settings.claude_model.includes(claude);
case chat_completion_sources.OPENROUTER:
return !oai_settings.openrouter_force_instruct && (oai_settings.openrouter_model.includes(gpt4v) || oai_settings.openrouter_model.includes(llava));
case chat_completion_sources.CUSTOM:
@ -4075,23 +4070,12 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#exclude_assistant').on('change', function () {
oai_settings.exclude_assistant = !!$('#exclude_assistant').prop('checked');
$('#claude_assistant_prefill_block').toggle(!oai_settings.exclude_assistant);
saveSettingsDebounced();
});
$('#claude_use_sysprompt').on('change', function () {
oai_settings.claude_use_sysprompt = !!$('#claude_use_sysprompt').prop('checked');
$('#claude_human_sysprompt_message_block').toggle(oai_settings.claude_use_sysprompt);
saveSettingsDebounced();
});
$('#claude_exclude_prefixes').on('change', function () {
oai_settings.claude_exclude_prefixes = !!$('#claude_exclude_prefixes').prop('checked');
saveSettingsDebounced();
});
$('#names_in_completion').on('change', function () {
oai_settings.names_in_completion = !!$('#names_in_completion').prop('checked');
saveSettingsDebounced();

3
server.js
View File

@ -505,6 +505,9 @@ app.use('/api/openai', require('./src/endpoints/openai').router);
//Google API
app.use('/api/google', require('./src/endpoints/google').router);
//Anthropic API
app.use('/api/anthropic', require('./src/endpoints/anthropic').router);
// Tokenizers
app.use('/api/tokenizers', require('./src/endpoints/tokenizers').router);

67
src/endpoints/anthropic.js Normal file
View File

@ -0,0 +1,67 @@
const { readSecret, SECRET_KEYS } = require('./secrets');
const fetch = require('node-fetch').default;
const express = require('express');
const { jsonParser } = require('../express-common');
const router = express.Router();
router.post('/caption-image', jsonParser, async (request, response) => {
try {
const mimeType = request.body.image.split(';')[0].split(':')[1];
const base64Data = request.body.image.split(',')[1];
const baseUrl = request.body.reverse_proxy ? request.body.reverse_proxy : 'https://api.anthropic.com/v1';
const url = `${baseUrl}/messages`;
const body = {
model: request.body.model,
messages: [
{
'role': 'user', 'content': [
{
'type': 'image',
'source': {
'type': 'base64',
'media_type': mimeType,
'data': base64Data,
},
},
{ 'type': 'text', 'text': request.body.prompt },
],
},
],
max_tokens: 800,
};
console.log('Multimodal captioning request', body);
const result = await fetch(url, {
body: JSON.stringify(body),
method: 'POST',
headers: {
'Content-Type': 'application/json',
'anthropic-version': '2023-06-01',
'x-api-key': request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.CLAUDE),
},
timeout: 0,
});
if (!result.ok) {
console.log(`Claude API returned error: ${result.status} ${result.statusText}`);
return response.status(result.status).send({ error: true });
}
const generateResponseJson = await result.json();
const caption = generateResponseJson.content[0].text;
console.log('Claude response:', generateResponseJson);
if (!caption) {
return response.status(500).send('No caption found');
}
return response.json({ caption });
} catch (error) {
console.error(error);
response.status(500).send('Internal server error');
}
});
module.exports = { router };

59
src/endpoints/backends/chat-completions.js
View File

@ -5,7 +5,7 @@ const { Readable } = require('stream');
const { jsonParser } = require('../../express-common');
const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY, OPENROUTER_HEADERS } = require('../../constants');
const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml, color } = require('../../util');
const { convertClaudePrompt, convertGooglePrompt, convertTextCompletionPrompt } = require('../prompt-converters');
const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt } = require('../prompt-converters');
const { readSecret, SECRET_KEYS } = require('../secrets');
const { getTokenizerModel, getSentencepiceTokenizer, getTiktokenTokenizer, sentencepieceTokenizers, TEXT_COMPLETION_MODELS } = require('../tokenizers');
@ -34,45 +34,8 @@ async function sendClaudeRequest(request, response) {
request.socket.on('close', function () {
controller.abort();
});
const isSysPromptSupported = request.body.model === 'claude-2' || request.body.model === 'claude-2.1';
const requestPrompt = convertClaudePrompt(request.body.messages, !request.body.exclude_assistant, request.body.assistant_prefill, isSysPromptSupported, request.body.claude_use_sysprompt, request.body.human_sysprompt_message, request.body.claude_exclude_prefixes);
// Check Claude messages sequence and prefixes presence.
let sequenceError = [];
const sequence = requestPrompt.split('\n').filter(x => x.startsWith('Human:') || x.startsWith('Assistant:'));
const humanFound = sequence.some(line => line.startsWith('Human:'));
const assistantFound = sequence.some(line => line.startsWith('Assistant:'));
let humanErrorCount = 0;
let assistantErrorCount = 0;
for (let i = 0; i < sequence.length - 1; i++) {
if (sequence[i].startsWith(sequence[i + 1].split(':')[0])) {
if (sequence[i].startsWith('Human:')) {
humanErrorCount++;
} else if (sequence[i].startsWith('Assistant:')) {
assistantErrorCount++;
}
}
}
if (!humanFound) {
sequenceError.push(`${divider}\nWarning: No 'Human:' prefix found in the prompt.\n${divider}`);
}
if (!assistantFound) {
sequenceError.push(`${divider}\nWarning: No 'Assistant: ' prefix found in the prompt.\n${divider}`);
}
if (sequence[0] && !sequence[0].startsWith('Human:')) {
sequenceError.push(`${divider}\nWarning: The messages sequence should start with 'Human:' prefix.\nMake sure you have '\\n\\nHuman:' prefix at the very beggining of the prompt, or after the system prompt.\n${divider}`);
}
if (humanErrorCount > 0 || assistantErrorCount > 0) {
sequenceError.push(`${divider}\nWarning: Detected incorrect Prefix sequence(s).`);
sequenceError.push(`Incorrect "Human:" prefix(es): ${humanErrorCount}.\nIncorrect "Assistant: " prefix(es): ${assistantErrorCount}.`);
sequenceError.push('Check the prompt above and fix it in the SillyTavern.');
sequenceError.push('\nThe correct sequence in the console should look like this:\n(System prompt msg) <-(for the sysprompt format only, else have \\n\\n above the first human\'s message.)');
sequenceError.push(`\\n + <-----(Each message beginning with the "Assistant:/Human:" prefix must have \\n\\n before it.)\n\\n +\nHuman: \\n +\n\\n +\nAssistant: \\n +\n...\n\\n +\nHuman: \\n +\n\\n +\nAssistant: \n${divider}`);
}
let use_system_prompt = (request.body.model.startsWith('claude-2') || request.body.model.startsWith('claude-3')) && request.body.claude_use_sysprompt;
let converted_prompt = convertClaudeMessages(request.body.messages, request.body.assistant_prefill, use_system_prompt, request.body.human_sysprompt_message);
// Add custom stop sequences
const stopSequences = ['\n\nHuman:', '\n\nSystem:', '\n\nAssistant:'];
if (Array.isArray(request.body.stop)) {
@ -80,23 +43,21 @@ async function sendClaudeRequest(request, response) {
}
const requestBody = {
prompt: requestPrompt,
messages: converted_prompt.messages,
model: request.body.model,
max_tokens_to_sample: request.body.max_tokens,
max_tokens: request.body.max_tokens,
stop_sequences: stopSequences,
temperature: request.body.temperature,
top_p: request.body.top_p,
top_k: request.body.top_k,
stream: request.body.stream,
};
if (use_system_prompt) {
requestBody.system = converted_prompt.systemPrompt;
}
console.log('Claude request:', requestBody);
sequenceError.forEach(sequenceError => {
console.log(color.red(sequenceError));
});
const generateResponse = await fetch(apiUrl + '/complete', {
const generateResponse = await fetch(apiUrl + '/messages', {
method: 'POST',
signal: controller.signal,
body: JSON.stringify(requestBody),
@ -118,7 +79,7 @@ async function sendClaudeRequest(request, response) {
}
const generateResponseJson = await generateResponse.json();
const responseText = generateResponseJson.completion;
const responseText = generateResponseJson.content[0].text;
console.log('Claude response:', generateResponseJson);
// Wrap it back to OAI format

97
src/endpoints/prompt-converters.js
View File

@ -71,6 +71,102 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill,
return requestPrompt;
}
/**
* Convert ChatML objects into working with Anthropic's new Messaging API.
* @param {object[]} messages Array of messages
* @param {string} prefillString User determined prefill string
* @param {boolean} useSysPrompt See if we want to use a system prompt
* @param {string} humanMsgFix Add Human message between system prompt and assistant.
*/
function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFix) {
let systemPrompt = '';
if (useSysPrompt) {
// Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
let i;
for (i = 0; i < messages.length; i++) {
if (messages[i].role !== 'system') {
break;
}
systemPrompt += `${messages[i].content}\n\n`;
}
messages.splice(0, i);
// Check if the first message in the array is of type user, if not, interject with humanMsgFix or a blank message.
if (messages.length > 0 && messages[0].role !== 'user') {
messages.unshift({
role: 'user',
content: humanMsgFix || '[Start a new chat]',
});
}
}
// Now replace all further messages that have the role 'system' with the role 'user'. (or all if we're not using one)
messages.forEach((message) => {
if (message.role === 'system') {
message.role = 'user';
}
});
// Since the messaging endpoint only supports user assistant roles in turns, we have to merge messages with the same role if they follow eachother
// Also handle multi-modality, holy slop.
let mergedMessages = [];
messages.forEach((message) => {
const imageEntry = message.content[1]?.image_url;
const imageData = imageEntry?.url;
const mimeType = imageData?.split(';')[0].split(':')[1];
const base64Data = imageData?.split(',')[1];
if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role) {
if(Array.isArray(message.content)) {
if(Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content[0].text;
} else {
mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content[0].text;
}
} else {
if(Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content;
} else {
mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content;
}
}
} else {
mergedMessages.push(message);
}
if (imageData) {
mergedMessages[mergedMessages.length - 1].content = [
{ type: 'text', text: mergedMessages[mergedMessages.length - 1].content[0]?.text || mergedMessages[mergedMessages.length - 1].content },
{
type: 'image', source: {
type: 'base64',
media_type: mimeType,
data: base64Data,
},
},
];
}
});
// Take care of name properties since claude messages don't support them
mergedMessages.forEach((message) => {
if (message.name) {
message.content = `${message.name}: ${message.content}`;
delete message.name;
}
});
// Shouldn't be conditional anymore, messages api expects the last role to be user unless we're explicitly prefilling
if (prefillString) {
mergedMessages.push({
role: 'assistant',
content: prefillString,
});
}
return { messages: mergedMessages, systemPrompt: systemPrompt.trim() };
}
/**
* Convert a prompt from the ChatML objects to the format used by Google MakerSuite models.
* @param {object[]} messages Array of messages
@ -160,6 +256,7 @@ function convertTextCompletionPrompt(messages) {
module.exports = {
convertClaudePrompt,
convertClaudeMessages,
convertGooglePrompt,
convertTextCompletionPrompt,
};

1
src/endpoints/tokenizers.js
View File

@ -249,6 +249,7 @@ async function loadClaudeTokenizer(modelPath) {
}
function countClaudeTokens(tokenizer, messages) {
// Should be fine if we use the old conversion method instead of the messages API one i think?
const convertedPrompt = convertClaudePrompt(messages, false, false, false);
// Fallback to strlen estimation