rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1890 from kingbased/nuclaude

Browse Source 
Claude 3
This commit is contained in:
Cohee
2024-03-05 20:42:36 +02:00
committed by GitHub
parent dc32884ced 00a4a12d7d
commit d303b36023
10 changed files with 219 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
default/settings.json
View File

@@ -623,7 +623,6 @@
"show_external_models": false, "show_external_models": false,
"proxy_password": "", "proxy_password": "",
"assistant_prefill": "", "assistant_prefill": "",
"use_ai21_tokenizer": false, "use_ai21_tokenizer": false
"exclude_assistant": false
} }
} }

56
public/index.html
View File

@@ -1654,12 +1654,12 @@
</span> </span>
</div> </div>
</div> </div>
<div class="range-block" data-source="openai,openrouter,makersuite,custom"> <div class="range-block" data-source="openai,openrouter,makersuite,claude,custom">
<label for="openai_image_inlining" class="checkbox_label flexWrap widthFreeExpand"> <label for="openai_image_inlining" class="checkbox_label flexWrap widthFreeExpand">
<input id="openai_image_inlining" type="checkbox" /> <input id="openai_image_inlining" type="checkbox" />
<span data-i18n="Send inline images">Send inline images</span> <span data-i18n="Send inline images">Send inline images</span>
<div id="image_inlining_hint" class="flexBasis100p toggle-description justifyLeft"> <div id="image_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
Sends images in prompts if the model supports it (e.g. GPT-4V or Llava 13B). Sends images in prompts if the model supports it (e.g. GPT-4V, Claude 3 or Llava 13B).
Use the <code><i class="fa-solid fa-paperclip"></i></code> action on any message or the Use the <code><i class="fa-solid fa-paperclip"></i></code> action on any message or the
<code><i class="fa-solid fa-wand-magic-sparkles"></i></code> menu to attach an image file to the chat. <code><i class="fa-solid fa-wand-magic-sparkles"></i></code> menu to attach an image file to the chat.
</div> </div>
@@ -1682,26 +1682,7 @@
</div> </div>
</div> </div>
<div data-newbie-hidden class="range-block" data-source="claude"> <div data-newbie-hidden class="range-block" data-source="claude">
<label for="claude_exclude_prefixes" title="Exclude Human/Assistant prefixes" class="checkbox_label widthFreeExpand"> <div class="wide100p">
<input id="claude_exclude_prefixes" type="checkbox" />
<span data-i18n="Exclude Human/Assistant prefixes">Exclude Human/Assistant prefixes</span>
</label>
<div class="toggle-description justifyLeft marginBot5">
<span data-i18n="Exclude Human/Assistant prefixes from being added to the prompt.">
Exclude Human/Assistant prefixes from being added to the prompt, except very first/last message, system prompt Human message and Assistant suffix.
Requires 'Add character names' checked.
</span>
</div>
<label for="exclude_assistant" title="Exclude Assistant suffix" class="checkbox_label widthFreeExpand">
<input id="exclude_assistant" type="checkbox" />
<span data-i18n="Exclude Assistant suffix">Exclude Assistant suffix</span>
</label>
<div class="toggle-description justifyLeft marginBot5">
<span data-i18n="Exclude the assistant suffix from being added to the end of prompt.">
Exclude the assistant suffix from being added to the end of prompt. Requires jailbreak with 'Assistant:' in it.
</span>
</div>
<div id="claude_assistant_prefill_block" class="wide100p">
<span id="claude_assistant_prefill_text" data-i18n="Assistant Prefill">Assistant Prefill</span> <span id="claude_assistant_prefill_text" data-i18n="Assistant Prefill">Assistant Prefill</span>
<textarea id="claude_assistant_prefill" class="text_pole textarea_compact" name="assistant_prefill autoSetHeight" rows="3" maxlength="10000" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea> <textarea id="claude_assistant_prefill" class="text_pole textarea_compact" name="assistant_prefill autoSetHeight" rows="3" maxlength="10000" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
</div> </div>
@@ -1712,19 +1693,18 @@
</span> </span>
</label> </label>
<div class="toggle-description justifyLeft marginBot5"> <div class="toggle-description justifyLeft marginBot5">
<span data-i18n="Exclude the 'Human: ' prefix from being added to the beginning of the prompt."> <span data-i18n="Send the system prompt for supported models.">
Exclude the 'Human: ' prefix from being added to the beginning of the prompt. Send the system prompt for supported models. If disabled, the user message is added to the beginning of the prompt.
Instead, place it between the system prompt and the first message with the role 'assistant' (right before 'Chat History' by default).
</span> </span>
</div> </div>
<div id="claude_human_sysprompt_message_block" class="wide100p"> <div id="claude_human_sysprompt_message_block" class="wide100p">
<div class="range-block-title openai_restorable"> <div class="range-block-title openai_restorable">
<span data-i18n="Human: first message">Human: first message</span> <span data-i18n="User first message">User first message</span>
<div id="claude_human_sysprompt_message_restore" title="Restore Human: first message" class="right_menu_button"> <div id="claude_human_sysprompt_message_restore" title="Restore User first message" class="right_menu_button">
<div class="fa-solid fa-clock-rotate-left"></div> <div class="fa-solid fa-clock-rotate-left"></div>
</div> </div>
</div> </div>
<textarea id="claude_human_sysprompt_textarea" class="text_pole textarea_compact" rows="4" maxlength="10000" data-i18n="[placeholder]Human message" placeholder="Human message, instruction, etc.&#10;Adds nothing when empty, i.e. requires a new prompt with the role 'user' or manually adding the 'Human: ' prefix."></textarea> <textarea id="claude_human_sysprompt_textarea" class="text_pole textarea_compact" rows="4" maxlength="10000" data-i18n="[placeholder]Human message" placeholder="Human message, instruction, etc.&#10;Adds nothing when empty, i.e. requires a new prompt with the role 'user'."></textarea>
</div> </div>
</div> </div>
</div> </div>
@@ -2347,24 +2327,14 @@
<div> <div>
<h4 data-i18n="Claude Model">Claude Model</h4> <h4 data-i18n="Claude Model">Claude Model</h4>
<select id="model_claude_select"> <select id="model_claude_select">
<optgroup label="Latest"> <optgroup label="Versions">
<option value="claude-2">claude-2</option> <option value="claude-3-opus-20240229">claude-3-opus-20240229</option>
<option value="claude-v1">claude-v1</option> <option value="claude-3-sonnet-20240229">claude-3-sonnet-20240229</option>
<option value="claude-v1-100k">claude-v1-100k</option>
<option value="claude-instant-v1">claude-instant-v1</option>
<option value="claude-instant-v1-100k">claude-instant-v1-100k</option>
</optgroup>
<optgroup label="Sub-versions">
<option value="claude-2.1">claude-2.1</option> <option value="claude-2.1">claude-2.1</option>
<option value="claude-2.0">claude-2.0</option> <option value="claude-2.0">claude-2.0</option>
<option value="claude-v1.3">claude-v1.3</option> <option value="claude-1.3">claude-1.3</option>
<option value="claude-v1.3-100k">claude-v1.3-100k</option>
<option value="claude-v1.2">claude-v1.2</option>
<option value="claude-v1.0">claude-v1.0</option>
<option value="claude-instant-1.2">claude-instant-1.2</option> <option value="claude-instant-1.2">claude-instant-1.2</option>
<option value="claude-instant-v1.1">claude-instant-v1.1</option> <option value="claude-instant-1.1">claude-instant-1.1</option>
<option value="claude-instant-v1.1-100k">claude-instant-v1.1-100k</option>
<option value="claude-instant-v1.0">claude-instant-v1.0</option>
</optgroup> </optgroup>
</select> </select>
</div> </div>

11
public/scripts/extensions/caption/index.js
View File

@@ -284,6 +284,7 @@ jQuery(function () {
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openai' && (secret_state[SECRET_KEYS.OPENAI] || extension_settings.caption.allow_reverse_proxy)) || (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openai' && (secret_state[SECRET_KEYS.OPENAI] || extension_settings.caption.allow_reverse_proxy)) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openrouter' && secret_state[SECRET_KEYS.OPENROUTER]) || (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openrouter' && secret_state[SECRET_KEYS.OPENROUTER]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && secret_state[SECRET_KEYS.MAKERSUITE]) || (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && secret_state[SECRET_KEYS.MAKERSUITE]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'anthropic' && secret_state[SECRET_KEYS.CLAUDE]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ollama' && textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) || (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ollama' && textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'llamacpp' && textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP]) || (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'llamacpp' && textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP]) ||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ooba' && textgenerationwebui_settings.server_urls[textgen_types.OOBA]) || (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ooba' && textgenerationwebui_settings.server_urls[textgen_types.OOBA]) ||
@@ -316,7 +317,8 @@ jQuery(function () {
$('#caption_multimodal_model').val(extension_settings.caption.multimodal_model); $('#caption_multimodal_model').val(extension_settings.caption.multimodal_model);
$('#caption_multimodal_block [data-type]').each(function () { $('#caption_multimodal_block [data-type]').each(function () {
const type = $(this).data('type'); const type = $(this).data('type');
$(this).toggle(type === extension_settings.caption.multimodal_api); const types = type.split(',');
$(this).toggle(types.includes(extension_settings.caption.multimodal_api));
}); });
$('#caption_multimodal_api').on('change', () => { $('#caption_multimodal_api').on('change', () => {
const api = String($('#caption_multimodal_api').val()); const api = String($('#caption_multimodal_api').val());
@@ -343,7 +345,7 @@ jQuery(function () {
<label for="caption_source">Source</label> <label for="caption_source">Source</label>
<select id="caption_source" class="text_pole"> <select id="caption_source" class="text_pole">
<option value="local">Local</option> <option value="local">Local</option>
<option value="multimodal">Multimodal (OpenAI / llama / Google)</option> <option value="multimodal">Multimodal (OpenAI / Anthropic / llama / Google)</option>
<option value="extras">Extras</option> <option value="extras">Extras</option>
<option value="horde">Horde</option> <option value="horde">Horde</option>
</select> </select>
@@ -355,6 +357,7 @@ jQuery(function () {
<option value="ooba">Text Generation WebUI (oobabooga)</option> <option value="ooba">Text Generation WebUI (oobabooga)</option>
<option value="ollama">Ollama</option> <option value="ollama">Ollama</option>
<option value="openai">OpenAI</option> <option value="openai">OpenAI</option>
<option value="anthropic">Anthropic</option>
<option value="openrouter">OpenRouter</option> <option value="openrouter">OpenRouter</option>
<option value="google">Google MakerSuite</option> <option value="google">Google MakerSuite</option>
<option value="custom">Custom (OpenAI-compatible)</option> <option value="custom">Custom (OpenAI-compatible)</option>
@@ -364,6 +367,8 @@ jQuery(function () {
<label for="caption_multimodal_model">Model</label> <label for="caption_multimodal_model">Model</label>
<select id="caption_multimodal_model" class="flex1 text_pole"> <select id="caption_multimodal_model" class="flex1 text_pole">
<option data-type="openai" value="gpt-4-vision-preview">gpt-4-vision-preview</option> <option data-type="openai" value="gpt-4-vision-preview">gpt-4-vision-preview</option>
<option data-type="anthropic" value="claude-3-opus-20240229">claude-3-opus-20240229</option>
<option data-type="anthropic" value="claude-3-sonnet-20240229">claude-3-sonnet-20240229</option>
<option data-type="google" value="gemini-pro-vision">gemini-pro-vision</option> <option data-type="google" value="gemini-pro-vision">gemini-pro-vision</option>
<option data-type="openrouter" value="openai/gpt-4-vision-preview">openai/gpt-4-vision-preview</option> <option data-type="openrouter" value="openai/gpt-4-vision-preview">openai/gpt-4-vision-preview</option>
<option data-type="openrouter" value="haotian-liu/llava-13b">haotian-liu/llava-13b</option> <option data-type="openrouter" value="haotian-liu/llava-13b">haotian-liu/llava-13b</option>
@@ -375,7 +380,7 @@ jQuery(function () {
<option data-type="custom" value="custom_current">[Currently selected]</option> <option data-type="custom" value="custom_current">[Currently selected]</option>
</select> </select>
</div> </div>
<label data-type="openai" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid."> <label data-type="openai,anthropic" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
<input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox"> <input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">
Allow reverse proxy Allow reverse proxy
</label> </label>

5
public/scripts/extensions/shared.js
View File

@@ -23,6 +23,7 @@ export async function getMultimodalCaption(base64Img, prompt) {
// OpenRouter has a payload limit of ~2MB. Google is 4MB, but we love democracy. // OpenRouter has a payload limit of ~2MB. Google is 4MB, but we love democracy.
// Ooba requires all images to be JPEGs. // Ooba requires all images to be JPEGs.
const isGoogle = extension_settings.caption.multimodal_api === 'google'; const isGoogle = extension_settings.caption.multimodal_api === 'google';
const isClaude = extension_settings.caption.multimodal_api === 'anthropic';
const isOllama = extension_settings.caption.multimodal_api === 'ollama'; const isOllama = extension_settings.caption.multimodal_api === 'ollama';
const isLlamaCpp = extension_settings.caption.multimodal_api === 'llamacpp'; const isLlamaCpp = extension_settings.caption.multimodal_api === 'llamacpp';
const isCustom = extension_settings.caption.multimodal_api === 'custom'; const isCustom = extension_settings.caption.multimodal_api === 'custom';
@@ -39,7 +40,7 @@ export async function getMultimodalCaption(base64Img, prompt) {
} }
const useReverseProxy = const useReverseProxy =
extension_settings.caption.multimodal_api === 'openai' (extension_settings.caption.multimodal_api === 'openai' || extension_settings.caption.multimodal_api === 'anthropic')
&& extension_settings.caption.allow_reverse_proxy && extension_settings.caption.allow_reverse_proxy
&& oai_settings.reverse_proxy && oai_settings.reverse_proxy
&& isValidUrl(oai_settings.reverse_proxy); && isValidUrl(oai_settings.reverse_proxy);
@@ -87,6 +88,8 @@ export async function getMultimodalCaption(base64Img, prompt) {
switch (extension_settings.caption.multimodal_api) { switch (extension_settings.caption.multimodal_api) {
case 'google': case 'google':
return '/api/google/caption-image'; return '/api/google/caption-image';
case 'anthropic':
return '/api/anthropic/caption-image';
case 'llamacpp': case 'llamacpp':
return '/api/backends/text-completions/llamacpp/caption-image'; return '/api/backends/text-completions/llamacpp/caption-image';
case 'ollama': case 'ollama':

46
public/scripts/openai.js
View File

@@ -213,7 +213,7 @@ const default_settings = {
scenario_format: default_scenario_format, scenario_format: default_scenario_format,
personality_format: default_personality_format, personality_format: default_personality_format,
openai_model: 'gpt-3.5-turbo', openai_model: 'gpt-3.5-turbo',
claude_model: 'claude-instant-v1', claude_model: 'claude-2.1',
google_model: 'gemini-pro', google_model: 'gemini-pro',
ai21_model: 'j2-ultra', ai21_model: 'j2-ultra',
mistralai_model: 'mistral-medium-latest', mistralai_model: 'mistral-medium-latest',
@@ -239,9 +239,7 @@ const default_settings = {
human_sysprompt_message: default_claude_human_sysprompt_message, human_sysprompt_message: default_claude_human_sysprompt_message,
use_ai21_tokenizer: false, use_ai21_tokenizer: false,
use_google_tokenizer: false, use_google_tokenizer: false,
exclude_assistant: false,
claude_use_sysprompt: false, claude_use_sysprompt: false,
claude_exclude_prefixes: false,
use_alt_scale: false, use_alt_scale: false,
squash_system_messages: false, squash_system_messages: false,
image_inlining: false, image_inlining: false,
@@ -282,7 +280,7 @@ const oai_settings = {
scenario_format: default_scenario_format, scenario_format: default_scenario_format,
personality_format: default_personality_format, personality_format: default_personality_format,
openai_model: 'gpt-3.5-turbo', openai_model: 'gpt-3.5-turbo',
claude_model: 'claude-instant-v1', claude_model: 'claude-2.1',
google_model: 'gemini-pro', google_model: 'gemini-pro',
ai21_model: 'j2-ultra', ai21_model: 'j2-ultra',
mistralai_model: 'mistral-medium-latest', mistralai_model: 'mistral-medium-latest',
@@ -308,9 +306,7 @@ const oai_settings = {
human_sysprompt_message: default_claude_human_sysprompt_message, human_sysprompt_message: default_claude_human_sysprompt_message,
use_ai21_tokenizer: false, use_ai21_tokenizer: false,
use_google_tokenizer: false, use_google_tokenizer: false,
exclude_assistant: false,
claude_use_sysprompt: false, claude_use_sysprompt: false,
claude_exclude_prefixes: false,
use_alt_scale: false, use_alt_scale: false,
squash_system_messages: false, squash_system_messages: false,
image_inlining: false, image_inlining: false,
@@ -1634,13 +1630,11 @@ async function sendOpenAIRequest(type, messages, signal) {
if (isClaude) { if (isClaude) {
generate_data['top_k'] = Number(oai_settings.top_k_openai); generate_data['top_k'] = Number(oai_settings.top_k_openai);
generate_data['exclude_assistant'] = oai_settings.exclude_assistant;
generate_data['claude_use_sysprompt'] = oai_settings.claude_use_sysprompt; generate_data['claude_use_sysprompt'] = oai_settings.claude_use_sysprompt;
generate_data['claude_exclude_prefixes'] = oai_settings.claude_exclude_prefixes;
generate_data['stop'] = getCustomStoppingStrings(); // Claude shouldn't have limits on stop strings. generate_data['stop'] = getCustomStoppingStrings(); // Claude shouldn't have limits on stop strings.
generate_data['human_sysprompt_message'] = substituteParams(oai_settings.human_sysprompt_message); generate_data['human_sysprompt_message'] = substituteParams(oai_settings.human_sysprompt_message);
// Don't add a prefill on quiet gens (summarization) // Don't add a prefill on quiet gens (summarization)
if (!isQuiet && !oai_settings.exclude_assistant) { if (!isQuiet) {
generate_data['assistant_prefill'] = substituteParams(oai_settings.assistant_prefill); generate_data['assistant_prefill'] = substituteParams(oai_settings.assistant_prefill);
} }
} }
@@ -1751,7 +1745,7 @@ async function sendOpenAIRequest(type, messages, signal) {
function getStreamingReply(data) { function getStreamingReply(data) {
if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) { if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {
return data?.completion || ''; return data?.delta?.text || '';
} else if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) { } else if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
return data?.candidates?.[0]?.content?.parts?.[0]?.text || ''; return data?.candidates?.[0]?.content?.parts?.[0]?.text || '';
} else { } else {
@@ -2564,9 +2558,7 @@ function loadOpenAISettings(data, settings) {
if (settings.openai_model !== undefined) oai_settings.openai_model = settings.openai_model; if (settings.openai_model !== undefined) oai_settings.openai_model = settings.openai_model;
if (settings.use_ai21_tokenizer !== undefined) { oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer; oai_settings.use_ai21_tokenizer ? ai21_max = 8191 : ai21_max = 9200; } if (settings.use_ai21_tokenizer !== undefined) { oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer; oai_settings.use_ai21_tokenizer ? ai21_max = 8191 : ai21_max = 9200; }
if (settings.use_google_tokenizer !== undefined) oai_settings.use_google_tokenizer = !!settings.use_google_tokenizer; if (settings.use_google_tokenizer !== undefined) oai_settings.use_google_tokenizer = !!settings.use_google_tokenizer;
if (settings.exclude_assistant !== undefined) oai_settings.exclude_assistant = !!settings.exclude_assistant;
if (settings.claude_use_sysprompt !== undefined) oai_settings.claude_use_sysprompt = !!settings.claude_use_sysprompt; if (settings.claude_use_sysprompt !== undefined) oai_settings.claude_use_sysprompt = !!settings.claude_use_sysprompt;
if (settings.claude_exclude_prefixes !== undefined) oai_settings.claude_exclude_prefixes = !!settings.claude_exclude_prefixes;
if (settings.use_alt_scale !== undefined) { oai_settings.use_alt_scale = !!settings.use_alt_scale; updateScaleForm(); } if (settings.use_alt_scale !== undefined) { oai_settings.use_alt_scale = !!settings.use_alt_scale; updateScaleForm(); }
$('#stream_toggle').prop('checked', oai_settings.stream_openai); $('#stream_toggle').prop('checked', oai_settings.stream_openai);
$('#api_url_scale').val(oai_settings.api_url_scale); $('#api_url_scale').val(oai_settings.api_url_scale);
@@ -2604,9 +2596,7 @@ function loadOpenAISettings(data, settings) {
$('#openai_external_category').toggle(oai_settings.show_external_models); $('#openai_external_category').toggle(oai_settings.show_external_models);
$('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer); $('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer);
$('#use_google_tokenizer').prop('checked', oai_settings.use_google_tokenizer); $('#use_google_tokenizer').prop('checked', oai_settings.use_google_tokenizer);
$('#exclude_assistant').prop('checked', oai_settings.exclude_assistant);
$('#claude_use_sysprompt').prop('checked', oai_settings.claude_use_sysprompt); $('#claude_use_sysprompt').prop('checked', oai_settings.claude_use_sysprompt);
$('#claude_exclude_prefixes').prop('checked', oai_settings.claude_exclude_prefixes);
$('#scale-alt').prop('checked', oai_settings.use_alt_scale); $('#scale-alt').prop('checked', oai_settings.use_alt_scale);
$('#openrouter_use_fallback').prop('checked', oai_settings.openrouter_use_fallback); $('#openrouter_use_fallback').prop('checked', oai_settings.openrouter_use_fallback);
$('#openrouter_force_instruct').prop('checked', oai_settings.openrouter_force_instruct); $('#openrouter_force_instruct').prop('checked', oai_settings.openrouter_force_instruct);
@@ -2828,9 +2818,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
human_sysprompt_message: settings.human_sysprompt_message, human_sysprompt_message: settings.human_sysprompt_message,
use_ai21_tokenizer: settings.use_ai21_tokenizer, use_ai21_tokenizer: settings.use_ai21_tokenizer,
use_google_tokenizer: settings.use_google_tokenizer, use_google_tokenizer: settings.use_google_tokenizer,
exclude_assistant: settings.exclude_assistant,
claude_use_sysprompt: settings.claude_use_sysprompt, claude_use_sysprompt: settings.claude_use_sysprompt,
claude_exclude_prefixes: settings.claude_exclude_prefixes,
use_alt_scale: settings.use_alt_scale, use_alt_scale: settings.use_alt_scale,
squash_system_messages: settings.squash_system_messages, squash_system_messages: settings.squash_system_messages,
image_inlining: settings.image_inlining, image_inlining: settings.image_inlining,
@@ -3205,9 +3193,7 @@ function onSettingsPresetChange() {
human_sysprompt_message: ['#claude_human_sysprompt_textarea', 'human_sysprompt_message', false], human_sysprompt_message: ['#claude_human_sysprompt_textarea', 'human_sysprompt_message', false],
use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', true], use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', true],
use_google_tokenizer: ['#use_google_tokenizer', 'use_google_tokenizer', true], use_google_tokenizer: ['#use_google_tokenizer', 'use_google_tokenizer', true],
exclude_assistant: ['#exclude_assistant', 'exclude_assistant', true],
claude_use_sysprompt: ['#claude_use_sysprompt', 'claude_use_sysprompt', true], claude_use_sysprompt: ['#claude_use_sysprompt', 'claude_use_sysprompt', true],
claude_exclude_prefixes: ['#claude_exclude_prefixes', 'claude_exclude_prefixes', true],
use_alt_scale: ['#use_alt_scale', 'use_alt_scale', true], use_alt_scale: ['#use_alt_scale', 'use_alt_scale', true],
squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true], squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true],
image_inlining: ['#openai_image_inlining', 'image_inlining', true], image_inlining: ['#openai_image_inlining', 'image_inlining', true],
@@ -3330,8 +3316,15 @@ async function onModelChange() {
let value = String($(this).val() || ''); let value = String($(this).val() || '');
if ($(this).is('#model_claude_select')) { if ($(this).is('#model_claude_select')) {
if (value.includes('-v')) {
value = value.replace('-v', '-');
} else if (value === '' || value === 'claude-2') {
value = default_settings.claude_model;
}
console.log('Claude model changed to', value); console.log('Claude model changed to', value);
oai_settings.claude_model = value; oai_settings.claude_model = value;
$('#model_claude_select').val(oai_settings.claude_model);
} }
if ($(this).is('#model_windowai_select')) { if ($(this).is('#model_windowai_select')) {
@@ -3439,7 +3432,7 @@ async function onModelChange() {
if (oai_settings.max_context_unlocked) { if (oai_settings.max_context_unlocked) {
$('#openai_max_context').attr('max', max_200k); $('#openai_max_context').attr('max', max_200k);
} }
else if (value == 'claude-2.1' || value == 'claude-2') { else if (value == 'claude-2.1' || value.startsWith('claude-3')) {
$('#openai_max_context').attr('max', max_200k); $('#openai_max_context').attr('max', max_200k);
} }
else if (value.endsWith('100k') || value.startsWith('claude-2') || value === 'claude-instant-1.2') { else if (value.endsWith('100k') || value.startsWith('claude-2') || value === 'claude-instant-1.2') {
@@ -3735,7 +3728,6 @@ function toggleChatCompletionForms() {
}); });
if (chat_completion_sources.CLAUDE == oai_settings.chat_completion_source) { if (chat_completion_sources.CLAUDE == oai_settings.chat_completion_source) {
$('#claude_assistant_prefill_block').toggle(!oai_settings.exclude_assistant);
$('#claude_human_sysprompt_message_block').toggle(oai_settings.claude_use_sysprompt); $('#claude_human_sysprompt_message_block').toggle(oai_settings.claude_use_sysprompt);
} }
} }
@@ -3829,6 +3821,7 @@ export function isImageInliningSupported() {
const gpt4v = 'gpt-4-vision'; const gpt4v = 'gpt-4-vision';
const geminiProV = 'gemini-pro-vision'; const geminiProV = 'gemini-pro-vision';
const claude = 'claude-3';
const llava = 'llava'; const llava = 'llava';
if (!oai_settings.image_inlining) { if (!oai_settings.image_inlining) {
@@ -3840,6 +3833,8 @@ export function isImageInliningSupported() {
return oai_settings.openai_model.includes(gpt4v); return oai_settings.openai_model.includes(gpt4v);
case chat_completion_sources.MAKERSUITE: case chat_completion_sources.MAKERSUITE:
return oai_settings.google_model.includes(geminiProV); return oai_settings.google_model.includes(geminiProV);
case chat_completion_sources.CLAUDE:
return oai_settings.claude_model.includes(claude);
case chat_completion_sources.OPENROUTER: case chat_completion_sources.OPENROUTER:
return !oai_settings.openrouter_force_instruct && (oai_settings.openrouter_model.includes(gpt4v) || oai_settings.openrouter_model.includes(llava)); return !oai_settings.openrouter_force_instruct && (oai_settings.openrouter_model.includes(gpt4v) || oai_settings.openrouter_model.includes(llava));
case chat_completion_sources.CUSTOM: case chat_completion_sources.CUSTOM:
@@ -4075,23 +4070,12 @@ $(document).ready(async function () {
saveSettingsDebounced(); saveSettingsDebounced();
}); });
$('#exclude_assistant').on('change', function () {
oai_settings.exclude_assistant = !!$('#exclude_assistant').prop('checked');
$('#claude_assistant_prefill_block').toggle(!oai_settings.exclude_assistant);
saveSettingsDebounced();
});
$('#claude_use_sysprompt').on('change', function () { $('#claude_use_sysprompt').on('change', function () {
oai_settings.claude_use_sysprompt = !!$('#claude_use_sysprompt').prop('checked'); oai_settings.claude_use_sysprompt = !!$('#claude_use_sysprompt').prop('checked');
$('#claude_human_sysprompt_message_block').toggle(oai_settings.claude_use_sysprompt); $('#claude_human_sysprompt_message_block').toggle(oai_settings.claude_use_sysprompt);
saveSettingsDebounced(); saveSettingsDebounced();
}); });
$('#claude_exclude_prefixes').on('change', function () {
oai_settings.claude_exclude_prefixes = !!$('#claude_exclude_prefixes').prop('checked');
saveSettingsDebounced();
});
$('#names_in_completion').on('change', function () { $('#names_in_completion').on('change', function () {
oai_settings.names_in_completion = !!$('#names_in_completion').prop('checked'); oai_settings.names_in_completion = !!$('#names_in_completion').prop('checked');
saveSettingsDebounced(); saveSettingsDebounced();

3
server.js
View File

@@ -505,6 +505,9 @@ app.use('/api/openai', require('./src/endpoints/openai').router);
//Google API //Google API
app.use('/api/google', require('./src/endpoints/google').router); app.use('/api/google', require('./src/endpoints/google').router);
//Anthropic API
app.use('/api/anthropic', require('./src/endpoints/anthropic').router);
// Tokenizers // Tokenizers
app.use('/api/tokenizers', require('./src/endpoints/tokenizers').router); app.use('/api/tokenizers', require('./src/endpoints/tokenizers').router);

67
src/endpoints/anthropic.js Normal file
View File

@@ -0,0 +1,67 @@
const { readSecret, SECRET_KEYS } = require('./secrets');
const fetch = require('node-fetch').default;
const express = require('express');
const { jsonParser } = require('../express-common');
const router = express.Router();
router.post('/caption-image', jsonParser, async (request, response) => {
try {
const mimeType = request.body.image.split(';')[0].split(':')[1];
const base64Data = request.body.image.split(',')[1];
const baseUrl = request.body.reverse_proxy ? request.body.reverse_proxy : 'https://api.anthropic.com/v1';
const url = `${baseUrl}/messages`;
const body = {
model: request.body.model,
messages: [
{
'role': 'user', 'content': [
{
'type': 'image',
'source': {
'type': 'base64',
'media_type': mimeType,
'data': base64Data,
},
},
{ 'type': 'text', 'text': request.body.prompt },
],
},
],
max_tokens: 800,
};
console.log('Multimodal captioning request', body);
const result = await fetch(url, {
body: JSON.stringify(body),
method: 'POST',
headers: {
'Content-Type': 'application/json',
'anthropic-version': '2023-06-01',
'x-api-key': request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.CLAUDE),
},
timeout: 0,
});
if (!result.ok) {
console.log(`Claude API returned error: ${result.status} ${result.statusText}`);
return response.status(result.status).send({ error: true });
}
const generateResponseJson = await result.json();
const caption = generateResponseJson.content[0].text;
console.log('Claude response:', generateResponseJson);
if (!caption) {
return response.status(500).send('No caption found');
}
return response.json({ caption });
} catch (error) {
console.error(error);
response.status(500).send('Internal server error');
}
});
module.exports = { router };

59
src/endpoints/backends/chat-completions.js
View File

@@ -5,7 +5,7 @@ const { Readable } = require('stream');
const { jsonParser } = require('../../express-common'); const { jsonParser } = require('../../express-common');
const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY, OPENROUTER_HEADERS } = require('../../constants'); const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY, OPENROUTER_HEADERS } = require('../../constants');
const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml, color } = require('../../util'); const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml, color } = require('../../util');
const { convertClaudePrompt, convertGooglePrompt, convertTextCompletionPrompt } = require('../prompt-converters'); const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt } = require('../prompt-converters');
const { readSecret, SECRET_KEYS } = require('../secrets'); const { readSecret, SECRET_KEYS } = require('../secrets');
const { getTokenizerModel, getSentencepiceTokenizer, getTiktokenTokenizer, sentencepieceTokenizers, TEXT_COMPLETION_MODELS } = require('../tokenizers'); const { getTokenizerModel, getSentencepiceTokenizer, getTiktokenTokenizer, sentencepieceTokenizers, TEXT_COMPLETION_MODELS } = require('../tokenizers');
@@ -34,45 +34,8 @@ async function sendClaudeRequest(request, response) {
request.socket.on('close', function () { request.socket.on('close', function () {
controller.abort(); controller.abort();
}); });
let use_system_prompt = (request.body.model.startsWith('claude-2') || request.body.model.startsWith('claude-3')) && request.body.claude_use_sysprompt;
const isSysPromptSupported = request.body.model === 'claude-2' || request.body.model === 'claude-2.1'; let converted_prompt = convertClaudeMessages(request.body.messages, request.body.assistant_prefill, use_system_prompt, request.body.human_sysprompt_message);
const requestPrompt = convertClaudePrompt(request.body.messages, !request.body.exclude_assistant, request.body.assistant_prefill, isSysPromptSupported, request.body.claude_use_sysprompt, request.body.human_sysprompt_message, request.body.claude_exclude_prefixes);
// Check Claude messages sequence and prefixes presence.
let sequenceError = [];
const sequence = requestPrompt.split('\n').filter(x => x.startsWith('Human:') || x.startsWith('Assistant:'));
const humanFound = sequence.some(line => line.startsWith('Human:'));
const assistantFound = sequence.some(line => line.startsWith('Assistant:'));
let humanErrorCount = 0;
let assistantErrorCount = 0;
for (let i = 0; i < sequence.length - 1; i++) {
if (sequence[i].startsWith(sequence[i + 1].split(':')[0])) {
if (sequence[i].startsWith('Human:')) {
humanErrorCount++;
} else if (sequence[i].startsWith('Assistant:')) {
assistantErrorCount++;
}
}
}
if (!humanFound) {
sequenceError.push(`${divider}\nWarning: No 'Human:' prefix found in the prompt.\n${divider}`);
}
if (!assistantFound) {
sequenceError.push(`${divider}\nWarning: No 'Assistant: ' prefix found in the prompt.\n${divider}`);
}
if (sequence[0] && !sequence[0].startsWith('Human:')) {
sequenceError.push(`${divider}\nWarning: The messages sequence should start with 'Human:' prefix.\nMake sure you have '\\n\\nHuman:' prefix at the very beggining of the prompt, or after the system prompt.\n${divider}`);
}
if (humanErrorCount > 0 || assistantErrorCount > 0) {
sequenceError.push(`${divider}\nWarning: Detected incorrect Prefix sequence(s).`);
sequenceError.push(`Incorrect "Human:" prefix(es): ${humanErrorCount}.\nIncorrect "Assistant: " prefix(es): ${assistantErrorCount}.`);
sequenceError.push('Check the prompt above and fix it in the SillyTavern.');
sequenceError.push('\nThe correct sequence in the console should look like this:\n(System prompt msg) <-(for the sysprompt format only, else have \\n\\n above the first human\'s message.)');
sequenceError.push(`\\n + <-----(Each message beginning with the "Assistant:/Human:" prefix must have \\n\\n before it.)\n\\n +\nHuman: \\n +\n\\n +\nAssistant: \\n +\n...\n\\n +\nHuman: \\n +\n\\n +\nAssistant: \n${divider}`);
}
// Add custom stop sequences // Add custom stop sequences
const stopSequences = ['\n\nHuman:', '\n\nSystem:', '\n\nAssistant:']; const stopSequences = ['\n\nHuman:', '\n\nSystem:', '\n\nAssistant:'];
if (Array.isArray(request.body.stop)) { if (Array.isArray(request.body.stop)) {
@@ -80,23 +43,21 @@ async function sendClaudeRequest(request, response) {
} }
const requestBody = { const requestBody = {
prompt: requestPrompt, messages: converted_prompt.messages,
model: request.body.model, model: request.body.model,
max_tokens_to_sample: request.body.max_tokens, max_tokens: request.body.max_tokens,
stop_sequences: stopSequences, stop_sequences: stopSequences,
temperature: request.body.temperature, temperature: request.body.temperature,
top_p: request.body.top_p, top_p: request.body.top_p,
top_k: request.body.top_k, top_k: request.body.top_k,
stream: request.body.stream, stream: request.body.stream,
}; };
if (use_system_prompt) {
requestBody.system = converted_prompt.systemPrompt;
}
console.log('Claude request:', requestBody); console.log('Claude request:', requestBody);
sequenceError.forEach(sequenceError => { const generateResponse = await fetch(apiUrl + '/messages', {
console.log(color.red(sequenceError));
});
const generateResponse = await fetch(apiUrl + '/complete', {
method: 'POST', method: 'POST',
signal: controller.signal, signal: controller.signal,
body: JSON.stringify(requestBody), body: JSON.stringify(requestBody),
@@ -118,7 +79,7 @@ async function sendClaudeRequest(request, response) {
} }
const generateResponseJson = await generateResponse.json(); const generateResponseJson = await generateResponse.json();
const responseText = generateResponseJson.completion; const responseText = generateResponseJson.content[0].text;
console.log('Claude response:', generateResponseJson); console.log('Claude response:', generateResponseJson);
// Wrap it back to OAI format // Wrap it back to OAI format

97
src/endpoints/prompt-converters.js
View File

@@ -71,6 +71,102 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill,
return requestPrompt; return requestPrompt;
} }
/**
* Convert ChatML objects into working with Anthropic's new Messaging API.
* @param {object[]} messages Array of messages
* @param {string} prefillString User determined prefill string
* @param {boolean} useSysPrompt See if we want to use a system prompt
* @param {string} humanMsgFix Add Human message between system prompt and assistant.
*/
function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFix) {
let systemPrompt = '';
if (useSysPrompt) {
// Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
let i;
for (i = 0; i < messages.length; i++) {
if (messages[i].role !== 'system') {
break;
}
systemPrompt += `${messages[i].content}\n\n`;
}
messages.splice(0, i);
// Check if the first message in the array is of type user, if not, interject with humanMsgFix or a blank message.
if (messages.length > 0 && messages[0].role !== 'user') {
messages.unshift({
role: 'user',
content: humanMsgFix || '[Start a new chat]',
});
}
}
// Now replace all further messages that have the role 'system' with the role 'user'. (or all if we're not using one)
messages.forEach((message) => {
if (message.role === 'system') {
message.role = 'user';
}
});
// Since the messaging endpoint only supports user assistant roles in turns, we have to merge messages with the same role if they follow eachother
// Also handle multi-modality, holy slop.
let mergedMessages = [];
messages.forEach((message) => {
const imageEntry = message.content[1]?.image_url;
const imageData = imageEntry?.url;
const mimeType = imageData?.split(';')[0].split(':')[1];
const base64Data = imageData?.split(',')[1];
if (mergedMessages.length > 0 && mergedMessages[mergedMessages.length - 1].role === message.role) {
if(Array.isArray(message.content)) {
if(Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content[0].text;
} else {
mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content[0].text;
}
} else {
if(Array.isArray(mergedMessages[mergedMessages.length - 1].content)) {
mergedMessages[mergedMessages.length - 1].content[0].text += '\n\n' + message.content;
} else {
mergedMessages[mergedMessages.length - 1].content += '\n\n' + message.content;
}
}
} else {
mergedMessages.push(message);
}
if (imageData) {
mergedMessages[mergedMessages.length - 1].content = [
{ type: 'text', text: mergedMessages[mergedMessages.length - 1].content[0]?.text || mergedMessages[mergedMessages.length - 1].content },
{
type: 'image', source: {
type: 'base64',
media_type: mimeType,
data: base64Data,
},
},
];
}
});
// Take care of name properties since claude messages don't support them
mergedMessages.forEach((message) => {
if (message.name) {
message.content = `${message.name}: ${message.content}`;
delete message.name;
}
});
// Shouldn't be conditional anymore, messages api expects the last role to be user unless we're explicitly prefilling
if (prefillString) {
mergedMessages.push({
role: 'assistant',
content: prefillString,
});
}
return { messages: mergedMessages, systemPrompt: systemPrompt.trim() };
}
/** /**
* Convert a prompt from the ChatML objects to the format used by Google MakerSuite models. * Convert a prompt from the ChatML objects to the format used by Google MakerSuite models.
* @param {object[]} messages Array of messages * @param {object[]} messages Array of messages
@@ -160,6 +256,7 @@ function convertTextCompletionPrompt(messages) {
module.exports = { module.exports = {
convertClaudePrompt, convertClaudePrompt,
convertClaudeMessages,
convertGooglePrompt, convertGooglePrompt,
convertTextCompletionPrompt, convertTextCompletionPrompt,
}; };

1
src/endpoints/tokenizers.js
View File

@@ -249,6 +249,7 @@ async function loadClaudeTokenizer(modelPath) {
} }
function countClaudeTokens(tokenizer, messages) { function countClaudeTokens(tokenizer, messages) {
// Should be fine if we use the old conversion method instead of the messages API one i think?
const convertedPrompt = convertClaudePrompt(messages, false, false, false); const convertedPrompt = convertClaudePrompt(messages, false, false, false);
// Fallback to strlen estimation // Fallback to strlen estimation