rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-02-23 15:37:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add prompt format override for OpenRouter

Browse Source
This commit is contained in:
Cohee 2023-11-03 00:34:22 +02:00
parent 814c62cc21
commit f10833a516
3 changed files with 121 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
public/index.html
View File

@ -1941,12 +1941,24 @@
<input id="openrouter_use_fallback" type="checkbox" />
<span data-i18n="Allow fallback routes">Allow fallback routes</span>
</label>
<div class="toggle-description justifyLeft">
<div class="toggle-description justifyLeft wide100p">
<span data-i18n="Allow fallback routes Description">
Automatically chooses an alternative model if the chosen model can't serve your request.
</span>
</div>
</div>
<div class="marginTopBot5">
<label for="openrouter_force_instruct" class="checkbox_label">
<input id="openrouter_force_instruct" type="checkbox" />
<span data-i18n="Force Instruct Mode formatting">Force Instruct Mode formatting</span>
</label>
<div class="toggle-description justifyLeft wide100p">
<span data-i18n="Force Instruct Mode formatting Description">
If both Instruct Mode and this are enabled, the prompt will be formatted by SillyTavern using the current
advanced formatting settings (except instruct System Prompt). If disabled, the prompt will be formatted by OpenRouter.
</span>
</div>
</div>
<h4 data-i18n="OpenRouter API Key">OpenRouter API Key</h4>
<div>
<small data-i18n="Click Authorize below or get the key from">

103
public/scripts/openai.js
View File

@ -55,6 +55,7 @@ import {
stringFormat,
} from "./utils.js";
import { countTokensOpenAI } from "./tokenizers.js";
import { formatInstructModeChat, formatInstructModeExamples, formatInstructModePrompt, formatInstructModeSystemPrompt } from "./instruct-mode.js";
export {
is_get_status_openai,
@ -205,6 +206,7 @@ const default_settings = {
windowai_model: '',
openrouter_model: openrouter_website_model,
openrouter_use_fallback: false,
openrouter_force_instruct: false,
jailbreak_system: false,
reverse_proxy: '',
legacy_streaming: false,
@ -250,6 +252,7 @@ const oai_settings = {
windowai_model: '',
openrouter_model: openrouter_website_model,
openrouter_use_fallback: false,
openrouter_force_instruct: false,
jailbreak_system: false,
reverse_proxy: '',
legacy_streaming: false,
@ -291,6 +294,90 @@ function setOpenAIOnlineStatus(value) {
is_get_status_openai = value;
}
function convertChatCompletionToInstruct(messages, type) {
messages = messages.filter(x => x.content !== oai_settings.new_chat_prompt && x.content !== oai_settings.new_example_chat_prompt);
let chatMessagesText = '';
let systemPromptText = '';
let examplesText = '';
function getPrefix(message) {
let prefix;
if (message.role === 'user' || message.name === 'example_user') {
if (selected_group) {
prefix = ''
} else if (message.name === 'example_user') {
prefix = name1;
} else {
prefix = message.name ?? name1;
}
}
if (message.role === 'assistant' || message.name === 'example_assistant') {
if (selected_group) {
prefix = ''
}
else if (message.name === 'example_assistant') {
prefix = name2;
} else {
prefix = message.name ?? name2;
}
}
return prefix;
}
function toString(message) {
if (message.role === 'system' && !message.name) {
return message.content;
}
const prefix = getPrefix(message);
return prefix ? `${prefix}: ${message.content}` : message.content;
}
const firstChatMessage = messages.findIndex(message => message.role === 'assistant' || message.role === 'user');
const systemPromptMessages = messages.slice(0, firstChatMessage).filter(message => message.role === 'system' && !message.name);
if (systemPromptMessages.length) {
systemPromptText = systemPromptMessages.map(message => message.content).join('\n');
systemPromptText = formatInstructModeSystemPrompt(systemPromptText);
}
const exampleMessages = messages.filter(x => x.role === 'system' && (x.name === 'example_user' || x.name === 'example_assistant'));
if (exampleMessages.length) {
examplesText = power_user.context.example_separator + '\n';
examplesText += exampleMessages.map(toString).join('\n');
examplesText = formatInstructModeExamples(examplesText, name1, name2);
}
const chatMessages = messages.slice(firstChatMessage);
if (chatMessages.length) {
chatMessagesText = power_user.context.chat_start + '\n';
for (const message of chatMessages) {
const name = getPrefix(message);
const isUser = message.role === 'user';
const isNarrator = message.role === 'system';
chatMessagesText += formatInstructModeChat(name, message.content, isUser, isNarrator, '', name1, name2, false);
}
}
const isImpersonate = type === 'impersonate';
const promptName = isImpersonate ? name1 : name2;
const promptLine = formatInstructModePrompt(promptName, isImpersonate, '', name1, name2).trimStart();
const prompt = [systemPromptText, examplesText, chatMessagesText, promptLine]
.filter(x => x)
.map(x => x.endsWith('\n') ? x : `${x}\n`)
.join('');
return prompt;
}
function setOpenAIMessages(chat) {
let j = 0;
// clean openai msgs
@ -1222,11 +1309,16 @@ async function sendOpenAIRequest(type, openai_msgs_tosend, signal) {
const isScale = oai_settings.chat_completion_source == chat_completion_sources.SCALE;
const isAI21 = oai_settings.chat_completion_source == chat_completion_sources.AI21;
const isPalm = oai_settings.chat_completion_source == chat_completion_sources.PALM;
const isTextCompletion = oai_settings.chat_completion_source == chat_completion_sources.OPENAI && textCompletionModels.includes(oai_settings.openai_model);
const isOAI = oai_settings.chat_completion_source == chat_completion_sources.OPENAI;
const isTextCompletion = (isOAI && textCompletionModels.includes(oai_settings.openai_model)) || (isOpenRouter && oai_settings.openrouter_force_instruct && power_user.instruct.enabled);
const isQuiet = type === 'quiet';
const isImpersonate = type === 'impersonate';
const stream = oai_settings.stream_openai && !isQuiet && !isScale && !isAI21 && !isPalm;
if (isTextCompletion && isOpenRouter) {
openai_msgs_tosend = convertChatCompletionToInstruct(openai_msgs_tosend, type);
}
if (isAI21 || isPalm) {
const joinedMsgs = openai_msgs_tosend.reduce((acc, obj) => {
const prefix = prefixMap[obj.role];
@ -2034,6 +2126,7 @@ function loadOpenAISettings(data, settings) {
oai_settings.windowai_model = settings.windowai_model ?? default_settings.windowai_model;
oai_settings.openrouter_model = settings.openrouter_model ?? default_settings.openrouter_model;
oai_settings.openrouter_use_fallback = settings.openrouter_use_fallback ?? default_settings.openrouter_use_fallback;
oai_settings.openrouter_force_instruct = settings.openrouter_force_instruct ?? default_settings.openrouter_force_instruct;
oai_settings.ai21_model = settings.ai21_model ?? default_settings.ai21_model;
oai_settings.chat_completion_source = settings.chat_completion_source ?? default_settings.chat_completion_source;
oai_settings.api_url_scale = settings.api_url_scale ?? default_settings.api_url_scale;
@ -2085,6 +2178,7 @@ function loadOpenAISettings(data, settings) {
$('#exclude_assistant').prop('checked', oai_settings.exclude_assistant);
$('#scale-alt').prop('checked', oai_settings.use_alt_scale);
$('#openrouter_use_fallback').prop('checked', oai_settings.openrouter_use_fallback);
$('#openrouter_force_instruct').prop('checked', oai_settings.openrouter_force_instruct);
$('#squash_system_messages').prop('checked', oai_settings.squash_system_messages);
if (settings.impersonation_prompt !== undefined) oai_settings.impersonation_prompt = settings.impersonation_prompt;
@ -2251,6 +2345,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
windowai_model: settings.windowai_model,
openrouter_model: settings.openrouter_model,
openrouter_use_fallback: settings.openrouter_use_fallback,
openrouter_force_instruct: settings.openrouter_force_instruct,
ai21_model: settings.ai21_model,
temperature: settings.temp_openai,
frequency_penalty: settings.freq_pen_openai,
@ -2612,6 +2707,7 @@ function onSettingsPresetChange() {
windowai_model: ['#model_windowai_select', 'windowai_model', false],
openrouter_model: ['#model_openrouter_select', 'openrouter_model', false],
openrouter_use_fallback: ['#openrouter_use_fallback', 'openrouter_use_fallback', true],
openrouter_force_instruct: ['#openrouter_force_instruct', 'openrouter_force_instruct', true],
ai21_model: ['#model_ai21_select', 'ai21_model', false],
openai_max_context: ['#openai_max_context', 'openai_max_context', false],
openai_max_tokens: ['#openai_max_tokens', 'openai_max_tokens', false],
@ -3344,6 +3440,11 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#openrouter_force_instruct').on('input', function () {
oai_settings.openrouter_force_instruct = !!$(this).prop('checked');
saveSettingsDebounced();
});
$('#squash_system_messages').on('input', function () {
oai_settings.squash_system_messages = !!$(this).prop('checked');
saveSettingsDebounced();

8
server.js
View File

@ -2849,6 +2849,10 @@ app.post("/openai_bias", jsonParser, async function (request, response) {
});
function convertChatMLPrompt(messages) {
if (typeof messages === 'string') {
return messages;
}
const messageStrings = [];
messages.forEach(m => {
if (m.role === 'system' && m.name === undefined) {
@ -3180,9 +3184,9 @@ app.post("/generate_openai", jsonParser, function (request, response_generate_op
bodyParams['stop'] = request.body.stop;
}
const isTextCompletion = Boolean(request.body.model && TEXT_COMPLETION_MODELS.includes(request.body.model));
const isTextCompletion = Boolean(request.body.model && TEXT_COMPLETION_MODELS.includes(request.body.model)) || typeof request.body.messages === 'string';
const textPrompt = isTextCompletion ? convertChatMLPrompt(request.body.messages) : '';
const endpointUrl = isTextCompletion ? `${api_url}/completions` : `${api_url}/chat/completions`;
const endpointUrl = isTextCompletion && !request.body.use_openrouter ? `${api_url}/completions` : `${api_url}/chat/completions`;
const controller = new AbortController();
request.socket.removeAllListeners('close');