diff --git a/public/script.js b/public/script.js
index d040f9192..5c46ca183 100644
--- a/public/script.js
+++ b/public/script.js
@@ -22,7 +22,7 @@ import {
parseTabbyLogprobs,
} from './scripts/textgen-settings.js';
-const { MANCER, TOGETHERAI, OOBA, APHRODITE, OLLAMA, INFERMATICAI } = textgen_types;
+const { MANCER, TOGETHERAI, OOBA, APHRODITE, OLLAMA, INFERMATICAI, OPENROUTER } = textgen_types;
import {
world_info,
@@ -196,7 +196,7 @@ import { createPersona, initPersonas, selectCurrentPersona, setPersonaDescriptio
import { getBackgrounds, initBackgrounds, loadBackgroundSettings, background_settings } from './scripts/backgrounds.js';
import { hideLoader, showLoader } from './scripts/loader.js';
import { BulkEditOverlay, CharacterContextMenu } from './scripts/BulkEditOverlay.js';
-import { loadMancerModels, loadOllamaModels, loadTogetherAIModels, loadInfermaticAIModels } from './scripts/textgen-models.js';
+import { loadMancerModels, loadOllamaModels, loadTogetherAIModels, loadInfermaticAIModels, loadOpenRouterModels } from './scripts/textgen-models.js';
import { appendFileContent, hasPendingFileAttachment, populateFileAttachment, decodeStyleTags, encodeStyleTags } from './scripts/chats.js';
import { initPresetManager } from './scripts/preset-manager.js';
import { evaluateMacros } from './scripts/macros.js';
@@ -1060,6 +1060,9 @@ async function getStatusTextgen() {
} else if (textgen_settings.type === INFERMATICAI) {
loadInfermaticAIModels(data?.data);
online_status = textgen_settings.infermaticai_model;
+ } else if (textgen_settings.type === OPENROUTER) {
+ loadOpenRouterModels(data?.data);
+ online_status = textgen_settings.openrouter_model;
} else {
online_status = data?.result;
}
@@ -5964,7 +5967,11 @@ function updateMessage(div) {
text = text.trim();
}
- const bias = extractMessageBias(text);
+ const bias = substituteParams(extractMessageBias(text));
+ text = substituteParams(text);
+ if (bias) {
+ text = removeMacros(text);
+ }
mes['mes'] = text;
if (mes['swipe_id'] !== undefined) {
mes['swipes'][mes['swipe_id']] = text;
@@ -6007,7 +6014,7 @@ function openMessageDelete(fromSlashCommand) {
}
function messageEditAuto(div) {
- const { mesBlock, text, mes } = updateMessage(div);
+ const { mesBlock, text, mes, bias } = updateMessage(div);
mesBlock.find('.mes_text').val('');
mesBlock.find('.mes_text').val(messageFormatting(
@@ -6017,6 +6024,8 @@ function messageEditAuto(div) {
mes.is_user,
this_edit_mes_id,
));
+ mesBlock.find('.mes_bias').empty();
+ mesBlock.find('.mes_bias').append(messageFormatting(bias, '', false, false, -1));
saveChatDebounced();
}
@@ -7706,6 +7715,11 @@ const CONNECT_API_MAP = {
button: '#api_button_textgenerationwebui',
type: textgen_types.INFERMATICAI,
},
+ 'openrouter-text': {
+ selected: 'textgenerationwebui',
+ button: '#api_button_textgenerationwebui',
+ type: textgen_types.OPENROUTER,
+ },
};
async function selectContextCallback(_, name) {
@@ -8649,6 +8663,11 @@ jQuery(async function () {
await writeSecret(SECRET_KEYS.INFERMATICAI, infermaticAIKey);
}
+ const openRouterKey = String($('#api_key_openrouter-tg').val()).trim();
+ if (openRouterKey.length) {
+ await writeSecret(SECRET_KEYS.OPENROUTER, openRouterKey);
+ }
+
validateTextGenUrl();
startStatusLoading();
main_api = 'textgenerationwebui';
diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js
index 8198de0e6..73bd5b7bc 100644
--- a/public/scripts/RossAscends-mods.js
+++ b/public/scripts/RossAscends-mods.js
@@ -391,7 +391,8 @@ function RA_autoconnect(PrevApi) {
case 'textgenerationwebui':
if ((textgen_settings.type === textgen_types.MANCER && secret_state[SECRET_KEYS.MANCER])
|| (textgen_settings.type === textgen_types.TOGETHERAI && secret_state[SECRET_KEYS.TOGETHERAI])
- || (textgen_settings.type === textgen_types.INFERMATICAI && secret_state[SECRET_KEYS.INFERMATICAI])
+ || (textgen_settings.type === textgen_types.INFERMATICAI && secret_state[SECRET_KEYS.INFERMATICAI]
+ || (textgen_settings.type === textgen_types.OPENROUTER && secret_state[SECRET_KEYS.OPENROUTER]))
) {
$('#api_button_textgenerationwebui').trigger('click');
}
diff --git a/public/scripts/openai.js b/public/scripts/openai.js
index b7a4eba56..6615c704e 100644
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -216,7 +216,7 @@ const default_settings = {
claude_model: 'claude-instant-v1',
google_model: 'gemini-pro',
ai21_model: 'j2-ultra',
- mistralai_model: 'mistral-medium',
+ mistralai_model: 'mistral-medium-latest',
custom_model: '',
custom_url: '',
custom_include_body: '',
@@ -285,7 +285,7 @@ const oai_settings = {
claude_model: 'claude-instant-v1',
google_model: 'gemini-pro',
ai21_model: 'j2-ultra',
- mistralai_model: 'mistral-medium',
+ mistralai_model: 'mistral-medium-latest',
custom_model: '',
custom_url: '',
custom_include_body: '',
@@ -3365,8 +3365,16 @@ async function onModelChange() {
}
if ($(this).is('#model_mistralai_select')) {
+ // Upgrade old mistral models to new naming scheme
+ // would have done this in loadOpenAISettings, but it wasn't updating on preset change?
+ if (value === 'mistral-medium' || value === 'mistral-small' || value === 'mistral-tiny') {
+ value = value + '-latest';
+ } else if (value === '') {
+ value = default_settings.mistralai_model;
+ }
console.log('MistralAI model changed to', value);
oai_settings.mistralai_model = value;
+ $('#model_mistralai_select').val(oai_settings.mistralai_model);
}
if (value && $(this).is('#model_custom_select')) {
diff --git a/public/scripts/preset-manager.js b/public/scripts/preset-manager.js
index fd6ae3b9b..ab9571032 100644
--- a/public/scripts/preset-manager.js
+++ b/public/scripts/preset-manager.js
@@ -313,6 +313,8 @@ class PresetManager {
'type',
'custom_model',
'bypass_status_check',
+ 'infermaticai_model',
+ 'openrouter_model',
];
const settings = Object.assign({}, getSettingsByApiId(this.apiId));
diff --git a/public/scripts/secrets.js b/public/scripts/secrets.js
index f1516e999..3ea3c5289 100644
--- a/public/scripts/secrets.js
+++ b/public/scripts/secrets.js
@@ -27,7 +27,7 @@ const INPUT_MAP = {
[SECRET_KEYS.OPENAI]: '#api_key_openai',
[SECRET_KEYS.NOVEL]: '#api_key_novel',
[SECRET_KEYS.CLAUDE]: '#api_key_claude',
- [SECRET_KEYS.OPENROUTER]: '#api_key_openrouter',
+ [SECRET_KEYS.OPENROUTER]: '.api_key_openrouter',
[SECRET_KEYS.SCALE]: '#api_key_scale',
[SECRET_KEYS.AI21]: '#api_key_ai21',
[SECRET_KEYS.SCALE_COOKIE]: '#scale_cookie',
@@ -199,5 +199,5 @@ jQuery(async () => {
const warningElement = $(`[data-for="${id}"]`);
warningElement.toggle(value.length > 0);
});
- $('#openrouter_authorize').on('click', authorizeOpenRouter);
+ $('.openrouter_authorize').on('click', authorizeOpenRouter);
});
diff --git a/public/scripts/textgen-models.js b/public/scripts/textgen-models.js
index d7612df5b..a30775af2 100644
--- a/public/scripts/textgen-models.js
+++ b/public/scripts/textgen-models.js
@@ -1,10 +1,12 @@
import { callPopup, getRequestHeaders, setGenerationParamsFromPreset } from '../script.js';
import { isMobile } from './RossAscends-mods.js';
import { textgenerationwebui_settings as textgen_settings, textgen_types } from './textgen-settings.js';
+import { tokenizers } from './tokenizers.js';
let mancerModels = [];
let togetherModels = [];
let infermaticAIModels = [];
+export let openRouterModels = [];
export async function loadOllamaModels(data) {
if (!Array.isArray(data)) {
@@ -101,6 +103,28 @@ export async function loadMancerModels(data) {
}
}
+export async function loadOpenRouterModels(data) {
+ if (!Array.isArray(data)) {
+ console.error('Invalid OpenRouter models data', data);
+ return;
+ }
+
+ openRouterModels = data;
+
+ if (!data.find(x => x.id === textgen_settings.openrouter_model)) {
+ textgen_settings.openrouter_model = data[0]?.id || '';
+ }
+
+ $('#openrouter_model').empty();
+ for (const model of data) {
+ const option = document.createElement('option');
+ option.value = model.id;
+ option.text = model.id;
+ option.selected = model.id === textgen_settings.openrouter_model;
+ $('#openrouter_model').append(option);
+ }
+}
+
function onMancerModelSelect() {
const modelId = String($('#mancer_model').val());
textgen_settings.mancer_model = modelId;
@@ -132,6 +156,14 @@ function onOllamaModelSelect() {
$('#api_button_textgenerationwebui').trigger('click');
}
+function onOpenRouterModelSelect() {
+ const modelId = String($('#openrouter_model').val());
+ textgen_settings.openrouter_model = modelId;
+ $('#api_button_textgenerationwebui').trigger('click');
+ const model = openRouterModels.find(x => x.id === modelId);
+ setGenerationParamsFromPreset({ max_length: model.context_length });
+}
+
function getMancerModelTemplate(option) {
const model = mancerModels.find(x => x.id === option?.element?.value);
@@ -179,6 +211,25 @@ function getInfermaticAIModelTemplate(option) {
`));
}
+function getOpenRouterModelTemplate(option) {
+ const model = openRouterModels.find(x => x.id === option?.element?.value);
+
+ if (!option.id || !model) {
+ return option.text;
+ }
+
+ let tokens_dollar = Number(1 / (1000 * model.pricing?.prompt));
+ let tokens_rounded = (Math.round(tokens_dollar * 1000) / 1000).toFixed(0);
+
+ const price = 0 === Number(model.pricing?.prompt) ? 'Free' : `${tokens_rounded}k t/$ `;
+
+ return $((`
+
+
${DOMPurify.sanitize(model.name)} | ${model.context_length} ctx | ${price}
+
+ `));
+}
+
async function downloadOllamaModel() {
try {
const serverUrl = textgen_settings.server_urls[textgen_types.OLLAMA];
@@ -220,11 +271,25 @@ async function downloadOllamaModel() {
}
}
+export function getCurrentOpenRouterModelTokenizer() {
+ const modelId = textgen_settings.openrouter_model;
+ const model = openRouterModels.find(x => x.id === modelId);
+ switch (model?.architecture?.tokenizer) {
+ case 'Llama2':
+ return tokenizers.LLAMA;
+ case 'Mistral':
+ return tokenizers.MISTRAL;
+ default:
+ return tokenizers.OPENAI;
+ }
+}
+
jQuery(function () {
$('#mancer_model').on('change', onMancerModelSelect);
$('#model_togetherai_select').on('change', onTogetherModelSelect);
$('#model_infermaticai_select').on('change', onInfermaticAIModelSelect);
$('#ollama_model').on('change', onOllamaModelSelect);
+ $('#openrouter_model').on('change', onOpenRouterModelSelect);
$('#ollama_download_model').on('click', downloadOllamaModel);
if (!isMobile()) {
@@ -255,5 +320,12 @@ jQuery(function () {
width: '100%',
templateResult: getInfermaticAIModelTemplate,
});
+ $('#openrouter_model').select2({
+ placeholder: 'Select a model',
+ searchInputPlaceholder: 'Search models...',
+ searchInputCssClass: 'text_pole',
+ width: '100%',
+ templateResult: getOpenRouterModelTemplate,
+ });
}
});
diff --git a/public/scripts/textgen-settings.js b/public/scripts/textgen-settings.js
index bb619959a..3b68b6533 100644
--- a/public/scripts/textgen-settings.js
+++ b/public/scripts/textgen-settings.js
@@ -14,6 +14,7 @@ import { BIAS_CACHE, createNewLogitBiasEntry, displayLogitBias, getLogitBiasList
import { power_user, registerDebugFunction } from './power-user.js';
import EventSourceStream from './sse-stream.js';
+import { getCurrentOpenRouterModelTokenizer } from './textgen-models.js';
import { SENTENCEPIECE_TOKENIZERS, TEXTGEN_TOKENIZERS, getTextTokens, tokenizers } from './tokenizers.js';
import { getSortableDelay, onlyUnique } from './utils.js';
@@ -34,9 +35,10 @@ export const textgen_types = {
LLAMACPP: 'llamacpp',
OLLAMA: 'ollama',
INFERMATICAI: 'infermaticai',
+ OPENROUTER: 'openrouter',
};
-const { MANCER, APHRODITE, TABBY, TOGETHERAI, OOBA, OLLAMA, LLAMACPP, INFERMATICAI } = textgen_types;
+const { MANCER, APHRODITE, TABBY, TOGETHERAI, OOBA, OLLAMA, LLAMACPP, INFERMATICAI, OPENROUTER } = textgen_types;
const LLAMACPP_DEFAULT_ORDER = [
'top_k',
@@ -69,6 +71,7 @@ const MANCER_SERVER_DEFAULT = 'https://neuro.mancer.tech';
let MANCER_SERVER = localStorage.getItem(MANCER_SERVER_KEY) ?? MANCER_SERVER_DEFAULT;
let TOGETHERAI_SERVER = 'https://api.together.xyz';
let INFERMATICAI_SERVER = 'https://api.totalgpt.ai';
+let OPENROUTER_SERVER = 'https://openrouter.ai/api';
const SERVER_INPUTS = {
[textgen_types.OOBA]: '#textgenerationwebui_api_url_text',
@@ -137,6 +140,7 @@ const settings = {
togetherai_model: 'Gryphe/MythoMax-L2-13b',
infermaticai_model: '',
ollama_model: '',
+ openrouter_model: 'openrouter/auto',
legacy_api: false,
sampler_order: KOBOLDCPP_ORDER,
logit_bias: [],
@@ -240,6 +244,10 @@ export function getTextGenServer() {
return INFERMATICAI_SERVER;
}
+ if (settings.type === OPENROUTER) {
+ return OPENROUTER_SERVER;
+ }
+
return settings.server_urls[settings.type] ?? '';
}
@@ -264,7 +272,7 @@ async function selectPreset(name) {
function formatTextGenURL(value) {
try {
// Mancer/Together/InfermaticAI doesn't need any formatting (it's hardcoded)
- if (settings.type === MANCER || settings.type === TOGETHERAI || settings.type === INFERMATICAI) {
+ if (settings.type === MANCER || settings.type === TOGETHERAI || settings.type === INFERMATICAI || settings.type === OPENROUTER) {
return value;
}
@@ -297,6 +305,10 @@ function getTokenizerForTokenIds() {
return power_user.tokenizer;
}
+ if (settings.type === OPENROUTER) {
+ return getCurrentOpenRouterModelTokenizer();
+ }
+
return tokenizers.LLAMA;
}
@@ -922,6 +934,10 @@ function getModel() {
return settings.infermaticai_model;
}
+ if (settings.type === OPENROUTER) {
+ return settings.openrouter_model;
+ }
+
if (settings.type === APHRODITE) {
return online_status;
}
diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js
index ac05be8b7..9fcc8f6e9 100644
--- a/public/scripts/tokenizers.js
+++ b/public/scripts/tokenizers.js
@@ -5,8 +5,9 @@ import { groups, selected_group } from './group-chats.js';
import { getStringHash } from './utils.js';
import { kai_flags } from './kai-settings.js';
import { textgen_types, textgenerationwebui_settings as textgen_settings, getTextGenServer } from './textgen-settings.js';
+import { getCurrentOpenRouterModelTokenizer, openRouterModels } from './textgen-models.js';
-const { OOBA, TABBY, KOBOLDCPP, APHRODITE, LLAMACPP } = textgen_types;
+const { OOBA, TABBY, KOBOLDCPP, APHRODITE, LLAMACPP, OPENROUTER } = textgen_types;
export const CHARACTERS_PER_TOKEN_RATIO = 3.35;
const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown';
@@ -202,6 +203,9 @@ export function getTokenizerBestMatch(forApi) {
if (forApi === 'textgenerationwebui' && isTokenizerSupported) {
return tokenizers.API_TEXTGENERATIONWEBUI;
}
+ if (forApi === 'textgenerationwebui' && textgen_settings.type === OPENROUTER) {
+ return getCurrentOpenRouterModelTokenizer();
+ }
}
return tokenizers.LLAMA;
@@ -349,8 +353,11 @@ export function getTokenizerModel() {
}
// And for OpenRouter (if not a site model, then it's impossible to determine the tokenizer)
- if (oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER && oai_settings.openrouter_model) {
- const model = model_list.find(x => x.id === oai_settings.openrouter_model);
+ if (main_api == 'openai' && oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER && oai_settings.openrouter_model ||
+ main_api == 'textgenerationwebui' && textgen_settings.type === OPENROUTER && textgen_settings.openrouter_model) {
+ const model = main_api == 'openai'
+ ? model_list.find(x => x.id === oai_settings.openrouter_model)
+ : openRouterModels.find(x => x.id === textgen_settings.openrouter_model);
if (model?.architecture?.tokenizer === 'Llama2') {
return llamaTokenizer;
diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js
index 9c8295027..5352fce5a 100644
--- a/public/scripts/world-info.js
+++ b/public/scripts/world-info.js
@@ -1542,7 +1542,7 @@ function getWorldEntry(name, data, entry) {
return;
}
- data.entries[uid].scanDepth = !isEmpty && !isNaN(value) && value >= 0 && value < MAX_SCAN_DEPTH ? Math.floor(value) : null;
+ data.entries[uid].scanDepth = !isEmpty && !isNaN(value) && value >= 0 && value <= MAX_SCAN_DEPTH ? Math.floor(value) : null;
setOriginalDataValue(data, uid, 'extensions.scan_depth', data.entries[uid].scanDepth);
saveWorldInfo(name, data);
});
diff --git a/src/additional-headers.js b/src/additional-headers.js
index 43aa4c44b..c1ae260c7 100644
--- a/src/additional-headers.js
+++ b/src/additional-headers.js
@@ -1,4 +1,4 @@
-const { TEXTGEN_TYPES } = require('./constants');
+const { TEXTGEN_TYPES, OPENROUTER_HEADERS } = require('./constants');
const { SECRET_KEYS, readSecret } = require('./endpoints/secrets');
const { getConfigValue } = require('./util');
@@ -27,6 +27,13 @@ function getInfermaticAIHeaders() {
}) : {};
}
+function getOpenRouterHeaders() {
+ const apiKey = readSecret(SECRET_KEYS.OPENROUTER);
+ const baseHeaders = { ...OPENROUTER_HEADERS };
+
+ return apiKey ? Object.assign(baseHeaders, { 'Authorization': `Bearer ${apiKey}` }) : baseHeaders;
+}
+
function getAphroditeHeaders() {
const apiKey = readSecret(SECRET_KEYS.APHRODITE);
@@ -91,6 +98,9 @@ function setAdditionalHeaders(request, args, server) {
case TEXTGEN_TYPES.INFERMATICAI:
headers = getInfermaticAIHeaders();
break;
+ case TEXTGEN_TYPES.OPENROUTER:
+ headers = getOpenRouterHeaders();
+ break;
default:
headers = server ? getOverrideHeaders((new URL(server))?.host) : {};
break;
diff --git a/src/constants.js b/src/constants.js
index 08ffdcd70..67939f90f 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -177,6 +177,7 @@ const TEXTGEN_TYPES = {
LLAMACPP: 'llamacpp',
OLLAMA: 'ollama',
INFERMATICAI: 'infermaticai',
+ OPENROUTER: 'openrouter',
};
const INFERMATICAI_KEYS = [
@@ -226,6 +227,29 @@ const OLLAMA_KEYS = [
const AVATAR_WIDTH = 400;
const AVATAR_HEIGHT = 600;
+const OPENROUTER_HEADERS = {
+ 'HTTP-Referer': 'https://sillytavern.app',
+ 'X-Title': 'SillyTavern',
+};
+
+const OPENROUTER_KEYS = [
+ 'max_tokens',
+ 'temperature',
+ 'top_k',
+ 'top_p',
+ 'presence_penalty',
+ 'frequency_penalty',
+ 'repetition_penalty',
+ 'min_p',
+ 'top_a',
+ 'seed',
+ 'logit_bias',
+ 'model',
+ 'stream',
+ 'prompt',
+ 'stop',
+];
+
module.exports = {
DIRECTORIES,
UNSAFE_EXTENSIONS,
@@ -239,4 +263,6 @@ module.exports = {
TOGETHERAI_KEYS,
OLLAMA_KEYS,
INFERMATICAI_KEYS,
+ OPENROUTER_HEADERS,
+ OPENROUTER_KEYS,
};
diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js
index b82a4391d..e4f936878 100644
--- a/src/endpoints/backends/chat-completions.js
+++ b/src/endpoints/backends/chat-completions.js
@@ -3,7 +3,7 @@ const fetch = require('node-fetch').default;
const { Readable } = require('stream');
const { jsonParser } = require('../../express-common');
-const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY } = require('../../constants');
+const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY, OPENROUTER_HEADERS } = require('../../constants');
const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml, color } = require('../../util');
const { convertClaudePrompt, convertGooglePrompt, convertTextCompletionPrompt } = require('../prompt-converters');
@@ -419,6 +419,9 @@ async function sendMistralAIRequest(request, response) {
try {
//must send a user role as last message
const messages = Array.isArray(request.body.messages) ? request.body.messages : [];
+ //large seems to be throwing a 500 error if we don't make the first message a user role, most likely a bug since the other models won't do this
+ if (request.body.model.includes('large'))
+ messages[0].role = 'user';
const lastMsg = messages[messages.length - 1];
if (messages.length > 0 && lastMsg && (lastMsg.role === 'system' || lastMsg.role === 'assistant')) {
if (lastMsg.role === 'assistant' && lastMsg.name) {
@@ -514,10 +517,7 @@ router.post('/status', jsonParser, async function (request, response_getstatus_o
api_url = 'https://openrouter.ai/api/v1';
api_key_openai = readSecret(SECRET_KEYS.OPENROUTER);
// OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
- headers = {
- 'HTTP-Referer': 'https://sillytavern.app',
- 'X-Title': 'SillyTavern',
- };
+ headers = { ...OPENROUTER_HEADERS };
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MISTRALAI) {
api_url = new URL(request.body.reverse_proxy || API_MISTRAL).toString();
api_key_openai = request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.MISTRALAI);
@@ -704,10 +704,7 @@ router.post('/generate', jsonParser, function (request, response) {
apiUrl = 'https://openrouter.ai/api/v1';
apiKey = readSecret(SECRET_KEYS.OPENROUTER);
// OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
- headers = {
- 'HTTP-Referer': 'https://sillytavern.app',
- 'X-Title': 'SillyTavern',
- };
+ headers = { ...OPENROUTER_HEADERS };
bodyParams = { 'transforms': ['middle-out'] };
if (request.body.min_p !== undefined) {
diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js
index 6c17883e5..d713a5aea 100644
--- a/src/endpoints/backends/text-completions.js
+++ b/src/endpoints/backends/text-completions.js
@@ -4,7 +4,7 @@ const _ = require('lodash');
const Readable = require('stream').Readable;
const { jsonParser } = require('../../express-common');
-const { TEXTGEN_TYPES, TOGETHERAI_KEYS, OLLAMA_KEYS, INFERMATICAI_KEYS } = require('../../constants');
+const { TEXTGEN_TYPES, TOGETHERAI_KEYS, OLLAMA_KEYS, INFERMATICAI_KEYS, OPENROUTER_KEYS } = require('../../constants');
const { forwardFetchResponse, trimV1 } = require('../../util');
const { setAdditionalHeaders } = require('../../additional-headers');
@@ -107,6 +107,7 @@ router.post('/status', jsonParser, async function (request, response) {
case TEXTGEN_TYPES.KOBOLDCPP:
case TEXTGEN_TYPES.LLAMACPP:
case TEXTGEN_TYPES.INFERMATICAI:
+ case TEXTGEN_TYPES.OPENROUTER:
url += '/v1/models';
break;
case TEXTGEN_TYPES.MANCER:
@@ -209,6 +210,7 @@ router.post('/generate', jsonParser, async function (request, response) {
request.body.api_server = request.body.api_server.replace('localhost', '127.0.0.1');
}
+ const apiType = request.body.api_type;
const baseUrl = request.body.api_server;
console.log(request.body);
@@ -245,6 +247,9 @@ router.post('/generate', jsonParser, async function (request, response) {
case TEXTGEN_TYPES.OLLAMA:
url += '/api/generate';
break;
+ case TEXTGEN_TYPES.OPENROUTER:
+ url += '/v1/chat/completions';
+ break;
}
}
@@ -268,11 +273,17 @@ router.post('/generate', jsonParser, async function (request, response) {
args.body = JSON.stringify(request.body);
}
+ if (request.body.api_type === TEXTGEN_TYPES.OPENROUTER) {
+ request.body = _.pickBy(request.body, (_, key) => OPENROUTER_KEYS.includes(key));
+ args.body = JSON.stringify(request.body);
+ }
+
if (request.body.api_type === TEXTGEN_TYPES.OLLAMA) {
args.body = JSON.stringify({
model: request.body.model,
prompt: request.body.prompt,
stream: request.body.stream ?? false,
+ keep_alive: -1,
raw: true,
options: _.pickBy(request.body, (_, key) => OLLAMA_KEYS.includes(key)),
});
@@ -300,7 +311,7 @@ router.post('/generate', jsonParser, async function (request, response) {
}
// Map InfermaticAI response to OAI completions format
- if (completionsReply.url.includes('https://api.totalgpt.ai')) {
+ if (apiType === TEXTGEN_TYPES.INFERMATICAI) {
data['choices'] = (data?.choices || []).map(choice => ({ text: choice.message.content }));
}