From 041957975a479fadb27cedec7cc6e6a57e2f4b7b Mon Sep 17 00:00:00 2001 From: based Date: Sat, 16 Dec 2023 06:08:41 +1000 Subject: [PATCH 01/13] add mistral completion source to UI --- public/index.html | 29 +++++++++++++++++++++++++---- public/scripts/openai.js | 15 +++++++++++++++ public/scripts/secrets.js | 2 ++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/public/index.html b/public/index.html index b0d21d081..00d4cc32f 100644 --- a/public/index.html +++ b/public/index.html @@ -444,7 +444,7 @@ complete. -
+
Temperature
@@ -509,7 +509,7 @@
-
+
Top P
@@ -749,7 +749,7 @@
-
+
Seed
@@ -1593,7 +1593,7 @@ - +
@@ -1842,6 +1842,7 @@ +

OpenAI API key

@@ -2126,6 +2127,26 @@
+
+

MistralAI API Key

+
+ +
+
+
+ For privacy reasons, your API key will be hidden after you reload the page. +
+
+

MistralAI Model

+ +
+
Cancel
diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 9007017f6..a925f5f1d 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -164,6 +164,7 @@ export const chat_completion_sources = { OPENROUTER: 'openrouter', AI21: 'ai21', MAKERSUITE: 'makersuite', + MISTRALAI: 'mistralai', }; const prefixMap = selected_group ? { @@ -208,6 +209,7 @@ const default_settings = { claude_model: 'claude-instant-v1', google_model: 'gemini-pro', ai21_model: 'j2-ultra', + mistralai_model: 'mistral-medium', windowai_model: '', openrouter_model: openrouter_website_model, openrouter_use_fallback: false, @@ -263,6 +265,7 @@ const oai_settings = { claude_model: 'claude-instant-v1', google_model: 'gemini-pro', ai21_model: 'j2-ultra', + mistralai_model: 'mistral-medium', windowai_model: '', openrouter_model: openrouter_website_model, openrouter_use_fallback: false, @@ -1261,6 +1264,8 @@ function getChatCompletionModel() { return oai_settings.openrouter_model !== openrouter_website_model ? oai_settings.openrouter_model : null; case chat_completion_sources.AI21: return oai_settings.ai21_model; + case chat_completion_sources.MISTRALAI: + return oai_settings.mistralai_model; default: throw new Error(`Unknown chat completion source: ${oai_settings.chat_completion_source}`); } @@ -2300,6 +2305,7 @@ function loadOpenAISettings(data, settings) { oai_settings.openrouter_use_fallback = settings.openrouter_use_fallback ?? default_settings.openrouter_use_fallback; oai_settings.openrouter_force_instruct = settings.openrouter_force_instruct ?? default_settings.openrouter_force_instruct; oai_settings.ai21_model = settings.ai21_model ?? default_settings.ai21_model; + oai_settings.mistralai_model = settings.mistralai_model ?? default_settings.mistralai_model; oai_settings.google_model = settings.google_model ?? default_settings.google_model; oai_settings.chat_completion_source = settings.chat_completion_source ?? default_settings.chat_completion_source; oai_settings.api_url_scale = settings.api_url_scale ?? default_settings.api_url_scale; @@ -2342,6 +2348,8 @@ function loadOpenAISettings(data, settings) { $(`#model_google_select option[value="${oai_settings.google_model}"`).attr('selected', true); $('#model_ai21_select').val(oai_settings.ai21_model); $(`#model_ai21_select option[value="${oai_settings.ai21_model}"`).attr('selected', true); + $('#model_mistralai_select').val(oai_settings.mistralai_model); + $(`#model_mistralai_select option[value="${oai_settings.mistralai_model}"`).attr('selected', true); $('#openai_max_context').val(oai_settings.openai_max_context); $('#openai_max_context_counter').val(`${oai_settings.openai_max_context}`); $('#model_openrouter_select').val(oai_settings.openrouter_model); @@ -2519,6 +2527,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) { openrouter_group_models: settings.openrouter_group_models, openrouter_sort_models: settings.openrouter_sort_models, ai21_model: settings.ai21_model, + mistralai_model: settings.mistralai_model, google_model: settings.google_model, temperature: settings.temp_openai, frequency_penalty: settings.freq_pen_openai, @@ -2890,6 +2899,7 @@ function onSettingsPresetChange() { openrouter_group_models: ['#openrouter_group_models', 'openrouter_group_models', false], openrouter_sort_models: ['#openrouter_sort_models', 'openrouter_sort_models', false], ai21_model: ['#model_ai21_select', 'ai21_model', false], + mistralai_model: ['#model_mistralai_select', 'mistralai_model', false], google_model: ['#model_google_select', 'google_model', false], openai_max_context: ['#openai_max_context', 'openai_max_context', false], openai_max_tokens: ['#openai_max_tokens', 'openai_max_tokens', false], @@ -3074,6 +3084,11 @@ async function onModelChange() { oai_settings.google_model = value; } + if ($(this).is('#model_mistralai_select')) { + console.log('MistralAI model changed to', value); + oai_settings.mistralai_model = value; + } + if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) { if (oai_settings.max_context_unlocked) { $('#openai_max_context').attr('max', unlocked_max); diff --git a/public/scripts/secrets.js b/public/scripts/secrets.js index 6afb538f1..5ccb6e54e 100644 --- a/public/scripts/secrets.js +++ b/public/scripts/secrets.js @@ -14,6 +14,7 @@ export const SECRET_KEYS = { SCALE_COOKIE: 'scale_cookie', MAKERSUITE: 'api_key_makersuite', SERPAPI: 'api_key_serpapi', + MISTRALAI: 'api_key_mistralai', }; const INPUT_MAP = { @@ -29,6 +30,7 @@ const INPUT_MAP = { [SECRET_KEYS.MAKERSUITE]: '#api_key_makersuite', [SECRET_KEYS.APHRODITE]: '#api_key_aphrodite', [SECRET_KEYS.TABBY]: '#api_key_tabby', + [SECRET_KEYS.MISTRALAI]: '#api_key_mistralai', }; async function clearSecret() { From 583f786d746031b241df845b99d1458a23decd64 Mon Sep 17 00:00:00 2001 From: based Date: Sat, 16 Dec 2023 07:15:57 +1000 Subject: [PATCH 02/13] finish mistral frontend integration + apikey status check --- public/script.js | 8 ++++- public/scripts/RossAscends-mods.js | 1 + public/scripts/openai.js | 36 ++++++++++++++++++++-- src/constants.js | 1 + src/endpoints/backends/chat-completions.js | 19 ++++++++++-- src/endpoints/secrets.js | 1 + 6 files changed, 61 insertions(+), 5 deletions(-) diff --git a/public/script.js b/public/script.js index d869f3eb6..da11e20dd 100644 --- a/public/script.js +++ b/public/script.js @@ -5396,6 +5396,7 @@ function changeMainAPI() { case chat_completion_sources.OPENAI: case chat_completion_sources.AI21: case chat_completion_sources.MAKERSUITE: + case chat_completion_sources.MISTRALAI: default: setupChatCompletionPromptManager(oai_settings); break; @@ -7546,6 +7547,11 @@ async function connectAPISlash(_, text) { source: 'makersuite', button: '#api_button_openai', }, + 'mistralai': { + selected: 'openai', + source: 'mistralai', + button: '#api_button_openai', + }, }; const apiConfig = apiMap[text.toLowerCase()]; @@ -7832,7 +7838,7 @@ jQuery(async function () { } registerSlashCommand('dupe', DupeChar, [], '– duplicates the currently selected character', true, true); - registerSlashCommand('api', connectAPISlash, [], '(kobold, horde, novel, ooba, tabby, mancer, aphrodite, kcpp, oai, claude, windowai, openrouter, scale, ai21, makersuite) – connect to an API', true, true); + registerSlashCommand('api', connectAPISlash, [], '(kobold, horde, novel, ooba, tabby, mancer, aphrodite, kcpp, oai, claude, windowai, openrouter, scale, ai21, makersuite, mistralai) – connect to an API', true, true); registerSlashCommand('impersonate', doImpersonate, ['imp'], '– calls an impersonation response', true, true); registerSlashCommand('delchat', doDeleteChat, [], '– deletes the current chat', true, true); registerSlashCommand('closechat', doCloseChat, [], '– closes the current chat', true, true); diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index c07a49437..e271d8c74 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -396,6 +396,7 @@ function RA_autoconnect(PrevApi) { || (secret_state[SECRET_KEYS.OPENROUTER] && oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER) || (secret_state[SECRET_KEYS.AI21] && oai_settings.chat_completion_source == chat_completion_sources.AI21) || (secret_state[SECRET_KEYS.MAKERSUITE] && oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) + || (secret_state[SECRET_KEYS.MISTRALAI] && oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) ) { $('#api_button_openai').trigger('click'); } diff --git a/public/scripts/openai.js b/public/scripts/openai.js index a925f5f1d..f7e1293e3 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -1453,6 +1453,7 @@ async function sendOpenAIRequest(type, messages, signal) { const isAI21 = oai_settings.chat_completion_source == chat_completion_sources.AI21; const isGoogle = oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE; const isOAI = oai_settings.chat_completion_source == chat_completion_sources.OPENAI; + const isMistral = oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI; const isTextCompletion = (isOAI && textCompletionModels.includes(oai_settings.openai_model)) || (isOpenRouter && oai_settings.openrouter_force_instruct && power_user.instruct.enabled); const isQuiet = type === 'quiet'; const isImpersonate = type === 'impersonate'; @@ -1560,7 +1561,11 @@ async function sendOpenAIRequest(type, messages, signal) { generate_data['stop_tokens'] = [name1 + ':', oai_settings.new_chat_prompt, oai_settings.new_group_chat_prompt]; } - if ((isOAI || isOpenRouter) && oai_settings.seed >= 0) { + if (isMistral) { + generate_data['safe_mode'] = false; // already defaults to false, but just incase they change that in the future. + } + + if ((isOAI || isOpenRouter || isMistral) && oai_settings.seed >= 0) { generate_data['seed'] = oai_settings.seed; } @@ -2457,7 +2462,7 @@ async function getStatusOpen() { chat_completion_source: oai_settings.chat_completion_source, }; - if (oai_settings.reverse_proxy && oai_settings.chat_completion_source !== chat_completion_sources.OPENROUTER) { + if (oai_settings.reverse_proxy && (oai_settings.chat_completion_source !== chat_completion_sources.OPENROUTER || oai_settings.chat_completion_source !== chat_completion_sources.MISTRALAI)) { validateReverseProxy(); } @@ -3194,6 +3199,16 @@ async function onModelChange() { $('#temp_openai').attr('max', oai_max_temp).val(oai_settings.temp_openai).trigger('input'); } + if (oai_settings.chat_completion_source === chat_completion_sources.MISTRALAI) { + $('#openai_max_context').attr('max', max_32k); + oai_settings.openai_max_context = Math.min(oai_settings.openai_max_context, Number($('#openai_max_context').attr('max'))); + $('#openai_max_context').val(oai_settings.openai_max_context).trigger('input'); + + //mistral also caps temp at 1.0 + oai_settings.temp_openai = Math.min(claude_max_temp, oai_settings.temp_openai); + $('#temp_openai').attr('max', claude_max_temp).val(oai_settings.temp_openai).trigger('input'); + } + if (oai_settings.chat_completion_source == chat_completion_sources.AI21) { if (oai_settings.max_context_unlocked) { $('#openai_max_context').attr('max', unlocked_max); @@ -3355,6 +3370,19 @@ async function onConnectButtonClick(e) { } } + if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) { + const api_key_mistralai = String($('#api_key_mistralai').val()).trim(); + + if (api_key_mistralai.length) { + await writeSecret(SECRET_KEYS.MISTRALAI, api_key_mistralai); + } + + if (!secret_state[SECRET_KEYS.MISTRALAI]) { + console.log('No secret key saved for MistralAI'); + return; + } + } + startStatusLoading(); saveSettingsDebounced(); await getStatusOpen(); @@ -3387,6 +3415,9 @@ function toggleChatCompletionForms() { else if (oai_settings.chat_completion_source == chat_completion_sources.AI21) { $('#model_ai21_select').trigger('change'); } + else if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) { + $('#model_mistralai_select').trigger('change'); + } $('[data-source]').each(function () { const validSources = $(this).data('source').split(','); $(this).toggle(validSources.includes(oai_settings.chat_completion_source)); @@ -3764,6 +3795,7 @@ $(document).ready(async function () { $('#openrouter_group_models').on('change', onOpenrouterModelSortChange); $('#openrouter_sort_models').on('change', onOpenrouterModelSortChange); $('#model_ai21_select').on('change', onModelChange); + $('#model_mistralai_select').on('change', onModelChange); $('#settings_preset_openai').on('change', onSettingsPresetChange); $('#new_oai_preset').on('click', onNewPresetClick); $('#delete_oai_preset').on('click', onDeletePresetClick); diff --git a/src/constants.js b/src/constants.js index 92af44cf2..3ca5caf0b 100644 --- a/src/constants.js +++ b/src/constants.js @@ -159,6 +159,7 @@ const CHAT_COMPLETION_SOURCES = { OPENROUTER: 'openrouter', AI21: 'ai21', MAKERSUITE: 'makersuite', + MISTRALAI: 'mistralai', }; const UPLOADS_PATH = './uploads'; diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index 13e09cd56..ab9bf2a21 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -392,6 +392,15 @@ async function sendAI21Request(request, response) { } +/** + * Sends a request to MistralAI API. + * @param {express.Request} request Express request + * @param {express.Response} response Express response + */ +async function sendMistralAIRequest(request, response) { + +} + const router = express.Router(); router.post('/status', jsonParser, async function (request, response_getstatus_openai) { @@ -401,15 +410,18 @@ router.post('/status', jsonParser, async function (request, response_getstatus_o let api_key_openai; let headers; - if (request.body.chat_completion_source !== CHAT_COMPLETION_SOURCES.OPENROUTER) { + if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) { api_url = new URL(request.body.reverse_proxy || API_OPENAI).toString(); api_key_openai = request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.OPENAI); headers = {}; - } else { + } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) { api_url = 'https://openrouter.ai/api/v1'; api_key_openai = readSecret(SECRET_KEYS.OPENROUTER); // OpenRouter needs to pass the referer: https://openrouter.ai/docs headers = { 'HTTP-Referer': request.headers.referer }; + } else { + api_url = 'https://api.mistral.ai/v1'; + api_key_openai = readSecret(SECRET_KEYS.MISTRALAI); } if (!api_key_openai && !request.body.reverse_proxy) { @@ -444,6 +456,9 @@ router.post('/status', jsonParser, async function (request, response_getstatus_o }); console.log('Available OpenRouter models:', models); + } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MISTRALAI) { + const models = data?.data; + console.log(models); } else { const models = data?.data; diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index 5da0cc730..d6f902df4 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -25,6 +25,7 @@ const SECRET_KEYS = { DEEPLX_URL: 'deeplx_url', MAKERSUITE: 'api_key_makersuite', SERPAPI: 'api_key_serpapi', + MISTRALAI: 'api_key_mistralai', }; /** From c517483141ab6599ead43e7f894eb9245183eead Mon Sep 17 00:00:00 2001 From: based Date: Sat, 16 Dec 2023 08:27:40 +1000 Subject: [PATCH 03/13] added mistral chat completion --- src/endpoints/backends/chat-completions.js | 55 ++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index ab9bf2a21..ad505fef3 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -398,7 +398,61 @@ async function sendAI21Request(request, response) { * @param {express.Response} response Express response */ async function sendMistralAIRequest(request, response) { + const apiKey = readSecret(SECRET_KEYS.MISTRALAI); + if (!apiKey) { + console.log('MistralAI API key is missing.'); + return response.status(400).send({ error: true }); + } + + try { + const controller = new AbortController(); + request.socket.removeAllListeners('close'); + request.socket.on('close', function () { + controller.abort(); + }); + + const config = { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': 'Bearer ' + apiKey, + }, + body: JSON.stringify({ + 'model': request.body.model, + 'messages': request.body.messages, + 'temperature': request.body.temperature, + 'top_p': request.body.top_p, + 'max_tokens': request.body.max_tokens, + 'stream': request.body.stream, + 'safe_mode': request.body.safe_mode, + 'random_seed': request.body.seed === -1 ? undefined : request.body.seed, + }), + signal: controller.signal, + timeout: 0, + }; + + const generateResponse = await fetch('https://api.mistral.ai/v1/chat/completions', config); + if (request.body.stream) { + forwardFetchResponse(generateResponse, response); + } else { + if (!generateResponse.ok) { + console.log(`MistralAI API returned error: ${generateResponse.status} ${generateResponse.statusText} ${await generateResponse.text()}`); + // a 401 unauthorized response breaks the frontend auth, so return a 500 instead. prob a better way of dealing with this. + // 401s are already handled by the streaming processor and dont pop up an error toast, that should probably be fixed too. + return response.status(generateResponse.status === 401 ? 500 : generateResponse.status).send({ error: true }); + } + const generateResponseJson = await generateResponse.json(); + return response.send(generateResponseJson); + } + } catch (error) { + console.log('Error communicating with MistralAI API: ', error); + if (!response.headersSent) { + response.send({ error: true }); + } else { + response.end(); + } + } } const router = express.Router(); @@ -566,6 +620,7 @@ router.post('/generate', jsonParser, function (request, response) { case CHAT_COMPLETION_SOURCES.SCALE: return sendScaleRequest(request, response); case CHAT_COMPLETION_SOURCES.AI21: return sendAI21Request(request, response); case CHAT_COMPLETION_SOURCES.MAKERSUITE: return sendMakerSuiteRequest(request, response); + case CHAT_COMPLETION_SOURCES.MISTRALAI: return sendMistralAIRequest(request, response); } let apiUrl; From 5dd2e8cd883aaf256f9c9e22d3314c8e6f80d723 Mon Sep 17 00:00:00 2001 From: based Date: Sat, 16 Dec 2023 08:37:39 +1000 Subject: [PATCH 04/13] added mistral vector support (off the back of oai's) --- public/scripts/extensions/vectors/index.js | 3 ++- public/scripts/extensions/vectors/settings.html | 1 + src/endpoints/vectors.js | 3 ++- src/openai-vectors.js | 12 +++++++----- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 214b1d887..9e8777333 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -394,7 +394,8 @@ async function getSavedHashes(collectionId) { */ async function insertVectorItems(collectionId, items) { if (settings.source === 'openai' && !secret_state[SECRET_KEYS.OPENAI] || - settings.source === 'palm' && !secret_state[SECRET_KEYS.MAKERSUITE]) { + settings.source === 'palm' && !secret_state[SECRET_KEYS.MAKERSUITE] || + settings.source === 'mistral' && !secret_state[SECRET_KEYS.MISTRALAI]) { throw new Error('Vectors: API key missing', { cause: 'api_key_missing' }); } diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html index cb904b8a3..b1d74c83d 100644 --- a/public/scripts/extensions/vectors/settings.html +++ b/public/scripts/extensions/vectors/settings.html @@ -13,6 +13,7 @@ +
diff --git a/src/endpoints/vectors.js b/src/endpoints/vectors.js index e49d157fa..45d4d55a6 100644 --- a/src/endpoints/vectors.js +++ b/src/endpoints/vectors.js @@ -12,8 +12,9 @@ const { jsonParser } = require('../express-common'); */ async function getVector(source, text) { switch (source) { + case 'mistral': case 'openai': - return require('../openai-vectors').getOpenAIVector(text); + return require('../openai-vectors').getOpenAIVector(text, source); case 'transformers': return require('../embedding').getTransformersVector(text); case 'palm': diff --git a/src/openai-vectors.js b/src/openai-vectors.js index ecb245065..40c54ae2f 100644 --- a/src/openai-vectors.js +++ b/src/openai-vectors.js @@ -2,19 +2,21 @@ const fetch = require('node-fetch').default; const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); /** - * Gets the vector for the given text from OpenAI ada model + * Gets the vector for the given text from an OpenAI compatible endpoint. * @param {string} text - The text to get the vector for * @returns {Promise} - The vector for the text */ -async function getOpenAIVector(text) { - const key = readSecret(SECRET_KEYS.OPENAI); +async function getOpenAIVector(text, source) { + const isMistral = source === 'mistral'; + const key = readSecret(isMistral ? SECRET_KEYS.MISTRALAI : SECRET_KEYS.OPENAI); if (!key) { console.log('No OpenAI key found'); throw new Error('No OpenAI key found'); } - const response = await fetch('https://api.openai.com/v1/embeddings', { + const url = isMistral ? 'api.mistral.ai' : 'api.openai.com'; + const response = await fetch(`https://${url}/v1/embeddings`, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -22,7 +24,7 @@ async function getOpenAIVector(text) { }, body: JSON.stringify({ input: text, - model: 'text-embedding-ada-002', + model: isMistral ? 'mistral-embed' : 'text-embedding-ada-002', }), }); From 7acb61ab681dcc796daa2a4053eff5a7f40484fe Mon Sep 17 00:00:00 2001 From: based Date: Sat, 16 Dec 2023 08:47:51 +1000 Subject: [PATCH 05/13] use appropriate tokenizer with mistral completion source --- public/scripts/tokenizers.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index 196f3ec9c..32cd44b5e 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -384,6 +384,10 @@ export function getTokenizerModel() { return claudeTokenizer; } + if (oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) { + return mistralTokenizer; + } + // Default to Turbo 3.5 return turboTokenizer; } From 65fa3335132f233bd6c093e33af0842301391f18 Mon Sep 17 00:00:00 2001 From: based Date: Sat, 16 Dec 2023 09:08:05 +1000 Subject: [PATCH 06/13] last role fix. wtf mistral...jbs le over? --- src/endpoints/backends/chat-completions.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index ad505fef3..2fc89fe03 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -405,6 +405,12 @@ async function sendMistralAIRequest(request, response) { return response.status(400).send({ error: true }); } + //can't send a system role as the last message. + const messages = Array.isArray(request.body.messages) ? request.body.messages : []; + if (messages.length > 0 && messages[messages.length - 1].role === 'system') { + messages[messages.length - 1].role = 'user'; + } + try { const controller = new AbortController(); request.socket.removeAllListeners('close'); @@ -420,7 +426,7 @@ async function sendMistralAIRequest(request, response) { }, body: JSON.stringify({ 'model': request.body.model, - 'messages': request.body.messages, + 'messages': messages, 'temperature': request.body.temperature, 'top_p': request.body.top_p, 'max_tokens': request.body.max_tokens, From af059a6af65af6d0bdd21d71f6db957269b3a2f9 Mon Sep 17 00:00:00 2001 From: based Date: Sat, 16 Dec 2023 09:44:47 +1000 Subject: [PATCH 07/13] more last prompt fixes + added an svg --- public/img/mistralai.svg | 15 +++++++++++++++ src/endpoints/backends/chat-completions.js | 8 ++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 public/img/mistralai.svg diff --git a/public/img/mistralai.svg b/public/img/mistralai.svg new file mode 100644 index 000000000..f3522fb5e --- /dev/null +++ b/public/img/mistralai.svg @@ -0,0 +1,15 @@ + + + + + + + + diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index 2fc89fe03..1ab53d79d 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -407,8 +407,12 @@ async function sendMistralAIRequest(request, response) { //can't send a system role as the last message. const messages = Array.isArray(request.body.messages) ? request.body.messages : []; - if (messages.length > 0 && messages[messages.length - 1].role === 'system') { - messages[messages.length - 1].role = 'user'; + const lastMsg = messages[messages.length - 1]; + if (messages.length > 0 && lastMsg && (lastMsg.role === 'system' || lastMsg.role === 'assistant')) { + lastMsg.role = 'user'; + if(lastMsg.role === 'assistant') { + lastMsg.content = lastMsg.name + ': ' + lastMsg.content; + } } try { From 47357da20c24854f04627891e0ec940cae2ced9f Mon Sep 17 00:00:00 2001 From: based Date: Sat, 16 Dec 2023 09:45:31 +1000 Subject: [PATCH 08/13] comment --- src/endpoints/backends/chat-completions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index 1ab53d79d..d93db150b 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -405,7 +405,7 @@ async function sendMistralAIRequest(request, response) { return response.status(400).send({ error: true }); } - //can't send a system role as the last message. + //must send a user role as last message const messages = Array.isArray(request.body.messages) ? request.body.messages : []; const lastMsg = messages[messages.length - 1]; if (messages.length > 0 && lastMsg && (lastMsg.role === 'system' || lastMsg.role === 'assistant')) { From f16ac8686ec214c863d7ad4062900ae5be7576a2 Mon Sep 17 00:00:00 2001 From: based Date: Sat, 16 Dec 2023 09:50:10 +1000 Subject: [PATCH 09/13] just incase --- public/img/mistralai.svg | 2 +- src/endpoints/backends/chat-completions.js | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/public/img/mistralai.svg b/public/img/mistralai.svg index f3522fb5e..383094210 100644 --- a/public/img/mistralai.svg +++ b/public/img/mistralai.svg @@ -2,7 +2,7 @@ 0 && lastMsg && (lastMsg.role === 'system' || lastMsg.role === 'assistant')) { - lastMsg.role = 'user'; - if(lastMsg.role === 'assistant') { - lastMsg.content = lastMsg.name + ': ' + lastMsg.content; - } - } - try { + //must send a user role as last message + const messages = Array.isArray(request.body.messages) ? request.body.messages : []; + const lastMsg = messages[messages.length - 1]; + if (messages.length > 0 && lastMsg && (lastMsg.role === 'system' || lastMsg.role === 'assistant')) { + lastMsg.role = 'user'; + if(lastMsg.role === 'assistant') { + lastMsg.content = lastMsg.name + ': ' + lastMsg.content; + } + } + const controller = new AbortController(); request.socket.removeAllListeners('close'); request.socket.on('close', function () { From ed96ec5c3e5345b63d8ec3346ca190c596919451 Mon Sep 17 00:00:00 2001 From: based Date: Sat, 16 Dec 2023 12:02:34 +1000 Subject: [PATCH 10/13] reverse proxy condition fix --- public/scripts/openai.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index f7e1293e3..f3c17f373 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -2462,7 +2462,7 @@ async function getStatusOpen() { chat_completion_source: oai_settings.chat_completion_source, }; - if (oai_settings.reverse_proxy && (oai_settings.chat_completion_source !== chat_completion_sources.OPENROUTER || oai_settings.chat_completion_source !== chat_completion_sources.MISTRALAI)) { + if (oai_settings.reverse_proxy && (oai_settings.chat_completion_source === chat_completion_sources.OPENAI || oai_settings.chat_completion_source === chat_completion_sources.CLAUDE)) { validateReverseProxy(); } From 60e1d10263445a570f06101ac255e53262c2e15c Mon Sep 17 00:00:00 2001 From: based Date: Sat, 16 Dec 2023 12:08:34 +1000 Subject: [PATCH 11/13] add fallback error for /status --- src/endpoints/backends/chat-completions.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index e4b7b3df7..c70d5c8f4 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -483,9 +483,12 @@ router.post('/status', jsonParser, async function (request, response_getstatus_o api_key_openai = readSecret(SECRET_KEYS.OPENROUTER); // OpenRouter needs to pass the referer: https://openrouter.ai/docs headers = { 'HTTP-Referer': request.headers.referer }; - } else { + } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MISTRALAI) { api_url = 'https://api.mistral.ai/v1'; api_key_openai = readSecret(SECRET_KEYS.MISTRALAI); + } else { + console.log('This chat completion source is not supported yet.'); + return response_getstatus_openai.status(400).send({ error: true }); } if (!api_key_openai && !request.body.reverse_proxy) { From b1f07eb9892b52d7718a0fbd2938be03e7c43044 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 17 Dec 2023 02:49:14 +0200 Subject: [PATCH 12/13] lint: format --- src/endpoints/backends/chat-completions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index c70d5c8f4..07775c1fd 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -411,7 +411,7 @@ async function sendMistralAIRequest(request, response) { const lastMsg = messages[messages.length - 1]; if (messages.length > 0 && lastMsg && (lastMsg.role === 'system' || lastMsg.role === 'assistant')) { lastMsg.role = 'user'; - if(lastMsg.role === 'assistant') { + if (lastMsg.role === 'assistant') { lastMsg.content = lastMsg.name + ': ' + lastMsg.content; } } From 2d8a62d0596127e9459da30adf7128ab7712f8bb Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 17 Dec 2023 02:56:47 +0200 Subject: [PATCH 13/13] Refactor openai vectors --- src/openai-vectors.js | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/src/openai-vectors.js b/src/openai-vectors.js index 40c54ae2f..3b19a4c96 100644 --- a/src/openai-vectors.js +++ b/src/openai-vectors.js @@ -1,21 +1,41 @@ const fetch = require('node-fetch').default; const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); +const SOURCES = { + 'mistral': { + secretKey: SECRET_KEYS.MISTRAL, + url: 'api.mistral.ai', + model: 'mistral-embed', + }, + 'openai': { + secretKey: SECRET_KEYS.OPENAI, + url: 'api.openai.com', + model: 'text-embedding-ada-002', + }, +}; + /** * Gets the vector for the given text from an OpenAI compatible endpoint. * @param {string} text - The text to get the vector for + * @param {string} source - The source of the vector * @returns {Promise} - The vector for the text */ async function getOpenAIVector(text, source) { - const isMistral = source === 'mistral'; - const key = readSecret(isMistral ? SECRET_KEYS.MISTRALAI : SECRET_KEYS.OPENAI); + const config = SOURCES[source]; - if (!key) { - console.log('No OpenAI key found'); - throw new Error('No OpenAI key found'); + if (!config) { + console.log('Unknown source', source); + throw new Error('Unknown source'); } - const url = isMistral ? 'api.mistral.ai' : 'api.openai.com'; + const key = readSecret(config.secretKey); + + if (!key) { + console.log('No API key found'); + throw new Error('No API key found'); + } + + const url = config.url; const response = await fetch(`https://${url}/v1/embeddings`, { method: 'POST', headers: { @@ -24,22 +44,22 @@ async function getOpenAIVector(text, source) { }, body: JSON.stringify({ input: text, - model: isMistral ? 'mistral-embed' : 'text-embedding-ada-002', + model: config.model, }), }); if (!response.ok) { const text = await response.text(); - console.log('OpenAI request failed', response.statusText, text); - throw new Error('OpenAI request failed'); + console.log('API request failed', response.statusText, text); + throw new Error('API request failed'); } const data = await response.json(); const vector = data?.data[0]?.embedding; if (!Array.isArray(vector)) { - console.log('OpenAI response was not an array'); - throw new Error('OpenAI response was not an array'); + console.log('API response was not an array'); + throw new Error('API response was not an array'); } return vector;