From f29f934c6bf4c5cc358a6516c9dfe5a199a6e56b Mon Sep 17 00:00:00 2001 From: NWilson Date: Wed, 24 Jan 2024 06:59:27 -0600 Subject: [PATCH 01/19] Progress --- public/index.html | 21 +++++++- public/script.js | 17 ++++++- public/scripts/RossAscends-mods.js | 1 + public/scripts/secrets.js | 2 + public/scripts/textgen-models.js | 57 ++++++++++++++++++++++ public/scripts/textgen-settings.js | 17 +++++-- src/additional-headers.js | 11 +++++ src/constants.js | 10 ++++ src/endpoints/backends/text-completions.js | 8 ++- 9 files changed, 137 insertions(+), 7 deletions(-) diff --git a/public/index.html b/public/index.html index cc468feae..64de8e586 100644 --- a/public/index.html +++ b/public/index.html @@ -1848,6 +1848,7 @@ +
@@ -1868,6 +1869,24 @@
+
+

InfermaticAI API Key

+
+ +
+
+
+ For privacy reasons, your API key will be hidden after you reload the page. +
+
+

InfermaticAI Model

+ +
+
@@ -5235,4 +5254,4 @@ - \ No newline at end of file + diff --git a/public/script.js b/public/script.js index 571b95d2f..bff268e5a 100644 --- a/public/script.js +++ b/public/script.js @@ -20,7 +20,7 @@ import { validateTextGenUrl, } from './scripts/textgen-settings.js'; -const { MANCER, TOGETHERAI, OOBA, APHRODITE, OLLAMA } = textgen_types; +const { MANCER, TOGETHERAI, OOBA, APHRODITE, OLLAMA, INFERMATICAI } = textgen_types; import { world_info, @@ -185,7 +185,7 @@ import { createPersona, initPersonas, selectCurrentPersona, setPersonaDescriptio import { getBackgrounds, initBackgrounds, loadBackgroundSettings, background_settings } from './scripts/backgrounds.js'; import { hideLoader, showLoader } from './scripts/loader.js'; import { BulkEditOverlay, CharacterContextMenu } from './scripts/BulkEditOverlay.js'; -import { loadMancerModels, loadOllamaModels, loadTogetherAIModels } from './scripts/textgen-models.js'; +import { loadMancerModels, loadOllamaModels, loadTogetherAIModels, loadInfermaticAIModels } from './scripts/textgen-models.js'; import { appendFileContent, hasPendingFileAttachment, populateFileAttachment, decodeStyleTags, encodeStyleTags } from './scripts/chats.js'; import { initPresetManager } from './scripts/preset-manager.js'; import { evaluateMacros } from './scripts/macros.js'; @@ -988,6 +988,9 @@ async function getStatusTextgen() { } else if (textgen_settings.type === OLLAMA) { loadOllamaModels(data?.data); online_status = textgen_settings.ollama_model || 'Connected'; + } else if (textgen_settings.type === INFERMATICAI) { + loadInfermaticAIModels(data?.data); + online_status = textgen_settings.infermaticai_model; } else { online_status = data?.result; } @@ -7399,6 +7402,11 @@ const CONNECT_API_MAP = { button: '#api_button_openai', source: chat_completion_sources.CUSTOM, }, + 'infermaticai': { + selected: 'openai', + button: '#api_button_openai', + type: textgen_types.INFERMATICAI, + }, }; /** @@ -8246,6 +8254,11 @@ jQuery(async function () { await writeSecret(SECRET_KEYS.TOGETHERAI, togetherKey); } + const infermaticAIKey = String($('#api_key_infermaticai').val()).trim(); + if (infermaticAIKey.length) { + await writeSecret(SECRET_KEYS.INFERMATICAI, infermaticAIKey); + } + validateTextGenUrl(); startStatusLoading(); main_api = 'textgenerationwebui'; diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index 7b76ba010..3284e6a6c 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -397,6 +397,7 @@ function RA_autoconnect(PrevApi) { || (secret_state[SECRET_KEYS.AI21] && oai_settings.chat_completion_source == chat_completion_sources.AI21) || (secret_state[SECRET_KEYS.MAKERSUITE] && oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) || (secret_state[SECRET_KEYS.MISTRALAI] && oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) + || (secret_state[SECRET_KEYS.INFERMATICAI] && oai_settings.chat_completion_source == chat_completion_sources.INFERMATICAI) || (isValidUrl(oai_settings.custom_url) && oai_settings.chat_completion_source == chat_completion_sources.CUSTOM) ) { $('#api_button_openai').trigger('click'); diff --git a/public/scripts/secrets.js b/public/scripts/secrets.js index 1c1154180..2a67e497b 100644 --- a/public/scripts/secrets.js +++ b/public/scripts/secrets.js @@ -16,6 +16,7 @@ export const SECRET_KEYS = { SERPAPI: 'api_key_serpapi', MISTRALAI: 'api_key_mistralai', TOGETHERAI: 'api_key_togetherai', + INFERMATICAI: 'api_key_infermaticai', CUSTOM: 'api_key_custom', }; @@ -35,6 +36,7 @@ const INPUT_MAP = { [SECRET_KEYS.MISTRALAI]: '#api_key_mistralai', [SECRET_KEYS.CUSTOM]: '#api_key_custom', [SECRET_KEYS.TOGETHERAI]: '#api_key_togetherai', + [SECRET_KEYS.INFERMATICAI]: '#api_key_infermaticai', }; async function clearSecret() { diff --git a/public/scripts/textgen-models.js b/public/scripts/textgen-models.js index c828f6404..ca9c53c77 100644 --- a/public/scripts/textgen-models.js +++ b/public/scripts/textgen-models.js @@ -4,6 +4,7 @@ import { textgenerationwebui_settings as textgen_settings, textgen_types } from let mancerModels = []; let togetherModels = []; +let infermaticAIModels = []; export async function loadOllamaModels(data) { if (!Array.isArray(data)) { @@ -52,6 +53,32 @@ export async function loadTogetherAIModels(data) { } } +export async function loadInfermaticAIModels(data) { + if (!Array.isArray(data)) { + console.error('Invalid Infermatic AI models data', data); + return; + } + + infermaticAIModels = data; + + if (!data.find(x => x.id === textgen_settings.infermaticai_model)) { + textgen_settings.infermaticai_model = data[0]?.id || ''; + } + + $('#model_infermaticai_select').empty(); + for (const model of data) { + if (model.display_type === 'image') { + continue; + } + + const option = document.createElement('option'); + option.value = model.id; + option.text = model.id; + option.selected = model.id === textgen_settings.infermaticai_model; + $('#model_infermaticai_select').append(option); + } +} + export async function loadMancerModels(data) { if (!Array.isArray(data)) { console.error('Invalid Mancer models data', data); @@ -91,6 +118,14 @@ function onTogetherModelSelect() { setGenerationParamsFromPreset({ max_length: model.context_length }); } +function onInfermaticAIModelSelect() { + const modelName = String($('#model_infermaticai_select').val()); + textgen_settings.infermaticai_model = modelName; + $('#api_button_openai').trigger('click'); + const model = infermaticAIModels.find(x => x.id === modelName); + setGenerationParamsFromPreset({ max_length: model.context_length }); +} + function onOllamaModelSelect() { const modelId = String($('#ollama_model').val()); textgen_settings.ollama_model = modelId; @@ -130,6 +165,20 @@ function getTogetherModelTemplate(option) { `)); } +function getInfermaticAIModelTemplate(option) { + const model = infermaticAIModels.find(x => x.id === option?.element?.value); + + if (!option.id || !model) { + return option.text; + } + + return $((` +
+
${DOMPurify.sanitize(model.id)}
+
+ `)); +} + async function downloadOllamaModel() { try { const serverUrl = textgen_settings.server_urls[textgen_types.OLLAMA]; @@ -174,6 +223,7 @@ async function downloadOllamaModel() { jQuery(function () { $('#mancer_model').on('change', onMancerModelSelect); $('#model_togetherai_select').on('change', onTogetherModelSelect); + $('#model_infermaticai_select').on('change', onInfermaticAIModelSelect); $('#ollama_model').on('change', onOllamaModelSelect); $('#ollama_download_model').on('click', downloadOllamaModel); @@ -198,5 +248,12 @@ jQuery(function () { searchInputCssClass: 'text_pole', width: '100%', }); + $('#model_infermaticai_select').select2({ + placeholder: 'Select a model', + searchInputPlaceholder: 'Search models...', + searchInputCssClass: 'text_pole', + width: '100%', + templateResult: getInfermaticAIModelTemplate, + }); } }); diff --git a/public/scripts/textgen-settings.js b/public/scripts/textgen-settings.js index 51b08538d..eb8227095 100644 --- a/public/scripts/textgen-settings.js +++ b/public/scripts/textgen-settings.js @@ -34,9 +34,10 @@ export const textgen_types = { TOGETHERAI: 'togetherai', LLAMACPP: 'llamacpp', OLLAMA: 'ollama', + INFERMATICAI: 'infermaticai', }; -const { MANCER, APHRODITE, TOGETHERAI, OOBA, OLLAMA, LLAMACPP } = textgen_types; +const { MANCER, APHRODITE, TOGETHERAI, OOBA, OLLAMA, LLAMACPP, INFERMATICAI } = textgen_types; const BIAS_KEY = '#textgenerationwebui_api-settings'; // Maybe let it be configurable in the future? @@ -45,6 +46,7 @@ const MANCER_SERVER_KEY = 'mancer_server'; const MANCER_SERVER_DEFAULT = 'https://neuro.mancer.tech'; let MANCER_SERVER = localStorage.getItem(MANCER_SERVER_KEY) ?? MANCER_SERVER_DEFAULT; let TOGETHERAI_SERVER = 'https://api.together.xyz'; +let INFERMATICAI_SERVER = 'https://api.totalgpt.ai'; const SERVER_INPUTS = { [textgen_types.OOBA]: '#textgenerationwebui_api_url_text', @@ -107,6 +109,7 @@ const settings = { type: textgen_types.OOBA, mancer_model: 'mytholite', togetherai_model: 'Gryphe/MythoMax-L2-13b', + infermaticai_model: '', ollama_model: '', legacy_api: false, sampler_order: KOBOLDCPP_ORDER, @@ -203,6 +206,10 @@ export function getTextGenServer() { return TOGETHERAI_SERVER; } + if (settings.type === INFERMATICAI) { + return INFERMATICAI_SERVER; + } + return settings.server_urls[settings.type] ?? ''; } @@ -226,8 +233,8 @@ async function selectPreset(name) { function formatTextGenURL(value) { try { - // Mancer/Together doesn't need any formatting (it's hardcoded) - if (settings.type === MANCER || settings.type === TOGETHERAI) { + // Mancer/Together/InfermaticAI doesn't need any formatting (it's hardcoded) + if (settings.type === MANCER || settings.type === TOGETHERAI || settings.type === INFERMATICAI) { return value; } @@ -746,6 +753,10 @@ function getModel() { return settings.togetherai_model; } + if (settings.type === INFERMATICAI) { + return settings.infermaticai_model; + } + if (settings.type === APHRODITE) { return online_status; } diff --git a/src/additional-headers.js b/src/additional-headers.js index de8b0d91a..a78c71fd3 100644 --- a/src/additional-headers.js +++ b/src/additional-headers.js @@ -19,6 +19,14 @@ function getTogetherAIHeaders() { }) : {}; } +function getInfermaticAIHeaders() { + const apiKey = readSecret(SECRET_KEYS.INFERMATICAI); + + return apiKey ? ({ + 'Authorization': `Bearer ${apiKey}`, + }) : {}; +} + function getAphroditeHeaders() { const apiKey = readSecret(SECRET_KEYS.APHRODITE); @@ -69,6 +77,9 @@ function setAdditionalHeaders(request, args, server) { case TEXTGEN_TYPES.TOGETHERAI: headers = getTogetherAIHeaders(); break; + case TEXTGEN_TYPES.INFERMATICAI: + headers = getInfermaticAIHeaders(); + break; default: headers = server ? getOverrideHeaders((new URL(server))?.host) : {}; break; diff --git a/src/constants.js b/src/constants.js index 80b7585b2..13c1fc410 100644 --- a/src/constants.js +++ b/src/constants.js @@ -175,8 +175,17 @@ const TEXTGEN_TYPES = { TOGETHERAI: 'togetherai', LLAMACPP: 'llamacpp', OLLAMA: 'ollama', + INFERMATICAI: 'infermaticai', }; +const INFERMATICAI_KEYS = [ + 'model', + 'max_tokens', + 'temperature', + 'repetition_penalty', + 'stream', +]; + // https://docs.together.ai/reference/completions const TOGETHERAI_KEYS = [ 'model', @@ -223,4 +232,5 @@ module.exports = { AVATAR_HEIGHT, TOGETHERAI_KEYS, OLLAMA_KEYS, + INFERMATICAI_KEYS, }; diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index f1ef4004d..d4f44978b 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -4,7 +4,7 @@ const _ = require('lodash'); const Readable = require('stream').Readable; const { jsonParser } = require('../../express-common'); -const { TEXTGEN_TYPES, TOGETHERAI_KEYS, OLLAMA_KEYS } = require('../../constants'); +const { TEXTGEN_TYPES, TOGETHERAI_KEYS, OLLAMA_KEYS, INFERMATICAI_KEYS } = require('../../constants'); const { forwardFetchResponse, trimV1 } = require('../../util'); const { setAdditionalHeaders } = require('../../additional-headers'); @@ -117,6 +117,9 @@ router.post('/status', jsonParser, async function (request, response) { case TEXTGEN_TYPES.TOGETHERAI: url += '/api/models?&info'; break; + case TEXTGEN_TYPES.INFERMATICAI: + url += '/models'; + break; case TEXTGEN_TYPES.OLLAMA: url += '/api/tags'; break; @@ -243,6 +246,9 @@ router.post('/generate', jsonParser, async function (request, response) { case TEXTGEN_TYPES.OLLAMA: url += '/api/generate'; break; + case TEXTGEN_TYPES.INFERMATICAI: + url += '/completions'; + break; } } From 8075e4cd1e57769784d4bb8be4ff3093d86090a7 Mon Sep 17 00:00:00 2001 From: NWilson Date: Fri, 16 Feb 2024 09:07:06 -0600 Subject: [PATCH 02/19] Changes --- public/script.js | 4 ++-- public/scripts/RossAscends-mods.js | 6 +++--- public/scripts/textgen-models.js | 2 +- src/endpoints/backends/text-completions.js | 8 ++------ 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/public/script.js b/public/script.js index ae6c4a7ea..a2ee76a85 100644 --- a/public/script.js +++ b/public/script.js @@ -7658,8 +7658,8 @@ const CONNECT_API_MAP = { source: chat_completion_sources.CUSTOM, }, 'infermaticai': { - selected: 'openai', - button: '#api_button_openai', + selected: 'textgenerationwebui', + button: '#api_button_textgenerationwebui', type: textgen_types.INFERMATICAI, }, }; diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index d41608b82..8198de0e6 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -389,8 +389,9 @@ function RA_autoconnect(PrevApi) { } break; case 'textgenerationwebui': - if ((textgen_settings.type === textgen_types.MANCER && secret_state[SECRET_KEYS.MANCER]) || - (textgen_settings.type === textgen_types.TOGETHERAI && secret_state[SECRET_KEYS.TOGETHERAI]) + if ((textgen_settings.type === textgen_types.MANCER && secret_state[SECRET_KEYS.MANCER]) + || (textgen_settings.type === textgen_types.TOGETHERAI && secret_state[SECRET_KEYS.TOGETHERAI]) + || (textgen_settings.type === textgen_types.INFERMATICAI && secret_state[SECRET_KEYS.INFERMATICAI]) ) { $('#api_button_textgenerationwebui').trigger('click'); } @@ -407,7 +408,6 @@ function RA_autoconnect(PrevApi) { || (secret_state[SECRET_KEYS.AI21] && oai_settings.chat_completion_source == chat_completion_sources.AI21) || (secret_state[SECRET_KEYS.MAKERSUITE] && oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) || (secret_state[SECRET_KEYS.MISTRALAI] && oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI) - || (secret_state[SECRET_KEYS.INFERMATICAI] && oai_settings.chat_completion_source == chat_completion_sources.INFERMATICAI) || (isValidUrl(oai_settings.custom_url) && oai_settings.chat_completion_source == chat_completion_sources.CUSTOM) ) { $('#api_button_openai').trigger('click'); diff --git a/public/scripts/textgen-models.js b/public/scripts/textgen-models.js index ca9c53c77..d7612df5b 100644 --- a/public/scripts/textgen-models.js +++ b/public/scripts/textgen-models.js @@ -121,7 +121,7 @@ function onTogetherModelSelect() { function onInfermaticAIModelSelect() { const modelName = String($('#model_infermaticai_select').val()); textgen_settings.infermaticai_model = modelName; - $('#api_button_openai').trigger('click'); + $('#api_button_textgenerationwebui').trigger('click'); const model = infermaticAIModels.find(x => x.id === modelName); setGenerationParamsFromPreset({ max_length: model.context_length }); } diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index a831fe751..5d7aae35c 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -106,6 +106,7 @@ router.post('/status', jsonParser, async function (request, response) { case TEXTGEN_TYPES.APHRODITE: case TEXTGEN_TYPES.KOBOLDCPP: case TEXTGEN_TYPES.LLAMACPP: + case TEXTGEN_TYPES.INFERMATICAI: url += '/v1/models'; break; case TEXTGEN_TYPES.MANCER: @@ -117,9 +118,6 @@ router.post('/status', jsonParser, async function (request, response) { case TEXTGEN_TYPES.TOGETHERAI: url += '/api/models?&info'; break; - case TEXTGEN_TYPES.INFERMATICAI: - url += '/models'; - break; case TEXTGEN_TYPES.OLLAMA: url += '/api/tags'; break; @@ -235,6 +233,7 @@ router.post('/generate', jsonParser, async function (request, response) { case TEXTGEN_TYPES.TABBY: case TEXTGEN_TYPES.KOBOLDCPP: case TEXTGEN_TYPES.TOGETHERAI: + case TEXTGEN_TYPES.INFERMATICAI: url += '/v1/completions'; break; case TEXTGEN_TYPES.MANCER: @@ -246,9 +245,6 @@ router.post('/generate', jsonParser, async function (request, response) { case TEXTGEN_TYPES.OLLAMA: url += '/api/generate'; break; - case TEXTGEN_TYPES.INFERMATICAI: - url += '/completions'; - break; } } From c6c73fedad91ea30ae239d3fcb219c7f7b56e3b0 Mon Sep 17 00:00:00 2001 From: NWilson Date: Fri, 16 Feb 2024 10:23:26 -0600 Subject: [PATCH 03/19] Key Fix --- src/endpoints/secrets.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index 3014f3e69..76958f768 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -30,6 +30,7 @@ const SECRET_KEYS = { MISTRALAI: 'api_key_mistralai', CUSTOM: 'api_key_custom', OOBA: 'api_key_ooba', + INFERMATICAI: 'api_key_infermaticai', }; /** From 90d5fbc182a86653fcf99fff5e8e6b5d91f6f0cf Mon Sep 17 00:00:00 2001 From: NWilson Date: Mon, 19 Feb 2024 09:46:56 -0600 Subject: [PATCH 04/19] Fix non streaming --- src/endpoints/backends/text-completions.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index 5d7aae35c..ddd30fbb0 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -294,6 +294,11 @@ router.post('/generate', jsonParser, async function (request, response) { data['choices'] = [{ text }]; } + // Map InfermaticAI response to OAI completions format + if (request.body.api_type === TEXTGEN_TYPES.INFERMATICAI) { + data['choices'] = (data?.choices || []).map(choice => ({ text: choice.message.content })); + } + return response.send(data); } else { const text = await completionsReply.text(); From e55d9036138677c02a9080796cc5d201f28b9033 Mon Sep 17 00:00:00 2001 From: NWilson Date: Mon, 19 Feb 2024 09:53:26 -0600 Subject: [PATCH 05/19] Support more settings --- src/constants.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/constants.js b/src/constants.js index dda58fd0b..08ffdcd70 100644 --- a/src/constants.js +++ b/src/constants.js @@ -181,10 +181,14 @@ const TEXTGEN_TYPES = { const INFERMATICAI_KEYS = [ 'model', + 'prompt', 'max_tokens', 'temperature', + 'top_p', + 'top_k', 'repetition_penalty', 'stream', + 'stop', ]; // https://docs.together.ai/reference/completions From 1e7c2820da5468d1dd1ef93651b6b86dea22bc20 Mon Sep 17 00:00:00 2001 From: NWilson Date: Tue, 20 Feb 2024 08:12:59 -0600 Subject: [PATCH 06/19] Add InfermaticAI Profile --- default/content/index.json | 4 +++ .../content/presets/textgen/InfermaticAI.json | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 default/content/presets/textgen/InfermaticAI.json diff --git a/default/content/index.json b/default/content/index.json index d7345c84e..b8accc8e0 100644 --- a/default/content/index.json +++ b/default/content/index.json @@ -352,6 +352,10 @@ "filename": "presets/textgen/simple-proxy-for-tavern.json", "type": "textgen_preset" }, + { + "filename": "presets/textgen/InfermaticAI.json", + "type": "textgen_preset" + }, { "filename": "presets/openai/Default.json", "type": "openai_preset" diff --git a/default/content/presets/textgen/InfermaticAI.json b/default/content/presets/textgen/InfermaticAI.json new file mode 100644 index 000000000..0fb1bef7c --- /dev/null +++ b/default/content/presets/textgen/InfermaticAI.json @@ -0,0 +1,25 @@ +{ + "temp": 0.7, + "top_p": 0.95, + "top_k": 40, + "typical_p": 0.95, + "top_a": 0, + "tfs": 1, + "epsilon_cutoff": 0, + "eta_cutoff": 0, + "rep_pen": 1.2, + "rep_pen_range": 1024, + "no_repeat_ngram_size": 0, + "penalty_alpha": 0, + "num_beams": 1, + "length_penalty": 1, + "min_length": 0, + "encoder_rep_pen": 1, + "do_sample": true, + "early_stopping": false, + "mirostat_mode": 0, + "mirostat_tau": 5, + "mirostat_eta": 0.1, + "streaming": true, + "skip_special_tokens": true +} From 48b9eb854227f591c2661dd469903a96fa519035 Mon Sep 17 00:00:00 2001 From: NWilson Date: Tue, 20 Feb 2024 09:37:39 -0600 Subject: [PATCH 07/19] Revert "Add InfermaticAI Profile" This reverts commit 1e7c2820da5468d1dd1ef93651b6b86dea22bc20. --- default/content/index.json | 4 --- .../content/presets/textgen/InfermaticAI.json | 25 ------------------- 2 files changed, 29 deletions(-) delete mode 100644 default/content/presets/textgen/InfermaticAI.json diff --git a/default/content/index.json b/default/content/index.json index b8accc8e0..d7345c84e 100644 --- a/default/content/index.json +++ b/default/content/index.json @@ -352,10 +352,6 @@ "filename": "presets/textgen/simple-proxy-for-tavern.json", "type": "textgen_preset" }, - { - "filename": "presets/textgen/InfermaticAI.json", - "type": "textgen_preset" - }, { "filename": "presets/openai/Default.json", "type": "openai_preset" diff --git a/default/content/presets/textgen/InfermaticAI.json b/default/content/presets/textgen/InfermaticAI.json deleted file mode 100644 index 0fb1bef7c..000000000 --- a/default/content/presets/textgen/InfermaticAI.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "temp": 0.7, - "top_p": 0.95, - "top_k": 40, - "typical_p": 0.95, - "top_a": 0, - "tfs": 1, - "epsilon_cutoff": 0, - "eta_cutoff": 0, - "rep_pen": 1.2, - "rep_pen_range": 1024, - "no_repeat_ngram_size": 0, - "penalty_alpha": 0, - "num_beams": 1, - "length_penalty": 1, - "min_length": 0, - "encoder_rep_pen": 1, - "do_sample": true, - "early_stopping": false, - "mirostat_mode": 0, - "mirostat_tau": 5, - "mirostat_eta": 0.1, - "streaming": true, - "skip_special_tokens": true -} From 7c12c836f21880fc1178508c965a6b78660516b2 Mon Sep 17 00:00:00 2001 From: NWilson Date: Tue, 20 Feb 2024 09:40:35 -0600 Subject: [PATCH 08/19] Implement Key Filter --- src/endpoints/backends/text-completions.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index ddd30fbb0..443e743aa 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -263,6 +263,11 @@ router.post('/generate', jsonParser, async function (request, response) { args.body = JSON.stringify(request.body); } + if (request.body.api_type === TEXTGEN_TYPES.INFERMATICAI) { + request.body = _.pickBy(request.body, (_, key) => INFERMATICAI_KEYS.includes(key)); + args.body = JSON.stringify(request.body); + } + if (request.body.api_type === TEXTGEN_TYPES.OLLAMA) { args.body = JSON.stringify({ model: request.body.model, From eb89337f510b8a6ae7d64cb8dde1cf41842d73d3 Mon Sep 17 00:00:00 2001 From: berbant <33601955+berbant@users.noreply.github.com> Date: Thu, 22 Feb 2024 23:49:47 +0400 Subject: [PATCH 09/19] Update index.js --- public/scripts/extensions/translate/index.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/translate/index.js b/public/scripts/extensions/translate/index.js index fa3af0a2a..3e2f39d97 100644 --- a/public/scripts/extensions/translate/index.js +++ b/public/scripts/extensions/translate/index.js @@ -598,13 +598,18 @@ jQuery(() => { 'deeplx': 'http://127.0.0.1:1188/translate', }; const popupText = `

${optionText} API URL

Example: ${String(exampleURLs[extension_settings.translate.provider])}`; - const url = await callPopup(popupText, 'input'); + const provider_name = extension_settings.translate.provider + '_url' + const saved_url = ( Boolean(secret_state[provider_name]) ) ? await findSecret(provider_name) : ''; + + const url = await callPopup(popupText, 'input', saved_url); + if (url == false) { return; } + + await writeSecret(provider_name, url); - await writeSecret(extension_settings.translate.provider + '_url', url); toastr.success('API URL saved'); $('#translate_url_button').addClass('success'); }); From bc2010a762ad58510787bde9a4262de2125836df Mon Sep 17 00:00:00 2001 From: berbant <33601955+berbant@users.noreply.github.com> Date: Thu, 22 Feb 2024 23:55:57 +0400 Subject: [PATCH 10/19] Update secrets.js --- src/endpoints/secrets.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index 3014f3e69..4bf8687b4 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -213,12 +213,13 @@ router.post('/view', jsonParser, async (_, response) => { router.post('/find', jsonParser, (request, response) => { const allowKeysExposure = getConfigValue('allowKeysExposure', false); - if (!allowKeysExposure) { + const key = request.body.key; + + if (!allowKeysExposure && key.slice(key.length-4) !== '_url' ) { console.error('Cannot fetch secrets unless allowKeysExposure in config.yaml is set to true'); return response.sendStatus(403); } - const key = request.body.key; try { const secret = readSecret(key); From f82740a2385dc7c6525eaf90db84397c4f609583 Mon Sep 17 00:00:00 2001 From: NWilson Date: Thu, 22 Feb 2024 15:51:11 -0600 Subject: [PATCH 11/19] Change Non-streaming Handler --- src/endpoints/backends/text-completions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index 443e743aa..6c17883e5 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -300,7 +300,7 @@ router.post('/generate', jsonParser, async function (request, response) { } // Map InfermaticAI response to OAI completions format - if (request.body.api_type === TEXTGEN_TYPES.INFERMATICAI) { + if (completionsReply.url.includes('https://api.totalgpt.ai')) { data['choices'] = (data?.choices || []).map(choice => ({ text: choice.message.content })); } From 344b9eedbcc24da097858c89b29c799ff445913e Mon Sep 17 00:00:00 2001 From: Deciare <1689220+deciare@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:01:46 -0500 Subject: [PATCH 12/19] Request token probabilities from llama.cpp backend llama.cpp server token probabilities are given as values ranging from 0 to 1 instead of as logarithms. --- public/scripts/logprobs.js | 16 ++++++++++++---- public/scripts/textgen-settings.js | 11 ++++++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/public/scripts/logprobs.js b/public/scripts/logprobs.js index 2f0636a0e..255b10964 100644 --- a/public/scripts/logprobs.js +++ b/public/scripts/logprobs.js @@ -12,6 +12,7 @@ import { import { debounce, delay, getStringHash } from './utils.js'; import { decodeTextTokens, getTokenizerBestMatch } from './tokenizers.js'; import { power_user } from './power-user.js'; +import { textgenerationwebui_settings, textgen_types } from './textgen-settings.js'; const TINTS = 4; const MAX_MESSAGE_LOGPROBS = 100; @@ -139,9 +140,14 @@ function renderTopLogprobs() { const candidates = topLogprobs .sort(([, logA], [, logB]) => logB - logA) .map(([text, log]) => { - const probability = Math.exp(log); - sum += probability; - return [text, probability, log]; + if (textgenerationwebui_settings.type !== textgen_types.LLAMACPP) { + const probability = Math.exp(log); + sum += probability; + return [text, probability, log]; + } + else { + return [text, log, null]; + } }); candidates.push(['', 1 - sum, 0]); @@ -157,7 +163,9 @@ function renderTopLogprobs() { const tokenText = $('').text(`${toVisibleWhitespace(token)}`); const percentText = $('').text(`${(probability * 100).toFixed(2)}%`); container.append(tokenText, percentText); - container.attr('title', `logarithm: ${log}`); + if (log) { + container.attr('title', `logarithm: ${log}`); + } addKeyboardProps(container); if (token !== '') { container.click(() => onAlternativeClicked(state.selectedTokenLogprobs, token)); diff --git a/public/scripts/textgen-settings.js b/public/scripts/textgen-settings.js index b1a292e68..76262ed4e 100644 --- a/public/scripts/textgen-settings.js +++ b/public/scripts/textgen-settings.js @@ -694,7 +694,7 @@ async function generateTextGenWithStreaming(generate_data, signal) { } else { const newText = data?.choices?.[0]?.text || data?.content || ''; text += newText; - logprobs = parseTextgenLogprobs(newText, data.choices?.[0]?.logprobs); + logprobs = parseTextgenLogprobs(newText, data.choices?.[0]?.logprobs || data?.completion_probabilities); } yield { text, swipes, logprobs }; @@ -727,6 +727,14 @@ function parseTextgenLogprobs(token, logprobs) { const candidates = Object.entries(topLogprobs[0]); return { token, topLogprobs: candidates }; } + case LLAMACPP: { + /** @type {Record[]} */ + if (!logprobs?.length) { + return null; + } + const candidates = logprobs[0].probs.map(x => [ x.tok_str, x.prob ]); + return { token, topLogprobs: candidates }; + } default: return null; } @@ -867,6 +875,7 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate, 'n_predict': maxTokens, 'mirostat': settings.mirostat_mode, 'ignore_eos': settings.ban_eos_token, + 'n_probs': power_user.request_token_probabilities ? 10 : undefined, }; const aphroditeParams = { 'n': canMultiSwipe ? settings.n : 1, From 4baefeba68caa2282a2ad5792da12e3a1cf9c27b Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 23 Feb 2024 21:18:40 +0200 Subject: [PATCH 13/19] Extend per-entry scan depth limit, add warnings on overflow --- public/index.html | 2 +- public/scripts/world-info.js | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/public/index.html b/public/index.html index 642344aa0..56257737c 100644 --- a/public/index.html +++ b/public/index.html @@ -4539,7 +4539,7 @@
Scan Depth - +
Case-Sensitive diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 5b3edd40e..79a80eaa6 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -70,7 +70,7 @@ const SORT_ORDER_KEY = 'world_info_sort_order'; const METADATA_KEY = 'world_info'; const DEFAULT_DEPTH = 4; -const MAX_SCAN_DEPTH = 100; +const MAX_SCAN_DEPTH = 1000; /** * Represents a scanning buffer for one evaluation of World Info. @@ -1513,11 +1513,13 @@ function getWorldEntry(name, data, entry) { // Clamp if necessary if (value < 0) { $(this).val(0).trigger('input'); + toastr.warning('Scan depth cannot be negative'); return; } if (value > MAX_SCAN_DEPTH) { $(this).val(MAX_SCAN_DEPTH).trigger('input'); + toastr.warning(`Scan depth cannot exceed ${MAX_SCAN_DEPTH}`); return; } From 82c5042bad80504f42fcaff52a44697870acb1f8 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 23 Feb 2024 21:23:44 +0200 Subject: [PATCH 14/19] Prevent extra loop iterations on buffer init --- public/scripts/world-info.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 79a80eaa6..cb819a055 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -113,6 +113,10 @@ class WorldInfoBuffer { if (messages[depth]) { this.#depthBuffer[depth] = messages[depth].trim(); } + // break if last message is reached + if (depth === messages.length - 1) { + break; + } } } From cb536a7611c56fde5ccb68ef3f40b8a68058a9ac Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 23 Feb 2024 21:41:54 +0200 Subject: [PATCH 15/19] Save a list of safe to export secret keys --- public/scripts/extensions/translate/index.js | 16 ++++++++-------- public/scripts/secrets.js | 5 +++++ src/endpoints/secrets.js | 14 ++++++++++---- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/public/scripts/extensions/translate/index.js b/public/scripts/extensions/translate/index.js index 3e2f39d97..a707e2e7e 100644 --- a/public/scripts/extensions/translate/index.js +++ b/public/scripts/extensions/translate/index.js @@ -11,7 +11,7 @@ import { updateMessageBlock, } from '../../../script.js'; import { extension_settings, getContext } from '../../extensions.js'; -import { secret_state, writeSecret } from '../../secrets.js'; +import { findSecret, secret_state, writeSecret } from '../../secrets.js'; import { splitRecursive } from '../../utils.js'; export const autoModeOptions = { @@ -599,16 +599,16 @@ jQuery(() => { }; const popupText = `

${optionText} API URL

Example: ${String(exampleURLs[extension_settings.translate.provider])}`; - const provider_name = extension_settings.translate.provider + '_url' - const saved_url = ( Boolean(secret_state[provider_name]) ) ? await findSecret(provider_name) : ''; + const secretKey = extension_settings.translate.provider + '_url'; + const savedUrl = secret_state[secretKey] ? await findSecret(secretKey) : ''; - const url = await callPopup(popupText, 'input', saved_url); - - if (url == false) { + const url = await callPopup(popupText, 'input', savedUrl); + + if (url == false || url == '') { return; } - - await writeSecret(provider_name, url); + + await writeSecret(secretKey, url); toastr.success('API URL saved'); $('#translate_url_button').addClass('success'); diff --git a/public/scripts/secrets.js b/public/scripts/secrets.js index 7fb1ee464..e2b8525e9 100644 --- a/public/scripts/secrets.js +++ b/public/scripts/secrets.js @@ -124,6 +124,11 @@ export async function readSecretState() { } } +/** + * Finds a secret value by key. + * @param {string} key Secret key + * @returns {Promise} Secret value, or undefined if keys are not exposed + */ export async function findSecret(key) { try { const response = await fetch('/api/secrets/find', { diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index 4bf8687b4..2ea0611da 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -32,6 +32,14 @@ const SECRET_KEYS = { OOBA: 'api_key_ooba', }; +// These are the keys that are safe to expose, even if allowKeysExposure is false +const EXPORTABLE_KEYS = [ + SECRET_KEYS.LIBRE_URL, + SECRET_KEYS.LINGVA_URL, + SECRET_KEYS.ONERING_URL, + SECRET_KEYS.DEEPLX_URL, +]; + /** * Writes a secret to the secrets file * @param {string} key Secret key @@ -212,15 +220,13 @@ router.post('/view', jsonParser, async (_, response) => { router.post('/find', jsonParser, (request, response) => { const allowKeysExposure = getConfigValue('allowKeysExposure', false); - const key = request.body.key; - - if (!allowKeysExposure && key.slice(key.length-4) !== '_url' ) { + + if (!allowKeysExposure && !EXPORTABLE_KEYS.includes(key)) { console.error('Cannot fetch secrets unless allowKeysExposure in config.yaml is set to true'); return response.sendStatus(403); } - try { const secret = readSecret(key); From 737a0bd3ae94966df073675893946c6abe1d691e Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 23 Feb 2024 22:37:00 +0200 Subject: [PATCH 16/19] Fix purge extras and mistral vectors --- public/scripts/extensions/vectors/index.js | 17 +++++++++++++---- src/endpoints/vectors.js | 6 ++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 5821a7a30..e318d6518 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -530,10 +530,15 @@ async function queryCollection(collectionId, searchText, topK) { return results; } +/** + * Purges the vector index for a collection. + * @param {string} collectionId Collection ID to purge + * @returns > True if deleted, false if not + */ async function purgeVectorIndex(collectionId) { try { if (!settings.enabled_chats) { - return; + return true; } const response = await fetch('/api/vector/purge', { @@ -549,9 +554,10 @@ async function purgeVectorIndex(collectionId) { } console.log(`Vectors: Purged vector index for collection ${collectionId}`); - + return true; } catch (error) { console.error('Vectors: Failed to purge', error); + return false; } } @@ -566,8 +572,11 @@ async function onPurgeClick() { toastr.info('No chat selected', 'Purge aborted'); return; } - await purgeVectorIndex(chatId); - toastr.success('Vector index purged', 'Purge successful'); + if (await purgeVectorIndex(chatId)) { + toastr.success('Vector index purged', 'Purge successful'); + } else { + toastr.error('Failed to purge vector index', 'Purge failed'); + } } async function onViewStatsClick() { diff --git a/src/endpoints/vectors.js b/src/endpoints/vectors.js index e50cc0bcf..b2ff9e2d0 100644 --- a/src/endpoints/vectors.js +++ b/src/endpoints/vectors.js @@ -4,6 +4,9 @@ const express = require('express'); const sanitize = require('sanitize-filename'); const { jsonParser } = require('../express-common'); +// Don't forget to add new sources to the SOURCES array +const SOURCES = ['transformers', 'mistral', 'openai', 'extras', 'palm']; + /** * Gets the vector for the given text from the given source. * @param {string} source - The source of the vector @@ -261,8 +264,7 @@ router.post('/purge', jsonParser, async (req, res) => { const collectionId = String(req.body.collectionId); - const sources = ['transformers', 'openai', 'palm']; - for (const source of sources) { + for (const source of SOURCES) { const index = await getIndex(collectionId, source, false); const exists = await index.isIndexCreated(); From 445cbda02f39aef1736efec3b1928a68b7e11120 Mon Sep 17 00:00:00 2001 From: Deciare <1689220+deciare@users.noreply.github.com> Date: Sat, 24 Feb 2024 00:13:00 -0500 Subject: [PATCH 17/19] If token probability is a logarithm it'll be < 0 No need to read settings to find out if llama.cpp backend is in use... --- public/scripts/logprobs.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/public/scripts/logprobs.js b/public/scripts/logprobs.js index 255b10964..683d8fa1a 100644 --- a/public/scripts/logprobs.js +++ b/public/scripts/logprobs.js @@ -12,7 +12,6 @@ import { import { debounce, delay, getStringHash } from './utils.js'; import { decodeTextTokens, getTokenizerBestMatch } from './tokenizers.js'; import { power_user } from './power-user.js'; -import { textgenerationwebui_settings, textgen_types } from './textgen-settings.js'; const TINTS = 4; const MAX_MESSAGE_LOGPROBS = 100; @@ -140,7 +139,7 @@ function renderTopLogprobs() { const candidates = topLogprobs .sort(([, logA], [, logB]) => logB - logA) .map(([text, log]) => { - if (textgenerationwebui_settings.type !== textgen_types.LLAMACPP) { + if (log < 0) { const probability = Math.exp(log); sum += probability; return [text, probability, log]; From 9287ff18decaab914f3a3f09711ab142c3d2a830 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 24 Feb 2024 14:50:06 +0200 Subject: [PATCH 18/19] Fix for non-streaming --- public/script.js | 6 ++++++ public/scripts/textgen-settings.js | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/public/script.js b/public/script.js index a17dd748c..9f9f82a5c 100644 --- a/public/script.js +++ b/public/script.js @@ -18,6 +18,7 @@ import { textgen_types, getTextGenServer, validateTextGenUrl, + parseTextgenLogprobs, } from './scripts/textgen-settings.js'; const { MANCER, TOGETHERAI, OOBA, APHRODITE, OLLAMA } = textgen_types; @@ -4478,6 +4479,11 @@ function parseAndSaveLogprobs(data, continueFrom) { // `sendOpenAIRequest`. `data` for these APIs is just a string with // the text of the generated message, logprobs are not included. return; + case 'textgenerationwebui': + if (textgen_settings.type === textgen_types.LLAMACPP) { + logprobs = data?.completion_probabilities?.map(x => parseTextgenLogprobs(x.content, [x])) || null; + } + break; default: return; } diff --git a/public/scripts/textgen-settings.js b/public/scripts/textgen-settings.js index 793081665..910a03615 100644 --- a/public/scripts/textgen-settings.js +++ b/public/scripts/textgen-settings.js @@ -764,7 +764,7 @@ async function generateTextGenWithStreaming(generate_data, signal) { * @param {Object} logprobs - logprobs object returned from the API * @returns {import('logprobs.js').TokenLogprobs | null} - converted logprobs */ -function parseTextgenLogprobs(token, logprobs) { +export function parseTextgenLogprobs(token, logprobs) { if (!logprobs) { return null; } From eaadfea63915d1f10eeaba9249399406f4ace69c Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 24 Feb 2024 15:03:57 +0200 Subject: [PATCH 19/19] Extend debounce duration of logprobs renderer --- public/script.js | 2 +- public/scripts/logprobs.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/public/script.js b/public/script.js index 9f9f82a5c..f01903faf 100644 --- a/public/script.js +++ b/public/script.js @@ -2666,8 +2666,8 @@ class StreamingProcessor { } const continueMsg = this.type === 'continue' ? this.messageAlreadyGenerated : undefined; - await saveChatConditional(); saveLogprobsForActiveMessage(this.messageLogprobs.filter(Boolean), continueMsg); + await saveChatConditional(); activateSendButtons(); showSwipeButtons(); setGenerationProgress(0); diff --git a/public/scripts/logprobs.js b/public/scripts/logprobs.js index 683d8fa1a..a066ea979 100644 --- a/public/scripts/logprobs.js +++ b/public/scripts/logprobs.js @@ -466,7 +466,7 @@ function convertTokenIdLogprobsToText(input) { } export function initLogprobs() { - const debouncedRender = debounce(renderAlternativeTokensView, 250); + const debouncedRender = debounce(renderAlternativeTokensView, 500); $('#logprobsViewerClose').click(onToggleLogprobsPanel); $('#option_toggle_logprobs').click(onToggleLogprobsPanel); eventSource.on(event_types.CHAT_CHANGED, debouncedRender);