From bd5592de7b341b49463fdb9fcf1452a4876852b1 Mon Sep 17 00:00:00 2001 From: DarokCx <77368869+DarokCx@users.noreply.github.com> Date: Thu, 27 Jun 2024 09:06:11 -0400 Subject: [PATCH 1/5] Added featherless, connect button not working --- public/index.html | 34 ++++++++++++++++ public/script.js | 7 +++- public/scripts/secrets.js | 1 + public/scripts/textgen-models.js | 31 +++++++++++++++ public/scripts/textgen-settings.js | 7 +++- src/constants.js | 45 ++++++++++++++++++++++ src/endpoints/backends/text-completions.js | 5 ++- 7 files changed, 126 insertions(+), 4 deletions(-) diff --git a/public/index.html b/public/index.html index 4c2ccdfcd..30ad41652 100644 --- a/public/index.html +++ b/public/index.html @@ -2038,6 +2038,7 @@ +
@@ -2182,6 +2183,39 @@
+ + + +
+
+ + featherless.ai + + +
+

API key

+
+ +
+
+
+
+ For privacy reasons, your API key will be hidden after you reload the page. +
+
+

Server URL

+ Try: Api.Featherless.ai/v1 + +
+ +
+ + + + + + +
diff --git a/public/script.js b/public/script.js index a9526a7a8..76c18a23b 100644 --- a/public/script.js +++ b/public/script.js @@ -22,7 +22,7 @@ import { parseTabbyLogprobs, } from './scripts/textgen-settings.js'; -const { MANCER, TOGETHERAI, OOBA, VLLM, APHRODITE, TABBY, OLLAMA, INFERMATICAI, DREAMGEN, OPENROUTER } = textgen_types; +const { MANCER, TOGETHERAI, OOBA, VLLM, APHRODITE, TABBY, OLLAMA, INFERMATICAI, DREAMGEN, OPENROUTER, FEATHERLESS} = textgen_types; import { world_info, @@ -222,7 +222,7 @@ import { import { getBackgrounds, initBackgrounds, loadBackgroundSettings, background_settings } from './scripts/backgrounds.js'; import { hideLoader, showLoader } from './scripts/loader.js'; import { BulkEditOverlay, CharacterContextMenu } from './scripts/BulkEditOverlay.js'; -import { loadMancerModels, loadOllamaModels, loadTogetherAIModels, loadInfermaticAIModels, loadOpenRouterModels, loadVllmModels, loadAphroditeModels, loadDreamGenModels } from './scripts/textgen-models.js'; +import { loadFeatherlessModels, loadMancerModels, loadOllamaModels, loadTogetherAIModels, loadInfermaticAIModels, loadOpenRouterModels, loadVllmModels, loadAphroditeModels, loadDreamGenModels } from './scripts/textgen-models.js'; import { appendFileContent, hasPendingFileAttachment, populateFileAttachment, decodeStyleTags, encodeStyleTags, isExternalMediaAllowed, getCurrentEntityId } from './scripts/chats.js'; import { initPresetManager } from './scripts/preset-manager.js'; import { evaluateMacros } from './scripts/macros.js'; @@ -1122,6 +1122,8 @@ async function getStatusTextgen() { } else if (textgen_settings.type === APHRODITE) { loadAphroditeModels(data?.data); online_status = textgen_settings.aphrodite_model; + } else if (textgen_settings.type === FEATHERLESS) { + loadFeatherlessModels(data?.data); } else { online_status = data?.result; } @@ -9322,6 +9324,7 @@ jQuery(async function () { { id: 'api_key_openrouter-tg', secret: SECRET_KEYS.OPENROUTER }, { id: 'api_key_koboldcpp', secret: SECRET_KEYS.KOBOLDCPP }, { id: 'api_key_llamacpp', secret: SECRET_KEYS.LLAMACPP }, + { id: 'api_key_featherless', secret: SECRET_KEYS.FEATHERLESS }, ]; for (const key of keys) { diff --git a/public/scripts/secrets.js b/public/scripts/secrets.js index cb4477a78..f9f3ec86d 100644 --- a/public/scripts/secrets.js +++ b/public/scripts/secrets.js @@ -28,6 +28,7 @@ export const SECRET_KEYS = { PERPLEXITY: 'api_key_perplexity', GROQ: 'api_key_groq', AZURE_TTS: 'api_key_azure_tts', + FEATHERLESS: 'api_key_featherless', }; const INPUT_MAP = { diff --git a/public/scripts/textgen-models.js b/public/scripts/textgen-models.js index d8f36cf45..5e5d69338 100644 --- a/public/scripts/textgen-models.js +++ b/public/scripts/textgen-models.js @@ -9,6 +9,7 @@ let infermaticAIModels = []; let dreamGenModels = []; let vllmModels = []; let aphroditeModels = []; +let featherlessModels = []; export let openRouterModels = []; /** @@ -231,6 +232,35 @@ export async function loadAphroditeModels(data) { } } +export async function loadFeatherlessModels(data) { + if (!Array.isArray(data)) { + console.error('Invalid Featherless models data', data); + return; + } + + featherlessModels = data; + + if (!data.find(x => x.id === textgen_settings.featherless_model)) { + textgen_settings.featherless_model = data[0]?.id || ''; + } + + $('#featherless_model').empty(); + for (const model of data) { + const option = document.createElement('option'); + option.value = model.id; + option.text = model.id; + option.selected = model.id === textgen_settings.featherless_model; + $('#featherless_model').append(option); + } +} + +function onFeatherlessModelSelect() { + const modelId = String($('#featherless_model').val()); + textgen_settings.featherless_model = modelId; + $('#api_button_textgenerationwebui').trigger('click'); +} + + function onMancerModelSelect() { const modelId = String($('#mancer_model').val()); textgen_settings.mancer_model = modelId; @@ -505,6 +535,7 @@ jQuery(function () { $('#ollama_download_model').on('click', downloadOllamaModel); $('#vllm_model').on('change', onVllmModelSelect); $('#aphrodite_model').on('change', onAphroditeModelSelect); + $('#featherless_model').on('change', onFeatherlessModelSelect); const providersSelect = $('.openrouter_providers'); for (const provider of OPENROUTER_PROVIDERS) { diff --git a/public/scripts/textgen-settings.js b/public/scripts/textgen-settings.js index dac12be1c..e97c62f17 100644 --- a/public/scripts/textgen-settings.js +++ b/public/scripts/textgen-settings.js @@ -38,9 +38,10 @@ export const textgen_types = { INFERMATICAI: 'infermaticai', DREAMGEN: 'dreamgen', OPENROUTER: 'openrouter', + FEATHERLESS: 'featherless', }; -const { MANCER, VLLM, APHRODITE, TABBY, TOGETHERAI, OOBA, OLLAMA, LLAMACPP, INFERMATICAI, DREAMGEN, OPENROUTER, KOBOLDCPP } = textgen_types; +const { MANCER, VLLM, APHRODITE, TABBY, TOGETHERAI, OOBA, OLLAMA, LLAMACPP, INFERMATICAI, DREAMGEN, OPENROUTER, KOBOLDCPP, FEATHERLESS } = textgen_types; const LLAMACPP_DEFAULT_ORDER = [ 'top_k', @@ -75,6 +76,7 @@ let TOGETHERAI_SERVER = 'https://api.together.xyz'; let INFERMATICAI_SERVER = 'https://api.totalgpt.ai'; let DREAMGEN_SERVER = 'https://dreamgen.com'; let OPENROUTER_SERVER = 'https://openrouter.ai/api'; +let FEATHERLESS_SERVER = 'https://api.featherless.ai/v1'; const SERVER_INPUTS = { [textgen_types.OOBA]: '#textgenerationwebui_api_url_text', @@ -84,6 +86,7 @@ const SERVER_INPUTS = { [textgen_types.KOBOLDCPP]: '#koboldcpp_api_url_text', [textgen_types.LLAMACPP]: '#llamacpp_api_url_text', [textgen_types.OLLAMA]: '#ollama_api_url_text', + [textgen_types.FEATHERLESS]: '#featherless_api_url_text', }; const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5]; @@ -265,6 +268,8 @@ export function validateTextGenUrl() { export function getTextGenServer() { switch (settings.type) { + case FEATHERLESS: + return FEATHERLESS_SERVER; case MANCER: return MANCER_SERVER; case TOGETHERAI: diff --git a/src/constants.js b/src/constants.js index b4945fa6f..e3556c97b 100644 --- a/src/constants.js +++ b/src/constants.js @@ -212,6 +212,7 @@ const TEXTGEN_TYPES = { INFERMATICAI: 'infermaticai', DREAMGEN: 'dreamgen', OPENROUTER: 'openrouter', + FEATHERLESS: 'featherless', }; const INFERMATICAI_KEYS = [ @@ -226,6 +227,49 @@ const INFERMATICAI_KEYS = [ 'stop', ]; +const FEATHERLESS_KEYS = [ + 'model', + 'prompt', + 'best_of', + 'echo', + 'frequency_penalty', + 'logit_bias', + 'logprobs', + 'max_tokens', + 'n', + 'presence_penalty', + 'seed', + 'stop', + 'stream', + 'suffix', + 'temperature', + 'top_p', + 'user', + + 'use_beam_search', + 'top_k', + 'min_p', + 'repetition_penalty', + 'length_penalty', + 'early_stopping', + 'stop_token_ids', + 'ignore_eos', + 'min_tokens', + 'skip_special_tokens', + 'spaces_between_special_tokens', + 'truncate_prompt_tokens', + + 'include_stop_str_in_output', + 'response_format', + 'guided_json', + 'guided_regex', + 'guided_choice', + 'guided_grammar', + 'guided_decoding_backend', + 'guided_whitespace_pattern', +]; + + // https://dreamgen.com/docs/api#openai-text const DREAMGEN_KEYS = [ 'model', @@ -366,4 +410,5 @@ module.exports = { OPENROUTER_HEADERS, OPENROUTER_KEYS, VLLM_KEYS, + FEATHERLESS_KEYS, }; diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index 49324e443..aeff56258 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -126,6 +126,9 @@ router.post('/status', jsonParser, async function (request, response) { case TEXTGEN_TYPES.OLLAMA: url += '/api/tags'; break; + case TEXTGEN_TYPES.FEATHERLESS: + url += '/v1/models'; + break; } } @@ -135,7 +138,7 @@ router.post('/status', jsonParser, async function (request, response) { console.log('Models endpoint is offline.'); return response.status(400); } - + console.log("url for models", url) let data = await modelsReply.json(); if (request.body.legacy_api) { From 8608bc92ae205b96ca9ee512d7bf95744e24ed86 Mon Sep 17 00:00:00 2001 From: DarokCx <77368869+DarokCx@users.noreply.github.com> Date: Thu, 27 Jun 2024 10:02:28 -0400 Subject: [PATCH 2/5] no authorization --- public/script.js | 1 + src/endpoints/backends/text-completions.js | 8 +++++++- src/endpoints/secrets.js | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/public/script.js b/public/script.js index 76c18a23b..995dd3e2e 100644 --- a/public/script.js +++ b/public/script.js @@ -1124,6 +1124,7 @@ async function getStatusTextgen() { online_status = textgen_settings.aphrodite_model; } else if (textgen_settings.type === FEATHERLESS) { loadFeatherlessModels(data?.data); + online_status = textgen_settings.featherless_model; } else { online_status = data?.result; } diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index aeff56258..782819058 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -4,7 +4,7 @@ const _ = require('lodash'); const Readable = require('stream').Readable; const { jsonParser } = require('../../express-common'); -const { TEXTGEN_TYPES, TOGETHERAI_KEYS, OLLAMA_KEYS, INFERMATICAI_KEYS, OPENROUTER_KEYS, VLLM_KEYS, DREAMGEN_KEYS } = require('../../constants'); +const { TEXTGEN_TYPES, TOGETHERAI_KEYS, OLLAMA_KEYS, INFERMATICAI_KEYS, OPENROUTER_KEYS, VLLM_KEYS, DREAMGEN_KEYS, FEATHERLESS_KEYS } = require('../../constants'); const { forwardFetchResponse, trimV1 } = require('../../util'); const { setAdditionalHeaders } = require('../../additional-headers'); @@ -238,6 +238,7 @@ router.post('/generate', jsonParser, async function (request, response) { } else { switch (request.body.api_type) { case TEXTGEN_TYPES.VLLM: + case TEXTGEN_TYPES.FEATHERLESS: case TEXTGEN_TYPES.APHRODITE: case TEXTGEN_TYPES.OOBA: case TEXTGEN_TYPES.TABBY: @@ -284,6 +285,11 @@ router.post('/generate', jsonParser, async function (request, response) { args.body = JSON.stringify(request.body); } + if (request.body.api_type === TEXTGEN_TYPES.FEATHERLESS) { + request.body = _.pickBy(request.body, (_, key) => FEATHERLESS_KEYS.includes(key)); + args.body = JSON.stringify(request.body); + } + if (request.body.api_type === TEXTGEN_TYPES.DREAMGEN) { request.body = _.pickBy(request.body, (_, key) => DREAMGEN_KEYS.includes(key)); // NOTE: DreamGen sometimes get confused by the unusual formatting in the character cards. diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index 9bf2eb765..df976df67 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -4,6 +4,7 @@ const express = require('express'); const { getConfigValue } = require('../util'); const writeFileAtomicSync = require('write-file-atomic').sync; const { jsonParser } = require('../express-common'); +const { FEATHERLESS_KEYS } = require('../constants'); const SECRETS_FILE = 'secrets.json'; const SECRET_KEYS = { @@ -40,6 +41,7 @@ const SECRET_KEYS = { PERPLEXITY: 'api_key_perplexity', GROQ: 'api_key_groq', AZURE_TTS: 'api_key_azure_tts', + FEATHERLESS: 'api_key_featherless', }; // These are the keys that are safe to expose, even if allowKeysExposure is false From 29ff0876a793ff27d9870d2d0fdf7ef2d9cf1827 Mon Sep 17 00:00:00 2001 From: DarokCx <77368869+DarokCx@users.noreply.github.com> Date: Fri, 28 Jun 2024 08:20:15 -0400 Subject: [PATCH 3/5] Added additional headers --- public/scripts/RossAscends-mods.js | 1 + public/scripts/preset-manager.js | 1 + public/scripts/secrets.js | 1 + public/scripts/textgen-settings.js | 2 ++ src/additional-headers.js | 9 +++++++++ 5 files changed, 14 insertions(+) diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index 3166b4afa..db03da327 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -360,6 +360,7 @@ function RA_autoconnect(PrevApi) { || (textgen_settings.type === textgen_types.INFERMATICAI && secret_state[SECRET_KEYS.INFERMATICAI]) || (textgen_settings.type === textgen_types.DREAMGEN && secret_state[SECRET_KEYS.DREAMGEN]) || (textgen_settings.type === textgen_types.OPENROUTER && secret_state[SECRET_KEYS.OPENROUTER]) + || (textgen_settings.type === textgen_types.FEATHERLESS && secret_state[SECRET_KEYS.FEATHERLESS]) ) { $('#api_button_textgenerationwebui').trigger('click'); } diff --git a/public/scripts/preset-manager.js b/public/scripts/preset-manager.js index 42cc7585a..94eff6971 100644 --- a/public/scripts/preset-manager.js +++ b/public/scripts/preset-manager.js @@ -322,6 +322,7 @@ class PresetManager { 'infermaticai_model', 'dreamgen_model', 'openrouter_model', + 'featherless_model', 'max_tokens_second', 'openrouter_providers', ]; diff --git a/public/scripts/secrets.js b/public/scripts/secrets.js index f9f3ec86d..ee6635791 100644 --- a/public/scripts/secrets.js +++ b/public/scripts/secrets.js @@ -57,6 +57,7 @@ const INPUT_MAP = { [SECRET_KEYS.COHERE]: '#api_key_cohere', [SECRET_KEYS.PERPLEXITY]: '#api_key_perplexity', [SECRET_KEYS.GROQ]: '#api_key_groq', + [SECRET_KEYS.FEATHERLESS]: '#api_key_featherless', }; async function clearSecret() { diff --git a/public/scripts/textgen-settings.js b/public/scripts/textgen-settings.js index e97c62f17..f33c38e9e 100644 --- a/public/scripts/textgen-settings.js +++ b/public/scripts/textgen-settings.js @@ -1013,6 +1013,8 @@ export function getTextGenModel() { throw new Error('No Ollama model selected'); } return settings.ollama_model; + case FEATHERLESS: + return settings.featherless_model; default: return undefined; } diff --git a/src/additional-headers.js b/src/additional-headers.js index b8a44b390..c11aeb719 100644 --- a/src/additional-headers.js +++ b/src/additional-headers.js @@ -147,6 +147,14 @@ function getKoboldCppHeaders(directories) { }) : {}; } +function getFeatherlessHeaders(directories) { + const apiKey = readSecret(directories, SECRET_KEYS.FEATHERLESS); + + return apiKey ? ({ + 'Authorization': `Bearer ${apiKey}`, + }) : {}; +} + function getOverrideHeaders(urlHost) { const requestOverrides = getConfigValue('requestOverrides', []); const overrideHeaders = requestOverrides?.find((e) => e.hosts?.includes(urlHost))?.headers; @@ -187,6 +195,7 @@ function setAdditionalHeadersByType(requestHeaders, type, server, directories) { [TEXTGEN_TYPES.OPENROUTER]: getOpenRouterHeaders, [TEXTGEN_TYPES.KOBOLDCPP]: getKoboldCppHeaders, [TEXTGEN_TYPES.LLAMACPP]: getLlamaCppHeaders, + [TEXTGEN_TYPES.FEATHERLESS]: getFeatherlessHeaders, }; const getHeaders = headerGetters[type]; From 4e33253a91bc3c708c0033dd9d99d97c23f54941 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 28 Jun 2024 19:12:56 +0300 Subject: [PATCH 4/5] Code clean-up --- public/index.html | 26 ++++++---------------- src/endpoints/backends/text-completions.js | 2 +- src/endpoints/secrets.js | 1 - 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/public/index.html b/public/index.html index 7827c3632..ad52d580f 100644 --- a/public/index.html +++ b/public/index.html @@ -2029,6 +2029,7 @@ + @@ -2039,7 +2040,6 @@ -
@@ -2184,15 +2184,11 @@
- - -
- + featherless.ai -

API key

@@ -2203,20 +2199,12 @@
For privacy reasons, your API key will be hidden after you reload the page.
-
-

Server URL

- Try: Api.Featherless.ai/v1 - -
- +
- - - - - - -
diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index 62ebb556d..68fa9ce14 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -142,7 +142,7 @@ router.post('/status', jsonParser, async function (request, response) { console.log('Models endpoint is offline.'); return response.status(400); } - console.log('url for models', url); + let data = await modelsReply.json(); if (request.body.legacy_api) { diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index 93597e7b9..dedb23096 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -4,7 +4,6 @@ const express = require('express'); const { getConfigValue } = require('../util'); const writeFileAtomicSync = require('write-file-atomic').sync; const { jsonParser } = require('../express-common'); -const { FEATHERLESS_KEYS } = require('../constants'); const SECRETS_FILE = 'secrets.json'; const SECRET_KEYS = { From cc9eca8427bfd261297c6c1c6c75310b833a810d Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 28 Jun 2024 19:13:46 +0300 Subject: [PATCH 5/5] Apply select2 to model selection --- public/scripts/textgen-models.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/scripts/textgen-models.js b/public/scripts/textgen-models.js index 371ac9d9d..a33c0d542 100644 --- a/public/scripts/textgen-models.js +++ b/public/scripts/textgen-models.js @@ -603,6 +603,12 @@ jQuery(function () { width: '100%', templateResult: getAphroditeModelTemplate, }); + $('#featherless_model').select2({ + placeholder: 'Select a model', + searchInputPlaceholder: 'Search models...', + searchInputCssClass: 'text_pole', + width: '100%', + }); providersSelect.select2({ sorter: data => data.sort((a, b) => a.text.localeCompare(b.text)), placeholder: 'Select providers. No selection = all providers.',