rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1437 from valadaptive/textgen-refactor

Browse Source 
Clean up some textgen API code
This commit is contained in:
Cohee
2023-12-04 00:15:57 +02:00
committed by GitHub
parent 8a1ead531c 0a092629ce
commit f7417e9fb3
8 changed files with 199 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
public/script.js
View File

@@ -11,22 +11,19 @@ import {
} from './scripts/kai-settings.js'; } from './scripts/kai-settings.js';
import { import {
textgenerationwebui_settings, textgenerationwebui_settings as textgen_settings,
loadTextGenSettings, loadTextGenSettings,
generateTextGenWithStreaming, generateTextGenWithStreaming,
getTextGenGenerationData, getTextGenGenerationData,
formatTextGenURL, formatTextGenURL,
getTextGenUrlSourceId, getTextGenUrlSourceId,
isMancer,
isAphrodite,
isTabby,
textgen_types, textgen_types,
textgenerationwebui_banned_in_macros, textgenerationwebui_banned_in_macros,
isOoba,
MANCER_SERVER, MANCER_SERVER,
isKoboldCpp,
} from './scripts/textgen-settings.js'; } from './scripts/textgen-settings.js';
const { MANCER } = textgen_types;
import { import {
world_info, world_info,
getWorldInfoPrompt, getWorldInfoPrompt,
@@ -885,22 +882,21 @@ async function getStatus() {
method: 'POST', method: 'POST',
headers: getRequestHeaders(), headers: getRequestHeaders(),
body: JSON.stringify({ body: JSON.stringify({
main_api: main_api, main_api,
api_server: endpoint, api_server: endpoint,
use_mancer: main_api == 'textgenerationwebui' ? isMancer() : false, api_type: textgen_settings.type,
use_aphrodite: main_api == 'textgenerationwebui' ? isAphrodite() : false, legacy_api: main_api == 'textgenerationwebui' ?
use_ooba: main_api == 'textgenerationwebui' ? isOoba() : false, textgen_settings.legacy_api &&
use_tabby: main_api == 'textgenerationwebui' ? isTabby() : false, textgen_settings.type !== MANCER :
use_koboldcpp: main_api == 'textgenerationwebui' ? isKoboldCpp() : false, false,
legacy_api: main_api == 'textgenerationwebui' ? textgenerationwebui_settings.legacy_api && !isMancer() : false,
}), }),
signal: abortStatusCheck.signal, signal: abortStatusCheck.signal,
}); });
const data = await response.json(); const data = await response.json();
if (main_api == 'textgenerationwebui' && isMancer()) { if (main_api == 'textgenerationwebui' && textgen_settings.type === MANCER) {
online_status = textgenerationwebui_settings.mancer_model; online_status = textgen_settings.mancer_model;
loadMancerModels(data?.data); loadMancerModels(data?.data);
} else { } else {
online_status = data?.result; online_status = data?.result;
@@ -947,7 +943,7 @@ export function resultCheckStatus() {
export function getAPIServerUrl() { export function getAPIServerUrl() {
if (main_api == 'textgenerationwebui') { if (main_api == 'textgenerationwebui') {
if (isMancer()) { if (textgen_settings.type === MANCER) {
return MANCER_SERVER; return MANCER_SERVER;
} }
@@ -2471,7 +2467,7 @@ function isStreamingEnabled() {
return ((main_api == 'openai' && oai_settings.stream_openai && !noStreamSources.includes(oai_settings.chat_completion_source)) return ((main_api == 'openai' && oai_settings.stream_openai && !noStreamSources.includes(oai_settings.chat_completion_source))
|| (main_api == 'kobold' && kai_settings.streaming_kobold && kai_flags.can_use_streaming) || (main_api == 'kobold' && kai_settings.streaming_kobold && kai_flags.can_use_streaming)
|| (main_api == 'novel' && nai_settings.streaming_novel) || (main_api == 'novel' && nai_settings.streaming_novel)
|| (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming)); || (main_api == 'textgenerationwebui' && textgen_settings.streaming));
} }
function showStopButton() { function showStopButton() {
@@ -2851,7 +2847,10 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
return; return;
} }
if (main_api === 'textgenerationwebui' && textgenerationwebui_settings.streaming && textgenerationwebui_settings.legacy_api && !isMancer()) { if (main_api === 'textgenerationwebui' &&
textgen_settings.streaming &&
textgen_settings.legacy_api &&
textgen_settings.type !== MANCER) {
toastr.error('Streaming is not supported for the Legacy API. Update Ooba and use --extensions openai to enable streaming.', undefined, { timeOut: 10000, preventDuplicates: true }); toastr.error('Streaming is not supported for the Legacy API. Update Ooba and use --extensions openai to enable streaming.', undefined, { timeOut: 10000, preventDuplicates: true });
unblockGeneration(); unblockGeneration();
return; return;
@@ -4694,7 +4693,7 @@ function getGeneratingApi() {
case 'openai': case 'openai':
return oai_settings.chat_completion_source || 'openai'; return oai_settings.chat_completion_source || 'openai';
case 'textgenerationwebui': case 'textgenerationwebui':
return textgenerationwebui_settings.type === textgen_types.OOBA ? 'textgenerationwebui' : textgenerationwebui_settings.type; return textgen_settings.type === textgen_types.OOBA ? 'textgenerationwebui' : textgen_settings.type;
default: default:
return main_api; return main_api;
} }
@@ -5661,7 +5660,7 @@ async function saveSettings(type) {
max_context: max_context, max_context: max_context,
main_api: main_api, main_api: main_api,
world_info_settings: getWorldInfoSettings(), world_info_settings: getWorldInfoSettings(),
textgenerationwebui_settings: textgenerationwebui_settings, textgenerationwebui_settings: textgen_settings,
swipes: swipes, swipes: swipes,
horde_settings: horde_settings, horde_settings: horde_settings,
power_user: power_user, power_user: power_user,

4
public/scripts/RossAscends-mods.js
View File

@@ -34,7 +34,7 @@ import {
import { debounce, delay, getStringHash, isValidUrl } from './utils.js'; import { debounce, delay, getStringHash, isValidUrl } from './utils.js';
import { chat_completion_sources, oai_settings } from './openai.js'; import { chat_completion_sources, oai_settings } from './openai.js';
import { getTokenCount } from './tokenizers.js'; import { getTokenCount } from './tokenizers.js';
import { isMancer } from './textgen-settings.js'; import { textgen_types, textgenerationwebui_settings as textgen_settings } from './textgen-settings.js';
var RPanelPin = document.getElementById('rm_button_panel_pin'); var RPanelPin = document.getElementById('rm_button_panel_pin');
@@ -401,7 +401,7 @@ function RA_autoconnect(PrevApi) {
} }
break; break;
case 'textgenerationwebui': case 'textgenerationwebui':
if (isMancer() && secret_state[SECRET_KEYS.MANCER]) { if (textgen_settings.type === textgen_types.MANCER && secret_state[SECRET_KEYS.MANCER]) {
$('#api_button_textgenerationwebui').trigger('click'); $('#api_button_textgenerationwebui').trigger('click');
} }
else if (api_server_textgenerationwebui && isValidUrl(api_server_textgenerationwebui)) { else if (api_server_textgenerationwebui && isValidUrl(api_server_textgenerationwebui)) {

6
public/scripts/mancer-settings.js
View File

@@ -1,6 +1,6 @@
import { setGenerationParamsFromPreset } from '../script.js'; import { setGenerationParamsFromPreset } from '../script.js';
import { getDeviceInfo } from './RossAscends-mods.js'; import { getDeviceInfo } from './RossAscends-mods.js';
import { textgenerationwebui_settings } from './textgen-settings.js'; import { textgenerationwebui_settings as textgen_settings } from './textgen-settings.js';
let models = []; let models = [];
@@ -17,14 +17,14 @@ export async function loadMancerModels(data) {
const option = document.createElement('option'); const option = document.createElement('option');
option.value = model.id; option.value = model.id;
option.text = model.name; option.text = model.name;
option.selected = model.id === textgenerationwebui_settings.mancer_model; option.selected = model.id === textgen_settings.mancer_model;
$('#mancer_model').append(option); $('#mancer_model').append(option);
} }
} }
function onMancerModelSelect() { function onMancerModelSelect() {
const modelId = String($('#mancer_model').val()); const modelId = String($('#mancer_model').val());
textgenerationwebui_settings.mancer_model = modelId; textgen_settings.mancer_model = modelId;
$('#api_button_textgenerationwebui').trigger('click'); $('#api_button_textgenerationwebui').trigger('click');
const limits = models.find(x => x.id === modelId)?.limits; const limits = models.find(x => x.id === modelId)?.limits;

4
public/scripts/preset-manager.js
View File

@@ -22,7 +22,7 @@ import { context_presets, getContextSettings, power_user } from './power-user.js
import { import {
textgenerationwebui_preset_names, textgenerationwebui_preset_names,
textgenerationwebui_presets, textgenerationwebui_presets,
textgenerationwebui_settings, textgenerationwebui_settings as textgen_settings,
} from './textgen-settings.js'; } from './textgen-settings.js';
import { download, parseJsonFile, waitUntilCondition } from './utils.js'; import { download, parseJsonFile, waitUntilCondition } from './utils.js';
@@ -236,7 +236,7 @@ class PresetManager {
case 'novel': case 'novel':
return nai_settings; return nai_settings;
case 'textgenerationwebui': case 'textgenerationwebui':
return textgenerationwebui_settings; return textgen_settings;
case 'context': { case 'context': {
const context_preset = getContextSettings(); const context_preset = getContextSettings();
context_preset['name'] = name || power_user.context.preset; context_preset['name'] = name || power_user.context.preset;

202
public/scripts/textgen-settings.js
View File

@@ -18,7 +18,7 @@ import { SENTENCEPIECE_TOKENIZERS, getTextTokens, tokenizers } from './tokenizer
import { getSortableDelay, onlyUnique } from './utils.js'; import { getSortableDelay, onlyUnique } from './utils.js';
export { export {
textgenerationwebui_settings, settings as textgenerationwebui_settings,
loadTextGenSettings, loadTextGenSettings,
generateTextGenWithStreaming, generateTextGenWithStreaming,
formatTextGenURL, formatTextGenURL,
@@ -32,6 +32,8 @@ export const textgen_types = {
KOBOLDCPP: 'koboldcpp', KOBOLDCPP: 'koboldcpp',
}; };
const { MANCER, APHRODITE } = textgen_types;
// Maybe let it be configurable in the future? // Maybe let it be configurable in the future?
// (7 days later) The future has come. // (7 days later) The future has come.
const MANCER_SERVER_KEY = 'mancer_server'; const MANCER_SERVER_KEY = 'mancer_server';
@@ -39,7 +41,7 @@ const MANCER_SERVER_DEFAULT = 'https://neuro.mancer.tech';
export let MANCER_SERVER = localStorage.getItem(MANCER_SERVER_KEY) ?? MANCER_SERVER_DEFAULT; export let MANCER_SERVER = localStorage.getItem(MANCER_SERVER_KEY) ?? MANCER_SERVER_DEFAULT;
const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5]; const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];
const textgenerationwebui_settings = { const settings = {
temp: 0.7, temp: 0.7,
temperature_last: true, temperature_last: true,
top_p: 0.5, top_p: 0.5,
@@ -150,7 +152,7 @@ async function selectPreset(name) {
return; return;
} }
textgenerationwebui_settings.preset = name; settings.preset = name;
for (const name of setting_names) { for (const name of setting_names) {
const value = preset[name]; const value = preset[name];
setSettingByName(name, value, true); setSettingByName(name, value, true);
@@ -162,17 +164,17 @@ async function selectPreset(name) {
function formatTextGenURL(value) { function formatTextGenURL(value) {
try { try {
// Mancer doesn't need any formatting (it's hardcoded) // Mancer doesn't need any formatting (it's hardcoded)
if (isMancer()) { if (settings.type === MANCER) {
return value; return value;
} }
const url = new URL(value); const url = new URL(value);
if (url.pathname === '/api' && !textgenerationwebui_settings.legacy_api) { if (url.pathname === '/api' && !settings.legacy_api) {
toastr.info('Enable Legacy API or start Ooba with the OpenAI extension enabled.', 'Legacy API URL detected. Generation may fail.', { preventDuplicates: true, timeOut: 10000, extendedTimeOut: 20000 }); toastr.info('Enable Legacy API or start Ooba with the OpenAI extension enabled.', 'Legacy API URL detected. Generation may fail.', { preventDuplicates: true, timeOut: 10000, extendedTimeOut: 20000 });
url.pathname = ''; url.pathname = '';
} }
if (!power_user.relaxed_api_urls && textgenerationwebui_settings.legacy_api) { if (!power_user.relaxed_api_urls && settings.legacy_api) {
url.pathname = '/api'; url.pathname = '/api';
} }
return url.toString(); return url.toString();
@@ -190,13 +192,13 @@ function convertPresets(presets) {
* @returns {string} String with comma-separated banned token IDs * @returns {string} String with comma-separated banned token IDs
*/ */
function getCustomTokenBans() { function getCustomTokenBans() {
if (!textgenerationwebui_settings.banned_tokens && !textgenerationwebui_banned_in_macros.length) { if (!settings.banned_tokens && !textgenerationwebui_banned_in_macros.length) {
return ''; return '';
} }
const tokenizer = SENTENCEPIECE_TOKENIZERS.includes(power_user.tokenizer) ? power_user.tokenizer : tokenizers.LLAMA; const tokenizer = SENTENCEPIECE_TOKENIZERS.includes(power_user.tokenizer) ? power_user.tokenizer : tokenizers.LLAMA;
const result = []; const result = [];
const sequences = textgenerationwebui_settings.banned_tokens const sequences = settings.banned_tokens
.split('\n') .split('\n')
.concat(textgenerationwebui_banned_in_macros) .concat(textgenerationwebui_banned_in_macros)
.filter(x => x.length > 0) .filter(x => x.length > 0)
@@ -237,13 +239,13 @@ function getCustomTokenBans() {
return result.filter(onlyUnique).map(x => String(x)).join(','); return result.filter(onlyUnique).map(x => String(x)).join(',');
} }
function loadTextGenSettings(data, settings) { function loadTextGenSettings(data, loadedSettings) {
textgenerationwebui_presets = convertPresets(data.textgenerationwebui_presets); textgenerationwebui_presets = convertPresets(data.textgenerationwebui_presets);
textgenerationwebui_preset_names = data.textgenerationwebui_preset_names ?? []; textgenerationwebui_preset_names = data.textgenerationwebui_preset_names ?? [];
Object.assign(textgenerationwebui_settings, settings.textgenerationwebui_settings ?? {}); Object.assign(settings, loadedSettings.textgenerationwebui_settings ?? {});
if (settings.api_use_mancer_webui) { if (loadedSettings.api_use_mancer_webui) {
textgenerationwebui_settings.type = textgen_types.MANCER; settings.type = MANCER;
} }
for (const name of textgenerationwebui_preset_names) { for (const name of textgenerationwebui_preset_names) {
@@ -253,19 +255,19 @@ function loadTextGenSettings(data, settings) {
$('#settings_preset_textgenerationwebui').append(option); $('#settings_preset_textgenerationwebui').append(option);
} }
if (textgenerationwebui_settings.preset) { if (settings.preset) {
$('#settings_preset_textgenerationwebui').val(textgenerationwebui_settings.preset); $('#settings_preset_textgenerationwebui').val(settings.preset);
} }
for (const i of setting_names) { for (const i of setting_names) {
const value = textgenerationwebui_settings[i]; const value = settings[i];
setSettingByName(i, value); setSettingByName(i, value);
} }
$('#textgen_type').val(textgenerationwebui_settings.type); $('#textgen_type').val(settings.type);
showTypeSpecificControls(textgenerationwebui_settings.type); showTypeSpecificControls(settings.type);
//this is needed because showTypeSpecificControls() does not handle NOT declarations //this is needed because showTypeSpecificControls() does not handle NOT declarations
if (isAphrodite()) { if (settings.type === textgen_types.APHRODITE) {
$('[data-forAphro=False]').each(function () { $('[data-forAphro=False]').each(function () {
$(this).hide(); $(this).hide();
}); });
@@ -285,28 +287,8 @@ function loadTextGenSettings(data, settings) {
}); });
} }
export function isMancer() {
return textgenerationwebui_settings.type === textgen_types.MANCER;
}
export function isAphrodite() {
return textgenerationwebui_settings.type === textgen_types.APHRODITE;
}
export function isTabby() {
return textgenerationwebui_settings.type === textgen_types.TABBY;
}
export function isOoba() {
return textgenerationwebui_settings.type === textgen_types.OOBA;
}
export function isKoboldCpp() {
return textgenerationwebui_settings.type === textgen_types.KOBOLDCPP;
}
export function getTextGenUrlSourceId() { export function getTextGenUrlSourceId() {
switch (textgenerationwebui_settings.type) { switch (settings.type) {
case textgen_types.OOBA: case textgen_types.OOBA:
return '#textgenerationwebui_api_url_text'; return '#textgenerationwebui_api_url_text';
case textgen_types.APHRODITE: case textgen_types.APHRODITE:
@@ -341,23 +323,23 @@ jQuery(function () {
$('#koboldcpp_order').children().each(function () { $('#koboldcpp_order').children().each(function () {
order.push($(this).data('id')); order.push($(this).data('id'));
}); });
textgenerationwebui_settings.sampler_order = order; settings.sampler_order = order;
console.log('Samplers reordered:', textgenerationwebui_settings.sampler_order); console.log('Samplers reordered:', settings.sampler_order);
saveSettingsDebounced(); saveSettingsDebounced();
}, },
}); });
$('#koboldcpp_default_order').on('click', function () { $('#koboldcpp_default_order').on('click', function () {
textgenerationwebui_settings.sampler_order = KOBOLDCPP_ORDER; settings.sampler_order = KOBOLDCPP_ORDER;
sortItemsByOrder(textgenerationwebui_settings.sampler_order); sortItemsByOrder(settings.sampler_order);
saveSettingsDebounced(); saveSettingsDebounced();
}); });
$('#textgen_type').on('change', function () { $('#textgen_type').on('change', function () {
const type = String($(this).val()); const type = String($(this).val());
textgenerationwebui_settings.type = type; settings.type = type;
if (isAphrodite()) { if (settings.type === textgen_types.APHRODITE) {
//this is needed because showTypeSpecificControls() does not handle NOT declarations //this is needed because showTypeSpecificControls() does not handle NOT declarations
$('[data-forAphro=False]').each(function () { $('[data-forAphro=False]').each(function () {
$(this).hide(); $(this).hide();
@@ -367,8 +349,8 @@ jQuery(function () {
$('#ban_eos_token_textgenerationwebui').prop('checked', false); //Aphro should not ban EOS, just ignore it; 'add token '2' to ban list do to this' $('#ban_eos_token_textgenerationwebui').prop('checked', false); //Aphro should not ban EOS, just ignore it; 'add token '2' to ban list do to this'
//special handling for Aphrodite topK -1 disable state //special handling for Aphrodite topK -1 disable state
$('#top_k_textgenerationwebui').attr('min', -1); $('#top_k_textgenerationwebui').attr('min', -1);
if ($('#top_k_textgenerationwebui').val() === '0' || textgenerationwebui_settings['top_k'] === 0) { if ($('#top_k_textgenerationwebui').val() === '0' || settings['top_k'] === 0) {
textgenerationwebui_settings['top_k'] = -1; settings['top_k'] = -1;
$('#top_k_textgenerationwebui').val('-1').trigger('input'); $('#top_k_textgenerationwebui').val('-1').trigger('input');
} }
} else { } else {
@@ -379,8 +361,8 @@ jQuery(function () {
$('#mirostat_mode_textgenerationwebui').attr('step', 1); $('#mirostat_mode_textgenerationwebui').attr('step', 1);
//undo special Aphrodite setup for topK //undo special Aphrodite setup for topK
$('#top_k_textgenerationwebui').attr('min', 0); $('#top_k_textgenerationwebui').attr('min', 0);
if ($('#top_k_textgenerationwebui').val() === '-1' || textgenerationwebui_settings['top_k'] === -1) { if ($('#top_k_textgenerationwebui').val() === '-1' || settings['top_k'] === -1) {
textgenerationwebui_settings['top_k'] = 0; settings['top_k'] = 0;
$('#top_k_textgenerationwebui').val('0').trigger('input'); $('#top_k_textgenerationwebui').val('0').trigger('input');
} }
} }
@@ -408,19 +390,21 @@ jQuery(function () {
if (isCheckbox) { if (isCheckbox) {
const value = $(this).prop('checked'); const value = $(this).prop('checked');
textgenerationwebui_settings[id] = value; settings[id] = value;
} }
else if (isText) { else if (isText) {
const value = $(this).val(); const value = $(this).val();
textgenerationwebui_settings[id] = value; settings[id] = value;
} }
else { else {
const value = Number($(this).val()); const value = Number($(this).val());
$(`#${id}_counter_textgenerationwebui`).val(value); $(`#${id}_counter_textgenerationwebui`).val(value);
textgenerationwebui_settings[id] = value; settings[id] = value;
//special handling for aphrodite using -1 as disabled instead of 0 //special handling for aphrodite using -1 as disabled instead of 0
if ($(this).attr('id') === 'top_k_textgenerationwebui' && isAphrodite() && value === 0) { if ($(this).attr('id') === 'top_k_textgenerationwebui' &&
textgenerationwebui_settings[id] = -1; settings.type === textgen_types.APHRODITE &&
value === 0) {
settings[id] = -1;
$(this).val(-1); $(this).val(-1);
} }
} }
@@ -448,7 +432,7 @@ function setSettingByName(setting, value, trigger) {
if ('sampler_order' === setting) { if ('sampler_order' === setting) {
value = Array.isArray(value) ? value : KOBOLDCPP_ORDER; value = Array.isArray(value) ? value : KOBOLDCPP_ORDER;
sortItemsByOrder(value); sortItemsByOrder(value);
textgenerationwebui_settings.sampler_order = value; settings.sampler_order = value;
return; return;
} }
@@ -575,11 +559,11 @@ function toIntArray(string) {
} }
function getModel() { function getModel() {
if (isMancer()) { if (settings.type === MANCER) {
return textgenerationwebui_settings.mancer_model; return settings.mancer_model;
} }
if (isAphrodite()) { if (settings.type === APHRODITE) {
return online_status; return online_status;
} }
@@ -593,64 +577,66 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate,
'model': getModel(), 'model': getModel(),
'max_new_tokens': maxTokens, 'max_new_tokens': maxTokens,
'max_tokens': maxTokens, 'max_tokens': maxTokens,
'temperature': textgenerationwebui_settings.temp, 'temperature': settings.temp,
'top_p': textgenerationwebui_settings.top_p, 'top_p': settings.top_p,
'typical_p': textgenerationwebui_settings.typical_p, 'typical_p': settings.typical_p,
'min_p': textgenerationwebui_settings.min_p, 'min_p': settings.min_p,
'repetition_penalty': textgenerationwebui_settings.rep_pen, 'repetition_penalty': settings.rep_pen,
'frequency_penalty': textgenerationwebui_settings.freq_pen, 'frequency_penalty': settings.freq_pen,
'presence_penalty': textgenerationwebui_settings.presence_pen, 'presence_penalty': settings.presence_pen,
'top_k': textgenerationwebui_settings.top_k, 'top_k': settings.top_k,
'min_length': textgenerationwebui_settings.min_length, 'min_length': settings.min_length,
'min_tokens': textgenerationwebui_settings.min_length, 'min_tokens': settings.min_length,
'num_beams': textgenerationwebui_settings.num_beams, 'num_beams': settings.num_beams,
'length_penalty': textgenerationwebui_settings.length_penalty, 'length_penalty': settings.length_penalty,
'early_stopping': textgenerationwebui_settings.early_stopping, 'early_stopping': settings.early_stopping,
'add_bos_token': textgenerationwebui_settings.add_bos_token, 'add_bos_token': settings.add_bos_token,
'stopping_strings': getStoppingStrings(isImpersonate, isContinue), 'stopping_strings': getStoppingStrings(isImpersonate, isContinue),
'stop': getStoppingStrings(isImpersonate, isContinue), 'stop': getStoppingStrings(isImpersonate, isContinue),
'truncation_length': max_context, 'truncation_length': max_context,
'ban_eos_token': textgenerationwebui_settings.ban_eos_token, 'ban_eos_token': settings.ban_eos_token,
'skip_special_tokens': textgenerationwebui_settings.skip_special_tokens, 'skip_special_tokens': settings.skip_special_tokens,
'top_a': textgenerationwebui_settings.top_a, 'top_a': settings.top_a,
'tfs': textgenerationwebui_settings.tfs, 'tfs': settings.tfs,
'epsilon_cutoff': textgenerationwebui_settings.epsilon_cutoff, 'epsilon_cutoff': settings.epsilon_cutoff,
'eta_cutoff': textgenerationwebui_settings.eta_cutoff, 'eta_cutoff': settings.eta_cutoff,
'mirostat_mode': textgenerationwebui_settings.mirostat_mode, 'mirostat_mode': settings.mirostat_mode,
'mirostat_tau': textgenerationwebui_settings.mirostat_tau, 'mirostat_tau': settings.mirostat_tau,
'mirostat_eta': textgenerationwebui_settings.mirostat_eta, 'mirostat_eta': settings.mirostat_eta,
'custom_token_bans': isAphrodite() ? toIntArray(getCustomTokenBans()) : getCustomTokenBans(), 'custom_token_bans': settings.type === textgen_types.APHRODITE ?
'use_mancer': isMancer(), toIntArray(getCustomTokenBans()) :
'use_aphrodite': isAphrodite(), getCustomTokenBans(),
'use_tabby': isTabby(), 'api_type': settings.type,
'use_koboldcpp': isKoboldCpp(), 'api_server': settings.type === MANCER ?
'use_ooba': isOoba(), MANCER_SERVER :
'api_server': isMancer() ? MANCER_SERVER : api_server_textgenerationwebui, api_server_textgenerationwebui,
'legacy_api': textgenerationwebui_settings.legacy_api && !isMancer(), 'legacy_api': settings.legacy_api && settings.type !== MANCER,
'sampler_order': isKoboldCpp() ? textgenerationwebui_settings.sampler_order : undefined, 'sampler_order': settings.type === textgen_types.KOBOLDCPP ?
settings.sampler_order :
undefined,
}; };
let aphroditeExclusionFlags = { let aphroditeExclusionFlags = {
'repetition_penalty_range': textgenerationwebui_settings.rep_pen_range, 'repetition_penalty_range': settings.rep_pen_range,
'encoder_repetition_penalty': textgenerationwebui_settings.encoder_rep_pen, 'encoder_repetition_penalty': settings.encoder_rep_pen,
'no_repeat_ngram_size': textgenerationwebui_settings.no_repeat_ngram_size, 'no_repeat_ngram_size': settings.no_repeat_ngram_size,
'penalty_alpha': textgenerationwebui_settings.penalty_alpha, 'penalty_alpha': settings.penalty_alpha,
'temperature_last': textgenerationwebui_settings.temperature_last, 'temperature_last': settings.temperature_last,
'do_sample': textgenerationwebui_settings.do_sample, 'do_sample': settings.do_sample,
'seed': textgenerationwebui_settings.seed, 'seed': settings.seed,
'guidance_scale': cfgValues?.guidanceScale?.value ?? textgenerationwebui_settings.guidance_scale ?? 1, 'guidance_scale': cfgValues?.guidanceScale?.value ?? settings.guidance_scale ?? 1,
'negative_prompt': cfgValues?.negativePrompt ?? substituteParams(textgenerationwebui_settings.negative_prompt) ?? '', 'negative_prompt': cfgValues?.negativePrompt ?? substituteParams(settings.negative_prompt) ?? '',
'grammar_string': textgenerationwebui_settings.grammar_string, 'grammar_string': settings.grammar_string,
}; };
let aphroditeFlags = { let aphroditeFlags = {
'n': canMultiSwipe ? textgenerationwebui_settings.n : 1, 'n': canMultiSwipe ? settings.n : 1,
'best_of': canMultiSwipe ? textgenerationwebui_settings.n : 1, 'best_of': canMultiSwipe ? settings.n : 1,
'ignore_eos': textgenerationwebui_settings.ignore_eos_token_aphrodite, 'ignore_eos': settings.ignore_eos_token_aphrodite,
'spaces_between_special_tokens': textgenerationwebui_settings.spaces_between_special_tokens_aphrodite, 'spaces_between_special_tokens': settings.spaces_between_special_tokens_aphrodite,
//'logits_processors': textgenerationwebui_settings.logits_processors_aphrodite, //'logits_processors': settings.logits_processors_aphrodite,
//'logprobs': textgenerationwebui_settings.log_probs_aphrodite, //'logprobs': settings.log_probs_aphrodite,
//'prompt_logprobs': textgenerationwebui_settings.prompt_log_probs_aphrodite, //'prompt_logprobs': settings.prompt_log_probs_aphrodite,
}; };
if (isAphrodite()) { if (settings.type === textgen_types.APHRODITE) {
APIflags = Object.assign(APIflags, aphroditeFlags); APIflags = Object.assign(APIflags, aphroditeFlags);
} else { } else {
APIflags = Object.assign(APIflags, aphroditeExclusionFlags); APIflags = Object.assign(APIflags, aphroditeExclusionFlags);

15
public/scripts/tokenizers.js
View File

@@ -4,7 +4,9 @@ import { chat_completion_sources, model_list, oai_settings } from './openai.js';
import { groups, selected_group } from './group-chats.js'; import { groups, selected_group } from './group-chats.js';
import { getStringHash } from './utils.js'; import { getStringHash } from './utils.js';
import { kai_flags } from './kai-settings.js'; import { kai_flags } from './kai-settings.js';
import { isKoboldCpp, isMancer, isOoba, isTabby, textgenerationwebui_settings } from './textgen-settings.js'; import { textgen_types, textgenerationwebui_settings as textgen_settings } from './textgen-settings.js';
const { OOBA, TABBY, KOBOLDCPP, MANCER } = textgen_types;
export const CHARACTERS_PER_TOKEN_RATIO = 3.35; export const CHARACTERS_PER_TOKEN_RATIO = 3.35;
const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown'; const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown';
@@ -129,7 +131,7 @@ export function getTokenizerBestMatch(forApi) {
// - Tokenizer haven't reported an error previously // - Tokenizer haven't reported an error previously
const hasTokenizerError = sessionStorage.getItem(TOKENIZER_WARNING_KEY); const hasTokenizerError = sessionStorage.getItem(TOKENIZER_WARNING_KEY);
const isConnected = online_status !== 'no_connection'; const isConnected = online_status !== 'no_connection';
const isTokenizerSupported = isOoba() || isTabby() || isKoboldCpp(); const isTokenizerSupported = textgen_settings.type === OOBA || textgen_settings.type === TABBY || textgen_settings.type === KOBOLDCPP;
if (!hasTokenizerError && isConnected) { if (!hasTokenizerError && isConnected) {
if (forApi === 'kobold' && kai_flags.can_use_tokenization) { if (forApi === 'kobold' && kai_flags.can_use_tokenization) {
@@ -392,11 +394,12 @@ function getTokenCacheObject() {
function getRemoteTokenizationParams(str) { function getRemoteTokenizationParams(str) {
return { return {
text: str, text: str,
api: main_api, main_api,
api_type: textgen_settings.type,
url: getAPIServerUrl(), url: getAPIServerUrl(),
legacy_api: main_api === 'textgenerationwebui' && textgenerationwebui_settings.legacy_api && !isMancer(), legacy_api: main_api === 'textgenerationwebui' &&
use_tabby: main_api === 'textgenerationwebui' && isTabby(), textgen_settings.legacy_api &&
use_koboldcpp: main_api === 'textgenerationwebui' && isKoboldCpp(), textgen_settings.type !== MANCER,
}; };
} }

109
server.js
View File

@@ -177,19 +177,24 @@ function getOverrideHeaders(urlHost) {
* @param {string|null} server API server for new request * @param {string|null} server API server for new request
*/ */
function setAdditionalHeaders(request, args, server) { function setAdditionalHeaders(request, args, server) {
let headers = {}; let headers;
if (request.body.use_mancer) { switch (request.body.api_type) {
headers = getMancerHeaders(); case TEXTGEN_TYPES.MANCER:
} else if (request.body.use_aphrodite) { headers = getMancerHeaders();
headers = getAphroditeHeaders(); break;
} else if (request.body.use_tabby) { case TEXTGEN_TYPES.APHRODITE:
headers = getTabbyHeaders(); headers = getAphroditeHeaders();
} else { break;
headers = server ? getOverrideHeaders((new URL(server))?.host) : {}; case TEXTGEN_TYPES.TABBY:
headers = getTabbyHeaders();
break;
default:
headers = server ? getOverrideHeaders((new URL(server))?.host) : {};
break;
} }
args.headers = Object.assign(args.headers, headers); Object.assign(args.headers, headers);
} }
function humanizedISO8601DateTime(date) { function humanizedISO8601DateTime(date) {
@@ -212,7 +217,7 @@ const AVATAR_WIDTH = 400;
const AVATAR_HEIGHT = 600; const AVATAR_HEIGHT = 600;
const jsonParser = express.json({ limit: '200mb' }); const jsonParser = express.json({ limit: '200mb' });
const urlencodedParser = express.urlencoded({ extended: true, limit: '200mb' }); const urlencodedParser = express.urlencoded({ extended: true, limit: '200mb' });
const { DIRECTORIES, UPLOADS_PATH, PALM_SAFETY } = require('./src/constants'); const { DIRECTORIES, UPLOADS_PATH, PALM_SAFETY, TEXTGEN_TYPES } = require('./src/constants');
const { TavernCardValidator } = require('./src/validator/TavernCardValidator'); const { TavernCardValidator } = require('./src/validator/TavernCardValidator');
// CSRF Protection // // CSRF Protection //
@@ -562,21 +567,20 @@ app.post('/api/textgenerationwebui/status', jsonParser, async function (request,
if (request.body.legacy_api) { if (request.body.legacy_api) {
url += '/v1/model'; url += '/v1/model';
} } else {
else if (request.body.use_ooba) { switch (request.body.api_type) {
url += '/v1/models'; case TEXTGEN_TYPES.OOBA:
} case TEXTGEN_TYPES.APHRODITE:
else if (request.body.use_aphrodite) { case TEXTGEN_TYPES.KOBOLDCPP:
url += '/v1/models'; url += '/v1/models';
} break;
else if (request.body.use_mancer) { case TEXTGEN_TYPES.MANCER:
url += '/oai/v1/models'; url += '/oai/v1/models';
} break;
else if (request.body.use_tabby) { case TEXTGEN_TYPES.TABBY:
url += '/v1/model/list'; url += '/v1/model/list';
} break;
else if (request.body.use_koboldcpp) { }
url += '/v1/models';
} }
const modelsReply = await fetch(url, args); const modelsReply = await fetch(url, args);
@@ -604,7 +608,7 @@ app.post('/api/textgenerationwebui/status', jsonParser, async function (request,
// Set result to the first model ID // Set result to the first model ID
result = modelIds[0] || 'Valid'; result = modelIds[0] || 'Valid';
if (request.body.use_ooba) { if (request.body.api_type === TEXTGEN_TYPES.OOBA) {
try { try {
const modelInfoUrl = baseUrl + '/v1/internal/model/info'; const modelInfoUrl = baseUrl + '/v1/internal/model/info';
const modelInfoReply = await fetch(modelInfoUrl, args); const modelInfoReply = await fetch(modelInfoUrl, args);
@@ -619,9 +623,7 @@ app.post('/api/textgenerationwebui/status', jsonParser, async function (request,
} catch (error) { } catch (error) {
console.error(`Failed to get Ooba model info: ${error}`); console.error(`Failed to get Ooba model info: ${error}`);
} }
} } else if (request.body.api_type === TEXTGEN_TYPES.TABBY) {
if (request.body.use_tabby) {
try { try {
const modelInfoUrl = baseUrl + '/v1/model'; const modelInfoUrl = baseUrl + '/v1/model';
const modelInfoReply = await fetch(modelInfoUrl, args); const modelInfoReply = await fetch(modelInfoUrl, args);
@@ -671,12 +673,18 @@ app.post('/api/textgenerationwebui/generate', jsonParser, async function (reques
if (request.body.legacy_api) { if (request.body.legacy_api) {
url += '/v1/generate'; url += '/v1/generate';
} } else {
else if (request.body.use_aphrodite || request.body.use_ooba || request.body.use_tabby || request.body.use_koboldcpp) { switch (request.body.api_type) {
url += '/v1/completions'; case TEXTGEN_TYPES.APHRODITE:
} case TEXTGEN_TYPES.OOBA:
else if (request.body.use_mancer) { case TEXTGEN_TYPES.TABBY:
url += '/oai/v1/completions'; case TEXTGEN_TYPES.KOBOLDCPP:
url += '/v1/completions';
break;
case TEXTGEN_TYPES.MANCER:
url += '/oai/v1/completions';
break;
}
} }
const args = { const args = {
@@ -3471,7 +3479,7 @@ app.post('/tokenize_via_api', jsonParser, async function (request, response) {
return response.sendStatus(400); return response.sendStatus(400);
} }
const text = String(request.body.text) || ''; const text = String(request.body.text) || '';
const api = String(request.body.api); const api = String(request.body.main_api);
const baseUrl = String(request.body.url); const baseUrl = String(request.body.url);
const legacyApi = Boolean(request.body.legacy_api); const legacyApi = Boolean(request.body.legacy_api);
@@ -3490,18 +3498,21 @@ app.post('/tokenize_via_api', jsonParser, async function (request, response) {
if (legacyApi) { if (legacyApi) {
url += '/v1/token-count'; url += '/v1/token-count';
args.body = JSON.stringify({ 'prompt': text }); args.body = JSON.stringify({ 'prompt': text });
} } else {
else if (request.body.use_tabby) { switch (request.body.api_type) {
url += '/v1/token/encode'; case TEXTGEN_TYPES.TABBY:
args.body = JSON.stringify({ 'text': text }); url += '/v1/token/encode';
} args.body = JSON.stringify({ 'text': text });
else if (request.body.use_koboldcpp) { break;
url += '/api/extra/tokencount'; case TEXTGEN_TYPES.KOBOLDCPP:
args.body = JSON.stringify({ 'prompt': text }); url += '/api/extra/tokencount';
} args.body = JSON.stringify({ 'prompt': text });
else { break;
url += '/v1/internal/encode'; default:
args.body = JSON.stringify({ 'text': text }); url += '/v1/internal/encode';
args.body = JSON.stringify({ 'text': text });
break;
}
} }
const result = await fetch(url, args); const result = await fetch(url, args);

10
src/constants.js
View File

@@ -134,9 +134,19 @@ const PALM_SAFETY = [
const UPLOADS_PATH = './uploads'; const UPLOADS_PATH = './uploads';
// TODO: this is copied from the client code; there should be a way to de-duplicate it eventually
const TEXTGEN_TYPES = {
OOBA: 'ooba',
MANCER: 'mancer',
APHRODITE: 'aphrodite',
TABBY: 'tabby',
KOBOLDCPP: 'koboldcpp',
};
module.exports = { module.exports = {
DIRECTORIES, DIRECTORIES,
UNSAFE_EXTENSIONS, UNSAFE_EXTENSIONS,
UPLOADS_PATH, UPLOADS_PATH,
PALM_SAFETY, PALM_SAFETY,
TEXTGEN_TYPES,
}; };