SillyTavern/public/scripts/textgen-settings.js

1310 lines
44 KiB
JavaScript
Raw Normal View History

2023-07-20 19:32:15 +02:00
import {
eventSource,
event_types,
2023-07-20 19:32:15 +02:00
getRequestHeaders,
getStoppingStrings,
main_api,
2023-07-20 19:32:15 +02:00
max_context,
saveSettingsDebounced,
setGenerationParamsFromPreset,
2023-11-08 01:52:03 +01:00
setOnlineStatus,
substituteParams,
2023-12-02 19:04:51 +01:00
} from '../script.js';
import { BIAS_CACHE, createNewLogitBiasEntry, displayLogitBias, getLogitBiasListResult } from './logit-bias.js';
2023-07-20 19:32:15 +02:00
import { power_user, registerDebugFunction } from './power-user.js';
2024-04-02 13:56:15 +02:00
import { getEventSourceStream } from './sse-stream.js';
import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer } from './textgen-models.js';
2024-01-05 15:17:06 +01:00
import { SENTENCEPIECE_TOKENIZERS, TEXTGEN_TOKENIZERS, getTextTokens, tokenizers } from './tokenizers.js';
2023-12-02 19:04:51 +01:00
import { getSortableDelay, onlyUnique } from './utils.js';
2023-07-20 19:32:15 +02:00
export {
settings as textgenerationwebui_settings,
2023-07-20 19:32:15 +02:00
loadTextGenSettings,
generateTextGenWithStreaming,
2023-08-03 12:07:54 +02:00
formatTextGenURL,
2023-12-02 20:11:06 +01:00
};
2023-07-20 19:32:15 +02:00
export const textgen_types = {
OOBA: 'ooba',
MANCER: 'mancer',
2024-05-03 00:40:40 +02:00
VLLM: 'vllm',
APHRODITE: 'aphrodite',
TABBY: 'tabby',
KOBOLDCPP: 'koboldcpp',
TOGETHERAI: 'togetherai',
LLAMACPP: 'llamacpp',
2023-12-19 15:38:11 +01:00
OLLAMA: 'ollama',
2024-01-24 13:59:27 +01:00
INFERMATICAI: 'infermaticai',
DREAMGEN: 'dreamgen',
OPENROUTER: 'openrouter',
FEATHERLESS: 'featherless',
HUGGINGFACE: 'huggingface',
};
const {
MANCER,
VLLM,
APHRODITE,
TABBY,
TOGETHERAI,
OOBA,
OLLAMA,
LLAMACPP,
INFERMATICAI,
DREAMGEN,
OPENROUTER,
KOBOLDCPP,
HUGGINGFACE,
FEATHERLESS,
} = textgen_types;
const LLAMACPP_DEFAULT_ORDER = [
'top_k',
'tfs_z',
'typical_p',
'top_p',
'min_p',
'temperature',
];
2024-02-20 23:53:54 +01:00
const OOBA_DEFAULT_ORDER = [
'temperature',
'dynamic_temperature',
'quadratic_sampling',
'top_k',
'top_p',
'typical_p',
'epsilon_cutoff',
'eta_cutoff',
'tfs',
'top_a',
'min_p',
'mirostat',
];
const BIAS_KEY = '#textgenerationwebui_api-settings';
// Maybe let it be configurable in the future?
2023-11-13 10:13:39 +01:00
// (7 days later) The future has come.
const MANCER_SERVER_KEY = 'mancer_server';
const MANCER_SERVER_DEFAULT = 'https://neuro.mancer.tech';
let MANCER_SERVER = localStorage.getItem(MANCER_SERVER_KEY) ?? MANCER_SERVER_DEFAULT;
let TOGETHERAI_SERVER = 'https://api.together.xyz';
2024-01-24 13:59:27 +01:00
let INFERMATICAI_SERVER = 'https://api.totalgpt.ai';
let DREAMGEN_SERVER = 'https://dreamgen.com';
let OPENROUTER_SERVER = 'https://openrouter.ai/api';
let FEATHERLESS_SERVER = 'https://api.featherless.ai/v1';
export const SERVER_INPUTS = {
2024-01-05 15:17:06 +01:00
[textgen_types.OOBA]: '#textgenerationwebui_api_url_text',
2024-05-03 00:40:40 +02:00
[textgen_types.VLLM]: '#vllm_api_url_text',
2023-12-19 15:38:11 +01:00
[textgen_types.APHRODITE]: '#aphrodite_api_url_text',
[textgen_types.TABBY]: '#tabby_api_url_text',
[textgen_types.KOBOLDCPP]: '#koboldcpp_api_url_text',
[textgen_types.LLAMACPP]: '#llamacpp_api_url_text',
[textgen_types.OLLAMA]: '#ollama_api_url_text',
[textgen_types.HUGGINGFACE]: '#huggingface_api_url_text',
2023-12-19 15:38:11 +01:00
};
const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];
const settings = {
2023-07-20 19:32:15 +02:00
temp: 0.7,
2023-11-05 15:50:07 +01:00
temperature_last: true,
2023-07-20 19:32:15 +02:00
top_p: 0.5,
top_k: 40,
top_a: 0,
tfs: 1,
epsilon_cutoff: 0,
eta_cutoff: 0,
typical_p: 1,
min_p: 0,
2023-07-20 19:32:15 +02:00
rep_pen: 1.2,
2023-07-22 00:38:35 +02:00
rep_pen_range: 0,
rep_pen_decay: 0,
rep_pen_slope: 1,
2023-07-20 19:32:15 +02:00
no_repeat_ngram_size: 0,
penalty_alpha: 0,
num_beams: 1,
length_penalty: 1,
min_length: 0,
encoder_rep_pen: 1,
freq_pen: 0,
presence_pen: 0,
skew: 0,
2023-07-20 19:32:15 +02:00
do_sample: true,
early_stopping: false,
dynatemp: false,
min_temp: 0,
max_temp: 2.0,
2024-01-28 21:14:21 +01:00
dynatemp_exponent: 1.0,
smoothing_factor: 0.0,
2024-03-01 07:06:34 +01:00
smoothing_curve: 1.0,
dry_allowed_length: 2,
dry_multiplier: 0.0,
dry_base: 1.75,
dry_sequence_breakers: '["\\n", ":", "\\"", "*"]',
dry_penalty_last_n: 0,
2024-02-29 00:55:25 +01:00
max_tokens_second: 0,
2023-07-20 19:32:15 +02:00
seed: -1,
preset: 'Default',
add_bos_token: true,
stopping_strings: [],
//truncation_length: 2048,
2023-07-20 19:32:15 +02:00
ban_eos_token: false,
skip_special_tokens: true,
streaming: false,
mirostat_mode: 0,
mirostat_tau: 5,
mirostat_eta: 0.1,
guidance_scale: 1,
negative_prompt: '',
grammar_string: '',
json_schema: {},
banned_tokens: '',
2024-02-20 23:53:54 +01:00
sampler_priority: OOBA_DEFAULT_ORDER,
samplers: LLAMACPP_DEFAULT_ORDER,
2024-05-03 00:40:40 +02:00
ignore_eos_token: false,
spaces_between_special_tokens: true,
speculative_ngram: false,
type: textgen_types.OOBA,
mancer_model: 'mytholite',
togetherai_model: 'Gryphe/MythoMax-L2-13b',
2024-01-24 13:59:27 +01:00
infermaticai_model: '',
2023-12-19 15:38:11 +01:00
ollama_model: '',
openrouter_model: 'openrouter/auto',
2024-05-06 18:26:20 +02:00
openrouter_providers: [],
2024-05-03 00:40:40 +02:00
vllm_model: '',
2024-03-01 22:02:43 +01:00
aphrodite_model: '',
dreamgen_model: 'opus-v1-xl/text',
2024-09-08 21:23:25 +02:00
tabby_model: '',
2023-11-08 09:13:28 +01:00
legacy_api: false,
sampler_order: KOBOLDCPP_ORDER,
logit_bias: [],
2023-12-03 19:40:09 +01:00
n: 1,
2023-12-19 15:38:11 +01:00
server_urls: {},
custom_model: '',
bypass_status_check: false,
openrouter_allow_fallbacks: true,
2024-08-19 06:32:45 +02:00
xtc_threshold: 0.1,
xtc_probability: 0,
2023-07-20 19:32:15 +02:00
};
export let textgenerationwebui_banned_in_macros = [];
export let textgenerationwebui_presets = [];
export let textgenerationwebui_preset_names = [];
2023-07-20 19:32:15 +02:00
2024-05-19 08:06:29 +02:00
export const setting_names = [
2023-12-02 19:04:51 +01:00
'temp',
'temperature_last',
'rep_pen',
'rep_pen_range',
'rep_pen_decay',
'rep_pen_slope',
2023-12-02 19:04:51 +01:00
'no_repeat_ngram_size',
'top_k',
'top_p',
'top_a',
'tfs',
'epsilon_cutoff',
'eta_cutoff',
'typical_p',
'min_p',
'penalty_alpha',
'num_beams',
'length_penalty',
'min_length',
'dynatemp',
'min_temp',
'max_temp',
2024-01-28 21:14:21 +01:00
'dynatemp_exponent',
'smoothing_factor',
2024-03-01 07:06:34 +01:00
'smoothing_curve',
'dry_allowed_length',
'dry_multiplier',
'dry_base',
'dry_sequence_breakers',
'dry_penalty_last_n',
2024-02-29 00:55:25 +01:00
'max_tokens_second',
2023-12-02 19:04:51 +01:00
'encoder_rep_pen',
'freq_pen',
'presence_pen',
'skew',
2023-12-02 19:04:51 +01:00
'do_sample',
'early_stopping',
'seed',
'add_bos_token',
'ban_eos_token',
'skip_special_tokens',
'streaming',
'mirostat_mode',
'mirostat_tau',
'mirostat_eta',
'guidance_scale',
'negative_prompt',
'grammar_string',
'json_schema',
2023-12-02 19:04:51 +01:00
'banned_tokens',
'legacy_api',
2024-05-03 00:40:40 +02:00
'ignore_eos_token',
'spaces_between_special_tokens',
'speculative_ngram',
2023-12-02 19:04:51 +01:00
'sampler_order',
2024-02-10 21:57:41 +01:00
'sampler_priority',
'samplers',
2023-12-03 19:40:09 +01:00
'n',
'logit_bias',
'custom_model',
'bypass_status_check',
'openrouter_allow_fallbacks',
2024-08-19 06:32:45 +02:00
'xtc_threshold',
'xtc_probability',
2023-07-20 19:32:15 +02:00
];
const DYNATEMP_BLOCK = document.getElementById('dynatemp_block_ooba');
2023-12-19 15:38:11 +01:00
export function validateTextGenUrl() {
const selector = SERVER_INPUTS[settings.type];
if (!selector) {
return;
}
const control = $(selector);
const url = String(control.val()).trim();
const formattedUrl = formatTextGenURL(url);
if (!formattedUrl) {
toastr.error('Enter a valid API URL', 'Text Completion API');
return;
}
control.val(formattedUrl);
}
export function getTextGenServer() {
2024-03-19 00:38:55 +01:00
switch (settings.type) {
case FEATHERLESS:
return FEATHERLESS_SERVER;
2024-03-19 00:38:55 +01:00
case MANCER:
return MANCER_SERVER;
case TOGETHERAI:
return TOGETHERAI_SERVER;
case INFERMATICAI:
return INFERMATICAI_SERVER;
case DREAMGEN:
return DREAMGEN_SERVER;
case OPENROUTER:
return OPENROUTER_SERVER;
default:
return settings.server_urls[settings.type] ?? '';
}
2023-12-19 15:38:11 +01:00
}
async function selectPreset(name) {
2023-07-20 19:32:15 +02:00
const preset = textgenerationwebui_presets[textgenerationwebui_preset_names.indexOf(name)];
if (!preset) {
return;
}
settings.preset = name;
2023-07-20 19:32:15 +02:00
for (const name of setting_names) {
const value = preset[name];
setSettingByName(name, value, true);
}
setGenerationParamsFromPreset(preset);
BIAS_CACHE.delete(BIAS_KEY);
displayLogitBias(preset.logit_bias, BIAS_KEY);
2023-07-20 19:32:15 +02:00
saveSettingsDebounced();
}
function formatTextGenURL(value) {
2023-08-03 12:07:54 +02:00
try {
2024-03-19 00:38:55 +01:00
const noFormatTypes = [MANCER, TOGETHERAI, INFERMATICAI, DREAMGEN, OPENROUTER];
const legacyApiTypes = [OOBA, APHRODITE];
2024-03-19 00:38:55 +01:00
if (noFormatTypes.includes(settings.type)) {
return value;
}
2023-08-03 12:07:54 +02:00
const url = new URL(value);
if (legacyApiTypes.includes(settings.type)) {
if (url.pathname === '/api' && !settings.legacy_api) {
toastr.info('Enable Legacy API or start Ooba with the OpenAI extension enabled.', 'Legacy API URL detected. Generation may fail.', { preventDuplicates: true, timeOut: 10000, extendedTimeOut: 20000 });
url.pathname = '';
}
if (!power_user.relaxed_api_urls && settings.legacy_api) {
url.pathname = '/api';
}
}
return url.toString();
2023-12-02 15:14:48 +01:00
} catch {
// Just using URL as a validation check
}
2023-08-03 12:07:54 +02:00
return null;
}
2023-07-20 19:32:15 +02:00
function convertPresets(presets) {
return Array.isArray(presets) ? presets.map((p) => JSON.parse(p)) : [];
2023-07-20 19:32:15 +02:00
}
2024-01-05 15:17:06 +01:00
function getTokenizerForTokenIds() {
if (power_user.tokenizer === tokenizers.API_CURRENT && TEXTGEN_TOKENIZERS.includes(settings.type)) {
return tokenizers.API_CURRENT;
}
if (SENTENCEPIECE_TOKENIZERS.includes(power_user.tokenizer)) {
return power_user.tokenizer;
}
if (settings.type === OPENROUTER) {
return getCurrentOpenRouterModelTokenizer();
}
if (settings.type === DREAMGEN) {
return getCurrentDreamGenModelTokenizer();
}
2024-01-05 15:17:06 +01:00
return tokenizers.LLAMA;
}
/**
* @typedef {{banned_tokens: string, banned_strings: string[]}} TokenBanResult
* @returns {TokenBanResult} String with comma-separated banned token IDs
*/
function getCustomTokenBans() {
if (!settings.banned_tokens && !textgenerationwebui_banned_in_macros.length) {
return {
banned_tokens: '',
banned_strings: [],
};
}
2024-01-05 15:17:06 +01:00
const tokenizer = getTokenizerForTokenIds();
const banned_tokens = [];
const banned_strings = [];
const sequences = settings.banned_tokens
.split('\n')
.concat(textgenerationwebui_banned_in_macros)
.filter(x => x.length > 0)
.filter(onlyUnique);
//debug
if (textgenerationwebui_banned_in_macros.length) {
2023-12-02 19:04:51 +01:00
console.log('=== Found banned word sequences in the macros:', textgenerationwebui_banned_in_macros, 'Resulting array of banned sequences (will be used this generation turn):', sequences);
}
//clean old temporary bans found in macros before, for the next generation turn.
textgenerationwebui_banned_in_macros = [];
for (const line of sequences) {
// Raw token ids, JSON serialized
if (line.startsWith('[') && line.endsWith(']')) {
try {
const tokens = JSON.parse(line);
if (Array.isArray(tokens) && tokens.every(t => Number.isInteger(t))) {
banned_tokens.push(...tokens);
} else {
throw new Error('Not an array of integers');
}
} catch (err) {
console.log(`Failed to parse bad word token list: ${line}`, err);
}
} else if (line.startsWith('"') && line.endsWith('"')) {
// Remove the enclosing quotes
2024-05-12 20:41:07 +02:00
banned_strings.push(line.slice(1, -1));
} else {
try {
const tokens = getTextTokens(tokenizer, line);
banned_tokens.push(...tokens);
} catch {
console.log(`Could not tokenize raw text: ${line}`);
}
}
}
return {
banned_tokens: banned_tokens.filter(onlyUnique).map(x => String(x)).join(','),
banned_strings: banned_strings,
};
}
/**
* Calculates logit bias object from the logit bias list.
* @returns {object} Logit bias object
*/
function calculateLogitBias() {
if (!Array.isArray(settings.logit_bias) || settings.logit_bias.length === 0) {
return {};
}
2024-01-05 15:17:06 +01:00
const tokenizer = getTokenizerForTokenIds();
const result = {};
/**
* Adds bias to the logit bias object.
* @param {number} bias
* @param {number[]} sequence
* @returns {object} Accumulated logit bias object
*/
function addBias(bias, sequence) {
if (sequence.length === 0) {
return;
}
for (const logit of sequence) {
const key = String(logit);
result[key] = bias;
}
return result;
}
getLogitBiasListResult(settings.logit_bias, tokenizer, addBias);
return result;
}
2023-12-03 21:44:36 +01:00
function loadTextGenSettings(data, loadedSettings) {
2023-07-20 19:32:15 +02:00
textgenerationwebui_presets = convertPresets(data.textgenerationwebui_presets);
textgenerationwebui_preset_names = data.textgenerationwebui_preset_names ?? [];
2023-12-03 21:44:36 +01:00
Object.assign(settings, loadedSettings.textgenerationwebui_settings ?? {});
2023-07-20 19:32:15 +02:00
2023-12-19 15:38:11 +01:00
if (loadedSettings.api_server_textgenerationwebui) {
for (const type of Object.keys(SERVER_INPUTS)) {
settings.server_urls[type] = loadedSettings.api_server_textgenerationwebui;
}
delete loadedSettings.api_server_textgenerationwebui;
}
for (const [type, selector] of Object.entries(SERVER_INPUTS)) {
const control = $(selector);
control.val(settings.server_urls[type] ?? '').on('input', function () {
settings.server_urls[type] = String($(this).val());
saveSettingsDebounced();
});
}
2023-12-03 21:44:36 +01:00
if (loadedSettings.api_use_mancer_webui) {
settings.type = MANCER;
}
2023-07-20 19:32:15 +02:00
for (const name of textgenerationwebui_preset_names) {
const option = document.createElement('option');
option.value = name;
option.innerText = name;
$('#settings_preset_textgenerationwebui').append(option);
}
if (settings.preset) {
$('#settings_preset_textgenerationwebui').val(settings.preset);
2023-07-20 19:32:15 +02:00
}
for (const i of setting_names) {
const value = settings[i];
2023-07-20 19:32:15 +02:00
setSettingByName(i, value);
}
$('#textgen_type').val(settings.type);
2024-05-06 18:26:20 +02:00
$('#openrouter_providers_text').val(settings.openrouter_providers).trigger('change');
showTypeSpecificControls(settings.type);
BIAS_CACHE.delete(BIAS_KEY);
displayLogitBias(settings.logit_bias, BIAS_KEY);
2023-11-13 10:13:39 +01:00
registerDebugFunction('change-mancer-url', 'Change Mancer base URL', 'Change Mancer API server base URL', () => {
const result = prompt(`Enter Mancer base URL\nDefault: ${MANCER_SERVER_DEFAULT}`, MANCER_SERVER);
if (result) {
localStorage.setItem(MANCER_SERVER_KEY, result);
MANCER_SERVER = result;
}
});
}
/**
* Sorts the sampler items by the given order.
* @param {any[]} orderArray Sampler order array.
*/
2024-02-20 23:53:54 +01:00
function sortKoboldItemsByOrder(orderArray) {
console.debug('Preset samplers order: ' + orderArray);
2023-12-02 19:04:51 +01:00
const $draggableItems = $('#koboldcpp_order');
for (let i = 0; i < orderArray.length; i++) {
const index = orderArray[i];
const $item = $draggableItems.find(`[data-id="${index}"]`).detach();
$draggableItems.append($item);
}
}
function sortLlamacppItemsByOrder(orderArray) {
console.debug('Preset samplers order: ', orderArray);
const $container = $('#llamacpp_samplers_sortable');
orderArray.forEach((name) => {
const $item = $container.find(`[data-name="${name}"]`).detach();
$container.append($item);
});
}
function sortOobaItemsByOrder(orderArray) {
console.debug('Preset samplers order: ', orderArray);
const $container = $('#sampler_priority_container');
orderArray.forEach((name) => {
const $item = $container.find(`[data-name="${name}"]`).detach();
$container.append($item);
});
}
jQuery(function () {
$('#koboldcpp_order').sortable({
delay: getSortableDelay(),
stop: function () {
const order = [];
$('#koboldcpp_order').children().each(function () {
order.push($(this).data('id'));
});
settings.sampler_order = order;
console.log('Samplers reordered:', settings.sampler_order);
saveSettingsDebounced();
},
});
$('#koboldcpp_default_order').on('click', function () {
settings.sampler_order = KOBOLDCPP_ORDER;
2024-02-20 23:53:54 +01:00
sortKoboldItemsByOrder(settings.sampler_order);
saveSettingsDebounced();
});
2024-02-20 23:53:54 +01:00
$('#llamacpp_samplers_sortable').sortable({
delay: getSortableDelay(),
stop: function () {
const order = [];
$('#llamacpp_samplers_sortable').children().each(function () {
order.push($(this).data('name'));
});
settings.samplers = order;
console.log('Samplers reordered:', settings.samplers);
saveSettingsDebounced();
},
});
$('#llamacpp_samplers_default_order').on('click', function () {
sortLlamacppItemsByOrder(LLAMACPP_DEFAULT_ORDER);
settings.samplers = LLAMACPP_DEFAULT_ORDER;
console.log('Default samplers order loaded:', settings.samplers);
saveSettingsDebounced();
});
$('#sampler_priority_container').sortable({
delay: getSortableDelay(),
2024-02-20 23:53:54 +01:00
stop: function () {
const order = [];
2024-02-20 23:53:54 +01:00
$('#sampler_priority_container').children().each(function () {
order.push($(this).data('name'));
});
2024-02-20 23:53:54 +01:00
settings.sampler_priority = order;
console.log('Samplers reordered:', settings.sampler_priority);
saveSettingsDebounced();
2024-02-20 23:53:54 +01:00
},
});
$('#tabby_json_schema').on('input', function () {
const json_schema_string = String($(this).val());
try {
settings.json_schema = JSON.parse(json_schema_string || '{}');
} catch {
// Ignore errors from here
}
saveSettingsDebounced();
});
2024-02-20 23:53:54 +01:00
$('#textgenerationwebui_default_order').on('click', function () {
sortOobaItemsByOrder(OOBA_DEFAULT_ORDER);
settings.sampler_priority = OOBA_DEFAULT_ORDER;
console.log('Default samplers order loaded:', settings.sampler_priority);
saveSettingsDebounced();
});
$('#textgen_type').on('change', function () {
const type = String($(this).val());
settings.type = type;
if ([VLLM, APHRODITE, INFERMATICAI].includes(settings.type)) {
2023-12-02 20:11:06 +01:00
$('#mirostat_mode_textgenerationwebui').attr('step', 2); //Aphro disallows mode 1
$('#do_sample_textgenerationwebui').prop('checked', true); //Aphro should always do sample; 'otherwise set temp to 0 to mimic no sample'
$('#ban_eos_token_textgenerationwebui').prop('checked', false); //Aphro should not ban EOS, just ignore it; 'add token '2' to ban list do to this'
2024-05-03 00:40:40 +02:00
//special handling for vLLM/Aphrodite topK -1 disable state
2023-12-02 20:11:06 +01:00
$('#top_k_textgenerationwebui').attr('min', -1);
if ($('#top_k_textgenerationwebui').val() === '0' || settings['top_k'] === 0) {
settings['top_k'] = -1;
2023-12-02 20:11:06 +01:00
$('#top_k_textgenerationwebui').val('-1').trigger('input');
}
} else {
2023-12-02 20:11:06 +01:00
$('#mirostat_mode_textgenerationwebui').attr('step', 1);
2024-05-03 00:40:40 +02:00
//undo special vLLM/Aphrodite setup for topK
2023-12-02 20:11:06 +01:00
$('#top_k_textgenerationwebui').attr('min', 0);
if ($('#top_k_textgenerationwebui').val() === '-1' || settings['top_k'] === -1) {
settings['top_k'] = 0;
2023-12-02 20:11:06 +01:00
$('#top_k_textgenerationwebui').val('0').trigger('input');
}
}
2023-11-04 03:16:12 +01:00
2023-11-08 01:52:03 +01:00
showTypeSpecificControls(type);
setOnlineStatus('no_connection');
BIAS_CACHE.delete(BIAS_KEY);
2023-11-08 01:52:03 +01:00
$('#main_api').trigger('change');
2023-12-19 15:38:11 +01:00
if (!SERVER_INPUTS[type] || settings.server_urls[type]) {
$('#api_button_textgenerationwebui').trigger('click');
}
saveSettingsDebounced();
});
$('#settings_preset_textgenerationwebui').on('change', function () {
2023-07-20 19:32:15 +02:00
const presetName = $(this).val();
selectPreset(presetName);
});
$('#samplerResetButton').off('click').on('click', function () {
const inputs = {
'temp_textgenerationwebui': 1,
'top_k_textgenerationwebui': [INFERMATICAI, APHRODITE, VLLM].includes(settings.type) ? -1 : 0,
'top_p_textgenerationwebui': 1,
'min_p_textgenerationwebui': 0,
'rep_pen_textgenerationwebui': 1,
'rep_pen_range_textgenerationwebui': 0,
'rep_pen_decay_textgenerationwebui': 0,
'dynatemp_textgenerationwebui': false,
'seed_textgenerationwebui': -1,
'ban_eos_token_textgenerationwebui': false,
'do_sample_textgenerationwebui': true,
'add_bos_token_textgenerationwebui': true,
'temperature_last_textgenerationwebui': true,
'skip_special_tokens_textgenerationwebui': true,
'top_a_textgenerationwebui': 0,
'top_a_counter_textgenerationwebui': 0,
'mirostat_mode_textgenerationwebui': 0,
'mirostat_tau_textgenerationwebui': 5,
'mirostat_eta_textgenerationwebui': 0.1,
'tfs_textgenerationwebui': 1,
'epsilon_cutoff_textgenerationwebui': 0,
'eta_cutoff_textgenerationwebui': 0,
'encoder_rep_pen_textgenerationwebui': 1,
'freq_pen_textgenerationwebui': 0,
'presence_pen_textgenerationwebui': 0,
'skew_textgenerationwebui': 0,
'no_repeat_ngram_size_textgenerationwebui': 0,
'speculative_ngram_textgenerationwebui': false,
'min_length_textgenerationwebui': 0,
'num_beams_textgenerationwebui': 1,
'length_penalty_textgenerationwebui': 1,
'penalty_alpha_textgenerationwebui': 0,
'typical_p_textgenerationwebui': 1, // Added entry
'guidance_scale_textgenerationwebui': 1,
'smoothing_factor_textgenerationwebui': 0,
2024-03-01 07:06:34 +01:00
'smoothing_curve_textgenerationwebui': 1,
'dry_allowed_length_textgenerationwebui': 2,
'dry_multiplier_textgenerationwebui': 0,
'dry_base_textgenerationwebui': 1.75,
'dry_penalty_last_n_textgenerationwebui': 0,
2024-08-19 06:32:45 +02:00
'xtc_threshold_textgenerationwebui': 0.1,
2024-08-19 20:36:28 +02:00
'xtc_probability_textgenerationwebui': 0,
};
for (const [id, value] of Object.entries(inputs)) {
const inputElement = $(`#${id}`);
if (inputElement.prop('type') === 'checkbox') {
2024-02-24 14:32:12 +01:00
inputElement.prop('checked', value).trigger('input');
} else if (inputElement.prop('type') === 'number') {
inputElement.val(value).trigger('input');
} else {
inputElement.val(value).trigger('input');
if (power_user.enableZenSliders) {
let masterElementID = inputElement.prop('id');
2024-01-20 19:40:40 +01:00
console.log(masterElementID);
let zenSlider = $(`#${masterElementID}_zenslider`).slider();
zenSlider.slider('option', 'value', value);
zenSlider.slider('option', 'slide')
.call(zenSlider, null, {
handle: $('.ui-slider-handle', zenSlider), value: value,
});
}
}
}
});
2023-07-20 19:32:15 +02:00
for (const i of setting_names) {
2023-12-02 19:04:51 +01:00
$(`#${i}_textgenerationwebui`).attr('x-setting-id', i);
$(document).on('input', `#${i}_textgenerationwebui`, function () {
2023-07-20 19:32:15 +02:00
const isCheckbox = $(this).attr('type') == 'checkbox';
const isText = $(this).attr('type') == 'text' || $(this).is('textarea');
2023-12-02 19:04:51 +01:00
const id = $(this).attr('x-setting-id');
2023-07-20 19:32:15 +02:00
if (isCheckbox) {
const value = $(this).prop('checked');
settings[id] = value;
2023-07-20 19:32:15 +02:00
}
else if (isText) {
const value = $(this).val();
settings[id] = value;
2023-07-20 19:32:15 +02:00
}
else {
2023-08-22 18:35:56 +02:00
const value = Number($(this).val());
2023-10-26 13:41:12 +02:00
$(`#${id}_counter_textgenerationwebui`).val(value);
settings[id] = value;
2024-05-03 00:40:40 +02:00
//special handling for vLLM/Aphrodite using -1 as disabled instead of 0
2024-05-11 10:38:22 +02:00
if ($(this).attr('id') === 'top_k_textgenerationwebui' && [INFERMATICAI, APHRODITE, VLLM].includes(settings.type) && value === 0) {
settings[id] = -1;
2023-12-02 20:11:06 +01:00
$(this).val(-1);
}
2023-07-20 19:32:15 +02:00
}
saveSettingsDebounced();
});
}
$('#textgen_logit_bias_new_entry').on('click', () => createNewLogitBiasEntry(settings.logit_bias, BIAS_KEY));
2024-05-06 20:21:03 +02:00
$('#openrouter_providers_text').on('change', function () {
const selectedProviders = $(this).val();
// Not a multiple select?
if (!Array.isArray(selectedProviders)) {
return;
}
settings.openrouter_providers = selectedProviders;
saveSettingsDebounced();
});
2023-12-02 20:11:06 +01:00
});
2023-07-20 19:32:15 +02:00
2023-11-08 01:52:03 +01:00
function showTypeSpecificControls(type) {
$('[data-tg-type]').each(function () {
2024-01-15 19:45:21 +01:00
const tgTypes = $(this).attr('data-tg-type').split(',').map(x => x.trim());
for (const tgType of tgTypes) {
if (tgType === type || tgType == 'all') {
$(this).show();
return;
} else {
$(this).hide();
}
2023-11-08 01:52:03 +01:00
}
});
}
function setSettingByName(setting, value, trigger) {
2023-07-20 19:32:15 +02:00
if (value === null || value === undefined) {
return;
}
if ('sampler_order' === setting) {
value = Array.isArray(value) ? value : KOBOLDCPP_ORDER;
2024-02-20 23:53:54 +01:00
sortKoboldItemsByOrder(value);
settings.sampler_order = value;
return;
}
2024-02-20 23:53:54 +01:00
if ('sampler_priority' === setting) {
value = Array.isArray(value) ? value : OOBA_DEFAULT_ORDER;
sortOobaItemsByOrder(value);
settings.sampler_priority = value;
return;
}
if ('samplers' === setting) {
value = Array.isArray(value) ? value : LLAMACPP_DEFAULT_ORDER;
sortLlamacppItemsByOrder(value);
settings.samplers = value;
return;
}
if ('logit_bias' === setting) {
settings.logit_bias = Array.isArray(value) ? value : [];
return;
}
if ('json_schema' === setting) {
settings.json_schema = value ?? {};
$('#tabby_json_schema').val(JSON.stringify(settings.json_schema, null, 2));
return;
}
const isCheckbox = $(`#${setting}_textgenerationwebui`).attr('type') == 'checkbox';
const isText = $(`#${setting}_textgenerationwebui`).attr('type') == 'text' || $(`#${setting}_textgenerationwebui`).is('textarea');
2023-07-20 19:32:15 +02:00
if (isCheckbox) {
const val = Boolean(value);
$(`#${setting}_textgenerationwebui`).prop('checked', val);
2023-07-20 19:32:15 +02:00
}
else if (isText) {
$(`#${setting}_textgenerationwebui`).val(value);
2023-07-20 19:32:15 +02:00
}
else {
const val = parseFloat(value);
$(`#${setting}_textgenerationwebui`).val(val);
$(`#${setting}_counter_textgenerationwebui`).val(val);
if (power_user.enableZenSliders) {
2023-12-02 20:11:06 +01:00
let zenSlider = $(`#${setting}_textgenerationwebui_zenslider`).slider();
zenSlider.slider('option', 'value', val);
zenSlider.slider('option', 'slide')
.call(zenSlider, null, {
2023-12-02 21:06:57 +01:00
handle: $('.ui-slider-handle', zenSlider), value: val,
});
}
2023-07-20 19:32:15 +02:00
}
if (trigger) {
$(`#${setting}_textgenerationwebui`).trigger('input');
2023-07-20 19:32:15 +02:00
}
}
async function generateTextGenWithStreaming(generate_data, signal) {
generate_data.stream = true;
const response = await fetch('/api/backends/text-completions/generate', {
2023-07-20 19:32:15 +02:00
headers: {
...getRequestHeaders(),
},
body: JSON.stringify(generate_data),
method: 'POST',
signal: signal,
});
if (!response.ok) {
tryParseStreamingError(response, await response.text());
throw new Error(`Got response status ${response.status}`);
}
2024-04-02 13:56:15 +02:00
const eventStream = getEventSourceStream();
response.body.pipeThrough(eventStream);
const reader = eventStream.readable.getReader();
2023-07-20 19:32:15 +02:00
return async function* streamData() {
let text = '';
/** @type {import('./logprobs.js').TokenLogprobs | null} */
let logprobs = null;
2023-12-03 19:40:09 +01:00
const swipes = [];
2023-07-20 19:32:15 +02:00
while (true) {
const { done, value } = await reader.read();
if (done) return;
if (value.data === '[DONE]') return;
2023-07-20 19:32:15 +02:00
tryParseStreamingError(response, value.data);
2023-07-20 19:32:15 +02:00
let data = JSON.parse(value.data);
if (data?.choices?.[0]?.index > 0) {
const swipeIndex = data.choices[0].index - 1;
swipes[swipeIndex] = (swipes[swipeIndex] || '') + data.choices[0].text;
} else {
const newText = data?.choices?.[0]?.text || data?.content || '';
text += newText;
logprobs = parseTextgenLogprobs(newText, data.choices?.[0]?.logprobs || data?.completion_probabilities);
}
yield { text, swipes, logprobs };
2023-07-20 19:32:15 +02:00
}
2023-12-02 20:11:06 +01:00
};
2023-07-20 19:32:15 +02:00
}
/**
* parseTextgenLogprobs converts a logprobs object returned from a textgen API
* for a single token into a TokenLogprobs object used by the Token
* Probabilities feature.
* @param {string} token - the text of the token that the logprobs are for
* @param {Object} logprobs - logprobs object returned from the API
* @returns {import('./logprobs.js').TokenLogprobs | null} - converted logprobs
*/
2024-02-24 13:50:06 +01:00
export function parseTextgenLogprobs(token, logprobs) {
if (!logprobs) {
return null;
}
switch (settings.type) {
case TABBY:
2024-05-03 00:40:40 +02:00
case VLLM:
case APHRODITE:
case MANCER:
2024-06-24 18:16:20 +02:00
case INFERMATICAI:
case OOBA: {
2024-02-02 19:06:46 +01:00
/** @type {Record<string, number>[]} */
const topLogprobs = logprobs.top_logprobs;
if (!topLogprobs?.length) {
return null;
}
const candidates = Object.entries(topLogprobs[0]);
return { token, topLogprobs: candidates };
}
case LLAMACPP: {
/** @type {Record<string, number>[]} */
if (!logprobs?.length) {
return null;
}
2024-03-19 00:38:55 +01:00
const candidates = logprobs[0].probs.map(x => [x.tok_str, x.prob]);
return { token, topLogprobs: candidates };
}
default:
return null;
}
}
2024-02-24 19:10:53 +01:00
export function parseTabbyLogprobs(data) {
const text = data?.choices?.[0]?.text;
const offsets = data?.choices?.[0]?.logprobs?.text_offset;
if (!text || !offsets) {
return null;
}
// Convert string offsets list to tokens
const tokens = offsets?.map((offset, index) => {
const nextOffset = offsets[index + 1] || text.length;
return text.substring(offset, nextOffset);
});
const topLogprobs = data?.choices?.[0]?.logprobs?.top_logprobs?.map(x => ({ top_logprobs: [x] }));
return tokens?.map((token, index) => parseTextgenLogprobs(token, topLogprobs[index])) || null;
}
/**
* Parses errors in streaming responses and displays them in toastr.
* @param {Response} response - Response from the server.
* @param {string} decoded - Decoded response body.
* @returns {void} Nothing.
*/
function tryParseStreamingError(response, decoded) {
let data = {};
try {
data = JSON.parse(decoded);
} catch {
// No JSON. Do nothing.
}
2024-09-08 21:24:03 +02:00
const message = data?.error?.message || data?.message || data?.detail;
2023-11-07 23:25:06 +01:00
if (message) {
toastr.error(message, 'Text Completion API');
2023-11-07 23:25:06 +01:00
throw new Error(message);
}
}
/**
* Converts a string of comma-separated integers to an array of integers.
* @param {string} string Input string
* @returns {number[]} Array of integers
*/
function toIntArray(string) {
if (!string) {
return [];
}
return string.split(',').map(x => parseInt(x)).filter(x => !isNaN(x));
}
2024-05-03 22:59:39 +02:00
export function getTextGenModel() {
2024-03-19 00:38:55 +01:00
switch (settings.type) {
case OOBA:
if (settings.custom_model) {
return settings.custom_model;
}
break;
case MANCER:
return settings.mancer_model;
case TOGETHERAI:
return settings.togetherai_model;
case INFERMATICAI:
return settings.infermaticai_model;
case DREAMGEN:
return settings.dreamgen_model;
case OPENROUTER:
return settings.openrouter_model;
2024-05-03 00:40:40 +02:00
case VLLM:
return settings.vllm_model;
2024-03-19 00:38:55 +01:00
case APHRODITE:
return settings.aphrodite_model;
case OLLAMA:
if (!settings.ollama_model) {
toastr.error('No Ollama model selected.', 'Text Completion API');
throw new Error('No Ollama model selected');
}
return settings.ollama_model;
2024-06-28 14:20:15 +02:00
case FEATHERLESS:
return settings.featherless_model;
case HUGGINGFACE:
return 'tgi';
2024-09-08 21:23:25 +02:00
case TABBY:
if (settings.tabby_model) {
return settings.tabby_model;
}
break;
2024-03-19 00:38:55 +01:00
default:
return undefined;
}
2023-12-19 15:38:11 +01:00
return undefined;
}
export function isJsonSchemaSupported() {
return [TABBY, LLAMACPP].includes(settings.type) && main_api === 'textgenerationwebui';
}
function isDynamicTemperatureSupported() {
return settings.dynatemp && DYNATEMP_BLOCK?.dataset?.tgType?.includes(settings.type);
}
2024-06-08 23:59:40 +02:00
function getLogprobsNumber() {
2024-06-24 18:16:20 +02:00
if (settings.type === VLLM || settings.type === INFERMATICAI) {
2024-06-08 23:59:40 +02:00
return 5;
}
return 10;
}
/**
* Replaces {{macro}} in a comma-separated or serialized JSON array string.
* @param {string} str Input string
* @returns {string} Output string
*/
function replaceMacrosInList(str) {
if (!str || typeof str !== 'string') {
return str;
}
try {
const array = JSON.parse(str);
if (!Array.isArray(array)) {
throw new Error('Not an array');
}
for (let i = 0; i < array.length; i++) {
array[i] = substituteParams(array[i]);
}
return JSON.stringify(array);
} catch {
const array = str.split(',');
for (let i = 0; i < array.length; i++) {
array[i] = substituteParams(array[i]);
}
return array.join(',');
}
}
2023-12-03 19:56:25 +01:00
export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate, isContinue, cfgValues, type) {
const canMultiSwipe = !isContinue && !isImpersonate && type !== 'quiet';
const dynatemp = isDynamicTemperatureSupported();
2024-05-11 10:38:22 +02:00
const { banned_tokens, banned_strings } = getCustomTokenBans();
let params = {
'prompt': finalPrompt,
2024-05-03 22:59:39 +02:00
'model': getTextGenModel(),
'max_new_tokens': maxTokens,
'max_tokens': maxTokens,
2024-06-08 23:59:40 +02:00
'logprobs': power_user.request_token_probabilities ? getLogprobsNumber() : undefined,
'temperature': dynatemp ? (settings.min_temp + settings.max_temp) / 2 : settings.temp,
'top_p': settings.top_p,
'typical_p': settings.typical_p,
2024-02-09 14:32:34 +01:00
'typical': settings.typical_p,
'sampler_seed': settings.seed,
'min_p': settings.min_p,
'repetition_penalty': settings.rep_pen,
'frequency_penalty': settings.freq_pen,
'presence_penalty': settings.presence_pen,
'top_k': settings.top_k,
'skew': settings.skew,
'min_length': settings.type === OOBA ? settings.min_length : undefined,
2024-03-07 17:28:38 +01:00
'minimum_message_content_tokens': settings.type === DREAMGEN ? settings.min_length : undefined,
'min_tokens': settings.min_length,
'num_beams': settings.type === OOBA ? settings.num_beams : undefined,
'length_penalty': settings.length_penalty,
'early_stopping': settings.early_stopping,
'add_bos_token': settings.add_bos_token,
'dynamic_temperature': dynatemp ? true : undefined,
'dynatemp_low': dynatemp ? settings.min_temp : undefined,
'dynatemp_high': dynatemp ? settings.max_temp : undefined,
'dynatemp_range': dynatemp ? (settings.max_temp - settings.min_temp) / 2 : undefined,
'dynatemp_exponent': dynatemp ? settings.dynatemp_exponent : undefined,
2024-01-28 21:14:21 +01:00
'smoothing_factor': settings.smoothing_factor,
2024-03-01 07:06:34 +01:00
'smoothing_curve': settings.smoothing_curve,
'dry_allowed_length': settings.dry_allowed_length,
'dry_multiplier': settings.dry_multiplier,
'dry_base': settings.dry_base,
'dry_sequence_breakers': replaceMacrosInList(settings.dry_sequence_breakers),
'dry_penalty_last_n': settings.dry_penalty_last_n,
2024-02-29 00:55:25 +01:00
'max_tokens_second': settings.max_tokens_second,
2024-02-22 01:45:35 +01:00
'sampler_priority': settings.type === OOBA ? settings.sampler_priority : undefined,
'samplers': settings.type === LLAMACPP ? settings.samplers : undefined,
'stopping_strings': getStoppingStrings(isImpersonate, isContinue),
'stop': getStoppingStrings(isImpersonate, isContinue),
2023-07-20 19:32:15 +02:00
'truncation_length': max_context,
'ban_eos_token': settings.ban_eos_token,
'skip_special_tokens': settings.skip_special_tokens,
'top_a': settings.top_a,
'tfs': settings.tfs,
'epsilon_cutoff': [OOBA, MANCER].includes(settings.type) ? settings.epsilon_cutoff : undefined,
'eta_cutoff': [OOBA, MANCER].includes(settings.type) ? settings.eta_cutoff : undefined,
'mirostat_mode': settings.mirostat_mode,
'mirostat_tau': settings.mirostat_tau,
'mirostat_eta': settings.mirostat_eta,
'custom_token_bans': [APHRODITE, MANCER].includes(settings.type) ?
toIntArray(banned_tokens) :
banned_tokens,
'banned_strings': banned_strings,
'api_type': settings.type,
'api_server': getTextGenServer(),
'legacy_api': settings.legacy_api && (settings.type === OOBA || settings.type === APHRODITE),
'sampler_order': settings.type === textgen_types.KOBOLDCPP ? settings.sampler_order : undefined,
2024-08-31 19:18:51 +02:00
'xtc_threshold': settings.xtc_threshold,
'xtc_probability': settings.xtc_probability,
};
const nonAphroditeParams = {
2024-01-12 11:12:53 +01:00
'rep_pen': settings.rep_pen,
'rep_pen_range': settings.rep_pen_range,
'repetition_decay': settings.type === TABBY ? settings.rep_pen_decay : undefined,
'repetition_penalty_range': settings.rep_pen_range,
'encoder_repetition_penalty': settings.type === OOBA ? settings.encoder_rep_pen : undefined,
'no_repeat_ngram_size': settings.type === OOBA ? settings.no_repeat_ngram_size : undefined,
'penalty_alpha': settings.type === OOBA ? settings.penalty_alpha : undefined,
'temperature_last': (settings.type === OOBA || settings.type === APHRODITE || settings.type == TABBY) ? settings.temperature_last : undefined,
'speculative_ngram': settings.type === TABBY ? settings.speculative_ngram : undefined,
'do_sample': settings.type === OOBA ? settings.do_sample : undefined,
'seed': settings.seed,
'guidance_scale': cfgValues?.guidanceScale?.value ?? settings.guidance_scale ?? 1,
'negative_prompt': cfgValues?.negativePrompt ?? substituteParams(settings.negative_prompt) ?? '',
'grammar_string': settings.grammar_string,
'json_schema': [TABBY, LLAMACPP].includes(settings.type) ? settings.json_schema : undefined,
2023-12-19 15:38:11 +01:00
// llama.cpp aliases. In case someone wants to use LM Studio as Text Completion API
'repeat_penalty': settings.rep_pen,
'tfs_z': settings.tfs,
'repeat_last_n': settings.rep_pen_range,
'n_predict': maxTokens,
'num_predict': maxTokens,
'num_ctx': max_context,
2023-12-19 15:38:11 +01:00
'mirostat': settings.mirostat_mode,
'ignore_eos': settings.ban_eos_token,
'n_probs': power_user.request_token_probabilities ? 10 : undefined,
'rep_pen_slope': settings.rep_pen_slope,
2023-12-02 20:11:06 +01:00
};
2024-05-03 00:40:40 +02:00
const vllmParams = {
'n': canMultiSwipe ? settings.n : 1,
'best_of': canMultiSwipe ? settings.n : 1,
'ignore_eos': settings.ignore_eos_token,
'spaces_between_special_tokens': settings.spaces_between_special_tokens,
2024-06-24 18:16:20 +02:00
'seed': settings.seed >= 0 ? settings.seed : undefined,
2024-05-03 00:40:40 +02:00
};
const aphroditeParams = {
2023-12-03 22:24:39 +01:00
'n': canMultiSwipe ? settings.n : 1,
'frequency_penalty': settings.freq_pen,
'presence_penalty': settings.presence_pen,
'repetition_penalty': settings.rep_pen,
'seed': settings.seed,
2024-09-14 14:38:19 +02:00
'stop': getStoppingStrings(isImpersonate, isContinue),
'temperature': settings.temp,
'temperature_last': settings.temperature_last,
'top_p': settings.top_p,
'top_k': settings.top_k,
'top_a': settings.top_a,
'min_p': settings.min_p,
'tfs': settings.tfs,
'eta_cutoff': settings.eta_cutoff,
'epsilon_cutoff': settings.epsilon_cutoff,
'typical_p': settings.typical_p,
'smoothing_factor': settings.smoothing_factor,
'smoothing_curve': settings.smoothing_curve,
2024-05-03 00:40:40 +02:00
'ignore_eos': settings.ignore_eos_token,
'min_tokens': settings.min_length,
'skip_special_tokens': settings.skip_special_tokens,
2024-05-03 00:40:40 +02:00
'spaces_between_special_tokens': settings.spaces_between_special_tokens,
'guided_grammar': settings.grammar_string,
'guided_json': settings.json_schema,
2024-09-14 15:32:50 +02:00
'early_stopping': false, // hacks
'include_stop_str_in_output': false,
2023-12-02 20:11:06 +01:00
};
2024-05-06 18:26:20 +02:00
if (settings.type === OPENROUTER) {
params.provider = settings.openrouter_providers;
params.allow_fallbacks = settings.openrouter_allow_fallbacks;
2024-05-06 18:26:20 +02:00
}
2024-03-29 16:28:28 +01:00
if (settings.type === KOBOLDCPP) {
params.grammar = settings.grammar_string;
}
if (settings.type === HUGGINGFACE) {
params.top_p = Math.min(Math.max(Number(params.top_p), 0.0), 0.999);
params.stop = Array.isArray(params.stop) ? params.stop.slice(0, 4) : [];
2024-07-26 14:50:39 +02:00
nonAphroditeParams.seed = settings.seed >= 0 ? settings.seed : Math.floor(Math.random() * Math.pow(2, 32));
}
if (settings.type === MANCER) {
2024-03-24 20:42:27 +01:00
params.n = canMultiSwipe ? settings.n : 1;
params.epsilon_cutoff /= 1000;
params.eta_cutoff /= 1000;
params.dynatemp_mode = params.dynamic_temperature ? 1 : 0;
params.dynatemp_min = params.dynatemp_low;
params.dynatemp_max = params.dynatemp_high;
2024-04-14 23:39:15 +02:00
delete params.dynatemp_low;
delete params.dynatemp_high;
}
if (settings.type === TABBY) {
params.n = canMultiSwipe ? settings.n : 1;
}
2024-05-03 00:40:40 +02:00
switch (settings.type) {
case VLLM:
2024-06-24 18:16:20 +02:00
case INFERMATICAI:
2024-05-03 00:40:40 +02:00
params = Object.assign(params, vllmParams);
break;
case APHRODITE:
2024-09-14 14:38:19 +02:00
// set params to aphroditeParams
params = Object.assign(params, aphroditeParams);
2024-05-03 00:40:40 +02:00
break;
default:
params = Object.assign(params, nonAphroditeParams);
break;
}
if (Array.isArray(settings.logit_bias) && settings.logit_bias.length) {
const logitBias = BIAS_CACHE.get(BIAS_KEY) || calculateLogitBias();
BIAS_CACHE.set(BIAS_KEY, logitBias);
params.logit_bias = logitBias;
}
2023-12-19 15:38:11 +01:00
if (settings.type === LLAMACPP || settings.type === OLLAMA) {
// Convert bias and token bans to array of arrays
const logitBiasArray = (params.logit_bias && typeof params.logit_bias === 'object' && Object.keys(params.logit_bias).length > 0)
? Object.entries(params.logit_bias).map(([key, value]) => [Number(key), value])
: [];
2024-05-12 20:41:07 +02:00
const tokenBans = toIntArray(banned_tokens);
logitBiasArray.push(...tokenBans.map(x => [Number(x), false]));
const llamaCppParams = {
'logit_bias': logitBiasArray,
2023-12-19 15:38:11 +01:00
// Conflicts with ooba's grammar_string
'grammar': settings.grammar_string,
2023-12-25 01:42:03 +01:00
'cache_prompt': true,
};
params = Object.assign(params, llamaCppParams);
}
eventSource.emitAndWait(event_types.TEXT_COMPLETION_SETTINGS_READY, params);
// Grammar conflicts with with json_schema
if (settings.type === LLAMACPP) {
if (params.json_schema && Object.keys(params.json_schema).length > 0) {
delete params.grammar_string;
delete params.grammar;
} else {
delete params.json_schema;
}
}
return params;
2023-07-20 19:32:15 +02:00
}