SillyTavern/public/scripts/kai-settings.js

395 lines
13 KiB
JavaScript
Raw Normal View History

2023-07-20 19:32:15 +02:00
import {
getRequestHeaders,
saveSettingsDebounced,
getStoppingStrings,
substituteParams,
api_server,
2023-12-02 19:04:51 +01:00
} from '../script.js';
2023-07-20 19:32:15 +02:00
2023-08-04 22:49:55 +02:00
import {
power_user,
2023-12-02 19:04:51 +01:00
} from './power-user.js';
2024-04-02 13:56:15 +02:00
import { getEventSourceStream } from './sse-stream.js';
2023-12-02 19:04:51 +01:00
import { getSortableDelay } from './utils.js';
2023-08-04 22:49:55 +02:00
export const kai_settings = {
2023-07-20 19:32:15 +02:00
temp: 1,
rep_pen: 1,
rep_pen_range: 0,
top_p: 1,
2023-11-02 06:53:57 +01:00
min_p: 0,
2023-07-20 19:32:15 +02:00
top_a: 1,
top_k: 0,
typical: 1,
tfs: 1,
rep_pen_slope: 0.9,
streaming_kobold: false,
sampler_order: [0, 1, 2, 3, 4, 5, 6],
2023-09-01 00:07:04 +02:00
mirostat: 0,
mirostat_tau: 5.0,
mirostat_eta: 0.1,
use_default_badwordsids: false,
2023-12-02 19:04:51 +01:00
grammar: '',
2023-10-26 20:22:00 +02:00
seed: -1,
2023-09-01 00:07:04 +02:00
};
/**
* Stable version of KoboldAI has a nasty payload validation.
* It will reject any payload that has a key that is not in the whitelist.
* @typedef {Object.<string, boolean>} kai_flags
*/
2023-09-01 00:07:04 +02:00
export const kai_flags = {
can_use_tokenization: false,
can_use_stop_sequence: false,
can_use_streaming: false,
can_use_default_badwordsids: false,
2023-09-01 00:07:04 +02:00
can_use_mirostat: false,
can_use_grammar: false,
can_use_min_p: false,
2023-07-20 19:32:15 +02:00
};
2023-09-01 00:07:04 +02:00
const defaultValues = Object.freeze(structuredClone(kai_settings));
2023-07-20 19:32:15 +02:00
const MIN_STOP_SEQUENCE_VERSION = '1.2.2';
2023-09-01 00:07:04 +02:00
const MIN_UNBAN_VERSION = '1.2.4';
2023-07-20 19:32:15 +02:00
const MIN_STREAMING_KCPPVERSION = '1.30';
const MIN_TOKENIZATION_KCPPVERSION = '1.41';
2023-09-01 00:07:04 +02:00
const MIN_MIROSTAT_KCPPVERSION = '1.35';
const MIN_GRAMMAR_KCPPVERSION = '1.44';
const MIN_MIN_P_KCPPVERSION = '1.48';
const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];
2023-07-20 19:32:15 +02:00
export function formatKoboldUrl(value) {
2023-07-20 19:32:15 +02:00
try {
const url = new URL(value);
if (!power_user.relaxed_api_urls) {
2023-08-04 22:49:55 +02:00
url.pathname = '/api';
}
return url.toString();
2023-12-02 15:14:48 +01:00
} catch {
// Just using URL as a validation check
}
return null;
2023-07-20 19:32:15 +02:00
}
export function loadKoboldSettings(preset) {
2023-07-20 19:32:15 +02:00
for (const name of Object.keys(kai_settings)) {
2023-09-01 00:07:04 +02:00
const value = preset[name] ?? defaultValues[name];
2023-07-20 19:32:15 +02:00
const slider = sliders.find(x => x.name === name);
2023-09-01 00:07:04 +02:00
if (!slider) {
2023-07-20 19:32:15 +02:00
continue;
}
const formattedValue = slider.format(value);
2023-09-01 00:07:04 +02:00
slider.setValue(value);
$(slider.sliderId).val(value);
2023-10-26 06:20:47 +02:00
$(slider.counterId).val(formattedValue);
2023-07-20 19:32:15 +02:00
}
2023-12-02 16:21:57 +01:00
if (Object.hasOwn(preset, 'streaming_kobold')) {
2023-07-20 19:32:15 +02:00
kai_settings.streaming_kobold = preset.streaming_kobold;
$('#streaming_kobold').prop('checked', kai_settings.streaming_kobold);
}
2023-12-02 16:21:57 +01:00
if (Object.hasOwn(preset, 'use_default_badwordsids')) {
2023-09-04 00:51:14 +02:00
kai_settings.use_default_badwordsids = preset.use_default_badwordsids;
$('#use_default_badwordsids').prop('checked', kai_settings.use_default_badwordsids);
2023-09-01 00:07:04 +02:00
}
2023-07-20 19:32:15 +02:00
}
2023-10-24 15:23:32 +02:00
/**
* Gets the Kobold generation data.
* @param {string} finalPrompt Final text prompt.
* @param {object} settings Settings preset object.
* @param {number} maxLength Maximum length.
* @param {number} maxContextLength Maximum context length.
* @param {boolean} isHorde True if the generation is for a horde, false otherwise.
* @param {string} type Generation type.
* @returns {object} Kobold generation data.
*/
export function getKoboldGenerationData(finalPrompt, settings, maxLength, maxContextLength, isHorde, type) {
const isImpersonate = type === 'impersonate';
const isContinue = type === 'continue';
2023-10-24 15:23:32 +02:00
const sampler_order = kai_settings.sampler_order || settings.sampler_order;
2023-07-20 19:32:15 +02:00
let generate_data = {
2023-08-22 13:30:49 +02:00
prompt: finalPrompt,
2023-07-20 19:32:15 +02:00
gui_settings: false,
sampler_order: sampler_order,
2023-10-24 15:23:32 +02:00
max_context_length: Number(maxContextLength),
max_length: maxLength,
2023-08-22 13:30:49 +02:00
rep_pen: Number(kai_settings.rep_pen),
rep_pen_range: Number(kai_settings.rep_pen_range),
2023-07-20 19:32:15 +02:00
rep_pen_slope: kai_settings.rep_pen_slope,
2023-08-22 13:30:49 +02:00
temperature: Number(kai_settings.temp),
2023-07-20 19:32:15 +02:00
tfs: kai_settings.tfs,
top_a: kai_settings.top_a,
top_k: kai_settings.top_k,
top_p: kai_settings.top_p,
min_p: (kai_flags.can_use_min_p || isHorde) ? kai_settings.min_p : undefined,
2023-07-20 19:32:15 +02:00
typical: kai_settings.typical,
use_world_info: false,
singleline: false,
stop_sequence: (kai_flags.can_use_stop_sequence || isHorde) ? getStoppingStrings(isImpersonate, isContinue) : undefined,
2023-09-01 00:07:04 +02:00
streaming: kai_settings.streaming_kobold && kai_flags.can_use_streaming && type !== 'quiet',
can_abort: kai_flags.can_use_streaming,
mirostat: (kai_flags.can_use_mirostat || isHorde) ? kai_settings.mirostat : undefined,
mirostat_tau: (kai_flags.can_use_mirostat || isHorde) ? kai_settings.mirostat_tau : undefined,
mirostat_eta: (kai_flags.can_use_mirostat || isHorde) ? kai_settings.mirostat_eta : undefined,
use_default_badwordsids: (kai_flags.can_use_default_badwordsids || isHorde) ? kai_settings.use_default_badwordsids : undefined,
grammar: (kai_flags.can_use_grammar || isHorde) ? substituteParams(kai_settings.grammar) : undefined,
2023-10-26 20:22:00 +02:00
sampler_seed: kai_settings.seed >= 0 ? kai_settings.seed : undefined,
api_server,
2023-07-20 19:32:15 +02:00
};
return generate_data;
}
function tryParseStreamingError(response, decoded) {
try {
const data = JSON.parse(decoded);
if (!data) {
return;
}
if (data.error) {
toastr.error(data.error.message || response.statusText, 'KoboldAI API');
throw new Error(data);
}
}
catch {
// No JSON. Do nothing.
}
}
2023-07-20 19:32:15 +02:00
export async function generateKoboldWithStreaming(generate_data, signal) {
2023-12-12 05:25:25 +01:00
const response = await fetch('/api/backends/kobold/generate', {
2023-07-20 19:32:15 +02:00
headers: getRequestHeaders(),
body: JSON.stringify(generate_data),
method: 'POST',
signal: signal,
});
if (!response.ok) {
tryParseStreamingError(response, await response.text());
throw new Error(`Got response status ${response.status}`);
}
2024-04-02 13:56:15 +02:00
const eventStream = getEventSourceStream();
response.body.pipeThrough(eventStream);
const reader = eventStream.readable.getReader();
2023-07-20 19:32:15 +02:00
return async function* streamData() {
let text = '';
2023-07-20 19:32:15 +02:00
while (true) {
const { done, value } = await reader.read();
if (done) return;
2023-07-20 19:32:15 +02:00
const data = JSON.parse(value.data);
if (data?.token) {
text += data.token;
}
yield { text, swipes: [] };
2023-07-20 19:32:15 +02:00
}
2023-12-02 20:11:06 +01:00
};
2023-07-20 19:32:15 +02:00
}
const sliders = [
{
2023-12-02 19:04:51 +01:00
name: 'temp',
sliderId: '#temp',
counterId: '#temp_counter',
2023-07-20 19:32:15 +02:00
format: (val) => Number(val).toFixed(2),
setValue: (val) => { kai_settings.temp = Number(val); },
},
{
2023-12-02 19:04:51 +01:00
name: 'rep_pen',
sliderId: '#rep_pen',
counterId: '#rep_pen_counter',
2023-07-20 19:32:15 +02:00
format: (val) => Number(val).toFixed(2),
setValue: (val) => { kai_settings.rep_pen = Number(val); },
},
{
2023-12-02 19:04:51 +01:00
name: 'rep_pen_range',
sliderId: '#rep_pen_range',
counterId: '#rep_pen_range_counter',
2023-07-20 19:32:15 +02:00
format: (val) => val,
setValue: (val) => { kai_settings.rep_pen_range = Number(val); },
},
{
2023-12-02 19:04:51 +01:00
name: 'top_p',
sliderId: '#top_p',
counterId: '#top_p_counter',
2023-07-20 19:32:15 +02:00
format: (val) => val,
setValue: (val) => { kai_settings.top_p = Number(val); },
},
2023-11-02 06:53:57 +01:00
{
2023-12-02 19:04:51 +01:00
name: 'min_p',
sliderId: '#min_p',
counterId: '#min_p_counter',
2023-11-02 06:53:57 +01:00
format: (val) => val,
setValue: (val) => { kai_settings.min_p = Number(val); },
},
2023-07-20 19:32:15 +02:00
{
2023-12-02 19:04:51 +01:00
name: 'top_a',
sliderId: '#top_a',
counterId: '#top_a_counter',
2023-07-20 19:32:15 +02:00
format: (val) => val,
setValue: (val) => { kai_settings.top_a = Number(val); },
},
{
2023-12-02 19:04:51 +01:00
name: 'top_k',
sliderId: '#top_k',
counterId: '#top_k_counter',
2023-07-20 19:32:15 +02:00
format: (val) => val,
setValue: (val) => { kai_settings.top_k = Number(val); },
},
{
2023-12-02 19:04:51 +01:00
name: 'typical',
sliderId: '#typical_p',
counterId: '#typical_p_counter',
2023-07-20 19:32:15 +02:00
format: (val) => val,
setValue: (val) => { kai_settings.typical = Number(val); },
},
{
2023-12-02 19:04:51 +01:00
name: 'tfs',
sliderId: '#tfs',
counterId: '#tfs_counter',
2023-07-20 19:32:15 +02:00
format: (val) => val,
setValue: (val) => { kai_settings.tfs = Number(val); },
},
{
2023-12-02 19:04:51 +01:00
name: 'rep_pen_slope',
sliderId: '#rep_pen_slope',
counterId: '#rep_pen_slope_counter',
2023-07-20 19:32:15 +02:00
format: (val) => val,
setValue: (val) => { kai_settings.rep_pen_slope = Number(val); },
},
{
2023-12-02 19:04:51 +01:00
name: 'sampler_order',
sliderId: '#no_op_selector',
counterId: '#no_op_selector',
2023-07-20 19:32:15 +02:00
format: (val) => val,
setValue: (val) => { sortItemsByOrder(val); kai_settings.sampler_order = val; },
2023-09-01 00:07:04 +02:00
},
{
2023-12-02 19:04:51 +01:00
name: 'mirostat',
sliderId: '#mirostat_mode_kobold',
counterId: '#mirostat_mode_counter_kobold',
2023-09-01 00:07:04 +02:00
format: (val) => val,
setValue: (val) => { kai_settings.mirostat = Number(val); },
},
{
2023-12-02 19:04:51 +01:00
name: 'mirostat_tau',
sliderId: '#mirostat_tau_kobold',
counterId: '#mirostat_tau_counter_kobold',
2023-09-01 00:07:04 +02:00
format: (val) => val,
setValue: (val) => { kai_settings.mirostat_tau = Number(val); },
},
{
2023-12-02 19:04:51 +01:00
name: 'mirostat_eta',
sliderId: '#mirostat_eta_kobold',
counterId: '#mirostat_eta_counter_kobold',
2023-09-01 00:07:04 +02:00
format: (val) => val,
setValue: (val) => { kai_settings.mirostat_eta = Number(val); },
},
{
2023-12-02 19:04:51 +01:00
name: 'grammar',
sliderId: '#grammar',
counterId: '#grammar_counter_kobold',
format: (val) => val,
setValue: (val) => { kai_settings.grammar = val; },
},
2023-10-26 20:22:00 +02:00
{
2023-12-02 19:04:51 +01:00
name: 'seed',
sliderId: '#seed_kobold',
counterId: '#seed_counter_kobold',
2023-10-26 20:22:00 +02:00
format: (val) => val,
setValue: (val) => { kai_settings.seed = Number(val); },
},
2023-07-20 19:32:15 +02:00
];
2024-01-07 00:22:43 +01:00
/**
* Sets the supported feature flags for the KoboldAI backend.
* @param {string} koboldUnitedVersion Kobold United version
* @param {string} koboldCppVersion KoboldCPP version
*/
export function setKoboldFlags(koboldUnitedVersion, koboldCppVersion) {
kai_flags.can_use_stop_sequence = versionCompare(koboldUnitedVersion, MIN_STOP_SEQUENCE_VERSION);
kai_flags.can_use_streaming = versionCompare(koboldCppVersion, MIN_STREAMING_KCPPVERSION);
kai_flags.can_use_tokenization = versionCompare(koboldCppVersion, MIN_TOKENIZATION_KCPPVERSION);
kai_flags.can_use_default_badwordsids = versionCompare(koboldUnitedVersion, MIN_UNBAN_VERSION);
kai_flags.can_use_mirostat = versionCompare(koboldCppVersion, MIN_MIROSTAT_KCPPVERSION);
kai_flags.can_use_grammar = versionCompare(koboldCppVersion, MIN_GRAMMAR_KCPPVERSION);
kai_flags.can_use_min_p = versionCompare(koboldCppVersion, MIN_MIN_P_KCPPVERSION);
2024-01-07 00:22:43 +01:00
const isKoboldCpp = versionCompare(koboldCppVersion, '1.0.0');
$('#koboldcpp_hint').toggleClass('displayNone', !isKoboldCpp);
2023-09-01 00:07:04 +02:00
}
2023-08-22 13:30:49 +02:00
/**
* Compares two version numbers, returning true if srcVersion >= minVersion
* @param {string} srcVersion The current version.
* @param {string} minVersion The target version number to test against
* @returns {boolean} True if srcVersion >= minVersion, false if not
2023-08-22 13:30:49 +02:00
*/
function versionCompare(srcVersion, minVersion) {
2023-12-11 19:44:29 +01:00
return (srcVersion || '0.0.0').localeCompare(minVersion, undefined, { numeric: true, sensitivity: 'base' }) > -1;
}
2023-08-22 13:30:49 +02:00
/**
* Sorts the sampler items by the given order.
* @param {any[]} orderArray Sampler order array.
*/
2023-07-20 19:32:15 +02:00
function sortItemsByOrder(orderArray) {
console.debug('Preset samplers order: ' + orderArray);
2023-12-02 19:04:51 +01:00
const $draggableItems = $('#kobold_order');
2023-07-20 19:32:15 +02:00
for (let i = 0; i < orderArray.length; i++) {
const index = orderArray[i];
const $item = $draggableItems.find(`[data-id="${index}"]`).detach();
$draggableItems.append($item);
}
}
jQuery(function () {
2023-07-20 19:32:15 +02:00
sliders.forEach(slider => {
2023-12-02 19:04:51 +01:00
$(document).on('input', slider.sliderId, function () {
2023-07-20 19:32:15 +02:00
const value = $(this).val();
const formattedValue = slider.format(value);
slider.setValue(value);
2023-10-26 06:20:47 +02:00
$(slider.counterId).val(formattedValue);
2023-07-20 19:32:15 +02:00
saveSettingsDebounced();
});
});
2023-12-02 19:04:51 +01:00
$('#streaming_kobold').on('input', function () {
2023-09-01 00:07:04 +02:00
const value = !!$(this).prop('checked');
2023-07-20 19:32:15 +02:00
kai_settings.streaming_kobold = value;
saveSettingsDebounced();
});
2023-12-02 19:04:51 +01:00
$('#use_default_badwordsids').on('input', function () {
2023-09-01 00:07:04 +02:00
const value = !!$(this).prop('checked');
2023-09-04 00:51:14 +02:00
kai_settings.use_default_badwordsids = value;
2023-09-01 00:07:04 +02:00
saveSettingsDebounced();
});
2023-07-20 19:32:15 +02:00
$('#kobold_order').sortable({
delay: getSortableDelay(),
2023-07-20 19:32:15 +02:00
stop: function () {
const order = [];
$('#kobold_order').children().each(function () {
order.push($(this).data('id'));
});
kai_settings.sampler_order = order;
console.log('Samplers reordered:', kai_settings.sampler_order);
saveSettingsDebounced();
},
});
$('#samplers_order_recommended').on('click', function () {
kai_settings.sampler_order = KOBOLDCPP_ORDER;
sortItemsByOrder(kai_settings.sampler_order);
saveSettingsDebounced();
});
2023-07-20 19:32:15 +02:00
});