mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-06-05 21:59:27 +02:00
Horde support (untested)
This commit is contained in:
SillyLossy
@ -36,6 +36,7 @@
|
||||
<script type="module" src="scripts/kai-settings.js"></script>
|
||||
<script type="module" src="scripts/textgen-settings.js"></script>
|
||||
<script type="module" src="scripts/bookmarks.js"></script>
|
||||
<script type="module" src="scripts/horde.js"></script>
|
||||
|
||||
<title>Tavern.AI</title>
|
||||
</head>
|
||||
@ -85,11 +86,29 @@
|
||||
</div>
|
||||
<div id="kobold_api" style="position: relative;"> <!-- shows the kobold settings -->
|
||||
<form action="javascript:void(null);" method="post" enctype="multipart/form-data">
|
||||
<h4>API url</h4>
|
||||
<h5>Example: http://127.0.0.1:5000/api </h5>
|
||||
<input id="api_url_text" name="api_url" class="text_pole" maxlength="500" value="" autocomplete="off">
|
||||
<input id="api_button" class="menu_button" type="submit" value="Connect">
|
||||
<img id="api_loading" src="img/load.svg">
|
||||
<label for="use_horde" class="checkbox_label">
|
||||
<input id="use_horde" type="checkbox" />
|
||||
Use Horde. <a target="_blank" href="https://stablehorde.net/">Learn more</a>
|
||||
</label>
|
||||
<div id="kobold_api_block">
|
||||
<h4>API url</h4>
|
||||
<h5>Example: http://127.0.0.1:5000/api </h5>
|
||||
<input id="api_url_text" name="api_url" class="text_pole" maxlength="500" value="" autocomplete="off">
|
||||
<input id="api_button" class="menu_button" type="submit" value="Connect">
|
||||
<img id="api_loading" src="img/load.svg">
|
||||
</div>
|
||||
<div id="kobold_horde_block">
|
||||
|
||||
<label for="horde_auto_adjust" class="checkbox_label">
|
||||
<input id="horde_auto_adjust" type="checkbox" />
|
||||
Adjust generation to worker capabilities
|
||||
</label>
|
||||
<h4>API key</h4>
|
||||
<h5>Get it here: <a target="_blank" href="https://stablehorde.net/register">Register</a></h5>
|
||||
<input id="horde_api_key" name="horde_api_key" class="text_pole" maxlength="500" value="0000000000" autocomplete="off">
|
||||
<h4>Model</h4>
|
||||
<select id="horde_model"></select>
|
||||
</div>
|
||||
<div id="online_status2">
|
||||
<div id="online_status_indicator2"></div>
|
||||
<div id="online_status_text2">Not connected</div>
|
||||
|
||||
@ -5,6 +5,7 @@ import {
|
||||
kai_settings,
|
||||
loadKoboldSettings,
|
||||
formatKoboldUrl,
|
||||
getKoboldGenerationData,
|
||||
} from "./scripts/kai-settings.js";
|
||||
|
||||
import {
|
||||
@ -72,6 +73,14 @@ import {
|
||||
|
||||
import { showBookmarksButtons } from "./scripts/bookmarks.js";
|
||||
|
||||
import {
|
||||
horde_settings,
|
||||
loadHordeSettings,
|
||||
generateHorde,
|
||||
checkHordeStatus,
|
||||
adjustHordeGenerationParams,
|
||||
} from "./scripts/horde.js";
|
||||
|
||||
import { debounce, delay } from "./scripts/utils.js";
|
||||
|
||||
//exporting functions and vars for mods
|
||||
@ -105,6 +114,7 @@ export {
|
||||
getExtensionPrompt,
|
||||
showSwipeButtons,
|
||||
hideSwipeButtons,
|
||||
changeMainAPI,
|
||||
chat,
|
||||
this_chid,
|
||||
settings,
|
||||
@ -169,6 +179,7 @@ let optionsPopper = Popper.createPopper(document.getElementById('send_form'), do
|
||||
const durationSaveEdit = 200;
|
||||
const saveSettingsDebounced = debounce(() => saveSettings(), durationSaveEdit);
|
||||
const saveCharacterDebounced = debounce(() => $("#create_button").click(), durationSaveEdit);
|
||||
const getStatusDebounced = debounce(() => getStatus(), 5000);
|
||||
|
||||
const system_message_types = {
|
||||
HELP: "help",
|
||||
@ -429,6 +440,24 @@ function checkOnlineStatus() {
|
||||
|
||||
async function getStatus() {
|
||||
if (is_get_status) {
|
||||
if (main_api == "kobold" && horde_settings.use_horde) {
|
||||
try {
|
||||
const hordeStatus = await checkHordeStatus();
|
||||
online_status = hordeStatus ? 'Connected' : 'no_connection';
|
||||
resultCheckStatus();
|
||||
|
||||
if (online_status !== "no_connection") {
|
||||
getStatusDebounced();
|
||||
}
|
||||
}
|
||||
catch {
|
||||
online_status = "no_connection";
|
||||
resultCheckStatus();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
jQuery.ajax({
|
||||
type: "POST", //
|
||||
url: "/getstatus", //
|
||||
@ -1292,6 +1321,13 @@ async function Generate(type, automatic_trigger, force_name2) {
|
||||
this_max_context = (max_context - amount_gen);
|
||||
}
|
||||
|
||||
let hordeAmountGen = null;
|
||||
if (main_api == 'kobold' && horde_settings.use_horde && horde_settings.auto_adjust) {
|
||||
const adjustedParams = await adjustHordeGenerationParams(this_max_context, amount_gen);
|
||||
this_max_context = adjustedParams.maxContextLength;
|
||||
hordeAmountGen = adjustedParams.maxLength;
|
||||
}
|
||||
|
||||
let { worldInfoString, worldInfoBefore, worldInfoAfter } = getWorldInfoPrompt(chat2);
|
||||
let extension_prompt = getExtensionPrompt(extension_prompt_types.AFTER_SCENARIO);
|
||||
const zeroDepthAnchor = getExtensionPrompt(extension_prompt_types.IN_CHAT, 0, ' ');
|
||||
@ -1556,6 +1592,10 @@ async function Generate(type, automatic_trigger, force_name2) {
|
||||
}
|
||||
}
|
||||
|
||||
if (main_api == 'kobold' && horde_settings.use_horde && hordeAmountGen) {
|
||||
this_amount_gen = Math.min(this_amount_gen, hordeAmountGen);
|
||||
}
|
||||
|
||||
var generate_data;
|
||||
if (main_api == 'kobold') {
|
||||
var generate_data = {
|
||||
@ -1566,33 +1606,9 @@ async function Generate(type, automatic_trigger, force_name2) {
|
||||
max_context_length: max_context,
|
||||
singleline: kai_settings.single_line,
|
||||
};
|
||||
if (preset_settings != 'gui') {
|
||||
|
||||
generate_data = {
|
||||
prompt: finalPromt,
|
||||
gui_settings: false,
|
||||
sampler_order: this_settings.sampler_order,
|
||||
max_context_length: parseInt(max_context),//this_settings.max_length,
|
||||
max_length: this_amount_gen,//parseInt(amount_gen),
|
||||
rep_pen: parseFloat(kai_settings.rep_pen),
|
||||
rep_pen_range: parseInt(kai_settings.rep_pen_range),
|
||||
rep_pen_slope: kai_settings.rep_pen_slope,
|
||||
temperature: parseFloat(kai_settings.temp),
|
||||
tfs: kai_settings.tfs,
|
||||
top_a: kai_settings.top_a,
|
||||
top_k: kai_settings.top_k,
|
||||
top_p: kai_settings.top_p,
|
||||
typical: kai_settings.typical,
|
||||
s1: this_settings.sampler_order[0],
|
||||
s2: this_settings.sampler_order[1],
|
||||
s3: this_settings.sampler_order[2],
|
||||
s4: this_settings.sampler_order[3],
|
||||
s5: this_settings.sampler_order[4],
|
||||
s6: this_settings.sampler_order[5],
|
||||
s7: this_settings.sampler_order[6],
|
||||
use_world_info: false,
|
||||
singleline: kai_settings.single_line,
|
||||
};
|
||||
if (preset_settings != 'gui' || horde_settings.use_horde) {
|
||||
generate_data = getKoboldGenerationData(finalPromt, this_settings, this_amount_gen, this_max_context);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1666,6 +1682,9 @@ async function Generate(type, automatic_trigger, force_name2) {
|
||||
let prompt = await prepareOpenAIMessages(name2, storyString, worldInfoBefore, worldInfoAfter, extension_prompt, promptBias);
|
||||
sendOpenAIRequest(prompt).then(onSuccess).catch(onError);
|
||||
}
|
||||
else if (main_api == 'kobold' && horde_settings.use_horde) {
|
||||
generateHorde(finalPromt, generate_data).then(onSuccess).catch(onError);
|
||||
}
|
||||
else {
|
||||
jQuery.ajax({
|
||||
type: 'POST', //
|
||||
@ -2123,6 +2142,11 @@ function changeMainAPI() {
|
||||
online_status = "Connected";
|
||||
checkOnlineStatus();
|
||||
}
|
||||
|
||||
if (main_api == "kobold" && horde_settings.use_horde) {
|
||||
is_get_status = true;
|
||||
getStatus();
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
@ -2294,6 +2318,9 @@ async function getSettings(type) {
|
||||
// OpenAI
|
||||
loadOpenAISettings(data, settings);
|
||||
|
||||
// Horde
|
||||
loadHordeSettings(settings);
|
||||
|
||||
//Enable GUI deference settings if GUI is selected for Kobold
|
||||
if (main_api === "kobold") {
|
||||
if (preset_settings == "gui") {
|
||||
@ -2389,6 +2416,7 @@ async function saveSettings(type) {
|
||||
active_character: active_character,
|
||||
textgenerationwebui_settings: textgenerationwebui_settings,
|
||||
swipes: swipes,
|
||||
horde_settings: horde_settings,
|
||||
...nai_settings,
|
||||
...kai_settings,
|
||||
...oai_settings,
|
||||
@ -3736,7 +3764,7 @@ $(document).ready(function () {
|
||||
|
||||
$("#api_button").click(function (e) {
|
||||
e.stopPropagation();
|
||||
if ($("#api_url_text").val() != "") {
|
||||
if ($("#api_url_text").val() != "" && !horde_settings.use_horde) {
|
||||
let value = formatKoboldUrl($.trim($("#api_url_text").val()));
|
||||
|
||||
if (!value) {
|
||||
@ -3757,6 +3785,12 @@ $(document).ready(function () {
|
||||
clearSoftPromptsList();
|
||||
getSoftPromptsList();
|
||||
}
|
||||
else if (horde_settings.use_horde) {
|
||||
main_api = "kobold";
|
||||
is_get_status = true;
|
||||
getStatus();
|
||||
clearSoftPromptsList();
|
||||
}
|
||||
});
|
||||
|
||||
$("#api_button_textgenerationwebui").click(function (e) {
|
||||
|
||||
172
public/scripts/horde.js
Normal file
172
public/scripts/horde.js
Normal file
@ -0,0 +1,172 @@
|
||||
import { saveSettingsDebounced, changeMainAPI, callPopup } from "../script.js";
|
||||
import { delay } from "./utils.js";
|
||||
|
||||
export {
|
||||
horde_settings,
|
||||
generateHorde,
|
||||
checkHordeStatus,
|
||||
loadHordeSettings,
|
||||
adjustHordeGenerationParams,
|
||||
}
|
||||
|
||||
let models = [];
|
||||
|
||||
let horde_settings = {
|
||||
api_key: '0000000000',
|
||||
model: null,
|
||||
use_horde: false,
|
||||
auto_adjust: true,
|
||||
};
|
||||
|
||||
const MAX_RETRIES = 100;
|
||||
const CHECK_INTERVAL = 1000;
|
||||
|
||||
async function getWorkers() {
|
||||
const response = await fetch('https://horde.koboldai.net/api/v2/workers?type=text');
|
||||
const data = await response.json();
|
||||
return data;
|
||||
}
|
||||
|
||||
async function adjustHordeGenerationParams(max_context_length, max_length) {
|
||||
const workers = await getWorkers();
|
||||
let maxContextLength = max_context_length;
|
||||
let maxLength = max_length;
|
||||
let availableWorkers = [];
|
||||
let selectedModel = models.find(m => m.name == horde_settings.model);
|
||||
|
||||
if (!selectedModel) {
|
||||
return { maxContextLength, maxLength };
|
||||
}
|
||||
|
||||
for (const worker of workers) {
|
||||
if (selectedModel.cluster == worker.cluster && worker.models.includes(selectedModel.name)) {
|
||||
availableWorkers.push(worker);
|
||||
}
|
||||
}
|
||||
|
||||
//get the minimum requires parameters, lowest common value for all selected
|
||||
for (const worker of availableWorkers) {
|
||||
maxContextLength = Math.min(worker.max_context_length, maxContextLength);
|
||||
maxLength = Math.min(worker.max_length, maxLength);
|
||||
}
|
||||
|
||||
return { maxContextLength, maxLength };
|
||||
}
|
||||
|
||||
async function generateHorde(prompt, params) {
|
||||
delete params.prompt;
|
||||
|
||||
const payload = {
|
||||
"prompt": prompt,
|
||||
"params": params,
|
||||
"trusted_workers": false,
|
||||
"slow_workers": false,
|
||||
"models": [horde_settings.model],
|
||||
};
|
||||
|
||||
const response = await fetch("https://horde.koboldai.net/api/v2/generate/text/async", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"apikey": horde_settings.api_key,
|
||||
},
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.json();
|
||||
callPopup(error.message, 'text');
|
||||
throw new Error('Horde generation failed: ' + error.message);
|
||||
}
|
||||
|
||||
const responseJson = await response.json();
|
||||
const task_id = responseJson.id;
|
||||
console.log(`Horde task id = ${task_id}`);
|
||||
|
||||
for (let retryNumber = 0; retryNumber < MAX_RETRIES; retryNumber++) {
|
||||
await delay(CHECK_INTERVAL);
|
||||
|
||||
const statusCheckResponse = await fetch(`https://horde.koboldai.net/api/v2/generate/text/status/${task_id}`, {
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"apikey": horde_settings.api_key,
|
||||
}
|
||||
});
|
||||
|
||||
const statusCheckJson = await statusCheckResponse.json();
|
||||
console.log(statusCheckJson);
|
||||
|
||||
if (statusCheckJson.done) {
|
||||
const generatedText = statusCheckJson.generations[0];
|
||||
console.log(generatedText);
|
||||
return generatedText;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function checkHordeStatus() {
|
||||
const response = await fetch('https://horde.koboldai.net/api/v2/status/heartbeat');
|
||||
return response.ok;
|
||||
}
|
||||
|
||||
async function getHordeModels() {
|
||||
$('#horde_model').empty();
|
||||
const response = await fetch('https://horde.koboldai.net/api/v2/status/models?type=text');
|
||||
models = await response.json();
|
||||
|
||||
for (const model of models) {
|
||||
const option = document.createElement('option');
|
||||
option.value = model.name;
|
||||
option.innerText = `${model.name} (Queue: ${model.queued}, Workers: ${model.count})`;
|
||||
option.selected = horde_settings.model === model.name;
|
||||
$('#horde_model').append(option);
|
||||
}
|
||||
}
|
||||
|
||||
function loadHordeSettings(settings) {
|
||||
if (settings.horde_settings) {
|
||||
Object.assign(horde_settings, settings.horde_settings);
|
||||
}
|
||||
|
||||
$('#use_horde').prop("checked", horde_settings.use_horde).trigger('input');
|
||||
$('#horde_api_key').val(horde_settings.api_key);
|
||||
$('#horde_auto_adjust').prop("checked", horde_settings.auto_adjust);
|
||||
}
|
||||
|
||||
$(document).ready(function () {
|
||||
$("#use_horde").on("input", async function () {
|
||||
horde_settings.use_horde = !!$(this).prop("checked");
|
||||
|
||||
if (horde_settings.use_horde) {
|
||||
$('#kobold_api_block').hide();
|
||||
$('#kobold_horde_block').show();
|
||||
}
|
||||
else {
|
||||
$('#kobold_api_block').show();
|
||||
$('#kobold_horde_block').hide();
|
||||
}
|
||||
|
||||
// Trigger status check
|
||||
changeMainAPI();
|
||||
saveSettingsDebounced();
|
||||
|
||||
if (horde_settings.use_horde) {
|
||||
await getHordeModels();
|
||||
}
|
||||
});
|
||||
|
||||
$("#horde_model").on("change", function () {
|
||||
horde_settings.model = $(this).val();
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
$("#horde_api_key").on("input", function () {
|
||||
horde_settings.api_key = $(this).val();
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
$("#horde_auto_adjust").on("input", function () {
|
||||
horde_settings.auto_adjust = !!$(this).prop("checked");
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
})
|
||||
@ -6,6 +6,7 @@ export {
|
||||
kai_settings,
|
||||
loadKoboldSettings,
|
||||
formatKoboldUrl,
|
||||
getKoboldGenerationData,
|
||||
};
|
||||
|
||||
const kai_settings = {
|
||||
@ -54,6 +55,35 @@ function loadKoboldSettings(preset) {
|
||||
}
|
||||
}
|
||||
|
||||
function getKoboldGenerationData(finalPromt, this_settings, this_amount_gen, this_max_context) {
|
||||
let generate_data = {
|
||||
prompt: finalPromt,
|
||||
gui_settings: false,
|
||||
sampler_order: this_settings.sampler_order,
|
||||
max_context_length: parseInt(this_max_context),
|
||||
max_length: this_amount_gen,
|
||||
rep_pen: parseFloat(kai_settings.rep_pen),
|
||||
rep_pen_range: parseInt(kai_settings.rep_pen_range),
|
||||
rep_pen_slope: kai_settings.rep_pen_slope,
|
||||
temperature: parseFloat(kai_settings.temp),
|
||||
tfs: kai_settings.tfs,
|
||||
top_a: kai_settings.top_a,
|
||||
top_k: kai_settings.top_k,
|
||||
top_p: kai_settings.top_p,
|
||||
typical: kai_settings.typical,
|
||||
s1: this_settings.sampler_order[0],
|
||||
s2: this_settings.sampler_order[1],
|
||||
s3: this_settings.sampler_order[2],
|
||||
s4: this_settings.sampler_order[3],
|
||||
s5: this_settings.sampler_order[4],
|
||||
s6: this_settings.sampler_order[5],
|
||||
s7: this_settings.sampler_order[6],
|
||||
use_world_info: false,
|
||||
singleline: kai_settings.single_line,
|
||||
};
|
||||
return generate_data;
|
||||
}
|
||||
|
||||
const sliders = [
|
||||
{
|
||||
name: "temp",
|
||||
|
||||
Reference in New Issue
Block a user