rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add warning if streaming is unsupported on kobold version.

Browse Source 
minimum version of koboldcpp 1.30 is the only known implementation
to support SSE token streaming, assuming 1.30 releases with the
feature enabled.
This commit is contained in:
Zixaphir
2023-06-11 18:08:36 -07:00
parent 698461c188
commit e9427a14d5
3 changed files with 28 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
public/script.js
View File

@ -8,6 +8,7 @@ import {
formatKoboldUrl,
getKoboldGenerationData,
canUseKoboldStopSequence,
canUseKoboldStreaming,
} from "./scripts/kai-settings.js";
import {
@ -747,6 +748,7 @@ async function getStatus() {
// determine if we can use stop sequence
if (main_api === "kobold" || main_api === "koboldhorde") {
kai_settings.use_stop_sequence = canUseKoboldStopSequence(data.version);
kai_settings.can_use_streaming = canUseKoboldStreaming(data.koboldVersion);
}
//console.log(online_status);
@ -1587,7 +1589,7 @@ function appendToStoryString(value, prefix) {
function isStreamingEnabled() {
return ((main_api == 'openai' && oai_settings.stream_openai)
|| (main_api == 'kobold' && kai_settings.streaming_kobold)
|| (main_api == 'kobold' && kai_settings.streaming_kobold && kai_settings.can_use_streaming)
|| (main_api == 'novel' && nai_settings.streaming_novel)
|| (main_api == 'poe' && poe_settings.streaming)
|| (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming))
@ -1855,6 +1857,10 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
return;
}
if (main_api == 'kobold' && kai_settings.streaming_kobold && !kai_settings.can_use_streaming) {
toastr.warning('Streaming is enabled, but the version of kobold used does not support token streaming.');
}
if (isHordeGenerationNotAllowed()) {
is_send_press = false;
return;

10
public/scripts/kai-settings.js
View File

@ -10,6 +10,7 @@ export {
formatKoboldUrl,
getKoboldGenerationData,
canUseKoboldStopSequence,
canUseKoboldStreaming,
};
const kai_settings = {
@ -28,6 +29,7 @@ const kai_settings = {
};
const MIN_STOP_SEQUENCE_VERSION = '1.2.2';
const MIN_STREAMING_KCPPVERSION = '1.30';
function formatKoboldUrl(value) {
try {
@ -92,7 +94,7 @@ function getKoboldGenerationData(finalPromt, this_settings, this_amount_gen, thi
use_world_info: false,
singleline: kai_settings.single_line,
stop_sequence: kai_settings.use_stop_sequence ? getStoppingStrings(isImpersonate, false) : undefined,
streaming: kai_settings.streaming_kobold,
streaming: kai_settings.streaming_kobold ? canUseKoboldStreaming() : false,
};
return generate_data;
}
@ -209,6 +211,12 @@ function canUseKoboldStopSequence(version) {
return (version || '0.0.0').localeCompare(MIN_STOP_SEQUENCE_VERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
}
function canUseKoboldStreaming(koboldVersion) {
if (koboldVersion = 'KoboldCpp') {
return (koboldVersion.version || '0.0').localeCompare(MIN_STREAMING_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
} else return false;
}
$(document).ready(function () {
sliders.forEach(slider => {
$(document).on("input", slider.sliderId, function () {

13
server.js
View File

@ -389,7 +389,7 @@ app.post("/generate", jsonParser, async function (request, response_generate = r
const fetch = require('node-fetch').default;
const url = request.body.streaming ? `${api_server}/extra/generate/stream` : `${api_server}/v1/generate`;
const response = await fetch(url, { method: 'POST', timeout: 0, ...args });
console.log(response);
if (request.body.streaming) {
// Pipe remote SSE stream to Express response
response.body.pipe(response_generate);
@ -581,6 +581,7 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
};
var url = api_server + "/v1/model";
let version = '';
let koboldVersion = {};
if (main_api == "kobold") {
try {
version = (await getAsync(api_server + "/v1/info/version")).result;
@ -588,6 +589,15 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
catch {
version = '0.0.0';
}
try {
koboldVersion = await getAsync(api_server + "/extra/version");
}
catch {
koboldVersion = {
result: 'Kobold',
version: '0.0',
};
}
}
client.get(url, args, function (data, response) {
if (typeof data !== 'object') {
@ -595,6 +605,7 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
}
if (response.statusCode == 200) {
data.version = version;
data.koboldVersion = koboldVersion;
if (data.result != "ReadOnly") {
} else {
data.result = "no_connection";