diff --git a/public/script.js b/public/script.js index 89f2e3646..6a2c0a7db 100644 --- a/public/script.js +++ b/public/script.js @@ -8,6 +8,7 @@ import { formatKoboldUrl, getKoboldGenerationData, canUseKoboldStopSequence, + canUseKoboldStreaming, } from "./scripts/kai-settings.js"; import { @@ -747,6 +748,7 @@ async function getStatus() { // determine if we can use stop sequence if (main_api === "kobold" || main_api === "koboldhorde") { kai_settings.use_stop_sequence = canUseKoboldStopSequence(data.version); + kai_settings.can_use_streaming = canUseKoboldStreaming(data.koboldVersion); } //console.log(online_status); @@ -1587,7 +1589,7 @@ function appendToStoryString(value, prefix) { function isStreamingEnabled() { return ((main_api == 'openai' && oai_settings.stream_openai) - || (main_api == 'kobold' && kai_settings.streaming_kobold) + || (main_api == 'kobold' && kai_settings.streaming_kobold && kai_settings.can_use_streaming) || (main_api == 'novel' && nai_settings.streaming_novel) || (main_api == 'poe' && poe_settings.streaming) || (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming)) @@ -1855,6 +1857,10 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject, return; } + if (main_api == 'kobold' && kai_settings.streaming_kobold && !kai_settings.can_use_streaming) { + toastr.warning('Streaming is enabled, but the version of kobold used does not support token streaming.'); + } + if (isHordeGenerationNotAllowed()) { is_send_press = false; return; diff --git a/public/scripts/kai-settings.js b/public/scripts/kai-settings.js index 63b523298..9c7eadb35 100644 --- a/public/scripts/kai-settings.js +++ b/public/scripts/kai-settings.js @@ -10,6 +10,7 @@ export { formatKoboldUrl, getKoboldGenerationData, canUseKoboldStopSequence, + canUseKoboldStreaming, }; const kai_settings = { @@ -28,6 +29,7 @@ const kai_settings = { }; const MIN_STOP_SEQUENCE_VERSION = '1.2.2'; +const MIN_STREAMING_KCPPVERSION = '1.30'; function formatKoboldUrl(value) { try { @@ -92,7 +94,7 @@ function getKoboldGenerationData(finalPromt, this_settings, this_amount_gen, thi use_world_info: false, singleline: kai_settings.single_line, stop_sequence: kai_settings.use_stop_sequence ? getStoppingStrings(isImpersonate, false) : undefined, - streaming: kai_settings.streaming_kobold, + streaming: kai_settings.streaming_kobold ? canUseKoboldStreaming() : false, }; return generate_data; } @@ -209,6 +211,12 @@ function canUseKoboldStopSequence(version) { return (version || '0.0.0').localeCompare(MIN_STOP_SEQUENCE_VERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1; } +function canUseKoboldStreaming(koboldVersion) { + if (koboldVersion = 'KoboldCpp') { + return (koboldVersion.version || '0.0').localeCompare(MIN_STREAMING_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1; + } else return false; +} + $(document).ready(function () { sliders.forEach(slider => { $(document).on("input", slider.sliderId, function () { diff --git a/server.js b/server.js index 1b9265480..e7f205272 100644 --- a/server.js +++ b/server.js @@ -389,7 +389,7 @@ app.post("/generate", jsonParser, async function (request, response_generate = r const fetch = require('node-fetch').default; const url = request.body.streaming ? `${api_server}/extra/generate/stream` : `${api_server}/v1/generate`; const response = await fetch(url, { method: 'POST', timeout: 0, ...args }); - console.log(response); + if (request.body.streaming) { // Pipe remote SSE stream to Express response response.body.pipe(response_generate); @@ -581,6 +581,7 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus = }; var url = api_server + "/v1/model"; let version = ''; + let koboldVersion = {}; if (main_api == "kobold") { try { version = (await getAsync(api_server + "/v1/info/version")).result; @@ -588,6 +589,15 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus = catch { version = '0.0.0'; } + try { + koboldVersion = await getAsync(api_server + "/extra/version"); + } + catch { + koboldVersion = { + result: 'Kobold', + version: '0.0', + }; + } } client.get(url, args, function (data, response) { if (typeof data !== 'object') { @@ -595,6 +605,7 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus = } if (response.statusCode == 200) { data.version = version; + data.koboldVersion = koboldVersion; if (data.result != "ReadOnly") { } else { data.result = "no_connection";