From e9427a14d5e97c74d95518da19ac4f25589cda24 Mon Sep 17 00:00:00 2001
From: Zixaphir <zixaphirmoxphar@gmail.com>
Date: Sun, 11 Jun 2023 18:08:36 -0700
Subject: [PATCH] Add warning if streaming is unsupported on kobold version.

minimum version of koboldcpp 1.30 is the only known implementation
to support SSE token streaming, assuming 1.30 releases with the
feature enabled.
---
 public/script.js               |  8 +++++++-
 public/scripts/kai-settings.js | 10 +++++++++-
 server.js                      | 13 ++++++++++++-
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/public/script.js b/public/script.js
index 89f2e3646..6a2c0a7db 100644
--- a/public/script.js
+++ b/public/script.js
@@ -8,6 +8,7 @@ import {
     formatKoboldUrl,
     getKoboldGenerationData,
     canUseKoboldStopSequence,
+    canUseKoboldStreaming,
 } from "./scripts/kai-settings.js";
 
 import {
@@ -747,6 +748,7 @@ async function getStatus() {
                 // determine if we can use stop sequence
                 if (main_api === "kobold" || main_api === "koboldhorde") {
                     kai_settings.use_stop_sequence = canUseKoboldStopSequence(data.version);
+                    kai_settings.can_use_streaming = canUseKoboldStreaming(data.koboldVersion);
                 }
 
                 //console.log(online_status);
@@ -1587,7 +1589,7 @@ function appendToStoryString(value, prefix) {
 
 function isStreamingEnabled() {
     return ((main_api == 'openai' && oai_settings.stream_openai)
-        || (main_api == 'kobold' && kai_settings.streaming_kobold)
+        || (main_api == 'kobold' && kai_settings.streaming_kobold && kai_settings.can_use_streaming)
         || (main_api == 'novel' && nai_settings.streaming_novel)
         || (main_api == 'poe' && poe_settings.streaming)
         || (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming))
@@ -1855,6 +1857,10 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
         return;
     }
 
+    if (main_api == 'kobold' && kai_settings.streaming_kobold && !kai_settings.can_use_streaming) {
+        toastr.warning('Streaming is enabled, but the version of kobold used does not support token streaming.');
+    }
+
     if (isHordeGenerationNotAllowed()) {
         is_send_press = false;
         return;
diff --git a/public/scripts/kai-settings.js b/public/scripts/kai-settings.js
index 63b523298..9c7eadb35 100644
--- a/public/scripts/kai-settings.js
+++ b/public/scripts/kai-settings.js
@@ -10,6 +10,7 @@ export {
     formatKoboldUrl,
     getKoboldGenerationData,
     canUseKoboldStopSequence,
+    canUseKoboldStreaming,
 };
 
 const kai_settings = {
@@ -28,6 +29,7 @@ const kai_settings = {
 };
 
 const MIN_STOP_SEQUENCE_VERSION = '1.2.2';
+const MIN_STREAMING_KCPPVERSION = '1.30';
 
 function formatKoboldUrl(value) {
     try {
@@ -92,7 +94,7 @@ function getKoboldGenerationData(finalPromt, this_settings, this_amount_gen, thi
         use_world_info: false,
         singleline: kai_settings.single_line,
         stop_sequence: kai_settings.use_stop_sequence ? getStoppingStrings(isImpersonate, false) : undefined,
-        streaming: kai_settings.streaming_kobold,
+        streaming: kai_settings.streaming_kobold ? canUseKoboldStreaming() : false,
     };
     return generate_data;
 }
@@ -209,6 +211,12 @@ function canUseKoboldStopSequence(version) {
     return (version || '0.0.0').localeCompare(MIN_STOP_SEQUENCE_VERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
 }
 
+function canUseKoboldStreaming(koboldVersion) {
+    if (koboldVersion = 'KoboldCpp') {
+        return (koboldVersion.version || '0.0').localeCompare(MIN_STREAMING_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
+    } else return false;
+}
+
 $(document).ready(function () {
     sliders.forEach(slider => {
         $(document).on("input", slider.sliderId, function () {
diff --git a/server.js b/server.js
index 1b9265480..e7f205272 100644
--- a/server.js
+++ b/server.js
@@ -389,7 +389,7 @@ app.post("/generate", jsonParser, async function (request, response_generate = r
         const fetch = require('node-fetch').default;
         const url = request.body.streaming ? `${api_server}/extra/generate/stream` : `${api_server}/v1/generate`;
         const response = await fetch(url, { method: 'POST', timeout: 0, ...args });
-        console.log(response);
+
         if (request.body.streaming) {
             // Pipe remote SSE stream to Express response
             response.body.pipe(response_generate);
@@ -581,6 +581,7 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
     };
     var url = api_server + "/v1/model";
     let version = '';
+    let koboldVersion = {};
     if (main_api == "kobold") {
         try {
             version = (await getAsync(api_server + "/v1/info/version")).result;
@@ -588,6 +589,15 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
         catch {
             version = '0.0.0';
         }
+        try {
+            koboldVersion = await getAsync(api_server + "/extra/version");
+        }
+        catch {
+            koboldVersion = {
+                result: 'Kobold',
+                version: '0.0',
+            };
+        }
     }
     client.get(url, args, function (data, response) {
         if (typeof data !== 'object') {
@@ -595,6 +605,7 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
         }
         if (response.statusCode == 200) {
             data.version = version;
+            data.koboldVersion = koboldVersion;
             if (data.result != "ReadOnly") {
             } else {
                 data.result = "no_connection";