Clean up server code. Fix oobabooga multigen with streaming. Pass Gradio fn_index as a request header

2025-06-05 21:59:27 +02:00 · 2023-04-13 21:50:20 +03:00
parent 58ed03dfe0
commit f7e89b345d
5 changed files with 52 additions and 103 deletions
--- a/public/index.html
+++ b/public/index.html
@ -607,6 +607,15 @@
                                </div>
                                <input type="number" id="seed_textgenerationwebui" class="text_pole" />
                            </div>
+                            <div class="range-block">
+                                <div class="range-block-title">
+                                    Gradio Streaming Function ID
+                                    <a href="/notes/textgen_streaming" class="notes-link" target="_blank">
+                                        <span class="note-link-span">?</span>
+                                    </a>
+                                </div>
+                                <input type="number" id="fn_index_textgenerationwebui" class="text_pole" />
+                            </div>
                        </div>
                        <div id="openai_settings">
                            <div class="">
--- a/public/notes/textgen_streaming.html
+++ b/public/notes/textgen_streaming.html
@ -0,0 +1,28 @@
+<html>
+
+<head>
+    <title>Gradio Streaming Function ID</title>
+    <link rel="stylesheet" href="/css/notes.css">
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
+    <link
+        href="https://fonts.googleapis.com/css2?family=Noto+Sans:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&amp;display=swap"
+        rel="stylesheet">
+</head>
+
+<body>
+    <div id="main">
+        <div id="content">
+            <p>
+                To use streaming with Text Generation Web UI, a Gradio function index needs to be provided.
+                It is impossible to be determined programmatically and should be typed in manually.
+                If the streaming doesn't work with the default value, get the most recent function ID here:
+                <a href="https://github.com/oobabooga/text-generation-webui/blob/main/api-example-stream.py#L15">GRADIO_FN</a>
+            </p>
+        </div>
+    </div>
+</body>
+
+</html>
--- a/public/script.js
+++ b/public/script.js
@ -1244,8 +1244,8 @@ class StreamingProcessor {
            }

            try {
-                this.result = this.prefix + text;
-                this.onProgressStreaming(this.messageId, this.result);
+                this.result = text;
+                this.onProgressStreaming(this.messageId, message_already_generated + text);
            }
            catch (err) {
                console.error(err);
@ -1887,7 +1887,7 @@ async function Generate(type, automatic_trigger, force_name2) {
                }
            }
            else if (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming) {
-                streamingProcessor.generator = await generateTextGenWithStreaming(generate_data, finalPromt);
+                streamingProcessor.generator = await generateTextGenWithStreaming(generate_data);
            }
            else {
                jQuery.ajax({
@ -1919,6 +1919,8 @@ async function Generate(type, automatic_trigger, force_name2) {
                        console.log('returning to make generate again');
                        return;
                    }
+
+                    getMessage = message_already_generated;
                }
                
                if (streamingProcessor.isFinished) {
--- a/public/scripts/textgen-settings.js
+++ b/public/scripts/textgen-settings.js
@ -30,6 +30,7 @@ let textgenerationwebui_settings = {
    truncation_length: 2048,
    ban_eos_token: false,
    streaming: false,
+    fn_index: 29,
 };

 let textgenerationwebui_presets = [];
@ -52,6 +53,7 @@ const setting_names = [
    "seed",
    "add_bos_token",
    "ban_eos_token",
+    "fn_index",
 ];

 function selectPreset(name) {
@ -143,12 +145,13 @@ function setSettingByName(i, value, trigger) {
    }
 }

-async function generateTextGenWithStreaming(generate_data, finalPromt) {
+async function generateTextGenWithStreaming(generate_data) {
    const response = await fetch('/generate_textgenerationwebui', {
        headers: {
            'X-CSRF-Token': token,
            'Content-Type': 'application/json',
            'X-Response-Streaming': true,
+            'X-Gradio-Streaming-Function': textgenerationwebui_settings.fn_index,
        },
        body: JSON.stringify(generate_data),
        method: 'POST',
--- a/server.js
+++ b/server.js
@ -218,18 +218,15 @@ app.use('/characters', (req, res) => {
            res.status(404).send('File not found');
            return;
        }
-        //res.contentType('image/jpeg');
        res.send(data);
    });
 });
 app.use(multer({ dest: "uploads" }).single("avatar"));
 app.get("/", function (request, response) {
    response.sendFile(__dirname + "/public/index.html");
-    //response.send("<h1>Главная страница</h1>");
 });
 app.get("/notes/*", function (request, response) {
    response.sendFile(__dirname + "/public" + request.url + ".html");
-    //response.send("<h1>Главная страница</h1>");
 });

 //**************Kobold api
@ -319,16 +316,16 @@ function randomHash() {
    return result;
 }

-function textGenProcessStartedHandler(websocket, content, session, prompt, SEND_PROMPT_GRADIO_FN) {
+function textGenProcessStartedHandler(websocket, content, session, prompt, fn_index) {
    switch (content.msg) {
        case "send_hash":
-            const send_hash = JSON.stringify({ "session_hash": session, "fn_index": SEND_PROMPT_GRADIO_FN });
+            const send_hash = JSON.stringify({ "session_hash": session, "fn_index": fn_index });
            websocket.send(send_hash);
            break;
        case "estimation":
            break;
        case "send_data":
-            const send_data = JSON.stringify({ "session_hash": session, "fn_index": SEND_PROMPT_GRADIO_FN, "data": prompt.data });
+            const send_data = JSON.stringify({ "session_hash": session, "fn_index": fn_index, "data": prompt.data });
            console.log(send_data);
            websocket.send(send_data);
            break;
@ -350,7 +347,7 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
    console.log(request.body);

    if (!!request.header('X-Response-Streaming')) {
-        const SEND_PARAMS_GRADIO_FN = 29;
+        const fn_index = Number(request.header('X-Gradio-Streaming-Function'));

        response_generate.writeHead(200, {
            'Content-Type': 'text/plain;charset=utf-8',
@ -381,7 +378,7 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
            websocket.on('message', async (message) => {
                const content = json5.parse(message);
                console.log(content);
-                text = textGenProcessStartedHandler(websocket, content, session, request.body, SEND_PARAMS_GRADIO_FN);
+                text = textGenProcessStartedHandler(websocket, content, session, request.body, fn_index);
            });

            while (true) {
@ -450,38 +447,20 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r


 app.post("/savechat", jsonParser, function (request, response) {
-    //console.log(humanizedISO8601DateTime()+':/savechat/ entered');
-    //console.log(request.data);
-    //console.log(request.body.bg);
-    //const data = request.body;
-    //console.log(request);
-    //console.log(request.body.chat);
-    //var bg = "body {background-image: linear-gradient(rgba(19,21,44,0.75), rgba(19,21,44,0.75)), url(../backgrounds/"+request.body.bg+");}";
    var dir_name = String(request.body.avatar_url).replace('.png', '');
-    //console.log(humanizedISO8601DateTime()+':/savechat sees '+dir_name+' as the character name (derived from avatar PNG filename)');
    let chat_data = request.body.chat;
    let jsonlData = chat_data.map(JSON.stringify).join('\n');
-    //console.log(humanizedISO8601DateTime()+':/savechat saving a chat named '+request.body.file_name+'.jsonl');
    fs.writeFile(chatsPath + dir_name + "/" + request.body.file_name + '.jsonl', jsonlData, 'utf8', function (err) {
        if (err) {
            response.send(err);
            return console.log(err);
-            //response.send(err);
        } else {
-            //response.redirect("/");
            response.send({ result: "ok" });
        }
    });

 });
 app.post("/getchat", jsonParser, function (request, response) {
-    //console.log(request.data);
-    //console.log(request.body.bg);
-    //const data = request.body;
-    //console.log(request);
-    //console.log(request.body.chat);
-    //var bg = "body {background-image: linear-gradient(rgba(19,21,44,0.75), rgba(19,21,44,0.75)), url(../backgrounds/"+request.body.bg+");}";
-    //console.log(humanizedISO8601DateTime()+':/getchat entered');
    var dir_name = String(request.body.avatar_url).replace('.png', '');

    fs.stat(chatsPath + dir_name, function (err, stat) {
@ -498,7 +477,6 @@ app.post("/getchat", jsonParser, function (request, response) {
                fs.stat(chatsPath + dir_name + "/" + request.body.file_name + ".jsonl", function (err, stat) {

                    if (err === null) { //if no error (the file exists), read the file
-                        //console.log(humanizedISO8601DateTime()+':/getchat tries to access: '+chatsPath+dir_name+'/'+request.body.file_name+'.jsonl');                        
                        if (stat !== undefined) {
                            fs.readFile(chatsPath + dir_name + "/" + request.body.file_name + ".jsonl", 'utf8', (err, data) => {
                                if (err) {
@ -574,10 +552,6 @@ app.post("/getstatus", jsonParser, function (request, response_getstatus = respo
            data.result = "no_connection";
        }
        response_getstatus.send(data);
-        //console.log(response.statusCode);
-        //console.log(data);
-        //response_getstatus.send(data);
-        //data.results[0].text
    }).on('error', function (err) {
        //console.log(url);
        //console.log('something went wrong on the request', err.request.options);
@ -629,20 +603,15 @@ app.post("/setsoftprompt", jsonParser, async function (request, response) {
 });

 function checkServer() {
-    //console.log('Check run###################################################');
    api_server = 'http://127.0.0.1:5000';
    var args = {
        headers: { "Content-Type": "application/json" }
    };
    client.get(api_server + "/v1/model", args, function (data, response) {
        console.log(data.result);
-        //console.log('###################################################');
        console.log(data);
    }).on('error', function (err) {
        console.log(err);
-        //console.log('');
-        //console.log('something went wrong on the request', err.request.options);
-        //console.log('errorrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr');
    });
 }

@ -688,10 +657,6 @@ app.post("/createcharacter", urlencodedParser, function (request, response) {
        response.send("Error: A character with that name already exists.");
        //response.send({error: true});
    }
-    //console.log(request.body);
-    //response.send(request.body.ch_name);
-
-    //response.redirect("https://metanit.com")
 });


@ -892,11 +857,6 @@ app.post("/getcharacters", jsonParser, function (request, response) {
        //console.log(characters);
        response.send(JSON.stringify(characters));
    });
-    //var directories = getDirectories("public/characters");
-    //console.log(directories[0]);
-    //characters = {};
-    //character_i = 0;
-    //getCharacterFile(directories, response,0);

 });
 app.post("/getbackgrounds", jsonParser, function (request, response) {
@ -918,11 +878,6 @@ app.post("/getuseravatars", jsonParser, function (request, response) {

 });
 app.post("/setbackground", jsonParser, function (request, response) {
-    //console.log(request.data);
-    //console.log(request.body.bg);
-    //const data = request.body;
-    //console.log(request);
-    //console.log(1);
    var bg = "#bg1 {background-image: url(../backgrounds/" + request.body.bg + ");}";
    fs.writeFile('public/css/bg_load.css', bg, 'utf8', function (err) {
        if (err) {
@ -1016,8 +971,6 @@ app.post("/downloadbackground", urlencodedParser, function (request, response) {
 });

 app.post("/savesettings", jsonParser, function (request, response) {
-
-
    fs.writeFile('public/settings.json', JSON.stringify(request.body), 'utf8', function (err) {
        if (err) {
            response.send(err);
@ -1028,8 +981,8 @@ app.post("/savesettings", jsonParser, function (request, response) {
            response.send({ result: "ok" });
        }
    });
-
 });
+
 app.post('/getsettings', jsonParser, (request, response) => { //Wintermute's code
    const koboldai_settings = [];
    const koboldai_setting_names = [];
@ -1238,34 +1191,6 @@ function readWorldInfoFile(worldInfoName) {
    return worldInfo;
 }

-function getCharacterFile(directories, response, i) { //old need del
-    if (directories.length > i) {
-
-        fs.stat(charactersPath + directories[i] + '/' + directories[i] + ".json", function (err, stat) {
-            if (err == null) {
-                fs.readFile(charactersPath + directories[i] + '/' + directories[i] + ".json", 'utf8', (err, data) => {
-                    if (err) {
-                        console.error(err);
-                        return;
-                    }
-                    //console.log(data);
-
-                    characters[character_i] = {};
-                    characters[character_i] = data;
-                    i++;
-                    character_i++;
-                    getCharacterFile(directories, response, i);
-                });
-            } else {
-                i++;
-                getCharacterFile(directories, response, i);
-            }
-        });
-
-    } else {
-        response.send(JSON.stringify(characters));
-    }
-}

 function getImages(path) {
    return fs
@ -1277,17 +1202,6 @@ function getImages(path) {
        .sort(Intl.Collator().compare);
 }

-function getKoboldSettingFiles(path) {
-    return fs.readdirSync(path).sort(function (a, b) {
-        return new Date(fs.statSync(path + '/' + a).mtime) - new Date(fs.statSync(path + '/' + b).mtime);
-    }).reverse();
-}
-function getDirectories(path) {
-    return fs.readdirSync(path).sort(function (a, b) {
-        return new Date(fs.statSync(path + '/' + a).mtime) - new Date(fs.statSync(path + '/' + b).mtime);
-    }).reverse();
-}
-
 //***********Novel.ai API 

 app.post("/getstatus_novelai", jsonParser, function (request, response_getstatus_novel = response) {
@ -1320,8 +1234,6 @@ app.post("/getstatus_novelai", jsonParser, function (request, response_getstatus
    });
 });

-
-
 app.post("/generate_novelai", jsonParser, function (request, response_generate_novel = response) {
    if (!request.body) return response_generate_novel.sendStatus(400);

@ -1581,17 +1493,12 @@ app.post("/exportcharacter", jsonParser, async function (request, response) {


 app.post("/importchat", urlencodedParser, function (request, response) {
-    //console.log(humanizedISO8601DateTime()+':/importchat begun');
    if (!request.body) return response.sendStatus(400);

    var format = request.body.file_type;
    let filedata = request.file;
    let avatar_url = (request.body.avatar_url).replace('.png', '');
    let ch_name = request.body.character_name;
-    //console.log(filedata.filename);
-    //var format = request.body.file_type;
-    //console.log(format);
-    //console.log(1);
    if (filedata) {

        if (format === 'json') {