diff --git a/public/notes/textgen_streaming.html b/public/notes/textgen_streaming.html
deleted file mode 100644
index 85fffc742..000000000
--- a/public/notes/textgen_streaming.html
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
-
-
Gradio Streaming Function ID
-
-
-
-
-
-
-
-
-
-
Gradio Streaming Function ID
-
- To use streaming with Text Generation Web UI, a Gradio function index needs to be provided.
- It is impossible to be determined programmatically and should be typed in manually.
- If the streaming doesn't work with the default value, get the most recent function ID here:
- GRADIO_FN
-
-
-
-
-
-
\ No newline at end of file
diff --git a/public/script.js b/public/script.js
index 7ddda2ace..b83f33f33 100644
--- a/public/script.js
+++ b/public/script.js
@@ -531,22 +531,6 @@ async function getStatus() {
kai_settings.use_stop_sequence = canUseKoboldStopSequence(data.version);
}
- // determine if streaming is enabled for ooba
- if (main_api == 'textgenerationwebui' && typeof data.gradio_config == 'string') {
- try {
- let textGenConfig = JSON.parse(data.gradio_config);
- let commandLineConfig = textGenConfig.components.filter(x => x.type == "checkboxgroup" && Array.isArray(x.props.choices) && x.props.choices.includes("no_stream"));
-
- if (commandLineConfig.length) {
- let selectedOptions = commandLineConfig[0].props.value;
- textgenerationwebui_settings.streaming = !selectedOptions.includes('no_stream');
- }
- }
- catch {
- textgenerationwebui_settings.streaming = false;
- }
- }
-
//console.log(online_status);
resultCheckStatus();
if (online_status !== "no_connection") {
@@ -1331,6 +1315,12 @@ async function Generate(type, automatic_trigger, force_name2) {
return;
}
+ if (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming && !textgenerationwebui_settings.streaming_url) {
+ callPopup('Streaming URL is not set. Look it up in the console window when starting TextGen Web UI', 'text');
+ is_send_press = false;
+ return;
+ }
+
if (isHordeGenerationNotAllowed()) {
is_send_press = false;
return;
@@ -1863,32 +1853,30 @@ async function Generate(type, automatic_trigger, force_name2) {
}
if (main_api == 'textgenerationwebui') {
- let data = [
- finalPromt,
- {
- 'max_new_tokens': this_amount_gen,
- 'do_sample': textgenerationwebui_settings.do_sample,
- 'temperature': textgenerationwebui_settings.temp,
- 'top_p': textgenerationwebui_settings.top_p,
- 'typical_p': textgenerationwebui_settings.typical_p,
- 'repetition_penalty': textgenerationwebui_settings.rep_pen,
- 'encoder_repetition_penalty': textgenerationwebui_settings.encoder_rep_pen,
- 'top_k': textgenerationwebui_settings.top_k,
- 'min_length': textgenerationwebui_settings.min_length,
- 'no_repeat_ngram_size': textgenerationwebui_settings.no_repeat_ngram_size,
- 'num_beams': textgenerationwebui_settings.num_beams,
- 'penalty_alpha': textgenerationwebui_settings.penalty_alpha,
- 'length_penalty': textgenerationwebui_settings.length_penalty,
- 'early_stopping': textgenerationwebui_settings.early_stopping,
- 'seed': textgenerationwebui_settings.seed,
- 'add_bos_token': textgenerationwebui_settings.add_bos_token,
- 'stopping_strings': getStoppingStrings(isImpersonate, false),
- 'truncation_length': max_context,
- 'ban_eos_token': textgenerationwebui_settings.ban_eos_token,
- 'skip_special_tokens': textgenerationwebui_settings.skip_special_tokens,
- }
- ];
- generate_data = { "data": [JSON.stringify(data)] };
+ generate_data =
+ {
+ 'prompt': finalPromt,
+ 'max_new_tokens': this_amount_gen,
+ 'do_sample': textgenerationwebui_settings.do_sample,
+ 'temperature': textgenerationwebui_settings.temp,
+ 'top_p': textgenerationwebui_settings.top_p,
+ 'typical_p': textgenerationwebui_settings.typical_p,
+ 'repetition_penalty': textgenerationwebui_settings.rep_pen,
+ 'encoder_repetition_penalty': textgenerationwebui_settings.encoder_rep_pen,
+ 'top_k': textgenerationwebui_settings.top_k,
+ 'min_length': textgenerationwebui_settings.min_length,
+ 'no_repeat_ngram_size': textgenerationwebui_settings.no_repeat_ngram_size,
+ 'num_beams': textgenerationwebui_settings.num_beams,
+ 'penalty_alpha': textgenerationwebui_settings.penalty_alpha,
+ 'length_penalty': textgenerationwebui_settings.length_penalty,
+ 'early_stopping': textgenerationwebui_settings.early_stopping,
+ 'seed': textgenerationwebui_settings.seed,
+ 'add_bos_token': textgenerationwebui_settings.add_bos_token,
+ 'stopping_strings': getStoppingStrings(isImpersonate, false),
+ 'truncation_length': max_context,
+ 'ban_eos_token': textgenerationwebui_settings.ban_eos_token,
+ 'skip_special_tokens': textgenerationwebui_settings.skip_special_tokens,
+ };
}
if (main_api == 'novel') {
@@ -2120,7 +2108,7 @@ function throwCircuitBreakerError() {
throw new Error('Generate circuit breaker interruption');
}
-function extractMessageFromData(data, finalPromt) {
+function extractMessageFromData(data) {
let getMessage = "";
if (main_api == 'kobold' && !horde_settings.use_horde) {
@@ -2132,13 +2120,12 @@ function extractMessageFromData(data, finalPromt) {
}
if (main_api == 'textgenerationwebui') {
- getMessage = data.data[0];
+ getMessage = data.results[0].text;
if (getMessage == null || data.error) {
activateSendButtons();
callPopup('
Got empty response from Text generation web UI. Try restarting the API with recommended options. ', 'text');
return;
}
- getMessage = getMessage.substring(finalPromt.length);
}
if (main_api == 'novel') {
@@ -4339,24 +4326,17 @@ $(document).ready(function () {
$("#api_button_textgenerationwebui").click(function (e) {
e.stopPropagation();
if ($("#textgenerationwebui_api_url_text").val() != "") {
+ let value = formatKoboldUrl($("#textgenerationwebui_api_url_text").val().trim());
+
+ if (!value) {
+ callPopup('Please enter a valid URL.', 'text');
+ return;
+ }
+
+ $("#textgenerationwebui_api_url_text").val(value);
$("#api_loading_textgenerationwebui").css("display", "inline-block");
$("#api_button_textgenerationwebui").css("display", "none");
- api_server_textgenerationwebui = $(
- "#textgenerationwebui_api_url_text"
- ).val();
- api_server_textgenerationwebui = $.trim(api_server_textgenerationwebui);
- if (
- api_server_textgenerationwebui.substr(
- api_server_textgenerationwebui.length - 1,
- 1
- ) == "/"
- ) {
- api_server_textgenerationwebui = api_server_textgenerationwebui.substr(
- 0,
- api_server_textgenerationwebui.length - 1
- );
- }
- //console.log("2: "+api_server_textgenerationwebui);
+ api_server_textgenerationwebui = value;
main_api = "textgenerationwebui";
saveSettingsDebounced();
is_get_status = true;
diff --git a/public/scripts/textgen-settings.js b/public/scripts/textgen-settings.js
index 358ddeb66..6ab0107fd 100644
--- a/public/scripts/textgen-settings.js
+++ b/public/scripts/textgen-settings.js
@@ -29,9 +29,9 @@ let textgenerationwebui_settings = {
stopping_strings: [],
truncation_length: 2048,
ban_eos_token: false,
- streaming: false,
- fn_index: 43,
skip_special_tokens: true,
+ streaming: false,
+ streaming_url: 'ws://127.0.0.1:5005/api/v1/stream',
};
let textgenerationwebui_presets = [];
@@ -54,8 +54,9 @@ const setting_names = [
"seed",
"add_bos_token",
"ban_eos_token",
- "fn_index",
"skip_special_tokens",
+ "streaming",
+ "streaming_url",
];
function selectPreset(name) {
@@ -109,12 +110,17 @@ $(document).ready(function () {
$(`#${i}_textgenerationwebui`).attr("x-setting-id", i);
$(document).on("input", `#${i}_textgenerationwebui`, function () {
const isCheckbox = $(this).attr('type') == 'checkbox';
+ const isText = $(this).attr('type') == 'text';
const id = $(this).attr("x-setting-id");
if (isCheckbox) {
const value = $(this).prop('checked');
textgenerationwebui_settings[id] = value;
}
+ else if (isText) {
+ const value = $(this).val();
+ textgenerationwebui_settings[id] = value;
+ }
else {
const value = parseFloat($(this).val());
$(`#${id}_counter_textgenerationwebui`).text(value.toFixed(2));
@@ -132,10 +138,14 @@ function setSettingByName(i, value, trigger) {
}
const isCheckbox = $(`#${i}_textgenerationwebui`).attr('type') == 'checkbox';
+ const isText = $(`#${i}_textgenerationwebui`).attr('type') == 'text';
if (isCheckbox) {
const val = Boolean(value);
$(`#${i}_textgenerationwebui`).prop('checked', val);
}
+ else if (isText) {
+ $(`#${i}_textgenerationwebui`).val(value);
+ }
else {
const val = parseFloat(value);
$(`#${i}_textgenerationwebui`).val(val);
@@ -150,10 +160,10 @@ function setSettingByName(i, value, trigger) {
async function generateTextGenWithStreaming(generate_data, signal) {
const response = await fetch('/generate_textgenerationwebui', {
headers: {
- 'X-CSRF-Token': token,
'Content-Type': 'application/json',
+ 'X-CSRF-Token': token,
'X-Response-Streaming': true,
- 'X-Gradio-Streaming-Function': textgenerationwebui_settings.fn_index,
+ 'X-Streaming-URL': textgenerationwebui_settings.streaming_url,
},
body: JSON.stringify(generate_data),
method: 'POST',
@@ -167,22 +177,7 @@ async function generateTextGenWithStreaming(generate_data, signal) {
while (true) {
const { done, value } = await reader.read();
let response = decoder.decode(value);
- let delta = '';
-
- try {
- delta = response.split('\n').map(x => {
- try {
- return JSON.parse(x).delta;
- } catch {
- return '';
- }
- }).join('');
- }
- catch {
- delta = '';
- }
-
- getMessage += delta;
+ getMessage += response;
if (done) {
return;
diff --git a/server.js b/server.js
index c4ace2273..46e2c0382 100644
--- a/server.js
+++ b/server.js
@@ -349,44 +349,6 @@ app.post("/generate", jsonParser, async function (request, response_generate = r
}
});
-function randomHash() {
- const letters = 'abcdefghijklmnopqrstuvwxyz0123456789';
- let result = '';
- for (let i = 0; i < 9; i++) {
- result += letters.charAt(Math.floor(Math.random() * letters.length));
- }
- return result;
-}
-
-function textGenProcessStartedHandler(websocket, content, session, prompt, fn_index) {
- switch (content.msg) {
- case "send_hash":
- const send_hash = JSON.stringify({ "session_hash": session, "fn_index": fn_index });
- websocket.send(send_hash);
- break;
- case "estimation":
- break;
- case "send_data":
- const send_data = JSON.stringify({ "session_hash": session, "fn_index": fn_index, "data": prompt.data });
- console.log(send_data);
- websocket.send(send_data);
- break;
- case "process_starts":
- break;
- case "process_generating":
- return { text: content.output.data[0], completed: false };
- case "process_completed":
- try {
- return { text: content.output.data[0], completed: true };
- }
- catch {
- return { text: '', completed: true };
- }
- }
-
- return { text: '', completed: false };
-}
-
//************** Text generation web UI
app.post("/generate_textgenerationwebui", jsonParser, async function (request, response_generate = response) {
if (!request.body) return response_generate.sendStatus(400);
@@ -394,7 +356,6 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
console.log(request.body);
if (!!request.header('X-Response-Streaming')) {
- const fn_index = Number(request.header('X-Gradio-Streaming-Function'));
let isStreamingStopped = false;
request.socket.on('close', function () {
isStreamingStopped = true;
@@ -407,14 +368,12 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
});
async function* readWebsocket() {
- const session = randomHash();
- const url = new URL(api_server);
- const websocket = new WebSocket(`ws://${url.host}/queue/join`, { perMessageDeflate: false });
- let text = '';
- let completed = false;
+ const streamingUrl = request.header('X-Streaming-URL');
+ const websocket = new WebSocket(streamingUrl);
websocket.on('open', async function () {
console.log('websocket open');
+ websocket.send(JSON.stringify(request.body));
});
websocket.on('error', (err) => {
@@ -427,69 +386,46 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
console.log(reason);
});
- websocket.on('message', async (message) => {
- const content = json5.parse(message);
- console.log(content);
- let result = textGenProcessStartedHandler(websocket, content, session, request.body, fn_index);
- text = result.text;
- completed = result.completed;
- });
-
while (true) {
if (isStreamingStopped) {
console.error('Streaming stopped by user. Closing websocket...');
websocket.close();
- return null;
+ return;
}
- if (websocket.readyState == 0 || websocket.readyState == 1 || websocket.readyState == 2) {
- await delay(50);
- yield text;
+ const rawMessage = await new Promise(resolve => websocket.once('message', resolve));
+ const message = json5.parse(rawMessage);
- if (completed || (!text && typeof text !== 'string')) {
- websocket.close();
- yield null;
+ switch (message.event) {
+ case 'text_stream':
+ yield message.text;
break;
- }
- }
- else {
- break;
+ case 'stream_end':
+ websocket.close();
+ return;
}
}
-
- return null;
}
- let result = JSON.parse(request.body.data)[0];
- let prompt = result;
- let stopping_strings = JSON.parse(request.body.data)[1].stopping_strings;
+ let reply = '';
try {
for await (const text of readWebsocket()) {
- if (text == null || typeof text !== 'string') {
+ if (typeof text !== 'string') {
break;
}
- let newText = text.substring(result.length);
+ let newText = text;
if (!newText) {
continue;
}
- result = text;
-
- const generatedText = result.substring(prompt.length);
-
- response_generate.write(JSON.stringify({ delta: newText }) + '\n');
-
- if (generatedText) {
- for (const str of stopping_strings) {
- if (generatedText.indexOf(str) !== -1) {
- break;
- }
- }
- }
+ reply += text;
+ response_generate.write(newText);
}
+
+ console.log(reply);
}
finally {
response_generate.end();
@@ -500,7 +436,7 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
data: request.body,
headers: { "Content-Type": "application/json" }
};
- client.post(api_server + "/run/textgen", args, function (data, response) {
+ client.post(api_server + "/v1/generate", args, function (data, response) {
console.log("####", data);
if (response.statusCode == 200) {
console.log(data);
@@ -599,10 +535,6 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
};
var url = api_server + "/v1/model";
let version = '';
- if (main_api == "textgenerationwebui") {
- url = api_server;
- args = {}
- }
if (main_api == "kobold") {
try {
version = (await getAsync(api_server + "/v1/info/version")).result;
@@ -613,34 +545,16 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
}
client.get(url, args, function (data, response) {
if (response.statusCode == 200) {
- if (main_api == "textgenerationwebui") {
- // console.log(body);
- try {
- var body = data.toString();
- var response = body.match(/gradio_config[ =]*(\{.*\});/)[1];
- if (!response)
- throw "no_connection";
- let model = json5.parse(response).components.filter((x) => x.props.label == "Model" && x.type == "dropdown")[0].props.value;
- data = { result: model, gradio_config: response };
- if (!data)
- throw "no_connection";
- } catch {
- data = { result: "no_connection" };
- }
+ data.version = version;
+ if (data.result != "ReadOnly") {
} else {
- data.version = version;
- if (data.result != "ReadOnly") {
- } else {
- data.result = "no_connection";
- }
+ data.result = "no_connection";
}
} else {
data.result = "no_connection";
}
response_getstatus.send(data);
}).on('error', function (err) {
- //console.log(url);
- //console.log('something went wrong on the request', err.request.options);
response_getstatus.send({ result: "no_connection" });
});
});