rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into dev

Browse Source
This commit is contained in:
SillyLossy
2023-04-24 01:34:20 +03:00
parent 42769e465f 2e24aea734
commit 87eeb0ef52
5 changed files with 92 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
public/index.html
View File

@ -519,6 +519,12 @@
<span id="typical_p_counter_textgenerationwebui">select</span>
</div>
</div>
<div class="range-block">
<label class="checkbox_label" for="streaming_textgenerationwebui">
<input type="checkbox" id="streaming_textgenerationwebui" />
Streaming
</label>
</div>
<div class="range-block">
<label class="checkbox_label" for="do_sample_textgenerationwebui">
<input type="checkbox" id="do_sample_textgenerationwebui" />
@ -598,15 +604,6 @@
</div>
<input type="number" id="seed_textgenerationwebui" class="text_pole" maxlength="100" />
</div>
<div class="range-block">
<div class="range-block-title">
Gradio Streaming Function ID
<a href="/notes/textgen_streaming" class="notes-link" target="_blank">
<span class="note-link-span">?</span>
</a>
</div>
<input type="number" id="fn_index_textgenerationwebui" class="text_pole" maxlength="100" />
</div>
</div>
<div id="openai_settings">
<div class="">
@ -886,15 +883,16 @@
</a>
</div>
<span>
Make sure you run it in notebook/default mode<br>(not
<pre>--cai-chat</pre> or
<pre>--chat</pre>)
Make sure you run it with <tt>--api</tt> flag
</span>
<form action="javascript:void(null);" method="post" enctype="multipart/form-data">
<h4>API url</h4>
<h5>Example: http://127.0.0.1:7860/ </h5>
<h4>Blocking API url</h4>
<h5>Example: http://127.0.0.1:5000/</h5>
<input id="textgenerationwebui_api_url_text" name="textgenerationwebui_api_url" class="text_pole" maxlength="500" value="" autocomplete="off">
<input id="api_button_textgenerationwebui" class="menu_button" type="submit" value="Connect">
<h4>Streaming API url</h4>
<h5>Example: ws://127.0.0.1:5005/api/v1/stream</h5>
<input id="streaming_url_textgenerationwebui" type="text" class="text_pole" maxlength="500" value="" autocomplete="off">
<div id="api_loading_textgenerationwebui" class="api-load-icon fa-solid fa-hourglass fa-spin"></div>
</form>
<div class="online_status4">

25
public/notes/textgen_streaming.html
View File

@ -1,25 +0,0 @@
<html>
<head>
<title>Gradio Streaming Function ID</title>
<link rel="stylesheet" href="/css/notes.css">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link href="/webfonts/NotoSans/stylesheet.css" rel="stylesheet">
</head>
<body>
<div id="main">
<div id="content">
<h2>Gradio Streaming Function ID</h2>
<p>
To use streaming with Text Generation Web UI, a Gradio function index needs to be provided.
It is impossible to be determined programmatically and should be typed in manually.
If the streaming doesn't work with the default value, get the most recent function ID here:
<a href="https://github.com/oobabooga/text-generation-webui/blob/main/api-example-stream.py#L15">GRADIO_FN</a>
</p>
</div>
</div>
</body>
</html>

60
public/script.js
View File

@ -531,22 +531,6 @@ async function getStatus() {
kai_settings.use_stop_sequence = canUseKoboldStopSequence(data.version);
}
// determine if streaming is enabled for ooba
if (main_api == 'textgenerationwebui' && typeof data.gradio_config == 'string') {
try {
let textGenConfig = JSON.parse(data.gradio_config);
let commandLineConfig = textGenConfig.components.filter(x => x.type == "checkboxgroup" && Array.isArray(x.props.choices) && x.props.choices.includes("no_stream"));
if (commandLineConfig.length) {
let selectedOptions = commandLineConfig[0].props.value;
textgenerationwebui_settings.streaming = !selectedOptions.includes('no_stream');
}
}
catch {
textgenerationwebui_settings.streaming = false;
}
}
//console.log(online_status);
resultCheckStatus();
if (online_status !== "no_connection") {
@ -1331,6 +1315,12 @@ async function Generate(type, automatic_trigger, force_name2) {
return;
}
if (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming && !textgenerationwebui_settings.streaming_url) {
callPopup('Streaming URL is not set. Look it up in the console window when starting TextGen Web UI', 'text');
is_send_press = false;
return;
}
if (isHordeGenerationNotAllowed()) {
is_send_press = false;
return;
@ -1863,9 +1853,9 @@ async function Generate(type, automatic_trigger, force_name2) {
}
if (main_api == 'textgenerationwebui') {
let data = [
finalPromt,
generate_data =
{
'prompt': finalPromt,
'max_new_tokens': this_amount_gen,
'do_sample': textgenerationwebui_settings.do_sample,
'temperature': textgenerationwebui_settings.temp,
@ -1886,9 +1876,7 @@ async function Generate(type, automatic_trigger, force_name2) {
'truncation_length': max_context,
'ban_eos_token': textgenerationwebui_settings.ban_eos_token,
'skip_special_tokens': textgenerationwebui_settings.skip_special_tokens,
}
];
generate_data = { "data": [JSON.stringify(data)] };
};
}
if (main_api == 'novel') {
@ -2120,7 +2108,7 @@ function throwCircuitBreakerError() {
throw new Error('Generate circuit breaker interruption');
}
function extractMessageFromData(data, finalPromt) {
function extractMessageFromData(data) {
let getMessage = "";
if (main_api == 'kobold' && !horde_settings.use_horde) {
@ -2132,13 +2120,12 @@ function extractMessageFromData(data, finalPromt) {
}
if (main_api == 'textgenerationwebui') {
getMessage = data.data[0];
getMessage = data.results[0].text;
if (getMessage == null || data.error) {
activateSendButtons();
callPopup('<h3>Got empty response from Text generation web UI. Try restarting the API with recommended options.</h3>', 'text');
return;
}
getMessage = getMessage.substring(finalPromt.length);
}
if (main_api == 'novel') {
@ -4339,24 +4326,17 @@ $(document).ready(function () {
$("#api_button_textgenerationwebui").click(function (e) {
e.stopPropagation();
if ($("#textgenerationwebui_api_url_text").val() != "") {
let value = formatKoboldUrl($("#textgenerationwebui_api_url_text").val().trim());
if (!value) {
callPopup('Please enter a valid URL.', 'text');
return;
}
$("#textgenerationwebui_api_url_text").val(value);
$("#api_loading_textgenerationwebui").css("display", "inline-block");
$("#api_button_textgenerationwebui").css("display", "none");
api_server_textgenerationwebui = $(
"#textgenerationwebui_api_url_text"
).val();
api_server_textgenerationwebui = $.trim(api_server_textgenerationwebui);
if (
api_server_textgenerationwebui.substr(
api_server_textgenerationwebui.length - 1,
1
) == "/"
) {
api_server_textgenerationwebui = api_server_textgenerationwebui.substr(
0,
api_server_textgenerationwebui.length - 1
);
}
//console.log("2: "+api_server_textgenerationwebui);
api_server_textgenerationwebui = value;
main_api = "textgenerationwebui";
saveSettingsDebounced();
is_get_status = true;

37
public/scripts/textgen-settings.js
View File

@ -29,9 +29,9 @@ let textgenerationwebui_settings = {
stopping_strings: [],
truncation_length: 2048,
ban_eos_token: false,
streaming: false,
fn_index: 43,
skip_special_tokens: true,
streaming: false,
streaming_url: 'ws://127.0.0.1:5005/api/v1/stream',
};
let textgenerationwebui_presets = [];
@ -54,8 +54,9 @@ const setting_names = [
"seed",
"add_bos_token",
"ban_eos_token",
"fn_index",
"skip_special_tokens",
"streaming",
"streaming_url",
];
function selectPreset(name) {
@ -109,12 +110,17 @@ $(document).ready(function () {
$(`#${i}_textgenerationwebui`).attr("x-setting-id", i);
$(document).on("input", `#${i}_textgenerationwebui`, function () {
const isCheckbox = $(this).attr('type') == 'checkbox';
const isText = $(this).attr('type') == 'text';
const id = $(this).attr("x-setting-id");
if (isCheckbox) {
const value = $(this).prop('checked');
textgenerationwebui_settings[id] = value;
}
else if (isText) {
const value = $(this).val();
textgenerationwebui_settings[id] = value;
}
else {
const value = parseFloat($(this).val());
$(`#${id}_counter_textgenerationwebui`).text(value.toFixed(2));
@ -132,10 +138,14 @@ function setSettingByName(i, value, trigger) {
}
const isCheckbox = $(`#${i}_textgenerationwebui`).attr('type') == 'checkbox';
const isText = $(`#${i}_textgenerationwebui`).attr('type') == 'text';
if (isCheckbox) {
const val = Boolean(value);
$(`#${i}_textgenerationwebui`).prop('checked', val);
}
else if (isText) {
$(`#${i}_textgenerationwebui`).val(value);
}
else {
const val = parseFloat(value);
$(`#${i}_textgenerationwebui`).val(val);
@ -150,10 +160,10 @@ function setSettingByName(i, value, trigger) {
async function generateTextGenWithStreaming(generate_data, signal) {
const response = await fetch('/generate_textgenerationwebui', {
headers: {
'X-CSRF-Token': token,
'Content-Type': 'application/json',
'X-CSRF-Token': token,
'X-Response-Streaming': true,
'X-Gradio-Streaming-Function': textgenerationwebui_settings.fn_index,
'X-Streaming-URL': textgenerationwebui_settings.streaming_url,
},
body: JSON.stringify(generate_data),
method: 'POST',
@ -167,22 +177,7 @@ async function generateTextGenWithStreaming(generate_data, signal) {
while (true) {
const { done, value } = await reader.read();
let response = decoder.decode(value);
let delta = '';
try {
delta = response.split('\n').map(x => {
try {
return JSON.parse(x).delta;
} catch {
return '';
}
}).join('');
}
catch {
delta = '';
}
getMessage += delta;
getMessage += response;
if (done) {
return;

126
server.js
View File

@ -349,44 +349,6 @@ app.post("/generate", jsonParser, async function (request, response_generate = r
}
});
function randomHash() {
const letters = 'abcdefghijklmnopqrstuvwxyz0123456789';
let result = '';
for (let i = 0; i < 9; i++) {
result += letters.charAt(Math.floor(Math.random() * letters.length));
}
return result;
}
function textGenProcessStartedHandler(websocket, content, session, prompt, fn_index) {
switch (content.msg) {
case "send_hash":
const send_hash = JSON.stringify({ "session_hash": session, "fn_index": fn_index });
websocket.send(send_hash);
break;
case "estimation":
break;
case "send_data":
const send_data = JSON.stringify({ "session_hash": session, "fn_index": fn_index, "data": prompt.data });
console.log(send_data);
websocket.send(send_data);
break;
case "process_starts":
break;
case "process_generating":
return { text: content.output.data[0], completed: false };
case "process_completed":
try {
return { text: content.output.data[0], completed: true };
}
catch {
return { text: '', completed: true };
}
}
return { text: '', completed: false };
}
//************** Text generation web UI
app.post("/generate_textgenerationwebui", jsonParser, async function (request, response_generate = response) {
if (!request.body) return response_generate.sendStatus(400);
@ -394,7 +356,6 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
console.log(request.body);
if (!!request.header('X-Response-Streaming')) {
const fn_index = Number(request.header('X-Gradio-Streaming-Function'));
let isStreamingStopped = false;
request.socket.on('close', function () {
isStreamingStopped = true;
@ -407,14 +368,12 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
});
async function* readWebsocket() {
const session = randomHash();
const url = new URL(api_server);
const websocket = new WebSocket(`ws://${url.host}/queue/join`, { perMessageDeflate: false });
let text = '';
let completed = false;
const streamingUrl = request.header('X-Streaming-URL');
const websocket = new WebSocket(streamingUrl);
websocket.on('open', async function () {
console.log('websocket open');
websocket.send(JSON.stringify(request.body));
});
websocket.on('error', (err) => {
@ -427,69 +386,46 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
console.log(reason);
});
websocket.on('message', async (message) => {
const content = json5.parse(message);
console.log(content);
let result = textGenProcessStartedHandler(websocket, content, session, request.body, fn_index);
text = result.text;
completed = result.completed;
});
while (true) {
if (isStreamingStopped) {
console.error('Streaming stopped by user. Closing websocket...');
websocket.close();
return null;
return;
}
if (websocket.readyState == 0 || websocket.readyState == 1 || websocket.readyState == 2) {
await delay(50);
yield text;
const rawMessage = await new Promise(resolve => websocket.once('message', resolve));
const message = json5.parse(rawMessage);
if (completed || (!text && typeof text !== 'string')) {
switch (message.event) {
case 'text_stream':
yield message.text;
break;
case 'stream_end':
websocket.close();
yield null;
break;
return;
}
}
else {
break;
}
}
return null;
}
let result = JSON.parse(request.body.data)[0];
let prompt = result;
let stopping_strings = JSON.parse(request.body.data)[1].stopping_strings;
let reply = '';
try {
for await (const text of readWebsocket()) {
if (text == null || typeof text !== 'string') {
if (typeof text !== 'string') {
break;
}
let newText = text.substring(result.length);
let newText = text;
if (!newText) {
continue;
}
result = text;
reply += text;
response_generate.write(newText);
}
const generatedText = result.substring(prompt.length);
response_generate.write(JSON.stringify({ delta: newText }) + '\n');
if (generatedText) {
for (const str of stopping_strings) {
if (generatedText.indexOf(str) !== -1) {
break;
}
}
}
}
console.log(reply);
}
finally {
response_generate.end();
@ -500,7 +436,7 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
data: request.body,
headers: { "Content-Type": "application/json" }
};
client.post(api_server + "/run/textgen", args, function (data, response) {
client.post(api_server + "/v1/generate", args, function (data, response) {
console.log("####", data);
if (response.statusCode == 200) {
console.log(data);
@ -599,10 +535,6 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
};
var url = api_server + "/v1/model";
let version = '';
if (main_api == "textgenerationwebui") {
url = api_server;
args = {}
}
if (main_api == "kobold") {
try {
version = (await getAsync(api_server + "/v1/info/version")).result;
@ -613,34 +545,16 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
}
client.get(url, args, function (data, response) {
if (response.statusCode == 200) {
if (main_api == "textgenerationwebui") {
// console.log(body);
try {
var body = data.toString();
var response = body.match(/gradio_config[ =]*(\{.*\});/)[1];
if (!response)
throw "no_connection";
let model = json5.parse(response).components.filter((x) => x.props.label == "Model" && x.type == "dropdown")[0].props.value;
data = { result: model, gradio_config: response };
if (!data)
throw "no_connection";
} catch {
data = { result: "no_connection" };
}
} else {
data.version = version;
if (data.result != "ReadOnly") {
} else {
data.result = "no_connection";
}
}
} else {
data.result = "no_connection";
}
response_getstatus.send(data);
}).on('error', function (err) {
//console.log(url);
//console.log('something went wrong on the request', err.request.options);
response_getstatus.send({ result: "no_connection" });
});
});