rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into dev

Browse Source
This commit is contained in:
SillyLossy
2023-04-24 01:34:20 +03:00
parent 42769e465f 2e24aea734
commit 87eeb0ef52
5 changed files with 92 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
public/index.html
View File

@ -519,6 +519,12 @@
<span id="typical_p_counter_textgenerationwebui">select</span> <span id="typical_p_counter_textgenerationwebui">select</span>
</div> </div>
</div> </div>
<div class="range-block">
<label class="checkbox_label" for="streaming_textgenerationwebui">
<input type="checkbox" id="streaming_textgenerationwebui" />
Streaming
</label>
</div>
<div class="range-block"> <div class="range-block">
<label class="checkbox_label" for="do_sample_textgenerationwebui"> <label class="checkbox_label" for="do_sample_textgenerationwebui">
<input type="checkbox" id="do_sample_textgenerationwebui" /> <input type="checkbox" id="do_sample_textgenerationwebui" />
@ -598,15 +604,6 @@
</div> </div>
<input type="number" id="seed_textgenerationwebui" class="text_pole" maxlength="100" /> <input type="number" id="seed_textgenerationwebui" class="text_pole" maxlength="100" />
</div> </div>
<div class="range-block">
<div class="range-block-title">
Gradio Streaming Function ID
<a href="/notes/textgen_streaming" class="notes-link" target="_blank">
<span class="note-link-span">?</span>
</a>
</div>
<input type="number" id="fn_index_textgenerationwebui" class="text_pole" maxlength="100" />
</div>
</div> </div>
<div id="openai_settings"> <div id="openai_settings">
<div class=""> <div class="">
@ -886,15 +883,16 @@
</a> </a>
</div> </div>
<span> <span>
Make sure you run it in notebook/default mode<br>(not Make sure you run it with <tt>--api</tt> flag
<pre>--cai-chat</pre> or
<pre>--chat</pre>)
</span> </span>
<form action="javascript:void(null);" method="post" enctype="multipart/form-data"> <form action="javascript:void(null);" method="post" enctype="multipart/form-data">
<h4>API url</h4> <h4>Blocking API url</h4>
<h5>Example: http://127.0.0.1:7860/ </h5> <h5>Example: http://127.0.0.1:5000/</h5>
<input id="textgenerationwebui_api_url_text" name="textgenerationwebui_api_url" class="text_pole" maxlength="500" value="" autocomplete="off"> <input id="textgenerationwebui_api_url_text" name="textgenerationwebui_api_url" class="text_pole" maxlength="500" value="" autocomplete="off">
<input id="api_button_textgenerationwebui" class="menu_button" type="submit" value="Connect"> <input id="api_button_textgenerationwebui" class="menu_button" type="submit" value="Connect">
<h4>Streaming API url</h4>
<h5>Example: ws://127.0.0.1:5005/api/v1/stream</h5>
<input id="streaming_url_textgenerationwebui" type="text" class="text_pole" maxlength="500" value="" autocomplete="off">
<div id="api_loading_textgenerationwebui" class="api-load-icon fa-solid fa-hourglass fa-spin"></div> <div id="api_loading_textgenerationwebui" class="api-load-icon fa-solid fa-hourglass fa-spin"></div>
</form> </form>
<div class="online_status4"> <div class="online_status4">

25
public/notes/textgen_streaming.html
View File

@ -1,25 +0,0 @@
<html>
<head>
<title>Gradio Streaming Function ID</title>
<link rel="stylesheet" href="/css/notes.css">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link href="/webfonts/NotoSans/stylesheet.css" rel="stylesheet">
</head>
<body>
<div id="main">
<div id="content">
<h2>Gradio Streaming Function ID</h2>
<p>
To use streaming with Text Generation Web UI, a Gradio function index needs to be provided.
It is impossible to be determined programmatically and should be typed in manually.
If the streaming doesn't work with the default value, get the most recent function ID here:
<a href="https://github.com/oobabooga/text-generation-webui/blob/main/api-example-stream.py#L15">GRADIO_FN</a>
</p>
</div>
</div>
</body>
</html>

102
public/script.js
View File

@ -531,22 +531,6 @@ async function getStatus() {
kai_settings.use_stop_sequence = canUseKoboldStopSequence(data.version); kai_settings.use_stop_sequence = canUseKoboldStopSequence(data.version);
} }
// determine if streaming is enabled for ooba
if (main_api == 'textgenerationwebui' && typeof data.gradio_config == 'string') {
try {
let textGenConfig = JSON.parse(data.gradio_config);
let commandLineConfig = textGenConfig.components.filter(x => x.type == "checkboxgroup" && Array.isArray(x.props.choices) && x.props.choices.includes("no_stream"));
if (commandLineConfig.length) {
let selectedOptions = commandLineConfig[0].props.value;
textgenerationwebui_settings.streaming = !selectedOptions.includes('no_stream');
}
}
catch {
textgenerationwebui_settings.streaming = false;
}
}
//console.log(online_status); //console.log(online_status);
resultCheckStatus(); resultCheckStatus();
if (online_status !== "no_connection") { if (online_status !== "no_connection") {
@ -1331,6 +1315,12 @@ async function Generate(type, automatic_trigger, force_name2) {
return; return;
} }
if (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming && !textgenerationwebui_settings.streaming_url) {
callPopup('Streaming URL is not set. Look it up in the console window when starting TextGen Web UI', 'text');
is_send_press = false;
return;
}
if (isHordeGenerationNotAllowed()) { if (isHordeGenerationNotAllowed()) {
is_send_press = false; is_send_press = false;
return; return;
@ -1863,32 +1853,30 @@ async function Generate(type, automatic_trigger, force_name2) {
} }
if (main_api == 'textgenerationwebui') { if (main_api == 'textgenerationwebui') {
let data = [ generate_data =
finalPromt, {
{ 'prompt': finalPromt,
'max_new_tokens': this_amount_gen, 'max_new_tokens': this_amount_gen,
'do_sample': textgenerationwebui_settings.do_sample, 'do_sample': textgenerationwebui_settings.do_sample,
'temperature': textgenerationwebui_settings.temp, 'temperature': textgenerationwebui_settings.temp,
'top_p': textgenerationwebui_settings.top_p, 'top_p': textgenerationwebui_settings.top_p,
'typical_p': textgenerationwebui_settings.typical_p, 'typical_p': textgenerationwebui_settings.typical_p,
'repetition_penalty': textgenerationwebui_settings.rep_pen, 'repetition_penalty': textgenerationwebui_settings.rep_pen,
'encoder_repetition_penalty': textgenerationwebui_settings.encoder_rep_pen, 'encoder_repetition_penalty': textgenerationwebui_settings.encoder_rep_pen,
'top_k': textgenerationwebui_settings.top_k, 'top_k': textgenerationwebui_settings.top_k,
'min_length': textgenerationwebui_settings.min_length, 'min_length': textgenerationwebui_settings.min_length,
'no_repeat_ngram_size': textgenerationwebui_settings.no_repeat_ngram_size, 'no_repeat_ngram_size': textgenerationwebui_settings.no_repeat_ngram_size,
'num_beams': textgenerationwebui_settings.num_beams, 'num_beams': textgenerationwebui_settings.num_beams,
'penalty_alpha': textgenerationwebui_settings.penalty_alpha, 'penalty_alpha': textgenerationwebui_settings.penalty_alpha,
'length_penalty': textgenerationwebui_settings.length_penalty, 'length_penalty': textgenerationwebui_settings.length_penalty,
'early_stopping': textgenerationwebui_settings.early_stopping, 'early_stopping': textgenerationwebui_settings.early_stopping,
'seed': textgenerationwebui_settings.seed, 'seed': textgenerationwebui_settings.seed,
'add_bos_token': textgenerationwebui_settings.add_bos_token, 'add_bos_token': textgenerationwebui_settings.add_bos_token,
'stopping_strings': getStoppingStrings(isImpersonate, false), 'stopping_strings': getStoppingStrings(isImpersonate, false),
'truncation_length': max_context, 'truncation_length': max_context,
'ban_eos_token': textgenerationwebui_settings.ban_eos_token, 'ban_eos_token': textgenerationwebui_settings.ban_eos_token,
'skip_special_tokens': textgenerationwebui_settings.skip_special_tokens, 'skip_special_tokens': textgenerationwebui_settings.skip_special_tokens,
} };
];
generate_data = { "data": [JSON.stringify(data)] };
} }
if (main_api == 'novel') { if (main_api == 'novel') {
@ -2120,7 +2108,7 @@ function throwCircuitBreakerError() {
throw new Error('Generate circuit breaker interruption'); throw new Error('Generate circuit breaker interruption');
} }
function extractMessageFromData(data, finalPromt) { function extractMessageFromData(data) {
let getMessage = ""; let getMessage = "";
if (main_api == 'kobold' && !horde_settings.use_horde) { if (main_api == 'kobold' && !horde_settings.use_horde) {
@ -2132,13 +2120,12 @@ function extractMessageFromData(data, finalPromt) {
} }
if (main_api == 'textgenerationwebui') { if (main_api == 'textgenerationwebui') {
getMessage = data.data[0]; getMessage = data.results[0].text;
if (getMessage == null || data.error) { if (getMessage == null || data.error) {
activateSendButtons(); activateSendButtons();
callPopup('<h3>Got empty response from Text generation web UI. Try restarting the API with recommended options.</h3>', 'text'); callPopup('<h3>Got empty response from Text generation web UI. Try restarting the API with recommended options.</h3>', 'text');
return; return;
} }
getMessage = getMessage.substring(finalPromt.length);
} }
if (main_api == 'novel') { if (main_api == 'novel') {
@ -4339,24 +4326,17 @@ $(document).ready(function () {
$("#api_button_textgenerationwebui").click(function (e) { $("#api_button_textgenerationwebui").click(function (e) {
e.stopPropagation(); e.stopPropagation();
if ($("#textgenerationwebui_api_url_text").val() != "") { if ($("#textgenerationwebui_api_url_text").val() != "") {
let value = formatKoboldUrl($("#textgenerationwebui_api_url_text").val().trim());
if (!value) {
callPopup('Please enter a valid URL.', 'text');
return;
}
$("#textgenerationwebui_api_url_text").val(value);
$("#api_loading_textgenerationwebui").css("display", "inline-block"); $("#api_loading_textgenerationwebui").css("display", "inline-block");
$("#api_button_textgenerationwebui").css("display", "none"); $("#api_button_textgenerationwebui").css("display", "none");
api_server_textgenerationwebui = $( api_server_textgenerationwebui = value;
"#textgenerationwebui_api_url_text"
).val();
api_server_textgenerationwebui = $.trim(api_server_textgenerationwebui);
if (
api_server_textgenerationwebui.substr(
api_server_textgenerationwebui.length - 1,
1
) == "/"
) {
api_server_textgenerationwebui = api_server_textgenerationwebui.substr(
0,
api_server_textgenerationwebui.length - 1
);
}
//console.log("2: "+api_server_textgenerationwebui);
main_api = "textgenerationwebui"; main_api = "textgenerationwebui";
saveSettingsDebounced(); saveSettingsDebounced();
is_get_status = true; is_get_status = true;

37
public/scripts/textgen-settings.js
View File

@ -29,9 +29,9 @@ let textgenerationwebui_settings = {
stopping_strings: [], stopping_strings: [],
truncation_length: 2048, truncation_length: 2048,
ban_eos_token: false, ban_eos_token: false,
streaming: false,
fn_index: 43,
skip_special_tokens: true, skip_special_tokens: true,
streaming: false,
streaming_url: 'ws://127.0.0.1:5005/api/v1/stream',
}; };
let textgenerationwebui_presets = []; let textgenerationwebui_presets = [];
@ -54,8 +54,9 @@ const setting_names = [
"seed", "seed",
"add_bos_token", "add_bos_token",
"ban_eos_token", "ban_eos_token",
"fn_index",
"skip_special_tokens", "skip_special_tokens",
"streaming",
"streaming_url",
]; ];
function selectPreset(name) { function selectPreset(name) {
@ -109,12 +110,17 @@ $(document).ready(function () {
$(`#${i}_textgenerationwebui`).attr("x-setting-id", i); $(`#${i}_textgenerationwebui`).attr("x-setting-id", i);
$(document).on("input", `#${i}_textgenerationwebui`, function () { $(document).on("input", `#${i}_textgenerationwebui`, function () {
const isCheckbox = $(this).attr('type') == 'checkbox'; const isCheckbox = $(this).attr('type') == 'checkbox';
const isText = $(this).attr('type') == 'text';
const id = $(this).attr("x-setting-id"); const id = $(this).attr("x-setting-id");
if (isCheckbox) { if (isCheckbox) {
const value = $(this).prop('checked'); const value = $(this).prop('checked');
textgenerationwebui_settings[id] = value; textgenerationwebui_settings[id] = value;
} }
else if (isText) {
const value = $(this).val();
textgenerationwebui_settings[id] = value;
}
else { else {
const value = parseFloat($(this).val()); const value = parseFloat($(this).val());
$(`#${id}_counter_textgenerationwebui`).text(value.toFixed(2)); $(`#${id}_counter_textgenerationwebui`).text(value.toFixed(2));
@ -132,10 +138,14 @@ function setSettingByName(i, value, trigger) {
} }
const isCheckbox = $(`#${i}_textgenerationwebui`).attr('type') == 'checkbox'; const isCheckbox = $(`#${i}_textgenerationwebui`).attr('type') == 'checkbox';
const isText = $(`#${i}_textgenerationwebui`).attr('type') == 'text';
if (isCheckbox) { if (isCheckbox) {
const val = Boolean(value); const val = Boolean(value);
$(`#${i}_textgenerationwebui`).prop('checked', val); $(`#${i}_textgenerationwebui`).prop('checked', val);
} }
else if (isText) {
$(`#${i}_textgenerationwebui`).val(value);
}
else { else {
const val = parseFloat(value); const val = parseFloat(value);
$(`#${i}_textgenerationwebui`).val(val); $(`#${i}_textgenerationwebui`).val(val);
@ -150,10 +160,10 @@ function setSettingByName(i, value, trigger) {
async function generateTextGenWithStreaming(generate_data, signal) { async function generateTextGenWithStreaming(generate_data, signal) {
const response = await fetch('/generate_textgenerationwebui', { const response = await fetch('/generate_textgenerationwebui', {
headers: { headers: {
'X-CSRF-Token': token,
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'X-CSRF-Token': token,
'X-Response-Streaming': true, 'X-Response-Streaming': true,
'X-Gradio-Streaming-Function': textgenerationwebui_settings.fn_index, 'X-Streaming-URL': textgenerationwebui_settings.streaming_url,
}, },
body: JSON.stringify(generate_data), body: JSON.stringify(generate_data),
method: 'POST', method: 'POST',
@ -167,22 +177,7 @@ async function generateTextGenWithStreaming(generate_data, signal) {
while (true) { while (true) {
const { done, value } = await reader.read(); const { done, value } = await reader.read();
let response = decoder.decode(value); let response = decoder.decode(value);
let delta = ''; getMessage += response;
try {
delta = response.split('\n').map(x => {
try {
return JSON.parse(x).delta;
} catch {
return '';
}
}).join('');
}
catch {
delta = '';
}
getMessage += delta;
if (done) { if (done) {
return; return;

132
server.js
View File

@ -349,44 +349,6 @@ app.post("/generate", jsonParser, async function (request, response_generate = r
} }
}); });
function randomHash() {
const letters = 'abcdefghijklmnopqrstuvwxyz0123456789';
let result = '';
for (let i = 0; i < 9; i++) {
result += letters.charAt(Math.floor(Math.random() * letters.length));
}
return result;
}
function textGenProcessStartedHandler(websocket, content, session, prompt, fn_index) {
switch (content.msg) {
case "send_hash":
const send_hash = JSON.stringify({ "session_hash": session, "fn_index": fn_index });
websocket.send(send_hash);
break;
case "estimation":
break;
case "send_data":
const send_data = JSON.stringify({ "session_hash": session, "fn_index": fn_index, "data": prompt.data });
console.log(send_data);
websocket.send(send_data);
break;
case "process_starts":
break;
case "process_generating":
return { text: content.output.data[0], completed: false };
case "process_completed":
try {
return { text: content.output.data[0], completed: true };
}
catch {
return { text: '', completed: true };
}
}
return { text: '', completed: false };
}
//************** Text generation web UI //************** Text generation web UI
app.post("/generate_textgenerationwebui", jsonParser, async function (request, response_generate = response) { app.post("/generate_textgenerationwebui", jsonParser, async function (request, response_generate = response) {
if (!request.body) return response_generate.sendStatus(400); if (!request.body) return response_generate.sendStatus(400);
@ -394,7 +356,6 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
console.log(request.body); console.log(request.body);
if (!!request.header('X-Response-Streaming')) { if (!!request.header('X-Response-Streaming')) {
const fn_index = Number(request.header('X-Gradio-Streaming-Function'));
let isStreamingStopped = false; let isStreamingStopped = false;
request.socket.on('close', function () { request.socket.on('close', function () {
isStreamingStopped = true; isStreamingStopped = true;
@ -407,14 +368,12 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
}); });
async function* readWebsocket() { async function* readWebsocket() {
const session = randomHash(); const streamingUrl = request.header('X-Streaming-URL');
const url = new URL(api_server); const websocket = new WebSocket(streamingUrl);
const websocket = new WebSocket(`ws://${url.host}/queue/join`, { perMessageDeflate: false });
let text = '';
let completed = false;
websocket.on('open', async function () { websocket.on('open', async function () {
console.log('websocket open'); console.log('websocket open');
websocket.send(JSON.stringify(request.body));
}); });
websocket.on('error', (err) => { websocket.on('error', (err) => {
@ -427,69 +386,46 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
console.log(reason); console.log(reason);
}); });
websocket.on('message', async (message) => {
const content = json5.parse(message);
console.log(content);
let result = textGenProcessStartedHandler(websocket, content, session, request.body, fn_index);
text = result.text;
completed = result.completed;
});
while (true) { while (true) {
if (isStreamingStopped) { if (isStreamingStopped) {
console.error('Streaming stopped by user. Closing websocket...'); console.error('Streaming stopped by user. Closing websocket...');
websocket.close(); websocket.close();
return null; return;
} }
if (websocket.readyState == 0 || websocket.readyState == 1 || websocket.readyState == 2) { const rawMessage = await new Promise(resolve => websocket.once('message', resolve));
await delay(50); const message = json5.parse(rawMessage);
yield text;
if (completed || (!text && typeof text !== 'string')) { switch (message.event) {
websocket.close(); case 'text_stream':
yield null; yield message.text;
break; break;
} case 'stream_end':
} websocket.close();
else { return;
break;
} }
} }
return null;
} }
let result = JSON.parse(request.body.data)[0]; let reply = '';
let prompt = result;
let stopping_strings = JSON.parse(request.body.data)[1].stopping_strings;
try { try {
for await (const text of readWebsocket()) { for await (const text of readWebsocket()) {
if (text == null || typeof text !== 'string') { if (typeof text !== 'string') {
break; break;
} }
let newText = text.substring(result.length); let newText = text;
if (!newText) { if (!newText) {
continue; continue;
} }
result = text; reply += text;
response_generate.write(newText);
const generatedText = result.substring(prompt.length);
response_generate.write(JSON.stringify({ delta: newText }) + '\n');
if (generatedText) {
for (const str of stopping_strings) {
if (generatedText.indexOf(str) !== -1) {
break;
}
}
}
} }
console.log(reply);
} }
finally { finally {
response_generate.end(); response_generate.end();
@ -500,7 +436,7 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
data: request.body, data: request.body,
headers: { "Content-Type": "application/json" } headers: { "Content-Type": "application/json" }
}; };
client.post(api_server + "/run/textgen", args, function (data, response) { client.post(api_server + "/v1/generate", args, function (data, response) {
console.log("####", data); console.log("####", data);
if (response.statusCode == 200) { if (response.statusCode == 200) {
console.log(data); console.log(data);
@ -599,10 +535,6 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
}; };
var url = api_server + "/v1/model"; var url = api_server + "/v1/model";
let version = ''; let version = '';
if (main_api == "textgenerationwebui") {
url = api_server;
args = {}
}
if (main_api == "kobold") { if (main_api == "kobold") {
try { try {
version = (await getAsync(api_server + "/v1/info/version")).result; version = (await getAsync(api_server + "/v1/info/version")).result;
@ -613,34 +545,16 @@ app.post("/getstatus", jsonParser, async function (request, response_getstatus =
} }
client.get(url, args, function (data, response) { client.get(url, args, function (data, response) {
if (response.statusCode == 200) { if (response.statusCode == 200) {
if (main_api == "textgenerationwebui") { data.version = version;
// console.log(body); if (data.result != "ReadOnly") {
try {
var body = data.toString();
var response = body.match(/gradio_config[ =]*(\{.*\});/)[1];
if (!response)
throw "no_connection";
let model = json5.parse(response).components.filter((x) => x.props.label == "Model" && x.type == "dropdown")[0].props.value;
data = { result: model, gradio_config: response };
if (!data)
throw "no_connection";
} catch {
data = { result: "no_connection" };
}
} else { } else {
data.version = version; data.result = "no_connection";
if (data.result != "ReadOnly") {
} else {
data.result = "no_connection";
}
} }
} else { } else {
data.result = "no_connection"; data.result = "no_connection";
} }
response_getstatus.send(data); response_getstatus.send(data);
}).on('error', function (err) { }).on('error', function (err) {
//console.log(url);
//console.log('something went wrong on the request', err.request.options);
response_getstatus.send({ result: "no_connection" }); response_getstatus.send({ result: "no_connection" });
}); });
}); });