rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Clean up server code. Fix oobabooga multigen with streaming. Pass Gradio fn_index as a request header

Browse Source
This commit is contained in:
SillyLossy
2023-04-13 21:50:20 +03:00
parent 58ed03dfe0
commit f7e89b345d
5 changed files with 52 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
public/index.html
View File

@ -607,6 +607,15 @@
</div> </div>
<input type="number" id="seed_textgenerationwebui" class="text_pole" /> <input type="number" id="seed_textgenerationwebui" class="text_pole" />
</div> </div>
<div class="range-block">
<div class="range-block-title">
Gradio Streaming Function ID
<a href="/notes/textgen_streaming" class="notes-link" target="_blank">
<span class="note-link-span">?</span>
</a>
</div>
<input type="number" id="fn_index_textgenerationwebui" class="text_pole" />
</div>
</div> </div>
<div id="openai_settings"> <div id="openai_settings">
<div class=""> <div class="">

28
public/notes/textgen_streaming.html Normal file
View File

@ -0,0 +1,28 @@
<html>
<head>
<title>Gradio Streaming Function ID</title>
<link rel="stylesheet" href="/css/notes.css">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
<link
href="https://fonts.googleapis.com/css2?family=Noto+Sans:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&amp;display=swap"
rel="stylesheet">
</head>
<body>
<div id="main">
<div id="content">
<p>
To use streaming with Text Generation Web UI, a Gradio function index needs to be provided.
It is impossible to be determined programmatically and should be typed in manually.
If the streaming doesn't work with the default value, get the most recent function ID here:
<a href="https://github.com/oobabooga/text-generation-webui/blob/main/api-example-stream.py#L15">GRADIO_FN</a>
</p>
</div>
</div>
</body>
</html>

8
public/script.js
View File

@ -1244,8 +1244,8 @@ class StreamingProcessor {
} }
try { try {
this.result = this.prefix + text; this.result = text;
this.onProgressStreaming(this.messageId, this.result); this.onProgressStreaming(this.messageId, message_already_generated + text);
} }
catch (err) { catch (err) {
console.error(err); console.error(err);
@ -1887,7 +1887,7 @@ async function Generate(type, automatic_trigger, force_name2) {
} }
} }
else if (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming) { else if (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming) {
streamingProcessor.generator = await generateTextGenWithStreaming(generate_data, finalPromt); streamingProcessor.generator = await generateTextGenWithStreaming(generate_data);
} }
else { else {
jQuery.ajax({ jQuery.ajax({
@ -1919,6 +1919,8 @@ async function Generate(type, automatic_trigger, force_name2) {
console.log('returning to make generate again'); console.log('returning to make generate again');
return; return;
} }
getMessage = message_already_generated;
} }
if (streamingProcessor.isFinished) { if (streamingProcessor.isFinished) {

5
public/scripts/textgen-settings.js
View File

@ -30,6 +30,7 @@ let textgenerationwebui_settings = {
truncation_length: 2048, truncation_length: 2048,
ban_eos_token: false, ban_eos_token: false,
streaming: false, streaming: false,
fn_index: 29,
}; };
let textgenerationwebui_presets = []; let textgenerationwebui_presets = [];
@ -52,6 +53,7 @@ const setting_names = [
"seed", "seed",
"add_bos_token", "add_bos_token",
"ban_eos_token", "ban_eos_token",
"fn_index",
]; ];
function selectPreset(name) { function selectPreset(name) {
@ -143,12 +145,13 @@ function setSettingByName(i, value, trigger) {
} }
} }
async function generateTextGenWithStreaming(generate_data, finalPromt) { async function generateTextGenWithStreaming(generate_data) {
const response = await fetch('/generate_textgenerationwebui', { const response = await fetch('/generate_textgenerationwebui', {
headers: { headers: {
'X-CSRF-Token': token, 'X-CSRF-Token': token,
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'X-Response-Streaming': true, 'X-Response-Streaming': true,
'X-Gradio-Streaming-Function': textgenerationwebui_settings.fn_index,
}, },
body: JSON.stringify(generate_data), body: JSON.stringify(generate_data),
method: 'POST', method: 'POST',

105
server.js
View File

@ -218,18 +218,15 @@ app.use('/characters', (req, res) => {
res.status(404).send('File not found'); res.status(404).send('File not found');
return; return;
} }
//res.contentType('image/jpeg');
res.send(data); res.send(data);
}); });
}); });
app.use(multer({ dest: "uploads" }).single("avatar")); app.use(multer({ dest: "uploads" }).single("avatar"));
app.get("/", function (request, response) { app.get("/", function (request, response) {
response.sendFile(__dirname + "/public/index.html"); response.sendFile(__dirname + "/public/index.html");
//response.send("<h1>Главная страница</h1>");
}); });
app.get("/notes/*", function (request, response) { app.get("/notes/*", function (request, response) {
response.sendFile(__dirname + "/public" + request.url + ".html"); response.sendFile(__dirname + "/public" + request.url + ".html");
//response.send("<h1>Главная страница</h1>");
}); });
//**************Kobold api //**************Kobold api
@ -319,16 +316,16 @@ function randomHash() {
return result; return result;
} }
function textGenProcessStartedHandler(websocket, content, session, prompt, SEND_PROMPT_GRADIO_FN) { function textGenProcessStartedHandler(websocket, content, session, prompt, fn_index) {
switch (content.msg) { switch (content.msg) {
case "send_hash": case "send_hash":
const send_hash = JSON.stringify({ "session_hash": session, "fn_index": SEND_PROMPT_GRADIO_FN }); const send_hash = JSON.stringify({ "session_hash": session, "fn_index": fn_index });
websocket.send(send_hash); websocket.send(send_hash);
break; break;
case "estimation": case "estimation":
break; break;
case "send_data": case "send_data":
const send_data = JSON.stringify({ "session_hash": session, "fn_index": SEND_PROMPT_GRADIO_FN, "data": prompt.data }); const send_data = JSON.stringify({ "session_hash": session, "fn_index": fn_index, "data": prompt.data });
console.log(send_data); console.log(send_data);
websocket.send(send_data); websocket.send(send_data);
break; break;
@ -350,7 +347,7 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
console.log(request.body); console.log(request.body);
if (!!request.header('X-Response-Streaming')) { if (!!request.header('X-Response-Streaming')) {
const SEND_PARAMS_GRADIO_FN = 29; const fn_index = Number(request.header('X-Gradio-Streaming-Function'));
response_generate.writeHead(200, { response_generate.writeHead(200, {
'Content-Type': 'text/plain;charset=utf-8', 'Content-Type': 'text/plain;charset=utf-8',
@ -381,7 +378,7 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
websocket.on('message', async (message) => { websocket.on('message', async (message) => {
const content = json5.parse(message); const content = json5.parse(message);
console.log(content); console.log(content);
text = textGenProcessStartedHandler(websocket, content, session, request.body, SEND_PARAMS_GRADIO_FN); text = textGenProcessStartedHandler(websocket, content, session, request.body, fn_index);
}); });
while (true) { while (true) {
@ -450,38 +447,20 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
app.post("/savechat", jsonParser, function (request, response) { app.post("/savechat", jsonParser, function (request, response) {
//console.log(humanizedISO8601DateTime()+':/savechat/ entered');
//console.log(request.data);
//console.log(request.body.bg);
//const data = request.body;
//console.log(request);
//console.log(request.body.chat);
//var bg = "body {background-image: linear-gradient(rgba(19,21,44,0.75), rgba(19,21,44,0.75)), url(../backgrounds/"+request.body.bg+");}";
var dir_name = String(request.body.avatar_url).replace('.png', ''); var dir_name = String(request.body.avatar_url).replace('.png', '');
//console.log(humanizedISO8601DateTime()+':/savechat sees '+dir_name+' as the character name (derived from avatar PNG filename)');
let chat_data = request.body.chat; let chat_data = request.body.chat;
let jsonlData = chat_data.map(JSON.stringify).join('\n'); let jsonlData = chat_data.map(JSON.stringify).join('\n');
//console.log(humanizedISO8601DateTime()+':/savechat saving a chat named '+request.body.file_name+'.jsonl');
fs.writeFile(chatsPath + dir_name + "/" + request.body.file_name + '.jsonl', jsonlData, 'utf8', function (err) { fs.writeFile(chatsPath + dir_name + "/" + request.body.file_name + '.jsonl', jsonlData, 'utf8', function (err) {
if (err) { if (err) {
response.send(err); response.send(err);
return console.log(err); return console.log(err);
//response.send(err);
} else { } else {
//response.redirect("/");
response.send({ result: "ok" }); response.send({ result: "ok" });
} }
}); });
}); });
app.post("/getchat", jsonParser, function (request, response) { app.post("/getchat", jsonParser, function (request, response) {
//console.log(request.data);
//console.log(request.body.bg);
//const data = request.body;
//console.log(request);
//console.log(request.body.chat);
//var bg = "body {background-image: linear-gradient(rgba(19,21,44,0.75), rgba(19,21,44,0.75)), url(../backgrounds/"+request.body.bg+");}";
//console.log(humanizedISO8601DateTime()+':/getchat entered');
var dir_name = String(request.body.avatar_url).replace('.png', ''); var dir_name = String(request.body.avatar_url).replace('.png', '');
fs.stat(chatsPath + dir_name, function (err, stat) { fs.stat(chatsPath + dir_name, function (err, stat) {
@ -498,7 +477,6 @@ app.post("/getchat", jsonParser, function (request, response) {
fs.stat(chatsPath + dir_name + "/" + request.body.file_name + ".jsonl", function (err, stat) { fs.stat(chatsPath + dir_name + "/" + request.body.file_name + ".jsonl", function (err, stat) {
if (err === null) { //if no error (the file exists), read the file if (err === null) { //if no error (the file exists), read the file
//console.log(humanizedISO8601DateTime()+':/getchat tries to access: '+chatsPath+dir_name+'/'+request.body.file_name+'.jsonl');
if (stat !== undefined) { if (stat !== undefined) {
fs.readFile(chatsPath + dir_name + "/" + request.body.file_name + ".jsonl", 'utf8', (err, data) => { fs.readFile(chatsPath + dir_name + "/" + request.body.file_name + ".jsonl", 'utf8', (err, data) => {
if (err) { if (err) {
@ -574,10 +552,6 @@ app.post("/getstatus", jsonParser, function (request, response_getstatus = respo
data.result = "no_connection"; data.result = "no_connection";
} }
response_getstatus.send(data); response_getstatus.send(data);
//console.log(response.statusCode);
//console.log(data);
//response_getstatus.send(data);
//data.results[0].text
}).on('error', function (err) { }).on('error', function (err) {
//console.log(url); //console.log(url);
//console.log('something went wrong on the request', err.request.options); //console.log('something went wrong on the request', err.request.options);
@ -629,20 +603,15 @@ app.post("/setsoftprompt", jsonParser, async function (request, response) {
}); });
function checkServer() { function checkServer() {
//console.log('Check run###################################################');
api_server = 'http://127.0.0.1:5000'; api_server = 'http://127.0.0.1:5000';
var args = { var args = {
headers: { "Content-Type": "application/json" } headers: { "Content-Type": "application/json" }
}; };
client.get(api_server + "/v1/model", args, function (data, response) { client.get(api_server + "/v1/model", args, function (data, response) {
console.log(data.result); console.log(data.result);
//console.log('###################################################');
console.log(data); console.log(data);
}).on('error', function (err) { }).on('error', function (err) {
console.log(err); console.log(err);
//console.log('');
//console.log('something went wrong on the request', err.request.options);
//console.log('errorrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr');
}); });
} }
@ -688,10 +657,6 @@ app.post("/createcharacter", urlencodedParser, function (request, response) {
response.send("Error: A character with that name already exists."); response.send("Error: A character with that name already exists.");
//response.send({error: true}); //response.send({error: true});
} }
//console.log(request.body);
//response.send(request.body.ch_name);
//response.redirect("https://metanit.com")
}); });
@ -892,11 +857,6 @@ app.post("/getcharacters", jsonParser, function (request, response) {
//console.log(characters); //console.log(characters);
response.send(JSON.stringify(characters)); response.send(JSON.stringify(characters));
}); });
//var directories = getDirectories("public/characters");
//console.log(directories[0]);
//characters = {};
//character_i = 0;
//getCharacterFile(directories, response,0);
}); });
app.post("/getbackgrounds", jsonParser, function (request, response) { app.post("/getbackgrounds", jsonParser, function (request, response) {
@ -918,11 +878,6 @@ app.post("/getuseravatars", jsonParser, function (request, response) {
}); });
app.post("/setbackground", jsonParser, function (request, response) { app.post("/setbackground", jsonParser, function (request, response) {
//console.log(request.data);
//console.log(request.body.bg);
//const data = request.body;
//console.log(request);
//console.log(1);
var bg = "#bg1 {background-image: url(../backgrounds/" + request.body.bg + ");}"; var bg = "#bg1 {background-image: url(../backgrounds/" + request.body.bg + ");}";
fs.writeFile('public/css/bg_load.css', bg, 'utf8', function (err) { fs.writeFile('public/css/bg_load.css', bg, 'utf8', function (err) {
if (err) { if (err) {
@ -1016,8 +971,6 @@ app.post("/downloadbackground", urlencodedParser, function (request, response) {
}); });
app.post("/savesettings", jsonParser, function (request, response) { app.post("/savesettings", jsonParser, function (request, response) {
fs.writeFile('public/settings.json', JSON.stringify(request.body), 'utf8', function (err) { fs.writeFile('public/settings.json', JSON.stringify(request.body), 'utf8', function (err) {
if (err) { if (err) {
response.send(err); response.send(err);
@ -1028,8 +981,8 @@ app.post("/savesettings", jsonParser, function (request, response) {
response.send({ result: "ok" }); response.send({ result: "ok" });
} }
}); });
}); });
app.post('/getsettings', jsonParser, (request, response) => { //Wintermute's code app.post('/getsettings', jsonParser, (request, response) => { //Wintermute's code
const koboldai_settings = []; const koboldai_settings = [];
const koboldai_setting_names = []; const koboldai_setting_names = [];
@ -1238,34 +1191,6 @@ function readWorldInfoFile(worldInfoName) {
return worldInfo; return worldInfo;
} }
function getCharacterFile(directories, response, i) { //old need del
if (directories.length > i) {
fs.stat(charactersPath + directories[i] + '/' + directories[i] + ".json", function (err, stat) {
if (err == null) {
fs.readFile(charactersPath + directories[i] + '/' + directories[i] + ".json", 'utf8', (err, data) => {
if (err) {
console.error(err);
return;
}
//console.log(data);
characters[character_i] = {};
characters[character_i] = data;
i++;
character_i++;
getCharacterFile(directories, response, i);
});
} else {
i++;
getCharacterFile(directories, response, i);
}
});
} else {
response.send(JSON.stringify(characters));
}
}
function getImages(path) { function getImages(path) {
return fs return fs
@ -1277,17 +1202,6 @@ function getImages(path) {
.sort(Intl.Collator().compare); .sort(Intl.Collator().compare);
} }
function getKoboldSettingFiles(path) {
return fs.readdirSync(path).sort(function (a, b) {
return new Date(fs.statSync(path + '/' + a).mtime) - new Date(fs.statSync(path + '/' + b).mtime);
}).reverse();
}
function getDirectories(path) {
return fs.readdirSync(path).sort(function (a, b) {
return new Date(fs.statSync(path + '/' + a).mtime) - new Date(fs.statSync(path + '/' + b).mtime);
}).reverse();
}
//***********Novel.ai API //***********Novel.ai API
app.post("/getstatus_novelai", jsonParser, function (request, response_getstatus_novel = response) { app.post("/getstatus_novelai", jsonParser, function (request, response_getstatus_novel = response) {
@ -1320,8 +1234,6 @@ app.post("/getstatus_novelai", jsonParser, function (request, response_getstatus
}); });
}); });
app.post("/generate_novelai", jsonParser, function (request, response_generate_novel = response) { app.post("/generate_novelai", jsonParser, function (request, response_generate_novel = response) {
if (!request.body) return response_generate_novel.sendStatus(400); if (!request.body) return response_generate_novel.sendStatus(400);
@ -1581,17 +1493,12 @@ app.post("/exportcharacter", jsonParser, async function (request, response) {
app.post("/importchat", urlencodedParser, function (request, response) { app.post("/importchat", urlencodedParser, function (request, response) {
//console.log(humanizedISO8601DateTime()+':/importchat begun');
if (!request.body) return response.sendStatus(400); if (!request.body) return response.sendStatus(400);
var format = request.body.file_type; var format = request.body.file_type;
let filedata = request.file; let filedata = request.file;
let avatar_url = (request.body.avatar_url).replace('.png', ''); let avatar_url = (request.body.avatar_url).replace('.png', '');
let ch_name = request.body.character_name; let ch_name = request.body.character_name;
//console.log(filedata.filename);
//var format = request.body.file_type;
//console.log(format);
//console.log(1);
if (filedata) { if (filedata) {
if (format === 'json') { if (format === 'json') {