rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use abort endpoint for all supported koboldcpp generations

Browse Source
This commit is contained in:
Zixaphir
2023-06-14 19:56:55 -07:00
parent 8f97d4a71b
commit ea49b33e3c
2 changed files with 15 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
server.js
View File

@ -354,7 +354,19 @@ app.post("/generate", jsonParser, async function (request, response_generate = r
const request_prompt = request.body.prompt;
const controller = new AbortController();
request.socket.removeAllListeners('close');
request.socket.on('close', function () {
request.socket.on('close', async function () {
if (request.body.can_abort && !response_generate.finished) {
try {
// send abort signal to koboldcpp
await fetch(`${api_server}/extra/abort`, {
method: 'POST',
});
} catch {
if ('status' in error) {
console.log('Status Code from Kobold:', error.status);
}
}
}
controller.abort();
});
@ -412,20 +424,7 @@ app.post("/generate", jsonParser, async function (request, response_generate = r
response = await fetch(url, { method: 'POST', timeout: 0, ...args });
if (request.body.streaming) {
request.socket.on('close', async function () {
if (controller.signal.aborted) {
try {
// send abort signal to koboldcpp
await fetch(`${api_server}/extra/abort`, {
method: 'POST',
});
} catch {
// likely endpoint not available on older versions of koboldcpp
if ('status' in error) {
console.log('Status Code from Kobold:', error.status);
}
}
}
request.socket.on('close', function () {
response.body.destroy(); // Close the remote stream
response_generate.end(); // End the Express response
});