rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use abort endpoint for all supported koboldcpp generations

Browse Source
This commit is contained in:
Zixaphir
2023-06-14 19:56:55 -07:00
parent 8f97d4a71b
commit ea49b33e3c
2 changed files with 15 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
public/scripts/kai-settings.js
View File

@ -95,6 +95,7 @@ function getKoboldGenerationData(finalPromt, this_settings, this_amount_gen, thi
singleline: kai_settings.single_line, singleline: kai_settings.single_line,
stop_sequence: kai_settings.use_stop_sequence ? getStoppingStrings(isImpersonate, false) : undefined, stop_sequence: kai_settings.use_stop_sequence ? getStoppingStrings(isImpersonate, false) : undefined,
streaming: kai_settings.streaming_kobold && kai_settings.can_use_streaming, streaming: kai_settings.streaming_kobold && kai_settings.can_use_streaming,
can_abort: kai_settings.can_use_streaming,
}; };
return generate_data; return generate_data;
} }

29
server.js
View File

@ -354,7 +354,19 @@ app.post("/generate", jsonParser, async function (request, response_generate = r
const request_prompt = request.body.prompt; const request_prompt = request.body.prompt;
const controller = new AbortController(); const controller = new AbortController();
request.socket.removeAllListeners('close'); request.socket.removeAllListeners('close');
request.socket.on('close', function () { request.socket.on('close', async function () {
if (request.body.can_abort && !response_generate.finished) {
try {
// send abort signal to koboldcpp
await fetch(`${api_server}/extra/abort`, {
method: 'POST',
});
} catch {
if ('status' in error) {
console.log('Status Code from Kobold:', error.status);
}
}
}
controller.abort(); controller.abort();
}); });
@ -412,20 +424,7 @@ app.post("/generate", jsonParser, async function (request, response_generate = r
response = await fetch(url, { method: 'POST', timeout: 0, ...args }); response = await fetch(url, { method: 'POST', timeout: 0, ...args });
if (request.body.streaming) { if (request.body.streaming) {
request.socket.on('close', async function () { request.socket.on('close', function () {
if (controller.signal.aborted) {
try {
// send abort signal to koboldcpp
await fetch(`${api_server}/extra/abort`, {
method: 'POST',
});
} catch {
// likely endpoint not available on older versions of koboldcpp
if ('status' in error) {
console.log('Status Code from Kobold:', error.status);
}
}
}
response.body.destroy(); // Close the remote stream response.body.destroy(); // Close the remote stream
response_generate.end(); // End the Express response response_generate.end(); // End the Express response
}); });