rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix aborting generation on KoboldCpp via Text Completion

Browse Source
This commit is contained in:
Cohee
2023-12-21 23:14:28 +02:00
parent b782a8cc03
commit 9cf6cef0a4
1 changed files with 25 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/endpoints/backends/text-completions.js
View File

@ -57,6 +57,26 @@ async function parseOllamaStream(jsonStream, request, response) {
}
}
/**
* Abort KoboldCpp generation request.
* @param {string} url Server base URL
* @returns {Promise<void>} Promise resolving when we are done
*/
async function abortKoboldCppRequest(url) {
try {
console.log('Aborting Kobold generation...');
const abortResponse = await fetch(`${url}/api/extra/abort`, {
method: 'POST',
});
if (!abortResponse.ok) {
console.log('Error sending abort request to Kobold:', abortResponse.status, abortResponse.statusText);
}
} catch (error) {
console.log(error);
}
}
//************** Ooba/OpenAI text completions API
router.post('/status', jsonParser, async function (request, response) {
if (!request.body) return response.sendStatus(400);
@ -193,7 +213,11 @@ router.post('/generate', jsonParser, async function (request, response) {
const controller = new AbortController();
request.socket.removeAllListeners('close');
request.socket.on('close', function () {
request.socket.on('close', async function () {
if (request.body.api_type === TEXTGEN_TYPES.KOBOLDCPP && !response.writableEnded) {
await abortKoboldCppRequest(trimV1(baseUrl));
}
controller.abort();
});