rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

template derivation: add support for llama.cpp server backend

This commit is contained in:
Karl-Johan Alm 2024-11-19 23:38:38 +09:00
parent cdc0147490
commit feb1b91619
No known key found for this signature in database
GPG Key ID: CF78C98086AB1ECA
2 changed files with 16 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
public/scripts/chat-cemplates.js
View File

@ -59,6 +59,6 @@ export async function deriveTemplatesFromChatTemplate(chat_template, hash) {
if (hash in derivations) { if (hash in derivations) {
return derivations[hash]; return derivations[hash];
} }
console.log(`Unknown chat template hash: ${hash}`); console.log(`Unknown chat template hash: ${hash} for [${chat_template}]`);
return null; return null;
} }

16
src/endpoints/backends/text-completions.js
View File

@ -231,6 +231,9 @@ router.post('/status', jsonParser, async function (request, response) {
} catch (error) { } catch (error) {
console.error(`Failed to fetch chat template info: ${error}`); console.error(`Failed to fetch chat template info: ${error}`);
} }
} else if (apiType == TEXTGEN_TYPES.LLAMACPP) {
// the /props endpoint includes chat template
response.setHeader('x-supports-chat-template', 'true');
} }
return response.send({ result, data: data.data }); return response.send({ result, data: data.data });
@ -240,6 +243,11 @@ router.post('/status', jsonParser, async function (request, response) {
} }
}); });
const chat_template_endpoints = {
koboldcpp: '/api/extra/chat_template',
llamacpp: '/props',
}
router.post('/chat_template', jsonParser, async function (request, response) { router.post('/chat_template', jsonParser, async function (request, response) {
if (!request.body.api_server) return response.sendStatus(400); if (!request.body.api_server) return response.sendStatus(400);
@ -251,7 +259,8 @@ router.post('/chat_template', jsonParser, async function (request, response) {
setAdditionalHeaders(request, args, baseUrl); setAdditionalHeaders(request, args, baseUrl);
const chatTemplateUrl = baseUrl + '/api/extra/chat_template'; const apiType = request.body.api_type;
const chatTemplateUrl = baseUrl + chat_template_endpoints[apiType];
const chatTemplateReply = await fetch(chatTemplateUrl, args); const chatTemplateReply = await fetch(chatTemplateUrl, args);
if (!chatTemplateReply.ok) { if (!chatTemplateReply.ok) {
@ -261,7 +270,12 @@ router.post('/chat_template', jsonParser, async function (request, response) {
/** @type {any} */ /** @type {any} */
const chatTemplate = await chatTemplateReply.json(); const chatTemplate = await chatTemplateReply.json();
// TEMPORARY: llama.cpp's /props endpoint includes a \u0000 at the end of the chat template, resulting in mismatching hashes
if (apiType === TEXTGEN_TYPES.LLAMACPP && chatTemplate['chat_template'].endsWith('\u0000')) {
chatTemplate['chat_template'] = chatTemplate['chat_template'].slice(0, -1);
}
chatTemplate['chat_template_hash'] = createHash('sha256').update(chatTemplate['chat_template']).digest('hex'); chatTemplate['chat_template_hash'] = createHash('sha256').update(chatTemplate['chat_template']).digest('hex');
console.log(`We have chat template stuff: ${JSON.stringify(chatTemplate)}`);
return response.send(chatTemplate); return response.send(chatTemplate);
} catch (error) { } catch (error) {
console.error(error); console.error(error);