diff --git a/public/scripts/chat-cemplates.js b/public/scripts/chat-cemplates.js index 2dfaebbb6..00b2f9771 100644 --- a/public/scripts/chat-cemplates.js +++ b/public/scripts/chat-cemplates.js @@ -59,6 +59,6 @@ export async function deriveTemplatesFromChatTemplate(chat_template, hash) { if (hash in derivations) { return derivations[hash]; } - console.log(`Unknown chat template hash: ${hash}`); + console.log(`Unknown chat template hash: ${hash} for [${chat_template}]`); return null; } diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index 81ed8ea7e..d0b9c17a8 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -231,6 +231,9 @@ router.post('/status', jsonParser, async function (request, response) { } catch (error) { console.error(`Failed to fetch chat template info: ${error}`); } + } else if (apiType == TEXTGEN_TYPES.LLAMACPP) { + // the /props endpoint includes chat template + response.setHeader('x-supports-chat-template', 'true'); } return response.send({ result, data: data.data }); @@ -240,6 +243,11 @@ router.post('/status', jsonParser, async function (request, response) { } }); +const chat_template_endpoints = { + koboldcpp: '/api/extra/chat_template', + llamacpp: '/props', +} + router.post('/chat_template', jsonParser, async function (request, response) { if (!request.body.api_server) return response.sendStatus(400); @@ -251,7 +259,8 @@ router.post('/chat_template', jsonParser, async function (request, response) { setAdditionalHeaders(request, args, baseUrl); - const chatTemplateUrl = baseUrl + '/api/extra/chat_template'; + const apiType = request.body.api_type; + const chatTemplateUrl = baseUrl + chat_template_endpoints[apiType]; const chatTemplateReply = await fetch(chatTemplateUrl, args); if (!chatTemplateReply.ok) { @@ -261,7 +270,12 @@ router.post('/chat_template', jsonParser, async function (request, response) { /** @type {any} */ const chatTemplate = await chatTemplateReply.json(); + // TEMPORARY: llama.cpp's /props endpoint includes a \u0000 at the end of the chat template, resulting in mismatching hashes + if (apiType === TEXTGEN_TYPES.LLAMACPP && chatTemplate['chat_template'].endsWith('\u0000')) { + chatTemplate['chat_template'] = chatTemplate['chat_template'].slice(0, -1); + } chatTemplate['chat_template_hash'] = createHash('sha256').update(chatTemplate['chat_template']).digest('hex'); + console.log(`We have chat template stuff: ${JSON.stringify(chatTemplate)}`); return response.send(chatTemplate); } catch (error) { console.error(error);