rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-02-03 12:47:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

template derivation: move hash part to backend

Browse Source
This commit is contained in:
Karl-Johan Alm 2024-11-19 20:09:29 +09:00
parent 0e2fdf37a8
commit f25ea9f6d6
No known key found for this signature in database
GPG Key ID: CF78C98086AB1ECA
5 changed files with 13 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
package-lock.json generated
View File

@ -43,6 +43,7 @@
"ip-matching": "^2.1.2", "ip-matching": "^2.1.2",
"ipaddr.js": "^2.0.1", "ipaddr.js": "^2.0.1",
"jimp": "^0.22.10", "jimp": "^0.22.10",
"js-sha256": "^0.11.0",
"localforage": "^1.10.0", "localforage": "^1.10.0",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"mime-types": "^2.1.35", "mime-types": "^2.1.35",
@ -4882,6 +4883,12 @@
"integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==",
"license": "BSD-3-Clause" "license": "BSD-3-Clause"
}, },
"node_modules/js-sha256": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.11.0.tgz",
"integrity": "sha512-6xNlKayMZvds9h1Y1VWc0fQHQ82BxTXizWPEtEeGvmOUYpBRy4gbWroHLpzowe6xiQhHpelCQiE7HEdznyBL9Q==",
"license": "MIT"
},
"node_modules/js-yaml": { "node_modules/js-yaml": {
"version": "4.1.0", "version": "4.1.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",

1
package.json
View File

@ -33,6 +33,7 @@
"ip-matching": "^2.1.2", "ip-matching": "^2.1.2",
"ipaddr.js": "^2.0.1", "ipaddr.js": "^2.0.1",
"jimp": "^0.22.10", "jimp": "^0.22.10",
"js-sha256": "^0.11.0",
"localforage": "^1.10.0", "localforage": "^1.10.0",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"mime-types": "^2.1.35", "mime-types": "^2.1.35",

4
public/script.js
View File

@ -1250,9 +1250,9 @@ async function getStatusTextgen() {
const data = await response.json(); const data = await response.json();
if (data) { if (data) {
const chat_template = data.chat_template; const { chat_template, chat_template_hash } = data;
console.log(`We have chat template ${chat_template.split('\n')[0]}...`); console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
const templates = await deriveTemplatesFromChatTemplate(chat_template); const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
if (templates) { if (templates) {
const { context, instruct } = templates; const { context, instruct } = templates;
selectContextPreset(context, { isAuto: true }); selectContextPreset(context, { isAuto: true });

14
public/scripts/chat-cemplates.js
View File

@ -1,14 +1,3 @@
// https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest
async function digestMessage(message) {
const msgUint8 = new TextEncoder().encode(message); // encode as (utf-8) Uint8Array
const hashBuffer = await window.crypto.subtle.digest('SHA-256', msgUint8); // hash the message
const hashArray = Array.from(new Uint8Array(hashBuffer)); // convert buffer to byte array
const hashHex = hashArray
.map((b) => b.toString(16).padStart(2, '0'))
.join(''); // convert bytes to hex string
return hashHex;
}
// the hash can be obtained from command line e.g. via: MODEL=path_to_model; python -c "import json, hashlib, sys; print(hashlib.sha256(json.load(open('"$MODEL"/tokenizer_config.json'))['chat_template'].strip().encode()).hexdigest())" // the hash can be obtained from command line e.g. via: MODEL=path_to_model; python -c "import json, hashlib, sys; print(hashlib.sha256(json.load(open('"$MODEL"/tokenizer_config.json'))['chat_template'].strip().encode()).hexdigest())"
// note that chat templates must be trimmed to match the llama.cpp metadata value // note that chat templates must be trimmed to match the llama.cpp metadata value
const derivations = { const derivations = {
@ -66,8 +55,7 @@ const derivations = {
}, },
}; };
export async function deriveTemplatesFromChatTemplate(chat_template) { export async function deriveTemplatesFromChatTemplate(chat_template, hash) {
const hash = await digestMessage(chat_template);
if (hash in derivations) { if (hash in derivations) {
return derivations[hash]; return derivations[hash];
} }

2
src/endpoints/backends/text-completions.js
View File

@ -16,6 +16,7 @@ import {
} from '../../constants.js'; } from '../../constants.js';
import { forwardFetchResponse, trimV1, getConfigValue } from '../../util.js'; import { forwardFetchResponse, trimV1, getConfigValue } from '../../util.js';
import { setAdditionalHeaders } from '../../additional-headers.js'; import { setAdditionalHeaders } from '../../additional-headers.js';
import { sha256 } from 'js-sha256';
export const router = express.Router(); export const router = express.Router();
@ -260,6 +261,7 @@ router.post('/chat_template', jsonParser, async function (request, response) {
/** @type {any} */ /** @type {any} */
const chatTemplate = await chatTemplateReply.json(); const chatTemplate = await chatTemplateReply.json();
chatTemplate['chat_template_hash'] = sha256.create().update(chatTemplate['chat_template']).hex();
return response.send(chatTemplate); return response.send(chatTemplate);
} catch (error) { } catch (error) {
console.error(error); console.error(error);