Merge pull request #3090 from kallewoof/202411-auto-templates
Derived templates
This commit is contained in:
Cohee
GitHub
commit
ba91845ced
|
|
@ -91,9 +91,6 @@
|
|||
<div class="margin0 title_restorable standoutHeader">
|
||||
<strong>
|
||||
<span data-i18n="kobldpresets">Kobold Presets</span>
|
||||
<a href="https://docs.sillytavern.app/usage/api-connections/koboldai/" class="notes-link" target="_blank">
|
||||
<span class="fa-solid fa-circle-question note-link-span"></span>
|
||||
</a>
|
||||
</strong>
|
||||
|
||||
<div class="flex-container gap3px">
|
||||
|
|
@ -3274,8 +3271,14 @@
|
|||
</div>
|
||||
<div id="AdvancedFormatting" class="drawer-content">
|
||||
<div class="flex-container alignItemsBaseline">
|
||||
<h3 class="margin0 flex1" data-i18n="Advanced Formatting">
|
||||
Advanced Formatting
|
||||
<h3 class="margin0 flex1 flex-container alignItemsBaseline">
|
||||
<span data-i18n="Advanced Formatting">
|
||||
Advanced Formatting
|
||||
</span>
|
||||
|
||||
<a href="https://docs.sillytavern.app/usage/prompts/" class="notes-link" target="_blank">
|
||||
<span class="fa-solid fa-circle-question note-link-span"></span>
|
||||
</a>
|
||||
</h3>
|
||||
<div class="flex-container">
|
||||
<input id="af_master_import_file" type="file" hidden accept=".json" class="displayNone">
|
||||
|
|
@ -3295,9 +3298,12 @@
|
|||
<h4 class="standoutHeader title_restorable">
|
||||
<div>
|
||||
<span data-i18n="Context Template">Context Template</span>
|
||||
<a href="https://docs.sillytavern.app/usage/core-concepts/advancedformatting/#context-template" class="notes-link" target="_blank">
|
||||
<span class="fa-solid fa-circle-question note-link-span"></span>
|
||||
</a>
|
||||
</div>
|
||||
<div class="flex-container">
|
||||
<label for="context_derived" class="checkbox_label flex1" title="Derive from Model Metadata, if possible." data-i18n="[title]context_derived">
|
||||
<input id="context_derived" type="checkbox" style="display:none;" />
|
||||
<small><i class="fa-solid fa-bolt menu_button margin0"></i></small>
|
||||
</label>
|
||||
</div>
|
||||
</h4>
|
||||
<div class="flex-container" title="Select your current Context Template" data-i18n="[title]Select your current Context Template">
|
||||
|
|
@ -3394,11 +3400,12 @@
|
|||
<h4 class="standoutHeader title_restorable justifySpaceBetween">
|
||||
<div class="flex-container">
|
||||
<span data-i18n="Instruct Template">Instruct Template</span>
|
||||
<a href="https://docs.sillytavern.app/usage/core-concepts/instructmode/" class="notes-link" target="_blank">
|
||||
<span class="fa-solid fa-circle-question note-link-span"></span>
|
||||
</a>
|
||||
</div>
|
||||
<div class="flex-container">
|
||||
<label for="instruct_derived" class="checkbox_label flex1" title="Derive from Model Metadata, if possible." data-i18n="[title]instruct_derived">
|
||||
<input id="instruct_derived" type="checkbox" style="display:none;" />
|
||||
<small><i class="fa-solid fa-bolt menu_button margin0"></i></small>
|
||||
</label>
|
||||
<label for="instruct_bind_to_context" class="checkbox_label flex1" title="Bind to Context
If enabled, Context templates will be automatically selected based on selected Instruct template name or by preference." data-i18n="[title]instruct_bind_to_context">
|
||||
<input id="instruct_bind_to_context" type="checkbox" style="display:none;" />
|
||||
<small><i class="fa-solid fa-link menu_button margin0"></i></small>
|
||||
|
|
@ -3579,9 +3586,6 @@
|
|||
<h4 class="standoutHeader title_restorable justifySpaceBetween">
|
||||
<div class="flex-container">
|
||||
<span data-i18n="System Prompt">System Prompt</span>
|
||||
<a href="https://docs.sillytavern.app/usage/core-concepts/advancedformatting/#system-prompt" class="notes-link" target="_blank">
|
||||
<span class="fa-solid fa-circle-question note-link-span"></span>
|
||||
</a>
|
||||
</div>
|
||||
<div class="flex-container">
|
||||
<label id="sysprompt_enabled_label" for="sysprompt_enabled" class="checkbox_label flex1" title="Enable System Prompt" data-i18n="[title]sysprompt_enabled">
|
||||
|
|
@ -3649,7 +3653,7 @@
|
|||
<div name="tokenizerSettingsBlock">
|
||||
<div name="tokenizerSelectorBlock">
|
||||
<h4 class="standoutHeader"><span data-i18n="Tokenizer">Tokenizer</span>
|
||||
<a href="https://docs.sillytavern.app/usage/core-concepts/advancedformatting/#tokenizer" class="notes-link" target="_blank">
|
||||
<a href="https://docs.sillytavern.app/usage/prompts/tokenizer/" class="notes-link" target="_blank">
|
||||
<span class="fa-solid fa-circle-question note-link-span"></span>
|
||||
</a>
|
||||
</h4>
|
||||
|
|
@ -3675,10 +3679,8 @@
|
|||
</div>
|
||||
<div class="range-block flex-container flexnowrap" name="tokenPaddingBlock">
|
||||
<div class="range-block-title justifyLeft">
|
||||
<small data-i18n="Token Padding" class="flex-container">Token Padding
|
||||
<a href="https://docs.sillytavern.app/usage/core-concepts/advancedformatting/#token-padding" class="notes-link" target="_blank">
|
||||
<span class="fa-solid fa-circle-question note-link-span"></span>
|
||||
</a>
|
||||
<small data-i18n="Token Padding">
|
||||
Token Padding
|
||||
</small>
|
||||
</div>
|
||||
<input id="token_padding" class="text_pole textarea_compact" type="number" min="-2048" max="2048" />
|
||||
|
|
@ -3736,7 +3738,7 @@
|
|||
</div>
|
||||
<h3 class="margin0">
|
||||
<span data-i18n="Worlds/Lorebooks">Worlds/Lorebooks</span>
|
||||
<a href="https://docs.sillytavern.app/usage/core-concepts/worldinfo/" class="notes-link" target="_blank">
|
||||
<a href="https://docs.sillytavern.app/usage/worldinfo/" class="notes-link" target="_blank">
|
||||
<span class="fa-solid fa-circle-question note-link-span"></span>
|
||||
</a>
|
||||
</h3>
|
||||
|
|
@ -4263,7 +4265,7 @@
|
|||
<audio id="audio_message_sound" src="sounds/message.mp3" hidden></audio>
|
||||
<span>
|
||||
<small data-i18n="Message Sound">Message Sound</small>
|
||||
<a href="https://docs.sillytavern.app/usage/core-concepts/uicustomization/#message-sound" class="notes-link" target="_blank">
|
||||
<a href="https://docs.sillytavern.app/usage/user_settings/uicustomization/#message-sound" class="notes-link" target="_blank">
|
||||
<span class="fa-solid fa-circle-question note-link-span"></span>
|
||||
</a>
|
||||
</span>
|
||||
|
|
@ -5795,7 +5797,7 @@
|
|||
<div class="flex-container justifySpaceBetween">
|
||||
<small for="group">
|
||||
<span data-i18n="Inclusion Group">Inclusion Group</span>
|
||||
<a href="https://docs.sillytavern.app/usage/core-concepts/worldinfo/#inclusion-group" class="notes-link" target="_blank" title="Inclusion Groups ensure only one entry from a group is activated at a time, if multiple are triggered. Supports multiple comma-separated groups. Documentation: World Info - Inclusion Group" data-i18n="[title]Inclusion Groups ensure only one entry from a group is activated at a time, if multiple are triggered.Documentation: World Info - Inclusion Group">
|
||||
<a href="https://docs.sillytavern.app/usage/worldinfo/#inclusion-group" class="notes-link" target="_blank" title="Inclusion Groups ensure only one entry from a group is activated at a time, if multiple are triggered. Supports multiple comma-separated groups. Documentation: World Info - Inclusion Group" data-i18n="[title]Inclusion Groups ensure only one entry from a group is activated at a time, if multiple are triggered.Documentation: World Info - Inclusion Group">
|
||||
<span class="fa-solid fa-circle-question note-link-span"></span>
|
||||
</a>
|
||||
</small>
|
||||
|
|
|
|||
|
|
@ -267,6 +267,7 @@ import { applyBrowserFixes } from './scripts/browser-fixes.js';
|
|||
import { initServerHistory } from './scripts/server-history.js';
|
||||
import { initSettingsSearch } from './scripts/setting-search.js';
|
||||
import { initBulkEdit } from './scripts/bulk-edit.js';
|
||||
import { deriveTemplatesFromChatTemplate } from './scripts/chat-templates.js';
|
||||
|
||||
//exporting functions and vars for mods
|
||||
export {
|
||||
|
|
@ -1235,6 +1236,38 @@ async function getStatusTextgen() {
|
|||
const supportsTokenization = response.headers.get('x-supports-tokenization') === 'true';
|
||||
supportsTokenization ? sessionStorage.setItem(TOKENIZER_SUPPORTED_KEY, 'true') : sessionStorage.removeItem(TOKENIZER_SUPPORTED_KEY);
|
||||
|
||||
const wantsInstructDerivation = (power_user.instruct.enabled && power_user.instruct.derived);
|
||||
const wantsContextDerivation = power_user.context_derived;
|
||||
const supportsChatTemplate = [textgen_types.KOBOLDCPP, textgen_types.LLAMACPP].includes(textgen_settings.type);
|
||||
if (supportsChatTemplate && (wantsInstructDerivation || wantsContextDerivation)) {
|
||||
const response = await fetch('/api/backends/text-completions/props', {
|
||||
method: 'POST',
|
||||
headers: getRequestHeaders(),
|
||||
body: JSON.stringify({
|
||||
api_server: endpoint,
|
||||
api_type: textgen_settings.type,
|
||||
}),
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
if (data) {
|
||||
const { chat_template, chat_template_hash } = data;
|
||||
console.log(`We have chat template ${chat_template.split('\n')[0]}...`);
|
||||
const templates = await deriveTemplatesFromChatTemplate(chat_template, chat_template_hash);
|
||||
if (templates) {
|
||||
const { context, instruct } = templates;
|
||||
if (wantsContextDerivation) {
|
||||
selectContextPreset(context, { isAuto: true });
|
||||
}
|
||||
if (wantsInstructDerivation) {
|
||||
selectInstructPreset(instruct, { isAuto: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress.
|
||||
if (online_status === 'no_connection' && data.response) {
|
||||
toastr.error(data.response, t`API Error`, { timeOut: 5000, preventDuplicates: true });
|
||||
|
|
|
|||
|
|
@ -0,0 +1,81 @@
|
|||
// the hash can be obtained from command line e.g. via: MODEL=path_to_model; python -c "import json, hashlib, sys; print(hashlib.sha256(json.load(open('"$MODEL"/tokenizer_config.json'))['chat_template'].encode()).hexdigest())"
|
||||
// note that chat templates must be trimmed to match the llama.cpp metadata value
|
||||
const hash_derivations = {
|
||||
// Meta
|
||||
'e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65':
|
||||
// Meta-Llama-3.1-8B-Instruct
|
||||
// Meta-Llama-3.1-70B-Instruct
|
||||
'Llama 3 Instruct'
|
||||
,
|
||||
'5816fce10444e03c2e9ee1ef8a4a1ea61ae7e69e438613f3b17b69d0426223a4':
|
||||
// Llama-3.2-1B-Instruct
|
||||
// Llama-3.2-3B-Instruct
|
||||
'Llama 3 Instruct'
|
||||
,
|
||||
'73e87b1667d87ab7d7b579107f01151b29ce7f3ccdd1018fdc397e78be76219d':
|
||||
// Nemotron 70B
|
||||
'Llama 3 Instruct'
|
||||
,
|
||||
|
||||
// Mistral
|
||||
// Mistral Reference: https://github.com/mistralai/mistral-common
|
||||
'e16746b40344d6c5b5265988e0328a0bf7277be86f1c335156eae07e29c82826':
|
||||
// Mistral-Small-Instruct-2409
|
||||
// Mistral-Large-Instruct-2407
|
||||
'Mistral V2 & V3'
|
||||
,
|
||||
'3c4ad5fa60dd8c7ccdf82fa4225864c903e107728fcaf859fa6052cb80c92ee9':
|
||||
// Mistral-Large-Instruct-2411
|
||||
'Mistral V7' // https://huggingface.co/mistralai/Mistral-Large-Instruct-2411
|
||||
,
|
||||
'e4676cb56dffea7782fd3e2b577cfaf1e123537e6ef49b3ec7caa6c095c62272':
|
||||
// Mistral-Nemo-Instruct-2407
|
||||
'Mistral V3-Tekken'
|
||||
,
|
||||
'26a59556925c987317ce5291811ba3b7f32ec4c647c400c6cc7e3a9993007ba7':
|
||||
// Mistral-7B-Instruct-v0.3
|
||||
'Mistral V2 & V3'
|
||||
,
|
||||
|
||||
// Gemma
|
||||
'ecd6ae513fe103f0eb62e8ab5bfa8d0fe45c1074fa398b089c93a7e70c15cfd6':
|
||||
// gemma-2-9b-it
|
||||
// gemma-2-27b-it
|
||||
'Gemma 2'
|
||||
,
|
||||
'87fa45af6cdc3d6a9e4dd34a0a6848eceaa73a35dcfe976bd2946a5822a38bf3':
|
||||
// gemma-2-2b-it
|
||||
'Gemma 2'
|
||||
,
|
||||
|
||||
// Cohere
|
||||
'3b54f5c219ae1caa5c0bb2cdc7c001863ca6807cf888e4240e8739fa7eb9e02e':
|
||||
// command-r-08-2024
|
||||
'Command R'
|
||||
,
|
||||
};
|
||||
|
||||
const substr_derivations = {
|
||||
'<|im_start|>': 'ChatML', // qwen2.5, ...
|
||||
};
|
||||
|
||||
const parse_derivation = derivation => (typeof derivation === 'string') ? {
|
||||
'context': derivation,
|
||||
'instruct': derivation,
|
||||
} : derivation;
|
||||
|
||||
export async function deriveTemplatesFromChatTemplate(chat_template, hash) {
|
||||
if (hash in hash_derivations) {
|
||||
return parse_derivation(hash_derivations[hash]);
|
||||
}
|
||||
|
||||
// heuristics
|
||||
for (const [substr, derivation] of Object.entries(substr_derivations) ) {
|
||||
if (chat_template.includes(substr)) {
|
||||
return parse_derivation(derivation);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Unknown chat template hash: ${hash} for [${chat_template}]`);
|
||||
return null;
|
||||
}
|
||||
|
|
@ -39,6 +39,7 @@ const controls = [
|
|||
{ id: 'instruct_first_input_sequence', property: 'first_input_sequence', isCheckbox: false },
|
||||
{ id: 'instruct_last_input_sequence', property: 'last_input_sequence', isCheckbox: false },
|
||||
{ id: 'instruct_activation_regex', property: 'activation_regex', isCheckbox: false },
|
||||
{ id: 'instruct_derived', property: 'derived', isCheckbox: true },
|
||||
{ id: 'instruct_bind_to_context', property: 'bind_to_context', isCheckbox: true },
|
||||
{ id: 'instruct_skip_examples', property: 'skip_examples', isCheckbox: true },
|
||||
{ id: 'instruct_names_behavior', property: 'names_behavior', isCheckbox: false },
|
||||
|
|
@ -100,6 +101,7 @@ export async function loadInstructMode(data) {
|
|||
|
||||
$('#instruct_enabled').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.enabled);
|
||||
$('#instructSettingsBlock, #InstructSequencesColumn').toggleClass('disabled', !power_user.instruct.enabled);
|
||||
$('#instruct_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.derived);
|
||||
$('#instruct_bind_to_context').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.bind_to_context);
|
||||
|
||||
controls.forEach(control => {
|
||||
|
|
@ -146,6 +148,12 @@ export async function loadInstructMode(data) {
|
|||
* @param {boolean} [options.isAuto=false] Is auto-select.
|
||||
*/
|
||||
export function selectContextPreset(preset, { quiet = false, isAuto = false } = {}) {
|
||||
const presetExists = context_presets.some(x => x.name === preset);
|
||||
if (!presetExists) {
|
||||
console.warn(`Context template "${preset}" not found`);
|
||||
return;
|
||||
}
|
||||
|
||||
// If context template is not already selected, select it
|
||||
if (preset !== power_user.context.preset) {
|
||||
$('#context_presets').val(preset).trigger('change');
|
||||
|
|
@ -163,6 +171,12 @@ export function selectContextPreset(preset, { quiet = false, isAuto = false } =
|
|||
* @param {boolean} [options.isAuto=false] Is auto-select.
|
||||
*/
|
||||
export function selectInstructPreset(preset, { quiet = false, isAuto = false } = {}) {
|
||||
const presetExists = instruct_presets.some(x => x.name === preset);
|
||||
if (!presetExists) {
|
||||
console.warn(`Instruct template "${preset}" not found`);
|
||||
return;
|
||||
}
|
||||
|
||||
// If instruct preset is not already selected, select it
|
||||
if (preset !== power_user.instruct.preset) {
|
||||
$('#instruct_presets').val(preset).trigger('change');
|
||||
|
|
@ -715,6 +729,10 @@ jQuery(() => {
|
|||
}
|
||||
});
|
||||
|
||||
$('#instruct_derived').on('change', function () {
|
||||
$('#instruct_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.derived);
|
||||
});
|
||||
|
||||
$('#instruct_bind_to_context').on('change', function () {
|
||||
$('#instruct_bind_to_context').parent().find('i').toggleClass('toggleEnabled', !!power_user.instruct.bind_to_context);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -226,6 +226,7 @@ let power_user = {
|
|||
macro: true,
|
||||
names_behavior: names_behavior_types.FORCE,
|
||||
activation_regex: '',
|
||||
derived: false,
|
||||
bind_to_context: false,
|
||||
user_alignment_message: '',
|
||||
system_same_as_user: false,
|
||||
|
|
@ -243,6 +244,8 @@ let power_user = {
|
|||
names_as_stop_strings: true,
|
||||
},
|
||||
|
||||
context_derived: false,
|
||||
|
||||
sysprompt: {
|
||||
enabled: true,
|
||||
name: 'Neutral - Chat',
|
||||
|
|
@ -1472,6 +1475,7 @@ async function loadPowerUserSettings(settings, data) {
|
|||
$('#encode_tags').prop('checked', power_user.encode_tags);
|
||||
$('#example_messages_behavior').val(getExampleMessagesBehavior());
|
||||
$(`#example_messages_behavior option[value="${getExampleMessagesBehavior()}"]`).prop('selected', true);
|
||||
$('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
|
||||
|
||||
$('#console_log_prompts').prop('checked', power_user.console_log_prompts);
|
||||
$('#request_token_probabilities').prop('checked', power_user.request_token_probabilities);
|
||||
|
|
@ -3057,6 +3061,16 @@ $(document).ready(() => {
|
|||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
$('#context_derived').on('input', function () {
|
||||
const value = !!$(this).prop('checked');
|
||||
power_user.context_derived = value;
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
$('#context_derived').on('change', function () {
|
||||
$('#context_derived').parent().find('i').toggleClass('toggleEnabled', !!power_user.context_derived);
|
||||
});
|
||||
|
||||
$('#always-force-name2-checkbox').change(function () {
|
||||
power_user.always_force_name2 = !!$(this).prop('checked');
|
||||
saveSettingsDebounced();
|
||||
|
|
|
|||
|
|
@ -584,6 +584,7 @@ class PresetManager {
|
|||
'openrouter_providers',
|
||||
'openrouter_allow_fallbacks',
|
||||
'tabby_model',
|
||||
'derived',
|
||||
];
|
||||
const settings = Object.assign({}, getSettingsByApiId(this.apiId));
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import {
|
|||
} from '../../constants.js';
|
||||
import { forwardFetchResponse, trimV1, getConfigValue } from '../../util.js';
|
||||
import { setAdditionalHeaders } from '../../additional-headers.js';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
export const router = express.Router();
|
||||
|
||||
|
|
@ -227,6 +228,40 @@ router.post('/status', jsonParser, async function (request, response) {
|
|||
}
|
||||
});
|
||||
|
||||
router.post('/props', jsonParser, async function (request, response) {
|
||||
if (!request.body.api_server) return response.sendStatus(400);
|
||||
|
||||
try {
|
||||
const baseUrl = trimV1(request.body.api_server);
|
||||
const args = {
|
||||
headers: {},
|
||||
};
|
||||
|
||||
setAdditionalHeaders(request, args, baseUrl);
|
||||
|
||||
const apiType = request.body.api_type;
|
||||
const propsUrl = baseUrl + '/props';
|
||||
const propsReply = await fetch(propsUrl, args);
|
||||
|
||||
if (!propsReply.ok) {
|
||||
return response.status(400);
|
||||
}
|
||||
|
||||
/** @type {any} */
|
||||
const props = await propsReply.json();
|
||||
// TEMPORARY: llama.cpp's /props endpoint has a bug which replaces the last newline with a \0
|
||||
if (apiType === TEXTGEN_TYPES.LLAMACPP && props['chat_template'].endsWith('\u0000')) {
|
||||
props['chat_template'] = props['chat_template'].slice(0, -1) + '\n';
|
||||
}
|
||||
props['chat_template_hash'] = createHash('sha256').update(props['chat_template']).digest('hex');
|
||||
console.log(`Model properties: ${JSON.stringify(props)}`);
|
||||
return response.send(props);
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
return response.status(500);
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/generate', jsonParser, async function (request, response) {
|
||||
if (!request.body) return response.sendStatus(400);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue