rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'staging' into connection-manager

This commit is contained in:
Cohee 2024-09-07 23:23:26 +03:00
parent 10ddf77948 37b3184cda
commit 669c49ebba
14 changed files with 248 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
default/config.yaml
View File

@ -95,6 +95,9 @@ requestOverrides: []
enableExtensions: true
# Automatically update extensions when a release version changes
enableExtensionsAutoUpdate: true
# Additional model tokenizers can be downloaded on demand.
# Disabling will fallback to another locally available tokenizer.
enableDownloadableTokenizers: true
# Extension settings
extras:
# Disables automatic model download from HuggingFace

8
public/index.html
View File

@ -1820,12 +1820,12 @@
<span id="claude_assistant_prefill_text" data-i18n="Assistant Prefill">Assistant Prefill</span>
<i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="claude_assistant_prefill" title="Expand the editor" data-i18n="[title]Expand the editor"></i>
</div>
<textarea id="claude_assistant_prefill" class="text_pole textarea_compact" name="assistant_prefill" rows="6" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
<textarea id="claude_assistant_prefill" class="text_pole textarea_compact autoSetHeight" name="assistant_prefill" rows="2" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
<div class="flex-container alignItemsCenter">
<span id="claude_assistant_impersonation_text" data-i18n="Assistant Impersonation Prefill">Assistant Impersonation Prefill</span>
<i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="claude_assistant_impersonation" title="Expand the editor" data-i18n="[title]Expand the editor"></i>
</div>
<textarea id="claude_assistant_impersonation" class="text_pole textarea_compact" name="assistant_impersonation" rows="6" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
<textarea id="claude_assistant_impersonation" class="text_pole textarea_compact autoSetHeight" name="assistant_impersonation" rows="2" data-i18n="[placeholder]Start Claude's answer with..." placeholder="Start Claude's answer with..."></textarea>
</div>
<label for="claude_use_sysprompt" class="checkbox_label widthFreeExpand">
<input id="claude_use_sysprompt" type="checkbox" />
@ -1845,7 +1845,7 @@
<div class="fa-solid fa-clock-rotate-left"></div>
</div>
</div>
<textarea id="claude_human_sysprompt_textarea" class="text_pole textarea_compact" rows="4" data-i18n="[placeholder]Human message" placeholder="Human message, instruction, etc.&#10;Adds nothing when empty, i.e. requires a new prompt with the role 'user'."></textarea>
<textarea id="claude_human_sysprompt_textarea" class="text_pole textarea_compact autoSetHeight" rows="2" data-i18n="[placeholder]Human message" placeholder="Human message, instruction, etc.&#10;Adds nothing when empty, i.e. requires a new prompt with the role 'user'."></textarea>
</div>
</div>
</div>
@ -3289,6 +3289,8 @@
<option value="12">Llama 3</option>
<option value="13">Gemma / Gemini</option>
<option value="14">Jamba</option>
<option value="15">Qwen2</option>
<option value="16">Command-R</option>
<option value="4">NerdStash (NovelAI Clio)</option>
<option value="5">NerdStash v2 (NovelAI Kayra)</option>
<option value="7">Mistral</option>

20
public/script.js
View File

@ -8504,22 +8504,23 @@ for (const chatCompletionSource of Object.values(chat_completion_sources)) {
};
}
async function selectContextCallback(_, name) {
async function selectContextCallback(args, name) {
if (!name) {
return power_user.context.preset;
}
const quiet = isTrueBoolean(args?.quiet);
const contextNames = context_presets.map(preset => preset.name);
const fuse = new Fuse(contextNames);
const result = fuse.search(name);
if (result.length === 0) {
toastr.warning(`Context template "${name}" not found`);
!quiet && toastr.warning(`Context template "${name}" not found`);
return '';
}
const foundName = result[0].item;
selectContextPreset(foundName);
selectContextPreset(foundName, quiet);
return foundName;
}
@ -8528,16 +8529,16 @@ async function selectInstructCallback(args, name) {
return power_user.instruct.preset;
}
const quiet = isTrueBoolean(args?.quiet);
const instructNames = instruct_presets.map(preset => preset.name);
const fuse = new Fuse(instructNames);
const result = fuse.search(name);
if (result.length === 0) {
toastr.warning(`Instruct template "${name}" not found`);
!quiet && toastr.warning(`Instruct template "${name}" not found`);
return '';
}
const quiet = isTrueBoolean(args?.quiet);
const foundName = result[0].item;
selectInstructPreset(foundName, quiet);
return foundName;
@ -9283,6 +9284,15 @@ jQuery(async function () {
name: 'context',
callback: selectContextCallback,
returns: 'template name',
namedArgumentList: [
SlashCommandNamedArgument.fromProps({
name: 'quiet',
description: 'Suppress the toast message on template change',
typeList: [ARGUMENT_TYPE.BOOLEAN],
defaultValue: 'false',
enumList: commonEnumProviders.boolean('trueFalse')(),
}),
],
unnamedArgumentList: [
SlashCommandArgument.fromProps({
description: 'context template name',

28
public/scripts/extensions/stable-diffusion/index.js
View File

@ -724,7 +724,7 @@ function onChatChanged() {
}
async function adjustElementScrollHeight() {
if (!$('.sd_settings').is(':visible')) {
if (CSS.supports('field-sizing', 'content') || !$('.sd_settings').is(':visible')) {
return;
}
@ -737,17 +737,19 @@ async function adjustElementScrollHeight() {
async function onCharacterPromptInput() {
const key = getCharaFilename(this_chid);
extension_settings.sd.character_prompts[key] = $('#sd_character_prompt').val();
await resetScrollHeight($(this));
saveSettingsDebounced();
writePromptFieldsDebounced(this_chid);
if (CSS.supports('field-sizing', 'content')) return;
await resetScrollHeight($(this));
}
async function onCharacterNegativePromptInput() {
const key = getCharaFilename(this_chid);
extension_settings.sd.character_negative_prompts[key] = $('#sd_character_negative_prompt').val();
await resetScrollHeight($(this));
saveSettingsDebounced();
writePromptFieldsDebounced(this_chid);
if (CSS.supports('field-sizing', 'content')) return;
await resetScrollHeight($(this));
}
function getCharacterPrefix() {
@ -856,14 +858,16 @@ function onStepsInput() {
async function onPromptPrefixInput() {
extension_settings.sd.prompt_prefix = $('#sd_prompt_prefix').val();
await resetScrollHeight($(this));
saveSettingsDebounced();
if (CSS.supports('field-sizing', 'content')) return;
await resetScrollHeight($(this));
}
async function onNegativePromptInput() {
extension_settings.sd.negative_prompt = $('#sd_negative_prompt').val();
await resetScrollHeight($(this));
saveSettingsDebounced();
if (CSS.supports('field-sizing', 'content')) return;
await resetScrollHeight($(this));
}
function onSamplerChange() {
@ -3911,12 +3915,14 @@ jQuery(async () => {
$('#sd_stability_style_preset').on('change', onStabilityStylePresetChange);
$('#sd_huggingface_model_id').on('input', onHFModelInput);
$('.sd_settings .inline-drawer-toggle').on('click', function () {
initScrollHeight($('#sd_prompt_prefix'));
initScrollHeight($('#sd_negative_prompt'));
initScrollHeight($('#sd_character_prompt'));
initScrollHeight($('#sd_character_negative_prompt'));
});
if (!CSS.supports('field-sizing', 'content')) {
$('.sd_settings .inline-drawer-toggle').on('click', function () {
initScrollHeight($('#sd_prompt_prefix'));
initScrollHeight($('#sd_negative_prompt'));
initScrollHeight($('#sd_character_prompt'));
initScrollHeight($('#sd_character_negative_prompt'));
});
}
for (const [key, value] of Object.entries(resolutionOptions)) {
const option = document.createElement('option');

8
public/scripts/extensions/stable-diffusion/settings.html
View File

@ -408,16 +408,16 @@
</div>
</div>
<label for="sd_prompt_prefix" data-i18n="Common prompt prefix">Common prompt prefix</label>
<textarea id="sd_prompt_prefix" class="text_pole textarea_compact" data-i18n="[placeholder]sd_prompt_prefix_placeholder" placeholder="Use {prompt} to specify where the generated prompt will be inserted"></textarea>
<textarea id="sd_prompt_prefix" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_prompt_prefix_placeholder" placeholder="Use {prompt} to specify where the generated prompt will be inserted"></textarea>
<label for="sd_negative_prompt" data-i18n="Negative common prompt prefix">Negative common prompt prefix</label>
<textarea id="sd_negative_prompt" class="text_pole textarea_compact"></textarea>
<textarea id="sd_negative_prompt" class="text_pole textarea_compact autoSetHeight"></textarea>
<div id="sd_character_prompt_block">
<label for="sd_character_prompt" data-i18n="Character-specific prompt prefix">Character-specific prompt prefix</label>
<small data-i18n="Won't be used in groups.">Won't be used in groups.</small>
<textarea id="sd_character_prompt" class="text_pole textarea_compact" data-i18n="[placeholder]sd_character_prompt_placeholder" placeholder="Any characteristics that describe the currently selected character. Will be added after a common prompt prefix.&#10;Example: female, green eyes, brown hair, pink shirt"></textarea>
<textarea id="sd_character_prompt" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_character_prompt_placeholder" placeholder="Any characteristics that describe the currently selected character. Will be added after a common prompt prefix.&#10;Example: female, green eyes, brown hair, pink shirt"></textarea>
<label for="sd_character_negative_prompt" data-i18n="Character-specific negative prompt prefix">Character-specific negative prompt prefix</label>
<small data-i18n="Won't be used in groups.">Won't be used in groups.</small>
<textarea id="sd_character_negative_prompt" class="text_pole textarea_compact" data-i18n="[placeholder]sd_character_negative_prompt_placeholder" placeholder="Any characteristics that should not appear for the selected character. Will be added after a negative common prompt prefix.&#10;Example: jewellery, shoes, glasses"></textarea>
<textarea id="sd_character_negative_prompt" class="text_pole textarea_compact autoSetHeight" data-i18n="[placeholder]sd_character_negative_prompt_placeholder" placeholder="Any characteristics that should not appear for the selected character. Will be added after a negative common prompt prefix.&#10;Example: jewellery, shoes, glasses"></textarea>
<label for="sd_character_prompt_share" class="checkbox_label flexWrap marginTop5">
<input id="sd_character_prompt_share" type="checkbox" />
<span data-i18n="Shareable">

6
public/scripts/extensions/token-counter/index.js
View File

@ -59,8 +59,10 @@ async function doTokenCounter() {
$('#tokenized_chunks_display').text('—');
}
await resetScrollHeight($('#token_counter_textarea'));
await resetScrollHeight($('#token_counter_ids'));
if (!CSS.supports('field-sizing', 'content')) {
await resetScrollHeight($('#token_counter_textarea'));
await resetScrollHeight($('#token_counter_ids'));
}
}, debounce_timeout.relaxed);
dialog.find('#token_counter_textarea').on('input', () => countDebounced());

5
public/scripts/extensions/token-counter/style.css
View File

@ -4,3 +4,8 @@
padding: 2px;
display: inline-block;
}
#token_counter_textarea,
#token_counter_ids {
field-sizing: content;
}

7
public/scripts/group-chats.js
View File

@ -1414,8 +1414,11 @@ function toggleHiddenControls(group, generationMode = null) {
const isJoin = [group_generation_mode.APPEND, group_generation_mode.APPEND_DISABLED].includes(generationMode ?? group?.generation_mode);
$('#rm_group_generation_mode_join_prefix').parent().toggle(isJoin);
$('#rm_group_generation_mode_join_suffix').parent().toggle(isJoin);
initScrollHeight($('#rm_group_generation_mode_join_prefix'));
initScrollHeight($('#rm_group_generation_mode_join_suffix'));
if (!CSS.supports('field-sizing', 'content')) {
initScrollHeight($('#rm_group_generation_mode_join_prefix'));
initScrollHeight($('#rm_group_generation_mode_join_suffix'));
}
}
function select_group_chats(groupId, skipAnimation) {

7
public/scripts/instruct-mode.js
View File

@ -130,19 +130,20 @@ function highlightDefaultPreset() {
/**
* Select context template if not already selected.
* @param {string} preset Preset name.
* @param {boolean} quiet Suppress info message.
*/
export function selectContextPreset(preset) {
export function selectContextPreset(preset, quiet) {
// If context template is not already selected, select it
if (preset !== power_user.context.preset) {
$('#context_presets').val(preset).trigger('change');
toastr.info(`Context Template: preset "${preset}" auto-selected`);
!quiet && toastr.info(`Context Template: preset "${preset}" auto-selected`);
}
// If instruct mode is disabled, enable it, except for default context template
if (!power_user.instruct.enabled && preset !== power_user.default_context) {
power_user.instruct.enabled = true;
$('#instruct_enabled').prop('checked', true).trigger('change');
toastr.info('Instruct Mode enabled');
!quiet && toastr.info('Instruct Mode enabled');
}
saveSettingsDebounced();

7
public/scripts/textgen-models.js
View File

@ -590,6 +590,9 @@ function calculateOpenRouterCost() {
export function getCurrentOpenRouterModelTokenizer() {
const modelId = textgen_settings.openrouter_model;
const model = openRouterModels.find(x => x.id === modelId);
if (modelId?.includes('jamba')) {
return tokenizers.JAMBA;
}
switch (model?.architecture?.tokenizer) {
case 'Llama2':
return tokenizers.LLAMA;
@ -603,6 +606,10 @@ export function getCurrentOpenRouterModelTokenizer() {
return tokenizers.GEMMA;
case 'Claude':
return tokenizers.CLAUDE;
case 'Cohere':
return tokenizers.COMMAND_R;
case 'Qwen':
return tokenizers.QWEN2;
default:
return tokenizers.OPENAI;
}

30
public/scripts/tokenizers.js
View File

@ -28,6 +28,8 @@ export const tokenizers = {
LLAMA3: 12,
GEMMA: 13,
JAMBA: 14,
QWEN2: 15,
COMMAND_R: 16,
BEST_MATCH: 99,
};
@ -105,6 +107,16 @@ const TOKENIZER_URLS = {
decode: '/api/tokenizers/jamba/decode',
count: '/api/tokenizers/jamba/encode',
},
[tokenizers.QWEN2]: {
encode: '/api/tokenizers/qwen2/encode',
decode: '/api/tokenizers/qwen2/decode',
count: '/api/tokenizers/qwen2/encode',
},
[tokenizers.COMMAND_R]: {
encode: '/api/tokenizers/command-r/encode',
decode: '/api/tokenizers/command-r/decode',
count: '/api/tokenizers/command-r/encode',
},
[tokenizers.API_TEXTGENERATIONWEBUI]: {
encode: '/api/tokenizers/remote/textgenerationwebui/encode',
count: '/api/tokenizers/remote/textgenerationwebui/encode',
@ -293,6 +305,12 @@ export function getTokenizerBestMatch(forApi) {
if (model.includes('jamba')) {
return tokenizers.JAMBA;
}
if (model.includes('command-r')) {
return tokenizers.COMMAND_R;
}
if (model.includes('qwen2')) {
return tokenizers.QWEN2;
}
}
return tokenizers.LLAMA;
@ -511,6 +529,8 @@ export function getTokenizerModel() {
const yiTokenizer = 'yi';
const gemmaTokenizer = 'gemma';
const jambaTokenizer = 'jamba';
const qwen2Tokenizer = 'qwen2';
const commandRTokenizer = 'command-r';
// Assuming no one would use it for different models.. right?
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
@ -558,6 +578,12 @@ export function getTokenizerModel() {
else if (model?.architecture?.tokenizer === 'Gemini') {
return gemmaTokenizer;
}
else if (model?.architecture?.tokenizer === 'Qwen') {
return qwen2Tokenizer;
}
else if (model?.architecture?.tokenizer === 'Cohere') {
return commandRTokenizer;
}
else if (oai_settings.openrouter_model.includes('gpt-4o')) {
return gpt4oTokenizer;
}
@ -581,6 +607,10 @@ export function getTokenizerModel() {
}
}
if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {
return commandRTokenizer;
}
if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
return gemmaTokenizer;
}

8
public/style.css
View File

@ -182,6 +182,10 @@ body.movingUI ::-webkit-scrollbar-thumb:vertical {
min-width: 40px;
}
::-webkit-scrollbar-corner {
background-color: transparent;
}
table.responsiveTable {
width: 100%;
margin: 10px 0;
@ -4675,10 +4679,6 @@ body:not(.sd) .mes_img_swipes {
width: 100%;
}
.img_enlarged_holder::-webkit-scrollbar-corner {
background-color: transparent;
}
.img_enlarged_container pre code {
position: relative;
display: block;

140
src/endpoints/tokenizers.js
View File

@ -4,13 +4,12 @@ const express = require('express');
const { SentencePieceProcessor } = require('@agnai/sentencepiece-js');
const tiktoken = require('tiktoken');
const { Tokenizer } = require('@agnai/web-tokenizers');
const { convertClaudePrompt, convertGooglePrompt } = require('../prompt-converters');
const { readSecret, SECRET_KEYS } = require('./secrets');
const { convertClaudePrompt } = require('../prompt-converters');
const { TEXTGEN_TYPES } = require('../constants');
const { jsonParser } = require('../express-common');
const { setAdditionalHeaders } = require('../additional-headers');
const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
const { getConfigValue, isValidUrl } = require('../util');
const writeFileAtomicSync = require('write-file-atomic').sync;
/**
* @typedef { (req: import('express').Request, res: import('express').Response) => Promise<any> } TokenizationHandler
@ -53,6 +52,65 @@ const TEXT_COMPLETION_MODELS = [
];
const CHARS_PER_TOKEN = 3.35;
const IS_DOWNLOAD_ALLOWED = getConfigValue('enableDownloadableTokenizers', true);
/**
* Gets a path to the tokenizer model. Downloads the model if it's a URL.
* @param {string} model Model URL or path
* @param {string|undefined} fallbackModel Fallback model path\
* @returns {Promise<string>} Path to the tokenizer model
*/
async function getPathToTokenizer(model, fallbackModel) {
if (!isValidUrl(model)) {
return model;
}
try {
const url = new URL(model);
if (!['https:', 'http:'].includes(url.protocol)) {
throw new Error('Invalid URL protocol');
}
const fileName = url.pathname.split('/').pop();
if (!fileName) {
throw new Error('Failed to extract the file name from the URL');
}
const CACHE_PATH = path.join(global.DATA_ROOT, '_cache');
if (!fs.existsSync(CACHE_PATH)) {
fs.mkdirSync(CACHE_PATH, { recursive: true });
}
const cachedFile = path.join(CACHE_PATH, fileName);
if (fs.existsSync(cachedFile)) {
return cachedFile;
}
if (!IS_DOWNLOAD_ALLOWED) {
throw new Error('Downloading tokenizers is disabled, the model is not cached');
}
console.log('Downloading tokenizer model:', model);
const response = await fetch(model);
if (!response.ok) {
throw new Error(`Failed to fetch the model: ${response.status} ${response.statusText}`);
}
const arrayBuffer = await response.arrayBuffer();
writeFileAtomicSync(cachedFile, Buffer.from(arrayBuffer));
return cachedFile;
} catch (error) {
const getLastSegment = str => str?.split('/')?.pop() || '';
if (fallbackModel) {
console.log(`Could not get a tokenizer from ${getLastSegment(model)}. Reason: ${error.message}. Using a fallback model: ${getLastSegment(fallbackModel)}.`);
return fallbackModel;
}
throw new Error(`Failed to instantiate a tokenizer and fallback is not provided. Reason: ${error.message}`);
}
}
/**
* Sentencepiece tokenizer for tokenizing text.
@ -66,13 +124,19 @@ class SentencePieceTokenizer {
* @type {string} Path to the tokenizer model
*/
#model;
/**
* @type {string|undefined} Path to the fallback model
*/
#fallbackModel;
/**
* Creates a new Sentencepiece tokenizer.
* @param {string} model Path to the tokenizer model
* @param {string} [fallbackModel] Path to the fallback model
*/
constructor(model) {
constructor(model, fallbackModel) {
this.#model = model;
this.#fallbackModel = fallbackModel;
}
/**
@ -85,9 +149,10 @@ class SentencePieceTokenizer {
}
try {
const pathToModel = await getPathToTokenizer(this.#model, this.#fallbackModel);
this.#instance = new SentencePieceProcessor();
await this.#instance.load(this.#model);
console.log('Instantiated the tokenizer for', path.parse(this.#model).name);
await this.#instance.load(pathToModel);
console.log('Instantiated the tokenizer for', path.parse(pathToModel).name);
return this.#instance;
} catch (error) {
console.error('Sentencepiece tokenizer failed to load: ' + this.#model, error);
@ -108,13 +173,19 @@ class WebTokenizer {
* @type {string} Path to the tokenizer model
*/
#model;
/**
* @type {string|undefined} Path to the fallback model
*/
#fallbackModel;
/**
* Creates a new Web tokenizer.
* @param {string} model Path to the tokenizer model
* @param {string} [fallbackModel] Path to the fallback model
*/
constructor(model) {
constructor(model, fallbackModel) {
this.#model = model;
this.#fallbackModel = fallbackModel;
}
/**
@ -127,9 +198,10 @@ class WebTokenizer {
}
try {
const arrayBuffer = fs.readFileSync(this.#model).buffer;
const pathToModel = await getPathToTokenizer(this.#model, this.#fallbackModel);
const arrayBuffer = fs.readFileSync(pathToModel).buffer;
this.#instance = await Tokenizer.fromJSON(arrayBuffer);
console.log('Instantiated the tokenizer for', path.parse(this.#model).name);
console.log('Instantiated the tokenizer for', path.parse(pathToModel).name);
return this.#instance;
} catch (error) {
console.error('Web tokenizer failed to load: ' + this.#model, error);
@ -147,6 +219,8 @@ const spp_gemma = new SentencePieceTokenizer('src/tokenizers/gemma.model');
const spp_jamba = new SentencePieceTokenizer('src/tokenizers/jamba.model');
const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
const llama3_tokenizer = new WebTokenizer('src/tokenizers/llama3.json');
const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/command-r.json', 'src/tokenizers/llama3.json');
const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');
const sentencepieceTokenizers = [
'llama',
@ -332,6 +406,14 @@ function getTokenizerModel(requestModel) {
return 'jamba';
}
if (requestModel.includes('qwen2')) {
return 'qwen2';
}
if (requestModel.includes('command-r')) {
return 'command-r';
}
// default
return 'gpt-3.5-turbo';
}
@ -557,6 +639,8 @@ router.post('/jamba/encode', jsonParser, createSentencepieceEncodingHandler(spp_
router.post('/gpt2/encode', jsonParser, createTiktokenEncodingHandler('gpt2'));
router.post('/claude/encode', jsonParser, createWebTokenizerEncodingHandler(claude_tokenizer));
router.post('/llama3/encode', jsonParser, createWebTokenizerEncodingHandler(llama3_tokenizer));
router.post('/qwen2/encode', jsonParser, createWebTokenizerEncodingHandler(qwen2Tokenizer));
router.post('/command-r/encode', jsonParser, createWebTokenizerEncodingHandler(commandTokenizer));
router.post('/llama/decode', jsonParser, createSentencepieceDecodingHandler(spp_llama));
router.post('/nerdstash/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd));
router.post('/nerdstash_v2/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2));
@ -567,6 +651,8 @@ router.post('/jamba/decode', jsonParser, createSentencepieceDecodingHandler(spp_
router.post('/gpt2/decode', jsonParser, createTiktokenDecodingHandler('gpt2'));
router.post('/claude/decode', jsonParser, createWebTokenizerDecodingHandler(claude_tokenizer));
router.post('/llama3/decode', jsonParser, createWebTokenizerDecodingHandler(llama3_tokenizer));
router.post('/qwen2/decode', jsonParser, createWebTokenizerDecodingHandler(qwen2Tokenizer));
router.post('/command-r/decode', jsonParser, createWebTokenizerDecodingHandler(commandTokenizer));
router.post('/openai/encode', jsonParser, async function (req, res) {
try {
@ -607,6 +693,16 @@ router.post('/openai/encode', jsonParser, async function (req, res) {
return handler(req, res);
}
if (queryModel.includes('qwen2')) {
const handler = createWebTokenizerEncodingHandler(qwen2Tokenizer);
return handler(req, res);
}
if (queryModel.includes('command-r')) {
const handler = createWebTokenizerEncodingHandler(commandTokenizer);
return handler(req, res);
}
const model = getTokenizerModel(queryModel);
const handler = createTiktokenEncodingHandler(model);
return handler(req, res);
@ -655,6 +751,16 @@ router.post('/openai/decode', jsonParser, async function (req, res) {
return handler(req, res);
}
if (queryModel.includes('qwen2')) {
const handler = createWebTokenizerDecodingHandler(qwen2Tokenizer);
return handler(req, res);
}
if (queryModel.includes('command-r')) {
const handler = createWebTokenizerDecodingHandler(commandTokenizer);
return handler(req, res);
}
const model = getTokenizerModel(queryModel);
const handler = createTiktokenDecodingHandler(model);
return handler(req, res);
@ -711,6 +817,20 @@ router.post('/openai/count', jsonParser, async function (req, res) {
return res.send({ 'token_count': num_tokens });
}
if (model === 'qwen2') {
const instance = await qwen2Tokenizer.get();
if (!instance) throw new Error('Failed to load the Qwen2 tokenizer');
num_tokens = countWebTokenizerTokens(instance, req.body);
return res.send({ 'token_count': num_tokens });
}
if (model === 'command-r') {
const instance = await commandTokenizer.get();
if (!instance) throw new Error('Failed to load the Command-R tokenizer');
num_tokens = countWebTokenizerTokens(instance, req.body);
return res.send({ 'token_count': num_tokens });
}
const tokensPerName = queryModel.includes('gpt-3.5-turbo-0301') ? -1 : 1;
const tokensPerMessage = queryModel.includes('gpt-3.5-turbo-0301') ? 4 : 3;
const tokensPadding = 3;

15
src/util.js
View File

@ -647,6 +647,20 @@ function getSeparator(n) {
return '='.repeat(n);
}
/**
* Checks if the string is a valid URL.
* @param {string} url String to check
* @returns {boolean} If the URL is valid
*/
function isValidUrl(url) {
try {
new URL(url);
return true;
} catch (error) {
return false;
}
}
module.exports = {
getConfig,
getConfigValue,
@ -676,4 +690,5 @@ module.exports = {
makeHttp2Request,
removeColorFormatting,
getSeparator,
isValidUrl,
};