diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index 32f13647e..6139122b0 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -28,6 +28,7 @@ import { callPopup, deactivateSendButtons, activateSendButtons, + main_api, } from "../script.js"; import { getMessageTimeStamp } from "./RossAscends-mods.js"; import { findGroupMemberId, groups, is_group_generating, resetSelectedGroup, saveGroupChat, selected_group } from "./group-chats.js"; @@ -36,8 +37,9 @@ import { addEphemeralStoppingString, chat_styles, flushEphemeralStoppingStrings, import { autoSelectPersona } from "./personas.js"; import { getContext } from "./extensions.js"; import { hideChatMessage, unhideChatMessage } from "./chats.js"; -import { delay, isFalseBoolean, isTrueBoolean, stringToRange } from "./utils.js"; +import { delay, isFalseBoolean, isTrueBoolean, stringToRange, trimToEndSentence, trimToStartSentence } from "./utils.js"; import { registerVariableCommands, resolveVariable } from "./variables.js"; +import { decodeTextTokens, getFriendlyTokenizerName, getTextTokens, getTokenCount } from "./tokenizers.js"; export { executeSlashCommands, registerSlashCommand, @@ -175,6 +177,9 @@ parser.addCommand('run', runCallback, ['call', 'exec'], '(names=off/on [message index or range]) – returns the specified message or range of messages as a string.', true, true); parser.addCommand('setinput', setInputCallback, [], '(text) – sets the user input to the specified text and passes it to the next command through the pipe.', true, true); parser.addCommand('popup', popupCallback, [], '(text) – shows a blocking popup with the specified text.', true, true); +parser.addCommand('trimtokens', trimTokensCallback, [], '(direction=start/end limit=number [text]) – trims the start or end of text to the specified number of tokens.', true, true); +parser.addCommand('trimstart', trimStartCallback, [], '(text) – trims the text to the start of the first full sentence.', true, true); +parser.addCommand('trimend', trimEndCallback, [], '(text) – trims the text to the end of the last full sentence.', true, true); registerVariableCommands(); const NARRATOR_NAME_KEY = 'narrator_name'; @@ -186,6 +191,70 @@ function setInputCallback(_, value) { return value; } +function trimStartCallback(_, value) { + if (!value) { + return ''; + } + + return trimToStartSentence(value); +} + +function trimEndCallback(_, value) { + if (!value) { + return ''; + } + + return trimToEndSentence(value); +} + +function trimTokensCallback(arg, value) { + if (!value) { + console.warn('WARN: No argument provided for /trimtokens command'); + return ''; + } + + const limit = Number(resolveVariable(arg.limit)); + + if (isNaN(limit)) { + console.warn(`WARN: Invalid limit provided for /trimtokens command: ${limit}`); + return value; + } + + if (limit <= 0) { + return ''; + } + + const direction = arg.direction || 'end'; + const tokenCount = getTokenCount(value) + + // Token count is less than the limit, do nothing + if (tokenCount <= limit) { + return value; + } + + const { tokenizerName, tokenizerId } = getFriendlyTokenizerName(main_api); + console.debug('Requesting tokenization for /trimtokens command', tokenizerName); + + try { + const textTokens = getTextTokens(tokenizerId, value); + + if (!Array.isArray(textTokens) || !textTokens.length) { + console.warn('WARN: No tokens returned for /trimtokens command, falling back to estimation'); + const percentage = limit / tokenCount; + const trimIndex = Math.floor(value.length * percentage); + const trimmedText = direction === 'start' ? value.substring(trimIndex) : value.substring(0, value.length - trimIndex); + return trimmedText; + } + + const sliceTokens = direction === 'start' ? textTokens.slice(0, limit) : textTokens.slice(-limit); + const decodedText = decodeTextTokens(tokenizerId, sliceTokens); + return decodedText; + } catch (error) { + console.warn('WARN: Tokenization failed for /trimtokens command, returning original', error); + return value; + } +} + async function popupCallback(_, value) { const safeValue = DOMPurify.sanitize(value || ''); await delay(1); diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index 079cbf33b..7493fabee 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -467,7 +467,11 @@ function getTextTokensRemote(endpoint, str, model = '') { * @param {string} endpoint API endpoint. * @param {number[]} ids Array of token ids */ -function decodeTextTokensRemote(endpoint, ids) { +function decodeTextTokensRemote(endpoint, ids, model = '') { + if (model) { + endpoint += `?model=${model}`; + } + let text = ''; jQuery.ajax({ async: false, @@ -533,6 +537,9 @@ export function decodeTextTokens(tokenizerType, ids) { return decodeTextTokensRemote('/api/decode/mistral', ids); case tokenizers.YI: return decodeTextTokensRemote('/api/decode/yi', ids); + case tokenizers.OPENAI: + const model = getTokenizerModel(); + return decodeTextTokensRemote('/api/decode/openai', ids, model); default: console.warn("Calling decodeTextTokens with unsupported tokenizer type", tokenizerType); return ''; diff --git a/src/tokenizers.js b/src/tokenizers.js index c1bd90f75..b7e7ded98 100644 --- a/src/tokenizers.js +++ b/src/tokenizers.js @@ -245,7 +245,7 @@ async function loadClaudeTokenizer(modelPath) { } function countClaudeTokens(tokenizer, messages) { - const convertedPrompt = convertClaudePrompt(messages, false, false); + const convertedPrompt = convertClaudePrompt(messages, false, false, false); // Fallback to strlen estimation if (!tokenizer) { @@ -435,6 +435,40 @@ function registerEndpoints(app, jsonParser) { } }); + app.post('/api/decode/openai', jsonParser, async function (req, res) { + try { + const queryModel = String(req.query.model || ''); + + if (queryModel.includes('llama')) { + const handler = createSentencepieceDecodingHandler(spp_llama); + return handler(req, res); + } + + if (queryModel.includes('mistral')) { + const handler = createSentencepieceDecodingHandler(spp_mistral); + return handler(req, res); + } + + if (queryModel.includes('yi')) { + const handler = createSentencepieceDecodingHandler(spp_yi); + return handler(req, res); + } + + if (queryModel.includes('claude')) { + const ids = req.body.ids || []; + const chunkText = await claude_tokenizer.decode(new Uint32Array(ids)); + return res.send({ text: chunkText }); + } + + const model = getTokenizerModel(queryModel); + const handler = createTiktokenDecodingHandler(model); + return handler(req, res); + } catch (error) { + console.log(error); + return res.send({ text: '' }); + } + }); + app.post("/api/tokenize/openai", jsonParser, async function (req, res) { try { if (!req.body) return res.sendStatus(400);