import { callPopup, main_api } from "../../../script.js";
import { getContext } from "../../extensions.js";
import { registerSlashCommand } from "../../slash-commands.js";
import { getTokenCount, getTokenizerModel } from "../../tokenizers.js";
async function doTokenCounter() {
const selectedTokenizer = main_api == 'openai'
? `tiktoken (${getTokenizerModel()})`
: $("#tokenizer").find(':selected').text();
const html = `
Token Counter
Type / paste in the box below to see the number of tokens in the text.
Selected tokenizer: ${selectedTokenizer}
Tokens: 0
`;
const dialog = $(html);
dialog.find('#token_counter_textarea').on('input', () => {
const text = $('#token_counter_textarea').val();
const context = getContext();
const count = context.getTokenCount(text);
$('#token_counter_result').text(count);
});
$('#dialogue_popup').addClass('wide_dialogue_popup');
callPopup(dialog, 'text');
}
function doCount() {
// get all of the messages in the chat
const context = getContext();
const messages = context.chat.filter(x => x.mes && !x.is_system).map(x => x.mes);
//concat all the messages into a single string
const allMessages = messages.join(' ');
console.debug('All messages:', allMessages);
//toastr success with the token count of the chat
toastr.success(`Token count: ${getTokenCount(allMessages)}`);
}
jQuery(() => {
const buttonHtml = `
Token Counter
`;
$('#extensionsMenu').prepend(buttonHtml);
$('#token_counter').on('click', doTokenCounter);
registerSlashCommand('count', doCount, [], '– counts the number of tokens in the current chat', true, false);
});