rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-02-10 00:50:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add a token counter for persona descriptions

Browse Source
This commit is contained in:
Cohee 2023-08-22 18:32:18 +03:00
parent ea4d4a8fd6
commit 042c0b84a1
4 changed files with 24 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
public/index.html
View File

@ -2981,7 +2981,10 @@
</div> </div>
<div> <div>
<h4 data-i18n="Persona Description">Persona Description</h4> <h4 data-i18n="Persona Description">Persona Description</h4>
<textarea id="persona_description" name="persona_description" placeholder="Example:&#10;[{{user}} is a 28-year-old Romanian cat girl.]" class="text_pole textarea_compact" maxlength="5000" value="" autocomplete="off" rows="4"></textarea> <textarea id="persona_description" name="persona_description" placeholder="Example:&#10;[{{user}} is a 28-year-old Romanian cat girl.]" class="text_pole textarea_compact" maxlength="5000" value="" autocomplete="off" rows="8"></textarea>
<div class="extension_token_counter">
Tokens: <span id="persona_description_token_count">0</span>
</div>
<label for="persona_description_position" data-i18n="Position:">Position:</label> <label for="persona_description_position" data-i18n="Position:">Position:</label>
<select id="persona_description_position"> <select id="persona_description_position">
<option value="0" data-i18n="In Story String / Chat Completion: Before Character Card">In Story String / Chat Completion: Before Character Card</option> <option value="0" data-i18n="In Story String / Chat Completion: Before Character Card">In Story String / Chat Completion: Before Character Card</option>

17
public/script.js
View File

@ -543,6 +543,12 @@ function getTokenizerBestMatch() {
return power_user.NONE; return power_user.NONE;
} }
/**
* Gets the token count for a string using the current model tokenizer.
* @param {string} str String to tokenize
* @param {number | undefined} padding Optional padding tokens. Defaults to 0.
* @returns {number} Token count.
*/
function getTokenCount(str, padding = undefined) { function getTokenCount(str, padding = undefined) {
if (typeof str !== 'string' || !str?.length) { if (typeof str !== 'string' || !str?.length) {
return 0; return 0;
@ -4520,6 +4526,7 @@ function setPersonaDescription() {
.val(power_user.persona_description_position) .val(power_user.persona_description_position)
.find(`option[value='${power_user.persona_description_position}']`) .find(`option[value='${power_user.persona_description_position}']`)
.attr("selected", true); .attr("selected", true);
countPersonaDescriptionTokens();
} }
function onPersonaDescriptionPositionInput() { function onPersonaDescriptionPositionInput() {
@ -4544,8 +4551,18 @@ function onPersonaDescriptionPositionInput() {
saveSettingsDebounced(); saveSettingsDebounced();
} }
/**
* Counts the number of tokens in a persona description.
*/
const countPersonaDescriptionTokens = debounce(() => {
const description = String($("#persona_description").val());
const count = getTokenCount(description);
$("#persona_description_token_count").text(String(count));
}, durationSaveEdit);
function onPersonaDescriptionInput() { function onPersonaDescriptionInput() {
power_user.persona_description = String($("#persona_description").val()); power_user.persona_description = String($("#persona_description").val());
countPersonaDescriptionTokens();
if (power_user.personas[user_avatar]) { if (power_user.personas[user_avatar]) {
let object = power_user.persona_descriptions[user_avatar]; let object = power_user.persona_descriptions[user_avatar];

2
public/scripts/RossAscends-mods.js
View File

@ -197,7 +197,7 @@ export function RA_CountCharTokens() {
$('[data-token-counter]').each(function () { $('[data-token-counter]').each(function () {
const counter = $(this); const counter = $(this);
const input = $(document.getElementById(counter.data('token-counter'))); const input = $(document.getElementById(counter.data('token-counter')));
const value = input.val(); const value = String(input.val());
if (input.length === 0) { if (input.length === 0) {
counter.text('Invalid input reference'); counter.text('Invalid input reference');

5
public/scripts/extensions/stable-diffusion/index.js
View File

@ -14,7 +14,7 @@ import {
import { getApiUrl, getContext, extension_settings, doExtrasFetch, modules } from "../../extensions.js"; import { getApiUrl, getContext, extension_settings, doExtrasFetch, modules } from "../../extensions.js";
import { selected_group } from "../../group-chats.js"; import { selected_group } from "../../group-chats.js";
import { stringFormat, initScrollHeight, resetScrollHeight, timestampToMoment, getCharaFilename, saveBase64AsFile } from "../../utils.js"; import { stringFormat, initScrollHeight, resetScrollHeight, timestampToMoment, getCharaFilename, saveBase64AsFile } from "../../utils.js";
import { humanizedDateTime } from "../../RossAscends-mods.js"; import { getMessageTimeStamp, humanizedDateTime } from "../../RossAscends-mods.js";
export { MODULE_NAME }; export { MODULE_NAME };
// Wraps a string into monospace font-face span // Wraps a string into monospace font-face span
@ -755,11 +755,10 @@ async function sendMessage(prompt, image) {
const messageText = `[${context.name2} sends a picture that contains: ${prompt}]`; const messageText = `[${context.name2} sends a picture that contains: ${prompt}]`;
const message = { const message = {
name: context.groupId ? systemUserName : context.name2, name: context.groupId ? systemUserName : context.name2,
is_system: context.groupId ? true : false,
is_user: false, is_user: false,
is_system: true, is_system: true,
is_name: true, is_name: true,
send_date: timestampToMoment(Date.now()).format('LL LT'), send_date: getMessageTimeStamp(),
mes: context.groupId ? p(messageText) : messageText, mes: context.groupId ? p(messageText) : messageText,
extra: { extra: {
image: image, image: image,