From cd2faea2b21c577222e508e7223bdbde869c11d1 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 24 Aug 2023 17:46:44 +0300 Subject: [PATCH 1/4] Fix group chats with streaming --- public/scripts/group-chats.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/scripts/group-chats.js b/public/scripts/group-chats.js index 339d477b5..db8753b35 100644 --- a/public/scripts/group-chats.js +++ b/public/scripts/group-chats.js @@ -8,6 +8,7 @@ import { extractAllWords, saveBase64AsFile, PAGINATION_TEMPLATE, + waitUntilCondition, } from './utils.js'; import { RA_CountCharTokens, humanizedDateTime, dragElement, favsToHotswap, getMessageTimeStamp } from "./RossAscends-mods.js"; import { loadMovingUIState, sortEntitiesList } from './power-user.js'; @@ -665,6 +666,7 @@ async function generateGroupWrapper(by_auto_mode, type = null, params = {}) { if (streamingProcessor && !streamingProcessor.isFinished) { await delay(100); } else { + await waitUntilCondition(() => streamingProcessor == null, 1000, 10); messagesBefore++; break; } From 7010e05f8e9ee90c5c53874829d1dfeb791b0714 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 24 Aug 2023 18:32:42 +0300 Subject: [PATCH 2/4] Sync bottom form height with the font size --- public/index.html | 2 +- public/scripts/RossAscends-mods.js | 2 +- public/style.css | 22 +++++++++++++--------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/public/index.html b/public/index.html index a3495686a..cc54fa665 100644 --- a/public/index.html +++ b/public/index.html @@ -2592,7 +2592,7 @@
- +
diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index 2d6b6da3f..ee2db4924 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -843,7 +843,7 @@ jQuery(async function () { //this makes the chat input text area resize vertically to match the text size (limited by CSS at 50% window height) $('#send_textarea').on('input', function () { - this.style.height = '30px'; + this.style.height = window.getComputedStyle(this).getPropertyValue('min-height'); this.style.height = (this.scrollHeight) + 'px'; }); diff --git a/public/style.css b/public/style.css index c1ced90c6..7218265c8 100644 --- a/public/style.css +++ b/public/style.css @@ -67,6 +67,10 @@ color-scheme: only light; + /* Send form variables */ + --bottomFormBlockPadding: calc(var(--mainFontSize) / 3); + --bottomFormIconSize: calc(var(--mainFontSize) * 2); + --bottomFormBlockSize: calc(var(--bottomFormIconSize) + var(--bottomFormBlockPadding)); /*styles for the color picker*/ --tool-cool-color-picker-btn-bg: transparent; @@ -513,19 +517,19 @@ hr { #send_but_sheld { padding: 0; border: 0; - height: 30px; + height: var(--bottomFormBlockSize); position: relative; background-position: center; display: flex; flex-direction: row; column-gap: 5px; - font-size: 25px; + font-size: var(--bottomFormIconSize); overflow: hidden; } #send_but_sheld>div { - width: 30px; - height: 30px; + width: var(--bottomFormBlockSize); + height: var(--bottomFormBlockSize); margin: 0; outline: none; border: none; @@ -556,8 +560,9 @@ hr { } #options_button { - width: 30px; - height: 30px; + width: var(--bottomFormBlockSize); + height: var(--bottomFormBlockSize); + font-size: var(--bottomFormIconSize); margin: 0; outline: none; border: none; @@ -568,7 +573,6 @@ hr { margin-left: 10px; padding: 0; transition: 0.3s; - font-size: 25px; display: flex; align-items: center; } @@ -879,11 +883,11 @@ select { } #send_textarea { - min-height: 30px; + min-height: var(--bottomFormBlockSize); + height: var(--bottomFormBlockSize); max-height: 50vh; max-height: 50svh; word-wrap: break-word; - height: 30px; resize: vertical; display: block; background-color: rgba(255, 0, 0, 0); From ab52af4fb5cc4ee27a0146f42d612c7ba9e42f33 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 24 Aug 2023 20:19:57 +0300 Subject: [PATCH 3/4] Add support for Koboldcpp tokenization endpoint --- public/index.html | 2 +- public/scripts/tokenizers.js | 37 ++++++++++++++++++++++++++++++++++-- server.js | 14 +++++++++++--- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/public/index.html b/public/index.html index b4726d0d8..6afcb8388 100644 --- a/public/index.html +++ b/public/index.html @@ -2245,7 +2245,7 @@ - +
diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index 8f9b82405..4900e30c8 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -24,6 +24,15 @@ const gpt3 = new GPT3BrowserTokenizer({ type: 'gpt3' }); let tokenCache = {}; +/** + * Guesstimates the token count for a string. + * @param {string} str String to tokenize. + * @returns {number} Token count. + */ +export function guesstimate(str) { + return Math.ceil(str.length / CHARACTERS_PER_TOKEN_RATIO); +} + async function loadTokenCache() { try { console.debug('Chat Completions: loading token cache') @@ -89,7 +98,7 @@ export function getTokenCount(str, padding = undefined) { function calculate(type) { switch (type) { case tokenizers.NONE: - return Math.ceil(str.length / CHARACTERS_PER_TOKEN_RATIO) + padding; + return guesstimate(str) + padding; case tokenizers.GPT3: return gpt3.encode(str).bpe.length + padding; case tokenizers.CLASSIC: @@ -291,8 +300,16 @@ function getTokenCacheObject() { return tokenCache[String(chatId)]; } +/** + * Counts token using the remote server API. + * @param {string} endpoint API endpoint. + * @param {string} str String to tokenize. + * @param {number} padding Number of padding tokens. + * @returns {number} Token count with padding. + */ function countTokensRemote(endpoint, str, padding) { let tokenCount = 0; + jQuery.ajax({ async: false, type: 'POST', @@ -301,9 +318,25 @@ function countTokensRemote(endpoint, str, padding) { dataType: "json", contentType: "application/json", success: function (data) { - tokenCount = data.count; + if (typeof data.count === 'number') { + tokenCount = data.count; + } else { + tokenCount = guesstimate(str); + console.error("Error counting tokens"); + + if (!sessionStorage.getItem('tokenizationWarningShown')) { + toastr.warning( + "Your selected API doesn't support the tokenization endpoint. Using estimated counts.", + "Error counting tokens", + { timeOut: 10000, preventDuplicates: true }, + ); + + sessionStorage.setItem('tokenizationWarningShown', String(true)); + } + } } }); + return tokenCount + padding; } diff --git a/server.js b/server.js index b46880ccb..50e4dcfb3 100644 --- a/server.js +++ b/server.js @@ -3842,11 +3842,19 @@ app.post("/tokenize_via_api", jsonParser, async function (request, response) { if (main_api == 'textgenerationwebui' && request.body.use_mancer) { args.headers = Object.assign(args.headers, get_mancer_headers()); + const data = await postAsync(api_server + "/v1/token-count", args); + return response.send({ count: data['results'][0]['tokens'] }); } - const data = await postAsync(api_server + "/v1/token-count", args); - console.log(data); - return response.send({ count: data['results'][0]['tokens'] }); + else if (main_api == 'kobold') { + const data = await postAsync(api_server + "/extra/tokencount", args); + const count = data['value']; + return response.send({ count: count }); + } + + else { + return response.send({ error: true }); + } } catch (error) { console.log(error); return response.send({ error: true }); From c91ab3b5e0d05f21963f42dd9fac829985346afc Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 24 Aug 2023 21:23:35 +0300 Subject: [PATCH 4/4] Add Kobold tokenization to best match logic. Fix not being able to stop group chat regeneration --- public/script.js | 6 ++++++ public/scripts/group-chats.js | 5 ++++- public/scripts/kai-settings.js | 34 ++++++++++++++++++-------------- public/scripts/preset-manager.js | 2 ++ public/scripts/tokenizers.js | 16 ++++++++++++--- 5 files changed, 44 insertions(+), 19 deletions(-) diff --git a/public/script.js b/public/script.js index 0aea4a0c5..44972edb2 100644 --- a/public/script.js +++ b/public/script.js @@ -8,6 +8,7 @@ import { getKoboldGenerationData, canUseKoboldStopSequence, canUseKoboldStreaming, + canUseKoboldTokenization, } from "./scripts/kai-settings.js"; import { @@ -783,6 +784,7 @@ async function getStatus() { if (main_api === "kobold" || main_api === "koboldhorde") { kai_settings.use_stop_sequence = canUseKoboldStopSequence(data.version); kai_settings.can_use_streaming = canUseKoboldStreaming(data.koboldVersion); + kai_settings.can_use_tokenization = canUseKoboldTokenization(data.koboldVersion); } // We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress. @@ -4007,6 +4009,10 @@ export function setMenuType(value) { menu_type = value; } +export function setExternalAbortController(controller) { + abortController = controller; +} + function setCharacterId(value) { this_chid = value; } diff --git a/public/scripts/group-chats.js b/public/scripts/group-chats.js index db8753b35..86ad3eead 100644 --- a/public/scripts/group-chats.js +++ b/public/scripts/group-chats.js @@ -65,6 +65,7 @@ import { getCropPopup, system_avatar, isChatSaving, + setExternalAbortController, } from "../script.js"; import { appendTagToList, createTagMapFromList, getTagsList, applyTagsOnCharacterSelect, tag_map, printTagFilters } from './tags.js'; import { FILTER_TYPES, FilterHelper } from './filters.js'; @@ -135,7 +136,9 @@ async function regenerateGroup() { await deleteLastMessage(); } - generateGroupWrapper(); + const abortController = new AbortController(); + setExternalAbortController(abortController); + generateGroupWrapper(false, 'normal', { signal: abortController.signal }); } async function loadGroupChat(chatId) { diff --git a/public/scripts/kai-settings.js b/public/scripts/kai-settings.js index 9dbb5ec21..726900a8a 100644 --- a/public/scripts/kai-settings.js +++ b/public/scripts/kai-settings.js @@ -9,16 +9,7 @@ import { } from "./power-user.js"; import { getSortableDelay } from "./utils.js"; -export { - kai_settings, - loadKoboldSettings, - formatKoboldUrl, - getKoboldGenerationData, - canUseKoboldStopSequence, - canUseKoboldStreaming, -}; - -const kai_settings = { +export const kai_settings = { temp: 1, rep_pen: 1, rep_pen_range: 0, @@ -30,15 +21,17 @@ const kai_settings = { rep_pen_slope: 0.9, single_line: false, use_stop_sequence: false, + can_use_tokenization: false, streaming_kobold: false, sampler_order: [0, 1, 2, 3, 4, 5, 6], }; const MIN_STOP_SEQUENCE_VERSION = '1.2.2'; const MIN_STREAMING_KCPPVERSION = '1.30'; +const MIN_TOKENIZATION_KCPPVERSION = '1.41'; const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5]; -function formatKoboldUrl(value) { +export function formatKoboldUrl(value) { try { const url = new URL(value); if (!power_user.relaxed_api_urls) { @@ -49,7 +42,7 @@ function formatKoboldUrl(value) { return null; } -function loadKoboldSettings(preset) { +export function loadKoboldSettings(preset) { for (const name of Object.keys(kai_settings)) { const value = preset[name]; const slider = sliders.find(x => x.name === name); @@ -75,7 +68,7 @@ function loadKoboldSettings(preset) { } } -function getKoboldGenerationData(finalPrompt, this_settings, this_amount_gen, this_max_context, isImpersonate, type) { +export function getKoboldGenerationData(finalPrompt, this_settings, this_amount_gen, this_max_context, isImpersonate, type) { const sampler_order = kai_settings.sampler_order || this_settings.sampler_order; let generate_data = { prompt: finalPrompt, @@ -228,7 +221,7 @@ const sliders = [ * @param {string} version KoboldAI version to check. * @returns {boolean} True if the Kobold stop sequence can be used, false otherwise. */ -function canUseKoboldStopSequence(version) { +export function canUseKoboldStopSequence(version) { return (version || '0.0.0').localeCompare(MIN_STOP_SEQUENCE_VERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1; } @@ -237,12 +230,23 @@ function canUseKoboldStopSequence(version) { * @param {{ result: string; version: string; }} koboldVersion KoboldAI version object. * @returns {boolean} True if the Kobold streaming API can be used, false otherwise. */ -function canUseKoboldStreaming(koboldVersion) { +export function canUseKoboldStreaming(koboldVersion) { if (koboldVersion && koboldVersion.result == 'KoboldCpp') { return (koboldVersion.version || '0.0').localeCompare(MIN_STREAMING_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1; } else return false; } +/** + * Determines if the Kobold tokenization API can be used with the given version. + * @param {{ result: string; version: string; }} koboldVersion KoboldAI version object. + * @returns {boolean} True if the Kobold tokenization API can be used, false otherwise. + */ +export function canUseKoboldTokenization(koboldVersion) { + if (koboldVersion && koboldVersion.result == 'KoboldCpp') { + return (koboldVersion.version || '0.0').localeCompare(MIN_TOKENIZATION_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1; + } else return false; +} + /** * Sorts the sampler items by the given order. * @param {any[]} orderArray Sampler order array. diff --git a/public/scripts/preset-manager.js b/public/scripts/preset-manager.js index 810ee070f..e90602d71 100644 --- a/public/scripts/preset-manager.js +++ b/public/scripts/preset-manager.js @@ -246,6 +246,8 @@ class PresetManager { 'streaming_url', 'stopping_strings', 'use_stop_sequence', + 'can_use_tokenization', + 'can_use_streaming', 'preset_settings_novel', 'streaming_novel', 'nai_preamble', diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index 4900e30c8..0b5a34668 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -1,12 +1,14 @@ -import { characters, main_api, nai_settings, this_chid } from "../script.js"; +import { characters, main_api, nai_settings, online_status, this_chid } from "../script.js"; import { power_user } from "./power-user.js"; import { encode } from "../lib/gpt-2-3-tokenizer/mod.js"; import { GPT3BrowserTokenizer } from "../lib/gpt-3-tokenizer/gpt3-tokenizer.js"; import { chat_completion_sources, oai_settings } from "./openai.js"; import { groups, selected_group } from "./group-chats.js"; import { getStringHash } from "./utils.js"; +import { kai_settings } from "./kai-settings.js"; export const CHARACTERS_PER_TOKEN_RATIO = 3.35; +const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown'; export const tokenizers = { NONE: 0, @@ -77,6 +79,14 @@ function getTokenizerBestMatch() { } } if (main_api === 'kobold' || main_api === 'textgenerationwebui' || main_api === 'koboldhorde') { + // Try to use the API tokenizer if possible: + // - API must be connected + // - Kobold must pass a version check + // - Tokenizer haven't reported an error previously + if (kai_settings.can_use_tokenization && !sessionStorage.getItem(TOKENIZER_WARNING_KEY) && online_status !== 'no_connection') { + return tokenizers.API; + } + return tokenizers.LLAMA; } @@ -324,14 +334,14 @@ function countTokensRemote(endpoint, str, padding) { tokenCount = guesstimate(str); console.error("Error counting tokens"); - if (!sessionStorage.getItem('tokenizationWarningShown')) { + if (!sessionStorage.getItem(TOKENIZER_WARNING_KEY)) { toastr.warning( "Your selected API doesn't support the tokenization endpoint. Using estimated counts.", "Error counting tokens", { timeOut: 10000, preventDuplicates: true }, ); - sessionStorage.setItem('tokenizationWarningShown', String(true)); + sessionStorage.setItem(TOKENIZER_WARNING_KEY, String(true)); } } }