diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js
index 2d6b6da3f..ee2db4924 100644
--- a/public/scripts/RossAscends-mods.js
+++ b/public/scripts/RossAscends-mods.js
@@ -843,7 +843,7 @@ jQuery(async function () {
//this makes the chat input text area resize vertically to match the text size (limited by CSS at 50% window height)
$('#send_textarea').on('input', function () {
- this.style.height = '30px';
+ this.style.height = window.getComputedStyle(this).getPropertyValue('min-height');
this.style.height = (this.scrollHeight) + 'px';
});
diff --git a/public/style.css b/public/style.css
index c1ced90c6..7218265c8 100644
--- a/public/style.css
+++ b/public/style.css
@@ -67,6 +67,10 @@
color-scheme: only light;
+ /* Send form variables */
+ --bottomFormBlockPadding: calc(var(--mainFontSize) / 3);
+ --bottomFormIconSize: calc(var(--mainFontSize) * 2);
+ --bottomFormBlockSize: calc(var(--bottomFormIconSize) + var(--bottomFormBlockPadding));
/*styles for the color picker*/
--tool-cool-color-picker-btn-bg: transparent;
@@ -513,19 +517,19 @@ hr {
#send_but_sheld {
padding: 0;
border: 0;
- height: 30px;
+ height: var(--bottomFormBlockSize);
position: relative;
background-position: center;
display: flex;
flex-direction: row;
column-gap: 5px;
- font-size: 25px;
+ font-size: var(--bottomFormIconSize);
overflow: hidden;
}
#send_but_sheld>div {
- width: 30px;
- height: 30px;
+ width: var(--bottomFormBlockSize);
+ height: var(--bottomFormBlockSize);
margin: 0;
outline: none;
border: none;
@@ -556,8 +560,9 @@ hr {
}
#options_button {
- width: 30px;
- height: 30px;
+ width: var(--bottomFormBlockSize);
+ height: var(--bottomFormBlockSize);
+ font-size: var(--bottomFormIconSize);
margin: 0;
outline: none;
border: none;
@@ -568,7 +573,6 @@ hr {
margin-left: 10px;
padding: 0;
transition: 0.3s;
- font-size: 25px;
display: flex;
align-items: center;
}
@@ -879,11 +883,11 @@ select {
}
#send_textarea {
- min-height: 30px;
+ min-height: var(--bottomFormBlockSize);
+ height: var(--bottomFormBlockSize);
max-height: 50vh;
max-height: 50svh;
word-wrap: break-word;
- height: 30px;
resize: vertical;
display: block;
background-color: rgba(255, 0, 0, 0);
From ab52af4fb5cc4ee27a0146f42d612c7ba9e42f33 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Thu, 24 Aug 2023 20:19:57 +0300
Subject: [PATCH 3/4] Add support for Koboldcpp tokenization endpoint
---
public/index.html | 2 +-
public/scripts/tokenizers.js | 37 ++++++++++++++++++++++++++++++++++--
server.js | 14 +++++++++++---
3 files changed, 47 insertions(+), 6 deletions(-)
diff --git a/public/index.html b/public/index.html
index b4726d0d8..6afcb8388 100644
--- a/public/index.html
+++ b/public/index.html
@@ -2245,7 +2245,7 @@
-
+
diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js
index 8f9b82405..4900e30c8 100644
--- a/public/scripts/tokenizers.js
+++ b/public/scripts/tokenizers.js
@@ -24,6 +24,15 @@ const gpt3 = new GPT3BrowserTokenizer({ type: 'gpt3' });
let tokenCache = {};
+/**
+ * Guesstimates the token count for a string.
+ * @param {string} str String to tokenize.
+ * @returns {number} Token count.
+ */
+export function guesstimate(str) {
+ return Math.ceil(str.length / CHARACTERS_PER_TOKEN_RATIO);
+}
+
async function loadTokenCache() {
try {
console.debug('Chat Completions: loading token cache')
@@ -89,7 +98,7 @@ export function getTokenCount(str, padding = undefined) {
function calculate(type) {
switch (type) {
case tokenizers.NONE:
- return Math.ceil(str.length / CHARACTERS_PER_TOKEN_RATIO) + padding;
+ return guesstimate(str) + padding;
case tokenizers.GPT3:
return gpt3.encode(str).bpe.length + padding;
case tokenizers.CLASSIC:
@@ -291,8 +300,16 @@ function getTokenCacheObject() {
return tokenCache[String(chatId)];
}
+/**
+ * Counts token using the remote server API.
+ * @param {string} endpoint API endpoint.
+ * @param {string} str String to tokenize.
+ * @param {number} padding Number of padding tokens.
+ * @returns {number} Token count with padding.
+ */
function countTokensRemote(endpoint, str, padding) {
let tokenCount = 0;
+
jQuery.ajax({
async: false,
type: 'POST',
@@ -301,9 +318,25 @@ function countTokensRemote(endpoint, str, padding) {
dataType: "json",
contentType: "application/json",
success: function (data) {
- tokenCount = data.count;
+ if (typeof data.count === 'number') {
+ tokenCount = data.count;
+ } else {
+ tokenCount = guesstimate(str);
+ console.error("Error counting tokens");
+
+ if (!sessionStorage.getItem('tokenizationWarningShown')) {
+ toastr.warning(
+ "Your selected API doesn't support the tokenization endpoint. Using estimated counts.",
+ "Error counting tokens",
+ { timeOut: 10000, preventDuplicates: true },
+ );
+
+ sessionStorage.setItem('tokenizationWarningShown', String(true));
+ }
+ }
}
});
+
return tokenCount + padding;
}
diff --git a/server.js b/server.js
index b46880ccb..50e4dcfb3 100644
--- a/server.js
+++ b/server.js
@@ -3842,11 +3842,19 @@ app.post("/tokenize_via_api", jsonParser, async function (request, response) {
if (main_api == 'textgenerationwebui' && request.body.use_mancer) {
args.headers = Object.assign(args.headers, get_mancer_headers());
+ const data = await postAsync(api_server + "/v1/token-count", args);
+ return response.send({ count: data['results'][0]['tokens'] });
}
- const data = await postAsync(api_server + "/v1/token-count", args);
- console.log(data);
- return response.send({ count: data['results'][0]['tokens'] });
+ else if (main_api == 'kobold') {
+ const data = await postAsync(api_server + "/extra/tokencount", args);
+ const count = data['value'];
+ return response.send({ count: count });
+ }
+
+ else {
+ return response.send({ error: true });
+ }
} catch (error) {
console.log(error);
return response.send({ error: true });
From c91ab3b5e0d05f21963f42dd9fac829985346afc Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Thu, 24 Aug 2023 21:23:35 +0300
Subject: [PATCH 4/4] Add Kobold tokenization to best match logic. Fix not
being able to stop group chat regeneration
---
public/script.js | 6 ++++++
public/scripts/group-chats.js | 5 ++++-
public/scripts/kai-settings.js | 34 ++++++++++++++++++--------------
public/scripts/preset-manager.js | 2 ++
public/scripts/tokenizers.js | 16 ++++++++++++---
5 files changed, 44 insertions(+), 19 deletions(-)
diff --git a/public/script.js b/public/script.js
index 0aea4a0c5..44972edb2 100644
--- a/public/script.js
+++ b/public/script.js
@@ -8,6 +8,7 @@ import {
getKoboldGenerationData,
canUseKoboldStopSequence,
canUseKoboldStreaming,
+ canUseKoboldTokenization,
} from "./scripts/kai-settings.js";
import {
@@ -783,6 +784,7 @@ async function getStatus() {
if (main_api === "kobold" || main_api === "koboldhorde") {
kai_settings.use_stop_sequence = canUseKoboldStopSequence(data.version);
kai_settings.can_use_streaming = canUseKoboldStreaming(data.koboldVersion);
+ kai_settings.can_use_tokenization = canUseKoboldTokenization(data.koboldVersion);
}
// We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress.
@@ -4007,6 +4009,10 @@ export function setMenuType(value) {
menu_type = value;
}
+export function setExternalAbortController(controller) {
+ abortController = controller;
+}
+
function setCharacterId(value) {
this_chid = value;
}
diff --git a/public/scripts/group-chats.js b/public/scripts/group-chats.js
index db8753b35..86ad3eead 100644
--- a/public/scripts/group-chats.js
+++ b/public/scripts/group-chats.js
@@ -65,6 +65,7 @@ import {
getCropPopup,
system_avatar,
isChatSaving,
+ setExternalAbortController,
} from "../script.js";
import { appendTagToList, createTagMapFromList, getTagsList, applyTagsOnCharacterSelect, tag_map, printTagFilters } from './tags.js';
import { FILTER_TYPES, FilterHelper } from './filters.js';
@@ -135,7 +136,9 @@ async function regenerateGroup() {
await deleteLastMessage();
}
- generateGroupWrapper();
+ const abortController = new AbortController();
+ setExternalAbortController(abortController);
+ generateGroupWrapper(false, 'normal', { signal: abortController.signal });
}
async function loadGroupChat(chatId) {
diff --git a/public/scripts/kai-settings.js b/public/scripts/kai-settings.js
index 9dbb5ec21..726900a8a 100644
--- a/public/scripts/kai-settings.js
+++ b/public/scripts/kai-settings.js
@@ -9,16 +9,7 @@ import {
} from "./power-user.js";
import { getSortableDelay } from "./utils.js";
-export {
- kai_settings,
- loadKoboldSettings,
- formatKoboldUrl,
- getKoboldGenerationData,
- canUseKoboldStopSequence,
- canUseKoboldStreaming,
-};
-
-const kai_settings = {
+export const kai_settings = {
temp: 1,
rep_pen: 1,
rep_pen_range: 0,
@@ -30,15 +21,17 @@ const kai_settings = {
rep_pen_slope: 0.9,
single_line: false,
use_stop_sequence: false,
+ can_use_tokenization: false,
streaming_kobold: false,
sampler_order: [0, 1, 2, 3, 4, 5, 6],
};
const MIN_STOP_SEQUENCE_VERSION = '1.2.2';
const MIN_STREAMING_KCPPVERSION = '1.30';
+const MIN_TOKENIZATION_KCPPVERSION = '1.41';
const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];
-function formatKoboldUrl(value) {
+export function formatKoboldUrl(value) {
try {
const url = new URL(value);
if (!power_user.relaxed_api_urls) {
@@ -49,7 +42,7 @@ function formatKoboldUrl(value) {
return null;
}
-function loadKoboldSettings(preset) {
+export function loadKoboldSettings(preset) {
for (const name of Object.keys(kai_settings)) {
const value = preset[name];
const slider = sliders.find(x => x.name === name);
@@ -75,7 +68,7 @@ function loadKoboldSettings(preset) {
}
}
-function getKoboldGenerationData(finalPrompt, this_settings, this_amount_gen, this_max_context, isImpersonate, type) {
+export function getKoboldGenerationData(finalPrompt, this_settings, this_amount_gen, this_max_context, isImpersonate, type) {
const sampler_order = kai_settings.sampler_order || this_settings.sampler_order;
let generate_data = {
prompt: finalPrompt,
@@ -228,7 +221,7 @@ const sliders = [
* @param {string} version KoboldAI version to check.
* @returns {boolean} True if the Kobold stop sequence can be used, false otherwise.
*/
-function canUseKoboldStopSequence(version) {
+export function canUseKoboldStopSequence(version) {
return (version || '0.0.0').localeCompare(MIN_STOP_SEQUENCE_VERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
}
@@ -237,12 +230,23 @@ function canUseKoboldStopSequence(version) {
* @param {{ result: string; version: string; }} koboldVersion KoboldAI version object.
* @returns {boolean} True if the Kobold streaming API can be used, false otherwise.
*/
-function canUseKoboldStreaming(koboldVersion) {
+export function canUseKoboldStreaming(koboldVersion) {
if (koboldVersion && koboldVersion.result == 'KoboldCpp') {
return (koboldVersion.version || '0.0').localeCompare(MIN_STREAMING_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
} else return false;
}
+/**
+ * Determines if the Kobold tokenization API can be used with the given version.
+ * @param {{ result: string; version: string; }} koboldVersion KoboldAI version object.
+ * @returns {boolean} True if the Kobold tokenization API can be used, false otherwise.
+ */
+export function canUseKoboldTokenization(koboldVersion) {
+ if (koboldVersion && koboldVersion.result == 'KoboldCpp') {
+ return (koboldVersion.version || '0.0').localeCompare(MIN_TOKENIZATION_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
+ } else return false;
+}
+
/**
* Sorts the sampler items by the given order.
* @param {any[]} orderArray Sampler order array.
diff --git a/public/scripts/preset-manager.js b/public/scripts/preset-manager.js
index 810ee070f..e90602d71 100644
--- a/public/scripts/preset-manager.js
+++ b/public/scripts/preset-manager.js
@@ -246,6 +246,8 @@ class PresetManager {
'streaming_url',
'stopping_strings',
'use_stop_sequence',
+ 'can_use_tokenization',
+ 'can_use_streaming',
'preset_settings_novel',
'streaming_novel',
'nai_preamble',
diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js
index 4900e30c8..0b5a34668 100644
--- a/public/scripts/tokenizers.js
+++ b/public/scripts/tokenizers.js
@@ -1,12 +1,14 @@
-import { characters, main_api, nai_settings, this_chid } from "../script.js";
+import { characters, main_api, nai_settings, online_status, this_chid } from "../script.js";
import { power_user } from "./power-user.js";
import { encode } from "../lib/gpt-2-3-tokenizer/mod.js";
import { GPT3BrowserTokenizer } from "../lib/gpt-3-tokenizer/gpt3-tokenizer.js";
import { chat_completion_sources, oai_settings } from "./openai.js";
import { groups, selected_group } from "./group-chats.js";
import { getStringHash } from "./utils.js";
+import { kai_settings } from "./kai-settings.js";
export const CHARACTERS_PER_TOKEN_RATIO = 3.35;
+const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown';
export const tokenizers = {
NONE: 0,
@@ -77,6 +79,14 @@ function getTokenizerBestMatch() {
}
}
if (main_api === 'kobold' || main_api === 'textgenerationwebui' || main_api === 'koboldhorde') {
+ // Try to use the API tokenizer if possible:
+ // - API must be connected
+ // - Kobold must pass a version check
+ // - Tokenizer haven't reported an error previously
+ if (kai_settings.can_use_tokenization && !sessionStorage.getItem(TOKENIZER_WARNING_KEY) && online_status !== 'no_connection') {
+ return tokenizers.API;
+ }
+
return tokenizers.LLAMA;
}
@@ -324,14 +334,14 @@ function countTokensRemote(endpoint, str, padding) {
tokenCount = guesstimate(str);
console.error("Error counting tokens");
- if (!sessionStorage.getItem('tokenizationWarningShown')) {
+ if (!sessionStorage.getItem(TOKENIZER_WARNING_KEY)) {
toastr.warning(
"Your selected API doesn't support the tokenization endpoint. Using estimated counts.",
"Error counting tokens",
{ timeOut: 10000, preventDuplicates: true },
);
- sessionStorage.setItem('tokenizationWarningShown', String(true));
+ sessionStorage.setItem(TOKENIZER_WARNING_KEY, String(true));
}
}
}