rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'SillyTavern:staging' into staging

This commit is contained in:
Tony Ribeiro 2023-08-24 21:05:12 +02:00 committed by GitHub
parent cd24de3c2e c91ab3b5e0
commit 3b6978010f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 106 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
public/index.html
View File

@ -2245,7 +2245,7 @@
<option value="3">Sentencepiece (LLaMA)</option>
<option value="4">NerdStash (NovelAI Clio)</option>
<option value="5">NerdStash v2 (NovelAI Kayra)</option>
<option value="6">API (WebUI)</option>
<option value="6">API (WebUI / koboldcpp)</option>
</select>
</div>
<div class="range-block">
@ -2592,7 +2592,7 @@
</div>
<div class="range-block-range-and-counter">
<div class="range-block-range">
<input type="range" id="font_scale" name="font_scale" min="0.8" max="1.2" step="0.05">
<input type="range" id="font_scale" name="font_scale" min="0.75" max="1.25" step="0.01">
</div>
<div class="range-block-counter">
<div contenteditable="true" data-for="font_scale" id="font_scale_counter">

6
public/script.js
View File

@ -8,6 +8,7 @@ import {
getKoboldGenerationData,
canUseKoboldStopSequence,
canUseKoboldStreaming,
canUseKoboldTokenization,
} from "./scripts/kai-settings.js";
import {
@ -783,6 +784,7 @@ async function getStatus() {
if (main_api === "kobold" || main_api === "koboldhorde") {
kai_settings.use_stop_sequence = canUseKoboldStopSequence(data.version);
kai_settings.can_use_streaming = canUseKoboldStreaming(data.koboldVersion);
kai_settings.can_use_tokenization = canUseKoboldTokenization(data.koboldVersion);
}
// We didn't get a 200 status code, but the endpoint has an explanation. Which means it DID connect, but I digress.
@ -4007,6 +4009,10 @@ export function setMenuType(value) {
menu_type = value;
}
export function setExternalAbortController(controller) {
abortController = controller;
}
function setCharacterId(value) {
this_chid = value;
}

2
public/scripts/RossAscends-mods.js
View File

@ -843,7 +843,7 @@ jQuery(async function () {
//this makes the chat input text area resize vertically to match the text size (limited by CSS at 50% window height)
$('#send_textarea').on('input', function () {
this.style.height = '30px';
this.style.height = window.getComputedStyle(this).getPropertyValue('min-height');
this.style.height = (this.scrollHeight) + 'px';
});

7
public/scripts/group-chats.js
View File

@ -8,6 +8,7 @@ import {
extractAllWords,
saveBase64AsFile,
PAGINATION_TEMPLATE,
waitUntilCondition,
} from './utils.js';
import { RA_CountCharTokens, humanizedDateTime, dragElement, favsToHotswap, getMessageTimeStamp } from "./RossAscends-mods.js";
import { loadMovingUIState, sortEntitiesList } from './power-user.js';
@ -64,6 +65,7 @@ import {
getCropPopup,
system_avatar,
isChatSaving,
setExternalAbortController,
} from "../script.js";
import { appendTagToList, createTagMapFromList, getTagsList, applyTagsOnCharacterSelect, tag_map, printTagFilters } from './tags.js';
import { FILTER_TYPES, FilterHelper } from './filters.js';
@ -134,7 +136,9 @@ async function regenerateGroup() {
await deleteLastMessage();
}
generateGroupWrapper();
const abortController = new AbortController();
setExternalAbortController(abortController);
generateGroupWrapper(false, 'normal', { signal: abortController.signal });
}
async function loadGroupChat(chatId) {
@ -665,6 +669,7 @@ async function generateGroupWrapper(by_auto_mode, type = null, params = {}) {
if (streamingProcessor && !streamingProcessor.isFinished) {
await delay(100);
} else {
await waitUntilCondition(() => streamingProcessor == null, 1000, 10);
messagesBefore++;
break;
}

34
public/scripts/kai-settings.js
View File

@ -9,16 +9,7 @@ import {
} from "./power-user.js";
import { getSortableDelay } from "./utils.js";
export {
kai_settings,
loadKoboldSettings,
formatKoboldUrl,
getKoboldGenerationData,
canUseKoboldStopSequence,
canUseKoboldStreaming,
};
const kai_settings = {
export const kai_settings = {
temp: 1,
rep_pen: 1,
rep_pen_range: 0,
@ -30,15 +21,17 @@ const kai_settings = {
rep_pen_slope: 0.9,
single_line: false,
use_stop_sequence: false,
can_use_tokenization: false,
streaming_kobold: false,
sampler_order: [0, 1, 2, 3, 4, 5, 6],
};
const MIN_STOP_SEQUENCE_VERSION = '1.2.2';
const MIN_STREAMING_KCPPVERSION = '1.30';
const MIN_TOKENIZATION_KCPPVERSION = '1.41';
const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];
function formatKoboldUrl(value) {
export function formatKoboldUrl(value) {
try {
const url = new URL(value);
if (!power_user.relaxed_api_urls) {
@ -49,7 +42,7 @@ function formatKoboldUrl(value) {
return null;
}
function loadKoboldSettings(preset) {
export function loadKoboldSettings(preset) {
for (const name of Object.keys(kai_settings)) {
const value = preset[name];
const slider = sliders.find(x => x.name === name);
@ -75,7 +68,7 @@ function loadKoboldSettings(preset) {
}
}
function getKoboldGenerationData(finalPrompt, this_settings, this_amount_gen, this_max_context, isImpersonate, type) {
export function getKoboldGenerationData(finalPrompt, this_settings, this_amount_gen, this_max_context, isImpersonate, type) {
const sampler_order = kai_settings.sampler_order || this_settings.sampler_order;
let generate_data = {
prompt: finalPrompt,
@ -228,7 +221,7 @@ const sliders = [
* @param {string} version KoboldAI version to check.
* @returns {boolean} True if the Kobold stop sequence can be used, false otherwise.
*/
function canUseKoboldStopSequence(version) {
export function canUseKoboldStopSequence(version) {
return (version || '0.0.0').localeCompare(MIN_STOP_SEQUENCE_VERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
}
@ -237,12 +230,23 @@ function canUseKoboldStopSequence(version) {
* @param {{ result: string; version: string; }} koboldVersion KoboldAI version object.
* @returns {boolean} True if the Kobold streaming API can be used, false otherwise.
*/
function canUseKoboldStreaming(koboldVersion) {
export function canUseKoboldStreaming(koboldVersion) {
if (koboldVersion && koboldVersion.result == 'KoboldCpp') {
return (koboldVersion.version || '0.0').localeCompare(MIN_STREAMING_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
} else return false;
}
/**
* Determines if the Kobold tokenization API can be used with the given version.
* @param {{ result: string; version: string; }} koboldVersion KoboldAI version object.
* @returns {boolean} True if the Kobold tokenization API can be used, false otherwise.
*/
export function canUseKoboldTokenization(koboldVersion) {
if (koboldVersion && koboldVersion.result == 'KoboldCpp') {
return (koboldVersion.version || '0.0').localeCompare(MIN_TOKENIZATION_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
} else return false;
}
/**
* Sorts the sampler items by the given order.
* @param {any[]} orderArray Sampler order array.

2
public/scripts/preset-manager.js
View File

@ -246,6 +246,8 @@ class PresetManager {
'streaming_url',
'stopping_strings',
'use_stop_sequence',
'can_use_tokenization',
'can_use_streaming',
'preset_settings_novel',
'streaming_novel',
'nai_preamble',

49
public/scripts/tokenizers.js
View File

@ -1,12 +1,14 @@
import { characters, main_api, nai_settings, this_chid } from "../script.js";
import { characters, main_api, nai_settings, online_status, this_chid } from "../script.js";
import { power_user } from "./power-user.js";
import { encode } from "../lib/gpt-2-3-tokenizer/mod.js";
import { GPT3BrowserTokenizer } from "../lib/gpt-3-tokenizer/gpt3-tokenizer.js";
import { chat_completion_sources, oai_settings } from "./openai.js";
import { groups, selected_group } from "./group-chats.js";
import { getStringHash } from "./utils.js";
import { kai_settings } from "./kai-settings.js";
export const CHARACTERS_PER_TOKEN_RATIO = 3.35;
const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown';
export const tokenizers = {
NONE: 0,
@ -24,6 +26,15 @@ const gpt3 = new GPT3BrowserTokenizer({ type: 'gpt3' });
let tokenCache = {};
/**
* Guesstimates the token count for a string.
* @param {string} str String to tokenize.
* @returns {number} Token count.
*/
export function guesstimate(str) {
return Math.ceil(str.length / CHARACTERS_PER_TOKEN_RATIO);
}
async function loadTokenCache() {
try {
console.debug('Chat Completions: loading token cache')
@ -68,6 +79,14 @@ function getTokenizerBestMatch() {
}
}
if (main_api === 'kobold' || main_api === 'textgenerationwebui' || main_api === 'koboldhorde') {
// Try to use the API tokenizer if possible:
// - API must be connected
// - Kobold must pass a version check
// - Tokenizer haven't reported an error previously
if (kai_settings.can_use_tokenization && !sessionStorage.getItem(TOKENIZER_WARNING_KEY) && online_status !== 'no_connection') {
return tokenizers.API;
}
return tokenizers.LLAMA;
}
@ -89,7 +108,7 @@ export function getTokenCount(str, padding = undefined) {
function calculate(type) {
switch (type) {
case tokenizers.NONE:
return Math.ceil(str.length / CHARACTERS_PER_TOKEN_RATIO) + padding;
return guesstimate(str) + padding;
case tokenizers.GPT3:
return gpt3.encode(str).bpe.length + padding;
case tokenizers.CLASSIC:
@ -291,8 +310,16 @@ function getTokenCacheObject() {
return tokenCache[String(chatId)];
}
/**
* Counts token using the remote server API.
* @param {string} endpoint API endpoint.
* @param {string} str String to tokenize.
* @param {number} padding Number of padding tokens.
* @returns {number} Token count with padding.
*/
function countTokensRemote(endpoint, str, padding) {
let tokenCount = 0;
jQuery.ajax({
async: false,
type: 'POST',
@ -301,9 +328,25 @@ function countTokensRemote(endpoint, str, padding) {
dataType: "json",
contentType: "application/json",
success: function (data) {
tokenCount = data.count;
if (typeof data.count === 'number') {
tokenCount = data.count;
} else {
tokenCount = guesstimate(str);
console.error("Error counting tokens");
if (!sessionStorage.getItem(TOKENIZER_WARNING_KEY)) {
toastr.warning(
"Your selected API doesn't support the tokenization endpoint. Using estimated counts.",
"Error counting tokens",
{ timeOut: 10000, preventDuplicates: true },
);
sessionStorage.setItem(TOKENIZER_WARNING_KEY, String(true));
}
}
}
});
return tokenCount + padding;
}

22
public/style.css
View File

@ -67,6 +67,10 @@
color-scheme: only light;
/* Send form variables */
--bottomFormBlockPadding: calc(var(--mainFontSize) / 3);
--bottomFormIconSize: calc(var(--mainFontSize) * 2);
--bottomFormBlockSize: calc(var(--bottomFormIconSize) + var(--bottomFormBlockPadding));
/*styles for the color picker*/
--tool-cool-color-picker-btn-bg: transparent;
@ -513,19 +517,19 @@ hr {
#send_but_sheld {
padding: 0;
border: 0;
height: 30px;
height: var(--bottomFormBlockSize);
position: relative;
background-position: center;
display: flex;
flex-direction: row;
column-gap: 5px;
font-size: 25px;
font-size: var(--bottomFormIconSize);
overflow: hidden;
}
#send_but_sheld>div {
width: 30px;
height: 30px;
width: var(--bottomFormBlockSize);
height: var(--bottomFormBlockSize);
margin: 0;
outline: none;
border: none;
@ -556,8 +560,9 @@ hr {
}
#options_button {
width: 30px;
height: 30px;
width: var(--bottomFormBlockSize);
height: var(--bottomFormBlockSize);
font-size: var(--bottomFormIconSize);
margin: 0;
outline: none;
border: none;
@ -568,7 +573,6 @@ hr {
margin-left: 10px;
padding: 0;
transition: 0.3s;
font-size: 25px;
display: flex;
align-items: center;
}
@ -879,11 +883,11 @@ select {
}
#send_textarea {
min-height: 30px;
min-height: var(--bottomFormBlockSize);
height: var(--bottomFormBlockSize);
max-height: 50vh;
max-height: 50svh;
word-wrap: break-word;
height: 30px;
resize: vertical;
display: block;
background-color: rgba(255, 0, 0, 0);

14
server.js
View File

@ -3843,11 +3843,19 @@ app.post("/tokenize_via_api", jsonParser, async function (request, response) {
if (main_api == 'textgenerationwebui' && request.body.use_mancer) {
args.headers = Object.assign(args.headers, get_mancer_headers());
const data = await postAsync(api_server + "/v1/token-count", args);
return response.send({ count: data['results'][0]['tokens'] });
}
const data = await postAsync(api_server + "/v1/token-count", args);
console.log(data);
return response.send({ count: data['results'][0]['tokens'] });
else if (main_api == 'kobold') {
const data = await postAsync(api_server + "/extra/tokencount", args);
const count = data['value'];
return response.send({ count: count });
}
else {
return response.send({ error: true });
}
} catch (error) {
console.log(error);
return response.send({ error: true });