rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Cache stopping strings rather than skipping them

Browse Source
This commit is contained in:
valadaptive
2023-12-12 16:32:54 -05:00
parent 3d7706e6b3
commit 87cbe361fc
1 changed files with 23 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
public/script.js
View File

@ -2629,12 +2629,12 @@ class StreamingProcessor {
if (!isImpersonate && !isContinue && Array.isArray(this.swipes) && this.swipes.length > 0) { if (!isImpersonate && !isContinue && Array.isArray(this.swipes) && this.swipes.length > 0) {
for (let i = 0; i < this.swipes.length; i++) { for (let i = 0; i < this.swipes.length; i++) {
this.swipes[i] = cleanUpMessage(this.removePrefix(this.swipes[i]), false, false, true, !isFinal); this.swipes[i] = cleanUpMessage(this.removePrefix(this.swipes[i]), false, false, true, this.stoppingStrings);
} }
} }
text = this.removePrefix(text); text = this.removePrefix(text);
let processedText = cleanUpMessage(text, isImpersonate, isContinue, !isFinal, !isFinal); let processedText = cleanUpMessage(text, isImpersonate, isContinue, !isFinal, this.stoppingStrings);
// Predict unbalanced asterisks / quotes during streaming // Predict unbalanced asterisks / quotes during streaming
const charsToBalance = ['*', '"', '```']; const charsToBalance = ['*', '"', '```'];
@ -2805,6 +2805,10 @@ class StreamingProcessor {
scrollLock = false; scrollLock = false;
} }
const isImpersonate = this.type == 'impersonate';
const isContinue = this.type == 'continue';
this.stoppingStrings = getStoppingStrings(isImpersonate, isContinue);
try { try {
const sw = new Stopwatch(1000 / power_user.streaming_fps); const sw = new Stopwatch(1000 / power_user.streaming_fps);
const timestamps = []; const timestamps = [];
@ -2907,7 +2911,7 @@ export async function generateRaw(prompt, api, instructOverride) {
throw new Error(data.error); throw new Error(data.error);
} }
const message = cleanUpMessage(extractMessageFromData(data), false, false, true, false); const message = cleanUpMessage(extractMessageFromData(data), false, false, true);
if (!message) { if (!message) {
throw new Error('No message generated'); throw new Error('No message generated');
@ -3814,7 +3818,7 @@ async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, qu
streamingProcessor.generator = streamingGenerator; streamingProcessor.generator = streamingGenerator;
hideSwipeButtons(); hideSwipeButtons();
let getMessage = await streamingProcessor.generate(); let getMessage = await streamingProcessor.generate();
let messageChunk = cleanUpMessage(getMessage, isImpersonate, isContinue, false, false); let messageChunk = cleanUpMessage(getMessage, isImpersonate, isContinue, false);
if (isContinue) { if (isContinue) {
getMessage = continue_mag + getMessage; getMessage = continue_mag + getMessage;
@ -3849,7 +3853,7 @@ async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, qu
const swipes = extractMultiSwipes(data, type); const swipes = extractMultiSwipes(data, type);
messageChunk = cleanUpMessage(getMessage, isImpersonate, isContinue, false, false); messageChunk = cleanUpMessage(getMessage, isImpersonate, isContinue, false);
if (isContinue) { if (isContinue) {
getMessage = continue_mag + getMessage; getMessage = continue_mag + getMessage;
@ -3857,7 +3861,7 @@ async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, qu
//Formating //Formating
const displayIncomplete = type === 'quiet' && !quietToLoud; const displayIncomplete = type === 'quiet' && !quietToLoud;
getMessage = cleanUpMessage(getMessage, isImpersonate, isContinue, displayIncomplete, false); getMessage = cleanUpMessage(getMessage, isImpersonate, isContinue, displayIncomplete);
if (getMessage.length > 0) { if (getMessage.length > 0) {
if (isImpersonate) { if (isImpersonate) {
@ -4487,7 +4491,7 @@ function extractMultiSwipes(data, type) {
} }
for (let i = 1; i < data.choices.length; i++) { for (let i = 1; i < data.choices.length; i++) {
const text = cleanUpMessage(data.choices[i].text, false, false, false, false); const text = cleanUpMessage(data.choices[i].text, false, false, false);
swipes.push(text); swipes.push(text);
} }
} }
@ -4495,7 +4499,7 @@ function extractMultiSwipes(data, type) {
return swipes; return swipes;
} }
function cleanUpMessage(getMessage, isImpersonate, isContinue, displayIncompleteSentences = false, skipStopStringCleanup = false) { function cleanUpMessage(getMessage, isImpersonate, isContinue, displayIncompleteSentences = false, stoppingStrings = null) {
if (!getMessage) { if (!getMessage) {
return ''; return '';
} }
@ -4510,8 +4514,11 @@ function cleanUpMessage(getMessage, isImpersonate, isContinue, displayIncomplete
getMessage = substituteParams(power_user.user_prompt_bias) + getMessage; getMessage = substituteParams(power_user.user_prompt_bias) + getMessage;
} }
if (!skipStopStringCleanup) { // Allow for caching of stopping strings. getStoppingStrings is an expensive function, especially with macros
const stoppingStrings = getStoppingStrings(isImpersonate, isContinue); // enabled, so for streaming, we call it once and then pass it into each cleanUpMessage call.
if (!stoppingStrings) {
stoppingStrings = getStoppingStrings(isImpersonate, isContinue);
}
for (const stoppingString of stoppingStrings) { for (const stoppingString of stoppingStrings) {
if (stoppingString.length) { if (stoppingString.length) {
@ -4523,7 +4530,6 @@ function cleanUpMessage(getMessage, isImpersonate, isContinue, displayIncomplete
} }
} }
} }
}
// Regex uses vars, so add before formatting // Regex uses vars, so add before formatting
getMessage = getRegexedString(getMessage, isImpersonate ? regex_placement.USER_INPUT : regex_placement.AI_OUTPUT); getMessage = getRegexedString(getMessage, isImpersonate ? regex_placement.USER_INPUT : regex_placement.AI_OUTPUT);