From c5f251c6e376701a273876affaa7a87910420453 Mon Sep 17 00:00:00 2001 From: bmen25124 Date: Wed, 26 Mar 2025 22:35:10 +0300 Subject: [PATCH 1/5] Added stop string cleanup, better stopping string param --- public/scripts/custom-request.js | 72 ++++++++++++++++++++++++++++---- public/scripts/instruct-mode.js | 28 +++++++------ 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/public/scripts/custom-request.js b/public/scripts/custom-request.js index 250e74f11..d6281a57a 100644 --- a/public/scripts/custom-request.js +++ b/public/scripts/custom-request.js @@ -2,7 +2,7 @@ import { getPresetManager } from './preset-manager.js'; import { extractMessageFromData, getGenerateUrl, getRequestHeaders } from '../script.js'; import { getTextGenServer } from './textgen-settings.js'; import { extractReasoningFromData } from './reasoning.js'; -import { formatInstructModeChat, formatInstructModePrompt, names_behavior_types } from './instruct-mode.js'; +import { formatInstructModeChat, formatInstructModePrompt, getInstructStoppingSequences, names_behavior_types } from './instruct-mode.js'; import { getStreamingReply, tryParseStreamingError } from './openai.js'; import EventSourceStream from './sse-stream.js'; @@ -222,10 +222,13 @@ export class TextCompletionService { } } + + /** @type {InstructSettings | undefined} */ + let instructPreset; // Handle instruct formatting if requested if (Array.isArray(prompt) && instructName) { const instructPresetManager = getPresetManager('instruct'); - let instructPreset = instructPresetManager?.getCompletionPresetByName(instructName); + instructPreset = instructPresetManager?.getCompletionPresetByName(instructName); if (instructPreset) { // Clone the preset to avoid modifying the original instructPreset = structuredClone(instructPreset); @@ -266,10 +269,9 @@ export class TextCompletionService { formattedMessages.push(messageContent); } requestData.prompt = formattedMessages.join(''); - if (instructPreset.output_suffix) { - requestData.stop = [instructPreset.output_suffix]; - requestData.stopping_strings = [instructPreset.output_suffix]; - } + const stoppingStrings = getInstructStoppingSequences({ customInstruct: instructPreset, useStopString: false }); + requestData.stop = stoppingStrings + requestData.stopping_strings = stoppingStrings; } else { console.warn(`Instruct preset "${instructName}" not found, using basic formatting`); requestData.prompt = prompt.map(x => x.content).join('\n\n'); @@ -283,7 +285,63 @@ export class TextCompletionService { // @ts-ignore const data = this.createRequestData(requestData); - return await this.sendRequest(data, extractData, signal); + const response = await this.sendRequest(data, extractData, signal); + // Remove stopping strings from the end + if (!data.stream && extractData) { + /** @type {ExtractedData} */ + // @ts-ignore + const extractedData = response; + + let message = extractedData.content; + + message = message.replace(/[^\S\r\n]+$/gm, ''); + + if (requestData.stopping_strings) { + for (const stoppingString of requestData.stopping_strings) { + if (stoppingString.length) { + for (let j = stoppingString.length; j > 0; j--) { + if (message.slice(-j) === stoppingString.slice(0, j)) { + message = message.slice(0, -j); + break; + } + } + } + } + } + + if (instructPreset) { + if (instructPreset.stop_sequence) { + const index = message.indexOf(instructPreset.stop_sequence); + if (index != -1) { + message = message.substring(0, index); + } + } + if (instructPreset.input_sequence && instructPreset.input_sequence.trim()) { + const index = message.indexOf(instructPreset.input_sequence); + if (index != -1) { + message = message.substring(0, index); + } + } + if (instructPreset.output_sequence) { + instructPreset.output_sequence.split('\n') + .filter(line => line.trim() !== '') + .forEach(line => { + message = message.replaceAll(line, ''); + }); + } + if (instructPreset.last_output_sequence) { + instructPreset.last_output_sequence.split('\n') + .filter(line => line.trim() !== '') + .forEach(line => { + message = message.replaceAll(line, ''); + }); + } + } + + extractedData.content = message; + } + + return response; } /** diff --git a/public/scripts/instruct-mode.js b/public/scripts/instruct-mode.js index 000d242b0..f5b93dd45 100644 --- a/public/scripts/instruct-mode.js +++ b/public/scripts/instruct-mode.js @@ -243,9 +243,12 @@ export function autoSelectInstructPreset(modelId) { /** * Converts instruct mode sequences to an array of stopping strings. + * @param {{customInstruct?: InstructSettings, useStopString?: boolean}} options * @returns {string[]} Array of instruct mode stopping strings. */ -export function getInstructStoppingSequences() { +export function getInstructStoppingSequences({ customInstruct = null, useStopString = false } = {}) { + const instruct = structuredClone(customInstruct ?? power_user.instruct); + /** * Adds instruct mode sequence to the result array. * @param {string} sequence Sequence string. @@ -254,7 +257,7 @@ export function getInstructStoppingSequences() { function addInstructSequence(sequence) { // Cohee: oobabooga's textgen always appends newline before the sequence as a stopping string // But it's a problem for Metharme which doesn't use newlines to separate them. - const wrap = (s) => power_user.instruct.wrap ? '\n' + s : s; + const wrap = (s) => instruct.wrap ? '\n' + s : s; // Sequence must be a non-empty string if (typeof sequence === 'string' && sequence.length > 0) { // If sequence is just a whitespace or newline - we don't want to make it a stopping string @@ -262,7 +265,7 @@ export function getInstructStoppingSequences() { if (sequence.trim().length > 0) { const wrappedSequence = wrap(sequence); // Need to respect "insert macro" setting - const stopString = power_user.instruct.macro ? substituteParams(wrappedSequence) : wrappedSequence; + const stopString = instruct.macro ? substituteParams(wrappedSequence) : wrappedSequence; result.push(stopString); } } @@ -270,14 +273,15 @@ export function getInstructStoppingSequences() { const result = []; - if (power_user.instruct.enabled) { - const stop_sequence = power_user.instruct.stop_sequence || ''; - const input_sequence = power_user.instruct.input_sequence?.replace(/{{name}}/gi, name1) || ''; - const output_sequence = power_user.instruct.output_sequence?.replace(/{{name}}/gi, name2) || ''; - const first_output_sequence = power_user.instruct.first_output_sequence?.replace(/{{name}}/gi, name2) || ''; - const last_output_sequence = power_user.instruct.last_output_sequence?.replace(/{{name}}/gi, name2) || ''; - const system_sequence = power_user.instruct.system_sequence?.replace(/{{name}}/gi, 'System') || ''; - const last_system_sequence = power_user.instruct.last_system_sequence?.replace(/{{name}}/gi, 'System') || ''; + // Since preset's don't have "enabled", we assume it's always enabled + if (customInstruct ?? instruct.enabled) { + const stop_sequence = instruct.stop_sequence || ''; + const input_sequence = instruct.input_sequence?.replace(/{{name}}/gi, name1) || ''; + const output_sequence = instruct.output_sequence?.replace(/{{name}}/gi, name2) || ''; + const first_output_sequence = instruct.first_output_sequence?.replace(/{{name}}/gi, name2) || ''; + const last_output_sequence = instruct.last_output_sequence?.replace(/{{name}}/gi, name2) || ''; + const system_sequence = instruct.system_sequence?.replace(/{{name}}/gi, 'System') || ''; + const last_system_sequence = instruct.last_system_sequence?.replace(/{{name}}/gi, 'System') || ''; const combined_sequence = [ stop_sequence, @@ -292,7 +296,7 @@ export function getInstructStoppingSequences() { combined_sequence.split('\n').filter((line, index, self) => self.indexOf(line) === index).forEach(addInstructSequence); } - if (power_user.context.use_stop_strings) { + if (useStopString ?? power_user.context.use_stop_strings) { if (power_user.context.chat_start) { result.push(`\n${substituteParams(power_user.context.chat_start)}`); } From a7d48b1aed317625bb5eab5aac8baa471883b374 Mon Sep 17 00:00:00 2001 From: bmen25124 Date: Wed, 26 Mar 2025 23:21:48 +0300 Subject: [PATCH 2/5] Added overridable instruct settings, removed macro override --- public/scripts/custom-request.js | 7 +++++-- public/scripts/extensions/shared.js | 12 ++++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/public/scripts/custom-request.js b/public/scripts/custom-request.js index d6281a57a..353e6cec0 100644 --- a/public/scripts/custom-request.js +++ b/public/scripts/custom-request.js @@ -190,6 +190,7 @@ export class TextCompletionService { * @param {Object} options - Configuration options * @param {string?} [options.presetName] - Name of the preset to use for generation settings * @param {string?} [options.instructName] - Name of instruct preset for message formatting + * @param {Partial?} [options.instructSettings] - Override instruct settings * @param {boolean} extractData - Whether to extract structured data from response * @param {AbortSignal?} [signal] * @returns {Promise AsyncGenerator)>} If not streaming, returns extracted data; if streaming, returns a function that creates an AsyncGenerator @@ -232,8 +233,10 @@ export class TextCompletionService { if (instructPreset) { // Clone the preset to avoid modifying the original instructPreset = structuredClone(instructPreset); - instructPreset.macro = false; instructPreset.names_behavior = names_behavior_types.NONE; + if (options.instructSettings) { + Object.assign(instructPreset, options.instructSettings); + } // Format messages using instruct formatting const formattedMessages = []; @@ -270,7 +273,7 @@ export class TextCompletionService { } requestData.prompt = formattedMessages.join(''); const stoppingStrings = getInstructStoppingSequences({ customInstruct: instructPreset, useStopString: false }); - requestData.stop = stoppingStrings + requestData.stop = stoppingStrings; requestData.stopping_strings = stoppingStrings; } else { console.warn(`Instruct preset "${instructName}" not found, using basic formatting`); diff --git a/public/scripts/extensions/shared.js b/public/scripts/extensions/shared.js index 144ac3d6b..9ca630fa9 100644 --- a/public/scripts/extensions/shared.js +++ b/public/scripts/extensions/shared.js @@ -285,6 +285,7 @@ export class ConnectionManagerRequestService { extractData: true, includePreset: true, includeInstruct: true, + instructSettings: {}, }; static getAllowedTypes() { @@ -298,11 +299,17 @@ export class ConnectionManagerRequestService { * @param {string} profileId * @param {string | (import('../custom-request.js').ChatCompletionMessage & {ignoreInstruct?: boolean})[]} prompt * @param {number} maxTokens - * @param {{stream?: boolean, signal?: AbortSignal, extractData?: boolean, includePreset?: boolean, includeInstruct?: boolean}} custom - default values are true + * @param {object} custom + * @param {boolean?} [custom.stream=false] + * @param {AbortSignal?} [custom.signal] + * @param {boolean?} [custom.extractData=true] + * @param {boolean?} [custom.includePreset=true] + * @param {boolean?} [custom.includeInstruct=true] + * @param {Partial?} [custom.instructSettings] Override instruct settings * @returns {Promise AsyncGenerator)>} If not streaming, returns extracted data; if streaming, returns a function that creates an AsyncGenerator */ static async sendRequest(profileId, prompt, maxTokens, custom = this.defaultSendRequestParams) { - const { stream, signal, extractData, includePreset, includeInstruct } = { ...this.defaultSendRequestParams, ...custom }; + const { stream, signal, extractData, includePreset, includeInstruct, instructSettings } = { ...this.defaultSendRequestParams, ...custom }; const context = SillyTavern.getContext(); if (context.extensionSettings.disabledExtensions.includes('connection-manager')) { @@ -346,6 +353,7 @@ export class ConnectionManagerRequestService { }, { instructName: includeInstruct ? profile.instruct : undefined, presetName: includePreset ? profile.preset : undefined, + instructSettings: includeInstruct ? instructSettings : undefined, }, extractData, signal); } default: { From 972b1e5fa7eb78fa0f5e858774c1889757cf041c Mon Sep 17 00:00:00 2001 From: bmen25124 Date: Wed, 26 Mar 2025 23:30:09 +0300 Subject: [PATCH 3/5] Fixed variable naming, better jsdoc --- public/scripts/custom-request.js | 2 +- public/scripts/extensions/shared.js | 2 +- public/scripts/instruct-mode.js | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/public/scripts/custom-request.js b/public/scripts/custom-request.js index 353e6cec0..7ea8f60eb 100644 --- a/public/scripts/custom-request.js +++ b/public/scripts/custom-request.js @@ -272,7 +272,7 @@ export class TextCompletionService { formattedMessages.push(messageContent); } requestData.prompt = formattedMessages.join(''); - const stoppingStrings = getInstructStoppingSequences({ customInstruct: instructPreset, useStopString: false }); + const stoppingStrings = getInstructStoppingSequences({ customInstruct: instructPreset, useStopStrings: false }); requestData.stop = stoppingStrings; requestData.stopping_strings = stoppingStrings; } else { diff --git a/public/scripts/extensions/shared.js b/public/scripts/extensions/shared.js index 9ca630fa9..e4cca98fa 100644 --- a/public/scripts/extensions/shared.js +++ b/public/scripts/extensions/shared.js @@ -299,7 +299,7 @@ export class ConnectionManagerRequestService { * @param {string} profileId * @param {string | (import('../custom-request.js').ChatCompletionMessage & {ignoreInstruct?: boolean})[]} prompt * @param {number} maxTokens - * @param {object} custom + * @param {Object} custom * @param {boolean?} [custom.stream=false] * @param {AbortSignal?} [custom.signal] * @param {boolean?} [custom.extractData=true] diff --git a/public/scripts/instruct-mode.js b/public/scripts/instruct-mode.js index f5b93dd45..7edfb1e0b 100644 --- a/public/scripts/instruct-mode.js +++ b/public/scripts/instruct-mode.js @@ -243,10 +243,12 @@ export function autoSelectInstructPreset(modelId) { /** * Converts instruct mode sequences to an array of stopping strings. - * @param {{customInstruct?: InstructSettings, useStopString?: boolean}} options + * @param {Object} options + * @param {InstructSettings?} [options.customInstruct=null] - Custom instruct settings. + * @param {boolean?} [options.useStopStrings=false] - Decides whether to use "Chat Start" and "Example Separator" * @returns {string[]} Array of instruct mode stopping strings. */ -export function getInstructStoppingSequences({ customInstruct = null, useStopString = false } = {}) { +export function getInstructStoppingSequences({ customInstruct = null, useStopStrings = false } = {}) { const instruct = structuredClone(customInstruct ?? power_user.instruct); /** @@ -296,7 +298,7 @@ export function getInstructStoppingSequences({ customInstruct = null, useStopStr combined_sequence.split('\n').filter((line, index, self) => self.indexOf(line) === index).forEach(addInstructSequence); } - if (useStopString ?? power_user.context.use_stop_strings) { + if (useStopStrings ?? power_user.context.use_stop_strings) { if (power_user.context.chat_start) { result.push(`\n${substituteParams(power_user.context.chat_start)}`); } From 4e207c2cf0123f84c9438512bef211cb8dd53598 Mon Sep 17 00:00:00 2001 From: bmen25124 Date: Wed, 26 Mar 2025 23:38:02 +0300 Subject: [PATCH 4/5] Removed duplicate codes --- public/scripts/custom-request.js | 46 +++++++++++++++----------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/public/scripts/custom-request.js b/public/scripts/custom-request.js index 7ea8f60eb..5cd3f4827 100644 --- a/public/scripts/custom-request.js +++ b/public/scripts/custom-request.js @@ -313,32 +313,30 @@ export class TextCompletionService { } if (instructPreset) { - if (instructPreset.stop_sequence) { - const index = message.indexOf(instructPreset.stop_sequence); - if (index != -1) { - message = message.substring(0, index); + [ + instructPreset.stop_sequence, + instructPreset.input_sequence, + ].forEach(sequence => { + if (sequence?.trim()) { + const index = message.indexOf(sequence); + if (index !== -1) { + message = message.substring(0, index); + } } - } - if (instructPreset.input_sequence && instructPreset.input_sequence.trim()) { - const index = message.indexOf(instructPreset.input_sequence); - if (index != -1) { - message = message.substring(0, index); + }); + + [ + instructPreset.output_sequence, + instructPreset.last_output_sequence, + ].forEach(sequences => { + if (sequences) { + sequences.split('\n') + .filter(line => line.trim() !== '') + .forEach(line => { + message = message.replaceAll(line, ''); + }); } - } - if (instructPreset.output_sequence) { - instructPreset.output_sequence.split('\n') - .filter(line => line.trim() !== '') - .forEach(line => { - message = message.replaceAll(line, ''); - }); - } - if (instructPreset.last_output_sequence) { - instructPreset.last_output_sequence.split('\n') - .filter(line => line.trim() !== '') - .forEach(line => { - message = message.replaceAll(line, ''); - }); - } + }); } extractedData.content = message; From d93aba5706a084e90dee5b5e26ff85151a00d3d7 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 27 Mar 2025 22:52:01 +0200 Subject: [PATCH 5/5] Fix useStopStrings defaulting --- public/scripts/instruct-mode.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/scripts/instruct-mode.js b/public/scripts/instruct-mode.js index 7edfb1e0b..56b16f560 100644 --- a/public/scripts/instruct-mode.js +++ b/public/scripts/instruct-mode.js @@ -245,10 +245,10 @@ export function autoSelectInstructPreset(modelId) { * Converts instruct mode sequences to an array of stopping strings. * @param {Object} options * @param {InstructSettings?} [options.customInstruct=null] - Custom instruct settings. - * @param {boolean?} [options.useStopStrings=false] - Decides whether to use "Chat Start" and "Example Separator" + * @param {boolean?} [options.useStopStrings] - Decides whether to use "Chat Start" and "Example Separator" * @returns {string[]} Array of instruct mode stopping strings. */ -export function getInstructStoppingSequences({ customInstruct = null, useStopStrings = false } = {}) { +export function getInstructStoppingSequences({ customInstruct = null, useStopStrings = null } = {}) { const instruct = structuredClone(customInstruct ?? power_user.instruct); /**