diff --git a/public/scripts/custom-request.js b/public/scripts/custom-request.js index 250e74f11..5cd3f4827 100644 --- a/public/scripts/custom-request.js +++ b/public/scripts/custom-request.js @@ -2,7 +2,7 @@ import { getPresetManager } from './preset-manager.js'; import { extractMessageFromData, getGenerateUrl, getRequestHeaders } from '../script.js'; import { getTextGenServer } from './textgen-settings.js'; import { extractReasoningFromData } from './reasoning.js'; -import { formatInstructModeChat, formatInstructModePrompt, names_behavior_types } from './instruct-mode.js'; +import { formatInstructModeChat, formatInstructModePrompt, getInstructStoppingSequences, names_behavior_types } from './instruct-mode.js'; import { getStreamingReply, tryParseStreamingError } from './openai.js'; import EventSourceStream from './sse-stream.js'; @@ -190,6 +190,7 @@ export class TextCompletionService { * @param {Object} options - Configuration options * @param {string?} [options.presetName] - Name of the preset to use for generation settings * @param {string?} [options.instructName] - Name of instruct preset for message formatting + * @param {Partial?} [options.instructSettings] - Override instruct settings * @param {boolean} extractData - Whether to extract structured data from response * @param {AbortSignal?} [signal] * @returns {Promise AsyncGenerator)>} If not streaming, returns extracted data; if streaming, returns a function that creates an AsyncGenerator @@ -222,15 +223,20 @@ export class TextCompletionService { } } + + /** @type {InstructSettings | undefined} */ + let instructPreset; // Handle instruct formatting if requested if (Array.isArray(prompt) && instructName) { const instructPresetManager = getPresetManager('instruct'); - let instructPreset = instructPresetManager?.getCompletionPresetByName(instructName); + instructPreset = instructPresetManager?.getCompletionPresetByName(instructName); if (instructPreset) { // Clone the preset to avoid modifying the original instructPreset = structuredClone(instructPreset); - instructPreset.macro = false; instructPreset.names_behavior = names_behavior_types.NONE; + if (options.instructSettings) { + Object.assign(instructPreset, options.instructSettings); + } // Format messages using instruct formatting const formattedMessages = []; @@ -266,10 +272,9 @@ export class TextCompletionService { formattedMessages.push(messageContent); } requestData.prompt = formattedMessages.join(''); - if (instructPreset.output_suffix) { - requestData.stop = [instructPreset.output_suffix]; - requestData.stopping_strings = [instructPreset.output_suffix]; - } + const stoppingStrings = getInstructStoppingSequences({ customInstruct: instructPreset, useStopStrings: false }); + requestData.stop = stoppingStrings; + requestData.stopping_strings = stoppingStrings; } else { console.warn(`Instruct preset "${instructName}" not found, using basic formatting`); requestData.prompt = prompt.map(x => x.content).join('\n\n'); @@ -283,7 +288,61 @@ export class TextCompletionService { // @ts-ignore const data = this.createRequestData(requestData); - return await this.sendRequest(data, extractData, signal); + const response = await this.sendRequest(data, extractData, signal); + // Remove stopping strings from the end + if (!data.stream && extractData) { + /** @type {ExtractedData} */ + // @ts-ignore + const extractedData = response; + + let message = extractedData.content; + + message = message.replace(/[^\S\r\n]+$/gm, ''); + + if (requestData.stopping_strings) { + for (const stoppingString of requestData.stopping_strings) { + if (stoppingString.length) { + for (let j = stoppingString.length; j > 0; j--) { + if (message.slice(-j) === stoppingString.slice(0, j)) { + message = message.slice(0, -j); + break; + } + } + } + } + } + + if (instructPreset) { + [ + instructPreset.stop_sequence, + instructPreset.input_sequence, + ].forEach(sequence => { + if (sequence?.trim()) { + const index = message.indexOf(sequence); + if (index !== -1) { + message = message.substring(0, index); + } + } + }); + + [ + instructPreset.output_sequence, + instructPreset.last_output_sequence, + ].forEach(sequences => { + if (sequences) { + sequences.split('\n') + .filter(line => line.trim() !== '') + .forEach(line => { + message = message.replaceAll(line, ''); + }); + } + }); + } + + extractedData.content = message; + } + + return response; } /** diff --git a/public/scripts/extensions/shared.js b/public/scripts/extensions/shared.js index 144ac3d6b..e4cca98fa 100644 --- a/public/scripts/extensions/shared.js +++ b/public/scripts/extensions/shared.js @@ -285,6 +285,7 @@ export class ConnectionManagerRequestService { extractData: true, includePreset: true, includeInstruct: true, + instructSettings: {}, }; static getAllowedTypes() { @@ -298,11 +299,17 @@ export class ConnectionManagerRequestService { * @param {string} profileId * @param {string | (import('../custom-request.js').ChatCompletionMessage & {ignoreInstruct?: boolean})[]} prompt * @param {number} maxTokens - * @param {{stream?: boolean, signal?: AbortSignal, extractData?: boolean, includePreset?: boolean, includeInstruct?: boolean}} custom - default values are true + * @param {Object} custom + * @param {boolean?} [custom.stream=false] + * @param {AbortSignal?} [custom.signal] + * @param {boolean?} [custom.extractData=true] + * @param {boolean?} [custom.includePreset=true] + * @param {boolean?} [custom.includeInstruct=true] + * @param {Partial?} [custom.instructSettings] Override instruct settings * @returns {Promise AsyncGenerator)>} If not streaming, returns extracted data; if streaming, returns a function that creates an AsyncGenerator */ static async sendRequest(profileId, prompt, maxTokens, custom = this.defaultSendRequestParams) { - const { stream, signal, extractData, includePreset, includeInstruct } = { ...this.defaultSendRequestParams, ...custom }; + const { stream, signal, extractData, includePreset, includeInstruct, instructSettings } = { ...this.defaultSendRequestParams, ...custom }; const context = SillyTavern.getContext(); if (context.extensionSettings.disabledExtensions.includes('connection-manager')) { @@ -346,6 +353,7 @@ export class ConnectionManagerRequestService { }, { instructName: includeInstruct ? profile.instruct : undefined, presetName: includePreset ? profile.preset : undefined, + instructSettings: includeInstruct ? instructSettings : undefined, }, extractData, signal); } default: { diff --git a/public/scripts/instruct-mode.js b/public/scripts/instruct-mode.js index 000d242b0..56b16f560 100644 --- a/public/scripts/instruct-mode.js +++ b/public/scripts/instruct-mode.js @@ -243,9 +243,14 @@ export function autoSelectInstructPreset(modelId) { /** * Converts instruct mode sequences to an array of stopping strings. + * @param {Object} options + * @param {InstructSettings?} [options.customInstruct=null] - Custom instruct settings. + * @param {boolean?} [options.useStopStrings] - Decides whether to use "Chat Start" and "Example Separator" * @returns {string[]} Array of instruct mode stopping strings. */ -export function getInstructStoppingSequences() { +export function getInstructStoppingSequences({ customInstruct = null, useStopStrings = null } = {}) { + const instruct = structuredClone(customInstruct ?? power_user.instruct); + /** * Adds instruct mode sequence to the result array. * @param {string} sequence Sequence string. @@ -254,7 +259,7 @@ export function getInstructStoppingSequences() { function addInstructSequence(sequence) { // Cohee: oobabooga's textgen always appends newline before the sequence as a stopping string // But it's a problem for Metharme which doesn't use newlines to separate them. - const wrap = (s) => power_user.instruct.wrap ? '\n' + s : s; + const wrap = (s) => instruct.wrap ? '\n' + s : s; // Sequence must be a non-empty string if (typeof sequence === 'string' && sequence.length > 0) { // If sequence is just a whitespace or newline - we don't want to make it a stopping string @@ -262,7 +267,7 @@ export function getInstructStoppingSequences() { if (sequence.trim().length > 0) { const wrappedSequence = wrap(sequence); // Need to respect "insert macro" setting - const stopString = power_user.instruct.macro ? substituteParams(wrappedSequence) : wrappedSequence; + const stopString = instruct.macro ? substituteParams(wrappedSequence) : wrappedSequence; result.push(stopString); } } @@ -270,14 +275,15 @@ export function getInstructStoppingSequences() { const result = []; - if (power_user.instruct.enabled) { - const stop_sequence = power_user.instruct.stop_sequence || ''; - const input_sequence = power_user.instruct.input_sequence?.replace(/{{name}}/gi, name1) || ''; - const output_sequence = power_user.instruct.output_sequence?.replace(/{{name}}/gi, name2) || ''; - const first_output_sequence = power_user.instruct.first_output_sequence?.replace(/{{name}}/gi, name2) || ''; - const last_output_sequence = power_user.instruct.last_output_sequence?.replace(/{{name}}/gi, name2) || ''; - const system_sequence = power_user.instruct.system_sequence?.replace(/{{name}}/gi, 'System') || ''; - const last_system_sequence = power_user.instruct.last_system_sequence?.replace(/{{name}}/gi, 'System') || ''; + // Since preset's don't have "enabled", we assume it's always enabled + if (customInstruct ?? instruct.enabled) { + const stop_sequence = instruct.stop_sequence || ''; + const input_sequence = instruct.input_sequence?.replace(/{{name}}/gi, name1) || ''; + const output_sequence = instruct.output_sequence?.replace(/{{name}}/gi, name2) || ''; + const first_output_sequence = instruct.first_output_sequence?.replace(/{{name}}/gi, name2) || ''; + const last_output_sequence = instruct.last_output_sequence?.replace(/{{name}}/gi, name2) || ''; + const system_sequence = instruct.system_sequence?.replace(/{{name}}/gi, 'System') || ''; + const last_system_sequence = instruct.last_system_sequence?.replace(/{{name}}/gi, 'System') || ''; const combined_sequence = [ stop_sequence, @@ -292,7 +298,7 @@ export function getInstructStoppingSequences() { combined_sequence.split('\n').filter((line, index, self) => self.indexOf(line) === index).forEach(addInstructSequence); } - if (power_user.context.use_stop_strings) { + if (useStopStrings ?? power_user.context.use_stop_strings) { if (power_user.context.chat_start) { result.push(`\n${substituteParams(power_user.context.chat_start)}`); }