rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Added stream support to "custom-request"

Browse Source
This commit is contained in:
bmen25124
2025-03-21 20:36:51 +03:00
parent ef56eda2c2
commit ec474f5571
3 changed files with 192 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
public/scripts/custom-request.js
View File

@ -3,10 +3,13 @@ import { extractMessageFromData, getGenerateUrl, getRequestHeaders } from '../sc
import { getTextGenServer } from './textgen-settings.js';
import { extractReasoningFromData } from './reasoning.js';
import { formatInstructModeChat, formatInstructModePrompt, names_behavior_types } from './instruct-mode.js';
import { getStreamingReply, tryParseStreamingError } from './openai.js';
import EventSourceStream from './sse-stream.js';
// #region Type Definitions
/**
* @typedef {Object} TextCompletionRequestBase
* @property {boolean?} [stream=false] - Whether to stream the response
* @property {number} max_tokens - Maximum number of tokens to generate
* @property {string} [model] - Optional model name
* @property {string} api_type - Type of API to use
@ -17,6 +20,7 @@ import { formatInstructModeChat, formatInstructModePrompt, names_behavior_types
/**
* @typedef {Object} TextCompletionPayloadBase
* @property {boolean?} [stream=false] - Whether to stream the response
* @property {string} prompt - The text prompt for completion
* @property {number} max_tokens - Maximum number of tokens to generate
* @property {number} max_new_tokens - Alias for max_tokens
@ -36,6 +40,7 @@ import { formatInstructModeChat, formatInstructModePrompt, names_behavior_types
/**
* @typedef {Object} ChatCompletionPayloadBase
* @property {boolean?} [stream=false] - Whether to stream the response
* @property {ChatCompletionMessage[]} messages - Array of chat messages
* @property {string} [model] - Optional model name to use for completion
* @property {string} chat_completion_source - Source provider for chat completion
@ -52,10 +57,20 @@ import { formatInstructModeChat, formatInstructModePrompt, names_behavior_types
* @property {string} reasoning - Extracted reasoning.
*/
/**
* @typedef {Object} StreamResponse
* @property {string} text - Generated text.
* @property {string[]} swipes - Generated swipes
* @property {Object} state - Generated state
* @property {string?} [state.reasoning] - Generated reasoning
* @property {string?} [state.image] - Generated image
* @returns {StreamResponse}
*/
// #endregion
/**
* Creates & sends a text completion request. Streaming is not supported.
* Creates & sends a text completion request.
*/
export class TextCompletionService {
static TYPE = 'textgenerationwebui';
@ -64,9 +79,10 @@ export class TextCompletionService {
* @param {Record<string, any> & TextCompletionRequestBase & {prompt: string}} custom
* @returns {TextCompletionPayload}
*/
static createRequestData({ prompt, max_tokens, model, api_type, api_server, temperature, min_p, ...props }) {
static createRequestData({ stream = false, prompt, max_tokens, model, api_type, api_server, temperature, min_p, ...props }) {
const payload = {
...props,
stream,
prompt,
max_tokens,
max_new_tokens: max_tokens,
@ -75,7 +91,6 @@ export class TextCompletionService {
api_server: api_server ?? getTextGenServer(api_type),
temperature,
min_p,
stream: false,
};
// Remove undefined values to avoid API errors
@ -92,16 +107,18 @@ export class TextCompletionService {
* Sends a text completion request to the specified server
* @param {TextCompletionPayload} data Request data
* @param {boolean?} extractData Extract message from the response. Default true
* @returns {Promise<ExtractedData | any>} Extracted data or the raw response
* @param {AbortSignal?} signal
* @returns {Promise<ExtractedData | (() => AsyncGenerator<StreamResponse>)>} If not streaming, returns extracted data; if streaming, returns a function that creates an AsyncGenerator
* @throws {Error}
*/
static async sendRequest(data, extractData = true) {
static async sendRequest(data, extractData = true, signal = null) {
if (!data.stream) {
const response = await fetch(getGenerateUrl(this.TYPE), {
method: 'POST',
headers: getRequestHeaders(),
cache: 'no-cache',
body: JSON.stringify(data),
signal: new AbortController().signal,
signal: signal ?? new AbortController().signal,
});
const json = await response.json();
@ -123,6 +140,51 @@ export class TextCompletionService {
};
}
const response = await fetch('/api/backends/text-completions/generate', {
method: 'POST',
headers: getRequestHeaders(),
cache: 'no-cache',
body: JSON.stringify(data),
signal: signal ?? new AbortController().signal,
});
if (!response.ok) {
const text = await response.text();
tryParseStreamingError(response, text, true);
throw new Error(`Got response status ${response.status}`);
}
const eventStream = new EventSourceStream();
response.body.pipeThrough(eventStream);
const reader = eventStream.readable.getReader();
return async function* streamData() {
let text = '';
const swipes = [];
const state = { reasoning: '' };
while (true) {
const { done, value } = await reader.read();
if (done) return;
if (value.data === '[DONE]') return;
tryParseStreamingError(response, value.data, true);
let data = JSON.parse(value.data);
if (data?.choices?.[0]?.index > 0) {
const swipeIndex = data.choices[0].index - 1;
swipes[swipeIndex] = (swipes[swipeIndex] || '') + data.choices[0].text;
} else {
const newText = data?.choices?.[0]?.text || data?.content || '';
text += newText;
state.reasoning += data?.choices?.[0]?.reasoning ?? '';
}
yield { text, swipes, state };
}
};
}
/**
* Process and send a text completion request with optional preset & instruct
* @param {Record<string, any> & TextCompletionRequestBase & {prompt: (ChatCompletionMessage & {ignoreInstruct?: boolean})[] |string}} custom
@ -130,13 +192,15 @@ export class TextCompletionService {
* @param {string?} [options.presetName] - Name of the preset to use for generation settings
* @param {string?} [options.instructName] - Name of instruct preset for message formatting
* @param {boolean} extractData - Whether to extract structured data from response
* @returns {Promise<ExtractedData | any>} Extracted data or the raw response
* @param {AbortSignal?} [signal]
* @returns {Promise<ExtractedData | (() => AsyncGenerator<StreamResponse>)>} If not streaming, returns extracted data; if streaming, returns a function that creates an AsyncGenerator
* @throws {Error}
*/
static async processRequest(
custom,
options = {},
extractData = true,
signal = null,
) {
const { presetName, instructName } = options;
let requestData = { ...custom };
@ -220,7 +284,7 @@ export class TextCompletionService {
// @ts-ignore
const data = this.createRequestData(requestData);
return await this.sendRequest(data, extractData);
return await this.sendRequest(data, extractData, signal);
}
/**
@ -256,7 +320,7 @@ export class TextCompletionService {
}
/**
* Creates & sends a chat completion request. Streaming is not supported.
* Creates & sends a chat completion request.
*/
export class ChatCompletionService {
static TYPE = 'openai';
@ -265,16 +329,16 @@ export class ChatCompletionService {
* @param {ChatCompletionPayload} custom
* @returns {ChatCompletionPayload}
*/
static createRequestData({ messages, model, chat_completion_source, max_tokens, temperature, custom_url, ...props }) {
static createRequestData({ stream = false, messages, model, chat_completion_source, max_tokens, temperature, custom_url, ...props }) {
const payload = {
...props,
stream,
messages,
model,
chat_completion_source,
max_tokens,
temperature,
custom_url,
stream: false,
};
// Remove undefined values to avoid API errors
@ -291,18 +355,20 @@ export class ChatCompletionService {
* Sends a chat completion request
* @param {ChatCompletionPayload} data Request data
* @param {boolean?} extractData Extract message from the response. Default true
* @returns {Promise<ExtractedData | any>} Extracted data or the raw response
* @param {AbortSignal?} signal Abort signal
* @returns {Promise<ExtractedData | (() => AsyncGenerator<StreamResponse>)>} If not streaming, returns extracted data; if streaming, returns a function that creates an AsyncGenerator
* @throws {Error}
*/
static async sendRequest(data, extractData = true) {
static async sendRequest(data, extractData = true, signal = null) {
const response = await fetch('/api/backends/chat-completions/generate', {
method: 'POST',
headers: getRequestHeaders(),
cache: 'no-cache',
body: JSON.stringify(data),
signal: new AbortController().signal,
signal: signal ?? new AbortController().signal,
});
if (!data.stream) {
const json = await response.json();
if (!response.ok || json.error) {
throw json;
@ -322,16 +388,55 @@ export class ChatCompletionService {
};
}
if (!response.ok) {
const text = await response.text();
tryParseStreamingError(response, text, true);
throw new Error(`Got response status ${response.status}`);
}
const eventStream = new EventSourceStream();
response.body.pipeThrough(eventStream);
const reader = eventStream.readable.getReader();
return async function* streamData() {
let text = '';
const swipes = [];
const state = { reasoning: '', image: '' };
while (true) {
const { done, value } = await reader.read();
if (done) return;
const rawData = value.data;
if (rawData === '[DONE]') return;
tryParseStreamingError(response, rawData, true);
const parsed = JSON.parse(rawData);
const reply = getStreamingReply(parsed, state, {
chatCompletionSource: data.chat_completion_source,
ignoreShowThoughts: true,
});
if (Array.isArray(parsed?.choices) && parsed?.choices?.[0]?.index > 0) {
const swipeIndex = parsed.choices[0].index - 1;
swipes[swipeIndex] = (swipes[swipeIndex] || '') + reply;
} else {
text += reply;
}
yield { text, swipes: swipes, state };
}
};
}
/**
* Process and send a chat completion request with optional preset
* @param {ChatCompletionPayload} custom
* @param {Object} options - Configuration options
* @param {string?} [options.presetName] - Name of the preset to use for generation settings
* @param {boolean} extractData - Whether to extract structured data from response
* @returns {Promise<ExtractedData | any>} Extracted data or the raw response
* @param {boolean} [extractData=true] - Whether to extract structured data from response
* @param {AbortSignal?} [signal] - Abort signal
* @returns {Promise<ExtractedData | (() => AsyncGenerator<StreamResponse>)>} If not streaming, returns extracted data; if streaming, returns a function that creates an AsyncGenerator
* @throws {Error}
*/
static async processRequest(custom, options, extractData = true) {
static async processRequest(custom, options, extractData = true, signal = null) {
const { presetName } = options;
let requestData = { ...custom };
@ -354,7 +459,7 @@ export class ChatCompletionService {
const data = this.createRequestData(requestData);
return await this.sendRequest(data, extractData);
return await this.sendRequest(data, extractData, signal);
}
/**

16
public/scripts/extensions/shared.js
View File

@ -276,10 +276,12 @@ export async function getWebLlmContextSize() {
}
/**
* It uses the profiles to send a generate request to the API. Doesn't support streaming.
* It uses the profiles to send a generate request to the API.
*/
export class ConnectionManagerRequestService {
static defaultSendRequestParams = {
stream: false,
signal: null,
extractData: true,
includePreset: true,
includeInstruct: true,
@ -296,11 +298,11 @@ export class ConnectionManagerRequestService {
* @param {string} profileId
* @param {string | (import('../custom-request.js').ChatCompletionMessage & {ignoreInstruct?: boolean})[]} prompt
* @param {number} maxTokens
* @param {{extractData?: boolean, includePreset?: boolean, includeInstruct?: boolean}} custom - default values are true
* @returns {Promise<import('../custom-request.js').ExtractedData | any>} Extracted data or the raw response
* @param {{stream?: boolean, signal?: AbortSignal, extractData?: boolean, includePreset?: boolean, includeInstruct?: boolean}} custom - default values are true
* @returns {Promise<import('../custom-request.js').ExtractedData | (() => AsyncGenerator<import('../custom-request.js').StreamResponse>)>} If not streaming, returns extracted data; if streaming, returns a function that creates an AsyncGenerator
*/
static async sendRequest(profileId, prompt, maxTokens, custom = this.defaultSendRequestParams) {
const { extractData, includePreset, includeInstruct } = { ...this.defaultSendRequestParams, ...custom };
const { stream, signal, extractData, includePreset, includeInstruct } = { ...this.defaultSendRequestParams, ...custom };
const context = SillyTavern.getContext();
if (context.extensionSettings.disabledExtensions.includes('connection-manager')) {
@ -319,6 +321,7 @@ export class ConnectionManagerRequestService {
const messages = Array.isArray(prompt) ? prompt : [{ role: 'user', content: prompt }];
return await context.ChatCompletionService.processRequest({
stream,
messages,
max_tokens: maxTokens,
model: profile.model,
@ -326,7 +329,7 @@ export class ConnectionManagerRequestService {
custom_url: profile['api-url'],
}, {
presetName: includePreset ? profile.preset : undefined,
}, extractData);
}, extractData, signal);
}
case 'textgenerationwebui': {
if (!selectedApiMap.type) {
@ -334,6 +337,7 @@ export class ConnectionManagerRequestService {
}
return await context.TextCompletionService.processRequest({
stream,
prompt,
max_tokens: maxTokens,
model: profile.model,
@ -342,7 +346,7 @@ export class ConnectionManagerRequestService {
}, {
instructName: includeInstruct ? profile.instruct : undefined,
presetName: includePreset ? profile.preset : undefined,
}, extractData);
}, extractData, signal);
}
default: {
throw new Error(`Unknown API type ${selectedApiMap.selected}`);

54
public/scripts/openai.js
View File

@ -1444,8 +1444,9 @@ export async function prepareOpenAIMessages({
* Handles errors during streaming requests.
* @param {Response} response
* @param {string} decoded - response text or decoded stream data
* @param {boolean?} [supressToastr=false]
*/
function tryParseStreamingError(response, decoded) {
export function tryParseStreamingError(response, decoded, supressToastr = false) {
try {
const data = JSON.parse(decoded);
@ -1453,19 +1454,19 @@ function tryParseStreamingError(response, decoded) {
return;
}
checkQuotaError(data);
checkModerationError(data);
checkQuotaError(data, supressToastr);
checkModerationError(data, supressToastr);
// these do not throw correctly (equiv to Error("[object Object]"))
// if trying to fix "[object Object]" displayed to users, start here
if (data.error) {
toastr.error(data.error.message || response.statusText, 'Chat Completion API');
!supressToastr && toastr.error(data.error.message || response.statusText, 'Chat Completion API');
throw new Error(data);
}
if (data.message) {
toastr.error(data.message, 'Chat Completion API');
!supressToastr && toastr.error(data.message, 'Chat Completion API');
throw new Error(data);
}
}
@ -1477,16 +1478,17 @@ function tryParseStreamingError(response, decoded) {
/**
* Checks if the response contains a quota error and displays a popup if it does.
* @param data
* @param {boolean?} [supressToastr=false]
* @returns {void}
* @throws {object} - response JSON
*/
function checkQuotaError(data) {
function checkQuotaError(data, supressToastr = false) {
if (!data) {
return;
}
if (data.quota_error) {
renderTemplateAsync('quotaError').then((html) => Popup.show.text('Quota Error', html));
!supressToastr && renderTemplateAsync('quotaError').then((html) => Popup.show.text('Quota Error', html));
// this does not throw correctly (equiv to Error("[object Object]"))
// if trying to fix "[object Object]" displayed to users, start here
@ -1494,9 +1496,13 @@ function checkQuotaError(data) {
}
}
function checkModerationError(data) {
/**
* @param {any} data
* @param {boolean?} [supressToastr=false]
*/
function checkModerationError(data, supressToastr = false) {
const moderationError = data?.error?.message?.includes('requires moderation');
if (moderationError) {
if (moderationError && !supressToastr) {
const moderationReason = `Reasons: ${data?.error?.metadata?.reasons?.join(', ') ?? '(N/A)'}`;
const flaggedText = data?.error?.metadata?.flagged_input ?? '(N/A)';
toastr.info(flaggedText, moderationReason, { timeOut: 10000 });
@ -2255,37 +2261,43 @@ async function sendOpenAIRequest(type, messages, signal) {
* Extracts the reply from the response data from a chat completions-like source
* @param {object} data Response data from the chat completions-like source
* @param {object} state Additional state to keep track of
* @param {object} options Additional options
* @param {string?} [options.chatCompletionSource] Chat completion source
* @param {boolean?} [options.ignoreShowThoughts] Ignore show thoughts
* @returns {string} The reply extracted from the response data
*/
function getStreamingReply(data, state) {
if (oai_settings.chat_completion_source === chat_completion_sources.CLAUDE) {
if (oai_settings.show_thoughts) {
export function getStreamingReply(data, state, { chatCompletionSource = null, ignoreShowThoughts = false } = {}) {
const chat_completion_source = chatCompletionSource ?? oai_settings.chat_completion_source;
const show_thoughts = ignoreShowThoughts ? true : oai_settings.show_thoughts;
if (chat_completion_source === chat_completion_sources.CLAUDE) {
if (show_thoughts) {
state.reasoning += data?.delta?.thinking || '';
}
return data?.delta?.text || '';
} else if (oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE) {
} else if (chat_completion_source === chat_completion_sources.MAKERSUITE) {
const inlineData = data?.candidates?.[0]?.content?.parts?.find(x => x.inlineData)?.inlineData;
if (inlineData) {
state.image = `data:${inlineData.mimeType};base64,${inlineData.data}`;
}
if (oai_settings.show_thoughts) {
if (show_thoughts) {
state.reasoning += (data?.candidates?.[0]?.content?.parts?.filter(x => x.thought)?.map(x => x.text)?.[0] || '');
}
return data?.candidates?.[0]?.content?.parts?.filter(x => !x.thought)?.map(x => x.text)?.[0] || '';
} else if (oai_settings.chat_completion_source === chat_completion_sources.COHERE) {
} else if (chat_completion_source === chat_completion_sources.COHERE) {
return data?.delta?.message?.content?.text || data?.delta?.message?.tool_plan || '';
} else if (oai_settings.chat_completion_source === chat_completion_sources.DEEPSEEK) {
if (oai_settings.show_thoughts) {
} else if (chat_completion_source === chat_completion_sources.DEEPSEEK) {
if (show_thoughts) {
state.reasoning += (data.choices?.filter(x => x?.delta?.reasoning_content)?.[0]?.delta?.reasoning_content || '');
}
return data.choices?.[0]?.delta?.content || '';
} else if (oai_settings.chat_completion_source === chat_completion_sources.OPENROUTER) {
if (oai_settings.show_thoughts) {
} else if (chat_completion_source === chat_completion_sources.OPENROUTER) {
if (show_thoughts) {
state.reasoning += (data.choices?.filter(x => x?.delta?.reasoning)?.[0]?.delta?.reasoning || '');
}
return data.choices?.[0]?.delta?.content ?? data.choices?.[0]?.message?.content ?? data.choices?.[0]?.text ?? '';
} else if (oai_settings.chat_completion_source === chat_completion_sources.CUSTOM) {
if (oai_settings.show_thoughts) {
} else if (chat_completion_source === chat_completion_sources.CUSTOM) {
if (show_thoughts) {
state.reasoning +=
data.choices?.filter(x => x?.delta?.reasoning_content)?.[0]?.delta?.reasoning_content ??
data.choices?.filter(x => x?.delta?.reasoning)?.[0]?.delta?.reasoning ??