From 51b3b8bfaa9f17e4b24050c3cca79c31c6de1361 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 2 Apr 2024 14:56:15 +0300 Subject: [PATCH 01/36] Add smooth streaming --- public/index.html | 9 ++ public/scripts/kai-settings.js | 4 +- public/scripts/nai-settings.js | 4 +- public/scripts/openai.js | 4 +- public/scripts/power-user.js | 8 ++ public/scripts/sse-stream.js | 167 +++++++++++++++++++++++++++++ public/scripts/textgen-settings.js | 4 +- 7 files changed, 192 insertions(+), 8 deletions(-) diff --git a/public/index.html b/public/index.html index 5d93ef362..d6ed82c83 100644 --- a/public/index.html +++ b/public/index.html @@ -3618,6 +3618,15 @@ + diff --git a/public/scripts/kai-settings.js b/public/scripts/kai-settings.js index b6d6b73b7..27a204c42 100644 --- a/public/scripts/kai-settings.js +++ b/public/scripts/kai-settings.js @@ -9,7 +9,7 @@ import { import { power_user, } from './power-user.js'; -import EventSourceStream from './sse-stream.js'; +import { getEventSourceStream } from './sse-stream.js'; import { getSortableDelay } from './utils.js'; export const kai_settings = { @@ -174,7 +174,7 @@ export async function generateKoboldWithStreaming(generate_data, signal) { tryParseStreamingError(response, await response.text()); throw new Error(`Got response status ${response.status}`); } - const eventStream = new EventSourceStream(); + const eventStream = getEventSourceStream(); response.body.pipeThrough(eventStream); const reader = eventStream.readable.getReader(); diff --git a/public/scripts/nai-settings.js b/public/scripts/nai-settings.js index 5fcc851e4..edc69d70b 100644 --- a/public/scripts/nai-settings.js +++ b/public/scripts/nai-settings.js @@ -10,7 +10,7 @@ import { import { getCfgPrompt } from './cfg-scale.js'; import { MAX_CONTEXT_DEFAULT, MAX_RESPONSE_DEFAULT, power_user } from './power-user.js'; import { getTextTokens, tokenizers } from './tokenizers.js'; -import EventSourceStream from './sse-stream.js'; +import { getEventSourceStream } from './sse-stream.js'; import { getSortableDelay, getStringHash, @@ -614,7 +614,7 @@ export async function generateNovelWithStreaming(generate_data, signal) { tryParseStreamingError(response, await response.text()); throw new Error(`Got response status ${response.status}`); } - const eventStream = new EventSourceStream(); + const eventStream = getEventSourceStream(); response.body.pipeThrough(eventStream); const reader = eventStream.readable.getReader(); diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 0c30c0640..a5eb38bab 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -45,7 +45,7 @@ import { import { getCustomStoppingStrings, persona_description_positions, power_user } from './power-user.js'; import { SECRET_KEYS, secret_state, writeSecret } from './secrets.js'; -import EventSourceStream from './sse-stream.js'; +import { getEventSourceStream } from './sse-stream.js'; import { delay, download, @@ -1772,7 +1772,7 @@ async function sendOpenAIRequest(type, messages, signal) { throw new Error(`Got response status ${response.status}`); } if (stream) { - const eventStream = new EventSourceStream(); + const eventStream = getEventSourceStream(); response.body.pipeThrough(eventStream); const reader = eventStream.readable.getReader(); return async function* streamData() { diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index ae5d503f5..e9909316d 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -118,6 +118,7 @@ let power_user = { markdown_escape_strings: '', chat_truncation: 100, streaming_fps: 30, + smooth_streaming: false, ui_mode: ui_mode.POWER, fast_ui_mode: true, @@ -1544,6 +1545,8 @@ function loadPowerUserSettings(settings, data) { $('#streaming_fps').val(power_user.streaming_fps); $('#streaming_fps_counter').val(power_user.streaming_fps); + $('#smooth_streaming').prop('checked', power_user.smooth_streaming); + $('#font_scale').val(power_user.font_scale); $('#font_scale_counter').val(power_user.font_scale); @@ -2941,6 +2944,11 @@ $(document).ready(() => { saveSettingsDebounced(); }); + $('#smooth_streaming').on('input', function () { + power_user.smooth_streaming = !!$(this).prop('checked'); + saveSettingsDebounced(); + }); + $('input[name="font_scale"]').on('input', async function (e) { power_user.font_scale = Number(e.target.value); $('#font_scale_counter').val(power_user.font_scale); diff --git a/public/scripts/sse-stream.js b/public/scripts/sse-stream.js index c9f7158d7..92737e9a9 100644 --- a/public/scripts/sse-stream.js +++ b/public/scripts/sse-stream.js @@ -1,3 +1,6 @@ +import { power_user } from './power-user.js'; +import { delay } from './utils.js'; + /** * A stream which handles Server-Sent Events from a binary ReadableStream like you get from the fetch API. */ @@ -74,4 +77,168 @@ class EventSourceStream { } } +/** + * Like the default one, but multiplies the events by the number of letters in the event data. + */ +export class SmoothEventSourceStream extends EventSourceStream { + constructor() { + super(); + const defaultDelayMs = 20; + const punctuationDelayMs = 500; + function getDelay(s) { + if (!s) { + return 0; + } + + if (s == ',') { + return punctuationDelayMs / 2; + } + + if (['.', '!', '?', '\n'].includes(s)) { + return punctuationDelayMs; + } + + return defaultDelayMs; + } + let lastStr = ''; + const transformStream = new TransformStream({ + async transform(chunk, controller) { + const event = chunk; + const data = event.data; + try { + const json = JSON.parse(data); + + if (!json) { + controller.enqueue(event); + return; + } + + // Claude + if (typeof json.delta === 'object') { + if (typeof json.delta.text === 'string' && json.delta.text.length > 0) { + for (let i = 0; i < json.delta.text.length; i++) { + await delay(getDelay(lastStr)); + const str = json.delta.text[i]; + controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, delta: { text: str } }) })); + lastStr = str; + } + } else { + controller.enqueue(event); + } + } + // MakerSuite + else if (Array.isArray(json.candidates)) { + for (let i = 0; i < json.candidates.length; i++) { + if (typeof json.candidates[i].content === 'string' && json.candidates[i].content.length > 0) { + for (let j = 0; j < json.candidates[i].content.length; j++) { + await delay(getDelay(lastStr)); + const str = json.candidates[i].content[j]; + const candidatesClone = structuredClone(json.candidates[i]); + candidatesClone[i].content = str; + controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, candidates: candidatesClone }) })); + lastStr = str; + } + } else { + controller.enqueue(event); + } + } + } + // NovelAI / KoboldCpp Classic + else if (typeof json.token === 'string' && json.token.length > 0) { + for (let i = 0; i < json.token.length; i++) { + await delay(getDelay(lastStr)); + const str = json.token[i]; + controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, token: str }) })); + lastStr = str; + } + } + // llama.cpp? + else if (typeof json.content === 'string' && json.content.length > 0) { + for (let i = 0; i < json.content.length; i++) { + await delay(getDelay(lastStr)); + const str = json.content[i]; + controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, content: str }) })); + lastStr = str; + } + } + // OpenAI-likes + else if (Array.isArray(json.choices)) { + const isNotPrimary = json?.choices?.[0]?.index > 0; + if (isNotPrimary || json.choices.length === 0) { + controller.enqueue(event); + return; + } + if (typeof json.choices[0].delta === 'object') { + if (typeof json.choices[0].delta.text === 'string' && json.choices[0].delta.text.length > 0) { + for (let j = 0; j < json.choices[0].delta.text.length; j++) { + await delay(getDelay(lastStr)); + const str = json.choices[0].delta.text[j]; + const choiceClone = structuredClone(json.choices[0]); + choiceClone.delta.text = str; + const choices = [choiceClone]; + controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, choices }) })); + lastStr = str; + } + } else if (typeof json.choices[0].delta.content === 'string' && json.choices[0].delta.content.length > 0) { + for (let j = 0; j < json.choices[0].delta.content.length; j++) { + await delay(getDelay(lastStr)); + const str = json.choices[0].delta.content[j]; + const choiceClone = structuredClone(json.choices[0]); + choiceClone.delta.content = str; + const choices = [choiceClone]; + controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, choices }) })); + lastStr = str; + } + + } else { + controller.enqueue(event); + } + } + else if (typeof json.choices[0].message === 'object') { + if (typeof json.choices[0].message.content === 'string' && json.choices[0].message.content.length > 0) { + for (let j = 0; j < json.choices[0].message.content.length; j++) { + await delay(getDelay(lastStr)); + const str = json.choices[0].message.content[j]; + const choiceClone = structuredClone(json.choices[0]); + choiceClone.message.content = str; + const choices = [choiceClone]; + controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, choices }) })); + lastStr = str; + } + } else { + controller.enqueue(event); + } + } + else if (typeof json.choices[0].text === 'string' && json.choices[0].text.length > 0) { + for (let j = 0; j < json.choices[0].text.length; j++) { + await delay(getDelay(lastStr)); + const str = json.choices[0].text[j]; + const choiceClone = structuredClone(json.choices[0]); + choiceClone.text = str; + const choices = [choiceClone]; + controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, choices }) })); + lastStr = str; + } + } else { + controller.enqueue(event); + } + } + } catch { + controller.enqueue(event); + } + }, + }); + + this.readable = this.readable.pipeThrough(transformStream); + } +} + +export function getEventSourceStream() { + if (power_user.smooth_streaming) { + return new SmoothEventSourceStream(); + } + + return new EventSourceStream(); +} + export default EventSourceStream; diff --git a/public/scripts/textgen-settings.js b/public/scripts/textgen-settings.js index 6653149ba..f871434a3 100644 --- a/public/scripts/textgen-settings.js +++ b/public/scripts/textgen-settings.js @@ -12,7 +12,7 @@ import { import { BIAS_CACHE, createNewLogitBiasEntry, displayLogitBias, getLogitBiasListResult } from './logit-bias.js'; import { power_user, registerDebugFunction } from './power-user.js'; -import EventSourceStream from './sse-stream.js'; +import { getEventSourceStream } from './sse-stream.js'; import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer } from './textgen-models.js'; import { SENTENCEPIECE_TOKENIZERS, TEXTGEN_TOKENIZERS, getTextTokens, tokenizers } from './tokenizers.js'; import { getSortableDelay, onlyUnique } from './utils.js'; @@ -821,7 +821,7 @@ async function generateTextGenWithStreaming(generate_data, signal) { throw new Error(`Got response status ${response.status}`); } - const eventStream = new EventSourceStream(); + const eventStream = getEventSourceStream(); response.body.pipeThrough(eventStream); const reader = eventStream.readable.getReader(); From 8176e09d4a1f354ac7ddccbaeb8a10ce7f16a523 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:25:23 +0300 Subject: [PATCH 02/36] Refactor event parsing --- public/scripts/sse-stream.js | 275 +++++++++++++++++++---------------- 1 file changed, 149 insertions(+), 126 deletions(-) diff --git a/public/scripts/sse-stream.js b/public/scripts/sse-stream.js index 92737e9a9..7cc8e15e9 100644 --- a/public/scripts/sse-stream.js +++ b/public/scripts/sse-stream.js @@ -77,29 +77,152 @@ class EventSourceStream { } } +const defaultDelayMs = 20; +const punctuationDelayMs = 500; + +/** + * Gets a delay based on the character. + * @param {string} s The character. + * @returns {number} The delay in milliseconds. + */ +function getDelay(s) { + if (!s) { + return 0; + } + + if (s == ',') { + return punctuationDelayMs / 2; + } + + if (['.', '!', '?', '\n'].includes(s)) { + return punctuationDelayMs; + } + + return defaultDelayMs; +} + +/** + * Parses the stream data and returns the parsed data and the chunk to be sent. + * @param {object} json The JSON data. + * @returns {AsyncGenerator<{data: object, chunk: string} | null>} The parsed data and the chunk to be sent. + */ +async function* parseStreamData(json) { + // Claude + if (typeof json.delta === 'object') { + if (typeof json.delta.text === 'string' && json.delta.text.length > 0) { + for (let i = 0; i < json.delta.text.length; i++) { + const str = json.delta.text[i]; + yield { + data: { ...json, delta: { text: str } }, + chunk: str, + }; + } + } + } + // MakerSuite + else if (Array.isArray(json.candidates)) { + for (let i = 0; i < json.candidates.length; i++) { + if (typeof json.candidates[i].content === 'string' && json.candidates[i].content.length > 0) { + for (let j = 0; j < json.candidates[i].content.length; j++) { + const str = json.candidates[i].content[j]; + const candidatesClone = structuredClone(json.candidates[i]); + candidatesClone[i].content = str; + yield { + data: { ...json, candidates: candidatesClone }, + chunk: str, + }; + } + } + } + } + // NovelAI / KoboldCpp Classic + else if (typeof json.token === 'string' && json.token.length > 0) { + for (let i = 0; i < json.token.length; i++) { + const str = json.token[i]; + yield { + data: { ...json, token: str }, + chunk: str, + }; + } + } + // llama.cpp? + else if (typeof json.content === 'string' && json.content.length > 0) { + for (let i = 0; i < json.content.length; i++) { + const str = json.content[i]; + yield { + data: { ...json, content: str }, + chunk: str, + }; + } + } + // OpenAI-likes + else if (Array.isArray(json.choices)) { + const isNotPrimary = json?.choices?.[0]?.index > 0; + if (isNotPrimary || json.choices.length === 0) { + return null; + } + if (typeof json.choices[0].delta === 'object') { + if (typeof json.choices[0].delta.text === 'string' && json.choices[0].delta.text.length > 0) { + for (let j = 0; j < json.choices[0].delta.text.length; j++) { + const str = json.choices[0].delta.text[j]; + const choiceClone = structuredClone(json.choices[0]); + choiceClone.delta.text = str; + const choices = [choiceClone]; + yield { + data: { ...json, choices }, + chunk: str, + }; + } + } else if (typeof json.choices[0].delta.content === 'string' && json.choices[0].delta.content.length > 0) { + for (let j = 0; j < json.choices[0].delta.content.length; j++) { + const str = json.choices[0].delta.content[j]; + const choiceClone = structuredClone(json.choices[0]); + choiceClone.delta.content = str; + const choices = [choiceClone]; + yield { + data: { ...json, choices }, + chunk: str, + }; + } + } + } + else if (typeof json.choices[0].message === 'object') { + if (typeof json.choices[0].message.content === 'string' && json.choices[0].message.content.length > 0) { + for (let j = 0; j < json.choices[0].message.content.length; j++) { + const str = json.choices[0].message.content[j]; + const choiceClone = structuredClone(json.choices[0]); + choiceClone.message.content = str; + const choices = [choiceClone]; + yield { + data: { ...json, choices }, + chunk: str, + }; + } + } + } + else if (typeof json.choices[0].text === 'string' && json.choices[0].text.length > 0) { + for (let j = 0; j < json.choices[0].text.length; j++) { + const str = json.choices[0].text[j]; + const choiceClone = structuredClone(json.choices[0]); + choiceClone.text = str; + const choices = [choiceClone]; + yield { + data: { ...json, choices }, + chunk: str, + }; + } + } + } + + return null; +} + /** * Like the default one, but multiplies the events by the number of letters in the event data. */ export class SmoothEventSourceStream extends EventSourceStream { constructor() { super(); - const defaultDelayMs = 20; - const punctuationDelayMs = 500; - function getDelay(s) { - if (!s) { - return 0; - } - - if (s == ',') { - return punctuationDelayMs / 2; - } - - if (['.', '!', '?', '\n'].includes(s)) { - return punctuationDelayMs; - } - - return defaultDelayMs; - } let lastStr = ''; const transformStream = new TransformStream({ async transform(chunk, controller) { @@ -109,119 +232,19 @@ export class SmoothEventSourceStream extends EventSourceStream { const json = JSON.parse(data); if (!json) { - controller.enqueue(event); - return; + lastStr = ''; + return controller.enqueue(event); } - // Claude - if (typeof json.delta === 'object') { - if (typeof json.delta.text === 'string' && json.delta.text.length > 0) { - for (let i = 0; i < json.delta.text.length; i++) { - await delay(getDelay(lastStr)); - const str = json.delta.text[i]; - controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, delta: { text: str } }) })); - lastStr = str; - } - } else { - controller.enqueue(event); + for await (const parsed of parseStreamData(json)) { + if (!parsed) { + lastStr = ''; + return controller.enqueue(event); } - } - // MakerSuite - else if (Array.isArray(json.candidates)) { - for (let i = 0; i < json.candidates.length; i++) { - if (typeof json.candidates[i].content === 'string' && json.candidates[i].content.length > 0) { - for (let j = 0; j < json.candidates[i].content.length; j++) { - await delay(getDelay(lastStr)); - const str = json.candidates[i].content[j]; - const candidatesClone = structuredClone(json.candidates[i]); - candidatesClone[i].content = str; - controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, candidates: candidatesClone }) })); - lastStr = str; - } - } else { - controller.enqueue(event); - } - } - } - // NovelAI / KoboldCpp Classic - else if (typeof json.token === 'string' && json.token.length > 0) { - for (let i = 0; i < json.token.length; i++) { - await delay(getDelay(lastStr)); - const str = json.token[i]; - controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, token: str }) })); - lastStr = str; - } - } - // llama.cpp? - else if (typeof json.content === 'string' && json.content.length > 0) { - for (let i = 0; i < json.content.length; i++) { - await delay(getDelay(lastStr)); - const str = json.content[i]; - controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, content: str }) })); - lastStr = str; - } - } - // OpenAI-likes - else if (Array.isArray(json.choices)) { - const isNotPrimary = json?.choices?.[0]?.index > 0; - if (isNotPrimary || json.choices.length === 0) { - controller.enqueue(event); - return; - } - if (typeof json.choices[0].delta === 'object') { - if (typeof json.choices[0].delta.text === 'string' && json.choices[0].delta.text.length > 0) { - for (let j = 0; j < json.choices[0].delta.text.length; j++) { - await delay(getDelay(lastStr)); - const str = json.choices[0].delta.text[j]; - const choiceClone = structuredClone(json.choices[0]); - choiceClone.delta.text = str; - const choices = [choiceClone]; - controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, choices }) })); - lastStr = str; - } - } else if (typeof json.choices[0].delta.content === 'string' && json.choices[0].delta.content.length > 0) { - for (let j = 0; j < json.choices[0].delta.content.length; j++) { - await delay(getDelay(lastStr)); - const str = json.choices[0].delta.content[j]; - const choiceClone = structuredClone(json.choices[0]); - choiceClone.delta.content = str; - const choices = [choiceClone]; - controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, choices }) })); - lastStr = str; - } - } else { - controller.enqueue(event); - } - } - else if (typeof json.choices[0].message === 'object') { - if (typeof json.choices[0].message.content === 'string' && json.choices[0].message.content.length > 0) { - for (let j = 0; j < json.choices[0].message.content.length; j++) { - await delay(getDelay(lastStr)); - const str = json.choices[0].message.content[j]; - const choiceClone = structuredClone(json.choices[0]); - choiceClone.message.content = str; - const choices = [choiceClone]; - controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, choices }) })); - lastStr = str; - } - } else { - controller.enqueue(event); - } - } - else if (typeof json.choices[0].text === 'string' && json.choices[0].text.length > 0) { - for (let j = 0; j < json.choices[0].text.length; j++) { - await delay(getDelay(lastStr)); - const str = json.choices[0].text[j]; - const choiceClone = structuredClone(json.choices[0]); - choiceClone.text = str; - const choices = [choiceClone]; - controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify({ ...json, choices }) })); - lastStr = str; - } - } else { - controller.enqueue(event); - } + await delay(getDelay(lastStr)); + controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify(parsed.data) })); + lastStr = parsed.chunk; } } catch { controller.enqueue(event); From 7389286862cf32ca9dfa0ba6ea64019ca9360bd5 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:51:00 +0300 Subject: [PATCH 03/36] Don't show logprobs when using smooth streaming --- public/scripts/logprobs.js | 11 ++++++++--- public/scripts/sse-stream.js | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/public/scripts/logprobs.js b/public/scripts/logprobs.js index 2aef6e61b..b2e682286 100644 --- a/public/scripts/logprobs.js +++ b/public/scripts/logprobs.js @@ -8,6 +8,7 @@ import { Generate, getGeneratingApi, is_send_press, + isStreamingEnabled, } from '../script.js'; import { debounce, delay, getStringHash } from './utils.js'; import { decodeTextTokens, getTokenizerBestMatch } from './tokenizers.js'; @@ -64,11 +65,15 @@ function renderAlternativeTokensView() { renderTopLogprobs(); const { messageLogprobs, continueFrom } = getActiveMessageLogprobData() || {}; - if (!messageLogprobs?.length) { + const usingSmoothStreaming = isStreamingEnabled() && power_user.smooth_streaming; + if (!messageLogprobs?.length || usingSmoothStreaming) { const emptyState = $('
'); + const noTokensMsg = usingSmoothStreaming + ? 'Token probabilities are not available when using Smooth Streaming.' + : 'No token probabilities available for the current message.'; const msg = power_user.request_token_probabilities - ? 'No token probabilities available for the current message.' - : `Enable Request token probabilities in the User Settings menu to use this feature.`; + ? noTokensMsg + : 'Enable Request token probabilities in the User Settings menu to use this feature.'; emptyState.html(msg); emptyState.addClass('logprobs_empty_state'); view.append(emptyState); diff --git a/public/scripts/sse-stream.js b/public/scripts/sse-stream.js index 7cc8e15e9..0cce6b1e2 100644 --- a/public/scripts/sse-stream.js +++ b/public/scripts/sse-stream.js @@ -90,11 +90,11 @@ function getDelay(s) { return 0; } - if (s == ',') { + if ([',', '\n'].includes(s)) { return punctuationDelayMs / 2; } - if (['.', '!', '?', '\n'].includes(s)) { + if (['.', '!', '?'].includes(s)) { return punctuationDelayMs; } From ca047034b7a8f142585780659068626f30d3b407 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 2 Apr 2024 16:13:01 +0300 Subject: [PATCH 04/36] Fix smooth stream for MakerSuite --- public/scripts/sse-stream.js | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/public/scripts/sse-stream.js b/public/scripts/sse-stream.js index 0cce6b1e2..50891e86e 100644 --- a/public/scripts/sse-stream.js +++ b/public/scripts/sse-stream.js @@ -122,15 +122,24 @@ async function* parseStreamData(json) { // MakerSuite else if (Array.isArray(json.candidates)) { for (let i = 0; i < json.candidates.length; i++) { - if (typeof json.candidates[i].content === 'string' && json.candidates[i].content.length > 0) { - for (let j = 0; j < json.candidates[i].content.length; j++) { - const str = json.candidates[i].content[j]; - const candidatesClone = structuredClone(json.candidates[i]); - candidatesClone[i].content = str; - yield { - data: { ...json, candidates: candidatesClone }, - chunk: str, - }; + const isNotPrimary = json.candidates?.[0]?.index > 0; + if (isNotPrimary || json.candidates.length === 0) { + return null; + } + if (typeof json.candidates[0].content === 'object' && Array.isArray(json.candidates[i].content.parts)) { + for (let j = 0; j < json.candidates[i].content.parts.length; j++) { + if (typeof json.candidates[i].content.parts[j].text === 'string') { + for (let k = 0; k < json.candidates[i].content.parts[j].text.length; k++) { + const str = json.candidates[i].content.parts[j].text[k]; + const candidateClone = structuredClone(json.candidates[0]); + candidateClone.content.parts[j].text = str; + const candidates = [candidateClone]; + yield { + data: { ...json, candidates }, + chunk: str, + }; + } + } } } } From 2859ae54abcd7502e39732569e863d79428e95f7 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 2 Apr 2024 16:21:55 +0300 Subject: [PATCH 05/36] Don't delay when not in focus --- public/scripts/sse-stream.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/scripts/sse-stream.js b/public/scripts/sse-stream.js index 50891e86e..b817c4e45 100644 --- a/public/scripts/sse-stream.js +++ b/public/scripts/sse-stream.js @@ -238,6 +238,7 @@ export class SmoothEventSourceStream extends EventSourceStream { const event = chunk; const data = event.data; try { + const hasFocus = document.hasFocus(); const json = JSON.parse(data); if (!json) { @@ -251,7 +252,7 @@ export class SmoothEventSourceStream extends EventSourceStream { return controller.enqueue(event); } - await delay(getDelay(lastStr)); + hasFocus && await delay(getDelay(lastStr)); controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify(parsed.data) })); lastStr = parsed.chunk; } From 759e8eed0c01de7a5615b99e82d5ab96dc409763 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 2 Apr 2024 16:38:39 +0300 Subject: [PATCH 06/36] Fix for Together --- public/scripts/sse-stream.js | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/public/scripts/sse-stream.js b/public/scripts/sse-stream.js index b817c4e45..ad50798d4 100644 --- a/public/scripts/sse-stream.js +++ b/public/scripts/sse-stream.js @@ -170,7 +170,20 @@ async function* parseStreamData(json) { if (isNotPrimary || json.choices.length === 0) { return null; } - if (typeof json.choices[0].delta === 'object') { + + if (typeof json.choices[0].text === 'string' && json.choices[0].text.length > 0) { + for (let j = 0; j < json.choices[0].text.length; j++) { + const str = json.choices[0].text[j]; + const choiceClone = structuredClone(json.choices[0]); + choiceClone.text = str; + const choices = [choiceClone]; + yield { + data: { ...json, choices }, + chunk: str, + }; + } + } + else if (typeof json.choices[0].delta === 'object') { if (typeof json.choices[0].delta.text === 'string' && json.choices[0].delta.text.length > 0) { for (let j = 0; j < json.choices[0].delta.text.length; j++) { const str = json.choices[0].delta.text[j]; @@ -182,7 +195,8 @@ async function* parseStreamData(json) { chunk: str, }; } - } else if (typeof json.choices[0].delta.content === 'string' && json.choices[0].delta.content.length > 0) { + } + else if (typeof json.choices[0].delta.content === 'string' && json.choices[0].delta.content.length > 0) { for (let j = 0; j < json.choices[0].delta.content.length; j++) { const str = json.choices[0].delta.content[j]; const choiceClone = structuredClone(json.choices[0]); @@ -209,18 +223,6 @@ async function* parseStreamData(json) { } } } - else if (typeof json.choices[0].text === 'string' && json.choices[0].text.length > 0) { - for (let j = 0; j < json.choices[0].text.length; j++) { - const str = json.choices[0].text[j]; - const choiceClone = structuredClone(json.choices[0]); - choiceClone.text = str; - const choices = [choiceClone]; - yield { - data: { ...json, choices }, - chunk: str, - }; - } - } } return null; From 422b9e1b631790f0cdd471b5217f46fae9bbb75f Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 2 Apr 2024 18:34:29 +0300 Subject: [PATCH 07/36] Fix sequences to stop strings if missing values --- public/scripts/instruct-mode.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/public/scripts/instruct-mode.js b/public/scripts/instruct-mode.js index 033794009..d90ffb9bc 100644 --- a/public/scripts/instruct-mode.js +++ b/public/scripts/instruct-mode.js @@ -243,12 +243,12 @@ export function getInstructStoppingSequences() { const result = []; if (power_user.instruct.enabled) { - const stop_sequence = power_user.instruct.stop_sequence; - const input_sequence = power_user.instruct.input_sequence.replace(/{{name}}/gi, name1); - const output_sequence = power_user.instruct.output_sequence.replace(/{{name}}/gi, name2); - const first_output_sequence = power_user.instruct.first_output_sequence.replace(/{{name}}/gi, name2); - const last_output_sequence = power_user.instruct.last_output_sequence.replace(/{{name}}/gi, name2); - const system_sequence = power_user.instruct.system_sequence.replace(/{{name}}/gi, 'System'); + const stop_sequence = power_user.instruct.stop_sequence || ''; + const input_sequence = power_user.instruct.input_sequence?.replace(/{{name}}/gi, name1) || ''; + const output_sequence = power_user.instruct.output_sequence?.replace(/{{name}}/gi, name2) || ''; + const first_output_sequence = power_user.instruct.first_output_sequence?.replace(/{{name}}/gi, name2) || ''; + const last_output_sequence = power_user.instruct.last_output_sequence?.replace(/{{name}}/gi, name2) || ''; + const system_sequence = power_user.instruct.system_sequence?.replace(/{{name}}/gi, 'System') || ''; const combined_sequence = `${stop_sequence}\n${input_sequence}\n${output_sequence}\n${first_output_sequence}\n${last_output_sequence}\n${system_sequence}`; From f13e718dc7297db24cb5b61e0f98cbdc10f07be0 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 2 Apr 2024 20:25:37 +0300 Subject: [PATCH 08/36] Compatibility with extensions --- public/lib/eventemitter.js | 6 ++++++ public/script.js | 1 + public/scripts/sse-stream.js | 2 ++ 3 files changed, 9 insertions(+) diff --git a/public/lib/eventemitter.js b/public/lib/eventemitter.js index 046991a05..a1b40e38c 100644 --- a/public/lib/eventemitter.js +++ b/public/lib/eventemitter.js @@ -29,6 +29,12 @@ var EventEmitter = function () { }; EventEmitter.prototype.on = function (event, listener) { + // Unknown event used by external libraries? + if (event === undefined) { + console.trace('EventEmitter: Cannot listen to undefined event'); + return; + } + if (typeof this.events[event] !== 'object') { this.events[event] = []; } diff --git a/public/script.js b/public/script.js index 3078d8a88..805f54dff 100644 --- a/public/script.js +++ b/public/script.js @@ -416,6 +416,7 @@ export const event_types = { // TODO: Naming convention is inconsistent with other events CHARACTER_DELETED: 'characterDeleted', CHARACTER_DUPLICATED: 'character_duplicated', + SMOOTH_STREAM_TOKEN_RECEIVED: 'smooth_stream_token_received', }; export const eventSource = new EventEmitter(); diff --git a/public/scripts/sse-stream.js b/public/scripts/sse-stream.js index ad50798d4..86caf38d7 100644 --- a/public/scripts/sse-stream.js +++ b/public/scripts/sse-stream.js @@ -1,3 +1,4 @@ +import { eventSource, event_types } from '../script.js'; import { power_user } from './power-user.js'; import { delay } from './utils.js'; @@ -257,6 +258,7 @@ export class SmoothEventSourceStream extends EventSourceStream { hasFocus && await delay(getDelay(lastStr)); controller.enqueue(new MessageEvent(event.type, { data: JSON.stringify(parsed.data) })); lastStr = parsed.chunk; + hasFocus && await eventSource.emit(event_types.SMOOTH_STREAM_TOKEN_RECEIVED, parsed.chunk); } } catch { controller.enqueue(event); From 54a6f4bc62ff47187b25439dc61514bd982d693b Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 2 Apr 2024 22:52:51 +0300 Subject: [PATCH 09/36] Add speed control --- public/css/toggle-dependent.css | 8 ++++++++ public/index.html | 11 ++++++++++- public/scripts/power-user.js | 7 +++++++ public/scripts/sse-stream.js | 7 ++++--- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/public/css/toggle-dependent.css b/public/css/toggle-dependent.css index 13067d50a..3f90edc42 100644 --- a/public/css/toggle-dependent.css +++ b/public/css/toggle-dependent.css @@ -439,3 +439,11 @@ body.expandMessageActions .mes .mes_buttons .extraMesButtonsHint { #openai_image_inlining:checked~#image_inlining_hint { display: block; } + +#smooth_streaming:not(:checked)~#smooth_streaming_speed_control { + display: none; +} + +#smooth_streaming:checked~#smooth_streaming_speed_control { + display: block; +} diff --git a/public/index.html b/public/index.html index d6ed82c83..ec7c1500f 100644 --- a/public/index.html +++ b/public/index.html @@ -3618,7 +3618,7 @@ - -
+
Automation ID
diff --git a/public/scripts/extensions/quick-reply/index.js b/public/scripts/extensions/quick-reply/index.js index a032a12d6..7b58f4aaa 100644 --- a/public/scripts/extensions/quick-reply/index.js +++ b/public/scripts/extensions/quick-reply/index.js @@ -104,7 +104,7 @@ const loadSets = async () => { qr.executeOnAi = slot.autoExecute_botMessage ?? false; qr.executeOnChatChange = slot.autoExecute_chatLoad ?? false; qr.executeOnGroupMemberDraft = slot.autoExecute_groupMemberDraft ?? false; - qr.automationId = slot.automationId ?? false; + qr.automationId = slot.automationId ?? ''; qr.contextList = (slot.contextMenu ?? []).map(it=>({ set: it.preset, isChained: it.chain,