From f5b68893d097d466d555eae68216cc23195b3651 Mon Sep 17 00:00:00 2001 From: Wolfsblvt Date: Thu, 12 Sep 2024 22:41:53 +0200 Subject: [PATCH 01/13] Improve error handling of /genraw and /gen - /generate TC backend returns more status/error texts - Fix /genraw and /gen returning null/undefined - Logging errors on /genraw if backend throws an error - Fixes #2836 --- public/script.js | 2 +- public/scripts/slash-commands.js | 10 +++++++++- src/endpoints/backends/text-completions.js | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/public/script.js b/public/script.js index 0305a68e7..4228d646c 100644 --- a/public/script.js +++ b/public/script.js @@ -3265,7 +3265,7 @@ export async function generateRaw(prompt, api, instructOverride, quietToLoud, sy } if (data.error) { - throw new Error(data.error); + throw new Error(data.response); } const message = cleanUpMessage(extractMessageFromData(data), false, false, true); diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index 68f001359..18b04a172 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -2241,7 +2241,7 @@ function setEphemeralStopStrings(value) { async function generateRawCallback(args, value) { if (!value) { console.warn('WARN: No argument provided for /genraw command'); - return; + return ''; } // Prevent generate recursion @@ -2260,12 +2260,16 @@ async function generateRawCallback(args, value) { setEphemeralStopStrings(resolveVariable(args?.stop)); const result = await generateRaw(value, '', isFalseBoolean(args?.instruct), quietToLoud, systemPrompt, length); return result; + } catch (err) { + console.error('Error on /genraw generation', err); + toastr.error(err.message, 'Error on Generate'); } finally { if (lock) { activateSendButtons(); } flushEphemeralStoppingStrings(); } + return ''; } /** @@ -2291,12 +2295,16 @@ async function generateCallback(args, value) { const name = args?.name; const result = await generateQuietPrompt(value, quietToLoud, false, '', name, length); return result; + } catch (err) { + console.error('Error on /genraw generation', err); + toastr.error(err.message, 'Error on Generate'); } finally { if (lock) { activateSendButtons(); } flushEphemeralStoppingStrings(); } + return ''; } /** diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index ba113f2fd..06f9e0325 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -375,7 +375,9 @@ router.post('/generate', jsonParser, async function (request, response) { } } } catch (error) { - let value = { error: true, status: error?.status, response: error?.statusText }; + const status = error?.status ?? error?.code ?? 'UNKNOWN'; + const text = error?.error ?? error?.statusText ?? error?.message ?? 'Unknown error on /generate endpoint'; + let value = { error: true, status: status, response: text }; console.log('Endpoint error:', error); if (!response.headersSent) { From 7a7673432a2168f45fe2b5759cd2e7d9907eb79f Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 10:47:25 +0300 Subject: [PATCH 02/13] Calculate entry hash before replacing macros Fixes #2762 --- public/scripts/world-info.js | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 77dae1cad..92eae32d1 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -108,6 +108,7 @@ const KNOWN_DECORATORS = ['@@activate', '@@dont_activate']; * @property {number} [cooldown] The cooldown of the entry * @property {number} [delay] The delay of the entry * @property {string[]} [decorators] Array of decorators for the entry + * @property {number} [hash] The hash of the entry */ /** @@ -382,12 +383,6 @@ class WorldInfoBuffer { * Represents a timed effects manager for World Info. */ class WorldInfoTimedEffects { - /** - * Cache for entry hashes. Uses weak map to avoid memory leaks. - * @type {WeakMap} - */ - #entryHashCache = new WeakMap(); - /** * Array of chat messages. * @type {string[]} @@ -485,13 +480,7 @@ class WorldInfoTimedEffects { * @returns {number} String hash */ #getEntryHash(entry) { - if (this.#entryHashCache.has(entry)) { - return this.#entryHashCache.get(entry); - } - - const hash = getStringHash(JSON.stringify(entry)); - this.#entryHashCache.set(entry, hash); - return hash; + return entry.hash; } /** @@ -3603,10 +3592,11 @@ export async function getSortedEntries() { // Chat lore always goes first entries = [...chatLore.sort(sortFn), ...entries]; - // Parse decorators + // Calculate hash and parse decorators entries = entries.map((entry) => { + const hash = getStringHash(JSON.stringify(entry)); const [decorators, content] = parseDecorators(entry.content || ''); - return { ...entry, decorators, content }; + return { ...entry, decorators, content, hash }; }); console.debug(`[WI] Found ${entries.length} world lore entries. Sorted by strategy`, Object.entries(world_info_insertion_strategy).find((x) => x[1] === world_info_character_strategy)); From 34b2707895daa74d256ba95081d308a7f6098369 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 10:52:35 +0300 Subject: [PATCH 03/13] Try to preserve old hashes --- public/scripts/world-info.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 92eae32d1..20ccfda0c 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -3592,11 +3592,13 @@ export async function getSortedEntries() { // Chat lore always goes first entries = [...chatLore.sort(sortFn), ...entries]; - // Calculate hash and parse decorators + // Calculate hash and parse decorators. Split maps to preserve old hashes. entries = entries.map((entry) => { - const hash = getStringHash(JSON.stringify(entry)); const [decorators, content] = parseDecorators(entry.content || ''); - return { ...entry, decorators, content, hash }; + return { ...entry, decorators, content }; + }).map((entry) => { + const hash = getStringHash(JSON.stringify(entry)); + return { ...entry, hash }; }); console.debug(`[WI] Found ${entries.length} world lore entries. Sorted by strategy`, Object.entries(world_info_insertion_strategy).find((x) => x[1] === world_info_character_strategy)); From 88fab65a8f9895098614f0a07af71fb88ac0d4cd Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 11:01:59 +0000 Subject: [PATCH 04/13] Fix /gen log --- public/scripts/slash-commands.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index 18b04a172..c1215f310 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -2261,7 +2261,7 @@ async function generateRawCallback(args, value) { const result = await generateRaw(value, '', isFalseBoolean(args?.instruct), quietToLoud, systemPrompt, length); return result; } catch (err) { - console.error('Error on /genraw generation', err); + console.error('Error on /gen generation', err); toastr.error(err.message, 'Error on Generate'); } finally { if (lock) { From 0aede54cf1f69229ae6e7a1b681535e63f0082a3 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 11:10:24 +0000 Subject: [PATCH 05/13] Fix throw on /gen --- public/script.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/script.js b/public/script.js index 4228d646c..3acb6f33f 100644 --- a/public/script.js +++ b/public/script.js @@ -4415,7 +4415,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro if (data?.response) { toastr.error(data.response, 'API Error'); } - throw data?.response; + throw new Error(data?.response); } //const getData = await response.json(); From 620847c4d6a971e27c6b370cd31f24b927d07efc Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 11:12:41 +0000 Subject: [PATCH 06/13] Fixed logs --- public/scripts/slash-commands.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index c1215f310..08dd6744c 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -2261,7 +2261,7 @@ async function generateRawCallback(args, value) { const result = await generateRaw(value, '', isFalseBoolean(args?.instruct), quietToLoud, systemPrompt, length); return result; } catch (err) { - console.error('Error on /gen generation', err); + console.error('Error on /genraw generation', err); toastr.error(err.message, 'Error on Generate'); } finally { if (lock) { @@ -2296,7 +2296,7 @@ async function generateCallback(args, value) { const result = await generateQuietPrompt(value, quietToLoud, false, '', name, length); return result; } catch (err) { - console.error('Error on /genraw generation', err); + console.error('Error on /gen generation', err); toastr.error(err.message, 'Error on Generate'); } finally { if (lock) { From 8dd7a97eb37922a324d2610d28ec60b9aa6806db Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 11:22:12 +0000 Subject: [PATCH 07/13] Prevent duplicate toasts --- public/script.js | 2 +- public/scripts/slash-commands.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/public/script.js b/public/script.js index 3acb6f33f..e512ec774 100644 --- a/public/script.js +++ b/public/script.js @@ -4413,7 +4413,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro generatedPromptCache = ''; if (data?.response) { - toastr.error(data.response, 'API Error'); + toastr.error(data.response, 'API Error', { preventDuplicates: true }); } throw new Error(data?.response); } diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index 08dd6744c..201dcb122 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -2262,7 +2262,7 @@ async function generateRawCallback(args, value) { return result; } catch (err) { console.error('Error on /genraw generation', err); - toastr.error(err.message, 'Error on Generate'); + toastr.error(err.message, 'API Error', { preventDuplicates: true }); } finally { if (lock) { activateSendButtons(); @@ -2297,7 +2297,7 @@ async function generateCallback(args, value) { return result; } catch (err) { console.error('Error on /gen generation', err); - toastr.error(err.message, 'Error on Generate'); + toastr.error(err.message, 'API Error', { preventDuplicates: true }); } finally { if (lock) { activateSendButtons(); From 6d79cc015ab8e9049a4b09b0d197c30aa709ef09 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 19:44:12 +0300 Subject: [PATCH 08/13] Add OpenAI o1 --- package-lock.json | 9 +++++---- package.json | 2 +- public/index.html | 6 +++++- public/script.js | 7 ++++++- public/scripts/openai.js | 30 +++++++++++++++++++++++++++--- src/endpoints/tokenizers.js | 4 ++++ 6 files changed, 48 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index ca83e8446..abba30c5b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,7 +46,7 @@ "sanitize-filename": "^1.6.3", "sillytavern-transformers": "2.14.6", "simple-git": "^3.19.1", - "tiktoken": "^1.0.15", + "tiktoken": "^1.0.16", "vectra": "^0.2.2", "wavefile": "^11.0.0", "write-file-atomic": "^5.0.1", @@ -5751,9 +5751,10 @@ "license": "MIT" }, "node_modules/tiktoken": { - "version": "1.0.15", - "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.15.tgz", - "integrity": "sha512-sCsrq/vMWUSEW29CJLNmPvWxlVp7yh2tlkAjpJltIKqp5CKf98ZNpdeHRmAlPVFlGEbswDc6SmI8vz64W/qErw==" + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.16.tgz", + "integrity": "sha512-hRcORIGF2YlAgWx3nzrGJOrKSJwLoc81HpXmMQk89632XAgURc7IeV2FgQ2iXo9z/J96fCvpsHg2kWoHcbj9fg==", + "license": "MIT" }, "node_modules/timm": { "version": "1.7.1", diff --git a/package.json b/package.json index 87d7fa508..99526e2f1 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "sanitize-filename": "^1.6.3", "sillytavern-transformers": "2.14.6", "simple-git": "^3.19.1", - "tiktoken": "^1.0.15", + "tiktoken": "^1.0.16", "vectra": "^0.2.2", "wavefile": "^11.0.0", "write-file-atomic": "^5.0.1", diff --git a/public/index.html b/public/index.html index 6f8c87112..397781653 100644 --- a/public/index.html +++ b/public/index.html @@ -383,7 +383,7 @@ Max Response Length (tokens)
- +
@@ -2611,6 +2611,10 @@ + + + + diff --git a/public/script.js b/public/script.js index e512ec774..8ab8daa36 100644 --- a/public/script.js +++ b/public/script.js @@ -2862,7 +2862,12 @@ export function getCharacterCardFields() { export function isStreamingEnabled() { const noStreamSources = [chat_completion_sources.SCALE]; - return ((main_api == 'openai' && oai_settings.stream_openai && !noStreamSources.includes(oai_settings.chat_completion_source) && !(oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE && oai_settings.google_model.includes('bison'))) + return ( + (main_api == 'openai' && + oai_settings.stream_openai && + !noStreamSources.includes(oai_settings.chat_completion_source) && + !(oai_settings.chat_completion_source == chat_completion_sources.OPENAI && oai_settings.openai_model.startsWith('o1-')) && + !(oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE && oai_settings.google_model.includes('bison'))) || (main_api == 'kobold' && kai_settings.streaming_kobold && kai_flags.can_use_streaming) || (main_api == 'novel' && nai_settings.streaming_novel) || (main_api == 'textgenerationwebui' && textgen_settings.streaming)); diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 09bb23131..bd86df72a 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -1797,7 +1797,7 @@ async function sendOpenAIRequest(type, messages, signal) { const isQuiet = type === 'quiet'; const isImpersonate = type === 'impersonate'; const isContinue = type === 'continue'; - const stream = oai_settings.stream_openai && !isQuiet && !isScale && !(isGoogle && oai_settings.google_model.includes('bison')); + const stream = oai_settings.stream_openai && !isQuiet && !isScale && !(isGoogle && oai_settings.google_model.includes('bison')) && !(isOAI && oai_settings.openai_model.startsWith('o1-')); const useLogprobs = !!power_user.request_token_probabilities; const canMultiSwipe = oai_settings.n > 1 && !isContinue && !isImpersonate && !isQuiet && (isOAI || isCustom); @@ -1960,12 +1960,33 @@ async function sendOpenAIRequest(type, messages, signal) { generate_data['seed'] = oai_settings.seed; } - await eventSource.emit(event_types.CHAT_COMPLETION_SETTINGS_READY, generate_data); - if (isFunctionCallingSupported() && !stream) { await registerFunctionTools(type, generate_data); } + if (isOAI && oai_settings.openai_model.startsWith('o1-')) { + generate_data.messages.forEach((msg) => { + if (msg.role === 'system') { + msg.role = 'user'; + } + }); + delete generate_data.stream; + delete generate_data.logprobs; + delete generate_data.top_logprobs; + delete generate_data.n; + delete generate_data.temperature; + delete generate_data.top_p; + delete generate_data.frequency_penalty; + delete generate_data.presence_penalty; + delete generate_data.tools; + delete generate_data.tool_choice; + // IDK if it supports it and I have no way to test it + // delete generate_data.logit_bias; + // delete generate_data.stop; + } + + await eventSource.emit(event_types.CHAT_COMPLETION_SETTINGS_READY, generate_data); + const generate_url = '/api/backends/chat-completions/generate'; const response = await fetch(generate_url, { method: 'POST', @@ -3905,6 +3926,9 @@ function getMaxContextOpenAI(value) { if (oai_settings.max_context_unlocked) { return unlocked_max; } + else if (value.startsWith('o1-')) { + return max_128k; + } else if (value.includes('chatgpt-4o-latest') || value.includes('gpt-4-turbo') || value.includes('gpt-4o') || value.includes('gpt-4-1106') || value.includes('gpt-4-0125') || value.includes('gpt-4-vision')) { return max_128k; } diff --git a/src/endpoints/tokenizers.js b/src/endpoints/tokenizers.js index cd42348c9..89d530174 100644 --- a/src/endpoints/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -350,6 +350,10 @@ function getWebTokenizersChunks(tokenizer, ids) { * @returns {string} Tokenizer model to use */ function getTokenizerModel(requestModel) { + if (requestModel.includes('o1-preview') || requestModel.includes('o1-mini')) { + return 'gpt-4o'; + } + if (requestModel.includes('gpt-4o')) { return 'gpt-4o'; } From 6371b4929f30512fd455b28da181bcc6b13c9b06 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 19:47:53 +0300 Subject: [PATCH 09/13] Fix HTML spacing --- public/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/index.html b/public/index.html index 397781653..5e3b73790 100644 --- a/public/index.html +++ b/public/index.html @@ -383,7 +383,7 @@ Max Response Length (tokens)
- +
From 45cfc532b1fbd22f8107becc0a8c012bb0cfdf1e Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 19:56:48 +0300 Subject: [PATCH 10/13] Redirect max_tokens to max_completion_tokens --- public/scripts/openai.js | 2 ++ src/endpoints/backends/chat-completions.js | 1 + 2 files changed, 3 insertions(+) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index bd86df72a..5455637e4 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -1970,6 +1970,8 @@ async function sendOpenAIRequest(type, messages, signal) { msg.role = 'user'; } }); + generate_data.max_completion_tokens = generate_data.max_tokens; + delete generate_data.max_tokens; delete generate_data.stream; delete generate_data.logprobs; delete generate_data.top_logprobs; diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index 1757b9e5f..88541d937 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -965,6 +965,7 @@ router.post('/generate', jsonParser, function (request, response) { 'model': request.body.model, 'temperature': request.body.temperature, 'max_tokens': request.body.max_tokens, + 'max_completion_tokens': request.body.max_completion_tokens, 'stream': request.body.stream, 'presence_penalty': request.body.presence_penalty, 'frequency_penalty': request.body.frequency_penalty, From db2b3569422c934ff0603b071447d0a8da8c2bf4 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 20:47:41 +0300 Subject: [PATCH 11/13] Remove stop for o1 --- public/scripts/openai.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 5455637e4..246766a46 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -1982,9 +1982,9 @@ async function sendOpenAIRequest(type, messages, signal) { delete generate_data.presence_penalty; delete generate_data.tools; delete generate_data.tool_choice; + delete generate_data.stop; // IDK if it supports it and I have no way to test it // delete generate_data.logit_bias; - // delete generate_data.stop; } await eventSource.emit(event_types.CHAT_COMPLETION_SETTINGS_READY, generate_data); @@ -2134,7 +2134,6 @@ async function checkFunctionToolCalls(data) { const args = toolCall.function; console.log('Function tool call:', toolCall); await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args); - data.allowEmptyResponse = true; } } @@ -2148,7 +2147,6 @@ async function checkFunctionToolCalls(data) { /** @type {FunctionToolCall} */ const args = { name: content.name, arguments: JSON.stringify(content.input) }; await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args); - data.allowEmptyResponse = true; } } } @@ -2163,7 +2161,6 @@ async function checkFunctionToolCalls(data) { const args = { name: toolCall.name, arguments: JSON.stringify(toolCall.parameters) }; console.log('Function tool call:', toolCall); await eventSource.emit(event_types.LLM_FUNCTION_TOOL_CALL, args); - data.allowEmptyResponse = true; } } } From d99a5eba2ab2ac8b58c12442a09aa85a7adf5b2c Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 20:52:34 +0300 Subject: [PATCH 12/13] Remove empty message looping --- public/script.js | 83 ++++++++++++++++-------------------------------- 1 file changed, 27 insertions(+), 56 deletions(-) diff --git a/public/script.js b/public/script.js index 8ab8daa36..6d9860278 100644 --- a/public/script.js +++ b/public/script.js @@ -881,7 +881,6 @@ let abortController; //css var css_send_form_display = $('
').css('display'); -const MAX_GENERATION_LOOPS = 5; var kobold_horde_model = ''; @@ -2864,10 +2863,10 @@ export function isStreamingEnabled() { const noStreamSources = [chat_completion_sources.SCALE]; return ( (main_api == 'openai' && - oai_settings.stream_openai && - !noStreamSources.includes(oai_settings.chat_completion_source) && - !(oai_settings.chat_completion_source == chat_completion_sources.OPENAI && oai_settings.openai_model.startsWith('o1-')) && - !(oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE && oai_settings.google_model.includes('bison'))) + oai_settings.stream_openai && + !noStreamSources.includes(oai_settings.chat_completion_source) && + !(oai_settings.chat_completion_source == chat_completion_sources.OPENAI && oai_settings.openai_model.startsWith('o1-')) && + !(oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE && oai_settings.google_model.includes('bison'))) || (main_api == 'kobold' && kai_settings.streaming_kobold && kai_flags.can_use_streaming) || (main_api == 'novel' && nai_settings.streaming_novel) || (main_api == 'textgenerationwebui' && textgen_settings.streaming)); @@ -3342,11 +3341,11 @@ function removeLastMessage() { * @param {GenerateOptions} options Generation options * @param {boolean} dryRun Whether to actually generate a message or just assemble the prompt * @returns {Promise} Returns a promise that resolves when the text is done generating. - * @typedef {{automatic_trigger?: boolean, force_name2?: boolean, quiet_prompt?: string, quietToLoud?: boolean, skipWIAN?: boolean, force_chid?: number, signal?: AbortSignal, quietImage?: string, maxLoops?: number, quietName?: string }} GenerateOptions + * @typedef {{automatic_trigger?: boolean, force_name2?: boolean, quiet_prompt?: string, quietToLoud?: boolean, skipWIAN?: boolean, force_chid?: number, signal?: AbortSignal, quietImage?: string, quietName?: string }} GenerateOptions */ -export async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, maxLoops, quietName } = {}, dryRun = false) { +export async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName } = {}, dryRun = false) { console.log('Generate entered'); - await eventSource.emit(event_types.GENERATION_STARTED, type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, maxLoops }, dryRun); + await eventSource.emit(event_types.GENERATION_STARTED, type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage }, dryRun); setGenerationProgress(0); generation_started = new Date(); @@ -3408,7 +3407,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro if (selected_group && !is_group_generating) { if (!dryRun) { // Returns the promise that generateGroupWrapper returns; resolves when generation is done - return generateGroupWrapper(false, type, { quiet_prompt, force_chid, signal: abortController.signal, quietImage, maxLoops }); + return generateGroupWrapper(false, type, { quiet_prompt, force_chid, signal: abortController.signal, quietImage }); } const characterIndexMap = new Map(characters.map((char, index) => [char.avatar, index])); @@ -4440,53 +4439,30 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro const displayIncomplete = type === 'quiet' && !quietToLoud; getMessage = cleanUpMessage(getMessage, isImpersonate, isContinue, displayIncomplete); - if (getMessage.length > 0 || data.allowEmptyResponse) { - if (isImpersonate) { - $('#send_textarea').val(getMessage)[0].dispatchEvent(new Event('input', { bubbles: true })); - generatedPromptCache = ''; - await eventSource.emit(event_types.IMPERSONATE_READY, getMessage); - } - else if (type == 'quiet') { - unblockGeneration(type); - return getMessage; + if (isImpersonate) { + $('#send_textarea').val(getMessage)[0].dispatchEvent(new Event('input', { bubbles: true })); + generatedPromptCache = ''; + await eventSource.emit(event_types.IMPERSONATE_READY, getMessage); + } + else if (type == 'quiet') { + unblockGeneration(type); + return getMessage; + } + else { + // Without streaming we'll be having a full message on continuation. Treat it as a last chunk. + if (originalType !== 'continue') { + ({ type, getMessage } = await saveReply(type, getMessage, false, title, swipes)); } else { - // Without streaming we'll be having a full message on continuation. Treat it as a last chunk. - if (originalType !== 'continue') { - ({ type, getMessage } = await saveReply(type, getMessage, false, title, swipes)); - } - else { - ({ type, getMessage } = await saveReply('appendFinal', getMessage, false, title, swipes)); - } - - // This relies on `saveReply` having been called to add the message to the chat, so it must be last. - parseAndSaveLogprobs(data, continue_mag); + ({ type, getMessage } = await saveReply('appendFinal', getMessage, false, title, swipes)); } - if (type !== 'quiet') { - playMessageSound(); - } - } else { - // If maxLoops is not passed in (e.g. first time generating), set it to MAX_GENERATION_LOOPS - maxLoops ??= MAX_GENERATION_LOOPS; + // This relies on `saveReply` having been called to add the message to the chat, so it must be last. + parseAndSaveLogprobs(data, continue_mag); + } - if (maxLoops === 0) { - if (type !== 'quiet') { - throwCircuitBreakerError(); - } - throw new Error('Generate circuit breaker interruption'); - } - - // regenerate with character speech reenforced - // to make sure we leave on swipe type while also adding the name2 appendage - await delay(1000); - // A message was already deleted on regeneration, so instead treat is as a normal gen - if (type === 'regenerate') { - type = 'normal'; - } - // The first await is for waiting for the generate to start. The second one is waiting for it to finish - const result = await await Generate(type, { automatic_trigger, force_name2: true, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName, maxLoops: maxLoops - 1 }); - return result; + if (type !== 'quiet') { + playMessageSound(); } if (power_user.auto_swipe) { @@ -5259,11 +5235,6 @@ function getGenerateUrl(api) { } } -function throwCircuitBreakerError() { - callPopup(`Could not extract reply in ${MAX_GENERATION_LOOPS} attempts. Try generating again`, 'text'); - unblockGeneration(); -} - function extractTitleFromData(data) { if (main_api == 'koboldhorde') { return data.workerName; From eeee2068879731a28a5442e7dbd0d1e6f0d4bfe6 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 13 Sep 2024 21:03:04 +0300 Subject: [PATCH 13/13] Fix logit_bias comment --- public/scripts/openai.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 246766a46..0b86d8f5e 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -1983,7 +1983,7 @@ async function sendOpenAIRequest(type, messages, signal) { delete generate_data.tools; delete generate_data.tool_choice; delete generate_data.stop; - // IDK if it supports it and I have no way to test it + // It does support logit_bias, but the tokenizer used and its effect is yet unknown. // delete generate_data.logit_bias; }