diff --git a/public/script.js b/public/script.js index 149359d01..f818e97b5 100644 --- a/public/script.js +++ b/public/script.js @@ -190,6 +190,7 @@ let exportPopper = Popper.createPopper(document.getElementById('export_button'), }); let dialogueResolve = null; let chat_metadata = {}; +let streamingProcessor = null; const durationSaveEdit = 200; const saveSettingsDebounced = debounce(() => saveSettings(), durationSaveEdit); @@ -1105,6 +1106,86 @@ function appendToStoryString(value, prefix) { return ''; } +function isStreamingEnabled() { + return (main_api == 'openai' && oai_settings.stream_openai); +} + +class StreamingProcessor { + onStartStreaming(text) { + saveReply(type, text); + return (count_view_mes - 1); + } + + onProgressStreaming(messageId, text) { + let processedText = cleanUpMessage(text); + ({isName, processedText} = extractNameFromMessage(processedText, force_name2)); + chat[messageId]['is_name'] = isName; + chat[messageId]['mes'] = processedText; + let formattedText = messageFormating(processedText, chat[messageId].name, chat[messageId].is_system, chat[messageId].force_avatar); + const mesText = $(`#chat .mes[mesid="${messageId}"] .mes_text`); + mesText.empty(); + mesText.append(formattedText); + } + + onFinishStreaming(messageId, text) { + this.onProgressStreaming(messageId, text); + playMessageSound(); + saveChatConditional(); + activateSendButtons(); + showSwipeButtons(); + setGenerationProgress(0); + $('.mes_edit:last').show(); + } + + onErrorStreaming() { + $("#send_textarea").removeAttr('disabled'); + is_send_press = false; + activateSendButtons(); + setGenerationProgress(0); + } + + onStopStreaming() { + this.onErrorStreaming(); + } + + nullStreamingGeneration() { + throw new Error('Generation function for streaming is not hooked up'); + } + + constructor() { + this.result = ""; + this.messageId = -1; + this.isStopped = false; + this.isFinished = false; + this.generator = this.nullStreamingGeneration; + } + + async generate() { + this.messageId = this.onStartStreaming(''); + + for await (const text of this.generator()) { + if (this.isStopped) { + this.onStopStreaming(); + return; + } + + try { + this.result = text; + this.onProgressStreaming(this.messageId, text); + } + catch (err) { + console.error(err); + this.onErrorStreaming(); + this.isStopped = true; + return; + } + } + + this.isFinished = true; + this.onFinishStreaming(this.messageId, this.result); + } +} + async function Generate(type, automatic_trigger, force_name2) { console.log('Generate entered'); setGenerationProgress(0); @@ -1696,10 +1777,18 @@ async function Generate(type, automatic_trigger, force_name2) { } console.log('rungenerate calling API'); + streamingProcessor = new StreamingProcessor(); if (main_api == 'openai') { let prompt = await prepareOpenAIMessages(name2, storyString, worldInfoBefore, worldInfoAfter, extension_prompt, promptBias); - sendOpenAIRequest(prompt).then(onSuccess).catch(onError); + + if (isStreamingEnabled()) { + streamingProcessor.generator = () => sendOpenAIRequest(prompt); + await streamingProcessor.generate(); + } + else { + sendOpenAIRequest(prompt).then(onSuccess).catch(onError); + } } else if (main_api == 'kobold' && horde_settings.use_horde) { generateHorde(finalPromt, generate_data).then(onSuccess).catch(onError); @@ -1729,32 +1818,7 @@ async function Generate(type, automatic_trigger, force_name2) { is_send_press = false; if (!data.error) { //const getData = await response.json(); - var getMessage = ""; - if (main_api == 'kobold' && !horde_settings.use_horde) { - getMessage = data.results[0].text; - } - else if (main_api == 'kobold' && horde_settings.use_horde) { - getMessage = data; - } - else if (main_api == 'textgenerationwebui') { - getMessage = data.data[0]; - if (getMessage == null || data.error) { - activateSendButtons(); - callPopup('

Got empty response from Text generation web UI. Try restarting the API with recommended options.

', 'text'); - return; - } - getMessage = getMessage.substring(finalPromt.length); - } - else if (main_api == 'novel') { - getMessage = data.output; - } - if (main_api == 'openai' || main_api == 'poe') { - getMessage = data; - } - - if (power_user.collapse_newlines) { - getMessage = collapseNewlines(getMessage); - } + let getMessage = extractMessageFromData(data, finalPromt); //Pygmalion run again // to make it continue generating so long as it's under max_amount and hasn't signaled @@ -1776,50 +1840,21 @@ async function Generate(type, automatic_trigger, force_name2) { } //Formating - getMessage = $.trim(getMessage); - if (is_pygmalion) { - getMessage = getMessage.replace(//g, name1); - getMessage = getMessage.replace(//g, name2); - getMessage = getMessage.replace(/You:/g, name1 + ':'); - } - if (getMessage.indexOf(name1 + ":") != -1) { - getMessage = getMessage.substr(0, getMessage.indexOf(name1 + ":")); + getMessage = cleanUpMessage(getMessage); - } - if (getMessage.indexOf('<|endoftext|>') != -1) { - getMessage = getMessage.substr(0, getMessage.indexOf('<|endoftext|>')); - - } - // clean-up group message from excessive generations - if (selected_group) { - getMessage = cleanGroupMessage(getMessage); - } - let this_mes_is_name = true; - if (getMessage.indexOf(name2 + ":") === 0) { - getMessage = getMessage.replace(name2 + ':', ''); - getMessage = getMessage.trimStart(); - } else { - this_mes_is_name = false; - } - if (force_name2) this_mes_is_name = true; + let this_mes_is_name; + ({ this_mes_is_name, getMessage } = extractNameFromMessage(getMessage, force_name2)); //getMessage = getMessage.replace(/^\s+/g, ''); if (getMessage.length > 0) { ({ type, getMessage } = saveReply(type, getMessage, this_mes_is_name)); + activateSendButtons(); playMessageSound(); generate_loop_counter = 0; } else { ++generate_loop_counter; if (generate_loop_counter > MAX_GENERATION_LOOPS) { - callPopup(`Could not extract reply in ${MAX_GENERATION_LOOPS} attempts. Try generating again`, 'text'); - generate_loop_counter = 0; - $("#send_textarea").removeAttr('disabled'); - is_send_press = false; - activateSendButtons(); - setGenerationProgress(0); - showSwipeButtons(); - $('.mes_edit:last').show(); - throw new Error('Generate circuit breaker interruption'); + throwCircuitBreakerError(); } // regenerate with character speech reenforced @@ -1837,7 +1872,6 @@ async function Generate(type, automatic_trigger, force_name2) { console.log('/savechat called by /Generate'); saveChatConditional(); - activateSendButtons(); showSwipeButtons(); setGenerationProgress(0); @@ -1866,6 +1900,89 @@ async function Generate(type, automatic_trigger, force_name2) { console.log('generate ending'); } //generate ends +function extractNameFromMessage(getMessage, force_name2) { + let this_mes_is_name = true; + if (getMessage.indexOf(name2 + ":") === 0) { + getMessage = getMessage.replace(name2 + ':', ''); + getMessage = getMessage.trimStart(); + } else { + this_mes_is_name = false; + } + if (force_name2) + this_mes_is_name = true; + return { this_mes_is_name, getMessage }; +} + +function throwCircuitBreakerError() { + callPopup(`Could not extract reply in ${MAX_GENERATION_LOOPS} attempts. Try generating again`, 'text'); + generate_loop_counter = 0; + $("#send_textarea").removeAttr('disabled'); + is_send_press = false; + activateSendButtons(); + setGenerationProgress(0); + showSwipeButtons(); + $('.mes_edit:last').show(); + throw new Error('Generate circuit breaker interruption'); +} + +function extractMessageFromData(data, finalPromt) { + let getMessage = ""; + + if (main_api == 'kobold' && !horde_settings.use_horde) { + getMessage = data.results[0].text; + } + + if (main_api == 'kobold' && horde_settings.use_horde) { + getMessage = data; + } + + if (main_api == 'textgenerationwebui') { + getMessage = data.data[0]; + if (getMessage == null || data.error) { + activateSendButtons(); + callPopup('

Got empty response from Text generation web UI. Try restarting the API with recommended options.

', 'text'); + return; + } + getMessage = getMessage.substring(finalPromt.length); + } + + if (main_api == 'novel') { + getMessage = data.output; + } + + if (main_api == 'openai' || main_api == 'poe') { + getMessage = data; + } + + return getMessage; +} + +function cleanUpMessage(getMessage) { + if (power_user.collapse_newlines) { + getMessage = collapseNewlines(getMessage); + } + + getMessage = $.trim(getMessage); + if (is_pygmalion) { + getMessage = getMessage.replace(//g, name1); + getMessage = getMessage.replace(//g, name2); + getMessage = getMessage.replace(/You:/g, name1 + ':'); + } + if (getMessage.indexOf(name1 + ":") != -1) { + getMessage = getMessage.substr(0, getMessage.indexOf(name1 + ":")); + + } + if (getMessage.indexOf('<|endoftext|>') != -1) { + getMessage = getMessage.substr(0, getMessage.indexOf('<|endoftext|>')); + + } + // clean-up group message from excessive generations + if (selected_group) { + getMessage = cleanGroupMessage(getMessage); + } + return getMessage; +} + function saveReply(type, getMessage, this_mes_is_name) { if (chat.length && (chat[chat.length - 1]['swipe_id'] === undefined || chat[chat.length - 1]['is_user'])) { @@ -1905,8 +2022,6 @@ function saveReply(type, getMessage, this_mes_is_name) { } //console.log('runGenerate calls addOneMessage'); addOneMessage(chat[chat.length - 1]); - - activateSendButtons(); } return { type, getMessage }; } diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 577a8b787..2ed483089 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -453,51 +453,30 @@ async function sendOpenAIRequest(openai_msgs_tosend) { } // Unused -function onStream(e, resolve, reject, last_view_mes) { - let end = false; - if (!oai_settings.stream_openai) +async function* onStream(e) { + if (!oai_settings.stream_openai) { return; + } + let response = e.currentTarget.response; + if (response == "{\"error\":true}") { - reject('', 'error'); + throw new Error('error during streaming'); } let eventList = response.split("\n"); let getMessage = ""; + for (let event of eventList) { if (!event.startsWith("data")) continue; if (event == "data: [DONE]") { - chat[chat.length - 1]['mes'] = getMessage; - $("#send_but").css("display", "block"); - $("#loading_mes").css("display", "none"); - saveChat(); - end = true; - break; + return getMessage; } let data = JSON.parse(event.substring(6)); // the first and last messages are undefined, protect against that getMessage += data.choices[0]["delta"]["content"] || ""; - } - - if ($("#chat").children().filter(`[mesid="${last_view_mes}"]`).length == 0) { - chat[chat.length] = {}; - chat[chat.length - 1]['name'] = name2; - chat[chat.length - 1]['is_user'] = false; - chat[chat.length - 1]['is_name'] = false; - chat[chat.length - 1]['send_date'] = Date.now(); - chat[chat.length - 1]['mes'] = ""; - addOneMessage(chat[chat.length - 1]); - } - - let messageText = messageFormating($.trim(getMessage), name1); - $("#chat").children().filter(`[mesid="${last_view_mes}"]`).children('.mes_block').children('.mes_text').html(messageText); - - let $textchat = $('#chat'); - $textchat.scrollTop($textchat[0].scrollHeight); - - if (end) { - resolve(); + yield getMessage; } }