#1980 Add raw summary prompt builder mode

public/scripts/extensions/memory/index.js

@@ -1,11 +1,25 @@
-import { getStringHash, debounce, waitUntilCondition, extractAllWords } from '../../utils.js';
+import { getStringHash, debounce, waitUntilCondition, extractAllWords, delay } from '../../utils.js';
 import { getContext, getApiUrl, extension_settings, doExtrasFetch, modules, renderExtensionTemplate } from '../../extensions.js';
-import { animation_duration, eventSource, event_types, extension_prompt_roles, extension_prompt_types, generateQuietPrompt, is_send_press, saveSettingsDebounced, substituteParams } from '../../../script.js';
+import {
+    activateSendButtons,
+    deactivateSendButtons,
+    animation_duration,
+    eventSource,
+    event_types,
+    extension_prompt_roles,
+    extension_prompt_types,
+    generateQuietPrompt,
+    is_send_press,
+    saveSettingsDebounced,
+    substituteParams,
+    generateRaw,
+    getMaxContextSize,
+} from '../../../script.js';
 import { is_group_generating, selected_group } from '../../group-chats.js';
 import { registerSlashCommand } from '../../slash-commands.js';
 import { loadMovingUIState } from '../../power-user.js';
 import { dragElement } from '../../RossAscends-mods.js';
-import { getTextTokens, tokenizers } from '../../tokenizers.js';
+import { getTextTokens, getTokenCount, tokenizers } from '../../tokenizers.js';
 export { MODULE_NAME };
 
 const MODULE_NAME = '1_memory';
@@ -39,7 +53,13 @@ const summary_sources = {
     'main': 'main',
 };
 
-const defaultPrompt = '[Pause your roleplay. Summarize the most important facts and events that have happened in the chat so far. If a summary already exists in your memory, use that as a base and expand with new facts. Limit the summary to {{words}} words or less. Your response should include nothing but the summary.]';
+const prompt_builders = {
+    DEFAULT: 0,
+    RAW_BLOCKING: 1,
+    RAW_NON_BLOCKING: 2,
+};
+
+const defaultPrompt = '[Pause your roleplay. Summarize the most important facts and events in the story so far. If a summary already exists in your memory, use that as a base and expand with new facts. Limit the summary to {{words}} words or less. Your response should include nothing but the summary.]';
 const defaultTemplate = '[Summary: {{summary}}]';
 
 const defaultSettings = {
@@ -57,12 +77,21 @@ const defaultSettings = {
     promptWordsStep: 25,
     promptInterval: 10,
     promptMinInterval: 0,
-    promptMaxInterval: 100,
+    promptMaxInterval: 250,
     promptIntervalStep: 1,
     promptForceWords: 0,
     promptForceWordsStep: 100,
     promptMinForceWords: 0,
     promptMaxForceWords: 10000,
+    overrideResponseLength: 0,
+    overrideResponseLengthMin: 0,
+    overrideResponseLengthMax: 4096,
+    overrideResponseLengthStep: 16,
+    maxMessagesPerRequest: 0,
+    maxMessagesPerRequestMin: 0,
+    maxMessagesPerRequestMax: 250,
+    maxMessagesPerRequestStep: 1,
+    prompt_builder: prompt_builders.RAW_BLOCKING,
 };
 
 function loadSettings() {
@@ -87,9 +116,50 @@ function loadSettings() {
     $('#memory_role').val(extension_settings.memory.role).trigger('input');
     $(`input[name="memory_position"][value="${extension_settings.memory.position}"]`).prop('checked', true).trigger('input');
     $('#memory_prompt_words_force').val(extension_settings.memory.promptForceWords).trigger('input');
+    $(`input[name="memory_prompt_builder"][value="${extension_settings.memory.prompt_builder}"]`).prop('checked', true).trigger('input');
+    $('#memory_override_response_length').val(extension_settings.memory.overrideResponseLength).trigger('input');
+    $('#memory_max_messages_per_request').val(extension_settings.memory.maxMessagesPerRequest).trigger('input');
     switchSourceControls(extension_settings.memory.source);
 }
 
+async function onPromptIntervalAutoClick() {
+    const context = getContext();
+    const maxPromptLength = getMaxContextSize(extension_settings.memory.overrideResponseLength);
+    const chat = context.chat;
+    const allMessages = chat.filter(m => !m.is_system && m.mes).map(m => m.mes);
+    const messagesWordCount = allMessages.map(m => extractAllWords(m)).flat().length;
+    const messagesTokenCount = getTokenCount(allMessages.join('\n'));
+    const tokensPerWord = messagesTokenCount / messagesWordCount;
+    const averageMessageTokenCount = messagesTokenCount / allMessages.length;
+    const targetSummaryTokens = Math.round(extension_settings.memory.promptWords * tokensPerWord);
+    const promptTokens = getTokenCount(extension_settings.memory.prompt);
+    const promptAllowance = maxPromptLength - promptTokens - targetSummaryTokens;
+    const maxMessagesPerSummary = extension_settings.memory.maxMessagesPerRequest || 0;
+    const averageMessagesPerPrompt = Math.floor(promptAllowance / averageMessageTokenCount);
+    const unfitMessages = maxMessagesPerSummary > 0 ? averageMessagesPerPrompt - maxMessagesPerSummary : 0;
+    const adjustedAverageMessagesPerPrompt = Math.max(1, averageMessagesPerPrompt - (unfitMessages > 0 ? Math.ceil(unfitMessages / 2) : 0));
+
+    console.table({
+        maxPromptLength,
+        promptAllowance,
+        targetSummaryTokens,
+        promptTokens,
+        messagesWordCount,
+        messagesTokenCount,
+        tokensPerWord,
+        averageMessageTokenCount,
+        averageMessagesPerPrompt,
+        adjustedAverageMessagesPerPrompt,
+        maxMessagesPerSummary,
+        unfitMessages,
+    });
+
+    const ROUNDING = 5;
+    extension_settings.memory.promptInterval = Math.max(1, Math.floor(adjustedAverageMessagesPerPrompt / ROUNDING) * ROUNDING);
+
+    $('#memory_prompt_interval').val(extension_settings.memory.promptInterval).trigger('input');
+}
+
 function onSummarySourceChange(event) {
     const value = event.target.value;
     extension_settings.memory.source = value;
@@ -130,6 +200,10 @@ function onMemoryPromptIntervalInput() {
     saveSettingsDebounced();
 }
 
+function onMemoryPromptRestoreClick() {
+    $('#memory_prompt').val(defaultPrompt).trigger('input');
+}
+
 function onMemoryPromptInput() {
     const value = $(this).val();
     extension_settings.memory.prompt = value;
@@ -171,6 +245,20 @@ function onMemoryPromptWordsForceInput() {
     saveSettingsDebounced();
 }
 
+function onOverrideResponseLengthInput() {
+    const value = $(this).val();
+    extension_settings.memory.overrideResponseLength = Number(value);
+    $('#memory_override_response_length_value').text(extension_settings.memory.overrideResponseLength);
+    saveSettingsDebounced();
+}
+
+function onMaxMessagesPerRequestInput() {
+    const value = $(this).val();
+    extension_settings.memory.maxMessagesPerRequest = Number(value);
+    $('#memory_max_messages_per_request_value').text(extension_settings.memory.maxMessagesPerRequest);
+    saveSettingsDebounced();
+}
+
 function saveLastValues() {
     const context = getContext();
     lastGroupId = context.groupId;
@@ -196,6 +284,22 @@ function getLatestMemoryFromChat(chat) {
     return '';
 }
 
+function getIndexOfLatestChatSummary(chat) {
+    if (!Array.isArray(chat) || !chat.length) {
+        return -1;
+    }
+
+    const reversedChat = chat.slice().reverse();
+    reversedChat.shift();
+    for (let mes of reversedChat) {
+        if (mes.extra && mes.extra.memory) {
+            return chat.indexOf(mes);
+        }
+    }
+
+    return -1;
+}
+
 async function onChatEvent() {
     // Module not enabled
     if (extension_settings.memory.source === summary_sources.extras) {
@@ -359,8 +463,41 @@ async function summarizeChatMain(context, force, skipWIAN) {
         console.debug('Summarization prompt is empty. Skipping summarization.');
         return;
     }
+
     console.log('sending summary prompt');
-    const summary = await generateQuietPrompt(prompt, false, skipWIAN);
+    let summary = '';
+    let index = null;
+
+    if (prompt_builders.DEFAULT === extension_settings.memory.prompt_builder) {
+        summary = await generateQuietPrompt(prompt, false, skipWIAN, '', '', extension_settings.memory.overrideResponseLength);
+    }
+
+    if ([prompt_builders.RAW_BLOCKING, prompt_builders.RAW_NON_BLOCKING].includes(extension_settings.memory.prompt_builder)) {
+        const lock = extension_settings.memory.prompt_builder === prompt_builders.RAW_BLOCKING;
+        try {
+            if (lock) {
+                deactivateSendButtons();
+            }
+
+            const { rawPrompt, lastUsedIndex } = await getRawSummaryPrompt(context, prompt);
+
+            if (lastUsedIndex === null || lastUsedIndex === -1) {
+                if (force) {
+                    toastr.info('To try again, remove the latest summary.', 'No messages found to summarize');
+                }
+
+                return null;
+            }
+
+            summary = await generateRaw(rawPrompt, '', false, false, prompt, extension_settings.memory.overrideResponseLength);
+            index = lastUsedIndex;
+        } finally {
+            if (lock) {
+                activateSendButtons();
+            }
+        }
+    }
+
     const newContext = getContext();
 
     // something changed during summarization request
@@ -371,10 +508,83 @@ async function summarizeChatMain(context, force, skipWIAN) {
         return;
     }
 
-    setMemoryContext(summary, true);
+    setMemoryContext(summary, true, index);
     return summary;
 }
 
+/**
+ * Get the raw summarization prompt from the chat context.
+ * @param {object} context ST context
+ * @param {string} prompt Summarization system prompt
+ * @returns {Promise<{rawPrompt: string, lastUsedIndex: number}>} Raw summarization prompt
+ */
+async function getRawSummaryPrompt(context, prompt) {
+    /**
+     * Get the memory string from the chat buffer.
+     * @param {boolean} includeSystem Include prompt into the memory string
+     * @returns {string} Memory string
+     */
+    function getMemoryString(includeSystem) {
+        const delimiter = '\n\n';
+        const stringBuilder = [];
+        const bufferString = chatBuffer.slice().join(delimiter);
+
+        if (includeSystem) {
+            stringBuilder.push(prompt);
+        }
+
+        if (latestSummary) {
+            stringBuilder.push(latestSummary);
+        }
+
+        stringBuilder.push(bufferString);
+
+        return stringBuilder.join(delimiter).trim();
+    }
+
+    const chat = context.chat.slice();
+    const latestSummary = getLatestMemoryFromChat(chat);
+    const latestSummaryIndex = getIndexOfLatestChatSummary(chat);
+    chat.pop(); // We always exclude the last message from the buffer
+    const chatBuffer = [];
+    const PADDING = 64;
+    const PROMPT_SIZE = getMaxContextSize(extension_settings.memory.overrideResponseLength);
+    let latestUsedMessage = null;
+
+    for (let index = latestSummaryIndex + 1; index < chat.length; index++) {
+        const message = chat[index];
+
+        if (!message) {
+            break;
+        }
+
+        if (message.is_system || !message.mes) {
+            continue;
+        }
+
+        const entry = `${message.name}:\n${message.mes}`;
+        chatBuffer.push(entry);
+
+        const tokens = getTokenCount(getMemoryString(true), PADDING);
+        await delay(1);
+
+        if (tokens > PROMPT_SIZE) {
+            chatBuffer.pop();
+            break;
+        }
+
+        latestUsedMessage = message;
+
+        if (extension_settings.memory.maxMessagesPerRequest > 0 && chatBuffer.length >= extension_settings.memory.maxMessagesPerRequest) {
+            break;
+        }
+    }
+
+    const lastUsedIndex = context.chat.indexOf(latestUsedMessage);
+    const rawPrompt = getMemoryString(false);
+    return { rawPrompt, lastUsedIndex };
+}
+
 async function summarizeChatExtras(context) {
     function getMemoryString() {
         return (longMemory + '\n\n' + memoryBuffer.slice().reverse().join('\n\n')).trim();
@@ -482,12 +692,24 @@ function onMemoryContentInput() {
     setMemoryContext(value, true);
 }
 
+function onMemoryPromptBuilderInput(e) {
+    const value = Number(e.target.value);
+    extension_settings.memory.prompt_builder = value;
+    saveSettingsDebounced();
+}
+
 function reinsertMemory() {
-    const existingValue = $('#memory_contents').val();
+    const existingValue = String($('#memory_contents').val());
     setMemoryContext(existingValue, false);
 }
 
-function setMemoryContext(value, saveToMessage) {
+/**
+ * Set the summary value to the context and save it to the chat message extra.
+ * @param {string} value Value of a summary
+ * @param {boolean} saveToMessage Should the summary be saved to the chat message extra
+ * @param {number|null} index Index of the chat message to save the summary to. If null, the pre-last message is used.
+ */
+function setMemoryContext(value, saveToMessage, index = null) {
     const context = getContext();
     context.setExtensionPrompt(MODULE_NAME, formatMemoryValue(value), extension_settings.memory.position, extension_settings.memory.depth, false, extension_settings.memory.role);
     $('#memory_contents').val(value);
@@ -497,7 +719,7 @@ function setMemoryContext(value, saveToMessage) {
     console.debug('Role: ' + extension_settings.memory.role);
 
     if (saveToMessage && context.chat.length) {
-        const idx = context.chat.length - 2;
+        const idx = index ?? context.chat.length - 2;
         const mes = context.chat[idx < 0 ? 0 : idx];
 
         if (!mes.extra) {
@@ -573,6 +795,13 @@ function setupListeners() {
     $('#memory_role').off('click').on('input', onMemoryRoleInput);
     $('input[name="memory_position"]').off('click').on('change', onMemoryPositionChange);
     $('#memory_prompt_words_force').off('click').on('input', onMemoryPromptWordsForceInput);
+    $('#memory_prompt_builder_default').off('click').on('input', onMemoryPromptBuilderInput);
+    $('#memory_prompt_builder_raw_blocking').off('click').on('input', onMemoryPromptBuilderInput);
+    $('#memory_prompt_builder_raw_non_blocking').off('click').on('input', onMemoryPromptBuilderInput);
+    $('#memory_prompt_restore').off('click').on('click', onMemoryPromptRestoreClick);
+    $('#memory_prompt_interval_auto').off('click').on('click', onPromptIntervalAutoClick);
+    $('#memory_override_response_length').off('click').on('input', onOverrideResponseLengthInput);
+    $('#memory_max_messages_per_request').off('click').on('input', onMaxMessagesPerRequestInput);
     $('#summarySettingsBlockToggle').off('click').on('click', function () {
         console.log('saw settings button click');
         $('#summarySettingsBlock').slideToggle(200, 'swing'); //toggleClass("hidden");

public/scripts/extensions/memory/settings.html

@@ -36,9 +36,67 @@
                     </div>
                 </div>
                 <div id="summarySettingsBlock" style="display:none;">
+                    <div data-summary-source="main">
+                        <label>
+                            Prompt builder
+                        </label>
+                        <label class="checkbox_label" for="memory_prompt_builder_raw_blocking" title="Extension will build its own prompt using messages that were not summarized yet. Blocks the chat until the summary is generated.">
+                            <input id="memory_prompt_builder_raw_blocking" type="radio" name="memory_prompt_builder" value="1" />
+                            <span>Raw, blocking</span>
+                        </label>
+                        <label class="checkbox_label" for="memory_prompt_builder_raw_non_blocking" title="Extension will build its own prompt using messages that were not summarized yet. Does not block the chat while the summary is being generated. Not all backends support this mode.">
+                            <input id="memory_prompt_builder_raw_non_blocking" type="radio" name="memory_prompt_builder" value="2" />
+                            <span>Raw, non-blocking</span>
+                        </label>
+                        <label class="checkbox_label" id="memory_prompt_builder_default" title="Extension will use the regular main prompt builder and add the summary request to it as the last system message.">
+                            <input id="memory_prompt_builder_default" type="radio" name="memory_prompt_builder" value="0" />
+                            <span>Classic, blocking</span>
+                        </label>
+                    </div>
+                    <div data-summary-source="main">
+                        <label for="memory_prompt" class="title_restorable">
+                            <span data-i18n="Summary Prompt">Summary Prompt</span>
+                            <div id="memory_prompt_restore" title="Restore default prompt" class="right_menu_button">
+                                <div class="fa-solid fa-clock-rotate-left"></div>
+                            </div>
+                        </label>
+                        <textarea id="memory_prompt" class="text_pole textarea_compact" rows="6" placeholder="This prompt will be sent to AI to request the summary generation. &lcub;&lcub;words&rcub;&rcub; will resolve to the 'Number of words' parameter."></textarea>
+                        <label for="memory_prompt_words">Target summary length (<span id="memory_prompt_words_value"></span> words)</label>
+                        <input id="memory_prompt_words" type="range" value="{{defaultSettings.promptWords}}" min="{{defaultSettings.promptMinWords}}" max="{{defaultSettings.promptMaxWords}}" step="{{defaultSettings.promptWordsStep}}" />
+                        <label for="memory_override_response_length">
+                            API response length (<span id="memory_override_response_length_value"></span> tokens)
+                            <small class="memory_disabled_hint">0 = default</small>
+                        </label>
+                        <input id="memory_override_response_length" type="range" value="{{defaultSettings.overrideResponseLength}}" min="{{defaultSettings.overrideResponseLengthMin}}" max="{{defaultSettings.overrideResponseLengthMax}}" step="{{defaultSettings.overrideResponseLengthStep}}" />
+                        <label for="memory_max_messages_per_request">
+                            [Raw] Max messages per request (<span id="memory_max_messages_per_request_value"></span>)
+                            <small class="memory_disabled_hint">0 = unlimited</small>
+                        </label>
+                        <input id="memory_max_messages_per_request" type="range" value="{{defaultSettings.maxMessagesPerRequest}}" min="{{defaultSettings.maxMessagesPerRequestMin}}" max="{{defaultSettings.maxMessagesPerRequestMax}}" step="{{defaultSettings.maxMessagesPerRequestStep}}" />
+                        <h4 data-i18n="Update frequency" class="textAlignCenter">
+                            Update frequency
+                        </h4>
+                        <label for="memory_prompt_interval" class="title_restorable">
+                            <span>
+                                Update every <span id="memory_prompt_interval_value"></span> messages
+                                <small class="memory_disabled_hint">0 = disable</small>
+                            </span>
+                            <div id="memory_prompt_interval_auto" title="Try to automatically adjust the interval based on the chat metrics." class="right_menu_button">
+                                <div class="fa-solid fa-wand-magic-sparkles"></div>
+                            </div>
+                        </label>
+                        <input id="memory_prompt_interval" type="range" value="{{defaultSettings.promptInterval}}" min="{{defaultSettings.promptMinInterval}}" max="{{defaultSettings.promptMaxInterval}}" step="{{defaultSettings.promptIntervalStep}}" />
+                        <label for="memory_prompt_words_force">
+                            Update every <span id="memory_prompt_words_force_value"></span> words
+                            <small class="memory_disabled_hint">0 = disable</small>
+                        </label>
+                        <input id="memory_prompt_words_force" type="range" value="{{defaultSettings.promptForceWords}}" min="{{defaultSettings.promptMinForceWords}}" max="{{defaultSettings.promptMaxForceWords}}" step="{{defaultSettings.promptForceWordsStep}}" />
+                        <small>If both sliders are non-zero, then both will trigger summary updates at their respective intervals.</small>
+                        <hr>
+                    </div>
                     <div class="memory_template">
-                        <label for="memory_template">Insertion Template</label>
+                        <label for="memory_template">Injection Template</label>
-                        <textarea id="memory_template" class="text_pole textarea_compact" rows="2" placeholder="&lcub;&lcub;summary&lcub;&lcub; will resolve to the current summary contents."></textarea>
+                        <textarea id="memory_template" class="text_pole textarea_compact" rows="2" placeholder="&lcub;&lcub;summary&rcub;&rcub; will resolve to the current summary contents."></textarea>
                     </div>
                     <label for="memory_position">Injection Position</label>
                     <div class="radio_group">
@@ -61,23 +119,6 @@
                             </select>
                         </label>
                     </div>
-                    <div data-summary-source="main" class="memory_contents_controls">
-                    </div>
-                    <div data-summary-source="main">
-                        <label for="memory_prompt" class="title_restorable">
-                            Summary Prompt
-                        </label>
-                        <textarea id="memory_prompt" class="text_pole textarea_compact" rows="6" placeholder="This prompt will be sent to AI to request the summary generation. &lcub;&lcub;words&rcub;&rcub; will resolve to the 'Number of words' parameter."></textarea>
-                        <label for="memory_prompt_words">Summary length (<span id="memory_prompt_words_value"></span> words)</label>
-                        <input id="memory_prompt_words" type="range" value="{{defaultSettings.promptWords}}" min="{{defaultSettings.promptMinWords}}" max="{{defaultSettings.promptMaxWords}}" step="{{defaultSettings.promptWordsStep}}" />
-                        <label for="memory_prompt_interval">Update every <span id="memory_prompt_interval_value"></span> messages</label>
-                        <small>0 = disable</small>
-                        <input id="memory_prompt_interval" type="range" value="{{defaultSettings.promptInterval}}" min="{{defaultSettings.promptMinInterval}}" max="{{defaultSettings.promptMaxInterval}}" step="{{defaultSettings.promptIntervalStep}}" />
-                        <label for="memory_prompt_words_force">Update every <span id="memory_prompt_words_force_value"></span> words</label>
-                        <small>0 = disable</small>
-                        <input id="memory_prompt_words_force" type="range" value="{{defaultSettings.promptForceWords}}" min="{{defaultSettings.promptMinForceWords}}" max="{{defaultSettings.promptMaxForceWords}}" step="{{defaultSettings.promptForceWordsStep}}" />
-                        <small>If both sliders are non-zero, then both will trigger summary updates a their respective intervals.</small>
-                    </div>
                 </div>
             </div>
         </div>

public/scripts/extensions/memory/style.css

@@ -25,3 +25,13 @@ label[for="memory_frozen"] input {
     align-items: center;
     justify-content: space-between;
 }
+
+.memory_disabled_hint {
+    margin-left: 2px;
+}
+
+#summarySettingsBlock {
+    display: flex;
+    flex-direction: column;
+    row-gap: 5px;
+}