From d2803f6451b7d80cb9817f99d45d0187ca0a2d77 Mon Sep 17 00:00:00 2001 From: Mark Ceter <133643956+maceter@users.noreply.github.com> Date: Wed, 17 May 2023 17:58:58 +0000 Subject: [PATCH 1/6] [WIP} infinity context --- public/script.js | 4 +- public/scripts/extensions.js | 15 ++++ .../extensions/infinity-context/index.js | 75 +++++++++++++++++++ .../extensions/infinity-context/manifest.json | 14 ++++ .../extensions/infinity-context/style.css | 0 5 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 public/scripts/extensions/infinity-context/index.js create mode 100644 public/scripts/extensions/infinity-context/manifest.json create mode 100644 public/scripts/extensions/infinity-context/style.css diff --git a/public/script.js b/public/script.js index 439c9799e..e03bfd575 100644 --- a/public/script.js +++ b/public/script.js @@ -107,7 +107,7 @@ import { } from "./scripts/poe.js"; import { debounce, delay, restoreCaretPosition, saveCaretPosition, end_trim_to_sentence } from "./scripts/utils.js"; -import { extension_settings, loadExtensionSettings } from "./scripts/extensions.js"; +import { extension_settings, loadExtensionSettings, runGenerationInterceptors } from "./scripts/extensions.js"; import { executeSlashCommands, getSlashCommandsHelp, registerSlashCommand } from "./scripts/slash-commands.js"; import { tag_map, @@ -1651,6 +1651,8 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject, setGenerationProgress(0); tokens_already_generated = 0; generation_started = new Date(); + + await runGenerationInterceptors(); const isImpersonate = type == "impersonate"; const isInstruct = power_user.instruct.enabled; diff --git a/public/scripts/extensions.js b/public/scripts/extensions.js index e5dfd8f15..4ffd249a6 100644 --- a/public/scripts/extensions.js +++ b/public/scripts/extensions.js @@ -4,6 +4,7 @@ export { getContext, getApiUrl, loadExtensionSettings, + runGenerationInterceptors, defaultRequestArgs, modules, extension_settings, @@ -26,6 +27,7 @@ const extension_settings = { dice: {}, tts: {}, sd: {}, + chromadb: {}, }; let modules = []; @@ -317,6 +319,19 @@ async function loadExtensionSettings(settings) { } } +async function runGenerationInterceptors() { + for (const manifest of Object.values(manifests)) { + const interceptorKey = manifest.generate_interceptor; + if (typeof window[interceptorKey] === 'function') { + try { + await window[interceptorKey](); + } catch(e) { + console.error(`Failed running interceptor for ${manifest.display_name}`, e); + } + } + } +} + $(document).ready(async function () { setTimeout(function () { addExtensionsButtonAndMenu(); }, 100) $("#extensions_connect").on('click', connectClickHandler); diff --git a/public/scripts/extensions/infinity-context/index.js b/public/scripts/extensions/infinity-context/index.js new file mode 100644 index 000000000..2f4371f0d --- /dev/null +++ b/public/scripts/extensions/infinity-context/index.js @@ -0,0 +1,75 @@ +import { + saveSettingsDebounced, +} from "../../../script.js"; +import { getApiUrl, getContext, extension_settings, defaultRequestArgs } from "../../extensions.js"; +export { MODULE_NAME, chromadb_interceptGeneration }; + +const MODULE_NAME = 'chromadb'; + +const defaultSettings = { + keep_context: 10, + keep_context_min: 1, + keep_context_max: 100, + keep_context_step: 1, + + n_results: 20, + n_results_min: 1, + n_results_max: 100, + n_results_step: 1, +} + +async function loadSettings() { + if (Object.keys(extension_settings.chromadb).length === 0) { + Object.assign(extension_settings.chromadb, defaultSettings); + } + + $('#chromadb_keep_context').val(extension_settings.chromadb.keep_context).trigger('input'); + $('#chromadb_n_results').val(extension_settings.chromadb.n_results).trigger('input'); +} + +function onKeepContextInput() { + extension_settings.chromadb.keep_context = Number($('#chromadb_keep_context').val()); + $('#chromadb_keep_context_value').text(extension_settings.chromadb.keep_context); + saveSettingsDebounced(); +} + +function onNResultsInput() { + extension_settings.chromadb.n_results = Number($('#chromadb_n_results').val()); + $('#chromadb_n_results_value').text(extension_settings.chromadb.n_results); + saveSettingsDebounced(); +} + +async function moduleWorker() { + // ??? +} + +setInterval(moduleWorker, UPDATE_INTERVAL); + +window.chromadb_interceptGeneration = async () => { + const context = getContext(); + + // TODO substitute context +} + +jQuery(async () => { + const settingsHtml = ` +
+
+
+ Infinity Context +
+
+
+ + + + +
+
`; + + $('#extensions_settings').append(settingsHtml); + $('#chromadb_keep_context').on('input', onKeepContextInput); + $('#chromadb_n_results').on('input', onNResultsInput); + + await loadSettings(); +}); \ No newline at end of file diff --git a/public/scripts/extensions/infinity-context/manifest.json b/public/scripts/extensions/infinity-context/manifest.json new file mode 100644 index 000000000..463e68199 --- /dev/null +++ b/public/scripts/extensions/infinity-context/manifest.json @@ -0,0 +1,14 @@ +{ + "display_name": "Infinity Context", + "loading_order": 11, + "requires": [ + "chromadb" + ], + "optional": [], + "generate_interceptor": "chromadb_interceptGeneration", + "js": "index.js", + "css": "style.css", + "author": "maceter636@proton.me", + "version": "1.0.0", + "homePage": "https://github.com/Cohee1207/SillyTavern" +} \ No newline at end of file diff --git a/public/scripts/extensions/infinity-context/style.css b/public/scripts/extensions/infinity-context/style.css new file mode 100644 index 000000000..e69de29bb From 64ac2b2f58487d187071e4495d5b93c28016368a Mon Sep 17 00:00:00 2001 From: Mark Ceter <133643956+maceter@users.noreply.github.com> Date: Wed, 17 May 2023 18:33:14 +0000 Subject: [PATCH 2/6] Form new chat context --- public/script.js | 10 ++ .../extensions/infinity-context/index.js | 92 +++++++++++++++++-- 2 files changed, 96 insertions(+), 6 deletions(-) diff --git a/public/script.js b/public/script.js index e03bfd575..c5438d457 100644 --- a/public/script.js +++ b/public/script.js @@ -169,6 +169,7 @@ export { getStoppingStrings, getStatus, reloadMarkdownProcessor, + getCurrentChatId, chat, this_chid, selected_button, @@ -485,6 +486,15 @@ function reloadMarkdownProcessor(render_formulas = false) { return converter; } +function getCurrentChatId() { + if (selected_group) { + return groups.find(x => x.id == selected_group)?.chat_id; + } + else if (this_chid) { + return characters[this_chid].chat; + } +} + const CHARACTERS_PER_TOKEN_RATIO = 3.35; const talkativeness_default = 0.5; diff --git a/public/scripts/extensions/infinity-context/index.js b/public/scripts/extensions/infinity-context/index.js index 2f4371f0d..06b01a694 100644 --- a/public/scripts/extensions/infinity-context/index.js +++ b/public/scripts/extensions/infinity-context/index.js @@ -1,7 +1,5 @@ -import { - saveSettingsDebounced, -} from "../../../script.js"; -import { getApiUrl, getContext, extension_settings, defaultRequestArgs } from "../../extensions.js"; +import { chat, saveSettingsDebounced, getCurrentChatId } from "../../../script.js"; +import { getApiUrl, extension_settings, getContext } from "../../extensions.js"; export { MODULE_NAME, chromadb_interceptGeneration }; const MODULE_NAME = 'chromadb'; @@ -16,7 +14,12 @@ const defaultSettings = { n_results_min: 1, n_results_max: 100, n_results_step: 1, -} +}; + +const postHeaders = { + 'Content-Type': 'application/json', + 'Bypass-Tunnel-Reminder': 'bypass', +}; async function loadSettings() { if (Object.keys(extension_settings.chromadb).length === 0) { @@ -43,12 +46,89 @@ async function moduleWorker() { // ??? } +async function addMessages(chat_id, messages) { + const url = new URL(getApiUrl()); + url.pathname = '/api/chroma'; + + const transformedMessages = messages.map((m, id) => ({ + id: id, + role: m.is_user ? 'user' : 'assistant', + content: m.mes, + date: m.send_date, + })); + + const addMessagesResult = await fetch(url, { + method: 'POST', + headers: postHeaders, + body: JSON.stringify({ chat_id, messages: transformedMessages }), + }); + + if (addMessagesResult.ok) { + const addMessagesData = await addMessagesResult.json(); + + return addMessagesData; // { count: 1 } + } + + return { count: 0 }; +} + +async function queryMessages(chat_id, query) { + const url = new URL(getApiUrl()); + url.pathname = '/api/chroma/query'; + + const queryMessagesResult = await fetch(url, { + method: 'POST', + headers: postHeaders, + body: JSON.stringify({ chat_id, query, n_results: extension_settings.chromadb.n_results }), + }); + + if (queryMessagesResult.ok) { + const queryMessagesData = await queryMessagesResult.json(); + + return queryMessagesData; + } + + return []; +} + setInterval(moduleWorker, UPDATE_INTERVAL); window.chromadb_interceptGeneration = async () => { const context = getContext(); + const currentChatId = getCurrentChatId(); - // TODO substitute context + if (currentChatId) { + const messagesToStore = chat.slice(0, -extension_settings.chromadb.keep_context); + const messagesToKeep = chat.slice(-extension_settings.chromadb.keep_context); + + if (messagesToStore.length > 0) { + await addMessages(currentChatId, messagesToStore); + } + + const lastMessage = messagesToKeep[messagesToKeep.length - 1]; + + if (lastMessage) { + const queriedMessages = await queryMessages(currentChatId, lastMessage.mes); + + queriedMessages.sort((a, b) => a.date - b.date); + + const newChat = [ + ...queriedMessages.map(m => ({ + name: m.role === 'user' ? context.name1: context.name2, + is_user: m.role === 'user', + is_name: true, + send_date: m.date, + mes: m.content, + })), + ...messagesToKeep, + ]; + + console.log(newChat); + + // TODO replace current chat temporarily somehow... + + } + } } jQuery(async () => { From c675f09c549fbad8d75ea1da7370670a91bae744 Mon Sep 17 00:00:00 2001 From: Mark Ceter <133643956+maceter@users.noreply.github.com> Date: Wed, 17 May 2023 18:39:34 +0000 Subject: [PATCH 3/6] Store original message as metadata --- public/scripts/extensions/infinity-context/index.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/public/scripts/extensions/infinity-context/index.js b/public/scripts/extensions/infinity-context/index.js index 06b01a694..0858e4a18 100644 --- a/public/scripts/extensions/infinity-context/index.js +++ b/public/scripts/extensions/infinity-context/index.js @@ -55,6 +55,7 @@ async function addMessages(chat_id, messages) { role: m.is_user ? 'user' : 'assistant', content: m.mes, date: m.send_date, + meta: JSON.stringify(m), })); const addMessagesResult = await fetch(url, { @@ -113,13 +114,7 @@ window.chromadb_interceptGeneration = async () => { queriedMessages.sort((a, b) => a.date - b.date); const newChat = [ - ...queriedMessages.map(m => ({ - name: m.role === 'user' ? context.name1: context.name2, - is_user: m.role === 'user', - is_name: true, - send_date: m.date, - mes: m.content, - })), + ...queriedMessages.map(m => JSON.parse(m.meta)), ...messagesToKeep, ]; From 5c7e14c287d7799f4a38fbed15b6d9c390633f3e Mon Sep 17 00:00:00 2001 From: Mark Ceter <133643956+maceter@users.noreply.github.com> Date: Sun, 21 May 2023 10:41:18 +0000 Subject: [PATCH 4/6] Split messages into chunks --- .../extensions/infinity-context/index.js | 52 +++++++++++++------ public/scripts/utils.js | 31 +++++++++++ 2 files changed, 66 insertions(+), 17 deletions(-) diff --git a/public/scripts/extensions/infinity-context/index.js b/public/scripts/extensions/infinity-context/index.js index 0858e4a18..eae7ad4ca 100644 --- a/public/scripts/extensions/infinity-context/index.js +++ b/public/scripts/extensions/infinity-context/index.js @@ -1,5 +1,6 @@ -import { chat, saveSettingsDebounced, getCurrentChatId } from "../../../script.js"; -import { getApiUrl, extension_settings, getContext } from "../../extensions.js"; +import { saveSettingsDebounced, getCurrentChatId } from "../../../script.js"; +import { getApiUrl, extension_settings } from "../../extensions.js"; +import { splitRecursive } from "../../utils.js"; export { MODULE_NAME, chromadb_interceptGeneration }; const MODULE_NAME = 'chromadb'; @@ -14,6 +15,11 @@ const defaultSettings = { n_results_min: 1, n_results_max: 100, n_results_step: 1, + + split_length: 384, + split_length_min: 64, + split_length_max: 4096, + split_length_step: 64, }; const postHeaders = { @@ -28,6 +34,7 @@ async function loadSettings() { $('#chromadb_keep_context').val(extension_settings.chromadb.keep_context).trigger('input'); $('#chromadb_n_results').val(extension_settings.chromadb.n_results).trigger('input'); + $('#chromadb_split_length').val(extension_settings.chromadb.split_length).trigger('input'); } function onKeepContextInput() { @@ -42,16 +49,32 @@ function onNResultsInput() { saveSettingsDebounced(); } -async function moduleWorker() { - // ??? +function onSplitLengthInput() { + extension_settings.chromadb.split_length = Number($('#chromadb_split_length').val()); + $('#chromadb_split_length_value').text(extension_settings.chromadb.split_length); + saveSettingsDebounced(); } async function addMessages(chat_id, messages) { const url = new URL(getApiUrl()); url.pathname = '/api/chroma'; - const transformedMessages = messages.map((m, id) => ({ - id: id, + const messagesDeepCopy = JSON.parse(JSON.stringify(messages)); + const splittedMessages = []; + + let id = 0; + messagesDeepCopy.forEach(m => { + const split = splitRecursive(m.mes, extension_settings.chromadb.split_length); + splittedMessages.push(...split.map(text => ({ + ...m, + mes: text, + send_date: id, + id: `msg-${id++}`, + }))); + }); + + const transformedMessages = splittedMessages.map((m) => ({ + id: m.id, role: m.is_user ? 'user' : 'assistant', content: m.mes, date: m.send_date, @@ -92,15 +115,11 @@ async function queryMessages(chat_id, query) { return []; } -setInterval(moduleWorker, UPDATE_INTERVAL); - -window.chromadb_interceptGeneration = async () => { - const context = getContext(); +window.chromadb_interceptGeneration = async (chat) => { const currentChatId = getCurrentChatId(); if (currentChatId) { const messagesToStore = chat.slice(0, -extension_settings.chromadb.keep_context); - const messagesToKeep = chat.slice(-extension_settings.chromadb.keep_context); if (messagesToStore.length > 0) { await addMessages(currentChatId, messagesToStore); @@ -113,15 +132,11 @@ window.chromadb_interceptGeneration = async () => { queriedMessages.sort((a, b) => a.date - b.date); - const newChat = [ - ...queriedMessages.map(m => JSON.parse(m.meta)), - ...messagesToKeep, - ]; + const newChat = queriedMessages.map(m => JSON.parse(m.meta)); console.log(newChat); - // TODO replace current chat temporarily somehow... - + chat.splice(0, messagesToStore.length, newChat); } } } @@ -139,12 +154,15 @@ jQuery(async () => { + +
`; $('#extensions_settings').append(settingsHtml); $('#chromadb_keep_context').on('input', onKeepContextInput); $('#chromadb_n_results').on('input', onNResultsInput); + $('#chromadb_split_length').on('input', onSplitLengthInput); await loadSettings(); }); \ No newline at end of file diff --git a/public/scripts/utils.js b/public/scripts/utils.js index 77763b536..158f95471 100644 --- a/public/scripts/utils.js +++ b/public/scripts/utils.js @@ -231,4 +231,35 @@ export function countOccurrences(string, character) { export function isOdd(number) { return number % 2 !== 0; +} + +/** Split string to parts no more than length in size */ +export function splitRecursive(input, length, delimitiers = ['\n\n', '\n', ' ', '']) { + const delim = delimitiers[0] ?? ''; + const parts = input.split(delim); + + const flatParts = parts.flatMap(p => { + if (p.length < length) return p; + return splitRecursive(input, length, delimitiers.slice(1)); + }); + + // Merge short chunks + const result = []; + let currentChunk = ''; + for (let i = 0; i < flatParts.length;) { + currentChunk = flatParts[i]; + let j = i + 1; + while (j < flatParts.length) { + const nextChunk = flatParts[j]; + if (currentChunk.length + nextChunk.length + delim.length <= length) { + currentChunk += delim + nextChunk; + } else { + break; + } + j++; + } + i = j; + result.push(currentChunk); + } + return result; } \ No newline at end of file From f2ecac0d792c8d43d5c19e28ad9f2a7cedd485bd Mon Sep 17 00:00:00 2001 From: Mark Ceter <133643956+maceter@users.noreply.github.com> Date: Sun, 21 May 2023 10:52:11 +0000 Subject: [PATCH 5/6] Fixes --- public/scripts/extensions/infinity-context/index.js | 4 ++-- public/scripts/utils.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/public/scripts/extensions/infinity-context/index.js b/public/scripts/extensions/infinity-context/index.js index eae7ad4ca..97af2dec6 100644 --- a/public/scripts/extensions/infinity-context/index.js +++ b/public/scripts/extensions/infinity-context/index.js @@ -1,7 +1,7 @@ import { saveSettingsDebounced, getCurrentChatId } from "../../../script.js"; import { getApiUrl, extension_settings } from "../../extensions.js"; import { splitRecursive } from "../../utils.js"; -export { MODULE_NAME, chromadb_interceptGeneration }; +export { MODULE_NAME }; const MODULE_NAME = 'chromadb'; @@ -125,7 +125,7 @@ window.chromadb_interceptGeneration = async (chat) => { await addMessages(currentChatId, messagesToStore); } - const lastMessage = messagesToKeep[messagesToKeep.length - 1]; + const lastMessage = chat[chat.length - 1]; if (lastMessage) { const queriedMessages = await queryMessages(currentChatId, lastMessage.mes); diff --git a/public/scripts/utils.js b/public/scripts/utils.js index 158f95471..915c60096 100644 --- a/public/scripts/utils.js +++ b/public/scripts/utils.js @@ -262,4 +262,4 @@ export function splitRecursive(input, length, delimitiers = ['\n\n', '\n', ' ', result.push(currentChunk); } return result; -} \ No newline at end of file +} From f62c26a9bad2c192d2db0f0a5f96a1681c4ac5df Mon Sep 17 00:00:00 2001 From: Mark Ceter <133643956+maceter@users.noreply.github.com> Date: Sun, 21 May 2023 11:33:10 +0000 Subject: [PATCH 6/6] First working version --- .../extensions/infinity-context/index.js | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/public/scripts/extensions/infinity-context/index.js b/public/scripts/extensions/infinity-context/index.js index 97af2dec6..f813e70b5 100644 --- a/public/scripts/extensions/infinity-context/index.js +++ b/public/scripts/extensions/infinity-context/index.js @@ -57,7 +57,7 @@ function onSplitLengthInput() { async function addMessages(chat_id, messages) { const url = new URL(getApiUrl()); - url.pathname = '/api/chroma'; + url.pathname = '/api/chromadb'; const messagesDeepCopy = JSON.parse(JSON.stringify(messages)); const splittedMessages = []; @@ -98,7 +98,7 @@ async function addMessages(chat_id, messages) { async function queryMessages(chat_id, query) { const url = new URL(getApiUrl()); - url.pathname = '/api/chroma/query'; + url.pathname = '/api/chromadb/query'; const queryMessagesResult = await fetch(url, { method: 'POST', @@ -123,20 +123,20 @@ window.chromadb_interceptGeneration = async (chat) => { if (messagesToStore.length > 0) { await addMessages(currentChatId, messagesToStore); - } - const lastMessage = chat[chat.length - 1]; + const lastMessage = chat[chat.length - 1]; - if (lastMessage) { - const queriedMessages = await queryMessages(currentChatId, lastMessage.mes); + if (lastMessage) { + const queriedMessages = await queryMessages(currentChatId, lastMessage.mes); - queriedMessages.sort((a, b) => a.date - b.date); + queriedMessages.sort((a, b) => a.date - b.date); - const newChat = queriedMessages.map(m => JSON.parse(m.meta)); - - console.log(newChat); - - chat.splice(0, messagesToStore.length, newChat); + const newChat = queriedMessages.map(m => JSON.parse(m.meta)); + + chat.splice(0, messagesToStore.length, ...newChat); + + console.log('ChromaDB chat after injection', chat); + } } } } @@ -165,4 +165,4 @@ jQuery(async () => { $('#chromadb_split_length').on('input', onSplitLengthInput); await loadSettings(); -}); \ No newline at end of file +});