From af38971a0149667fc1f73da3dc1cd77817aa6558 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 9 Sep 2023 22:15:47 +0300 Subject: [PATCH] Delete vectors on deleting chats --- public/script.js | 16 ++++++++-- public/scripts/extensions/vectors/index.js | 27 +++++++++++++++++ public/scripts/group-chats.js | 10 +++++++ src/openai-vectors.js | 3 +- src/vectors.js | 35 ++++++++++++++++++++-- 5 files changed, 86 insertions(+), 5 deletions(-) diff --git a/public/script.js b/public/script.js index 91d84ca44..2c1b09b0c 100644 --- a/public/script.js +++ b/public/script.js @@ -285,7 +285,9 @@ export const event_types = { CHARACTER_EDITED: 'character_edited', USER_MESSAGE_RENDERED: 'user_message_rendered', CHARACTER_MESSAGE_RENDERED: 'character_message_rendered', - FORCE_SET_BACKGROUND: 'force_set_background,' + FORCE_SET_BACKGROUND: 'force_set_background', + CHAT_DELETED : 'chat_deleted', + GROUP_CHAT_DELETED: 'group_chat_deleted', } export const eventSource = new EventEmitter(); @@ -1106,10 +1108,12 @@ async function delChat(chatfile) { }); if (response.ok === true) { // choose another chat if current was deleted - if (chatfile.replace('.jsonl', '') === characters[this_chid].chat) { + const name = chatfile.replace('.jsonl', ''); + if (name === characters[this_chid].chat) { chat_metadata = {}; await replaceCurrentChat(); } + await eventSource.emit(event_types.CHAT_DELETED, name); } } @@ -6891,6 +6895,7 @@ export async function handleDeleteCharacter(popup_type, this_chid, delete_chats) const avatar = characters[this_chid].avatar; const name = characters[this_chid].name; + const pastChats = await getPastCharacterChats(); const msg = { avatar_url: avatar, delete_chats: delete_chats }; @@ -6903,6 +6908,13 @@ export async function handleDeleteCharacter(popup_type, this_chid, delete_chats) if (response.ok) { await deleteCharacter(name, avatar); + + if (delete_chats) { + for (const chat of pastChats) { + const name = chat.file_name.replace('.jsonl', ''); + await eventSource.emit(event_types.CHAT_DELETED, name); + } + } } else { console.error('Failed to delete character: ', response.status, response.statusText); } diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index a1addc4f4..1383ab46a 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -323,6 +323,31 @@ async function queryCollection(collectionId, searchText, topK) { return results; } +async function purgeVectorIndex(collectionId) { + try { + if (!settings.enabled) { + return; + } + + const response = await fetch('/api/vector/purge', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ + collectionId: collectionId, + }), + }); + + if (!response.ok) { + throw new Error(`Could not delete vector index for collection ${collectionId}`); + } + + console.log(`Vectors: Purged vector index for collection ${collectionId}`); + + } catch (error) { + console.error('Vectors: Failed to purge', error); + } +} + jQuery(async () => { if (!extension_settings.vectors) { extension_settings.vectors = settings; @@ -381,4 +406,6 @@ jQuery(async () => { eventSource.on(event_types.MESSAGE_SENT, onChatEvent); eventSource.on(event_types.MESSAGE_RECEIVED, onChatEvent); eventSource.on(event_types.MESSAGE_SWIPED, onChatEvent); + eventSource.on(event_types.CHAT_DELETED, purgeVectorIndex); + eventSource.on(event_types.GROUP_CHAT_DELETED, purgeVectorIndex); }); diff --git a/public/scripts/group-chats.js b/public/scripts/group-chats.js index d204579cf..ddac1ef48 100644 --- a/public/scripts/group-chats.js +++ b/public/scripts/group-chats.js @@ -816,12 +816,20 @@ function activateNaturalOrder(members, input, lastMessage, allowSelfResponses, i } async function deleteGroup(id) { + const group = groups.find((x) => x.id === id); + const response = await fetch("/deletegroup", { method: "POST", headers: getRequestHeaders(), body: JSON.stringify({ id: id }), }); + if (group && Array.isArray(group.chats)) { + for (const chatId of group.chats) { + await eventSource.emit(event_types.GROUP_CHAT_DELETED, chatId); + } + } + if (response.ok) { selected_group = null; delete tag_map[id]; @@ -1493,6 +1501,8 @@ export async function deleteGroupChat(groupId, chatId) { } else { await createNewGroupChat(groupId); } + + await eventSource.emit(event_types.GROUP_CHAT_DELETED, chatId); } } diff --git a/src/openai-vectors.js b/src/openai-vectors.js index f81590f88..b7eb89162 100644 --- a/src/openai-vectors.js +++ b/src/openai-vectors.js @@ -27,7 +27,8 @@ async function getOpenAIVector(text) { }); if (!response.ok) { - console.log('OpenAI request failed'); + const text = await response.text(); + console.log('OpenAI request failed', response.statusText, text); throw new Error('OpenAI request failed'); } diff --git a/src/vectors.js b/src/vectors.js index 15a7c3871..d42a812f9 100644 --- a/src/vectors.js +++ b/src/vectors.js @@ -24,12 +24,13 @@ async function getVector(source, text) { * Gets the index for the vector collection * @param {string} collectionId - The collection ID * @param {string} source - The source of the vector + * @param {boolean} create - Whether to create the index if it doesn't exist * @returns {Promise} - The index for the collection */ -async function getIndex(collectionId, source) { +async function getIndex(collectionId, source, create = true) { const index = new vectra.LocalIndex(path.join(process.cwd(), 'vectors', sanitize(source), sanitize(collectionId))); - if (!await index.isIndexCreated()) { + if (create && !await index.isIndexCreated()) { await index.createIndex(); } @@ -185,6 +186,36 @@ async function registerEndpoints(app, jsonParser) { return res.sendStatus(500); } }); + + app.post('/api/vector/purge', jsonParser, async (req, res) => { + try { + if (!req.body.collectionId) { + return res.sendStatus(400); + } + + const collectionId = String(req.body.collectionId); + + const sources = ['local', 'openai']; + for (const source of sources) { + const index = await getIndex(collectionId, source, false); + + const exists = await index.isIndexCreated(); + + if (!exists) { + continue; + } + + const path = index.folderPath; + await index.deleteIndex(); + console.log(`Deleted vector index at ${path}`); + } + + return res.sendStatus(200); + } catch (error) { + console.error(error); + return res.sendStatus(500); + } + }); } module.exports = { registerEndpoints };