Merge branch 'staging' into ruRuJan

public/scripts/extensions/vectors/index.js

@@ -30,6 +30,8 @@ import { textgen_types, textgenerationwebui_settings } from '../../textgen-setti
 import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
 import { SlashCommand } from '../../slash-commands/SlashCommand.js';
 import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '../../slash-commands/SlashCommandArgument.js';
+import { SlashCommandEnumValue, enumTypes } from '../../slash-commands/SlashCommandEnumValue.js';
+import { slashCommandReturnHelper } from '../../slash-commands/SlashCommandReturnHelper.js';
 import { callGenericPopup, POPUP_RESULT, POPUP_TYPE } from '../../popup.js';
 import { generateWebLlmChatPrompt, isWebLlmSupported } from '../shared.js';
 
@@ -1613,25 +1615,55 @@ jQuery(async () => {
         callback: async (args, query) => {
             const clamp = (v) => Number.isNaN(v) ? null : Math.min(1, Math.max(0, v));
             const threshold = clamp(Number(args?.threshold ?? settings.score_threshold));
+            const validateCount = (v) => Number.isNaN(v) || !Number.isInteger(v) || v < 1 ? null : v;
+            const count = validateCount(Number(args?.count)) ?? settings.chunk_count_db;
             const source = String(args?.source ?? '');
             const attachments = source ? getDataBankAttachmentsForSource(source, false) : getDataBankAttachments(false);
             const collectionIds = await ingestDataBankAttachments(String(source));
-            const queryResults = await queryMultipleCollections(collectionIds, String(query), settings.chunk_count_db, threshold);
-
-            // Map collection IDs to file URLs
+            const queryResults = await queryMultipleCollections(collectionIds, String(query), count, threshold);
+    
+            // Get URLs
             const urls = Object
                 .keys(queryResults)
                 .map(x => attachments.find(y => getFileCollectionId(y.url) === x))
                 .filter(x => x)
                 .map(x => x.url);
+    
+            // Gets the actual text content of chunks
+            const getChunksText = () => {
+                let textResult = '';
+                for (const collectionId in queryResults) {
+                    const metadata = queryResults[collectionId].metadata?.filter(x => x.text)?.sort((a, b) => a.index - b.index)?.map(x => x.text)?.filter(onlyUnique) || [];
+                    textResult += metadata.join('\n') + '\n\n';
+                }
+                return textResult;
+            };
+            
+            if (args.return === 'chunks') {
+                return getChunksText();
+            }
 
-            return JSON.stringify(urls);
+            // @ts-ignore
+            return slashCommandReturnHelper.doReturn(args.return ?? 'object', urls, { objectToStringFunc: list => list.join('\n') });
+            
         },
         aliases: ['databank-search', 'data-bank-search'],
         helpString: 'Search the Data Bank for a specific query using vector similarity. Returns a list of file URLs with the most relevant content.',
         namedArgumentList: [
             new SlashCommandNamedArgument('threshold', 'Threshold for the similarity score in the [0, 1] range. Uses the global config value if not set.', ARGUMENT_TYPE.NUMBER, false, false, ''),
+            new SlashCommandNamedArgument('count', 'Maximum number of query results to return.', ARGUMENT_TYPE.NUMBER, false, false, ''),
             new SlashCommandNamedArgument('source', 'Optional filter for the attachments by source.', ARGUMENT_TYPE.STRING, false, false, '', ['global', 'character', 'chat']),
+            SlashCommandNamedArgument.fromProps({
+                name: 'return',
+                description: 'How you want the return value to be provided',
+                typeList: [ARGUMENT_TYPE.STRING],
+                defaultValue: 'object',
+                enumList: [
+                    new SlashCommandEnumValue('chunks', 'Return the actual content chunks', enumTypes.enum, '{}'),
+                    ...slashCommandReturnHelper.enumList({ allowObject: true })
+                ],
+                forceEnum: true,
+            })
         ],
         unnamedArgumentList: [
             new SlashCommandArgument('Query to search by.', ARGUMENT_TYPE.STRING, true, false),

public/scripts/macros.js

@@ -26,6 +26,12 @@ Handlebars.registerHelper('helperMissing', function () {
  * @typedef {(nonce: string) => string} MacroFunction
  */
 
+/**
+ * @typedef {Object} CustomMacro
+ * @property {string} key - Macro name (key)
+ * @property {string} description - Optional description of the macro
+ */
+
 export class MacrosParser {
     /**
      * A map of registered macros.
@@ -33,12 +39,29 @@ export class MacrosParser {
      */
     static #macros = new Map();
 
+    /**
+     * A map of macro descriptions.
+     * @type {Map<string, string>}
+     */
+    static #descriptions = new Map();
+
+    /**
+     * Returns an iterator over all registered macros.
+     * @returns {IterableIterator<CustomMacro>}
+     */
+    static [Symbol.iterator] = function* () {
+        for (const macro of MacrosParser.#macros.keys()) {
+            yield { key: macro, description: MacrosParser.#descriptions.get(macro) };
+        }
+    };
+
     /**
      * Registers a global macro that can be used anywhere where substitution is allowed.
      * @param {string} key Macro name (key)
      * @param {string|MacroFunction} value A string or a function that returns a string
+     * @param {string} [description] Optional description of the macro
      */
-    static registerMacro(key, value) {
+    static registerMacro(key, value, description = '') {
         if (typeof key !== 'string') {
             throw new Error('Macro key must be a string');
         }
@@ -64,6 +87,10 @@ export class MacrosParser {
         }
 
         this.#macros.set(key, value);
+
+        if (typeof description === 'string' && description) {
+            this.#descriptions.set(key, description);
+        }
     }
 
     /**
@@ -88,6 +115,8 @@ export class MacrosParser {
         if (!deleted) {
             console.warn(`Macro ${key} was not registered`);
         }
+
+        this.#descriptions.delete(key);
     }
 
     /**

public/scripts/slash-commands/SlashCommandParser.js

@@ -22,6 +22,8 @@ import { SlashCommandBreakPoint } from './SlashCommandBreakPoint.js';
 import { SlashCommandDebugController } from './SlashCommandDebugController.js';
 import { commonEnumProviders } from './SlashCommandCommonEnumsProvider.js';
 import { SlashCommandBreak } from './SlashCommandBreak.js';
+import { MacrosParser } from '../macros.js';
+import { t } from '../i18n.js';
 
 /** @typedef {import('./SlashCommand.js').NamedArgumentsCapture} NamedArgumentsCapture */
 /** @typedef {import('./SlashCommand.js').NamedArguments} NamedArguments */
@@ -494,6 +496,10 @@ export class SlashCommandParser {
                     li.querySelector('tt').textContent,
                     (li.querySelector('tt').remove(),li.innerHTML),
                 ));
+                for (const macro of MacrosParser) {
+                    if (options.find(it => it.name === macro.key)) continue;
+                    options.push(new MacroAutoCompleteOption(macro.key, `{{${macro.key}}}`, macro.description || t`No description provided`));
+                }
                 const result = new AutoCompleteNameResult(
                     macro.name,
                     macro.start + 2,