rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'staging' into tooltips-vol1

Browse Source
This commit is contained in:
Juha Jeronen
2024-01-24 17:05:09 +02:00
parent 12d3ab92af 532c86b051
commit 0683d7a538
19 changed files with 514 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
public/scripts/extensions/regex/editor.html
View File

@@ -76,7 +76,7 @@
</div>
<div class="flex-container">
<div class="wi-enter-footer-text flex-container flexFlowColumn flexNoGap alignitemsstart">
<div class="flex1 wi-enter-footer-text flex-container flexFlowColumn flexNoGap alignitemsstart">
<small>Affects</small>
<div>
<label class="checkbox flex-container">
@@ -97,7 +97,7 @@
</label>
</div>
</div>
<div class="wi-enter-footer-text flex-container flexFlowColumn flexNoGap alignitemsstart">
<div class="flex1 wi-enter-footer-text flex-container flexFlowColumn flexNoGap alignitemsstart">
<small>Other Options</small>
<label class="checkbox flex-container">
<input type="checkbox" name="disabled" />
@@ -120,13 +120,6 @@
<span data-i18n="Substitute Regex">Substitute Regex (?)</span>
</label>
</div>
<div class="flex-container flexFlowColumn alignitemsstart">
<small>Replacement Strategy</small>
<select name="replace_strategy_select" class="margin0">
<option value="0">Replace</option>
<option value="1">Overlay (currently broken)</option>
</select>
</div>
</div>
</div>
</div>

91
public/scripts/extensions/regex/engine.js
View File

@@ -19,14 +19,6 @@ const regex_placement = {
SLASH_COMMAND: 3,
};
/**
* @enum {number} How the regex script should replace the matched string
*/
const regex_replace_strategy = {
REPLACE: 0,
OVERLAY: 1,
};
/**
* Instantiates a regular expression from a string.
* @param {string} input The input string.
@@ -153,86 +145,3 @@ function filterString(rawString, trimStrings, { characterOverride } = {}) {
return finalString;
}
/**
* Substitutes regex-specific and normal parameters
* @param {string} rawString
* @param {string} regexMatch
* @param {RegexSubstituteParams} params The parameters to use for the regex substitution
* @returns {string} The substituted string
* @typedef {{characterOverride?: string, replaceStrategy?: number}} RegexSubstituteParams The parameters to use for the regex substitution
*/
function substituteRegexParams(rawString, regexMatch, { characterOverride, replaceStrategy } = {}) {
let finalString = rawString;
finalString = substituteParams(finalString, undefined, characterOverride);
let overlaidMatch = regexMatch;
// TODO: Maybe move the for loops into a separate function?
if (replaceStrategy === regex_replace_strategy.OVERLAY) {
const splitReplace = finalString.split('{{match}}');
// There's a prefix
if (splitReplace[0]) {
// Fetch the prefix
const splicedPrefix = spliceSymbols(splitReplace[0], false);
// Sequentially remove all occurrences of prefix from start of split
const splitMatch = overlaidMatch.split(splicedPrefix);
let sliceNum = 0;
for (let index = 0; index < splitMatch.length; index++) {
if (splitMatch[index].length === 0) {
sliceNum++;
} else {
break;
}
}
overlaidMatch = splitMatch.slice(sliceNum, splitMatch.length).join(splicedPrefix);
}
// There's a suffix
if (splitReplace[1]) {
// Fetch the suffix
const splicedSuffix = spliceSymbols(splitReplace[1], true);
// Sequential removal of all suffix occurrences from end of split
const splitMatch = overlaidMatch.split(splicedSuffix);
let sliceNum = 0;
for (let index = splitMatch.length - 1; index >= 0; index--) {
if (splitMatch[index].length === 0) {
sliceNum++;
} else {
break;
}
}
overlaidMatch = splitMatch.slice(0, splitMatch.length - sliceNum).join(splicedSuffix);
}
}
// Only one match is replaced. This is by design
finalString = finalString.replace('{{match}}', overlaidMatch) || finalString.replace('{{match}}', regexMatch);
return finalString;
}
/**
* Splices common sentence symbols and whitespace from the beginning and end of a string.
* Using a for loop due to sequential ordering.
* @param {string} rawString The raw string to splice
* @param {boolean} isSuffix String is a suffix
* @returns {string} The spliced string
*/
function spliceSymbols(rawString, isSuffix) {
let offset = 0;
for (const ch of isSuffix ? rawString.split('').reverse() : rawString) {
if (ch.match(/[^\w.,?'!]/)) {
offset++;
} else {
break;
}
}
return isSuffix ? rawString.substring(0, rawString.length - offset) : rawString.substring(offset);
}

9
public/scripts/extensions/regex/index.js
View File

@@ -141,9 +141,6 @@ async function onRegexEditorOpenClick(existingId) {
editorHtml
.find('input[name="substitute_regex"]')
.prop('checked', existingScript.substituteRegex ?? false);
editorHtml
.find('select[name="replace_strategy_select"]')
.val(existingScript.replaceStrategy ?? 0);
existingScript.placement.forEach((element) => {
editorHtml
@@ -181,7 +178,6 @@ async function onRegexEditorOpenClick(existingId) {
replaceString: editorHtml.find('.regex_replace_string').val(),
trimStrings: String(editorHtml.find('.regex_trim_strings').val()).split('\n').filter((e) => e.length !== 0) || [],
substituteRegex: editorHtml.find('input[name="substitute_regex"]').prop('checked'),
replaceStrategy: Number(editorHtml.find('select[name="replace_strategy_select"]').find(':selected').val()) ?? 0,
};
const rawTestString = String(editorHtml.find('#regex_test_input').val());
const result = runRegexScript(testScript, rawTestString);
@@ -224,11 +220,6 @@ async function onRegexEditorOpenClick(existingId) {
editorHtml
.find('input[name="substitute_regex"]')
.prop('checked'),
replaceStrategy:
parseInt(editorHtml
.find('select[name="replace_strategy_select"]')
.find(':selected')
.val()) ?? 0,
};
saveRegexScript(newRegexScript, existingScriptIndex);

51
public/scripts/extensions/vectors/index.js
View File

@@ -1,5 +1,5 @@
import { eventSource, event_types, extension_prompt_types, getCurrentChatId, getRequestHeaders, is_send_press, saveSettingsDebounced, setExtensionPrompt, substituteParams } from '../../../script.js';
import { ModuleWorkerWrapper, extension_settings, getContext, renderExtensionTemplate } from '../../extensions.js';
import { ModuleWorkerWrapper, extension_settings, getContext, modules, renderExtensionTemplate } from '../../extensions.js';
import { collapseNewlines } from '../../power-user.js';
import { SECRET_KEYS, secret_state } from '../../secrets.js';
import { debounce, getStringHash as calculateHash, waitUntilCondition, onlyUnique, splitRecursive } from '../../utils.js';
@@ -152,8 +152,25 @@ async function synchronizeChat(batchSize = 5) {
return newVectorItems.length - batchSize;
} catch (error) {
/**
* Gets the error message for a given cause
* @param {string} cause Error cause key
* @returns {string} Error message
*/
function getErrorMessage(cause) {
switch (cause) {
case 'api_key_missing':
return 'API key missing. Save it in the "API Connections" panel.';
case 'extras_module_missing':
return 'Extras API must provide an "embeddings" module.';
default:
return 'Check server console for more details';
}
}
console.error('Vectors: Failed to synchronize chat', error);
const message = error.cause === 'api_key_missing' ? 'API key missing. Save it in the "API Connections" panel.' : 'Check server console for more details';
const message = getErrorMessage(error.cause);
toastr.error(message, 'Vectorization failed');
return -1;
} finally {
@@ -411,6 +428,18 @@ async function getSavedHashes(collectionId) {
return hashes;
}
/**
* Add headers for the Extras API source.
* @param {object} headers Headers object
*/
function addExtrasHeaders(headers) {
console.log(`Vector source is extras, populating API URL: ${extension_settings.apiUrl}`);
Object.assign(headers, {
'X-Extras-Url': extension_settings.apiUrl,
'X-Extras-Key': extension_settings.apiKey,
});
}
/**
* Inserts vector items into a collection
* @param {string} collectionId - The collection to insert into
@@ -424,9 +453,18 @@ async function insertVectorItems(collectionId, items) {
throw new Error('Vectors: API key missing', { cause: 'api_key_missing' });
}
if (settings.source === 'extras' && !modules.includes('embeddings')) {
throw new Error('Vectors: Embeddings module missing', { cause: 'extras_module_missing' });
}
const headers = getRequestHeaders();
if (settings.source === 'extras') {
addExtrasHeaders(headers);
}
const response = await fetch('/api/vector/insert', {
method: 'POST',
headers: getRequestHeaders(),
headers: headers,
body: JSON.stringify({
collectionId: collectionId,
items: items,
@@ -468,9 +506,14 @@ async function deleteVectorItems(collectionId, hashes) {
* @returns {Promise<{ hashes: number[], metadata: object[]}>} - Hashes of the results
*/
async function queryCollection(collectionId, searchText, topK) {
const headers = getRequestHeaders();
if (settings.source === 'extras') {
addExtrasHeaders(headers);
}
const response = await fetch('/api/vector/query', {
method: 'POST',
headers: getRequestHeaders(),
headers: headers,
body: JSON.stringify({
collectionId: collectionId,
searchText: searchText,

1
public/scripts/extensions/vectors/settings.html
View File

@@ -11,6 +11,7 @@
</label>
<select id="vectors_source" class="text_pole">
<option value="transformers">Local (Transformers)</option>
<option value="extras">Extras</option>
<option value="openai">OpenAI</option>
<option value="palm">Google MakerSuite (PaLM)</option>
<option value="mistral">MistralAI</option>