rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'staging' into smol-tag-improvements

Browse Source
This commit is contained in:
Wolfsblvt
2024-06-08 21:13:11 +02:00
parent d1824acee0 66fd973830
commit d98d811cc1
39 changed files with 18308 additions and 11333 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
public/scripts/extensions/caption/index.js
View File

@@ -1,5 +1,5 @@
import { getBase64Async, isTrueBoolean, saveBase64AsFile } from '../../utils.js';
import { getContext, getApiUrl, doExtrasFetch, extension_settings, modules } from '../../extensions.js';
import { getContext, getApiUrl, doExtrasFetch, extension_settings, modules, renderExtensionTemplateAsync } from '../../extensions.js';
import { callPopup, getRequestHeaders, saveSettingsDebounced, substituteParams } from '../../../script.js';
import { getMessageTimeStamp } from '../../RossAscends-mods.js';
import { SECRET_KEYS, secret_state } from '../../secrets.js';
@@ -334,7 +334,7 @@ async function captionCommandCallback(args, prompt) {
});
}
jQuery(function () {
jQuery(async function () {
function addSendPictureButton() {
const sendButton = $(`
<div id="send_picture" class="list-group-item flex-container flexGap5">
@@ -399,102 +399,12 @@ jQuery(function () {
saveSettingsDebounced();
});
}
function addSettings() {
const html = `
<div class="caption_settings">
<div class="inline-drawer">
<div class="inline-drawer-toggle inline-drawer-header">
<b>Image Captioning</b>
<div class="inline-drawer-icon fa-solid fa-circle-chevron-down down"></div>
</div>
<div class="inline-drawer-content">
<label for="caption_source">Source</label>
<select id="caption_source" class="text_pole">
<option value="local">Local</option>
<option value="multimodal">Multimodal (OpenAI / Anthropic / llama / Google)</option>
<option value="extras">Extras</option>
<option value="horde">Horde</option>
</select>
<div id="caption_multimodal_block" class="flex-container wide100p">
<div class="flex1 flex-container flexFlowColumn flexNoGap">
<label for="caption_multimodal_api">API</label>
<select id="caption_multimodal_api" class="flex1 text_pole">
<option value="anthropic">Anthropic</option>
<option value="custom">Custom (OpenAI-compatible)</option>
<option value="google">Google MakerSuite</option>
<option value="koboldcpp">KoboldCpp</option>
<option value="llamacpp">llama.cpp</option>
<option value="ollama">Ollama</option>
<option value="openai">OpenAI</option>
<option value="openrouter">OpenRouter</option>
<option value="ooba">Text Generation WebUI (oobabooga)</option>
</select>
</div>
<div class="flex1 flex-container flexFlowColumn flexNoGap">
<label for="caption_multimodal_model">Model</label>
<select id="caption_multimodal_model" class="flex1 text_pole">
<option data-type="openai" value="gpt-4-vision-preview">gpt-4-vision-preview</option>
<option data-type="openai" value="gpt-4-turbo">gpt-4-turbo</option>
<option data-type="openai" value="gpt-4o">gpt-4o</option>
<option data-type="anthropic" value="claude-3-opus-20240229">claude-3-opus-20240229</option>
<option data-type="anthropic" value="claude-3-sonnet-20240229">claude-3-sonnet-20240229</option>
<option data-type="anthropic" value="claude-3-haiku-20240307">claude-3-haiku-20240307</option>
<option data-type="google" value="gemini-pro-vision">gemini-pro-vision</option>
<option data-type="google" value="gemini-1.5-flash-latest">gemini-1.5-flash-latest</option>
<option data-type="openrouter" value="openai/gpt-4-vision-preview">openai/gpt-4-vision-preview</option>
<option data-type="openrouter" value="openai/gpt-4o">openai/gpt-4o</option>
<option data-type="openrouter" value="openai/gpt-4-turbo">openai/gpt-4-turbo</option>
<option data-type="openrouter" value="haotian-liu/llava-13b">haotian-liu/llava-13b</option>
<option data-type="openrouter" value="fireworks/firellava-13b">fireworks/firellava-13b</option>
<option data-type="openrouter" value="anthropic/claude-3-haiku">anthropic/claude-3-haiku</option>
<option data-type="openrouter" value="anthropic/claude-3-sonnet">anthropic/claude-3-sonnet</option>
<option data-type="openrouter" value="anthropic/claude-3-opus">anthropic/claude-3-opus</option>
<option data-type="openrouter" value="anthropic/claude-3-haiku:beta">anthropic/claude-3-haiku:beta</option>
<option data-type="openrouter" value="anthropic/claude-3-sonnet:beta">anthropic/claude-3-sonnet:beta</option>
<option data-type="openrouter" value="anthropic/claude-3-opus:beta">anthropic/claude-3-opus:beta</option>
<option data-type="openrouter" value="nousresearch/nous-hermes-2-vision-7b">nousresearch/nous-hermes-2-vision-7b</option>
<option data-type="openrouter" value="google/gemini-pro-vision">google/gemini-pro-vision</option>
<option data-type="openrouter" value="google/gemini-flash-1.5">google/gemini-flash-1.5</option>
<option data-type="openrouter" value="liuhaotian/llava-yi-34b">liuhaotian/llava-yi-34b</option>
<option data-type="ollama" value="ollama_current">[Currently selected]</option>
<option data-type="ollama" value="bakllava:latest">bakllava:latest</option>
<option data-type="ollama" value="llava:latest">llava:latest</option>
<option data-type="llamacpp" value="llamacpp_current">[Currently loaded]</option>
<option data-type="ooba" value="ooba_current">[Currently loaded]</option>
<option data-type="koboldcpp" value="koboldcpp_current">[Currently loaded]</option>
<option data-type="custom" value="custom_current">[Currently selected]</option>
</select>
</div>
<label data-type="openai,anthropic,google" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
<input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">
Allow reverse proxy
</label>
<div class="flexBasis100p m-b-1">
<small><b>Hint:</b> Set your API keys and endpoints in the 'API Connections' tab first.</small>
</div>
</div>
<div id="caption_prompt_block">
<label for="caption_prompt">Caption Prompt</label>
<textarea id="caption_prompt" class="text_pole" rows="1" placeholder="&lt; Use default &gt;">${PROMPT_DEFAULT}</textarea>
<label class="checkbox_label margin-bot-10px" for="caption_prompt_ask" title="Ask for a custom prompt every time an image is captioned.">
<input id="caption_prompt_ask" type="checkbox" class="checkbox">
Ask every time
</label>
</div>
<label for="caption_template">Message Template <small>(use <code>{{caption}}</code> macro)</small></label>
<textarea id="caption_template" class="text_pole" rows="2" placeholder="&lt; Use default &gt;">${TEMPLATE_DEFAULT}</textarea>
<label class="checkbox_label margin-bot-10px" for="caption_refine_mode">
<input id="caption_refine_mode" type="checkbox" class="checkbox">
Edit captions before saving
</label>
</div>
</div>
</div>
`;
async function addSettings() {
const html = await renderExtensionTemplateAsync('caption', 'settings');
$('#extensions_settings2').append(html);
}
addSettings();
await addSettings();
addPictureSendForm();
addSendPictureButton();
setImageIcon();

90
public/scripts/extensions/caption/settings.html Normal file
View File

@@ -0,0 +1,90 @@
<div class="caption_settings">
<div class="inline-drawer">
<div class="inline-drawer-toggle inline-drawer-header">
<b>Image Captioning</b>
<div class="inline-drawer-icon fa-solid fa-circle-chevron-down down"></div>
</div>
<div class="inline-drawer-content">
<label for="caption_source">Source</label>
<select id="caption_source" class="text_pole">
<option value="local">Local</option>
<option value="multimodal">Multimodal (OpenAI / Anthropic / llama / Google)</option>
<option value="extras">Extras</option>
<option value="horde">Horde</option>
</select>
<div id="caption_multimodal_block" class="flex-container wide100p">
<div class="flex1 flex-container flexFlowColumn flexNoGap">
<label for="caption_multimodal_api">API</label>
<select id="caption_multimodal_api" class="flex1 text_pole">
<option value="anthropic">Anthropic</option>
<option value="custom">Custom (OpenAI-compatible)</option>
<option value="google">Google MakerSuite</option>
<option value="koboldcpp">KoboldCpp</option>
<option value="llamacpp">llama.cpp</option>
<option value="ollama">Ollama</option>
<option value="openai">OpenAI</option>
<option value="openrouter">OpenRouter</option>
<option value="ooba">Text Generation WebUI (oobabooga)</option>
</select>
</div>
<div class="flex1 flex-container flexFlowColumn flexNoGap">
<label for="caption_multimodal_model">Model</label>
<select id="caption_multimodal_model" class="flex1 text_pole">
<option data-type="openai" value="gpt-4-vision-preview">gpt-4-vision-preview</option>
<option data-type="openai" value="gpt-4-turbo">gpt-4-turbo</option>
<option data-type="openai" value="gpt-4o">gpt-4o</option>
<option data-type="anthropic" value="claude-3-opus-20240229">claude-3-opus-20240229</option>
<option data-type="anthropic" value="claude-3-sonnet-20240229">claude-3-sonnet-20240229</option>
<option data-type="anthropic" value="claude-3-haiku-20240307">claude-3-haiku-20240307</option>
<option data-type="google" value="gemini-pro-vision">gemini-pro-vision</option>
<option data-type="google" value="gemini-1.5-flash-latest">gemini-1.5-flash-latest</option>
<option data-type="openrouter" value="openai/gpt-4-vision-preview">openai/gpt-4-vision-preview</option>
<option data-type="openrouter" value="openai/gpt-4o">openai/gpt-4o</option>
<option data-type="openrouter" value="openai/gpt-4-turbo">openai/gpt-4-turbo</option>
<option data-type="openrouter" value="haotian-liu/llava-13b">haotian-liu/llava-13b</option>
<option data-type="openrouter" value="fireworks/firellava-13b">fireworks/firellava-13b</option>
<option data-type="openrouter" value="anthropic/claude-3-haiku">anthropic/claude-3-haiku</option>
<option data-type="openrouter" value="anthropic/claude-3-sonnet">anthropic/claude-3-sonnet</option>
<option data-type="openrouter" value="anthropic/claude-3-opus">anthropic/claude-3-opus</option>
<option data-type="openrouter" value="anthropic/claude-3-haiku:beta">anthropic/claude-3-haiku:beta</option>
<option data-type="openrouter" value="anthropic/claude-3-sonnet:beta">anthropic/claude-3-sonnet:beta</option>
<option data-type="openrouter" value="anthropic/claude-3-opus:beta">anthropic/claude-3-opus:beta</option>
<option data-type="openrouter" value="nousresearch/nous-hermes-2-vision-7b">nousresearch/nous-hermes-2-vision-7b</option>
<option data-type="openrouter" value="google/gemini-pro-vision">google/gemini-pro-vision</option>
<option data-type="openrouter" value="google/gemini-flash-1.5">google/gemini-flash-1.5</option>
<option data-type="openrouter" value="liuhaotian/llava-yi-34b">liuhaotian/llava-yi-34b</option>
<option data-type="ollama" value="ollama_current">[Currently selected]</option>
<option data-type="ollama" value="bakllava:latest">bakllava:latest</option>
<option data-type="ollama" value="llava:latest">llava:latest</option>
<option data-type="llamacpp" value="llamacpp_current">[Currently loaded]</option>
<option data-type="ooba" value="ooba_current">[Currently loaded]</option>
<option data-type="koboldcpp" value="koboldcpp_current">[Currently loaded]</option>
<option data-type="custom" value="custom_current">[Currently selected]</option>
</select>
</div>
<label data-type="openai,anthropic,google" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
<input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">
Allow reverse proxy
</label>
<div class="flexBasis100p m-b-1">
<small><b>Hint:</b> Set your API keys and endpoints in the 'API Connections' tab first.</small>
</div>
</div>
<div id="caption_prompt_block">
<label for="caption_prompt">Caption Prompt</label>
<textarea id="caption_prompt" class="text_pole" rows="1" placeholder="&lt; Use default &gt;">${PROMPT_DEFAULT}</textarea>
<label class="checkbox_label margin-bot-10px" for="caption_prompt_ask" title="Ask for a custom prompt every time an image is captioned.">
<input id="caption_prompt_ask" type="checkbox" class="checkbox">
Ask every time
</label>
</div>
<label for="caption_template">Message Template <small>(use <code>&lcub;&lcub;caption&rcub;&rcub;</code> macro)</small></label>
<textarea id="caption_template" class="text_pole" rows="2" placeholder="&lt; Use default &gt;">${TEMPLATE_DEFAULT}</textarea>
<label class="checkbox_label margin-bot-10px" for="caption_refine_mode">
<input id="caption_refine_mode" type="checkbox" class="checkbox">
Edit captions before saving
</label>
</div>
</div>
</div>

75
public/scripts/extensions/vectors/index.js
View File

@@ -22,7 +22,7 @@ import {
import { collapseNewlines } from '../../power-user.js';
import { SECRET_KEYS, secret_state, writeSecret } from '../../secrets.js';
import { getDataBankAttachments, getDataBankAttachmentsForSource, getFileAttachment } from '../../chats.js';
import { debounce, getStringHash as calculateHash, waitUntilCondition, onlyUnique, splitRecursive } from '../../utils.js';
import { debounce, getStringHash as calculateHash, waitUntilCondition, onlyUnique, splitRecursive, trimToStartSentence, trimToEndSentence } from '../../utils.js';
import { debounce_timeout } from '../../constants.js';
import { getSortedEntries } from '../../world-info.js';
import { textgen_types, textgenerationwebui_settings } from '../../textgen-settings.js';
@@ -66,11 +66,13 @@ const settings = {
size_threshold: 10,
chunk_size: 5000,
chunk_count: 2,
overlap_percent: 0,
// For Data Bank
size_threshold_db: 5,
chunk_size_db: 2500,
chunk_count_db: 5,
overlap_percent_db: 0,
file_template_db: 'Related information:\n{{text}}',
file_position_db: extension_prompt_types.IN_PROMPT,
file_depth_db: 4,
@@ -369,7 +371,7 @@ async function processFiles(chat) {
// File is already in the collection
if (!hashesInCollection.length) {
await vectorizeFile(fileText, fileName, collectionId, settings.chunk_size);
await vectorizeFile(fileText, fileName, collectionId, settings.chunk_size, settings.overlap_percent);
}
const queryText = await getQueryText(chat);
@@ -409,7 +411,7 @@ async function ingestDataBankAttachments(source) {
const thresholdLength = settings.size_threshold_db * 1024;
// Use chunk size from settings if file is larger than threshold
const chunkSize = file.size > thresholdLength ? settings.chunk_size_db : -1;
await vectorizeFile(file.text, file.name, collectionId, chunkSize);
await vectorizeFile(file.text, file.name, collectionId, chunkSize, settings.overlap_percent_db);
}
return dataBankCollectionIds;
@@ -467,9 +469,10 @@ async function retrieveFileChunks(queryText, collectionId) {
* @param {string} fileName File name
* @param {string} collectionId File collection ID
* @param {number} chunkSize Chunk size
* @param {number} overlapPercent Overlap size (in %)
* @returns {Promise<boolean>} True if successful, false if not
*/
async function vectorizeFile(fileText, fileName, collectionId, chunkSize) {
async function vectorizeFile(fileText, fileName, collectionId, chunkSize, overlapPercent) {
try {
if (settings.translate_files && typeof window['translate'] === 'function') {
console.log(`Vectors: Translating file ${fileName} to English...`);
@@ -478,8 +481,11 @@ async function vectorizeFile(fileText, fileName, collectionId, chunkSize) {
}
const toast = toastr.info('Vectorization may take some time, please wait...', `Ingesting file ${fileName}`);
const chunks = splitRecursive(fileText, chunkSize);
console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks`, chunks);
const overlapSize = Math.round(chunkSize * overlapPercent / 100);
// Overlap should not be included in chunk size. It will be later compensated by overlapChunks
chunkSize = overlapSize > 0 ? (chunkSize - overlapSize) : chunkSize;
const chunks = splitRecursive(fileText, chunkSize).map((x, y, z) => overlapSize > 0 ? overlapChunks(x, y, z, overlapSize) : x);
console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks with ${overlapPercent}% overlap`, chunks);
const items = chunks.map((chunk, index) => ({ hash: getStringHash(chunk), text: chunk, index: index }));
await insertVectorItems(collectionId, items);
@@ -588,6 +594,26 @@ function getPromptText(queriedMessages) {
return substituteParams(settings.template.replace(/{{text}}/i, queriedText));
}
/**
* Modifies text chunks to include overlap with adjacent chunks.
* @param {string} chunk Current item
* @param {number} index Current index
* @param {string[]} chunks List of chunks
* @param {number} overlapSize Size of the overlap
* @returns {string} Overlapped chunks, with overlap trimmed to sentence boundaries
*/
function overlapChunks(chunk, index, chunks, overlapSize) {
const halfOverlap = Math.floor(overlapSize / 2);
const nextChunk = chunks[index + 1];
const prevChunk = chunks[index - 1];
const nextOverlap = trimToEndSentence(nextChunk?.substring(0, halfOverlap)) || '';
const prevOverlap = trimToStartSentence(prevChunk?.substring(prevChunk.length - halfOverlap)) || '';
const overlappedChunk = [prevOverlap, chunk, nextOverlap].filter(x => x).join(' ');
return overlappedChunk;
}
window['vectors_rearrangeChat'] = rearrangeChat;
const onChatEvent = debounce(async () => await moduleWorker.update(), debounce_timeout.relaxed);
@@ -969,8 +995,9 @@ async function onViewStatsClick() {
toastr.info(`Total hashes: <b>${totalHashes}</b><br>
Unique hashes: <b>${uniqueHashes}</b><br><br>
I'll mark collected messages with a green circle.`,
`Stats for chat ${chatId}`,
{ timeOut: 10000, escapeHtml: false });
`Stats for chat ${chatId}`,
{ timeOut: 10000, escapeHtml: false },
);
const chat = getContext().chat;
for (const message of chat) {
@@ -1010,6 +1037,23 @@ async function onVectorizeAllFilesClick() {
return -1;
}
/**
* Gets the overlap percent for a file attachment.
* @param file {import('../../chats.js').FileAttachment} File attachment
* @returns {number} Overlap percent for the file
*/
function getOverlapPercent(file) {
if (chatAttachments.includes(file)) {
return settings.overlap_percent;
}
if (dataBank.includes(file)) {
return settings.overlap_percent_db;
}
return 0;
}
let allSuccess = true;
for (const file of allFiles) {
@@ -1023,7 +1067,8 @@ async function onVectorizeAllFilesClick() {
}
const chunkSize = getChunkSize(file);
const result = await vectorizeFile(text, file.name, collectionId, chunkSize);
const overlapPercent = getOverlapPercent(file);
const result = await vectorizeFile(text, file.name, collectionId, chunkSize, overlapPercent);
if (!result) {
allSuccess = false;
@@ -1343,6 +1388,18 @@ jQuery(async () => {
saveSettingsDebounced();
});
$('#vectors_overlap_percent').val(settings.overlap_percent).on('input', () => {
settings.overlap_percent = Number($('#vectors_overlap_percent').val());
Object.assign(extension_settings.vectors, settings);
saveSettingsDebounced();
});
$('#vectors_overlap_percent_db').val(settings.overlap_percent_db).on('input', () => {
settings.overlap_percent_db = Number($('#vectors_overlap_percent_db').val());
Object.assign(extension_settings.vectors, settings);
saveSettingsDebounced();
});
$('#vectors_file_template_db').val(settings.file_template_db).on('input', () => {
settings.file_template_db = String($('#vectors_file_template_db').val());
Object.assign(extension_settings.vectors, settings);

24
public/scripts/extensions/vectors/settings.html
View File

@@ -193,19 +193,25 @@
<label for="vectors_size_threshold">
<small>Size threshold (KB)</small>
</label>
<input id="vectors_size_threshold" type="number" class="text_pole widthUnset" min="1" max="99999" />
<input id="vectors_size_threshold" type="number" class="text_pole" min="1" max="99999" />
</div>
<div class="flex1" title="Chunk size for file splitting.">
<label for="vectors_chunk_size">
<small>Chunk size (chars)</small>
</label>
<input id="vectors_chunk_size" type="number" class="text_pole widthUnset" min="1" max="99999" />
<input id="vectors_chunk_size" type="number" class="text_pole" min="1" max="99999" />
</div>
<div class="flex1" title="The overlap between adjacent chunks in % from chunk size. The overlap text is trimmed to sentence boundaries. 0 = disabled.">
<label for="vectors_overlap_percent">
<small>Chunk overlap (%)</small>
</label>
<input id="vectors_overlap_percent" type="number" class="text_pole" min="0" max="99" step="1" />
</div>
<div class="flex1" title="How many chunks to retrieve when querying.">
<label for="vectors_chunk_count">
<small>Retrieve chunks</small>
</label>
<input id="vectors_chunk_count" type="number" class="text_pole widthUnset" min="1" max="99999" />
<input id="vectors_chunk_count" type="number" class="text_pole" min="1" max="99999" />
</div>
</div>
<div class="flex justifyCenter" title="These settings apply to files stored in the Data Bank.">
@@ -216,19 +222,25 @@
<label for="vectors_size_threshold_db">
<small>Size threshold (KB)</small>
</label>
<input id="vectors_size_threshold_db" type="number" class="text_pole widthUnset" min="1" max="99999" />
<input id="vectors_size_threshold_db" type="number" class="text_pole" min="1" max="99999" />
</div>
<div class="flex1" title="Chunk size for file splitting.">
<label for="vectors_chunk_size_db">
<small>Chunk size (chars)</small>
</label>
<input id="vectors_chunk_size_db" type="number" class="text_pole widthUnset" min="1" max="99999" />
<input id="vectors_chunk_size_db" type="number" class="text_pole" min="1" max="99999" />
</div>
<div class="flex1" title="The overlap between adjacent chunks in % from chunk size. The overlap text is trimmed to sentence boundaries. 0 = disabled.">
<label for="vectors_overlap_percent_db">
<small>Chunk overlap (%)</small>
</label>
<input id="vectors_overlap_percent_db" type="number" class="text_pole" min="0" max="99" step="1" />
</div>
<div class="flex1" title="How many chunks to retrieve when querying.">
<label for="vectors_chunk_count_db">
<small>Retrieve chunks</small>
</label>
<input id="vectors_chunk_count_db" type="number" class="text_pole widthUnset" min="1" max="99999" />
<input id="vectors_chunk_count_db" type="number" class="text_pole" min="1" max="99999" />
</div>
</div>
<div class="flex-container flexFlowColumn">