Compare commits

..

5 Commits

Author SHA1 Message Date
Cohee 4665db62f4 #1954 Remove backtick wrapping for inserted files 2024-04-16 22:28:10 +03:00
Cohee ab5b497562 Add filters to data bank manager 2024-04-16 22:23:59 +03:00
Cohee 5a614b5173 Integrate data bank with Fandom plugin 2024-04-16 20:16:21 +03:00
Cohee 8546490bcc Improve Scale JWT error handling 2024-04-16 18:59:01 +03:00
Cohee 3dcea41c4e Preserve a query string when redirecting to and from login 2024-04-16 18:44:11 +03:00
10 changed files with 328 additions and 95 deletions

View File

@ -41,6 +41,7 @@ import { POPUP_RESULT, POPUP_TYPE, callGenericPopup } from './popup.js';
* @property {string} url File URL
* @property {number} size File size
* @property {string} name File name
* @property {number} created Timestamp
* @property {string} [text] File text
*/
@ -168,6 +169,7 @@ export async function populateFileAttachment(message, inputId = 'file_form_input
url: fileUrl,
size: file.size,
name: file.name,
created: Date.now(),
};
}
@ -383,7 +385,7 @@ export async function appendFileContent(message, messageText) {
const fileText = message.extra.file.text || (await getFileAttachment(message.extra.file.url));
if (fileText) {
const fileWrapped = `\`\`\`\n${fileText}\n\`\`\`\n\n`;
const fileWrapped = `${fileText}\n\n`;
message.extra.fileLength = fileWrapped.length;
messageText = fileWrapped + messageText;
}
@ -618,11 +620,38 @@ async function deleteAttachment(attachment, source, callback) {
*/
async function openAttachmentManager() {
/**
*
* Renders a list of attachments.
* @param {FileAttachment[]} attachments List of attachments
* @param {string} source Source of the attachments
*/
async function renderList(attachments, source) {
/**
* Sorts attachments by sortField and sortOrder.
* @param {FileAttachment} a First attachment
* @param {FileAttachment} b Second attachment
* @returns {number} Sort order
*/
function sortFn(a, b) {
const sortValueA = a[sortField];
const sortValueB = b[sortField];
if (typeof sortValueA === 'string' && typeof sortValueB === 'string') {
return sortValueA.localeCompare(sortValueB) * (sortOrder === 'asc' ? 1 : -1);
}
return (sortValueA - sortValueB) * (sortOrder === 'asc' ? 1 : -1);
}
/**
* Filters attachments by name.
* @param {FileAttachment} a Attachment
* @returns {boolean} True if attachment matches the filter, false otherwise.
*/
function filterFn(a) {
if (!filterString) {
return true;
}
return a.name.toLowerCase().includes(filterString.toLowerCase());
}
const sources = {
[ATTACHMENT_SOURCE.GLOBAL]: '.globalAttachmentsList',
[ATTACHMENT_SOURCE.CHARACTER]: '.characterAttachmentsList',
@ -630,16 +659,37 @@ async function openAttachmentManager() {
};
template.find(sources[source]).empty();
for (const attachment of attachments) {
// Sort attachments by sortField and sortOrder, and apply filter
const sortedAttachmentList = attachments.slice().filter(filterFn).sort(sortFn);
for (const attachment of sortedAttachmentList) {
const attachmentTemplate = template.find('.attachmentListItemTemplate .attachmentListItem').clone();
attachmentTemplate.find('.attachmentListItemName').text(attachment.name);
attachmentTemplate.find('.attachmentListItemSize').text(humanFileSize(attachment.size));
attachmentTemplate.find('.attachmentListItemCreated').text(new Date(attachment.created).toLocaleString());
attachmentTemplate.find('.viewAttachmentButton').on('click', () => openFilePopup(attachment));
attachmentTemplate.find('.deleteAttachmentButton').on('click', () => deleteAttachment(attachment, source, renderAttachments));
template.find(sources[source]).append(attachmentTemplate);
}
}
/**
* Renders buttons for the attachment manager.
* @param {string} source Source of the buttons
*/
function renderButtons(source) {
const sources = {
[ATTACHMENT_SOURCE.GLOBAL]: '.globalAttachmentsTitle',
[ATTACHMENT_SOURCE.CHARACTER]: '.characterAttachmentsTitle',
[ATTACHMENT_SOURCE.CHAT]: '.chatAttachmentsTitle',
};
const buttonsList = template.find('.actionButtonsTemplate .actionButtons').clone();
buttonsList.find('.menu_button').data('attachment-manager-target', source);
template.find(sources[source]).append(buttonsList);
}
async function renderAttachments() {
/** @type {FileAttachment[]} */
const globalAttachments = extension_settings.attachments ?? [];
@ -664,8 +714,14 @@ async function openAttachmentManager() {
template.find('.chatAttachmentsName').text(chatName);
}
let sortField = localStorage.getItem('DataBank_sortField') || 'created';
let sortOrder = localStorage.getItem('DataBank_sortOrder') || 'desc';
let filterString = '';
const hasFandomPlugin = await isFandomPluginAvailable();
const template = $(await renderExtensionTemplateAsync('attachments', 'manager', {}));
renderButtons(ATTACHMENT_SOURCE.GLOBAL);
renderButtons(ATTACHMENT_SOURCE.CHARACTER);
renderButtons(ATTACHMENT_SOURCE.CHAT);
template.find('.scrapeWebpageButton').on('click', function () {
openWebpageScraper(String($(this).data('attachment-manager-target')), renderAttachments);
});
@ -675,6 +731,21 @@ async function openAttachmentManager() {
template.find('.uploadFileButton').on('click', function () {
openFileUploader(String($(this).data('attachment-manager-target')), renderAttachments);
});
template.find('.attachmentSearch').on('input', function () {
filterString = String($(this).val());
renderAttachments();
});
template.find('.attachmentSort').on('change', function () {
if (!(this instanceof HTMLSelectElement) || this.selectedOptions.length === 0) {
return;
}
sortField = this.selectedOptions[0].dataset.sortField;
sortOrder = this.selectedOptions[0].dataset.sortOrder;
localStorage.setItem('DataBank_sortField', sortField);
localStorage.setItem('DataBank_sortOrder', sortOrder);
renderAttachments();
});
await renderAttachments();
callGenericPopup(template, POPUP_TYPE.TEXT, '', { wide: true, large: true, okButton: 'Close' });
}
@ -734,8 +805,85 @@ async function getTitleFromHtmlBlob(blob) {
* @param {function} callback Callback function
*/
async function openFandomScraper(target, callback) {
toastr.info('Not implemented yet', target);
if (!await isFandomPluginAvailable()) {
toastr.error('Fandom scraper plugin is not available');
return;
}
let fandom = '';
let filter = '';
let output = 'single';
const template = $(await renderExtensionTemplateAsync('attachments', 'fandom-scrape', {}));
template.find('input[name="fandomScrapeInput"]').on('input', function () {
fandom = String($(this).val());
});
template.find('input[name="fandomScrapeFilter"]').on('input', function () {
filter = String($(this).val());
});
template.find('input[name="fandomScrapeOutput"]').on('input', function () {
output = String($(this).val());
});
const confirm = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: false, large: false });
if (confirm !== POPUP_RESULT.AFFIRMATIVE) {
return;
}
if (!fandom) {
toastr.error('Fandom name is required');
return;
}
try {
const result = await fetch('/api/plugins/fandom/scrape', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ fandom, filter }),
});
if (!result.ok) {
const error = await result.text();
throw new Error(error);
}
// Get domain name part if it's a URL
try {
const url = new URL(fandom);
const fandomId = url.hostname.split('.')[0] || fandom;
fandom = fandomId;
} catch {
// Ignore
}
const data = await result.json();
let numberOfAttachments;
if (output === 'multi') {
numberOfAttachments = data.length;
for (const attachment of data) {
const file = new File([String(attachment.content).trim()], `${String(attachment.title).trim()}.txt`, { type: 'text/plain' });
await uploadFileAttachmentToServer(file, target);
}
}
if (output === 'single') {
numberOfAttachments = 1;
const combinedContent = data.map((a) => String(a.title).trim() + '\n\n' + String(a.content).trim()).join('\n\n\n\n');
const file = new File([combinedContent], `${fandom}.txt`, { type: 'text/plain' });
await uploadFileAttachmentToServer(file, target);
}
if (numberOfAttachments) {
toastr.success(`Scraped ${numberOfAttachments} attachments from ${fandom}`);
}
callback();
} catch (error) {
console.error('Fandom scraping failed', error);
toastr.error('Check browser console for details.', 'Fandom scraping failed');
}
}
/**
@ -791,6 +939,7 @@ async function uploadFileAttachmentToServer(file, target) {
}
const fileUrl = await uploadFileAttachment(uniqueFileName, base64Data);
const convertedSize = Math.round(base64Data.length * 0.75);
if (!fileUrl) {
return;
@ -798,8 +947,9 @@ async function uploadFileAttachmentToServer(file, target) {
const attachment = {
url: fileUrl,
size: file.size,
size: convertedSize,
name: file.name,
created: Date.now(),
};
ensureAttachmentsExist();

View File

@ -0,0 +1,51 @@
<div>
<div class="flex-container flexFlowColumn">
<label for="fandomScrapeInput" data-i18n="Enter a URL or the ID of a Fandom wiki page to scrape:">
Enter a URL or the ID of a Fandom wiki page to scrape:
</label>
<small>
<span data-i18n=Examples:">Examples:</span>
<code>https://harrypotter.fandom.com/</code>
<span data-i18n="or">or</span>
<code>harrypotter</code>
</small>
<input type="text" id="fandomScrapeInput" name="fandomScrapeInput" class="text_pole" placeholder="">
</div>
<div class="flex-container flexFlowColumn">
<label for="fandomScrapeFilter">
Optional regex to pick the content by its title:
</label>
<small>
<span data-i18n="Example:">Example:</span>
<code>/(Azkaban|Weasley)/gi</code>
</small>
<input type="text" id="fandomScrapeFilter" name="fandomScrapeFilter" class="text_pole" placeholder="">
</div>
<div class="flex-container flexFlowColumn">
<label>
Output format:
</label>
<label class="checkbox_label justifyLeft" for="fandomScrapeOutputSingle">
<input id="fandomScrapeOutputSingle" type="radio" name="fandomScrapeOutput" value="single" checked>
<div class="flex-container flexFlowColumn flexNoGap">
<span data-i18n="Single file">
Single file
</span>
<small data-i18n="All articles will be concatenated into a single file.">
All articles will be concatenated into a single file.
</small>
</div>
</label>
<label class="checkbox_label justifyLeft" for="fandomScrapeOutputMulti">
<input id="fandomScrapeOutputMulti" type="radio" name="fandomScrapeOutput" value="multi">
<div class="flex-container flexFlowColumn flexNoGap">
<span data-i18n="File per article">
File per article
</span>
<small data-i18n="Each article will be saved as a separate file.">
Not recommended. Each article will be saved as a separate file.
</small>
</div>
</label>
</div>
</div>

View File

@ -1,4 +1,4 @@
<div class="wide100p paddingTopBot5">
<div class="wide100p padding5">
<h2 class="marginBot5">
<span data-i18n="Data Bank">
Data Bank
@ -10,31 +10,35 @@
<div data-i18n="Supported file types: Plain Text, PDF, Markdown, HTML." class="marginTopBot5">
Supported file types: Plain Text, PDF, Markdown, HTML.
</div>
<div class="flex-container marginTopBot5">
<input type="search" id="attachmentSearch" class="attachmentSearch text_pole margin0 flex1" placeholder="Search...">
<select id="attachmentSort" class="attachmentSort text_pole margin0 flex1">
<option data-sort-field="created" data-sort-order="desc" data-i18n="Date (Newest First)">
Date (Newest First)
</option>
<option data-sort-field="created" data-sort-order="asc" data-i18n="Date (Oldest First)">
Date (Oldest First)
</option>
<option data-sort-field="name" data-sort-order="asc" data-i18n="Name (A-Z)">
Name (A-Z)
</option>
<option data-sort-field="name" data-sort-order="desc" data-i18n="Name (Z-A)">
Name (Z-A)
</option>
<option data-sort-field="size" data-sort-order="asc" data-i18n="Size (Smallest First)">
Size (Smallest First)
</option>
<option data-sort-field="size" data-sort-order="desc" data-i18n="Size (Largest First)">
Size (Largest First)
</option>
</select>
</div>
<div class="justifyLeft globalAttachmentsBlock marginBot10">
<h3 class="margin0 title_restorable">
<h3 class="globalAttachmentsTitle margin0 title_restorable">
<span data-i18n="Global Attachments">
Global Attachments
</span>
<div class="flex-container flexGap10">
<div class="scrapeWebpageButton menu_button_icon menu_button" data-attachment-manager-target="global" title="Download a page from the web.">
<i class="fa-fw fa-solid fa-globe"></i>
<span data-i18n="From Web">
From Web
</span>
</div>
<div class="scrapeFandomButton menu_button_icon menu_button" data-attachment-manager-target="global" title="Download a page from the Fandom wiki.">
<i class="fa-fw fa-solid fa-fan"></i>
<span data-i18n="From Fandom">
From Fandom
</span>
</div>
<div class="uploadFileButton menu_button_icon menu_button" data-attachment-manager-target="global" title="Upload a file from your computer.">
<i class="fa-fw fa-solid fa-upload"></i>
<span data-i18n="From File">
From File
</span>
</div>
</div>
</h3>
<small data-i18n="These files are available for all characters in all chats.">
These files are available for all characters in all chats.
@ -43,30 +47,10 @@
<hr>
</div>
<div class="justifyLeft characterAttachmentsBlock marginBot10">
<h3 class="margin0 title_restorable">
<h3 class="characterAttachmentsTitle margin0 title_restorable">
<span data-i18n="Character Attachments">
Character Attachments
</span>
<div class="flex-container flexGap10">
<div class="scrapeWebpageButton menu_button_icon menu_button" data-attachment-manager-target="character" title="Download a page from the web.">
<i class="fa-fw fa-solid fa-globe"></i>
<span data-i18n="From Web">
From Web
</span>
</div>
<div class="scrapeFandomButton menu_button_icon menu_button" data-attachment-manager-target="character" title="Download a page from the Fandom wiki.">
<i class="fa-fw fa-solid fa-fan"></i>
<span data-i18n="From Fandom">
From Fandom
</span>
</div>
<div class="uploadFileButton menu_button_icon menu_button" data-attachment-manager-target="character" title="Upload a file from your computer.">
<i class="fa-fw fa-solid fa-upload"></i>
<span data-i18n="From File">
From File
</span>
</div>
</div>
</h3>
<div class="flex-container flexFlowColumn">
<strong><small class="characterAttachmentsName"></small></strong>
@ -78,30 +62,10 @@
<hr>
</div>
<div class="justifyLeft chatAttachmentsBlock marginBot10">
<h3 class="margin0 title_restorable">
<h3 class="chatAttachmentsTitle margin0 title_restorable">
<span data-i18n="Chat Attachments">
Chat Attachments
</span>
<div class="flex-container flexGap10">
<div class="scrapeWebpageButton menu_button_icon menu_button" data-attachment-manager-target="chat" title="Download a page from the web.">
<i class="fa-fw fa-solid fa-globe"></i>
<span data-i18n="From Web">
From Web
</span>
</div>
<div class="scrapeFandomButton menu_button_icon menu_button" data-attachment-manager-target="chat" title="Download a page from the Fandom wiki.">
<i class="fa-fw fa-solid fa-fan"></i>
<span data-i18n="From Fandom">
From Fandom
</span>
</div>
<div class="uploadFileButton menu_button_icon menu_button" data-attachment-manager-target="chat" title="Upload a file from your computer.">
<i class="fa-fw fa-solid fa-upload"></i>
<span data-i18n="From File">
From File
</span>
</div>
</div>
</h3>
<div class="flex-container flexFlowColumn">
<strong><small class="chatAttachmentsName"></small></strong>
@ -116,9 +80,33 @@
<div class="attachmentListItem flex-container alignItemsCenter flexGap10">
<div class="attachmentFileIcon fa-solid fa-file-alt"></div>
<div class="attachmentListItemName flex1"></div>
<small class="attachmentListItemCreated"></small>
<small class="attachmentListItemSize"></small>
<div class="viewAttachmentButton right_menu_button fa-solid fa-magnifying-glass" title="View attachment content"></div>
<div class="deleteAttachmentButton right_menu_button fa-solid fa-trash" title="Delete attachment"></div>
</div>
</div>
<div class="actionButtonsTemplate template_element">
<div class="actionButtons flex-container flexGap10">
<div class="scrapeWebpageButton menu_button_icon menu_button" data-attachment-manager-target="" title="Download a page from the web.">
<i class="fa-fw fa-solid fa-globe"></i>
<span data-i18n="From Web">
From Web
</span>
</div>
<div class="scrapeFandomButton menu_button_icon menu_button" data-attachment-manager-target="" title="Download a page from the Fandom wiki.">
<i class="fa-fw fa-solid fa-fire"></i>
<span data-i18n="From Fandom">
From Fandom
</span>
</div>
<div class="uploadFileButton menu_button_icon menu_button" data-attachment-manager-target="" title="Upload a file from your computer.">
<i class="fa-fw fa-solid fa-upload"></i>
<span data-i18n="From File">
From File
</span>
</div>
</div>
</div>
</div>

View File

@ -18,3 +18,12 @@
.attachmentListItem {
padding: 10px;
}
.attachmentListItemSize {
min-width: 4em;
text-align: right;
}
.attachmentListItemCreated {
text-align: right;
}

View File

@ -222,8 +222,7 @@ async function processFiles(chat) {
// Trim file inserted by the script
const fileText = String(message.mes)
.substring(0, message.extra.fileLength).trim()
.replace(/^```/, '').replace(/```$/, '').trim();
.substring(0, message.extra.fileLength).trim();
// Convert kilobytes to string length
const thresholdLength = settings.size_threshold * 1024;
@ -247,8 +246,7 @@ async function processFiles(chat) {
const queryText = getQueryText(chat);
const fileChunks = await retrieveFileChunks(queryText, collectionId);
// Wrap it back in a code block
message.mes = `\`\`\`\n${fileChunks}\n\`\`\`\n\n${message.mes}`;
message.mes = `${fileChunks}\n\n${message.mes}`;
}
} catch (error) {
console.error('Vectors: Failed to retrieve files', error);

View File

@ -177,9 +177,10 @@ function displayError(message) {
/**
* Redirects the user to the home page.
* Preserves the query string.
*/
function redirectToHome() {
window.location.href = '/';
window.location.href = '/' + window.location.search;
}
/**

View File

@ -1594,6 +1594,11 @@ async function sendAltScaleRequest(messages, logit_bias, signal, type) {
signal: signal,
});
if (!response.ok) {
tryParseStreamingError(response, await response.text());
throw new Error('Scale response does not indicate success.');
}
const data = await response.json();
return data.output;
}

View File

@ -248,7 +248,9 @@ if (!disableCsrf) {
// Host index page
app.get('/', (request, response) => {
if (userModule.shouldRedirectToLogin(request)) {
return response.redirect('/login');
const query = request.url.split('?')[1];
const redirectUrl = query ? `/login?${query}` : '/login';
return response.redirect(redirectUrl);
}
return response.sendFile('index.html', { root: path.join(process.cwd(), 'public') });

View File

@ -7,16 +7,18 @@ const { readSecret, SECRET_KEYS } = require('../secrets');
const router = express.Router();
router.post('/generate', jsonParser, function (request, response) {
router.post('/generate', jsonParser, async function (request, response) {
if (!request.body) return response.sendStatus(400);
fetch('https://dashboard.scale.com/spellbook/api/trpc/v2.variant.run', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'cookie': `_jwt=${readSecret(request.user.directories, SECRET_KEYS.SCALE_COOKIE)}`,
},
body: JSON.stringify({
try {
const cookie = readSecret(request.user.directories, SECRET_KEYS.SCALE_COOKIE);
if (!cookie) {
console.log('No Scale cookie found');
return response.sendStatus(400);
}
const body = {
json: {
variant: {
name: 'New Variant',
@ -59,18 +61,41 @@ router.post('/generate', jsonParser, function (request, response) {
'modelParameters.logprobs': ['undefined'],
},
},
}),
})
.then(res => res.json())
.then(data => {
console.log(data.result.data.json.outputs[0]);
return response.send({ output: data.result.data.json.outputs[0] });
})
.catch((error) => {
console.error('Error:', error);
return response.send({ error: true });
};
console.log('Scale request:', body);
const result = await fetch('https://dashboard.scale.com/spellbook/api/trpc/v2.variant.run', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'cookie': `_jwt=${cookie}`,
},
timeout: 0,
body: JSON.stringify(body),
});
if (!result.ok) {
const text = await result.text();
console.log('Scale request failed', result.statusText, text);
return response.status(500).send({ error: { message: result.statusText } });
}
const data = await result.json();
const output = data?.result?.data?.json?.outputs?.[0] || '';
console.log('Scale response:', data);
if (!output) {
console.warn('Scale response is empty');
return response.sendStatus(500).send({ error: { message: 'Empty response' } });
}
return response.json({ output });
} catch (error) {
console.log(error);
return response.sendStatus(500);
}
});
module.exports = { router };

View File

@ -326,7 +326,7 @@ function toAvatarKey(handle) {
}
/**
* Initializes the user storage. Currently a no-op.
* Initializes the user storage.
* @param {string} dataRoot The root directory for user data
* @returns {Promise<void>}
*/
@ -655,6 +655,10 @@ async function createBackupArchive(handle, response) {
archive.finalize();
}
/**
* Checks if any admin users are not password protected. If so, logs a warning.
* @returns {Promise<void>}
*/
async function checkAccountsProtection() {
if (!ENABLE_ACCOUNTS) {
return;