Merge branch 'feature/extensions' of https://github.com/BlipRanger/SillyTavern into dev

This commit is contained in:
Cohee
2023-07-13 16:49:52 +03:00
7 changed files with 504 additions and 39 deletions

3
.gitignore vendored
View File

@@ -9,6 +9,7 @@ public/worlds/
public/css/bg_load.css
public/themes/
public/OpenAI Settings/
public/scripts/extensions/third-party/
/uploads/
*.jsonl
config.conf
@@ -22,4 +23,4 @@ secrets.json
poe_device.json
/backups/
poe-error.log
poe-success.log
poe-success.log

View File

@@ -32,6 +32,7 @@
"rimraf": "^3.0.2",
"sanitize-filename": "^1.6.3",
"sentencepiece-js": "^1.1.0",
"simple-git": "^3.19.1",
"uniqolor": "^1.1.0",
"webp-converter": "2.3.2",
"ws": "^8.13.0",

View File

@@ -2506,6 +2506,7 @@
<input id="extensions_connect" class="menu_button" type="submit" value="Connect">
</div>
<input id="extensions_details" class="alignitemsflexstart menu_button" type="button" value="Manage extensions">
<div id="third_party_extension_button" title="Import Extension From Git Repo" class="menu_button fa-solid fa-cloud-arrow-down faSmallFontSquareFix"></div>
</div>
</div>
<div id="extensions_settings" class="flex1 wide50p">

View File

@@ -5577,6 +5577,9 @@ function callPopup(text, type, inputValue = '', { okButton, rows } = {}) {
$("#dialogue_popup_ok").text(okButton ?? "Ok");
$("#dialogue_popup_cancel").css("display", "none");
break;
case "delete_extension":
$("#dialogue_popup_ok").text(okButton ?? "Ok");
break;
case "new_chat":
case "confirm":
$("#dialogue_popup_ok").text(okButton ?? "Yes");
@@ -8525,6 +8528,52 @@ $(document).ready(function () {
}
});
/**
* Handles the click event for the third-party extension import button.
* Prompts the user to enter the Git URL of the extension to import.
* After obtaining the Git URL, makes a POST request to '/get_extension' to import the extension.
* If the extension is imported successfully, a success message is displayed.
* If the extension import fails, an error message is displayed and the error is logged to the console.
* After successfully importing the extension, the extension settings are reloaded and a 'EXTENSION_SETTINGS_LOADED' event is emitted.
*
* @listens #third_party_extension_button#click - The click event of the '#third_party_extension_button' element.
*/
$('#third_party_extension_button').on('click', async () => {
const html = `<h3>Enter the Git URL of the extension to import</h3>
<br>
<p><b>Disclaimer:</b> Please be aware that using external extensions can have unintended side effects and may pose security risks. Always make sure you trust the source before importing an extension. We are not responsible for any damage caused by third-party extensions.</p>
<br>
<p>Example: <tt> https://github.com/author/extension-name </tt></p>`
const input = await callPopup(html, 'input');
if (!input) {
console.debug('Extension import cancelled');
return;
}
const url = input.trim();
console.debug('Extension import started', url);
const request = await fetch('/get_extension', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ url }),
});
if (!request.ok) {
toastr.info(request.statusText, 'Extension import failed');
console.error('Extension import failed', request.status, request.statusText);
return;
}
const response = await request.json();
toastr.success(`Extension "${response.display_name}" by ${response.author} (version ${response.version}) has been imported successfully!`, 'Extension import successful');
console.debug(`Extension "${response.display_name}" has been imported successfully at ${response.extensionPath}`);
await loadExtensionSettings(settings);
eventSource.emit(event_types.EXTENSION_SETTINGS_LOADED);
});
const $dropzone = $(document.body);
$dropzone.on('dragover', (event) => {

View File

@@ -1,4 +1,4 @@
import { callPopup, eventSource, event_types, saveSettings, saveSettingsDebounced } from "../script.js";
import { callPopup, eventSource, event_types, saveSettings, saveSettingsDebounced, getRequestHeaders } from "../script.js";
import { isSubsetOf, debounce } from "./utils.js";
export {
getContext,
@@ -376,50 +376,218 @@ function addExtensionScript(name, manifest) {
return Promise.resolve();
}
function showExtensionsDetails() {
let html = '<h3>Modules provided by your Extensions API:</h3>';
html += modules.length ? `<p>${DOMPurify.sanitize(modules.join(', '))}</p>` : '<p class="failure">Not connected to the API!</p>';
html += '<h3>Available extensions:</h3>';
Object.entries(manifests).sort((a, b) => a[1].loading_order - b[1].loading_order).forEach(extension => {
const name = extension[0];
const manifest = extension[1];
const isActive = activeExtensions.has(name);
const isDisabled = extension_settings.disabledExtensions.includes(name);
const titleClass = isActive ? "extension_enabled" : isDisabled ? "extension_disabled" : "extension_missing";
const iconString = isActive ? "\u2705" : isDisabled ? "\u274C" : "";
/**
* Generates HTML string for displaying an extension in the UI.
*
* @param {string} name - The name of the extension.
* @param {object} manifest - The manifest of the extension.
* @param {boolean} isActive - Whether the extension is active or not.
* @param {boolean} isDisabled - Whether the extension is disabled or not.
* @param {boolean} isExternal - Whether the extension is external or not.
* @param {string} checkboxClass - The class for the checkbox HTML element.
* @return {string} - The HTML string that represents the extension.
*/
async function generateExtensionHtml(name, manifest, isActive, isDisabled, isExternal, checkboxClass) {
const displayName = manifest.display_name;
let displayVersion = manifest.version ? ` v${manifest.version}` : "";
let isUpToDate = true;
let updateButton = '';
let originHtml = '';
if (isExternal) {
let data = await getExtensionVersion(name.replace('third-party', ''));
let branch = data.currentBranchName;
let commitHash = data.currentCommitHash;
let origin = data.remoteUrl
isUpToDate = data.isUpToDate;
displayVersion = ` (${branch}-${commitHash.substring(0, 7)})`;
updateButton = isUpToDate ?
`<span class="update-button"><button class="btn_update menu_button" data-name="${name.replace('third-party', '')}" title="Up to date"><i class="fa-solid fa-code-commit"></i></button></span>` :
`<span class="update-button"><button class="btn_update menu_button" data-name="${name.replace('third-party', '')}" title="Update available"><i class="fa-solid fa-download"></i></button></span>`;
originHtml = `<a href="${origin}" target="_blank" rel="noopener noreferrer">`;
}
let toggleElement = `<span title="Cannot enable extension" data-name="${name}" class="extension_missing">Unavailable</span>`;
if (isActive) {
toggleElement = `<a href="javascript:void" title="Click to disable" data-name="${name}" class="toggle_disable">Disable</a>`;
let toggleElement = isActive || isDisabled ?
`<input type="checkbox" title="Click to toggle" data-name="${name}" class="${isActive ? 'toggle_disable' : 'toggle_enable'} ${checkboxClass}" ${isActive ? 'checked' : ''}>` :
`<input type="checkbox" title="Cannot enable extension" data-name="${name}" class="extension_missing ${checkboxClass}" disabled>`;
let deleteButton = isExternal ? `<span class="delete-button"><button class="btn_delete menu_button" data-name="${name.replace('third-party', '')}" title="Delete"><i class="fa-solid fa-trash-can"></i></button></span>` : '';
// if external, wrap the name in a link to the repo
let extensionHtml = `<hr>
<h4>
${updateButton}
${deleteButton}
${originHtml}
<span class="${isActive ? "extension_enabled" : isDisabled ? "extension_disabled" : "extension_missing"}">
${DOMPurify.sanitize(displayName)}${displayVersion}
</span>
${isExternal ? '</a>' : ''}
<span style="float:right;">${toggleElement}</span>
</h4>`;
if (isActive && Array.isArray(manifest.optional)) {
const optional = new Set(manifest.optional);
modules.forEach(x => optional.delete(x));
if (optional.size > 0) {
const optionalString = DOMPurify.sanitize([...optional].join(', '));
extensionHtml += `<p>Optional modules: <span class="optional">${optionalString}</span></p>`;
}
else if (isDisabled) {
toggleElement = `<a href="javascript:void" title="Click to enable" data-name="${name}" class="toggle_enable">Enable</a>`;
}
html += `<hr><h4>${iconString} <span class="${titleClass}">${DOMPurify.sanitize(manifest.display_name)}</span> <span style="float:right;">${toggleElement}<span></h4>`;
} else if (!isDisabled) { // Neither active nor disabled
const requirements = new Set(manifest.requires);
modules.forEach(x => requirements.delete(x));
const requirementsString = DOMPurify.sanitize([...requirements].join(', '));
extensionHtml += `<p>Missing modules: <span class="failure">${requirementsString}</span></p>`;
}
if (isActive) {
if (Array.isArray(manifest.optional)) {
const optional = new Set(manifest.optional);
modules.forEach(x => optional.delete(x));
if (optional.size > 0) {
const optionalString = DOMPurify.sanitize([...optional].join(', '));
html += `<p>Optional modules: <span class="optional">${optionalString}</span></p>`;
}
}
}
else if (!isDisabled) { // Neither active nor disabled
const requirements = new Set(manifest.requires);
modules.forEach(x => requirements.delete(x));
const requirementsString = DOMPurify.sanitize([...requirements].join(', '));
html += `<p>Missing modules: <span class="failure">${requirementsString}</span></p>`
}
});
return extensionHtml;
}
/**
* Gets extension data and generates the corresponding HTML for displaying the extension.
*
* @param {Array} extension - An array where the first element is the extension name and the second element is the extension manifest.
* @return {object} - An object with 'isExternal' indicating whether the extension is external, and 'extensionHtml' for the extension's HTML string.
*/
async function getExtensionData(extension) {
const name = extension[0];
const manifest = extension[1];
const isActive = activeExtensions.has(name);
const isDisabled = extension_settings.disabledExtensions.includes(name);
const isExternal = name.startsWith('third-party');
const checkboxClass = isDisabled ? "checkbox_disabled" : "";
const extensionHtml = await generateExtensionHtml(name, manifest, isActive, isDisabled, isExternal, checkboxClass);
return { isExternal, extensionHtml };
}
/**
* Gets the module information to be displayed.
*
* @return {string} - The HTML string for the module information.
*/
function getModuleInformation() {
let moduleInfo = modules.length ? `<p>${DOMPurify.sanitize(modules.join(', '))}</p>` : '<p class="failure">Not connected to the API!</p>';
return `
<h3>Modules provided by your Extensions API:</h3>
${moduleInfo}
`;
}
/**
* Generates the HTML strings for all extensions and displays them in a popup.
*/
async function showExtensionsDetails() {
let htmlDefault = '<h3>Default Extensions:</h3>';
let htmlExternal = '<h3>External Extensions:</h3>';
const extensions = Object.entries(manifests).sort((a, b) => a[1].loading_order - b[1].loading_order);
for (const extension of extensions) {
const { isExternal, extensionHtml } = await getExtensionData(extension);
if (isExternal) {
htmlExternal += extensionHtml;
} else {
htmlDefault += extensionHtml;
}
}
const html = `
${getModuleInformation()}
${htmlDefault}
${htmlExternal}
`;
callPopup(`<div class="extensions_info">${html}</div>`, 'text');
}
/**
* Handles the click event for the update button of an extension.
* This function makes a POST request to '/update_extension' with the extension's name.
* If the extension is already up to date, it displays a success message.
* If the extension is not up to date, it updates the extension and displays a success message with the new commit hash.
*/
async function onUpdateClick() {
const extensionName = $(this).data('name');
try {
const response = await fetch('/update_extension', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ extensionName })
});
const data = await response.json();
if (data.isUpToDate) {
toastr.success('Extension is already up to date');
} else {
toastr.success(`Extension updated to ${data.shortCommitHash}`);
}
showExtensionsDetails();
} catch (error) {
console.error('Error:', error);
}
};
/**
* Handles the click event for the delete button of an extension.
* This function makes a POST request to '/delete_extension' with the extension's name.
* If the extension is deleted, it displays a success message.
* Creates a popup for the user to confirm before delete.
*/
async function onDeleteClick() {
const extensionName = $(this).data('name');
// use callPopup to create a popup for the user to confirm before delete
const confirmation = await callPopup(`Are you sure you want to delete ${extensionName}?`, 'delete_extension');
if (confirmation) {
try {
const response = await fetch('/delete_extension', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ extensionName })
});
} catch (error) {
console.error('Error:', error);
}
toastr.success(`Extension ${extensionName} deleted`);
showExtensionsDetails();
// reload the page to remove the extension from the list
location.reload();
}
};
/**
* Fetches the version details of a specific extension.
*
* @param {string} extensionName - The name of the extension.
* @return {object} - An object containing the extension's version details.
* This object includes the currentBranchName, currentCommitHash, isUpToDate, and remoteUrl.
* @throws {error} - If there is an error during the fetch operation, it logs the error to the console.
*/
async function getExtensionVersion(extensionName) {
try {
const response = await fetch('/get_extension_version', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ extensionName })
});
const data = await response.json();
return data;
} catch (error) {
console.error('Error:', error);
}
}
async function loadExtensionSettings(settings) {
if (settings.extension_settings) {
Object.assign(extension_settings, settings.extension_settings);
@@ -463,4 +631,6 @@ $(document).ready(async function () {
$("#extensions_details").on('click', showExtensionsDetails);
$(document).on('click', '.toggle_disable', onDisableExtensionClick);
$(document).on('click', '.toggle_enable', onEnableExtensionClick);
$(document).on('click', '.btn_update', onUpdateClick);
$(document).on('click', '.btn_delete', onDeleteClick);
});

View File

@@ -3840,12 +3840,27 @@ label[for="extensions_autoconnect"] {
.extensions_info .extension_missing {
color: gray;
}
input.extension_missing[type="checkbox"] {
opacity: 0.5;
}
#extensions_list .disabled {
text-decoration: line-through;
color: lightgray;
}
.update-button {
margin-right: 10px;
display: inline-flex;
}
/* Align the content of this span to the right */
.delete-button {
margin-right: 10px;
display: inline-flex;
}
/* possible place for WI Entry header styling */
/* .world_entry_form .inline-drawer-header {
background-color: var(--SmartThemeShadowColor);

232
server.js
View File

@@ -33,7 +33,9 @@ process.chdir(directory);
const express = require('express');
const compression = require('compression');
const app = express();
const responseTime = require('response-time')
const responseTime = require('response-time');
const simpleGit = require('simple-git');
app.use(compression());
app.use(responseTime());
@@ -2761,10 +2763,32 @@ app.post('/poe_suggest', jsonParser, async function (request, response) {
});
/**
* Discover the extension folders
* If the folder is called third-party, search for subfolders instead
*/
app.get('/discover_extensions', jsonParser, function (_, response) {
// get all folders in the extensions folder, except third-party
const extensions = fs
.readdirSync(directories.extensions)
.filter(f => fs.statSync(path.join(directories.extensions, f)).isDirectory());
.filter(f => fs.statSync(path.join(directories.extensions, f)).isDirectory())
.filter(f => f !== 'third-party');
// get all folders in the third-party folder, if it exists
if (!fs.existsSync(path.join(directories.extensions, 'third-party'))) {
return response.send(extensions);
}
const thirdPartyExtensions = fs
.readdirSync(path.join(directories.extensions, 'third-party'))
.filter(f => fs.statSync(path.join(directories.extensions, 'third-party', f)).isDirectory());
// add the third-party extensions to the extensions array
extensions.push(...thirdPartyExtensions.map(f => `third-party/${f}`));
console.log(extensions);
return response.send(extensions);
});
@@ -4417,3 +4441,207 @@ async function getImageBuffers(zipFilePath) {
});
});
}
/**
* This function extracts the extension information from the manifest file.
* @param {string} extensionPath - The path of the extension folder
* @returns {Object} - Returns the manifest data as an object
*/
async function getManifest(extensionPath) {
const manifestPath = path.join(extensionPath, 'manifest.json');
// Check if manifest.json exists
if (!fs.existsSync(manifestPath)) {
throw new Error(`Manifest file not found at ${manifestPath}`);
}
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
return manifest;
}
async function checkIfRepoIsUpToDate(extensionPath) {
const git = simpleGit();
await git.cwd(extensionPath).fetch('origin');
const currentBranch = await git.cwd(extensionPath).branch();
const currentCommitHash = await git.cwd(extensionPath).revparse(['HEAD']);
const log = await git.cwd(extensionPath).log({
from: currentCommitHash,
to: `origin/${currentBranch.current}`,
});
// Fetch remote repository information
const remotes = await git.cwd(extensionPath).getRemotes(true);
return {
isUpToDate: log.total === 0,
remoteUrl: remotes[0].refs.fetch, // URL of the remote repository
};
}
/**
* HTTP POST handler function to clone a git repository from a provided URL, read the extension manifest,
* and return extension information and path.
*
* @param {Object} request - HTTP Request object, expects a JSON body with a 'url' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/get_extension', jsonParser, async (request, response) => {
const git = simpleGit();
if (!request.body.url) {
return response.status(400).send('Bad Request: URL is required in the request body.');
}
try {
// make sure the third-party directory exists
if (!fs.existsSync(directories.extensions + '/third-party')) {
fs.mkdirSync(directories.extensions + '/third-party');
}
const url = request.body.url;
const extensionPath = path.join(directories.extensions, 'third-party', path.basename(url, '.git'));
if (fs.existsSync(extensionPath)) {
return response.status(409).send(`Directory already exists at ${extensionPath}`);
}
await git.clone(url, extensionPath);
console.log(`Extension has been cloned at ${extensionPath}`);
const { version, author, display_name } = await getManifest(extensionPath);
return response.send({ version, author, display_name, extensionPath });
} catch (error) {
console.log('Importing custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to pull the latest updates from a git repository
* based on the extension name provided in the request body. It returns the latest commit hash,
* the path of the extension, the status of the repository (whether it's up-to-date or not),
* and the remote URL of the repository.
*
* @param {Object} request - HTTP Request object, expects a JSON body with an 'extensionName' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/update_extension', jsonParser, async (request, response) => {
const git = simpleGit();
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(directories.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
const { isUpToDate, remoteUrl } = await checkIfRepoIsUpToDate(extensionPath);
const currentBranch = await git.cwd(extensionPath).branch();
if (!isUpToDate) {
await git.cwd(extensionPath).pull('origin', currentBranch.current);
console.log(`Extension has been updated at ${extensionPath}`);
} else {
console.log(`Extension is up to date at ${extensionPath}`);
}
await git.cwd(extensionPath).fetch('origin');
const fullCommitHash = await git.cwd(extensionPath).revparse(['HEAD']);
const shortCommitHash = fullCommitHash.slice(0, 7);
return response.send({ shortCommitHash, extensionPath, isUpToDate, remoteUrl });
} catch (error) {
console.log('Updating custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to get the current git commit hash and branch name for a given extension.
* It checks whether the repository is up-to-date with the remote, and returns the status along with
* the remote URL of the repository.
*
* @param {Object} request - HTTP Request object, expects a JSON body with an 'extensionName' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/get_extension_version', jsonParser, async (request, response) => {
const git = simpleGit();
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(directories.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
const currentBranch = await git.cwd(extensionPath).branch();
// get only the working branch
const currentBranchName = currentBranch.current;
await git.cwd(extensionPath).fetch('origin');
const currentCommitHash = await git.cwd(extensionPath).revparse(['HEAD']);
console.log(currentBranch, currentCommitHash);
const { isUpToDate, remoteUrl } = await checkIfRepoIsUpToDate(extensionPath);
return response.send({ currentBranchName, currentCommitHash, isUpToDate, remoteUrl });
} catch (error) {
console.log('Getting extension version failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
}
);
/**
* HTTP POST handler function to delete a git repository based on the extension name provided in the request body.
*
* @param {Object} request - HTTP Request object, expects a JSON body with a 'url' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/delete_extension', jsonParser, async (request, response) => {
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
// Sanatize the extension name to prevent directory traversal
const extensionName = sanitize(request.body.extensionName);
try {
const extensionPath = path.join(directories.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
await fs.promises.rmdir(extensionPath, { recursive: true });
console.log(`Extension has been deleted at ${extensionPath}`);
return response.send(`Extension has been deleted at ${extensionPath}`);
} catch (error) {
console.log('Deleting custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});