rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extract endpoints for secrets and assets to separate files

This commit is contained in:
Cohee 2023-09-16 16:39:07 +03:00
parent 6e562bd1ff
commit 4e1630c17d
8 changed files with 394 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
public/scripts/extensions/assets/index.js
View File

@ -150,7 +150,7 @@ async function installAsset(url, assetType, filename) {
const category = assetType;
try {
const body = { url, category, filename };
const result = await fetch('/asset_download', {
const result = await fetch('/api/assets/download', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify(body),
@ -171,7 +171,7 @@ async function deleteAsset(assetType, filename) {
const category = assetType;
try {
const body = { category, filename };
const result = await fetch('/asset_delete', {
const result = await fetch('/api/assets/delete', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify(body),
@ -194,7 +194,7 @@ async function deleteAsset(assetType, filename) {
async function updateCurrentAssets() {
console.debug(DEBUG_PREFIX, "Checking installed assets...")
try {
const result = await fetch(`/get_assets`, {
const result = await fetch(`/api/assets/get`, {
method: 'POST',
headers: getRequestHeaders(),
});

4
public/scripts/extensions/audio/index.js
View File

@ -191,7 +191,7 @@ async function getAssetsList(type) {
console.debug(DEBUG_PREFIX, "getting assets of type", type);
try {
const result = await fetch(`/get_assets`, {
const result = await fetch(`/api/assets/get`, {
method: 'POST',
headers: getRequestHeaders(),
});
@ -209,7 +209,7 @@ async function getCharacterBgmList(name) {
console.debug(DEBUG_PREFIX, "getting bgm list for", name);
try {
const result = await fetch(`/get_character_assets_list?name=${encodeURIComponent(name)}&category=${CHARACTER_BGM_FOLDER}`, {
const result = await fetch(`/api/assets/character?name=${encodeURIComponent(name)}&category=${CHARACTER_BGM_FOLDER}`, {
method: 'POST',
headers: getRequestHeaders(),
});

4
public/scripts/secrets.js
View File

@ -73,7 +73,7 @@ export let secret_state = {};
export async function writeSecret(key, value) {
try {
const response = await fetch('/writesecret', {
const response = await fetch('/api/secrets/write', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ key, value }),
@ -94,7 +94,7 @@ export async function writeSecret(key, value) {
export async function readSecretState() {
try {
const response = await fetch('/readsecretstate', {
const response = await fetch('/api/secrets/read', {
method: 'POST',
headers: getRequestHeaders(),
});

278
server.js
View File

@ -9,7 +9,6 @@ const path = require('path');
const readline = require('readline');
const util = require('util');
const { Readable } = require('stream');
const { finished } = require('stream/promises');
const { TextDecoder } = require('util');
// cli/fs related library imports
@ -62,7 +61,7 @@ const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware');
const characterCardParser = require('./src/character-card-parser.js');
const contentManager = require('./src/content-manager');
const statsHelpers = require('./statsHelpers.js');
const { writeSecret, readSecret, readSecretState, migrateSecrets, SECRET_KEYS, getAllSecrets } = require('./src/secrets');
const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/secrets');
const { delay, getVersion, getImageBuffers } = require('./src/util');
const { invalidateThumbnail, ensureThumbnailCache } = require('./src/thumbnails');
@ -125,7 +124,6 @@ const whitelistMode = config.whitelistMode;
const autorun = config.autorun && !cliArguments.ssl;
const enableExtensions = config.enableExtensions;
const listen = config.listen;
const allowKeysExposure = config.allowKeysExposure;
const API_OPENAI = "https://api.openai.com/v1";
const API_CLAUDE = "https://api.anthropic.com/v1";
@ -3791,45 +3789,6 @@ function ensurePublicDirectoriesExist() {
}
}
app.post('/writesecret', jsonParser, (request, response) => {
const key = request.body.key;
const value = request.body.value;
writeSecret(key, value);
return response.send('ok');
});
app.post('/readsecretstate', jsonParser, (_, response) => {
try {
const state = readSecretState();
return response.send(state);
} catch (error) {
console.error(error);
return response.send({});
}
});
app.post('/viewsecrets', jsonParser, async (_, response) => {
if (!allowKeysExposure) {
console.error('secrets.json could not be viewed unless the value of allowKeysExposure in config.conf is set to true');
return response.sendStatus(403);
}
try {
const secrets = getAllSecrets();
if (!secrets) {
return response.sendStatus(404);
}
return response.send(secrets);
} catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
app.post('/delete_sprite', jsonParser, async (request, response) => {
const label = request.body.label;
const name = request.body.name;
@ -4134,237 +4093,9 @@ function importRisuSprites(data) {
}
}
/**
* HTTP POST handler function to retrieve name of all files of a given folder path.
*
* @param {Object} request - HTTP Request object. Require folder path in query
* @param {Object} response - HTTP Response object will contain a list of file path.
*
* @returns {void}
*/
app.post('/get_assets', jsonParser, async (request, response) => {
const folderPath = path.join(directories.assets);
let output = {}
//console.info("Checking files into",folderPath);
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
const folders = fs.readdirSync(folderPath)
.filter(filename => {
return fs.statSync(path.join(folderPath, filename)).isDirectory();
});
for (const folder of folders) {
if (folder == "temp")
continue;
const files = fs.readdirSync(path.join(folderPath, folder))
.filter(filename => {
return filename != ".placeholder";
});
output[folder] = [];
for (const file of files) {
output[folder].push(path.join("assets", folder, file));
}
}
}
}
catch (err) {
console.log(err);
}
finally {
return response.send(output);
}
});
function checkAssetFileName(inputFilename) {
// Sanitize filename
if (inputFilename.indexOf('\0') !== -1) {
console.debug("Bad request: poisong null bytes in filename.");
return '';
}
if (!/^[a-zA-Z0-9_\-\.]+$/.test(inputFilename)) {
console.debug("Bad request: illegal character in filename, only alphanumeric, '_', '-' are accepted.");
return '';
}
if (contentManager.unsafeExtensions.some(ext => inputFilename.toLowerCase().endsWith(ext))) {
console.debug("Bad request: forbidden file extension.");
return '';
}
if (inputFilename.startsWith('.')) {
console.debug("Bad request: filename cannot start with '.'");
return '';
}
return path.normalize(inputFilename).replace(/^(\.\.(\/|\\|$))+/, '');;
}
/**
* HTTP POST handler function to download the requested asset.
*
* @param {Object} request - HTTP Request object, expects a url, a category and a filename.
* @param {Object} response - HTTP Response only gives status.
*
* @returns {void}
*/
app.post('/asset_download', jsonParser, async (request, response) => {
const url = request.body.url;
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
const validCategories = ["bgm", "ambient"];
const fetch = require('node-fetch').default;
// Check category
let category = null;
for (let i of validCategories)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug("Bad request: unsuported asset category.");
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const temp_path = path.join(directories.assets, "temp", safe_input)
const file_path = path.join(directories.assets, category, safe_input)
console.debug("Request received to download", url, "to", file_path);
try {
// Download to temp
const res = await fetch(url);
if (!res.ok || res.body === null) {
throw new Error(`Unexpected response ${res.statusText}`);
}
const destination = path.resolve(temp_path);
// Delete if previous download failed
if (fs.existsSync(temp_path)) {
fs.unlink(temp_path, (err) => {
if (err) throw err;
});
}
const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
await finished(res.body.pipe(fileStream));
// Move into asset place
console.debug("Download finished, moving file from", temp_path, "to", file_path);
fs.renameSync(temp_path, file_path);
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
/**
* HTTP POST handler function to delete the requested asset.
*
* @param {Object} request - HTTP Request object, expects a category and a filename
* @param {Object} response - HTTP Response only gives stats.
*
* @returns {void}
*/
app.post('/asset_delete', jsonParser, async (request, response) => {
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
const validCategories = ["bgm", "ambient"];
// Check category
let category = null;
for (let i of validCategories)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug("Bad request: unsuported asset category.");
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const file_path = path.join(directories.assets, category, safe_input)
console.debug("Request received to delete", category, file_path);
try {
// Delete if previous download failed
if (fs.existsSync(file_path)) {
fs.unlink(file_path, (err) => {
if (err) throw err;
});
console.debug("Asset deleted.");
}
else {
console.debug("Asset not found.");
response.sendStatus(400);
}
// Move into asset place
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
///////////////////////////////
/**
* HTTP POST handler function to retrieve a character background music list.
*
* @param {Object} request - HTTP Request object, expects a character name in the query.
* @param {Object} response - HTTP Response object will contain a list of audio file path.
*
* @returns {void}
*/
app.post('/get_character_assets_list', jsonParser, async (request, response) => {
if (request.query.name === undefined) return response.sendStatus(400);
const name = sanitize(request.query.name.toString());
const inputCategory = request.query.category;
const validCategories = ["bgm", "ambient"]
// Check category
let category = null
for (let i of validCategories)
if (i == inputCategory)
category = i
if (category === null) {
console.debug("Bad request: unsuported asset category.");
return response.sendStatus(400);
}
const folderPath = path.join(directories.characters, name, category);
let output = [];
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
const files = fs.readdirSync(folderPath)
.filter(filename => {
return filename != ".placeholder";
});
for (let i of files)
output.push(`/characters/${name}/${category}/${i}`);
}
return response.send(output);
}
catch (err) {
console.log(err);
return response.sendStatus(500);
}
});
// Secrets managemenet
require('./src/secrets').registerEndpoints(app, jsonParser);
// Thumbnail generation
require('./src/thumbnails').registerEndpoints(app, jsonParser);
@ -4375,6 +4106,9 @@ require('./src/novelai').registerEndpoints(app, jsonParser);
// Third-party extensions
require('./src/extensions').registerEndpoints(app, jsonParser);
// Asset management
require('./src/assets').registerEndpoints(app, jsonParser);
// Stable Diffusion generation
require('./src/stable-diffusion').registerEndpoints(app, jsonParser);

250
src/assets.js Normal file
View File

@ -0,0 +1,250 @@
const path = require('path');
const fs = require('fs');
const sanitize = require('sanitize-filename');
const fetch = require('node-fetch').default;
const { finished } = require('stream/promises');
const { directories, UNSAFE_EXTENSIONS } = require('./constants');
const VALID_CATEGORIES = ["bgm", "ambient"];
/**
* Sanitizes the input filename for theasset.
* @param {string} inputFilename Input filename
* @returns {string} Normalized or empty path if invalid
*/
function checkAssetFileName(inputFilename) {
// Sanitize filename
if (inputFilename.indexOf('\0') !== -1) {
console.debug("Bad request: poisong null bytes in filename.");
return '';
}
if (!/^[a-zA-Z0-9_\-\.]+$/.test(inputFilename)) {
console.debug("Bad request: illegal character in filename, only alphanumeric, '_', '-' are accepted.");
return '';
}
if (UNSAFE_EXTENSIONS.some(ext => inputFilename.toLowerCase().endsWith(ext))) {
console.debug("Bad request: forbidden file extension.");
return '';
}
if (inputFilename.startsWith('.')) {
console.debug("Bad request: filename cannot start with '.'");
return '';
}
return path.normalize(inputFilename).replace(/^(\.\.(\/|\\|$))+/, '');;
}
/**
* Registers the endpoints for the asset management.
* @param {import('express').Express} app Express app
* @param {any} jsonParser JSON parser middleware
*/
function registerEndpoints(app, jsonParser) {
/**
* HTTP POST handler function to retrieve name of all files of a given folder path.
*
* @param {Object} request - HTTP Request object. Require folder path in query
* @param {Object} response - HTTP Response object will contain a list of file path.
*
* @returns {void}
*/
app.post('/api/assets/get', jsonParser, async (_, response) => {
const folderPath = path.join(directories.assets);
let output = {}
//console.info("Checking files into",folderPath);
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
const folders = fs.readdirSync(folderPath)
.filter(filename => {
return fs.statSync(path.join(folderPath, filename)).isDirectory();
});
for (const folder of folders) {
if (folder == "temp")
continue;
const files = fs.readdirSync(path.join(folderPath, folder))
.filter(filename => {
return filename != ".placeholder";
});
output[folder] = [];
for (const file of files) {
output[folder].push(path.join("assets", folder, file));
}
}
}
}
catch (err) {
console.log(err);
}
finally {
return response.send(output);
}
});
/**
* HTTP POST handler function to download the requested asset.
*
* @param {Object} request - HTTP Request object, expects a url, a category and a filename.
* @param {Object} response - HTTP Response only gives status.
*
* @returns {void}
*/
app.post('/api/assets/download', jsonParser, async (request, response) => {
const url = request.body.url;
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug("Bad request: unsuported asset category.");
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const temp_path = path.join(directories.assets, "temp", safe_input)
const file_path = path.join(directories.assets, category, safe_input)
console.debug("Request received to download", url, "to", file_path);
try {
// Download to temp
const res = await fetch(url);
if (!res.ok || res.body === null) {
throw new Error(`Unexpected response ${res.statusText}`);
}
const destination = path.resolve(temp_path);
// Delete if previous download failed
if (fs.existsSync(temp_path)) {
fs.unlink(temp_path, (err) => {
if (err) throw err;
});
}
const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
await finished(res.body.pipe(fileStream));
// Move into asset place
console.debug("Download finished, moving file from", temp_path, "to", file_path);
fs.renameSync(temp_path, file_path);
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
/**
* HTTP POST handler function to delete the requested asset.
*
* @param {Object} request - HTTP Request object, expects a category and a filename
* @param {Object} response - HTTP Response only gives stats.
*
* @returns {void}
*/
app.post('/api/assets/delete', jsonParser, async (request, response) => {
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug("Bad request: unsuported asset category.");
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const file_path = path.join(directories.assets, category, safe_input)
console.debug("Request received to delete", category, file_path);
try {
// Delete if previous download failed
if (fs.existsSync(file_path)) {
fs.unlink(file_path, (err) => {
if (err) throw err;
});
console.debug("Asset deleted.");
}
else {
console.debug("Asset not found.");
response.sendStatus(400);
}
// Move into asset place
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
///////////////////////////////
/**
* HTTP POST handler function to retrieve a character background music list.
*
* @param {Object} request - HTTP Request object, expects a character name in the query.
* @param {Object} response - HTTP Response object will contain a list of audio file path.
*
* @returns {void}
*/
app.post('/api/assets/character', jsonParser, async (request, response) => {
if (request.query.name === undefined) return response.sendStatus(400);
const name = sanitize(request.query.name.toString());
const inputCategory = request.query.category;
// Check category
let category = null
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i
if (category === null) {
console.debug("Bad request: unsuported asset category.");
return response.sendStatus(400);
}
const folderPath = path.join(directories.characters, name, category);
let output = [];
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
const files = fs.readdirSync(folderPath)
.filter(filename => {
return filename != ".placeholder";
});
for (let i of files)
output.push(`/characters/${name}/${category}/${i}`);
}
return response.send(output);
}
catch (err) {
console.log(err);
return response.sendStatus(500);
}
});
}
module.exports = {
registerEndpoints,
}

78
src/constants.js
View File

@ -25,6 +25,84 @@ const directories = {
assets: 'public/assets',
};
const UNSAFE_EXTENSIONS = [
".php",
".exe",
".com",
".dll",
".pif",
".application",
".gadget",
".msi",
".jar",
".cmd",
".bat",
".reg",
".sh",
".py",
".js",
".jse",
".jsp",
".pdf",
".html",
".htm",
".hta",
".vb",
".vbs",
".vbe",
".cpl",
".msc",
".scr",
".sql",
".iso",
".img",
".dmg",
".ps1",
".ps1xml",
".ps2",
".ps2xml",
".psc1",
".psc2",
".msh",
".msh1",
".msh2",
".mshxml",
".msh1xml",
".msh2xml",
".scf",
".lnk",
".inf",
".reg",
".doc",
".docm",
".docx",
".dot",
".dotm",
".dotx",
".xls",
".xlsm",
".xlsx",
".xlt",
".xltm",
".xltx",
".xlam",
".ppt",
".pptm",
".pptx",
".pot",
".potm",
".potx",
".ppam",
".ppsx",
".ppsm",
".pps",
".ppam",
".sldx",
".sldm",
".ws",
];
module.exports = {
directories,
UNSAFE_EXTENSIONS,
}

78
src/content-manager.js
View File

@ -5,83 +5,6 @@ const contentDirectory = path.join(process.cwd(), 'default/content');
const contentLogPath = path.join(contentDirectory, 'content.log');
const contentIndexPath = path.join(contentDirectory, 'index.json');
const unsafeExtensions = [
".php",
".exe",
".com",
".dll",
".pif",
".application",
".gadget",
".msi",
".jar",
".cmd",
".bat",
".reg",
".sh",
".py",
".js",
".jse",
".jsp",
".pdf",
".html",
".htm",
".hta",
".vb",
".vbs",
".vbe",
".cpl",
".msc",
".scr",
".sql",
".iso",
".img",
".dmg",
".ps1",
".ps1xml",
".ps2",
".ps2xml",
".psc1",
".psc2",
".msh",
".msh1",
".msh2",
".mshxml",
".msh1xml",
".msh2xml",
".scf",
".lnk",
".inf",
".reg",
".doc",
".docm",
".docx",
".dot",
".dotm",
".dotx",
".xls",
".xlsm",
".xlsx",
".xlt",
".xltm",
".xltx",
".xlam",
".ppt",
".pptm",
".pptx",
".pot",
".potm",
".potx",
".ppam",
".ppsx",
".ppsm",
".pps",
".ppam",
".sldx",
".sldm",
".ws",
];
function checkForNewContent() {
try {
if (config.skipContentCheck) {
@ -162,5 +85,4 @@ function getContentLog() {
module.exports = {
checkForNewContent,
unsafeExtensions,
}

55
src/secrets.js
View File

@ -1,5 +1,6 @@
const fs = require('fs');
const path = require('path');
const { getConfigValue } = require('./util');
const writeFileAtomicSync = require('write-file-atomic').sync;
const SECRETS_FILE = path.join(process.cwd(), './secrets.json');
@ -34,7 +35,7 @@ function writeSecret(key, value) {
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8');
const secrets = JSON.parse(fileContents);
secrets[key] = value;
writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets), "utf-8");
writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets, null, 4), "utf-8");
}
/**
@ -114,7 +115,7 @@ function migrateSecrets(settingsFile) {
if (modified) {
console.log('Writing updated settings.json...');
const settingsContent = JSON.stringify(settings);
const settingsContent = JSON.stringify(settings, null, 4);
writeFileAtomicSync(settingsFile, settingsContent, "utf-8");
}
}
@ -138,11 +139,61 @@ function getAllSecrets() {
return secrets;
}
/**
* Registers endpoints for the secret management API
* @param {import('express').Express} app Express app
* @param {any} jsonParser JSON parser middleware
*/
function registerEndpoints(app, jsonParser) {
app.post('/api/secrets/write', jsonParser, (request, response) => {
const key = request.body.key;
const value = request.body.value;
writeSecret(key, value);
return response.send('ok');
});
app.post('/api/secrets/read', jsonParser, (_, response) => {
try {
const state = readSecretState();
return response.send(state);
} catch (error) {
console.error(error);
return response.send({});
}
});
app.post('/viewsecrets', jsonParser, async (_, response) => {
const allowKeysExposure = getConfigValue('allowKeysExposure', false);
if (!allowKeysExposure) {
console.error('secrets.json could not be viewed unless the value of allowKeysExposure in config.conf is set to true');
return response.sendStatus(403);
}
try {
const secrets = getAllSecrets();
if (!secrets) {
return response.sendStatus(404);
}
return response.send(secrets);
} catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
}
module.exports = {
writeSecret,
readSecret,
readSecretState,
migrateSecrets,
getAllSecrets,
registerEndpoints,
SECRET_KEYS,
};