rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1472 from valadaptive/files-cleanup

Browse Source 
Clean up assets and files API
This commit is contained in:
Cohee
2023-12-07 00:52:04 +02:00
committed by GitHub
parent 100dae5fd0 9c850b0b0c
commit a83a6bc97b
6 changed files with 91 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
public/scripts/chats.js
View File

@@ -167,7 +167,7 @@ export async function uploadFileAttachment(fileName, base64Data) {
}
const responseData = await result.json();
return responseData.path.replace(/\\/g, '/');
return responseData.path;
} catch (error) {
toastr.error(String(error), 'Could not upload file');
console.error('Could not upload file', error);

2
public/scripts/utils.js
View File

@@ -950,7 +950,7 @@ export async function saveBase64AsFile(base64Data, characterName, filename = '',
// If the response is successful, get the saved image path from the server's response
if (response.ok) {
const responseData = await response.json();
return responseData.path.replace(/\\/g, '/'); // Replace backslashes with forward slashes
return responseData.path;
} else {
const errorData = await response.json();
throw new Error(errorData.error || 'Failed to upload the image to the server');

12
server.js
View File

@@ -48,7 +48,7 @@ const { jsonParser, urlencodedParser } = require('./src/express-common.js');
const contentManager = require('./src/endpoints/content-manager');
const statsHelpers = require('./statsHelpers.js');
const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets');
const { delay, getVersion, getConfigValue, color, uuidv4, humanizedISO8601DateTime, tryParse } = require('./src/util');
const { delay, getVersion, getConfigValue, color, uuidv4, humanizedISO8601DateTime, tryParse, clientRelativePath, removeFileExtension } = require('./src/util');
const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails');
const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers');
const { convertClaudePrompt } = require('./src/chat-completion');
@@ -1573,9 +1573,11 @@ app.post('/uploadimage', jsonParser, async (request, response) => {
}
// Constructing filename and path
let filename = `${Date.now()}.${format}`;
let filename;
if (request.body.filename) {
filename = `${request.body.filename}.${format}`;
filename = `${removeFileExtension(request.body.filename)}.${format}`;
} else {
filename = `${Date.now()}.${format}`;
}
// if character is defined, save to a sub folder for that character
@@ -1587,9 +1589,7 @@ app.post('/uploadimage', jsonParser, async (request, response) => {
ensureDirectoryExistence(pathToNewFile);
const imageBuffer = Buffer.from(base64Data, 'base64');
await fs.promises.writeFile(pathToNewFile, imageBuffer);
// send the path to the image, relative to the client folder, which means removing the first folder from the path which is 'public'
pathToNewFile = pathToNewFile.split(path.sep).slice(1).join(path.sep);
response.send({ path: pathToNewFile });
response.send({ path: clientRelativePath(pathToNewFile) });
} catch (error) {
console.log(error);
response.status(500).send({ error: 'Failed to save the image' });

97
src/endpoints/assets.js
View File

@@ -6,43 +6,57 @@ const fetch = require('node-fetch').default;
const { finished } = require('stream/promises');
const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants');
const { jsonParser } = require('../express-common');
const { clientRelativePath } = require('../util');
const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d'];
/**
* Sanitizes the input filename for theasset.
* Validates the input filename for the asset.
* @param {string} inputFilename Input filename
* @returns {string} Normalized or empty path if invalid
* @returns {{error: boolean, message?: string}} Whether validation failed, and why if so
*/
function sanitizeAssetFileName(inputFilename) {
function validateAssetFileName(inputFilename) {
if (!/^[a-zA-Z0-9_\-.]+$/.test(inputFilename)) {
console.debug('Bad request: illegal character in filename, only alphanumeric, \'_\', \'-\' are accepted.');
return '';
return {
error: true,
message: 'Illegal character in filename; only alphanumeric, \'_\', \'-\' are accepted.',
};
}
const inputExtension = path.extname(inputFilename).toLowerCase();
if (UNSAFE_EXTENSIONS.some(ext => ext === inputExtension)) {
console.debug('Bad request: forbidden file extension.');
return '';
return {
error: true,
message: 'Forbidden file extension.',
};
}
if (inputFilename.startsWith('.')) {
console.debug('Bad request: filename cannot start with \'.\'');
return '';
return {
error: true,
message: 'Filename cannot start with \'.\'',
};
}
return inputFilename;
if (sanitize(inputFilename) !== inputFilename) {
return {
error: true,
message: 'Reserved or long filename.',
};
}
return { error: false };
}
// Recursive function to get files
function getFiles(dir, files = []) {
// Get an array of all files and directories in the passed directory using fs.readdirSync
const fileList = fs.readdirSync(dir);
const fileList = fs.readdirSync(dir, { withFileTypes: true });
// Create the full path of the file/directory by concatenating the passed directory and file/directory name
for (const file of fileList) {
const name = `${dir}/${file}`;
const name = path.join(dir, file.name);
// Check if the current file/directory is a directory using fs.statSync
if (fs.statSync(name).isDirectory()) {
if (file.isDirectory()) {
// If it is a directory, recursively call the getFiles function with the directory path and the files array
getFiles(name, files);
} else {
@@ -70,12 +84,10 @@ router.post('/get', jsonParser, async (_, response) => {
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
const folders = fs.readdirSync(folderPath)
.filter(filename => {
return fs.statSync(path.join(folderPath, filename)).isDirectory();
});
const folders = fs.readdirSync(folderPath, { withFileTypes: true })
.filter(file => file.isDirectory());
for (const folder of folders) {
for (const { name: folder } of folders) {
if (folder == 'temp')
continue;
@@ -86,10 +98,9 @@ router.post('/get', jsonParser, async (_, response) => {
const files = getFiles(live2d_folder);
//console.debug("FILE FOUND:",files)
for (let file of files) {
file = path.normalize(file.replace('public' + path.sep, ''));
if (file.includes('model') && file.endsWith('.json')) {
//console.debug("Asset live2d model found:",file)
output[folder].push(path.normalize(path.join(file)));
output[folder].push(clientRelativePath(file));
}
}
continue;
@@ -102,7 +113,7 @@ router.post('/get', jsonParser, async (_, response) => {
});
output[folder] = [];
for (const file of files) {
output[folder].push(path.join('assets', folder, file));
output[folder].push(`assets/${folder}/${file}`);
}
}
}
@@ -124,7 +135,6 @@ router.post('/get', jsonParser, async (_, response) => {
router.post('/download', jsonParser, async (request, response) => {
const url = request.body.url;
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
@@ -137,13 +147,13 @@ router.post('/download', jsonParser, async (request, response) => {
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = sanitizeAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
// Validate filename
const validation = validateAssetFileName(request.body.filename);
if (validation.error)
return response.status(400).send(validation.message);
const temp_path = path.join(DIRECTORIES.assets, 'temp', safe_input);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
const temp_path = path.join(DIRECTORIES.assets, 'temp', request.body.filename);
const file_path = path.join(DIRECTORIES.assets, category, request.body.filename);
console.debug('Request received to download', url, 'to', file_path);
try {
@@ -183,7 +193,6 @@ router.post('/download', jsonParser, async (request, response) => {
*/
router.post('/delete', jsonParser, async (request, response) => {
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
@@ -196,12 +205,12 @@ router.post('/delete', jsonParser, async (request, response) => {
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = sanitizeAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
// Validate filename
const validation = validateAssetFileName(request.body.filename);
if (validation.error)
return response.status(400).send(validation.message);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
const file_path = path.join(DIRECTORIES.assets, category, request.body.filename);
console.debug('Request received to delete', category, file_path);
try {
@@ -236,6 +245,7 @@ router.post('/delete', jsonParser, async (request, response) => {
*/
router.post('/character', jsonParser, async (request, response) => {
if (request.query.name === undefined) return response.sendStatus(400);
// For backwards compatibility, don't reject invalid character names, just sanitize them
const name = sanitize(request.query.name.toString());
const inputCategory = request.query.category;
@@ -258,15 +268,16 @@ router.post('/character', jsonParser, async (request, response) => {
// Live2d assets
if (category == 'live2d') {
const folders = fs.readdirSync(folderPath);
for (let modelFolder of folders) {
const folders = fs.readdirSync(folderPath, { withFileTypes: true });
for (const folderInfo of folders) {
if (!folderInfo.isDirectory()) continue;
const modelFolder = folderInfo.name;
const live2dModelPath = path.join(folderPath, modelFolder);
if (fs.statSync(live2dModelPath).isDirectory()) {
for (let file of fs.readdirSync(live2dModelPath)) {
//console.debug("Character live2d model found:", file)
if (file.includes('model') && file.endsWith('.json'))
output.push(path.join('characters', name, category, modelFolder, file));
}
for (let file of fs.readdirSync(live2dModelPath)) {
//console.debug("Character live2d model found:", file)
if (file.includes('model') && file.endsWith('.json'))
output.push(path.join('characters', name, category, modelFolder, file));
}
}
return response.send(output);
@@ -289,4 +300,4 @@ router.post('/character', jsonParser, async (request, response) => {
}
});
module.exports = { router, sanitizeAssetFileName };
module.exports = { router, validateAssetFileName };

16
src/endpoints/files.js
View File

@@ -2,9 +2,10 @@ const path = require('path');
const writeFileSyncAtomic = require('write-file-atomic').sync;
const express = require('express');
const router = express.Router();
const { sanitizeAssetFileName } = require('./assets');
const { validateAssetFileName } = require('./assets');
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
const { clientRelativePath } = require('../util');
router.post('/upload', jsonParser, async (request, response) => {
try {
@@ -16,15 +17,14 @@ router.post('/upload', jsonParser, async (request, response) => {
return response.status(400).send('No upload data specified');
}
const safeInput = sanitizeAssetFileName(request.body.name);
// Validate filename
const validation = validateAssetFileName(request.body.name);
if (validation.error)
return response.status(400).send(validation.message);
if (!safeInput) {
return response.status(400).send('Invalid upload name');
}
const pathToUpload = path.join(DIRECTORIES.files, safeInput);
const pathToUpload = path.join(DIRECTORIES.files, request.body.name);
writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
const url = clientRelativePath(pathToUpload);
return response.send({ path: url });
} catch (error) {
console.log(error);

21
src/util.js
View File

@@ -288,6 +288,25 @@ function tryParse(str) {
}
}
/**
* Takes a path to a client-accessible file in the `public` folder and converts it to a relative URL segment that the
* client can fetch it from. This involves stripping the `public/` prefix and always using `/` as the separator.
* @param {string} inputPath The path to be converted.
* @returns The relative URL path from which the client can access the file.
*/
function clientRelativePath(inputPath) {
return path.normalize(inputPath).split(path.sep).slice(1).join('/');
}
/**
* Strip the last file extension from a given file name. If there are multiple extensions, only the last is removed.
* @param {string} filename The file name to remove the extension from.
* @returns The file name, sans extension
*/
function removeFileExtension(filename) {
return filename.replace(/\.[^.]+$/, '');
}
module.exports = {
getConfig,
getConfigValue,
@@ -302,4 +321,6 @@ module.exports = {
uuidv4,
humanizedISO8601DateTime,
tryParse,
clientRelativePath,
removeFileExtension,
};