rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1452 from valadaptive/assets-router 

Use Express router for assets + "files" endpoints
This commit is contained in:
Cohee 2023-12-04 21:29:52 +02:00 committed by GitHub
parent 8db1a719b8 1ac494d468
commit ddd16c1469
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 290 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
public/script.js
View File

@ -2914,7 +2914,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
let textareaText;
if (type !== 'regenerate' && type !== 'swipe' && type !== 'quiet' && !isImpersonate && !dryRun) {
is_send_press = true;
textareaText = $('#send_textarea').val();
textareaText = String($('#send_textarea').val());
$('#send_textarea').val('').trigger('input');
} else {
textareaText = '';
@ -2960,7 +2960,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
//*********************************
//for normal messages sent from user..
if ((textareaText != '' || hasPendingFileAttachment()) && !automatic_trigger && type !== 'quiet') {
if ((textareaText != '' || hasPendingFileAttachment()) && !automatic_trigger && type !== 'quiet' && !dryRun) {
// If user message contains no text other than bias - send as a system message
if (messageBias && replaceBiasMarkup(textareaText).trim().length === 0) {
sendSystemMessage(system_message_types.GENERIC, ' ', { bias: messageBias });
@ -2969,7 +2969,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
await sendMessageAsUser(textareaText, messageBias);
}
}
else if (textareaText == '' && !automatic_trigger && type === undefined && main_api == 'openai' && oai_settings.send_if_empty.trim().length > 0) {
else if (textareaText == '' && !automatic_trigger && !dryRun && type === undefined && main_api == 'openai' && oai_settings.send_if_empty.trim().length > 0) {
// Use send_if_empty if set and the user message is empty. Only when sending messages normally
await sendMessageAsUser(oai_settings.send_if_empty.trim(), messageBias);
}

4
public/scripts/chats.js
View File

@ -152,7 +152,7 @@ export async function populateFileAttachment(message, inputId = 'file_form_input
*/
export async function uploadFileAttachment(fileName, base64Data) {
try {
const result = await fetch('/api/file/upload', {
const result = await fetch('/api/files/upload', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
@ -302,7 +302,7 @@ async function viewMessageFile(messageId) {
modalTemplate.addClass('file_modal');
addCopyToCodeBlocks(modalTemplate);
callPopup(modalTemplate, 'text');
callPopup(modalTemplate, 'text', '', { wide: true, large: true });
}
/**

5
server.js
View File

@ -3597,7 +3597,10 @@ require('./src/endpoints/novelai').registerEndpoints(app, jsonParser);
require('./src/endpoints/extensions').registerEndpoints(app, jsonParser);
// Asset management
require('./src/endpoints/assets').registerEndpoints(app, jsonParser);
app.use('/api/assets', require('./src/endpoints/assets').router);
// File management
app.use('/api/files', require('./src/endpoints/files').router);
// Character sprite management
require('./src/endpoints/sprites').registerEndpoints(app, jsonParser, urlencodedParser);

497
src/endpoints/assets.js
View File

@ -1,10 +1,12 @@
const path = require('path');
const fs = require('fs');
const express = require('express');
const sanitize = require('sanitize-filename');
const fetch = require('node-fetch').default;
const { finished } = require('stream/promises');
const writeFileSyncAtomic = require('write-file-atomic').sync;
const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants');
const { jsonParser } = require('../express-common');
const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d'];
@ -57,273 +59,266 @@ function getFiles(dir, files = []) {
return files;
}
const router = express.Router();
/**
* Registers the endpoints for the asset management.
* @param {import('express').Express} app Express app
* @param {any} jsonParser JSON parser middleware
* HTTP POST handler function to retrieve name of all files of a given folder path.
*
* @param {Object} request - HTTP Request object. Require folder path in query
* @param {Object} response - HTTP Response object will contain a list of file path.
*
* @returns {void}
*/
function registerEndpoints(app, jsonParser) {
/**
* HTTP POST handler function to retrieve name of all files of a given folder path.
*
* @param {Object} request - HTTP Request object. Require folder path in query
* @param {Object} response - HTTP Response object will contain a list of file path.
*
* @returns {void}
*/
app.post('/api/assets/get', jsonParser, async (_, response) => {
const folderPath = path.join(DIRECTORIES.assets);
let output = {};
//console.info("Checking files into",folderPath);
router.post('/get', jsonParser, async (_, response) => {
const folderPath = path.join(DIRECTORIES.assets);
let output = {};
//console.info("Checking files into",folderPath);
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
const folders = fs.readdirSync(folderPath)
.filter(filename => {
return fs.statSync(path.join(folderPath, filename)).isDirectory();
});
for (const folder of folders) {
if (folder == 'temp')
continue;
// Live2d assets
if (folder == 'live2d') {
output[folder] = [];
const live2d_folder = path.normalize(path.join(folderPath, folder));
const files = getFiles(live2d_folder);
//console.debug("FILE FOUND:",files)
for (let file of files) {
file = path.normalize(file.replace('public' + path.sep, ''));
if (file.includes('model') && file.endsWith('.json')) {
//console.debug("Asset live2d model found:",file)
output[folder].push(path.normalize(path.join(file)));
}
}
continue;
}
// Other assets (bgm/ambient/blip)
const files = fs.readdirSync(path.join(folderPath, folder))
.filter(filename => {
return filename != '.placeholder';
});
output[folder] = [];
for (const file of files) {
output[folder].push(path.join('assets', folder, file));
}
}
}
}
catch (err) {
console.log(err);
}
return response.send(output);
});
/**
* HTTP POST handler function to download the requested asset.
*
* @param {Object} request - HTTP Request object, expects a url, a category and a filename.
* @param {Object} response - HTTP Response only gives status.
*
* @returns {void}
*/
app.post('/api/assets/download', jsonParser, async (request, response) => {
const url = request.body.url;
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const temp_path = path.join(DIRECTORIES.assets, 'temp', safe_input);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to download', url, 'to', file_path);
try {
// Download to temp
const res = await fetch(url);
if (!res.ok || res.body === null) {
throw new Error(`Unexpected response ${res.statusText}`);
}
const destination = path.resolve(temp_path);
// Delete if previous download failed
if (fs.existsSync(temp_path)) {
fs.unlink(temp_path, (err) => {
if (err) throw err;
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
const folders = fs.readdirSync(folderPath)
.filter(filename => {
return fs.statSync(path.join(folderPath, filename)).isDirectory();
});
}
const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
await finished(res.body.pipe(fileStream));
// Move into asset place
console.debug('Download finished, moving file from', temp_path, 'to', file_path);
fs.renameSync(temp_path, file_path);
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
/**
* HTTP POST handler function to delete the requested asset.
*
* @param {Object} request - HTTP Request object, expects a category and a filename
* @param {Object} response - HTTP Response only gives stats.
*
* @returns {void}
*/
app.post('/api/assets/delete', jsonParser, async (request, response) => {
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to delete', category, file_path);
try {
// Delete if previous download failed
if (fs.existsSync(file_path)) {
fs.unlink(file_path, (err) => {
if (err) throw err;
});
console.debug('Asset deleted.');
}
else {
console.debug('Asset not found.');
response.sendStatus(400);
}
// Move into asset place
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
///////////////////////////////
/**
* HTTP POST handler function to retrieve a character background music list.
*
* @param {Object} request - HTTP Request object, expects a character name in the query.
* @param {Object} response - HTTP Response object will contain a list of audio file path.
*
* @returns {void}
*/
app.post('/api/assets/character', jsonParser, async (request, response) => {
if (request.query.name === undefined) return response.sendStatus(400);
const name = sanitize(request.query.name.toString());
const inputCategory = request.query.category;
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
const folderPath = path.join(DIRECTORIES.characters, name, category);
let output = [];
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
for (const folder of folders) {
if (folder == 'temp')
continue;
// Live2d assets
if (category == 'live2d') {
const folders = fs.readdirSync(folderPath);
for (let modelFolder of folders) {
const live2dModelPath = path.join(folderPath, modelFolder);
if (fs.statSync(live2dModelPath).isDirectory()) {
for (let file of fs.readdirSync(live2dModelPath)) {
//console.debug("Character live2d model found:", file)
if (file.includes('model') && file.endsWith('.json'))
output.push(path.join('characters', name, category, modelFolder, file));
}
if (folder == 'live2d') {
output[folder] = [];
const live2d_folder = path.normalize(path.join(folderPath, folder));
const files = getFiles(live2d_folder);
//console.debug("FILE FOUND:",files)
for (let file of files) {
file = path.normalize(file.replace('public' + path.sep, ''));
if (file.includes('model') && file.endsWith('.json')) {
//console.debug("Asset live2d model found:",file)
output[folder].push(path.normalize(path.join(file)));
}
}
return response.send(output);
continue;
}
// Other assets
const files = fs.readdirSync(folderPath)
// Other assets (bgm/ambient/blip)
const files = fs.readdirSync(path.join(folderPath, folder))
.filter(filename => {
return filename != '.placeholder';
});
for (let i of files)
output.push(`/characters/${name}/${category}/${i}`);
output[folder] = [];
for (const file of files) {
output[folder].push(path.join('assets', folder, file));
}
}
return response.send(output);
}
catch (err) {
console.log(err);
return response.sendStatus(500);
}
catch (err) {
console.log(err);
}
return response.send(output);
});
/**
* HTTP POST handler function to download the requested asset.
*
* @param {Object} request - HTTP Request object, expects a url, a category and a filename.
* @param {Object} response - HTTP Response only gives status.
*
* @returns {void}
*/
router.post('/download', jsonParser, async (request, response) => {
const url = request.body.url;
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const temp_path = path.join(DIRECTORIES.assets, 'temp', safe_input);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to download', url, 'to', file_path);
try {
// Download to temp
const res = await fetch(url);
if (!res.ok || res.body === null) {
throw new Error(`Unexpected response ${res.statusText}`);
}
});
app.post('/api/file/upload', jsonParser, async (request, response) => {
try {
if (!request.body.name) {
return response.status(400).send('No upload name specified');
}
if (!request.body.data) {
return response.status(400).send('No upload data specified');
}
const safeInput = checkAssetFileName(request.body.name);
if (!safeInput) {
return response.status(400).send('Invalid upload name');
}
const pathToUpload = path.join(DIRECTORIES.files, safeInput);
writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
return response.send({ path: url });
} catch (error) {
console.log(error);
return response.sendStatus(500);
const destination = path.resolve(temp_path);
// Delete if previous download failed
if (fs.existsSync(temp_path)) {
fs.unlink(temp_path, (err) => {
if (err) throw err;
});
}
});
}
const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
await finished(res.body.pipe(fileStream));
module.exports = {
registerEndpoints,
};
// Move into asset place
console.debug('Download finished, moving file from', temp_path, 'to', file_path);
fs.renameSync(temp_path, file_path);
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
/**
* HTTP POST handler function to delete the requested asset.
*
* @param {Object} request - HTTP Request object, expects a category and a filename
* @param {Object} response - HTTP Response only gives stats.
*
* @returns {void}
*/
router.post('/delete', jsonParser, async (request, response) => {
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to delete', category, file_path);
try {
// Delete if previous download failed
if (fs.existsSync(file_path)) {
fs.unlink(file_path, (err) => {
if (err) throw err;
});
console.debug('Asset deleted.');
}
else {
console.debug('Asset not found.');
response.sendStatus(400);
}
// Move into asset place
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
///////////////////////////////
/**
* HTTP POST handler function to retrieve a character background music list.
*
* @param {Object} request - HTTP Request object, expects a character name in the query.
* @param {Object} response - HTTP Response object will contain a list of audio file path.
*
* @returns {void}
*/
router.post('/character', jsonParser, async (request, response) => {
if (request.query.name === undefined) return response.sendStatus(400);
const name = sanitize(request.query.name.toString());
const inputCategory = request.query.category;
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
const folderPath = path.join(DIRECTORIES.characters, name, category);
let output = [];
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
// Live2d assets
if (category == 'live2d') {
const folders = fs.readdirSync(folderPath);
for (let modelFolder of folders) {
const live2dModelPath = path.join(folderPath, modelFolder);
if (fs.statSync(live2dModelPath).isDirectory()) {
for (let file of fs.readdirSync(live2dModelPath)) {
//console.debug("Character live2d model found:", file)
if (file.includes('model') && file.endsWith('.json'))
output.push(path.join('characters', name, category, modelFolder, file));
}
}
}
return response.send(output);
}
// Other assets
const files = fs.readdirSync(folderPath)
.filter(filename => {
return filename != '.placeholder';
});
for (let i of files)
output.push(`/characters/${name}/${category}/${i}`);
}
return response.send(output);
}
catch (err) {
console.log(err);
return response.sendStatus(500);
}
});
router.post('/upload', jsonParser, async (request, response) => {
try {
if (!request.body.name) {
return response.status(400).send('No upload name specified');
}
if (!request.body.data) {
return response.status(400).send('No upload data specified');
}
const safeInput = checkAssetFileName(request.body.name);
if (!safeInput) {
return response.status(400).send('Invalid upload name');
}
const pathToUpload = path.join(DIRECTORIES.files, safeInput);
writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
return response.send({ path: url });
} catch (error) {
console.log(error);
return response.sendStatus(500);
}
});
module.exports = { router, checkAssetFileName };

35
src/endpoints/files.js Normal file
View File

@ -0,0 +1,35 @@
const path = require('path');
const writeFileSyncAtomic = require('write-file-atomic').sync;
const express = require('express');
const router = express.Router();
const { checkAssetFileName } = require('./assets');
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
router.post('/upload', jsonParser, async (request, response) => {
try {
if (!request.body.name) {
return response.status(400).send('No upload name specified');
}
if (!request.body.data) {
return response.status(400).send('No upload data specified');
}
const safeInput = checkAssetFileName(request.body.name);
if (!safeInput) {
return response.status(400).send('Invalid upload name');
}
const pathToUpload = path.join(DIRECTORIES.files, safeInput);
writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
return response.send({ path: url });
} catch (error) {
console.log(error);
return response.sendStatus(500);
}
});
module.exports = { router };