rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'staging' into novelai-router

Browse Source
This commit is contained in:
Cohee
2023-12-04 22:14:46 +02:00
parent e6b549bc48 9c9b600403
commit dca470c9e9
16 changed files with 809 additions and 793 deletions
Download Patch File Download Diff File Expand all files Collapse all files

497
src/endpoints/assets.js
View File

@@ -1,10 +1,12 @@
const path = require('path');
const fs = require('fs');
const express = require('express');
const sanitize = require('sanitize-filename');
const fetch = require('node-fetch').default;
const { finished } = require('stream/promises');
const writeFileSyncAtomic = require('write-file-atomic').sync;
const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants');
const { jsonParser } = require('../express-common');
const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d'];
@@ -57,273 +59,266 @@ function getFiles(dir, files = []) {
return files;
}
const router = express.Router();
/**
* Registers the endpoints for the asset management.
* @param {import('express').Express} app Express app
* @param {any} jsonParser JSON parser middleware
* HTTP POST handler function to retrieve name of all files of a given folder path.
*
* @param {Object} request - HTTP Request object. Require folder path in query
* @param {Object} response - HTTP Response object will contain a list of file path.
*
* @returns {void}
*/
function registerEndpoints(app, jsonParser) {
/**
* HTTP POST handler function to retrieve name of all files of a given folder path.
*
* @param {Object} request - HTTP Request object. Require folder path in query
* @param {Object} response - HTTP Response object will contain a list of file path.
*
* @returns {void}
*/
app.post('/api/assets/get', jsonParser, async (_, response) => {
const folderPath = path.join(DIRECTORIES.assets);
let output = {};
//console.info("Checking files into",folderPath);
router.post('/get', jsonParser, async (_, response) => {
const folderPath = path.join(DIRECTORIES.assets);
let output = {};
//console.info("Checking files into",folderPath);
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
const folders = fs.readdirSync(folderPath)
.filter(filename => {
return fs.statSync(path.join(folderPath, filename)).isDirectory();
});
for (const folder of folders) {
if (folder == 'temp')
continue;
// Live2d assets
if (folder == 'live2d') {
output[folder] = [];
const live2d_folder = path.normalize(path.join(folderPath, folder));
const files = getFiles(live2d_folder);
//console.debug("FILE FOUND:",files)
for (let file of files) {
file = path.normalize(file.replace('public' + path.sep, ''));
if (file.includes('model') && file.endsWith('.json')) {
//console.debug("Asset live2d model found:",file)
output[folder].push(path.normalize(path.join(file)));
}
}
continue;
}
// Other assets (bgm/ambient/blip)
const files = fs.readdirSync(path.join(folderPath, folder))
.filter(filename => {
return filename != '.placeholder';
});
output[folder] = [];
for (const file of files) {
output[folder].push(path.join('assets', folder, file));
}
}
}
}
catch (err) {
console.log(err);
}
return response.send(output);
});
/**
* HTTP POST handler function to download the requested asset.
*
* @param {Object} request - HTTP Request object, expects a url, a category and a filename.
* @param {Object} response - HTTP Response only gives status.
*
* @returns {void}
*/
app.post('/api/assets/download', jsonParser, async (request, response) => {
const url = request.body.url;
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const temp_path = path.join(DIRECTORIES.assets, 'temp', safe_input);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to download', url, 'to', file_path);
try {
// Download to temp
const res = await fetch(url);
if (!res.ok || res.body === null) {
throw new Error(`Unexpected response ${res.statusText}`);
}
const destination = path.resolve(temp_path);
// Delete if previous download failed
if (fs.existsSync(temp_path)) {
fs.unlink(temp_path, (err) => {
if (err) throw err;
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
const folders = fs.readdirSync(folderPath)
.filter(filename => {
return fs.statSync(path.join(folderPath, filename)).isDirectory();
});
}
const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
await finished(res.body.pipe(fileStream));
// Move into asset place
console.debug('Download finished, moving file from', temp_path, 'to', file_path);
fs.renameSync(temp_path, file_path);
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
/**
* HTTP POST handler function to delete the requested asset.
*
* @param {Object} request - HTTP Request object, expects a category and a filename
* @param {Object} response - HTTP Response only gives stats.
*
* @returns {void}
*/
app.post('/api/assets/delete', jsonParser, async (request, response) => {
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to delete', category, file_path);
try {
// Delete if previous download failed
if (fs.existsSync(file_path)) {
fs.unlink(file_path, (err) => {
if (err) throw err;
});
console.debug('Asset deleted.');
}
else {
console.debug('Asset not found.');
response.sendStatus(400);
}
// Move into asset place
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
///////////////////////////////
/**
* HTTP POST handler function to retrieve a character background music list.
*
* @param {Object} request - HTTP Request object, expects a character name in the query.
* @param {Object} response - HTTP Response object will contain a list of audio file path.
*
* @returns {void}
*/
app.post('/api/assets/character', jsonParser, async (request, response) => {
if (request.query.name === undefined) return response.sendStatus(400);
const name = sanitize(request.query.name.toString());
const inputCategory = request.query.category;
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
const folderPath = path.join(DIRECTORIES.characters, name, category);
let output = [];
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
for (const folder of folders) {
if (folder == 'temp')
continue;
// Live2d assets
if (category == 'live2d') {
const folders = fs.readdirSync(folderPath);
for (let modelFolder of folders) {
const live2dModelPath = path.join(folderPath, modelFolder);
if (fs.statSync(live2dModelPath).isDirectory()) {
for (let file of fs.readdirSync(live2dModelPath)) {
//console.debug("Character live2d model found:", file)
if (file.includes('model') && file.endsWith('.json'))
output.push(path.join('characters', name, category, modelFolder, file));
}
if (folder == 'live2d') {
output[folder] = [];
const live2d_folder = path.normalize(path.join(folderPath, folder));
const files = getFiles(live2d_folder);
//console.debug("FILE FOUND:",files)
for (let file of files) {
file = path.normalize(file.replace('public' + path.sep, ''));
if (file.includes('model') && file.endsWith('.json')) {
//console.debug("Asset live2d model found:",file)
output[folder].push(path.normalize(path.join(file)));
}
}
return response.send(output);
continue;
}
// Other assets
const files = fs.readdirSync(folderPath)
// Other assets (bgm/ambient/blip)
const files = fs.readdirSync(path.join(folderPath, folder))
.filter(filename => {
return filename != '.placeholder';
});
for (let i of files)
output.push(`/characters/${name}/${category}/${i}`);
output[folder] = [];
for (const file of files) {
output[folder].push(path.join('assets', folder, file));
}
}
return response.send(output);
}
catch (err) {
console.log(err);
return response.sendStatus(500);
}
catch (err) {
console.log(err);
}
return response.send(output);
});
/**
* HTTP POST handler function to download the requested asset.
*
* @param {Object} request - HTTP Request object, expects a url, a category and a filename.
* @param {Object} response - HTTP Response only gives status.
*
* @returns {void}
*/
router.post('/download', jsonParser, async (request, response) => {
const url = request.body.url;
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const temp_path = path.join(DIRECTORIES.assets, 'temp', safe_input);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to download', url, 'to', file_path);
try {
// Download to temp
const res = await fetch(url);
if (!res.ok || res.body === null) {
throw new Error(`Unexpected response ${res.statusText}`);
}
});
app.post('/api/file/upload', jsonParser, async (request, response) => {
try {
if (!request.body.name) {
return response.status(400).send('No upload name specified');
}
if (!request.body.data) {
return response.status(400).send('No upload data specified');
}
const safeInput = checkAssetFileName(request.body.name);
if (!safeInput) {
return response.status(400).send('Invalid upload name');
}
const pathToUpload = path.join(DIRECTORIES.files, safeInput);
writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
return response.send({ path: url });
} catch (error) {
console.log(error);
return response.sendStatus(500);
const destination = path.resolve(temp_path);
// Delete if previous download failed
if (fs.existsSync(temp_path)) {
fs.unlink(temp_path, (err) => {
if (err) throw err;
});
}
});
}
const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
await finished(res.body.pipe(fileStream));
module.exports = {
registerEndpoints,
};
// Move into asset place
console.debug('Download finished, moving file from', temp_path, 'to', file_path);
fs.renameSync(temp_path, file_path);
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
/**
* HTTP POST handler function to delete the requested asset.
*
* @param {Object} request - HTTP Request object, expects a category and a filename
* @param {Object} response - HTTP Response only gives stats.
*
* @returns {void}
*/
router.post('/delete', jsonParser, async (request, response) => {
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to delete', category, file_path);
try {
// Delete if previous download failed
if (fs.existsSync(file_path)) {
fs.unlink(file_path, (err) => {
if (err) throw err;
});
console.debug('Asset deleted.');
}
else {
console.debug('Asset not found.');
response.sendStatus(400);
}
// Move into asset place
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
///////////////////////////////
/**
* HTTP POST handler function to retrieve a character background music list.
*
* @param {Object} request - HTTP Request object, expects a character name in the query.
* @param {Object} response - HTTP Response object will contain a list of audio file path.
*
* @returns {void}
*/
router.post('/character', jsonParser, async (request, response) => {
if (request.query.name === undefined) return response.sendStatus(400);
const name = sanitize(request.query.name.toString());
const inputCategory = request.query.category;
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
const folderPath = path.join(DIRECTORIES.characters, name, category);
let output = [];
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
// Live2d assets
if (category == 'live2d') {
const folders = fs.readdirSync(folderPath);
for (let modelFolder of folders) {
const live2dModelPath = path.join(folderPath, modelFolder);
if (fs.statSync(live2dModelPath).isDirectory()) {
for (let file of fs.readdirSync(live2dModelPath)) {
//console.debug("Character live2d model found:", file)
if (file.includes('model') && file.endsWith('.json'))
output.push(path.join('characters', name, category, modelFolder, file));
}
}
}
return response.send(output);
}
// Other assets
const files = fs.readdirSync(folderPath)
.filter(filename => {
return filename != '.placeholder';
});
for (let i of files)
output.push(`/characters/${name}/${category}/${i}`);
}
return response.send(output);
}
catch (err) {
console.log(err);
return response.sendStatus(500);
}
});
router.post('/upload', jsonParser, async (request, response) => {
try {
if (!request.body.name) {
return response.status(400).send('No upload name specified');
}
if (!request.body.data) {
return response.status(400).send('No upload data specified');
}
const safeInput = checkAssetFileName(request.body.name);
if (!safeInput) {
return response.status(400).send('Invalid upload name');
}
const pathToUpload = path.join(DIRECTORIES.files, safeInput);
writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
return response.send({ path: url });
} catch (error) {
console.log(error);
return response.sendStatus(500);
}
});
module.exports = { router, checkAssetFileName };

53
src/endpoints/caption.js
View File

@@ -1,35 +1,32 @@
const express = require('express');
const { jsonParser } = require('../express-common');
const TASK = 'image-to-text';
/**
* @param {import("express").Express} app
* @param {any} jsonParser
*/
function registerEndpoints(app, jsonParser) {
app.post('/api/extra/caption', jsonParser, async (req, res) => {
try {
const { image } = req.body;
const router = express.Router();
const module = await import('../transformers.mjs');
const rawImage = await module.default.getRawImage(image);
router.post('/', jsonParser, async (req, res) => {
try {
const { image } = req.body;
if (!rawImage) {
console.log('Failed to parse captioned image');
return res.sendStatus(400);
}
const module = await import('../transformers.mjs');
const rawImage = await module.default.getRawImage(image);
const pipe = await module.default.getPipeline(TASK);
const result = await pipe(rawImage);
const text = result[0].generated_text;
console.log('Image caption:', text);
return res.json({ caption: text });
} catch (error) {
console.error(error);
return res.sendStatus(500);
if (!rawImage) {
console.log('Failed to parse captioned image');
return res.sendStatus(400);
}
});
}
module.exports = {
registerEndpoints,
};
const pipe = await module.default.getPipeline(TASK);
const result = await pipe(rawImage);
const text = result[0].generated_text;
console.log('Image caption:', text);
return res.json({ caption: text });
} catch (error) {
console.error(error);
return res.sendStatus(500);
}
});
module.exports = { router };

87
src/endpoints/classify.js
View File

@@ -1,53 +1,50 @@
const express = require('express');
const { jsonParser } = require('../express-common');
const TASK = 'text-classification';
/**
* @param {import("express").Express} app
* @param {any} jsonParser
*/
function registerEndpoints(app, jsonParser) {
const cacheObject = {};
const router = express.Router();
app.post('/api/extra/classify/labels', jsonParser, async (req, res) => {
try {
const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(TASK);
const result = Object.keys(pipe.model.config.label2id);
return res.json({ labels: result });
} catch (error) {
console.error(error);
return res.sendStatus(500);
}
});
const cacheObject = {};
app.post('/api/extra/classify', jsonParser, async (req, res) => {
try {
const { text } = req.body;
router.post('/labels', jsonParser, async (req, res) => {
try {
const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(TASK);
const result = Object.keys(pipe.model.config.label2id);
return res.json({ labels: result });
} catch (error) {
console.error(error);
return res.sendStatus(500);
}
});
async function getResult(text) {
if (Object.hasOwn(cacheObject, text)) {
return cacheObject[text];
} else {
const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(TASK);
const result = await pipe(text, { topk: 5 });
result.sort((a, b) => b.score - a.score);
cacheObject[text] = result;
return result;
}
router.post('/', jsonParser, async (req, res) => {
try {
const { text } = req.body;
async function getResult(text) {
if (Object.hasOwn(cacheObject, text)) {
return cacheObject[text];
} else {
const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(TASK);
const result = await pipe(text, { topk: 5 });
result.sort((a, b) => b.score - a.score);
cacheObject[text] = result;
return result;
}
console.log('Classify input:', text);
const result = await getResult(text);
console.log('Classify output:', result);
return res.json({ classification: result });
} catch (error) {
console.error(error);
return res.sendStatus(500);
}
});
}
module.exports = {
registerEndpoints,
};
console.log('Classify input:', text);
const result = await getResult(text);
console.log('Classify output:', result);
return res.json({ classification: result });
} catch (error) {
console.error(error);
return res.sendStatus(500);
}
});
module.exports = { router };

91
src/endpoints/content-manager.js
View File

@@ -1,8 +1,10 @@
const fs = require('fs');
const path = require('path');
const express = require('express');
const fetch = require('node-fetch').default;
const sanitize = require('sanitize-filename');
const { getConfigValue } = require('../util');
const { jsonParser } = require('../express-common');
const contentDirectory = path.join(process.cwd(), 'default/content');
const contentLogPath = path.join(contentDirectory, 'content.log');
const contentIndexPath = path.join(contentDirectory, 'index.json');
@@ -302,62 +304,57 @@ function parseJannyUrl(url) {
return uuid;
}
/**
* Registers endpoints for custom content management
* @param {import('express').Express} app Express app
* @param {any} jsonParser JSON parser middleware
*/
function registerEndpoints(app, jsonParser) {
app.post('/api/content/import', jsonParser, async (request, response) => {
if (!request.body.url) {
return response.sendStatus(400);
}
const router = express.Router();
try {
const url = request.body.url;
let result;
let type;
router.post('/import', jsonParser, async (request, response) => {
if (!request.body.url) {
return response.sendStatus(400);
}
const isJannnyContent = url.includes('janitorai');
if (isJannnyContent) {
const uuid = parseJannyUrl(url);
if (!uuid) {
return response.sendStatus(404);
}
try {
const url = request.body.url;
let result;
let type;
type = 'character';
result = await downloadJannyCharacter(uuid);
} else {
const chubParsed = parseChubUrl(url);
type = chubParsed?.type;
if (chubParsed?.type === 'character') {
console.log('Downloading chub character:', chubParsed.id);
result = await downloadChubCharacter(chubParsed.id);
}
else if (chubParsed?.type === 'lorebook') {
console.log('Downloading chub lorebook:', chubParsed.id);
result = await downloadChubLorebook(chubParsed.id);
}
else {
return response.sendStatus(404);
}
const isJannnyContent = url.includes('janitorai');
if (isJannnyContent) {
const uuid = parseJannyUrl(url);
if (!uuid) {
return response.sendStatus(404);
}
if (result.fileType) response.set('Content-Type', result.fileType);
response.set('Content-Disposition', `attachment; filename="${result.fileName}"`);
response.set('X-Custom-Content-Type', type);
return response.send(result.buffer);
} catch (error) {
console.log('Importing custom content failed', error);
return response.sendStatus(500);
type = 'character';
result = await downloadJannyCharacter(uuid);
} else {
const chubParsed = parseChubUrl(url);
type = chubParsed?.type;
if (chubParsed?.type === 'character') {
console.log('Downloading chub character:', chubParsed.id);
result = await downloadChubCharacter(chubParsed.id);
}
else if (chubParsed?.type === 'lorebook') {
console.log('Downloading chub lorebook:', chubParsed.id);
result = await downloadChubLorebook(chubParsed.id);
}
else {
return response.sendStatus(404);
}
}
});
}
if (result.fileType) response.set('Content-Type', result.fileType);
response.set('Content-Disposition', `attachment; filename="${result.fileName}"`);
response.set('X-Custom-Content-Type', type);
return response.send(result.buffer);
} catch (error) {
console.log('Importing custom content failed', error);
return response.sendStatus(500);
}
});
module.exports = {
checkForNewContent,
registerEndpoints,
getDefaultPresets,
getDefaultPresetFile,
router,
};

374
src/endpoints/extensions.js
View File

@@ -1,8 +1,10 @@
const path = require('path');
const fs = require('fs');
const express = require('express');
const { default: simpleGit } = require('simple-git');
const sanitize = require('sanitize-filename');
const { DIRECTORIES } = require('../constants');
const { jsonParser } = require('../express-common');
/**
* This function extracts the extension information from the manifest file.
@@ -45,206 +47,198 @@ async function checkIfRepoIsUpToDate(extensionPath) {
};
}
const router = express.Router();
/**
* Registers the endpoints for the third-party extensions API.
* @param {import('express').Express} app - Express app
* @param {any} jsonParser - JSON parser middleware
* HTTP POST handler function to clone a git repository from a provided URL, read the extension manifest,
* and return extension information and path.
*
* @param {Object} request - HTTP Request object, expects a JSON body with a 'url' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
function registerEndpoints(app, jsonParser) {
/**
* HTTP POST handler function to clone a git repository from a provided URL, read the extension manifest,
* and return extension information and path.
*
* @param {Object} request - HTTP Request object, expects a JSON body with a 'url' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/api/extensions/install', jsonParser, async (request, response) => {
if (!request.body.url) {
return response.status(400).send('Bad Request: URL is required in the request body.');
}
router.post('/install', jsonParser, async (request, response) => {
if (!request.body.url) {
return response.status(400).send('Bad Request: URL is required in the request body.');
}
try {
const git = simpleGit();
// make sure the third-party directory exists
if (!fs.existsSync(path.join(DIRECTORIES.extensions, 'third-party'))) {
fs.mkdirSync(path.join(DIRECTORIES.extensions, 'third-party'));
}
const url = request.body.url;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', path.basename(url, '.git'));
if (fs.existsSync(extensionPath)) {
return response.status(409).send(`Directory already exists at ${extensionPath}`);
}
await git.clone(url, extensionPath, { '--depth': 1 });
console.log(`Extension has been cloned at ${extensionPath}`);
const { version, author, display_name } = await getManifest(extensionPath);
return response.send({ version, author, display_name, extensionPath });
} catch (error) {
console.log('Importing custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to pull the latest updates from a git repository
* based on the extension name provided in the request body. It returns the latest commit hash,
* the path of the extension, the status of the repository (whether it's up-to-date or not),
* and the remote URL of the repository.
*
* @param {Object} request - HTTP Request object, expects a JSON body with an 'extensionName' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/api/extensions/update', jsonParser, async (request, response) => {
try {
const git = simpleGit();
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
const { isUpToDate, remoteUrl } = await checkIfRepoIsUpToDate(extensionPath);
const currentBranch = await git.cwd(extensionPath).branch();
if (!isUpToDate) {
await git.cwd(extensionPath).pull('origin', currentBranch.current);
console.log(`Extension has been updated at ${extensionPath}`);
} else {
console.log(`Extension is up to date at ${extensionPath}`);
}
await git.cwd(extensionPath).fetch('origin');
const fullCommitHash = await git.cwd(extensionPath).revparse(['HEAD']);
const shortCommitHash = fullCommitHash.slice(0, 7);
return response.send({ shortCommitHash, extensionPath, isUpToDate, remoteUrl });
} catch (error) {
console.log('Updating custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to get the current git commit hash and branch name for a given extension.
* It checks whether the repository is up-to-date with the remote, and returns the status along with
* the remote URL of the repository.
*
* @param {Object} request - HTTP Request object, expects a JSON body with an 'extensionName' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/api/extensions/version', jsonParser, async (request, response) => {
const git = simpleGit();
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
const currentBranch = await git.cwd(extensionPath).branch();
// get only the working branch
const currentBranchName = currentBranch.current;
await git.cwd(extensionPath).fetch('origin');
const currentCommitHash = await git.cwd(extensionPath).revparse(['HEAD']);
console.log(currentBranch, currentCommitHash);
const { isUpToDate, remoteUrl } = await checkIfRepoIsUpToDate(extensionPath);
return response.send({ currentBranchName, currentCommitHash, isUpToDate, remoteUrl });
} catch (error) {
console.log('Getting extension version failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to delete a git repository based on the extension name provided in the request body.
*
* @param {Object} request - HTTP Request object, expects a JSON body with a 'url' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/api/extensions/delete', jsonParser, async (request, response) => {
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
// Sanatize the extension name to prevent directory traversal
const extensionName = sanitize(request.body.extensionName);
try {
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
await fs.promises.rmdir(extensionPath, { recursive: true });
console.log(`Extension has been deleted at ${extensionPath}`);
return response.send(`Extension has been deleted at ${extensionPath}`);
} catch (error) {
console.log('Deleting custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* Discover the extension folders
* If the folder is called third-party, search for subfolders instead
*/
app.get('/api/extensions/discover', jsonParser, function (_, response) {
// get all folders in the extensions folder, except third-party
const extensions = fs
.readdirSync(DIRECTORIES.extensions)
.filter(f => fs.statSync(path.join(DIRECTORIES.extensions, f)).isDirectory())
.filter(f => f !== 'third-party');
// get all folders in the third-party folder, if it exists
// make sure the third-party directory exists
if (!fs.existsSync(path.join(DIRECTORIES.extensions, 'third-party'))) {
return response.send(extensions);
fs.mkdirSync(path.join(DIRECTORIES.extensions, 'third-party'));
}
const thirdPartyExtensions = fs
.readdirSync(path.join(DIRECTORIES.extensions, 'third-party'))
.filter(f => fs.statSync(path.join(DIRECTORIES.extensions, 'third-party', f)).isDirectory());
const url = request.body.url;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', path.basename(url, '.git'));
// add the third-party extensions to the extensions array
extensions.push(...thirdPartyExtensions.map(f => `third-party/${f}`));
console.log(extensions);
if (fs.existsSync(extensionPath)) {
return response.status(409).send(`Directory already exists at ${extensionPath}`);
}
await git.clone(url, extensionPath, { '--depth': 1 });
console.log(`Extension has been cloned at ${extensionPath}`);
const { version, author, display_name } = await getManifest(extensionPath);
return response.send({ version, author, display_name, extensionPath });
} catch (error) {
console.log('Importing custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to pull the latest updates from a git repository
* based on the extension name provided in the request body. It returns the latest commit hash,
* the path of the extension, the status of the repository (whether it's up-to-date or not),
* and the remote URL of the repository.
*
* @param {Object} request - HTTP Request object, expects a JSON body with an 'extensionName' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
router.post('/update', jsonParser, async (request, response) => {
const git = simpleGit();
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
const { isUpToDate, remoteUrl } = await checkIfRepoIsUpToDate(extensionPath);
const currentBranch = await git.cwd(extensionPath).branch();
if (!isUpToDate) {
await git.cwd(extensionPath).pull('origin', currentBranch.current);
console.log(`Extension has been updated at ${extensionPath}`);
} else {
console.log(`Extension is up to date at ${extensionPath}`);
}
await git.cwd(extensionPath).fetch('origin');
const fullCommitHash = await git.cwd(extensionPath).revparse(['HEAD']);
const shortCommitHash = fullCommitHash.slice(0, 7);
return response.send({ shortCommitHash, extensionPath, isUpToDate, remoteUrl });
} catch (error) {
console.log('Updating custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to get the current git commit hash and branch name for a given extension.
* It checks whether the repository is up-to-date with the remote, and returns the status along with
* the remote URL of the repository.
*
* @param {Object} request - HTTP Request object, expects a JSON body with an 'extensionName' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
router.post('/version', jsonParser, async (request, response) => {
const git = simpleGit();
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
const currentBranch = await git.cwd(extensionPath).branch();
// get only the working branch
const currentBranchName = currentBranch.current;
await git.cwd(extensionPath).fetch('origin');
const currentCommitHash = await git.cwd(extensionPath).revparse(['HEAD']);
console.log(currentBranch, currentCommitHash);
const { isUpToDate, remoteUrl } = await checkIfRepoIsUpToDate(extensionPath);
return response.send({ currentBranchName, currentCommitHash, isUpToDate, remoteUrl });
} catch (error) {
console.log('Getting extension version failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to delete a git repository based on the extension name provided in the request body.
*
* @param {Object} request - HTTP Request object, expects a JSON body with a 'url' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
router.post('/delete', jsonParser, async (request, response) => {
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
// Sanatize the extension name to prevent directory traversal
const extensionName = sanitize(request.body.extensionName);
try {
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
await fs.promises.rmdir(extensionPath, { recursive: true });
console.log(`Extension has been deleted at ${extensionPath}`);
return response.send(`Extension has been deleted at ${extensionPath}`);
} catch (error) {
console.log('Deleting custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* Discover the extension folders
* If the folder is called third-party, search for subfolders instead
*/
router.get('/discover', jsonParser, function (_, response) {
// get all folders in the extensions folder, except third-party
const extensions = fs
.readdirSync(DIRECTORIES.extensions)
.filter(f => fs.statSync(path.join(DIRECTORIES.extensions, f)).isDirectory())
.filter(f => f !== 'third-party');
// get all folders in the third-party folder, if it exists
if (!fs.existsSync(path.join(DIRECTORIES.extensions, 'third-party'))) {
return response.send(extensions);
});
}
}
module.exports = {
registerEndpoints,
};
const thirdPartyExtensions = fs
.readdirSync(path.join(DIRECTORIES.extensions, 'third-party'))
.filter(f => fs.statSync(path.join(DIRECTORIES.extensions, 'third-party', f)).isDirectory());
// add the third-party extensions to the extensions array
extensions.push(...thirdPartyExtensions.map(f => `third-party/${f}`));
console.log(extensions);
return response.send(extensions);
});
module.exports = { router };

35
src/endpoints/files.js Normal file
View File

@@ -0,0 +1,35 @@
const path = require('path');
const writeFileSyncAtomic = require('write-file-atomic').sync;
const express = require('express');
const router = express.Router();
const { checkAssetFileName } = require('./assets');
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
router.post('/upload', jsonParser, async (request, response) => {
try {
if (!request.body.name) {
return response.status(400).send('No upload name specified');
}
if (!request.body.data) {
return response.status(400).send('No upload data specified');
}
const safeInput = checkAssetFileName(request.body.name);
if (!safeInput) {
return response.status(400).send('Invalid upload name');
}
const pathToUpload = path.join(DIRECTORIES.files, safeInput);
writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
return response.send({ path: url });
} catch (error) {
console.log(error);
return response.sendStatus(500);
}
});
module.exports = { router };

399
src/endpoints/horde.js
View File

@@ -1,7 +1,9 @@
const fetch = require('node-fetch').default;
const express = require('express');
const AIHorde = require('../ai_horde');
const { getVersion, delay } = require('../util');
const { readSecret, SECRET_KEYS } = require('./secrets');
const { jsonParser } = require('../express-common');
const ANONYMOUS_KEY = '0000000000';
@@ -52,221 +54,214 @@ function sanitizeHordeImagePrompt(prompt) {
return prompt;
}
/**
*
* @param {import("express").Express} app
* @param {any} jsonParser
*/
function registerEndpoints(app, jsonParser) {
app.post('/api/horde/generate-text', jsonParser, async (request, response) => {
const router = express.Router();
router.post('/generate-text', jsonParser, async (request, response) => {
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
const url = 'https://horde.koboldai.net/api/v2/generate/text/async';
console.log(request.body);
try {
const result = await fetch(url, {
method: 'POST',
body: JSON.stringify(request.body),
headers: {
'Content-Type': 'application/json',
'apikey': api_key_horde,
'Client-Agent': String(request.header('Client-Agent')),
},
});
if (!result.ok) {
const message = await result.text();
console.log('Horde returned an error:', message);
return response.send({ error: { message } });
}
const data = await result.json();
return response.send(data);
} catch (error) {
console.log(error);
return response.send({ error: true });
}
});
router.post('/sd-samplers', jsonParser, async (_, response) => {
try {
const ai_horde = await getHordeClient();
const samplers = Object.values(ai_horde.ModelGenerationInputStableSamplers);
response.send(samplers);
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
router.post('/sd-models', jsonParser, async (_, response) => {
try {
const ai_horde = await getHordeClient();
const models = await ai_horde.getModels();
response.send(models);
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
router.post('/caption-image', jsonParser, async (request, response) => {
try {
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
const url = 'https://horde.koboldai.net/api/v2/generate/text/async';
const ai_horde = await getHordeClient();
const result = await ai_horde.postAsyncInterrogate({
source_image: request.body.image,
forms: [{ name: AIHorde.ModelInterrogationFormTypes.caption }],
}, { token: api_key_horde });
console.log(request.body);
try {
const result = await fetch(url, {
method: 'POST',
body: JSON.stringify(request.body),
headers: {
'Content-Type': 'application/json',
'apikey': api_key_horde,
'Client-Agent': String(request.header('Client-Agent')),
},
});
if (!result.ok) {
const message = await result.text();
console.log('Horde returned an error:', message);
return response.send({ error: { message } });
}
const data = await result.json();
return response.send(data);
} catch (error) {
console.log(error);
return response.send({ error: true });
}
});
app.post('/api/horde/sd-samplers', jsonParser, async (_, response) => {
try {
const ai_horde = await getHordeClient();
const samplers = Object.values(ai_horde.ModelGenerationInputStableSamplers);
response.send(samplers);
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
app.post('/api/horde/sd-models', jsonParser, async (_, response) => {
try {
const ai_horde = await getHordeClient();
const models = await ai_horde.getModels();
response.send(models);
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
app.post('/api/horde/caption-image', jsonParser, async (request, response) => {
try {
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
const ai_horde = await getHordeClient();
const result = await ai_horde.postAsyncInterrogate({
source_image: request.body.image,
forms: [{ name: AIHorde.ModelInterrogationFormTypes.caption }],
}, { token: api_key_horde });
if (!result.id) {
console.error('Image interrogation request is not satisfyable:', result.message || 'unknown error');
return response.sendStatus(400);
}
const MAX_ATTEMPTS = 200;
const CHECK_INTERVAL = 3000;
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
await delay(CHECK_INTERVAL);
const status = await ai_horde.getInterrogationStatus(result.id);
console.log(status);
if (status.state === AIHorde.HordeAsyncRequestStates.done) {
if (status.forms === undefined) {
console.error('Image interrogation request failed: no forms found.');
return response.sendStatus(500);
}
console.log('Image interrogation result:', status);
const caption = status?.forms[0]?.result?.caption || '';
if (!caption) {
console.error('Image interrogation request failed: no caption found.');
return response.sendStatus(500);
}
return response.send({ caption });
}
if (status.state === AIHorde.HordeAsyncRequestStates.faulted || status.state === AIHorde.HordeAsyncRequestStates.cancelled) {
console.log('Image interrogation request is not successful.');
return response.sendStatus(503);
}
}
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
app.post('/api/horde/user-info', jsonParser, async (_, response) => {
const api_key_horde = readSecret(SECRET_KEYS.HORDE);
if (!api_key_horde) {
return response.send({ anonymous: true });
}
try {
const ai_horde = await getHordeClient();
const user = await ai_horde.findUser({ token: api_key_horde });
return response.send(user);
} catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
app.post('/api/horde/generate-image', jsonParser, async (request, response) => {
if (!request.body.prompt) {
if (!result.id) {
console.error('Image interrogation request is not satisfyable:', result.message || 'unknown error');
return response.sendStatus(400);
}
const MAX_ATTEMPTS = 200;
const CHECK_INTERVAL = 3000;
const PROMPT_THRESHOLD = 5000;
try {
const maxLength = PROMPT_THRESHOLD - String(request.body.negative_prompt).length - 5;
if (String(request.body.prompt).length > maxLength) {
console.log('Stable Horde prompt is too long, truncating...');
request.body.prompt = String(request.body.prompt).substring(0, maxLength);
}
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
await delay(CHECK_INTERVAL);
const status = await ai_horde.getInterrogationStatus(result.id);
console.log(status);
// Sanitize prompt if requested
if (request.body.sanitize) {
const sanitized = sanitizeHordeImagePrompt(request.body.prompt);
if (status.state === AIHorde.HordeAsyncRequestStates.done) {
if (request.body.prompt !== sanitized) {
console.log('Stable Horde prompt was sanitized.');
}
request.body.prompt = sanitized;
}
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
console.log('Stable Horde request:', request.body);
const ai_horde = await getHordeClient();
const generation = await ai_horde.postAsyncImageGenerate(
{
prompt: `${request.body.prompt} ### ${request.body.negative_prompt}`,
params:
{
sampler_name: request.body.sampler,
hires_fix: request.body.enable_hr,
// @ts-ignore - use_gfpgan param is not in the type definition, need to update to new ai_horde @ https://github.com/ZeldaFan0225/ai_horde/blob/main/index.ts
use_gfpgan: request.body.restore_faces,
cfg_scale: request.body.scale,
steps: request.body.steps,
width: request.body.width,
height: request.body.height,
karras: Boolean(request.body.karras),
n: 1,
},
r2: false,
nsfw: request.body.nfsw,
models: [request.body.model],
},
{ token: api_key_horde });
if (!generation.id) {
console.error('Image generation request is not satisfyable:', generation.message || 'unknown error');
return response.sendStatus(400);
}
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
await delay(CHECK_INTERVAL);
const check = await ai_horde.getImageGenerationCheck(generation.id);
console.log(check);
if (check.done) {
const result = await ai_horde.getImageGenerationStatus(generation.id);
if (result.generations === undefined) return response.sendStatus(500);
return response.send(result.generations[0].img);
}
/*
if (!check.is_possible) {
return response.sendStatus(503);
}
*/
if (check.faulted) {
if (status.forms === undefined) {
console.error('Image interrogation request failed: no forms found.');
return response.sendStatus(500);
}
console.log('Image interrogation result:', status);
const caption = status?.forms[0]?.result?.caption || '';
if (!caption) {
console.error('Image interrogation request failed: no caption found.');
return response.sendStatus(500);
}
return response.send({ caption });
}
return response.sendStatus(504);
} catch (error) {
console.error(error);
return response.sendStatus(500);
if (status.state === AIHorde.HordeAsyncRequestStates.faulted || status.state === AIHorde.HordeAsyncRequestStates.cancelled) {
console.log('Image interrogation request is not successful.');
return response.sendStatus(503);
}
}
});
}
module.exports = {
registerEndpoints,
};
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
router.post('/user-info', jsonParser, async (_, response) => {
const api_key_horde = readSecret(SECRET_KEYS.HORDE);
if (!api_key_horde) {
return response.send({ anonymous: true });
}
try {
const ai_horde = await getHordeClient();
const user = await ai_horde.findUser({ token: api_key_horde });
return response.send(user);
} catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
router.post('/generate-image', jsonParser, async (request, response) => {
if (!request.body.prompt) {
return response.sendStatus(400);
}
const MAX_ATTEMPTS = 200;
const CHECK_INTERVAL = 3000;
const PROMPT_THRESHOLD = 5000;
try {
const maxLength = PROMPT_THRESHOLD - String(request.body.negative_prompt).length - 5;
if (String(request.body.prompt).length > maxLength) {
console.log('Stable Horde prompt is too long, truncating...');
request.body.prompt = String(request.body.prompt).substring(0, maxLength);
}
// Sanitize prompt if requested
if (request.body.sanitize) {
const sanitized = sanitizeHordeImagePrompt(request.body.prompt);
if (request.body.prompt !== sanitized) {
console.log('Stable Horde prompt was sanitized.');
}
request.body.prompt = sanitized;
}
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
console.log('Stable Horde request:', request.body);
const ai_horde = await getHordeClient();
const generation = await ai_horde.postAsyncImageGenerate(
{
prompt: `${request.body.prompt} ### ${request.body.negative_prompt}`,
params:
{
sampler_name: request.body.sampler,
hires_fix: request.body.enable_hr,
// @ts-ignore - use_gfpgan param is not in the type definition, need to update to new ai_horde @ https://github.com/ZeldaFan0225/ai_horde/blob/main/index.ts
use_gfpgan: request.body.restore_faces,
cfg_scale: request.body.scale,
steps: request.body.steps,
width: request.body.width,
height: request.body.height,
karras: Boolean(request.body.karras),
n: 1,
},
r2: false,
nsfw: request.body.nfsw,
models: [request.body.model],
},
{ token: api_key_horde });
if (!generation.id) {
console.error('Image generation request is not satisfyable:', generation.message || 'unknown error');
return response.sendStatus(400);
}
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
await delay(CHECK_INTERVAL);
const check = await ai_horde.getImageGenerationCheck(generation.id);
console.log(check);
if (check.done) {
const result = await ai_horde.getImageGenerationStatus(generation.id);
if (result.generations === undefined) return response.sendStatus(500);
return response.send(result.generations[0].img);
}
/*
if (!check.is_possible) {
return response.sendStatus(503);
}
*/
if (check.faulted) {
return response.sendStatus(500);
}
}
return response.sendStatus(504);
} catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
module.exports = { router };