rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-02-02 12:26:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

Update all endpoints to use user directories

Browse Source
This commit is contained in:
Cohee 2024-04-07 01:47:07 +03:00
parent cd5aec7368
commit b07a6a9a78
39 changed files with 941 additions and 751 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
jsconfig.json
View File

@ -12,6 +12,7 @@
},
"exclude": [
"node_modules",
"**/node_modules/*"
"**/node_modules/*",
"public/lib"
]
}

10
public/script.js
View File

@ -1543,7 +1543,7 @@ function getCharacterSource(chId = this_chid) {
}
async function getCharacters() {
var response = await fetch('/api/characters/all', {
const response = await fetch('/api/characters/all', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
@ -1551,11 +1551,9 @@ async function getCharacters() {
}),
});
if (response.ok === true) {
var getData = ''; //RossAscends: reset to force array to update to account for deleted character.
getData = await response.json();
const load_ch_count = Object.getOwnPropertyNames(getData);
for (var i = 0; i < load_ch_count.length; i++) {
characters[i] = [];
characters.splice(0, characters.length);
const getData = await response.json();
for (let i = 0; i < getData.length; i++) {
characters[i] = getData[i];
characters[i]['name'] = DOMPurify.sanitize(characters[i]['name']);

3
server.js
View File

@ -212,7 +212,8 @@ if (enableCorsProxy) {
}
app.use(express.static(process.cwd() + '/public', {}));
app.use(userDataMiddleware(app));
app.use(userDataMiddleware());
app.use('/', require('./src/users').router);
app.use(multer({ dest: UPLOADS_PATH, limits: { fieldSize: 10 * 1024 * 1024 } }).single('avatar'));
app.get('/', function (request, response) {

94
src/additional-headers.js
View File

@ -2,8 +2,13 @@ const { TEXTGEN_TYPES, OPENROUTER_HEADERS } = require('./constants');
const { SECRET_KEYS, readSecret } = require('./endpoints/secrets');
const { getConfigValue } = require('./util');
function getMancerHeaders() {
const apiKey = readSecret(SECRET_KEYS.MANCER);
/**
* Gets the headers for the Mancer API.
* @param {import('./users').UserDirectoryList} directories User directories
* @returns {object} Headers for the request
*/
function getMancerHeaders(directories) {
const apiKey = readSecret(directories, SECRET_KEYS.MANCER);
return apiKey ? ({
'X-API-KEY': apiKey,
@ -11,39 +16,64 @@ function getMancerHeaders() {
}) : {};
}
function getTogetherAIHeaders() {
const apiKey = readSecret(SECRET_KEYS.TOGETHERAI);
/**
* Gets the headers for the TogetherAI API.
* @param {import('./users').UserDirectoryList} directories User directories
* @returns {object} Headers for the request
*/
function getTogetherAIHeaders(directories) {
const apiKey = readSecret(directories, SECRET_KEYS.TOGETHERAI);
return apiKey ? ({
'Authorization': `Bearer ${apiKey}`,
}) : {};
}
function getInfermaticAIHeaders() {
const apiKey = readSecret(SECRET_KEYS.INFERMATICAI);
/**
* Gets the headers for the InfermaticAI API.
* @param {import('./users').UserDirectoryList} directories User directories
* @returns {object} Headers for the request
*/
function getInfermaticAIHeaders(directories) {
const apiKey = readSecret(directories, SECRET_KEYS.INFERMATICAI);
return apiKey ? ({
'Authorization': `Bearer ${apiKey}`,
}) : {};
}
function getDreamGenHeaders() {
const apiKey = readSecret(SECRET_KEYS.DREAMGEN);
/**
* Gets the headers for the DreamGen API.
* @param {import('./users').UserDirectoryList} directories User directories
* @returns {object} Headers for the request
*/
function getDreamGenHeaders(directories) {
const apiKey = readSecret(directories, SECRET_KEYS.DREAMGEN);
return apiKey ? ({
'Authorization': `Bearer ${apiKey}`,
}) : {};
}
function getOpenRouterHeaders() {
const apiKey = readSecret(SECRET_KEYS.OPENROUTER);
/**
* Gets the headers for the OpenRouter API.
* @param {import('./users').UserDirectoryList} directories User directories
* @returns {object} Headers for the request
*/
function getOpenRouterHeaders(directories) {
const apiKey = readSecret(directories, SECRET_KEYS.OPENROUTER);
const baseHeaders = { ...OPENROUTER_HEADERS };
return apiKey ? Object.assign(baseHeaders, { 'Authorization': `Bearer ${apiKey}` }) : baseHeaders;
}
function getAphroditeHeaders() {
const apiKey = readSecret(SECRET_KEYS.APHRODITE);
/**
* Gets the headers for the Aphrodite API.
* @param {import('./users').UserDirectoryList} directories User directories
* @returns {object} Headers for the request
*/
function getAphroditeHeaders(directories) {
const apiKey = readSecret(directories, SECRET_KEYS.APHRODITE);
return apiKey ? ({
'X-API-KEY': apiKey,
@ -51,8 +81,13 @@ function getAphroditeHeaders() {
}) : {};
}
function getTabbyHeaders() {
const apiKey = readSecret(SECRET_KEYS.TABBY);
/**
* Gets the headers for the Tabby API.
* @param {import('./users').UserDirectoryList} directories User directories
* @returns {object} Headers for the request
*/
function getTabbyHeaders(directories) {
const apiKey = readSecret(directories, SECRET_KEYS.TABBY);
return apiKey ? ({
'x-api-key': apiKey,
@ -60,24 +95,39 @@ function getTabbyHeaders() {
}) : {};
}
function getLlamaCppHeaders() {
const apiKey = readSecret(SECRET_KEYS.LLAMACPP);
/**
* Gets the headers for the LlamaCPP API.
* @param {import('./users').UserDirectoryList} directories User directories
* @returns {object} Headers for the request
*/
function getLlamaCppHeaders(directories) {
const apiKey = readSecret(directories, SECRET_KEYS.LLAMACPP);
return apiKey ? ({
'Authorization': `Bearer ${apiKey}`,
}) : {};
}
function getOobaHeaders() {
const apiKey = readSecret(SECRET_KEYS.OOBA);
/**
* Gets the headers for the Ooba API.
* @param {import('./users').UserDirectoryList} directories
* @returns {object} Headers for the request
*/
function getOobaHeaders(directories) {
const apiKey = readSecret(directories, SECRET_KEYS.OOBA);
return apiKey ? ({
'Authorization': `Bearer ${apiKey}`,
}) : {};
}
function getKoboldCppHeaders() {
const apiKey = readSecret(SECRET_KEYS.KOBOLDCPP);
/**
* Gets the headers for the KoboldCpp API.
* @param {import('./users').UserDirectoryList} directories
* @returns {object} Headers for the request
*/
function getKoboldCppHeaders(directories) {
const apiKey = readSecret(directories, SECRET_KEYS.KOBOLDCPP);
return apiKey ? ({
'Authorization': `Bearer ${apiKey}`,
@ -96,7 +146,7 @@ function getOverrideHeaders(urlHost) {
/**
* Sets additional headers for the request.
* @param {object} request Original request body
* @param {import('express').Request} request Original request body
* @param {object} args New request arguments
* @param {string|null} server API server for new request
*/
@ -115,7 +165,7 @@ function setAdditionalHeaders(request, args, server) {
};
const getHeaders = headerGetters[request.body.api_type];
const headers = getHeaders ? getHeaders() : {};
const headers = getHeaders ? getHeaders(request.user.directories) : {};
if (typeof server === 'string' && server.length > 0) {
try {

4
src/constants.js
View File

@ -2,6 +2,7 @@ const PUBLIC_DIRECTORIES = {
images: 'public/img/',
backups: 'backups/',
sounds: 'public/sounds',
extensions: 'public/scripts/extensions',
};
/**
@ -29,13 +30,14 @@ const USER_DIRECTORY_TEMPLATE = Object.freeze({
textGen_Settings: 'TextGen Settings',
themes: 'themes',
movingUI: 'movingUI',
extensions: 'scripts/extensions',
extensions: 'extensions',
instruct: 'instruct',
context: 'context',
quickreplies: 'QuickReplies',
assets: 'assets',
comfyWorkflows: 'user/workflows',
files: 'user/files',
vectors: 'vectors',
});
const DEFAULT_USER = Object.freeze({

2
src/endpoints/anthropic.js
View File

@ -39,7 +39,7 @@ router.post('/caption-image', jsonParser, async (request, response) => {
headers: {
'Content-Type': 'application/json',
'anthropic-version': '2023-06-01',
'x-api-key': request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.CLAUDE),
'x-api-key': request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE),
},
timeout: 0,
});

48
src/endpoints/assets.js
View File

@ -4,11 +4,11 @@ const express = require('express');
const sanitize = require('sanitize-filename');
const fetch = require('node-fetch').default;
const { finished } = require('stream/promises');
const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants');
const { UNSAFE_EXTENSIONS } = require('../constants');
const { jsonParser } = require('../express-common');
const { clientRelativePath } = require('../util');
const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d', 'vrm', 'character'];
const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d', 'vrm', 'character', 'temp'];
/**
* Validates the input filename for the asset.
@ -48,7 +48,12 @@ function validateAssetFileName(inputFilename) {
return { error: false };
}
// Recursive function to get files
/**
* Recursive function to get files
* @param {string} dir - The directory to search for files
* @param {string[]} files - The array of files to return
* @returns {string[]} - The array of files
*/
function getFiles(dir, files = []) {
// Get an array of all files and directories in the passed directory using fs.readdirSync
const fileList = fs.readdirSync(dir, { withFileTypes: true });
@ -77,13 +82,23 @@ const router = express.Router();
*
* @returns {void}
*/
router.post('/get', jsonParser, async (_, response) => {
const folderPath = path.join(DIRECTORIES.assets);
router.post('/get', jsonParser, async (request, response) => {
const folderPath = path.join(request.user.directories.assets);
let output = {};
//console.info("Checking files into",folderPath);
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
for (const category of VALID_CATEGORIES) {
const assetCategoryPath = path.join(folderPath, category);
if (fs.existsSync(assetCategoryPath) && !fs.statSync(assetCategoryPath).isDirectory()) {
fs.unlinkSync(assetCategoryPath);
}
if (!fs.existsSync(assetCategoryPath)) {
fs.mkdirSync(assetCategoryPath);
}
}
const folders = fs.readdirSync(folderPath, { withFileTypes: true })
.filter(file => file.isDirectory());
@ -100,7 +115,7 @@ router.post('/get', jsonParser, async (_, response) => {
for (let file of files) {
if (file.includes('model') && file.endsWith('.json')) {
//console.debug("Asset live2d model found:",file)
output[folder].push(clientRelativePath(file));
output[folder].push(clientRelativePath(request.user.directories.root, file));
}
}
continue;
@ -116,7 +131,7 @@ router.post('/get', jsonParser, async (_, response) => {
for (let file of files) {
if (!file.endsWith('.placeholder')) {
//console.debug("Asset VRM model found:",file)
output['vrm']['model'].push(clientRelativePath(file));
output['vrm']['model'].push(clientRelativePath(request.user.directories.root, file));
}
}
@ -127,7 +142,7 @@ router.post('/get', jsonParser, async (_, response) => {
for (let file of files) {
if (!file.endsWith('.placeholder')) {
//console.debug("Asset VRM animation found:",file)
output['vrm']['animation'].push(clientRelativePath(file));
output['vrm']['animation'].push(clientRelativePath(request.user.directories.root, file));
}
}
continue;
@ -170,7 +185,7 @@ router.post('/download', jsonParser, async (request, response) => {
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
console.debug('Bad request: unsupported asset category.');
return response.sendStatus(400);
}
@ -179,8 +194,8 @@ router.post('/download', jsonParser, async (request, response) => {
if (validation.error)
return response.status(400).send(validation.message);
const temp_path = path.join(DIRECTORIES.assets, 'temp', request.body.filename);
const file_path = path.join(DIRECTORIES.assets, category, request.body.filename);
const temp_path = path.join(request.user.directories.assets, 'temp', request.body.filename);
const file_path = path.join(request.user.directories.assets, category, request.body.filename);
console.debug('Request received to download', url, 'to', file_path);
try {
@ -197,6 +212,7 @@ router.post('/download', jsonParser, async (request, response) => {
});
}
const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
// @ts-ignore
await finished(res.body.pipe(fileStream));
if (category === 'character') {
@ -235,7 +251,7 @@ router.post('/delete', jsonParser, async (request, response) => {
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
console.debug('Bad request: unsupported asset category.');
return response.sendStatus(400);
}
@ -244,7 +260,7 @@ router.post('/delete', jsonParser, async (request, response) => {
if (validation.error)
return response.status(400).send(validation.message);
const file_path = path.join(DIRECTORIES.assets, category, request.body.filename);
const file_path = path.join(request.user.directories.assets, category, request.body.filename);
console.debug('Request received to delete', category, file_path);
try {
@ -290,11 +306,11 @@ router.post('/character', jsonParser, async (request, response) => {
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
console.debug('Bad request: unsupported asset category.');
return response.sendStatus(400);
}
const folderPath = path.join(DIRECTORIES.characters, name, category);
const folderPath = path.join(request.user.directories.characters, name, category);
let output = [];
try {

8
src/endpoints/avatars.js
View File

@ -4,7 +4,7 @@ const fs = require('fs');
const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser, urlencodedParser } = require('../express-common');
const { DIRECTORIES, AVATAR_WIDTH, AVATAR_HEIGHT, UPLOADS_PATH } = require('../constants');
const { AVATAR_WIDTH, AVATAR_HEIGHT, UPLOADS_PATH } = require('../constants');
const { getImages, tryParse } = require('../util');
// image processing related library imports
@ -13,7 +13,7 @@ const jimp = require('jimp');
const router = express.Router();
router.post('/get', jsonParser, function (request, response) {
var images = getImages(DIRECTORIES.avatars);
var images = getImages(request.user.directories.avatars);
response.send(JSON.stringify(images));
});
@ -25,7 +25,7 @@ router.post('/delete', jsonParser, function (request, response) {
return response.sendStatus(403);
}
const fileName = path.join(DIRECTORIES.avatars, sanitize(request.body.avatar));
const fileName = path.join(request.user.directories.avatars, sanitize(request.body.avatar));
if (fs.existsSync(fileName)) {
fs.rmSync(fileName);
@ -50,7 +50,7 @@ router.post('/upload', urlencodedParser, async (request, response) => {
const image = await rawImg.cover(AVATAR_WIDTH, AVATAR_HEIGHT).getBufferAsync(jimp.MIME_PNG);
const filename = request.body.overwrite_name || `${Date.now()}.png`;
const pathToNewFile = path.join(DIRECTORIES.avatars, filename);
const pathToNewFile = path.join(request.user.directories.avatars, filename);
writeFileAtomicSync(pathToNewFile, image);
fs.rmSync(pathToUpload);
return response.send({ path: filename });

36
src/endpoints/backends/chat-completions.js
View File

@ -81,7 +81,7 @@ async function parseCohereStream(jsonStream, request, response) {
*/
async function sendClaudeRequest(request, response) {
const apiUrl = new URL(request.body.reverse_proxy || API_CLAUDE).toString();
const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.CLAUDE);
const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.CLAUDE);
const divider = '-'.repeat(process.stdout.columns);
if (!apiKey) {
@ -162,7 +162,7 @@ async function sendClaudeRequest(request, response) {
*/
async function sendScaleRequest(request, response) {
const apiUrl = new URL(request.body.api_url_scale).toString();
const apiKey = readSecret(SECRET_KEYS.SCALE);
const apiKey = readSecret(request.user.directories, SECRET_KEYS.SCALE);
if (!apiKey) {
console.log('Scale API key is missing.');
@ -213,7 +213,7 @@ async function sendScaleRequest(request, response) {
* @param {express.Response} response Express response
*/
async function sendMakerSuiteRequest(request, response) {
const apiKey = readSecret(SECRET_KEYS.MAKERSUITE);
const apiKey = readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
if (!apiKey) {
console.log('MakerSuite API key is missing.');
@ -367,7 +367,7 @@ async function sendAI21Request(request, response) {
headers: {
accept: 'application/json',
'content-type': 'application/json',
Authorization: `Bearer ${readSecret(SECRET_KEYS.AI21)}`,
Authorization: `Bearer ${readSecret(request.user.directories, SECRET_KEYS.AI21)}`,
},
body: JSON.stringify({
numResults: 1,
@ -431,7 +431,7 @@ async function sendAI21Request(request, response) {
*/
async function sendMistralAIRequest(request, response) {
const apiUrl = new URL(request.body.reverse_proxy || API_MISTRAL).toString();
const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.MISTRALAI);
const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MISTRALAI);
if (!apiKey) {
console.log('MistralAI API key is missing.');
@ -522,8 +522,14 @@ async function sendMistralAIRequest(request, response) {
}
}
/**
* Sends a request to Cohere API.
* @param {import('express').Request} request
* @param {import('express').Response} response
* @returns {Promise<any>}
*/
async function sendCohereRequest(request, response) {
const apiKey = readSecret(SECRET_KEYS.COHERE);
const apiKey = readSecret(request.user.directories, SECRET_KEYS.COHERE);
const controller = new AbortController();
request.socket.removeAllListeners('close');
request.socket.on('close', function () {
@ -612,25 +618,25 @@ router.post('/status', jsonParser, async function (request, response_getstatus_o
if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
api_url = new URL(request.body.reverse_proxy || API_OPENAI).toString();
api_key_openai = request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.OPENAI);
api_key_openai = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI);
headers = {};
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
api_url = 'https://openrouter.ai/api/v1';
api_key_openai = readSecret(SECRET_KEYS.OPENROUTER);
api_key_openai = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
// OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
headers = { ...OPENROUTER_HEADERS };
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.MISTRALAI) {
api_url = new URL(request.body.reverse_proxy || API_MISTRAL).toString();
api_key_openai = request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.MISTRALAI);
api_key_openai = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MISTRALAI);
headers = {};
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
api_url = request.body.custom_url;
api_key_openai = readSecret(SECRET_KEYS.CUSTOM);
api_key_openai = readSecret(request.user.directories, SECRET_KEYS.CUSTOM);
headers = {};
mergeObjectWithYaml(headers, request.body.custom_include_headers);
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COHERE) {
api_url = API_COHERE;
api_key_openai = readSecret(SECRET_KEYS.COHERE);
api_key_openai = readSecret(request.user.directories, SECRET_KEYS.COHERE);
headers = {};
} else {
console.log('This chat completion source is not supported yet.');
@ -795,10 +801,11 @@ router.post('/generate', jsonParser, function (request, response) {
if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENAI) {
apiUrl = new URL(request.body.reverse_proxy || API_OPENAI).toString();
apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.OPENAI);
apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.OPENAI);
headers = {};
bodyParams = {
logprobs: request.body.logprobs,
top_logprobs: undefined,
};
// Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }
@ -812,7 +819,7 @@ router.post('/generate', jsonParser, function (request, response) {
}
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.OPENROUTER) {
apiUrl = 'https://openrouter.ai/api/v1';
apiKey = readSecret(SECRET_KEYS.OPENROUTER);
apiKey = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
// OpenRouter needs to pass the Referer and X-Title: https://openrouter.ai/docs#requests
headers = { ...OPENROUTER_HEADERS };
bodyParams = { 'transforms': ['middle-out'] };
@ -834,10 +841,11 @@ router.post('/generate', jsonParser, function (request, response) {
}
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.CUSTOM) {
apiUrl = request.body.custom_url;
apiKey = readSecret(SECRET_KEYS.CUSTOM);
apiKey = readSecret(request.user.directories, SECRET_KEYS.CUSTOM);
headers = {};
bodyParams = {
logprobs: request.body.logprobs,
top_logprobs: undefined,
};
// Adjust logprobs params for Chat Completions API, which expects { top_logprobs: number; logprobs: boolean; }

2
src/endpoints/backends/scale-alt.js
View File

@ -14,7 +14,7 @@ router.post('/generate', jsonParser, function (request, response) {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'cookie': `_jwt=${readSecret(SECRET_KEYS.SCALE_COOKIE)}`,
'cookie': `_jwt=${readSecret(request.user.directories, SECRET_KEYS.SCALE_COOKIE)}`,
},
body: JSON.stringify({
json: {

19
src/endpoints/backgrounds.js
View File

@ -4,16 +4,15 @@ const express = require('express');
const sanitize = require('sanitize-filename');
const { jsonParser, urlencodedParser } = require('../express-common');
const { DIRECTORIES, UPLOADS_PATH } = require('../constants');
const { UPLOADS_PATH } = require('../constants');
const { invalidateThumbnail } = require('./thumbnails');
const { getImages } = require('../util');
const router = express.Router();
router.post('/all', jsonParser, function (request, response) {
var images = getImages('public/backgrounds');
var images = getImages(request.user.directories.backgrounds);
response.send(JSON.stringify(images));
});
router.post('/delete', jsonParser, function (request, response) {
@ -24,7 +23,7 @@ router.post('/delete', jsonParser, function (request, response) {
return response.sendStatus(403);
}
const fileName = path.join('public/backgrounds/', sanitize(request.body.bg));
const fileName = path.join(request.user.directories.backgrounds, sanitize(request.body.bg));
if (!fs.existsSync(fileName)) {
console.log('BG file not found');
@ -32,15 +31,15 @@ router.post('/delete', jsonParser, function (request, response) {
}
fs.rmSync(fileName);
invalidateThumbnail('bg', request.body.bg);
invalidateThumbnail(request.user.directories, 'bg', request.body.bg);
return response.send('ok');
});
router.post('/rename', jsonParser, function (request, response) {
if (!request.body) return response.sendStatus(400);
const oldFileName = path.join(DIRECTORIES.backgrounds, sanitize(request.body.old_bg));
const newFileName = path.join(DIRECTORIES.backgrounds, sanitize(request.body.new_bg));
const oldFileName = path.join(request.user.directories.backgrounds, sanitize(request.body.old_bg));
const newFileName = path.join(request.user.directories.backgrounds, sanitize(request.body.new_bg));
if (!fs.existsSync(oldFileName)) {
console.log('BG file not found');
@ -53,7 +52,7 @@ router.post('/rename', jsonParser, function (request, response) {
}
fs.renameSync(oldFileName, newFileName);
invalidateThumbnail('bg', request.body.old_bg);
invalidateThumbnail(request.user.directories, 'bg', request.body.old_bg);
return response.send('ok');
});
@ -64,8 +63,8 @@ router.post('/upload', urlencodedParser, function (request, response) {
const filename = request.file.originalname;
try {
fs.renameSync(img_path, path.join('public/backgrounds/', filename));
invalidateThumbnail('bg', filename);
fs.renameSync(img_path, path.join(request.user.directories.backgrounds, filename));
invalidateThumbnail(request.user.directories, 'bg', filename);
response.send(filename);
} catch (err) {
console.error(err);

647
src/endpoints/characters.js
View File

@ -9,7 +9,7 @@ const _ = require('lodash');
const jimp = require('jimp');
const { DIRECTORIES, UPLOADS_PATH, AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants');
const { UPLOADS_PATH, AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants');
const { jsonParser, urlencodedParser } = require('../express-common');
const { deepMerge, humanizedISO8601DateTime, tryParse } = require('../util');
const { TavernCardValidator } = require('../validator/TavernCardValidator');
@ -19,82 +19,99 @@ const { invalidateThumbnail } = require('./thumbnails');
const { importRisuSprites } = require('./sprites');
const defaultAvatarPath = './public/img/ai4.png';
let characters = {};
// KV-store for parsed character data
const characterDataCache = new Map();
/**
* Reads the character card from the specified image file.
* @param {string} img_url - Path to the image file
* @param {string} input_format - 'png'
* @param {string} inputFile - Path to the image file
* @param {string} inputFormat - 'png'
* @returns {Promise<string | undefined>} - Character card data
*/
async function charaRead(img_url, input_format = 'png') {
const stat = fs.statSync(img_url);
const cacheKey = `${img_url}-${stat.mtimeMs}`;
async function readCharacterData(inputFile, inputFormat = 'png') {
const stat = fs.statSync(inputFile);
const cacheKey = `${inputFile}-${stat.mtimeMs}`;
if (characterDataCache.has(cacheKey)) {
return characterDataCache.get(cacheKey);
}
const result = characterCardParser.parse(img_url, input_format);
const result = characterCardParser.parse(inputFile, inputFormat);
characterDataCache.set(cacheKey, result);
return result;
}
/**
* @param {express.Response | undefined} response
* @param {{file_name: string} | string} mes
* Writes the character card to the specified image file.
* @param {string} inputFile - Path to the image file
* @param {string} data - Character card data
* @param {string} outputFile - Target image file name
* @param {import('express').Request} request - Express request obejct
* @param {Crop|undefined} crop - Crop parameters
* @returns {Promise<boolean>} - True if the operation was successful
*/
async function charaWrite(img_url, data, target_img, response = undefined, mes = 'ok', crop = undefined) {
async function writeCharacterData(inputFile, data, outputFile, request, crop = undefined) {
try {
// Reset the cache
for (const key of characterDataCache.keys()) {
if (key.startsWith(img_url)) {
if (key.startsWith(inputFile)) {
characterDataCache.delete(key);
break;
}
}
// Read the image, resize, and save it as a PNG into the buffer
const inputImage = await tryReadImage(img_url, crop);
const inputImage = await tryReadImage(inputFile, crop);
// Get the chunks
const outputImage = characterCardParser.write(inputImage, data);
const outputImagePath = path.join(request.user.directories.characters, `${outputFile}.png`);
writeFileAtomicSync(DIRECTORIES.characters + target_img + '.png', outputImage);
if (response !== undefined) response.send(mes);
writeFileAtomicSync(outputImagePath, outputImage);
return true;
} catch (err) {
console.log(err);
if (response !== undefined) response.status(500).send(err);
return false;
}
}
async function tryReadImage(img_url, crop) {
/**
* @typedef {Object} Crop
* @property {number} x X-coordinate
* @property {number} y Y-coordinate
* @property {number} width Width
* @property {number} height Height
* @property {boolean} want_resize Resize the image to the standard avatar size
*/
/**
* Reads an image file and applies crop if defined.
* @param {string} imgPath Path to the image file
* @param {Crop|undefined} crop Crop parameters
* @returns {Promise<Buffer>} Image buffer
*/
async function tryReadImage(imgPath, crop) {
try {
let rawImg = await jimp.read(img_url);
let final_width = rawImg.bitmap.width, final_height = rawImg.bitmap.height;
let rawImg = await jimp.read(imgPath);
let finalWidth = rawImg.bitmap.width, finalHeight = rawImg.bitmap.height;
// Apply crop if defined
if (typeof crop == 'object' && [crop.x, crop.y, crop.width, crop.height].every(x => typeof x === 'number')) {
rawImg = rawImg.crop(crop.x, crop.y, crop.width, crop.height);
// Apply standard resize if requested
if (crop.want_resize) {
final_width = AVATAR_WIDTH;
final_height = AVATAR_HEIGHT;
finalWidth = AVATAR_WIDTH;
finalHeight = AVATAR_HEIGHT;
} else {
final_width = crop.width;
final_height = crop.height;
finalWidth = crop.width;
finalHeight = crop.height;
}
}
const image = await rawImg.cover(final_width, final_height).getBufferAsync(jimp.MIME_PNG);
const image = await rawImg.cover(finalWidth, finalHeight).getBufferAsync(jimp.MIME_PNG);
return image;
}
// If it's an unsupported type of image (APNG) - just read the file as buffer
catch {
return fs.readFileSync(img_url);
return fs.readFileSync(imgPath);
}
}
@ -131,54 +148,57 @@ const calculateDataSize = (data) => {
* processCharacter - Process a given character, read its data and calculate its statistics.
*
* @param {string} item The name of the character.
* @param {number} i The index of the character in the characters list.
* @return {Promise} A Promise that resolves when the character processing is done.
* @param {import('../users').UserDirectoryList} directories User directories
* @return {Promise<object>} A Promise that resolves when the character processing is done.
*/
const processCharacter = async (item, i) => {
const processCharacter = async (item, directories) => {
try {
const img_data = await charaRead(DIRECTORIES.characters + item);
if (img_data === undefined) throw new Error('Failed to read character file');
const imgFile = path.join(directories.characters, item);
const imgData = await readCharacterData(imgFile);
if (imgData === undefined) throw new Error('Failed to read character file');
let jsonObject = getCharaCardV2(JSON.parse(img_data), false);
let jsonObject = getCharaCardV2(JSON.parse(imgData), directories, false);
jsonObject.avatar = item;
characters[i] = jsonObject;
characters[i]['json_data'] = img_data;
const charStat = fs.statSync(path.join(DIRECTORIES.characters, item));
characters[i]['date_added'] = charStat.ctimeMs;
characters[i]['create_date'] = jsonObject['create_date'] || humanizedISO8601DateTime(charStat.ctimeMs);
const char_dir = path.join(DIRECTORIES.chats, item.replace('.png', ''));
const character = jsonObject;
character['json_data'] = imgData;
const charStat = fs.statSync(path.join(directories.characters, item));
character['date_added'] = charStat.ctimeMs;
character['create_date'] = jsonObject['create_date'] || humanizedISO8601DateTime(charStat.ctimeMs);
const chatsDirectory = path.join(directories.chats, item.replace('.png', ''));
const { chatSize, dateLastChat } = calculateChatSize(char_dir);
characters[i]['chat_size'] = chatSize;
characters[i]['date_last_chat'] = dateLastChat;
characters[i]['data_size'] = calculateDataSize(jsonObject?.data);
const { chatSize, dateLastChat } = calculateChatSize(chatsDirectory);
character['chat_size'] = chatSize;
character['date_last_chat'] = dateLastChat;
character['data_size'] = calculateDataSize(jsonObject?.data);
return character;
}
catch (err) {
characters[i] = {
console.log(`Could not process character: ${item}`);
if (err instanceof SyntaxError) {
console.log(`${item} does not contain a valid JSON object.`);
} else {
console.log('An unexpected error occurred: ', err);
}
return {
date_added: 0,
date_last_chat: 0,
chat_size: 0,
};
console.log(`Could not process character: ${item}`);
if (err instanceof SyntaxError) {
console.log('String [' + i + '] is not valid JSON!');
} else {
console.log('An unexpected error occurred: ', err);
}
}
};
/**
* Convert a character object to Spec V2 format.
* @param {object} jsonObject Character object
* @param {import('../users').UserDirectoryList} directories User directories
* @param {boolean} hoistDate Will set the chat and create_date fields to the current date if they are missing
* @returns {object} Character object in Spec V2 format
*/
function getCharaCardV2(jsonObject, hoistDate = true) {
function getCharaCardV2(jsonObject, directories, hoistDate = true) {
if (jsonObject.spec === undefined) {
jsonObject = convertToV2(jsonObject);
jsonObject = convertToV2(jsonObject, directories);
if (hoistDate && !jsonObject.create_date) {
jsonObject.create_date = humanizedISO8601DateTime();
@ -192,9 +212,10 @@ function getCharaCardV2(jsonObject, hoistDate = true) {
/**
* Convert a character object to Spec V2 format.
* @param {object} char Character object
* @param {import('../users').UserDirectoryList} directories User directories
* @returns {object} Character object in Spec V2 format
*/
function convertToV2(char) {
function convertToV2(char, directories) {
// Simulate incoming data from frontend form
const result = charaFormatData({
json_data: JSON.stringify(char),
@ -212,7 +233,7 @@ function convertToV2(char) {
depth_prompt_prompt: char.depth_prompt_prompt,
depth_prompt_depth: char.depth_prompt_depth,
depth_prompt_role: char.depth_prompt_role,
});
}, directories);
result.chat = char.chat ?? humanizedISO8601DateTime();
result.create_date = char.create_date;
@ -278,8 +299,13 @@ function readFromV2(char) {
return char;
}
//***************** Main functions
function charaFormatData(data) {
/**
* Format character data to Spec V2 format.
* @param {object} data Character data
* @param {import('../users').UserDirectoryList} directories User directories
* @returns
*/
function charaFormatData(data, directories) {
// This is supposed to save all the foreign keys that ST doesn't care about
const char = tryParse(data.json_data) || {};
@ -344,7 +370,7 @@ function charaFormatData(data) {
if (data.world) {
try {
const file = readWorldInfoFile(data.world, false);
const file = readWorldInfoFile(directories, data.world, false);
// File was imported - save it to the character book
if (file && file.originalData) {
@ -423,15 +449,16 @@ function convertWorldInfoToCharacterBook(name, entries) {
/**
* Import a character from a YAML file.
* @param {string} uploadPath Path to the uploaded file
* @param {import('express').Response} response Express response object
* @param {{ request: import('express').Request, response: import('express').Response }} context Express request and response objects
* @returns {Promise<string>} Internal name of the character
*/
function importFromYaml(uploadPath, response) {
async function importFromYaml(uploadPath, context) {
const fileText = fs.readFileSync(uploadPath, 'utf8');
fs.rmSync(uploadPath);
const yamlData = yaml.parse(fileText);
console.log('importing from yaml');
console.log('Importing from YAML');
yamlData.name = sanitize(yamlData.name);
const fileName = getPngName(yamlData.name);
const fileName = getPngName(yamlData.name, context.request.user.directories);
let char = convertToV2({
'name': yamlData.name,
'description': yamlData.context ?? '',
@ -446,32 +473,177 @@ function importFromYaml(uploadPath, response) {
'talkativeness': 0.5,
'creator': '',
'tags': '',
});
charaWrite(defaultAvatarPath, JSON.stringify(char), fileName, response, { file_name: fileName });
}, context.request.user.directories);
const result = await writeCharacterData(defaultAvatarPath, JSON.stringify(char), fileName, context.request);
return result ? fileName : '';
}
/**
* Import a character from a JSON file.
* @param {string} uploadPath Path to the uploaded file
* @param {{ request: import('express').Request, response: import('express').Response }} context Express request and response objects
* @returns {Promise<string>} Internal name of the character
*/
async function importFromJson(uploadPath, { request }) {
const data = fs.readFileSync(uploadPath, 'utf8');
fs.unlinkSync(uploadPath);
let jsonData = JSON.parse(data);
if (jsonData.spec !== undefined) {
console.log('Importing from v2 json');
importRisuSprites(request.user.directories, jsonData);
unsetFavFlag(jsonData);
jsonData = readFromV2(jsonData);
jsonData['create_date'] = humanizedISO8601DateTime();
const pngName = getPngName(jsonData.data?.name || jsonData.name, request.user.directories);
const char = JSON.stringify(jsonData);
const result = await writeCharacterData(defaultAvatarPath, char, pngName, request);
return result ? pngName : '';
} else if (jsonData.name !== undefined) {
console.log('Importing from v1 json');
jsonData.name = sanitize(jsonData.name);
if (jsonData.creator_notes) {
jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', '');
}
const pngName = getPngName(jsonData.name, request.user.directories);
let char = {
'name': jsonData.name,
'description': jsonData.description ?? '',
'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '',
'personality': jsonData.personality ?? '',
'first_mes': jsonData.first_mes ?? '',
'avatar': 'none',
'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(),
'mes_example': jsonData.mes_example ?? '',
'scenario': jsonData.scenario ?? '',
'create_date': humanizedISO8601DateTime(),
'talkativeness': jsonData.talkativeness ?? 0.5,
'creator': jsonData.creator ?? '',
'tags': jsonData.tags ?? '',
};
char = convertToV2(char, request.user.directories);
let charJSON = JSON.stringify(char);
const result = await writeCharacterData(defaultAvatarPath, charJSON, pngName, request);
return result ? pngName : '';
} else if (jsonData.char_name !== undefined) {//json Pygmalion notepad
console.log('Importing from gradio json');
jsonData.char_name = sanitize(jsonData.char_name);
if (jsonData.creator_notes) {
jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', '');
}
const pngName = getPngName(jsonData.char_name, request.user.directories);
let char = {
'name': jsonData.char_name,
'description': jsonData.char_persona ?? '',
'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '',
'personality': '',
'first_mes': jsonData.char_greeting ?? '',
'avatar': 'none',
'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(),
'mes_example': jsonData.example_dialogue ?? '',
'scenario': jsonData.world_scenario ?? '',
'create_date': humanizedISO8601DateTime(),
'talkativeness': jsonData.talkativeness ?? 0.5,
'creator': jsonData.creator ?? '',
'tags': jsonData.tags ?? '',
};
char = convertToV2(char, request.user.directories);
const charJSON = JSON.stringify(char);
const result = await writeCharacterData(defaultAvatarPath, charJSON, pngName, request);
return result ? pngName : '';
}
return '';
}
/**
* Import a character from a PNG file.
* @param {string} uploadPath Path to the uploaded file
* @param {{ request: import('express').Request, response: import('express').Response }} context Express request and response objects
* @param {string|undefined} preservedFileName Preserved file name
* @returns {Promise<string>} Internal name of the character
*/
async function importFromPng(uploadPath, { request }, preservedFileName) {
const imgData = await readCharacterData(uploadPath);
if (imgData === undefined) throw new Error('Failed to read character data');
let jsonData = JSON.parse(imgData);
jsonData.name = sanitize(jsonData.data?.name || jsonData.name);
const pngName = preservedFileName || getPngName(jsonData.name, request.user.directories);
if (jsonData.spec !== undefined) {
console.log('Found a v2 character file.');
importRisuSprites(request.user.directories, jsonData);
unsetFavFlag(jsonData);
jsonData = readFromV2(jsonData);
jsonData['create_date'] = humanizedISO8601DateTime();
const char = JSON.stringify(jsonData);
const result = await writeCharacterData(uploadPath, char, pngName, request);
fs.unlinkSync(uploadPath);
return result ? pngName : '';
} else if (jsonData.name !== undefined) {
console.log('Found a v1 character file.');
if (jsonData.creator_notes) {
jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', '');
}
let char = {
'name': jsonData.name,
'description': jsonData.description ?? '',
'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '',
'personality': jsonData.personality ?? '',
'first_mes': jsonData.first_mes ?? '',
'avatar': 'none',
'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(),
'mes_example': jsonData.mes_example ?? '',
'scenario': jsonData.scenario ?? '',
'create_date': humanizedISO8601DateTime(),
'talkativeness': jsonData.talkativeness ?? 0.5,
'creator': jsonData.creator ?? '',
'tags': jsonData.tags ?? '',
};
char = convertToV2(char, request.user.directories);
const charJSON = JSON.stringify(char);
const result = await writeCharacterData(uploadPath, charJSON, pngName, request);
fs.unlinkSync(uploadPath);
return result ? pngName : '';
}
return '';
}
const router = express.Router();
router.post('/create', urlencodedParser, async function (request, response) {
if (!request.body) return response.sendStatus(400);
try {
if (!request.body) return response.sendStatus(400);
request.body.ch_name = sanitize(request.body.ch_name);
request.body.ch_name = sanitize(request.body.ch_name);
const char = JSON.stringify(charaFormatData(request.body));
const internalName = getPngName(request.body.ch_name);
const avatarName = `${internalName}.png`;
const defaultAvatar = './public/img/ai4.png';
const chatsPath = DIRECTORIES.chats + internalName; //path.join(chatsPath, internalName);
const char = JSON.stringify(charaFormatData(request.body, request.user.directories));
const internalName = getPngName(request.body.ch_name, request.user.directories);
const avatarName = `${internalName}.png`;
const defaultAvatar = './public/img/ai4.png';
const chatsPath = path.join(request.user.directories.chats, internalName);
if (!fs.existsSync(chatsPath)) fs.mkdirSync(chatsPath);
if (!fs.existsSync(chatsPath)) fs.mkdirSync(chatsPath);
if (!request.file) {
charaWrite(defaultAvatar, char, internalName, response, avatarName);
} else {
const crop = tryParse(request.query.crop);
const uploadPath = path.join(UPLOADS_PATH, request.file.filename);
await charaWrite(uploadPath, char, internalName, response, avatarName, crop);
fs.unlinkSync(uploadPath);
if (!request.file) {
await writeCharacterData(defaultAvatar, char, internalName, request);
return response.send(avatarName);
} else {
const crop = tryParse(request.query.crop);
const uploadPath = path.join(UPLOADS_PATH, request.file.filename);
await writeCharacterData(uploadPath, char, internalName, request, crop);
fs.unlinkSync(uploadPath);
return response.send(avatarName);
}
} catch (err) {
console.error(err);
response.sendStatus(500);
}
});
@ -483,26 +655,26 @@ router.post('/rename', jsonParser, async function (request, response) {
const oldAvatarName = request.body.avatar_url;
const newName = sanitize(request.body.new_name);
const oldInternalName = path.parse(request.body.avatar_url).name;
const newInternalName = getPngName(newName);
const newInternalName = getPngName(newName, request.user.directories);
const newAvatarName = `${newInternalName}.png`;
const oldAvatarPath = path.join(DIRECTORIES.characters, oldAvatarName);
const oldAvatarPath = path.join(request.user.directories.characters, oldAvatarName);
const oldChatsPath = path.join(DIRECTORIES.chats, oldInternalName);
const newChatsPath = path.join(DIRECTORIES.chats, newInternalName);
const oldChatsPath = path.join(request.user.directories.chats, oldInternalName);
const newChatsPath = path.join(request.user.directories.chats, newInternalName);
try {
// Read old file, replace name int it
const rawOldData = await charaRead(oldAvatarPath);
const rawOldData = await readCharacterData(oldAvatarPath);
if (rawOldData === undefined) throw new Error('Failed to read character file');
const oldData = getCharaCardV2(JSON.parse(rawOldData));
const oldData = getCharaCardV2(JSON.parse(rawOldData), request.user.directories);
_.set(oldData, 'data.name', newName);
_.set(oldData, 'name', newName);
const newData = JSON.stringify(oldData);
// Write data to new location
await charaWrite(oldAvatarPath, newData, newInternalName);
await writeCharacterData(oldAvatarPath, newData, newInternalName, request);
// Rename chats folder
if (fs.existsSync(oldChatsPath) && !fs.existsSync(newChatsPath)) {
@ -513,7 +685,7 @@ router.post('/rename', jsonParser, async function (request, response) {
fs.rmSync(oldAvatarPath);
// Return new avatar name to ST
return response.send({ 'avatar': newAvatarName });
return response.send({ avatar: newAvatarName });
}
catch (err) {
console.error(err);
@ -534,23 +706,25 @@ router.post('/edit', urlencodedParser, async function (request, response) {
return;
}
let char = charaFormatData(request.body);
let char = charaFormatData(request.body, request.user.directories);
char.chat = request.body.chat;
char.create_date = request.body.create_date;
char = JSON.stringify(char);
let target_img = (request.body.avatar_url).replace('.png', '');
let targetFile = (request.body.avatar_url).replace('.png', '');
try {
if (!request.file) {
const avatarPath = path.join(DIRECTORIES.characters, request.body.avatar_url);
await charaWrite(avatarPath, char, target_img, response, 'Character saved');
const avatarPath = path.join(request.user.directories.characters, request.body.avatar_url);
await writeCharacterData(avatarPath, char, targetFile, request);
} else {
const crop = tryParse(request.query.crop);
const newAvatarPath = path.join(UPLOADS_PATH, request.file.filename);
invalidateThumbnail('avatar', request.body.avatar_url);
await charaWrite(newAvatarPath, char, target_img, response, 'Character saved', crop);
invalidateThumbnail(request.user.directories, 'avatar', request.body.avatar_url);
await writeCharacterData(newAvatarPath, char, targetFile, request, crop);
fs.unlinkSync(newAvatarPath);
}
return response.sendStatus(200);
}
catch {
console.error('An error occured, character edit invalidated.');
@ -572,22 +746,20 @@ router.post('/edit-attribute', jsonParser, async function (request, response) {
console.log(request.body);
if (!request.body) {
console.error('Error: no response body detected');
response.status(400).send('Error: no response body detected');
return;
return response.status(400).send('Error: no response body detected');
}
if (request.body.ch_name === '' || request.body.ch_name === undefined || request.body.ch_name === '.') {
console.error('Error: invalid name.');
response.status(400).send('Error: invalid name.');
return;
return response.status(400).send('Error: invalid name.');
}
try {
const avatarPath = path.join(DIRECTORIES.characters, request.body.avatar_url);
let charJSON = await charaRead(avatarPath);
const avatarPath = path.join(request.user.directories.characters, request.body.avatar_url);
const charJSON = await readCharacterData(avatarPath);
if (typeof charJSON !== 'string') throw new Error('Failed to read character file');
let char = JSON.parse(charJSON);
const char = JSON.parse(charJSON);
//check if the field exists
if (char[request.body.field] === undefined && char.data[request.body.field] === undefined) {
console.error('Error: invalid field.');
@ -597,7 +769,9 @@ router.post('/edit-attribute', jsonParser, async function (request, response) {
char[request.body.field] = request.body.value;
char.data[request.body.field] = request.body.value;
let newCharJSON = JSON.stringify(char);
await charaWrite(avatarPath, newCharJSON, (request.body.avatar_url).replace('.png', ''), response, 'Character saved');
const targetFile = (request.body.avatar_url).replace('.png', '');
await writeCharacterData(avatarPath, newCharJSON, targetFile, request);
return response.sendStatus(200);
} catch (err) {
console.error('An error occured, character edit invalidated.', err);
}
@ -617,30 +791,25 @@ router.post('/edit-attribute', jsonParser, async function (request, response) {
router.post('/merge-attributes', jsonParser, async function (request, response) {
try {
const update = request.body;
const avatarPath = path.join(DIRECTORIES.characters, update.avatar);
const avatarPath = path.join(request.user.directories.characters, update.avatar);
const pngStringData = await charaRead(avatarPath);
const pngStringData = await readCharacterData(avatarPath);
if (!pngStringData) {
console.error('Error: invalid character file.');
response.status(400).send('Error: invalid character file.');
return;
return response.status(400).send('Error: invalid character file.');
}
let character = JSON.parse(pngStringData);
character = deepMerge(character, update);
const validator = new TavernCardValidator(character);
const targetImg = (update.avatar).replace('.png', '');
//Accept either V1 or V2.
if (validator.validate()) {
await charaWrite(
avatarPath,
JSON.stringify(character),
(update.avatar).replace('.png', ''),
response,
'Character saved',
);
await writeCharacterData(avatarPath, JSON.stringify(character), targetImg, request);
response.sendStatus(200);
} else {
console.log(validator.lastValidationError);
response.status(400).send({ message: `Validation failed for ${character.name}`, error: validator.lastValidationError });
@ -660,13 +829,13 @@ router.post('/delete', jsonParser, async function (request, response) {
return response.sendStatus(403);
}
const avatarPath = DIRECTORIES.characters + request.body.avatar_url;
const avatarPath = path.join(request.user.directories.characters, request.body.avatar_url);
if (!fs.existsSync(avatarPath)) {
return response.sendStatus(400);
}
fs.rmSync(avatarPath);
invalidateThumbnail('avatar', request.body.avatar_url);
invalidateThumbnail(request.user.directories, 'avatar', request.body.avatar_url);
let dir_name = (request.body.avatar_url.replace('.png', ''));
if (!dir_name.length) {
@ -676,7 +845,7 @@ router.post('/delete', jsonParser, async function (request, response) {
if (request.body.delete_chats == true) {
try {
await fs.promises.rm(path.join(DIRECTORIES.chats, sanitize(dir_name)), { recursive: true, force: true });
await fs.promises.rm(path.join(request.user.directories.chats, sanitize(dir_name)), { recursive: true, force: true });
} catch (err) {
console.error(err);
return response.sendStatus(500);
@ -696,46 +865,40 @@ router.post('/delete', jsonParser, async function (request, response) {
* The stats are calculated by the `calculateStats` function.
* The characters are processed by the `processCharacter` function.
*
* @param {object} request The HTTP request object.
* @param {object} response The HTTP response object.
* @return {undefined} Does not return a value.
* @param {import("express").Request} request The HTTP request object.
* @param {import("express").Response} response The HTTP response object.
* @return {void}
*/
router.post('/all', jsonParser, function (request, response) {
fs.readdir(DIRECTORIES.characters, async (err, files) => {
if (err) {
console.error(err);
return;
}
router.post('/all', jsonParser, async function (request, response) {
try {
const files = fs.readdirSync(request.user.directories.characters);
const pngFiles = files.filter(file => file.endsWith('.png'));
characters = {};
let processingPromises = pngFiles.map((file, index) => processCharacter(file, index));
await Promise.all(processingPromises); performance.mark('B');
// Filter out invalid/broken characters
characters = Object.values(characters).filter(x => x?.name).reduce((acc, val, index) => {
acc[index] = val;
return acc;
}, {});
response.send(JSON.stringify(characters));
});
const processingPromises = pngFiles.map(file => processCharacter(file, request.user.directories));
const data = await Promise.all(processingPromises);
return response.send(data);
} catch (err) {
console.error(err);
response.sendStatus(500);
}
});
router.post('/get', jsonParser, async function (request, response) {
if (!request.body) return response.sendStatus(400);
const item = request.body.avatar_url;
const filePath = path.join(DIRECTORIES.characters, item);
try {
if (!request.body) return response.sendStatus(400);
const item = request.body.avatar_url;
const filePath = path.join(request.user.directories.characters, item);
if (!fs.existsSync(filePath)) {
return response.sendStatus(404);
if (!fs.existsSync(filePath)) {
return response.sendStatus(404);
}
const data = await processCharacter(item, request.user.directories);
return response.send(data);
} catch (err) {
console.error(err);
response.sendStatus(500);
}
characters = {};
await processCharacter(item, 0);
return response.send(characters[0]);
});
router.post('/chats', jsonParser, async function (request, response) {
@ -744,7 +907,7 @@ router.post('/chats', jsonParser, async function (request, response) {
const characterDirectory = (request.body.avatar_url).replace('.png', '');
try {
const chatsDirectory = path.join(DIRECTORIES.chats, characterDirectory);
const chatsDirectory = path.join(request.user.directories.chats, characterDirectory);
const files = fs.readdirSync(chatsDirectory);
const jsonFiles = files.filter(file => path.extname(file) === '.jsonl');
@ -755,7 +918,7 @@ router.post('/chats', jsonParser, async function (request, response) {
const jsonFilesPromise = jsonFiles.map((file) => {
return new Promise(async (res) => {
const pathToFile = path.join(DIRECTORIES.chats, characterDirectory, file);
const pathToFile = path.join(request.user.directories.chats, characterDirectory, file);
const fileStream = fs.createReadStream(pathToFile);
const stats = fs.statSync(pathToFile);
const fileSizeInKB = `${(stats.size / 1024).toFixed(2)}kb`;
@ -805,11 +968,17 @@ router.post('/chats', jsonParser, async function (request, response) {
}
});
function getPngName(file) {
/**
* Gets the name for the uploaded PNG file.
* @param {string} file File name
* @param {import('../users').UserDirectoryList} directories User directories
* @returns {string} - The name for the uploaded PNG file
*/
function getPngName(file, directories) {
let i = 1;
let base_name = file;
while (fs.existsSync(DIRECTORIES.characters + file + '.png')) {
file = base_name + i;
const baseName = file;
while (fs.existsSync(path.join(directories.characters, `${file}.png`))) {
file = baseName + i;
i++;
}
return file;
@ -829,147 +998,35 @@ function getPreservedName(request) {
router.post('/import', urlencodedParser, async function (request, response) {
if (!request.body || !request.file) return response.sendStatus(400);
let png_name = '';
let filedata = request.file;
let uploadPath = path.join(UPLOADS_PATH, filedata.filename);
let format = request.body.file_type;
const uploadPath = path.join(UPLOADS_PATH, request.file.filename);
const format = request.body.file_type;
const preservedFileName = getPreservedName(request);
if (format == 'yaml' || format == 'yml') {
try {
importFromYaml(uploadPath, response);
} catch (err) {
console.log(err);
response.send({ error: true });
const formatImportFunctions = {
'yaml': importFromYaml,
'yml': importFromYaml,
'json': importFromJson,
'png': importFromPng,
};
try {
const importFunction = formatImportFunctions[format];
if (!importFunction) {
throw new Error(`Unsupported format: ${format}`);
}
} else if (format == 'json') {
fs.readFile(uploadPath, 'utf8', async (err, data) => {
fs.unlinkSync(uploadPath);
if (err) {
console.log(err);
response.send({ error: true });
}
const fileName = await importFunction(uploadPath, { request, response }, preservedFileName);
let jsonData = JSON.parse(data);
if (jsonData.spec !== undefined) {
console.log('importing from v2 json');
importRisuSprites(jsonData);
unsetFavFlag(jsonData);
jsonData = readFromV2(jsonData);
jsonData['create_date'] = humanizedISO8601DateTime();
png_name = getPngName(jsonData.data?.name || jsonData.name);
let char = JSON.stringify(jsonData);
charaWrite(defaultAvatarPath, char, png_name, response, { file_name: png_name });
} else if (jsonData.name !== undefined) {
console.log('importing from v1 json');
jsonData.name = sanitize(jsonData.name);
if (jsonData.creator_notes) {
jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', '');
}
png_name = getPngName(jsonData.name);
let char = {
'name': jsonData.name,
'description': jsonData.description ?? '',
'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '',
'personality': jsonData.personality ?? '',
'first_mes': jsonData.first_mes ?? '',
'avatar': 'none',
'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(),
'mes_example': jsonData.mes_example ?? '',
'scenario': jsonData.scenario ?? '',
'create_date': humanizedISO8601DateTime(),
'talkativeness': jsonData.talkativeness ?? 0.5,
'creator': jsonData.creator ?? '',
'tags': jsonData.tags ?? '',
};
char = convertToV2(char);
let charJSON = JSON.stringify(char);
charaWrite(defaultAvatarPath, charJSON, png_name, response, { file_name: png_name });
} else if (jsonData.char_name !== undefined) {//json Pygmalion notepad
console.log('importing from gradio json');
jsonData.char_name = sanitize(jsonData.char_name);
if (jsonData.creator_notes) {
jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', '');
}
png_name = getPngName(jsonData.char_name);
let char = {
'name': jsonData.char_name,
'description': jsonData.char_persona ?? '',
'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '',
'personality': '',
'first_mes': jsonData.char_greeting ?? '',
'avatar': 'none',
'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(),
'mes_example': jsonData.example_dialogue ?? '',
'scenario': jsonData.world_scenario ?? '',
'create_date': humanizedISO8601DateTime(),
'talkativeness': jsonData.talkativeness ?? 0.5,
'creator': jsonData.creator ?? '',
'tags': jsonData.tags ?? '',
};
char = convertToV2(char);
let charJSON = JSON.stringify(char);
charaWrite(defaultAvatarPath, charJSON, png_name, response, { file_name: png_name });
} else {
console.log('Incorrect character format .json');
response.send({ error: true });
}
});
} else {
try {
var img_data = await charaRead(uploadPath, format);
if (img_data === undefined) throw new Error('Failed to read character data');
let jsonData = JSON.parse(img_data);
jsonData.name = sanitize(jsonData.data?.name || jsonData.name);
png_name = preservedFileName || getPngName(jsonData.name);
if (jsonData.spec !== undefined) {
console.log('Found a v2 character file.');
importRisuSprites(jsonData);
unsetFavFlag(jsonData);
jsonData = readFromV2(jsonData);
jsonData['create_date'] = humanizedISO8601DateTime();
const char = JSON.stringify(jsonData);
await charaWrite(uploadPath, char, png_name, response, { file_name: png_name });
fs.unlinkSync(uploadPath);
} else if (jsonData.name !== undefined) {
console.log('Found a v1 character file.');
if (jsonData.creator_notes) {
jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', '');
}
let char = {
'name': jsonData.name,
'description': jsonData.description ?? '',
'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '',
'personality': jsonData.personality ?? '',
'first_mes': jsonData.first_mes ?? '',
'avatar': 'none',
'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(),
'mes_example': jsonData.mes_example ?? '',
'scenario': jsonData.scenario ?? '',
'create_date': humanizedISO8601DateTime(),
'talkativeness': jsonData.talkativeness ?? 0.5,
'creator': jsonData.creator ?? '',
'tags': jsonData.tags ?? '',
};
char = convertToV2(char);
const charJSON = JSON.stringify(char);
await charaWrite(uploadPath, charJSON, png_name, response, { file_name: png_name });
fs.unlinkSync(uploadPath);
} else {
console.log('Unknown character card format');
response.send({ error: true });
}
} catch (err) {
console.log(err);
response.send({ error: true });
if (!fileName) {
console.error('Failed to import character');
return response.sendStatus(400);
}
response.send({ file_name: fileName });
} catch (err) {
console.log(err);
response.send({ error: true });
}
});
@ -980,7 +1037,7 @@ router.post('/duplicate', jsonParser, async function (request, response) {
console.log(request.body);
return response.sendStatus(400);
}
let filename = path.join(DIRECTORIES.characters, sanitize(request.body.avatar_url));
let filename = path.join(request.user.directories.characters, sanitize(request.body.avatar_url));
if (!fs.existsSync(filename)) {
console.log('file for dupe not found');
console.log(filename);
@ -1002,11 +1059,11 @@ router.post('/duplicate', jsonParser, async function (request, response) {
baseName = nameParts.join('_'); // original filename is completely the baseName
}
newFilename = path.join(DIRECTORIES.characters, `${baseName}_${suffix}${path.extname(filename)}`);
newFilename = path.join(request.user.directories.characters, `${baseName}_${suffix}${path.extname(filename)}`);
while (fs.existsSync(newFilename)) {
let suffixStr = '_' + suffix;
newFilename = path.join(DIRECTORIES.characters, `${baseName}${suffixStr}${path.extname(filename)}`);
newFilename = path.join(request.user.directories.characters, `${baseName}${suffixStr}${path.extname(filename)}`);
suffix++;
}
@ -1025,7 +1082,7 @@ router.post('/export', jsonParser, async function (request, response) {
return response.sendStatus(400);
}
let filename = path.join(DIRECTORIES.characters, sanitize(request.body.avatar_url));
let filename = path.join(request.user.directories.characters, sanitize(request.body.avatar_url));
if (!fs.existsSync(filename)) {
return response.sendStatus(404);
@ -1036,9 +1093,9 @@ router.post('/export', jsonParser, async function (request, response) {
return response.sendFile(filename, { root: process.cwd() });
case 'json': {
try {
let json = await charaRead(filename);
let json = await readCharacterData(filename);
if (json === undefined) return response.sendStatus(400);
let jsonObject = getCharaCardV2(JSON.parse(json));
let jsonObject = getCharaCardV2(JSON.parse(json), request.user.directories);
return response.type('json').send(JSON.stringify(jsonObject, null, 4));
}
catch {

192
src/endpoints/chats.js
View File

@ -6,7 +6,7 @@ const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser, urlencodedParser } = require('../express-common');
const { DIRECTORIES, UPLOADS_PATH } = require('../constants');
const { PUBLIC_DIRECTORIES, UPLOADS_PATH } = require('../constants');
const { getConfigValue, humanizedISO8601DateTime, tryParse, generateTimestamp, removeOldBackups } = require('../util');
/**
@ -22,14 +22,14 @@ function backupChat(name, chat) {
return;
}
if (!fs.existsSync(DIRECTORIES.backups)) {
fs.mkdirSync(DIRECTORIES.backups);
if (!fs.existsSync(PUBLIC_DIRECTORIES.backups)) {
fs.mkdirSync(PUBLIC_DIRECTORIES.backups);
}
// replace non-alphanumeric characters with underscores
name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();
const backupFile = path.join(DIRECTORIES.backups, `chat_${name}_${generateTimestamp()}.jsonl`);
const backupFile = path.join(PUBLIC_DIRECTORIES.backups, `chat_${name}_${generateTimestamp()}.jsonl`);
writeFileAtomicSync(backupFile, chat, 'utf-8');
removeOldBackups(`chat_${name}_`);
@ -38,18 +38,25 @@ function backupChat(name, chat) {
}
}
function importOobaChat(user_name, ch_name, jsonData, avatar_url) {
/**
* Imports a chat from Ooba's format.
* @param {string} userName User name
* @param {string} characterName Character name
* @param {object} jsonData JSON data
* @returns {string} Chat data
*/
function importOobaChat(userName, characterName, jsonData) {
/** @type {object[]} */
const chat = [{
user_name: user_name,
character_name: ch_name,
user_name: userName,
character_name: characterName,
create_date: humanizedISO8601DateTime(),
}];
for (const arr of jsonData.data_visible) {
if (arr[0]) {
const userMessage = {
name: user_name,
name: userName,
is_user: true,
send_date: humanizedISO8601DateTime(),
mes: arr[0],
@ -58,7 +65,7 @@ function importOobaChat(user_name, ch_name, jsonData, avatar_url) {
}
if (arr[1]) {
const charMessage = {
name: ch_name,
name: characterName,
is_user: false,
send_date: humanizedISO8601DateTime(),
mes: arr[1],
@ -68,21 +75,28 @@ function importOobaChat(user_name, ch_name, jsonData, avatar_url) {
}
const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');
writeFileAtomicSync(`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chatContent, 'utf8');
return chatContent;
}
function importAgnaiChat(user_name, ch_name, jsonData, avatar_url) {
/**
* Imports a chat from Agnai's format.
* @param {string} userName User name
* @param {string} characterName Character name
* @param {object} jsonData Chat data
* @returns {string} Chat data
*/
function importAgnaiChat(userName, characterName, jsonData) {
/** @type {object[]} */
const chat = [{
user_name: user_name,
character_name: ch_name,
user_name: userName,
character_name: characterName,
create_date: humanizedISO8601DateTime(),
}];
for (const message of jsonData.messages) {
const isUser = !!message.userId;
chat.push({
name: isUser ? user_name : ch_name,
name: isUser ? userName : characterName,
is_user: isUser,
send_date: humanizedISO8601DateTime(),
mes: message.msg,
@ -90,60 +104,54 @@ function importAgnaiChat(user_name, ch_name, jsonData, avatar_url) {
}
const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');
writeFileAtomicSync(`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chatContent, 'utf8');
return chatContent;
}
function importCAIChat(user_name, ch_name, jsonData, avatar_url) {
const chat = {
from(history) {
return [
{
user_name: user_name,
character_name: ch_name,
create_date: humanizedISO8601DateTime(),
},
...history.msgs.map(
(message) => ({
name: message.src.is_human ? user_name : ch_name,
is_user: message.src.is_human,
send_date: humanizedISO8601DateTime(),
mes: message.text,
}),
),
];
},
};
/**
* Imports a chat from CAI Tools format.
* @param {string} userName User name
* @param {string} characterName Character name
* @param {object} jsonData JSON data
* @returns {string[]} Converted data
*/
function importCAIChat(userName, characterName, jsonData) {
/**
* Converts the chat data to suitable format.
* @param {object} history Imported chat data
* @returns {object[]} Converted chat data
*/
function convert(history) {
const starter = {
user_name: userName,
character_name: characterName,
create_date: humanizedISO8601DateTime(),
};
const newChats = [];
(jsonData.histories.histories ?? []).forEach((history) => {
newChats.push(chat.from(history));
});
const historyData = history.msgs.map((msg) => ({
name: msg.src.is_human ? userName : characterName,
is_user: msg.src.is_human,
send_date: humanizedISO8601DateTime(),
mes: msg.text,
}));
const errors = [];
for (const chat of newChats) {
const filePath = `${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`;
const fileContent = chat.map(tryParse).filter(x => x).join('\n');
try {
writeFileAtomicSync(filePath, fileContent, 'utf8');
} catch (err) {
errors.push(err);
}
return [starter, ...historyData];
}
return errors;
const newChats = (jsonData.histories.histories ?? []).map(history => newChats.push(convert(history).map(obj => JSON.stringify(obj)).join('\n')));
return newChats;
}
const router = express.Router();
router.post('/save', jsonParser, function (request, response) {
try {
var dir_name = String(request.body.avatar_url).replace('.png', '');
let chat_data = request.body.chat;
let jsonlData = chat_data.map(JSON.stringify).join('\n');
writeFileAtomicSync(`${DIRECTORIES.chats + sanitize(dir_name)}/${sanitize(String(request.body.file_name))}.jsonl`, jsonlData, 'utf8');
backupChat(dir_name, jsonlData);
const directoryName = String(request.body.avatar_url).replace('.png', '');
const chatData = request.body.chat;
const jsonlData = chatData.map(JSON.stringify).join('\n');
const fileName = `${sanitize(String(request.body.file_name))}.jsonl`;
const filePath = path.join(request.user.directories.chats, directoryName, fileName);
writeFileAtomicSync(filePath, jsonlData, 'utf8');
backupChat(directoryName, jsonlData);
return response.send({ result: 'ok' });
} catch (error) {
response.send(error);
@ -154,11 +162,12 @@ router.post('/save', jsonParser, function (request, response) {
router.post('/get', jsonParser, function (request, response) {
try {
const dirName = String(request.body.avatar_url).replace('.png', '');
const chatDirExists = fs.existsSync(DIRECTORIES.chats + dirName);
const directoryPath = path.join(request.user.directories.chats, dirName);
const chatDirExists = fs.existsSync(directoryPath);
//if no chat dir for the character is found, make one with the character name
if (!chatDirExists) {
fs.mkdirSync(DIRECTORIES.chats + dirName);
fs.mkdirSync(directoryPath);
return response.send({});
}
@ -166,7 +175,7 @@ router.post('/get', jsonParser, function (request, response) {
return response.send({});
}
const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.file_name))}.jsonl`;
const fileName = path.join(directoryPath, `${sanitize(String(request.body.file_name))}.jsonl`);
const chatFileExists = fs.existsSync(fileName);
if (!chatFileExists) {
@ -192,8 +201,8 @@ router.post('/rename', jsonParser, async function (request, response) {
}
const pathToFolder = request.body.is_group
? DIRECTORIES.groupChats
: path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', ''));
? request.user.directories.groupChats
: path.join(request.user.directories.chats, String(request.body.avatar_url).replace('.png', ''));
const pathToOriginalFile = path.join(pathToFolder, request.body.original_file);
const pathToRenamedFile = path.join(pathToFolder, request.body.renamed_file);
console.log('Old chat name', pathToOriginalFile);
@ -210,7 +219,6 @@ router.post('/rename', jsonParser, async function (request, response) {
});
router.post('/delete', jsonParser, function (request, response) {
console.log('/api/chats/delete entered');
if (!request.body) {
console.log('no request body seen');
return response.sendStatus(400);
@ -222,18 +230,15 @@ router.post('/delete', jsonParser, function (request, response) {
}
const dirName = String(request.body.avatar_url).replace('.png', '');
const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.chatfile))}`;
const fileName = path.join(request.user.directories.chats, dirName, sanitize(String(request.body.chatfile)));
const chatFileExists = fs.existsSync(fileName);
if (!chatFileExists) {
console.log(`Chat file not found '${fileName}'`);
return response.sendStatus(400);
} else {
console.log('found the chat file: ' + fileName);
/* fs.unlinkSync(fileName); */
fs.rmSync(fileName);
console.log('deleted chat file: ' + fileName);
console.log('Deleted chat file: ' + fileName);
}
return response.send('ok');
@ -244,8 +249,8 @@ router.post('/export', jsonParser, async function (request, response) {
return response.sendStatus(400);
}
const pathToFolder = request.body.is_group
? DIRECTORIES.groupChats
: path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', ''));
? request.user.directories.groupChats
: path.join(request.user.directories.chats, String(request.body.avatar_url).replace('.png', ''));
let filename = path.join(pathToFolder, request.body.file);
let exportfilename = request.body.exportfilename;
if (!fs.existsSync(filename)) {
@ -321,7 +326,7 @@ router.post('/group/import', urlencodedParser, function (request, response) {
const chatname = humanizedISO8601DateTime();
const pathToUpload = path.join(UPLOADS_PATH, filedata.filename);
const pathToNewFile = path.join(DIRECTORIES.groupChats, `${chatname}.jsonl`);
const pathToNewFile = path.join(request.user.directories.groupChats, `${chatname}.jsonl`);
fs.copyFileSync(pathToUpload, pathToNewFile);
fs.unlinkSync(pathToUpload);
return response.send({ res: chatname });
@ -334,35 +339,42 @@ router.post('/group/import', urlencodedParser, function (request, response) {
router.post('/import', urlencodedParser, function (request, response) {
if (!request.body) return response.sendStatus(400);
var format = request.body.file_type;
let filedata = request.file;
let avatar_url = (request.body.avatar_url).replace('.png', '');
let ch_name = request.body.character_name;
let user_name = request.body.user_name || 'You';
const format = request.body.file_type;
const avatarUrl = (request.body.avatar_url).replace('.png', '');
const characterName = request.body.character_name;
const userName = request.body.user_name || 'You';
if (!filedata) {
if (!request.file) {
return response.sendStatus(400);
}
try {
const data = fs.readFileSync(path.join(UPLOADS_PATH, filedata.filename), 'utf8');
const data = fs.readFileSync(path.join(UPLOADS_PATH, request.file.filename), 'utf8');
if (format === 'json') {
const jsonData = JSON.parse(data);
if (jsonData.histories !== undefined) {
// CAI Tools format
const errors = importCAIChat(user_name, ch_name, jsonData, avatar_url);
if (0 < errors.length) {
return response.send('Errors occurred while writing character files. Errors: ' + JSON.stringify(errors));
const chats = importCAIChat(userName, characterName, jsonData);
for (const chat of chats) {
const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
writeFileAtomicSync(filePath, chat, 'utf8');
}
return response.send({ res: true });
} else if (Array.isArray(jsonData.data_visible)) {
// oobabooga's format
importOobaChat(user_name, ch_name, jsonData, avatar_url);
const chat = importOobaChat(userName, characterName, jsonData);
const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
writeFileAtomicSync(filePath, chat, 'utf8');
return response.send({ res: true });
} else if (Array.isArray(jsonData.messages)) {
// Agnai format
importAgnaiChat(user_name, ch_name, jsonData, avatar_url);
const chat = importAgnaiChat(userName, characterName, jsonData);
const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
writeFileAtomicSync(filePath, chat, 'utf8');
return response.send({ res: true });
} else {
console.log('Incorrect chat format .json');
@ -373,10 +385,12 @@ router.post('/import', urlencodedParser, function (request, response) {
if (format === 'jsonl') {
const line = data.split('\n')[0];
let jsonData = JSON.parse(line);
const jsonData = JSON.parse(line);
if (jsonData.user_name !== undefined || jsonData.name !== undefined) {
fs.copyFileSync(path.join(UPLOADS_PATH, filedata.filename), (`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()}.jsonl`));
const fileName = `${characterName} - ${humanizedISO8601DateTime()} imported.jsonl`;
const filePath = path.join(request.user.directories.chats, avatarUrl, fileName);
fs.copyFileSync(path.join(UPLOADS_PATH, request.file.filename), filePath);
response.send({ res: true });
} else {
console.log('Incorrect chat format .jsonl');
@ -395,7 +409,7 @@ router.post('/group/get', jsonParser, (request, response) => {
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
const pathToFile = path.join(request.user.directories.groupChats, `${id}.jsonl`);
if (fs.existsSync(pathToFile)) {
const data = fs.readFileSync(pathToFile, 'utf8');
@ -415,7 +429,7 @@ router.post('/group/delete', jsonParser, (request, response) => {
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
const pathToFile = path.join(request.user.directories.groupChats, `${id}.jsonl`);
if (fs.existsSync(pathToFile)) {
fs.rmSync(pathToFile);
@ -431,10 +445,10 @@ router.post('/group/save', jsonParser, (request, response) => {
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
const pathToFile = path.join(request.user.directories.groupChats, `${id}.jsonl`);
if (!fs.existsSync(DIRECTORIES.groupChats)) {
fs.mkdirSync(DIRECTORIES.groupChats);
if (!fs.existsSync(request.user.directories.groupChats)) {
fs.mkdirSync(request.user.directories.groupChats);
}
let chat_data = request.body.chat;

26
src/endpoints/extensions.js
View File

@ -3,7 +3,7 @@ const fs = require('fs');
const express = require('express');
const { default: simpleGit } = require('simple-git');
const sanitize = require('sanitize-filename');
const { DIRECTORIES } = require('../constants');
const { PUBLIC_DIRECTORIES } = require('../constants');
const { jsonParser } = require('../express-common');
/**
@ -67,12 +67,12 @@ router.post('/install', jsonParser, async (request, response) => {
const git = simpleGit();
// make sure the third-party directory exists
if (!fs.existsSync(path.join(DIRECTORIES.extensions, 'third-party'))) {
fs.mkdirSync(path.join(DIRECTORIES.extensions, 'third-party'));
if (!fs.existsSync(path.join(request.user.directories.extensions))) {
fs.mkdirSync(path.join(request.user.directories.extensions));
}
const url = request.body.url;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', path.basename(url, '.git'));
const extensionPath = path.join(request.user.directories.extensions, path.basename(url, '.git'));
if (fs.existsSync(extensionPath)) {
return response.status(409).send(`Directory already exists at ${extensionPath}`);
@ -111,7 +111,7 @@ router.post('/update', jsonParser, async (request, response) => {
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
const extensionPath = path.join(request.user.directories.extensions, extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
@ -156,7 +156,7 @@ router.post('/version', jsonParser, async (request, response) => {
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
const extensionPath = path.join(request.user.directories.extensions, extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
@ -195,7 +195,7 @@ router.post('/delete', jsonParser, async (request, response) => {
const extensionName = sanitize(request.body.extensionName);
try {
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
const extensionPath = path.join(request.user.directories.extensions, extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
@ -216,22 +216,22 @@ router.post('/delete', jsonParser, async (request, response) => {
* Discover the extension folders
* If the folder is called third-party, search for subfolders instead
*/
router.get('/discover', jsonParser, function (_, response) {
router.get('/discover', jsonParser, function (request, response) {
// get all folders in the extensions folder, except third-party
const extensions = fs
.readdirSync(DIRECTORIES.extensions)
.filter(f => fs.statSync(path.join(DIRECTORIES.extensions, f)).isDirectory())
.readdirSync(PUBLIC_DIRECTORIES.extensions)
.filter(f => fs.statSync(path.join(PUBLIC_DIRECTORIES.extensions, f)).isDirectory())
.filter(f => f !== 'third-party');
// get all folders in the third-party folder, if it exists
if (!fs.existsSync(path.join(DIRECTORIES.extensions, 'third-party'))) {
if (!fs.existsSync(path.join(request.user.directories.extensions))) {
return response.send(extensions);
}
const thirdPartyExtensions = fs
.readdirSync(path.join(DIRECTORIES.extensions, 'third-party'))
.filter(f => fs.statSync(path.join(DIRECTORIES.extensions, 'third-party', f)).isDirectory());
.readdirSync(path.join(request.user.directories.extensions))
.filter(f => fs.statSync(path.join(request.user.directories.extensions, f)).isDirectory());
// add the third-party extensions to the extensions array
extensions.push(...thirdPartyExtensions.map(f => `third-party/${f}`));

5
src/endpoints/files.js
View File

@ -4,7 +4,6 @@ const express = require('express');
const router = express.Router();
const { validateAssetFileName } = require('./assets');
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
const { clientRelativePath } = require('../util');
router.post('/upload', jsonParser, async (request, response) => {
@ -22,9 +21,9 @@ router.post('/upload', jsonParser, async (request, response) => {
if (validation.error)
return response.status(400).send(validation.message);
const pathToUpload = path.join(DIRECTORIES.files, request.body.name);
const pathToUpload = path.join(request.user.directories.files, request.body.name);
writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
const url = clientRelativePath(pathToUpload);
const url = clientRelativePath(request.user.directories.root, pathToUpload);
return response.send({ path: url });
} catch (error) {
console.log(error);

3
src/endpoints/google.js
View File

@ -10,7 +10,8 @@ router.post('/caption-image', jsonParser, async (request, response) => {
try {
const mimeType = request.body.image.split(';')[0].split(':')[1];
const base64Data = request.body.image.split(',')[1];
const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-pro-vision:generateContent?key=${readSecret(SECRET_KEYS.MAKERSUITE)}`;
const key = readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-pro-vision:generateContent?key=${key}`;
const body = {
contents: [{
parts: [

27
src/endpoints/groups.js
View File

@ -5,24 +5,23 @@ const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
const { humanizedISO8601DateTime } = require('../util');
const router = express.Router();
router.post('/all', jsonParser, (_, response) => {
router.post('/all', jsonParser, (request, response) => {
const groups = [];
if (!fs.existsSync(DIRECTORIES.groups)) {
fs.mkdirSync(DIRECTORIES.groups);
if (!fs.existsSync(request.user.directories.groups)) {
fs.mkdirSync(request.user.directories.groups);
}
const files = fs.readdirSync(DIRECTORIES.groups).filter(x => path.extname(x) === '.json');
const chats = fs.readdirSync(DIRECTORIES.groupChats).filter(x => path.extname(x) === '.jsonl');
const files = fs.readdirSync(request.user.directories.groups).filter(x => path.extname(x) === '.json');
const chats = fs.readdirSync(request.user.directories.groupChats).filter(x => path.extname(x) === '.jsonl');
files.forEach(function (file) {
try {
const filePath = path.join(DIRECTORIES.groups, file);
const filePath = path.join(request.user.directories.groups, file);
const fileContents = fs.readFileSync(filePath, 'utf8');
const group = JSON.parse(fileContents);
const groupStat = fs.statSync(filePath);
@ -35,7 +34,7 @@ router.post('/all', jsonParser, (_, response) => {
if (Array.isArray(group.chats) && Array.isArray(chats)) {
for (const chat of chats) {
if (group.chats.includes(path.parse(chat).name)) {
const chatStat = fs.statSync(path.join(DIRECTORIES.groupChats, chat));
const chatStat = fs.statSync(path.join(request.user.directories.groupChats, chat));
chat_size += chatStat.size;
date_last_chat = Math.max(date_last_chat, chatStat.mtimeMs);
}
@ -75,11 +74,11 @@ router.post('/create', jsonParser, (request, response) => {
chats: request.body.chats ?? [id],
auto_mode_delay: request.body.auto_mode_delay ?? 5,
};
const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`);
const pathToFile = path.join(request.user.directories.groups, `${id}.json`);
const fileData = JSON.stringify(groupMetadata);
if (!fs.existsSync(DIRECTORIES.groups)) {
fs.mkdirSync(DIRECTORIES.groups);
if (!fs.existsSync(request.user.directories.groups)) {
fs.mkdirSync(request.user.directories.groups);
}
writeFileAtomicSync(pathToFile, fileData);
@ -91,7 +90,7 @@ router.post('/edit', jsonParser, (request, response) => {
return response.sendStatus(400);
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`);
const pathToFile = path.join(request.user.directories.groups, `${id}.json`);
const fileData = JSON.stringify(request.body);
writeFileAtomicSync(pathToFile, fileData);
@ -104,7 +103,7 @@ router.post('/delete', jsonParser, async (request, response) => {
}
const id = request.body.id;
const pathToGroup = path.join(DIRECTORIES.groups, sanitize(`${id}.json`));
const pathToGroup = path.join(request.user.directories.groups, sanitize(`${id}.json`));
try {
// Delete group chats
@ -113,7 +112,7 @@ router.post('/delete', jsonParser, async (request, response) => {
if (group && Array.isArray(group.chats)) {
for (const chat of group.chats) {
console.log('Deleting group chat', chat);
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
const pathToFile = path.join(request.user.directories.groupChats, `${id}.jsonl`);
if (fs.existsSync(pathToFile)) {
fs.rmSync(pathToFile);

10
src/endpoints/horde.js
View File

@ -159,7 +159,7 @@ router.post('/task-status', jsonParser, async (request, response) => {
});
router.post('/generate-text', jsonParser, async (request, response) => {
const apiKey = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
const apiKey = readSecret(request.user.directories, SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
const url = 'https://horde.koboldai.net/api/v2/generate/text/async';
const agent = await getClientAgent();
@ -213,7 +213,7 @@ router.post('/sd-models', jsonParser, async (_, response) => {
router.post('/caption-image', jsonParser, async (request, response) => {
try {
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
const api_key_horde = readSecret(request.user.directories, SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
const ai_horde = await getHordeClient();
const result = await ai_horde.postAsyncInterrogate({
source_image: request.body.image,
@ -263,8 +263,8 @@ router.post('/caption-image', jsonParser, async (request, response) => {
}
});
router.post('/user-info', jsonParser, async (_, response) => {
const api_key_horde = readSecret(SECRET_KEYS.HORDE);
router.post('/user-info', jsonParser, async (request, response) => {
const api_key_horde = readSecret(request.user.directories, SECRET_KEYS.HORDE);
if (!api_key_horde) {
return response.send({ anonymous: true });
@ -307,7 +307,7 @@ router.post('/generate-image', jsonParser, async (request, response) => {
request.body.prompt = sanitized;
}
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
const api_key_horde = readSecret(request.user.directories, SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
console.log('Stable Horde request:', request.body);
const ai_horde = await getHordeClient();

15
src/endpoints/images.js
View File

@ -4,7 +4,6 @@ const express = require('express');
const sanitize = require('sanitize-filename');
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
const { clientRelativePath, removeFileExtension, getImages } = require('../util');
/**
@ -60,23 +59,23 @@ router.post('/upload', jsonParser, async (request, response) => {
}
// if character is defined, save to a sub folder for that character
let pathToNewFile = path.join(DIRECTORIES.userImages, sanitize(filename));
let pathToNewFile = path.join(request.user.directories.userImages, sanitize(filename));
if (request.body.ch_name) {
pathToNewFile = path.join(DIRECTORIES.userImages, sanitize(request.body.ch_name), sanitize(filename));
pathToNewFile = path.join(request.user.directories.userImages, sanitize(request.body.ch_name), sanitize(filename));
}
ensureDirectoryExistence(pathToNewFile);
const imageBuffer = Buffer.from(base64Data, 'base64');
await fs.promises.writeFile(pathToNewFile, imageBuffer);
response.send({ path: clientRelativePath(pathToNewFile) });
response.send({ path: clientRelativePath(request.user.directories.root, pathToNewFile) });
} catch (error) {
console.log(error);
response.status(500).send({ error: 'Failed to save the image' });
}
});
router.post('/list/:folder', (req, res) => {
const directoryPath = path.join(process.cwd(), DIRECTORIES.userImages, sanitize(req.params.folder));
router.post('/list/:folder', (request, response) => {
const directoryPath = path.join(request.user.directories.userImages, sanitize(request.params.folder));
if (!fs.existsSync(directoryPath)) {
fs.mkdirSync(directoryPath, { recursive: true });
@ -84,10 +83,10 @@ router.post('/list/:folder', (req, res) => {
try {
const images = getImages(directoryPath);
return res.send(images);
return response.send(images);
} catch (error) {
console.error(error);
return res.status(500).send({ error: 'Unable to retrieve files' });
return response.status(500).send({ error: 'Unable to retrieve files' });
}
});

3
src/endpoints/moving-ui.js
View File

@ -4,7 +4,6 @@ const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
const router = express.Router();
@ -13,7 +12,7 @@ router.post('/save', jsonParser, (request, response) => {
return response.sendStatus(400);
}
const filename = path.join(DIRECTORIES.movingUI, sanitize(request.body.name) + '.json');
const filename = path.join(request.user.directories.movingUI, sanitize(request.body.name) + '.json');
writeFileAtomicSync(filename, JSON.stringify(request.body, null, 4), 'utf8');
return response.sendStatus(200);

8
src/endpoints/novelai.js
View File

@ -66,7 +66,7 @@ const router = express.Router();
router.post('/status', jsonParser, async function (req, res) {
if (!req.body) return res.sendStatus(400);
const api_key_novel = readSecret(SECRET_KEYS.NOVEL);
const api_key_novel = readSecret(req.user.directories, SECRET_KEYS.NOVEL);
if (!api_key_novel) {
console.log('NovelAI Access Token is missing.');
@ -102,7 +102,7 @@ router.post('/status', jsonParser, async function (req, res) {
router.post('/generate', jsonParser, async function (req, res) {
if (!req.body) return res.sendStatus(400);
const api_key_novel = readSecret(SECRET_KEYS.NOVEL);
const api_key_novel = readSecret(req.user.directories, SECRET_KEYS.NOVEL);
if (!api_key_novel) {
console.log('NovelAI Access Token is missing.');
@ -230,7 +230,7 @@ router.post('/generate-image', jsonParser, async (request, response) => {
return response.sendStatus(400);
}
const key = readSecret(SECRET_KEYS.NOVEL);
const key = readSecret(request.user.directories, SECRET_KEYS.NOVEL);
if (!key) {
console.log('NovelAI Access Token is missing.');
@ -325,7 +325,7 @@ router.post('/generate-image', jsonParser, async (request, response) => {
});
router.post('/generate-voice', jsonParser, async (request, response) => {
const token = readSecret(SECRET_KEYS.NOVEL);
const token = readSecret(request.user.directories, SECRET_KEYS.NOVEL);
if (!token) {
console.log('NovelAI Access Token is missing.');

16
src/endpoints/openai.js
View File

@ -16,11 +16,11 @@ router.post('/caption-image', jsonParser, async (request, response) => {
let bodyParams = {};
if (request.body.api === 'openai' && !request.body.reverse_proxy) {
key = readSecret(SECRET_KEYS.OPENAI);
key = readSecret(request.user.directories, SECRET_KEYS.OPENAI);
}
if (request.body.api === 'openrouter' && !request.body.reverse_proxy) {
key = readSecret(SECRET_KEYS.OPENROUTER);
key = readSecret(request.user.directories, SECRET_KEYS.OPENROUTER);
}
if (request.body.reverse_proxy && request.body.proxy_password) {
@ -28,18 +28,18 @@ router.post('/caption-image', jsonParser, async (request, response) => {
}
if (request.body.api === 'custom') {
key = readSecret(SECRET_KEYS.CUSTOM);
key = readSecret(request.user.directories, SECRET_KEYS.CUSTOM);
mergeObjectWithYaml(bodyParams, request.body.custom_include_body);
mergeObjectWithYaml(headers, request.body.custom_include_headers);
}
if (request.body.api === 'ooba') {
key = readSecret(SECRET_KEYS.OOBA);
key = readSecret(request.user.directories, SECRET_KEYS.OOBA);
bodyParams.temperature = 0.1;
}
if (request.body.api === 'koboldcpp') {
key = readSecret(SECRET_KEYS.KOBOLDCPP);
key = readSecret(request.user.directories, SECRET_KEYS.KOBOLDCPP);
}
if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp'].includes(request.body.api) === false) {
@ -150,7 +150,7 @@ router.post('/caption-image', jsonParser, async (request, response) => {
router.post('/transcribe-audio', urlencodedParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.OPENAI);
const key = readSecret(request.user.directories, SECRET_KEYS.OPENAI);
if (!key) {
console.log('No OpenAI key found');
@ -198,7 +198,7 @@ router.post('/transcribe-audio', urlencodedParser, async (request, response) =>
router.post('/generate-voice', jsonParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.OPENAI);
const key = readSecret(request.user.directories, SECRET_KEYS.OPENAI);
if (!key) {
console.log('No OpenAI key found');
@ -237,7 +237,7 @@ router.post('/generate-voice', jsonParser, async (request, response) => {
router.post('/generate-image', jsonParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.OPENAI);
const key = readSecret(request.user.directories, SECRET_KEYS.OPENAI);
if (!key) {
console.log('No OpenAI key found');

5
src/endpoints/quick-replies.js
View File

@ -5,7 +5,6 @@ const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
const router = express.Router();
@ -14,7 +13,7 @@ router.post('/save', jsonParser, (request, response) => {
return response.sendStatus(400);
}
const filename = path.join(DIRECTORIES.quickreplies, sanitize(request.body.name) + '.json');
const filename = path.join(request.user.directories.quickreplies, sanitize(request.body.name) + '.json');
writeFileAtomicSync(filename, JSON.stringify(request.body, null, 4), 'utf8');
return response.sendStatus(200);
@ -25,7 +24,7 @@ router.post('/delete', jsonParser, (request, response) => {
return response.sendStatus(400);
}
const filename = path.join(DIRECTORIES.quickreplies, sanitize(request.body.name) + '.json');
const filename = path.join(request.user.directories.quickreplies, sanitize(request.body.name) + '.json');
if (fs.existsSync(filename)) {
fs.unlinkSync(filename);
}

71
src/endpoints/secrets.js
View File

@ -5,7 +5,7 @@ const { getConfigValue } = require('../util');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser } = require('../express-common');
const SECRETS_FILE = path.join(process.cwd(), './secrets.json');
const SECRETS_FILE = 'secrets.json';
const SECRET_KEYS = {
HORDE: 'api_key_horde',
MANCER: 'api_key_mancer',
@ -48,57 +48,74 @@ const EXPORTABLE_KEYS = [
/**
* Writes a secret to the secrets file
* @param {import('../users').UserDirectoryList} directories User directories
* @param {string} key Secret key
* @param {string} value Secret value
*/
function writeSecret(key, value) {
if (!fs.existsSync(SECRETS_FILE)) {
function writeSecret(directories, key, value) {
const filePath = path.join(directories.root, SECRETS_FILE);
if (!fs.existsSync(filePath)) {
const emptyFile = JSON.stringify({});
writeFileAtomicSync(SECRETS_FILE, emptyFile, 'utf-8');
writeFileAtomicSync(filePath, emptyFile, 'utf-8');
}
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8');
const fileContents = fs.readFileSync(filePath, 'utf-8');
const secrets = JSON.parse(fileContents);
secrets[key] = value;
writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets, null, 4), 'utf-8');
writeFileAtomicSync(filePath, JSON.stringify(secrets, null, 4), 'utf-8');
}
function deleteSecret(key) {
if (!fs.existsSync(SECRETS_FILE)) {
/**
* Deletes a secret from the secrets file
* @param {import('../users').UserDirectoryList} directories User directories
* @param {string} key Secret key
* @returns
*/
function deleteSecret(directories, key) {
const filePath = path.join(directories.root, SECRETS_FILE);
if (!fs.existsSync(filePath)) {
return;
}
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8');
const fileContents = fs.readFileSync(filePath, 'utf-8');
const secrets = JSON.parse(fileContents);
delete secrets[key];
writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets, null, 4), 'utf-8');
writeFileAtomicSync(filePath, JSON.stringify(secrets, null, 4), 'utf-8');
}
/**
* Reads a secret from the secrets file
* @param {import('../users').UserDirectoryList} directories User directories
* @param {string} key Secret key
* @returns {string} Secret value
*/
function readSecret(key) {
if (!fs.existsSync(SECRETS_FILE)) {
function readSecret(directories, key) {
const filePath = path.join(directories.root, SECRETS_FILE);
if (!fs.existsSync(filePath)) {
return '';
}
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8');
const fileContents = fs.readFileSync(filePath, 'utf-8');
const secrets = JSON.parse(fileContents);
return secrets[key];
}
/**
* Reads the secret state from the secrets file
* @param {import('../users').UserDirectoryList} directories User directories
* @returns {object} Secret state
*/
function readSecretState() {
if (!fs.existsSync(SECRETS_FILE)) {
function readSecretState(directories) {
const filePath = path.join(directories.root, SECRETS_FILE);
if (!fs.existsSync(filePath)) {
return {};
}
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf8');
const fileContents = fs.readFileSync(filePath, 'utf8');
const secrets = JSON.parse(fileContents);
const state = {};
@ -111,15 +128,18 @@ function readSecretState() {
/**
* Reads all secrets from the secrets file
* @param {import('../users').UserDirectoryList} directories User directories
* @returns {Record<string, string> | undefined} Secrets
*/
function getAllSecrets() {
if (!fs.existsSync(SECRETS_FILE)) {
function getAllSecrets(directories) {
const filePath = path.join(directories.root, SECRETS_FILE);
if (!fs.existsSync(filePath)) {
console.log('Secrets file does not exist');
return undefined;
}
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf8');
const fileContents = fs.readFileSync(filePath, 'utf8');
const secrets = JSON.parse(fileContents);
return secrets;
}
@ -130,13 +150,13 @@ router.post('/write', jsonParser, (request, response) => {
const key = request.body.key;
const value = request.body.value;
writeSecret(key, value);
writeSecret(request.user.directories, key, value);
return response.send('ok');
});
router.post('/read', jsonParser, (_, response) => {
router.post('/read', jsonParser, (request, response) => {
try {
const state = readSecretState();
const state = readSecretState(request.user.directories);
return response.send(state);
} catch (error) {
console.error(error);
@ -144,7 +164,7 @@ router.post('/read', jsonParser, (_, response) => {
}
});
router.post('/view', jsonParser, async (_, response) => {
router.post('/view', jsonParser, async (request, response) => {
const allowKeysExposure = getConfigValue('allowKeysExposure', false);
if (!allowKeysExposure) {
@ -153,7 +173,7 @@ router.post('/view', jsonParser, async (_, response) => {
}
try {
const secrets = getAllSecrets();
const secrets = getAllSecrets(request.user.directories);
if (!secrets) {
return response.sendStatus(404);
@ -176,7 +196,7 @@ router.post('/find', jsonParser, (request, response) => {
}
try {
const secret = readSecret(key);
const secret = readSecret(request.user.directories, key);
if (!secret) {
response.sendStatus(404);
@ -192,6 +212,7 @@ router.post('/find', jsonParser, (request, response) => {
module.exports = {
writeSecret,
readSecret,
deleteSecret,
readSecretState,
getAllSecrets,
SECRET_KEYS,

2
src/endpoints/serpapi.js
View File

@ -24,7 +24,7 @@ const visitHeaders = {
router.post('/search', jsonParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.SERPAPI);
const key = readSecret(request.user.directories, SECRET_KEYS.SERPAPI);
if (!key) {
console.log('No SerpApi key found');

6
src/endpoints/settings.js
View File

@ -73,8 +73,12 @@ async function backupSettings() {
const userDirectories = getUserDirectories(handle);
const backupFile = path.join(PUBLIC_DIRECTORIES.backups, `settings_${handle}_${generateTimestamp()}.json`);
const sourceFile = path.join(userDirectories.root, SETTINGS_FILE);
fs.copyFileSync(sourceFile, backupFile);
if (!fs.existsSync(sourceFile)) {
continue;
}
fs.copyFileSync(sourceFile, backupFile);
removeOldBackups(`settings_${handle}`);
}
} catch (err) {

24
src/endpoints/sprites.js
View File

@ -5,17 +5,18 @@ const express = require('express');
const mime = require('mime-types');
const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { DIRECTORIES, UPLOADS_PATH } = require('../constants');
const { UPLOADS_PATH } = require('../constants');
const { getImageBuffers } = require('../util');
const { jsonParser, urlencodedParser } = require('../express-common');
/**
* Gets the path to the sprites folder for the provided character name
* @param {import('../users').UserDirectoryList} directories - User directories
* @param {string} name - The name of the character
* @param {boolean} isSubfolder - Whether the name contains a subfolder
* @returns {string | null} The path to the sprites folder. Null if the name is invalid.
*/
function getSpritesPath(name, isSubfolder) {
function getSpritesPath(directories, name, isSubfolder) {
if (isSubfolder) {
const nameParts = name.split('/');
const characterName = sanitize(nameParts[0]);
@ -25,7 +26,7 @@ function getSpritesPath(name, isSubfolder) {
return null;
}
return path.join(DIRECTORIES.characters, characterName, subfolderName);
return path.join(directories.characters, characterName, subfolderName);
}
name = sanitize(name);
@ -34,15 +35,18 @@ function getSpritesPath(name, isSubfolder) {
return null;
}
return path.join(DIRECTORIES.characters, name);
return path.join(directories.characters, name);
}
/**
* Imports base64 encoded sprites from RisuAI character data.
* The sprites are saved in the character's sprites folder.
* The additionalAssets and emotions are removed from the data.
* @param {import('../users').UserDirectoryList} directories User directories
* @param {object} data RisuAI character data
* @returns {void}
*/
function importRisuSprites(data) {
function importRisuSprites(directories, data) {
try {
const name = data?.data?.name;
const risuData = data?.data?.extensions?.risuai;
@ -68,7 +72,7 @@ function importRisuSprites(data) {
}
// Create sprites folder if it doesn't exist
const spritesPath = path.join(DIRECTORIES.characters, name);
const spritesPath = path.join(directories.characters, name);
if (!fs.existsSync(spritesPath)) {
fs.mkdirSync(spritesPath);
}
@ -108,7 +112,7 @@ const router = express.Router();
router.get('/get', jsonParser, function (request, response) {
const name = String(request.query.name);
const isSubfolder = name.includes('/');
const spritesPath = getSpritesPath(name, isSubfolder);
const spritesPath = getSpritesPath(request.user.directories, name, isSubfolder);
let sprites = [];
try {
@ -142,7 +146,7 @@ router.post('/delete', jsonParser, async (request, response) => {
}
try {
const spritesPath = path.join(DIRECTORIES.characters, name);
const spritesPath = path.join(request.user.directories.characters, name);
// No sprites folder exists, or not a directory
if (!fs.existsSync(spritesPath) || !fs.statSync(spritesPath).isDirectory()) {
@ -174,7 +178,7 @@ router.post('/upload-zip', urlencodedParser, async (request, response) => {
}
try {
const spritesPath = path.join(DIRECTORIES.characters, name);
const spritesPath = path.join(request.user.directories.characters, name);
// Create sprites folder if it doesn't exist
if (!fs.existsSync(spritesPath)) {
@ -222,7 +226,7 @@ router.post('/upload', urlencodedParser, async (request, response) => {
}
try {
const spritesPath = path.join(DIRECTORIES.characters, name);
const spritesPath = path.join(request.user.directories.characters, name);
// Create sprites folder if it doesn't exist
if (!fs.existsSync(spritesPath)) {

47
src/endpoints/stable-diffusion.js
View File

@ -3,7 +3,7 @@ const fetch = require('node-fetch').default;
const sanitize = require('sanitize-filename');
const { getBasicAuthHeader, delay, getHexString } = require('../util.js');
const fs = require('fs');
const { DIRECTORIES } = require('../constants.js');
const path = require('path');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser } = require('../express-common');
const { readSecret, SECRET_KEYS } = require('./secrets.js');
@ -43,9 +43,14 @@ function removePattern(x, pattern) {
return x;
}
function getComfyWorkflows() {
/**
* Gets the comfy workflows.
* @param {import('../users.js').UserDirectoryList} directories
* @returns {string[]} List of comfy workflows
*/
function getComfyWorkflows(directories) {
return fs
.readdirSync(DIRECTORIES.comfyWorkflows)
.readdirSync(directories.comfyWorkflows)
.filter(file => file[0] != '.' && file.toLowerCase().endsWith('.json'))
.sort(Intl.Collator().compare);
}
@ -448,7 +453,7 @@ comfy.post('/vaes', jsonParser, async (request, response) => {
comfy.post('/workflows', jsonParser, async (request, response) => {
try {
const data = getComfyWorkflows();
const data = getComfyWorkflows(request.user.directories);
return response.send(data);
} catch (error) {
console.log(error);
@ -458,14 +463,11 @@ comfy.post('/workflows', jsonParser, async (request, response) => {
comfy.post('/workflow', jsonParser, async (request, response) => {
try {
let path = `${DIRECTORIES.comfyWorkflows}/${sanitize(String(request.body.file_name))}`;
if (!fs.existsSync(path)) {
path = `${DIRECTORIES.comfyWorkflows}/Default_Comfy_Workflow.json`;
let filePath = path.join(request.user.directories.comfyWorkflows, sanitize(String(request.body.file_name)));
if (!fs.existsSync(filePath)) {
filePath = path.join(request.user.directories.comfyWorkflows, 'Default_Comfy_Workflow.json');
}
const data = fs.readFileSync(
path,
{ encoding: 'utf-8' },
);
const data = fs.readFileSync(filePath, { encoding: 'utf-8' });
return response.send(JSON.stringify(data));
} catch (error) {
console.log(error);
@ -475,12 +477,9 @@ comfy.post('/workflow', jsonParser, async (request, response) => {
comfy.post('/save-workflow', jsonParser, async (request, response) => {
try {
writeFileAtomicSync(
`${DIRECTORIES.comfyWorkflows}/${sanitize(String(request.body.file_name))}`,
request.body.workflow,
'utf8',
);
const data = getComfyWorkflows();
const filePath = path.join(request.user.directories.comfyWorkflows, sanitize(String(request.body.file_name)));
writeFileAtomicSync(filePath, request.body.workflow, 'utf8');
const data = getComfyWorkflows(request.user.directories);
return response.send(data);
} catch (error) {
console.log(error);
@ -490,9 +489,9 @@ comfy.post('/save-workflow', jsonParser, async (request, response) => {
comfy.post('/delete-workflow', jsonParser, async (request, response) => {
try {
let path = `${DIRECTORIES.comfyWorkflows}/${sanitize(String(request.body.file_name))}`;
if (fs.existsSync(path)) {
fs.unlinkSync(path);
const filePath = path.join(request.user.directories.comfyWorkflows, sanitize(String(request.body.file_name)));
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
}
return response.sendStatus(200);
} catch (error) {
@ -548,9 +547,9 @@ comfy.post('/generate', jsonParser, async (request, response) => {
const together = express.Router();
together.post('/models', jsonParser, async (_, response) => {
together.post('/models', jsonParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.TOGETHERAI);
const key = readSecret(request.user.directories, SECRET_KEYS.TOGETHERAI);
if (!key) {
console.log('TogetherAI key not found.');
@ -589,7 +588,7 @@ together.post('/models', jsonParser, async (_, response) => {
together.post('/generate', jsonParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.TOGETHERAI);
const key = readSecret(request.user.directories, SECRET_KEYS.TOGETHERAI);
if (!key) {
console.log('TogetherAI key not found.');
@ -684,7 +683,7 @@ drawthings.post('/generate', jsonParser, async (request, response) => {
const url = new URL(request.body.url);
url.pathname = '/sdapi/v1/txt2img';
const body = {...request.body};
const body = { ...request.body };
delete body.url;
const result = await fetch(url, {

149
src/endpoints/stats.js
View File

@ -8,11 +8,15 @@ const readFile = fs.promises.readFile;
const readdir = fs.promises.readdir;
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
const { getAllUserHandles, getUserDirectories } = require('../users');
let charStats = {};
const STATS_FILE = 'stats.json';
/**
* @type {Map<string, Object>} The stats object for each user.
*/
const STATS = new Map();
let lastSaveTimestamp = 0;
const statsFilePath = 'public/stats.json';
/**
* Convert a timestamp to an integer timestamp.
@ -26,19 +30,19 @@ const statsFilePath = 'public/stats.json';
* the Unix Epoch, which can be converted to a JavaScript Date object with new Date().
*
* @param {string|number} timestamp - The timestamp to convert.
* @returns {number|null} The timestamp in milliseconds since the Unix Epoch, or null if the input cannot be parsed.
* @returns {number} The timestamp in milliseconds since the Unix Epoch, or 0 if the input cannot be parsed.
*
* @example
* // Unix timestamp
* timestampToMoment(1609459200);
* // ST humanized timestamp
* timestampToMoment("2021-01-01 @00h 00m 00s 000ms");
* timestampToMoment("2021-01-01 \@00h 00m 00s 000ms");
* // Date string
* timestampToMoment("January 1, 2021 12:00am");
*/
function timestampToMoment(timestamp) {
if (!timestamp) {
return null;
return 0;
}
if (typeof timestamp === 'number') {
@ -66,7 +70,7 @@ function timestampToMoment(timestamp) {
)}:${second.padStart(2, '0')}.${millisecond.padStart(3, '0')}Z`;
};
const isoTimestamp1 = timestamp.replace(pattern1, replacement1);
if (!isNaN(new Date(isoTimestamp1))) {
if (!isNaN(Number(new Date(isoTimestamp1)))) {
return new Date(isoTimestamp1).getTime();
}
@ -100,11 +104,11 @@ function timestampToMoment(timestamp) {
)}:00Z`;
};
const isoTimestamp2 = timestamp.replace(pattern2, replacement2);
if (!isNaN(new Date(isoTimestamp2))) {
if (!isNaN(Number(new Date(isoTimestamp2)))) {
return new Date(isoTimestamp2).getTime();
}
return null;
return 0;
}
/**
@ -112,7 +116,7 @@ function timestampToMoment(timestamp) {
*
* @param {string} chatsPath - The path to the directory containing the chat files.
* @param {string} charactersPath - The path to the directory containing the character files.
* @returns {Object} The aggregated stats object.
* @returns {Promise<Object>} The aggregated stats object.
*/
async function collectAndCreateStats(chatsPath, charactersPath) {
console.log('Collecting and creating stats...');
@ -120,8 +124,8 @@ async function collectAndCreateStats(chatsPath, charactersPath) {
const pngFiles = files.filter((file) => file.endsWith('.png'));
let processingPromises = pngFiles.map((file, index) =>
calculateStats(chatsPath, file, index),
let processingPromises = pngFiles.map((file) =>
calculateStats(chatsPath, file),
);
const statsArr = await Promise.all(processingPromises);
@ -134,8 +138,15 @@ async function collectAndCreateStats(chatsPath, charactersPath) {
return finalStats;
}
async function recreateStats(chatsPath, charactersPath) {
charStats = await collectAndCreateStats(chatsPath, charactersPath);
/**
* Recreates the stats object for a user.
* @param {string} handle User handle
* @param {string} chatsPath Path to the directory containing the chat files.
* @param {string} charactersPath Path to the directory containing the character files.
*/
async function recreateStats(handle, chatsPath, charactersPath) {
const stats = await collectAndCreateStats(chatsPath, charactersPath);
STATS.set(handle, stats);
await saveStatsToFile();
console.debug('Stats (re)created and saved to file.');
}
@ -146,15 +157,24 @@ async function recreateStats(chatsPath, charactersPath) {
*/
async function init() {
try {
const statsFileContent = await readFile(statsFilePath, 'utf-8');
charStats = JSON.parse(statsFileContent);
} catch (err) {
// If the file doesn't exist or is invalid, initialize stats
if (err.code === 'ENOENT' || err instanceof SyntaxError) {
recreateStats(DIRECTORIES.chats, DIRECTORIES.characters);
} else {
throw err; // Rethrow the error if it's something we didn't expect
const userHandles = await getAllUserHandles();
for (const handle of userHandles) {
const directories = getUserDirectories(handle);
try {
const statsFilePath = path.join(directories.root, STATS_FILE);
const statsFileContent = await readFile(statsFilePath, 'utf-8');
STATS.set(handle, JSON.parse(statsFileContent));
} catch (err) {
// If the file doesn't exist or is invalid, initialize stats
if (err.code === 'ENOENT' || err instanceof SyntaxError) {
recreateStats(handle, directories.chats, directories.characters);
} else {
throw err; // Rethrow the error if it's something we didn't expect
}
}
}
} catch (err) {
console.error('Failed to initialize stats:', err);
}
// Save stats every 5 minutes
setInterval(saveStatsToFile, 5 * 60 * 1000);
@ -163,16 +183,19 @@ async function init() {
* Saves the current state of charStats to a file, only if the data has changed since the last save.
*/
async function saveStatsToFile() {
if (charStats.timestamp > lastSaveTimestamp) {
//console.debug("Saving stats to file...");
try {
await writeFileAtomic(statsFilePath, JSON.stringify(charStats));
lastSaveTimestamp = Date.now();
} catch (error) {
console.log('Failed to save stats to file.', error);
const userHandles = await getAllUserHandles();
for (const handle of userHandles) {
const charStats = STATS.get(handle) || {};
if (charStats.timestamp > lastSaveTimestamp) {
try {
const directories = getUserDirectories(handle);
const statsFilePath = path.join(directories.root, STATS_FILE);
await writeFileAtomic(statsFilePath, JSON.stringify(charStats));
lastSaveTimestamp = Date.now();
} catch (error) {
console.log('Failed to save stats to file.', error);
}
}
} else {
//console.debug('Stats have not changed since last save. Skipping file write.');
}
}
@ -216,7 +239,7 @@ function readAndParseFile(filepath) {
function calculateGenTime(gen_started, gen_finished) {
let startDate = new Date(gen_started);
let endDate = new Date(gen_finished);
return endDate - startDate;
return Number(endDate) - Number(startDate);
}
/**
@ -233,12 +256,12 @@ function countWordsInString(str) {
/**
* calculateStats - Calculate statistics for a given character chat directory.
*
* @param {string} char_dir The directory containing the chat files.
* @param {string} chatsPath The directory containing the chat files.
* @param {string} item The name of the character.
* @return {object} An object containing the calculated statistics.
*/
const calculateStats = (chatsPath, item, index) => {
const char_dir = path.join(chatsPath, item.replace('.png', ''));
const calculateStats = (chatsPath, item) => {
const chatDir = path.join(chatsPath, item.replace('.png', ''));
const stats = {
total_gen_time: 0,
user_word_count: 0,
@ -252,12 +275,12 @@ const calculateStats = (chatsPath, item, index) => {
};
let uniqueGenStartTimes = new Set();
if (fs.existsSync(char_dir)) {
const chats = fs.readdirSync(char_dir);
if (fs.existsSync(chatDir)) {
const chats = fs.readdirSync(chatDir);
if (Array.isArray(chats) && chats.length) {
for (const chat of chats) {
const result = calculateTotalGenTimeAndWordCount(
char_dir,
chatDir,
chat,
uniqueGenStartTimes,
);
@ -268,7 +291,7 @@ const calculateStats = (chatsPath, item, index) => {
stats.non_user_msg_count += result.nonUserMsgCount || 0;
stats.total_swipe_count += result.totalSwipeCount || 0;
const chatStat = fs.statSync(path.join(char_dir, chat));
const chatStat = fs.statSync(path.join(chatDir, chat));
stats.chat_size += chatStat.size;
stats.date_last_chat = Math.max(
stats.date_last_chat,
@ -285,37 +308,30 @@ const calculateStats = (chatsPath, item, index) => {
return { [item]: stats };
};
/**
* Returns the current charStats object.
* @returns {Object} The current charStats object.
**/
function getCharStats() {
return charStats;
}
/**
* Sets the current charStats object.
* @param {string} handle - The user handle.
* @param {Object} stats - The new charStats object.
**/
function setCharStats(stats) {
charStats = stats;
charStats.timestamp = Date.now();
function setCharStats(handle, stats) {
stats.timestamp = Date.now();
STATS.set(handle, stats);
}
/**
* Calculates the total generation time and word count for a chat with a character.
*
* @param {string} char_dir - The directory path where character chat files are stored.
* @param {string} chatDir - The directory path where character chat files are stored.
* @param {string} chat - The name of the chat file.
* @returns {Object} - An object containing the total generation time, user word count, and non-user word count.
* @throws Will throw an error if the file cannot be read or parsed.
*/
function calculateTotalGenTimeAndWordCount(
char_dir,
chatDir,
chat,
uniqueGenStartTimes,
) {
let filepath = path.join(char_dir, chat);
let filepath = path.join(chatDir, chat);
let lines = readAndParseFile(filepath);
let totalGenTime = 0;
@ -416,29 +432,18 @@ const router = express.Router();
/**
* Handle a POST request to get the stats object
*
* This function returns the stats object that was calculated by the `calculateStats` function.
*
*
* @param {Object} request - The HTTP request object.
* @param {Object} response - The HTTP response object.
* @returns {void}
*/
router.post('/get', jsonParser, function (request, response) {
response.send(JSON.stringify(getCharStats()));
const stats = STATS.get(request.user.profile.handle) || {};
response.send(stats);
});
/**
* Triggers the recreation of statistics from chat files.
* - If successful: returns a 200 OK status.
* - On failure: returns a 500 Internal Server Error status.
*
* @param {Object} request - Express request object.
* @param {Object} response - Express response object.
*/
router.post('/recreate', jsonParser, async function (request, response) {
try {
await recreateStats(DIRECTORIES.chats, DIRECTORIES.characters);
await recreateStats(request.user.profile.handle, request.user.directories.chats, request.user.directories.characters);
return response.sendStatus(200);
} catch (error) {
console.error(error);
@ -446,20 +451,12 @@ router.post('/recreate', jsonParser, async function (request, response) {
}
});
/**
* Handle a POST request to update the stats object
*
* This function updates the stats object with the data from the request body.
*
* @param {Object} request - The HTTP request object.
* @param {Object} response - The HTTP response object.
* @returns {void}
*
*/
router.post('/update', jsonParser, function (request, response) {
if (!request.body) return response.sendStatus(400);
setCharStats(request.body);
setCharStats(request.user.profile.handle, request.body);
return response.sendStatus(200);
});

6
src/endpoints/tokenizers.js
View File

@ -370,12 +370,13 @@ const router = express.Router();
router.post('/ai21/count', jsonParser, async function (req, res) {
if (!req.body) return res.sendStatus(400);
const key = readSecret(req.user.directories, SECRET_KEYS.AI21);
const options = {
method: 'POST',
headers: {
accept: 'application/json',
'content-type': 'application/json',
Authorization: `Bearer ${readSecret(SECRET_KEYS.AI21)}`,
Authorization: `Bearer ${key}`,
},
body: JSON.stringify({ text: req.body[0].content }),
};
@ -401,7 +402,8 @@ router.post('/google/count', jsonParser, async function (req, res) {
body: JSON.stringify({ contents: convertGooglePrompt(req.body, String(req.query.model)) }),
};
try {
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${req.query.model}:countTokens?key=${readSecret(SECRET_KEYS.MAKERSUITE)}`, options);
const key = readSecret(req.user.directories, SECRET_KEYS.MAKERSUITE);
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${req.query.model}:countTokens?key=${key}`, options);
const data = await response.json();
return res.send({ 'token_count': data?.totalTokens || 0 });
} catch (err) {

12
src/endpoints/translate.js
View File

@ -11,8 +11,8 @@ const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate';
const router = express.Router();
router.post('/libre', jsonParser, async (request, response) => {
const key = readSecret(SECRET_KEYS.LIBRE);
const url = readSecret(SECRET_KEYS.LIBRE_URL);
const key = readSecret(request.user.directories, SECRET_KEYS.LIBRE);
const url = readSecret(request.user.directories, SECRET_KEYS.LIBRE_URL);
if (!url) {
console.log('LibreTranslate URL is not configured.');
@ -104,7 +104,7 @@ router.post('/google', jsonParser, async (request, response) => {
router.post('/lingva', jsonParser, async (request, response) => {
try {
const baseUrl = readSecret(SECRET_KEYS.LINGVA_URL);
const baseUrl = readSecret(request.user.directories, SECRET_KEYS.LINGVA_URL);
if (!baseUrl) {
console.log('Lingva URL is not configured.');
@ -149,7 +149,7 @@ router.post('/lingva', jsonParser, async (request, response) => {
});
router.post('/deepl', jsonParser, async (request, response) => {
const key = readSecret(SECRET_KEYS.DEEPL);
const key = readSecret(request.user.directories, SECRET_KEYS.DEEPL);
if (!key) {
console.log('DeepL key is not configured.');
@ -208,7 +208,7 @@ router.post('/deepl', jsonParser, async (request, response) => {
});
router.post('/onering', jsonParser, async (request, response) => {
const secretUrl = readSecret(SECRET_KEYS.ONERING_URL);
const secretUrl = readSecret(request.user.directories, SECRET_KEYS.ONERING_URL);
const url = secretUrl || ONERING_URL_DEFAULT;
if (!url) {
@ -261,7 +261,7 @@ router.post('/onering', jsonParser, async (request, response) => {
});
router.post('/deeplx', jsonParser, async (request, response) => {
const secretUrl = readSecret(SECRET_KEYS.DEEPLX_URL);
const secretUrl = readSecret(request.user.directories, SECRET_KEYS.DEEPLX_URL);
const url = secretUrl || DEEPLX_URL_DEFAULT;
if (!url) {

55
src/endpoints/vectors.js
View File

@ -12,22 +12,23 @@ const SOURCES = ['transformers', 'mistral', 'openai', 'extras', 'palm', 'togethe
* @param {string} source - The source of the vector
* @param {Object} sourceSettings - Settings for the source, if it needs any
* @param {string} text - The text to get the vector for
* @param {import('../users').UserDirectoryList} directories - The directories object for the user
* @returns {Promise<number[]>} - The vector for the text
*/
async function getVector(source, sourceSettings, text) {
async function getVector(source, sourceSettings, text, directories) {
switch (source) {
case 'nomicai':
return require('../nomicai-vectors').getNomicAIVector(text, source);
return require('../nomicai-vectors').getNomicAIVector(text, source, directories);
case 'togetherai':
case 'mistral':
case 'openai':
return require('../openai-vectors').getOpenAIVector(text, source, sourceSettings.model);
return require('../openai-vectors').getOpenAIVector(text, source, directories, sourceSettings.model);
case 'transformers':
return require('../embedding').getTransformersVector(text);
case 'extras':
return require('../extras-vectors').getExtrasVector(text, sourceSettings.extrasUrl, sourceSettings.extrasKey);
case 'palm':
return require('../makersuite-vectors').getMakerSuiteVector(text);
return require('../makersuite-vectors').getMakerSuiteVector(text, directories);
}
throw new Error(`Unknown vector source ${source}`);
@ -38,9 +39,10 @@ async function getVector(source, sourceSettings, text) {
* @param {string} source - The source of the vector
* @param {Object} sourceSettings - Settings for the source, if it needs any
* @param {string[]} texts - The array of texts to get the vector for
* @param {import('../users').UserDirectoryList} directories - The directories object for the user
* @returns {Promise<number[][]>} - The array of vectors for the texts
*/
async function getBatchVector(source, sourceSettings, texts) {
async function getBatchVector(source, sourceSettings, texts, directories) {
const batchSize = 10;
const batches = Array(Math.ceil(texts.length / batchSize)).fill(undefined).map((_, i) => texts.slice(i * batchSize, i * batchSize + batchSize));
@ -48,7 +50,7 @@ async function getBatchVector(source, sourceSettings, texts) {
for (let batch of batches) {
switch (source) {
case 'nomicai':
results.push(...await require('../nomicai-vectors').getNomicAIBatchVector(batch, source));
results.push(...await require('../nomicai-vectors').getNomicAIBatchVector(batch, source, directories));
break;
case 'togetherai':
case 'mistral':
@ -62,7 +64,7 @@ async function getBatchVector(source, sourceSettings, texts) {
results.push(...await require('../extras-vectors').getExtrasBatchVector(batch, sourceSettings.extrasUrl, sourceSettings.extrasKey));
break;
case 'palm':
results.push(...await require('../makersuite-vectors').getMakerSuiteBatchVector(batch));
results.push(...await require('../makersuite-vectors').getMakerSuiteBatchVector(batch, directories));
break;
default:
throw new Error(`Unknown vector source ${source}`);
@ -74,13 +76,15 @@ async function getBatchVector(source, sourceSettings, texts) {
/**
* Gets the index for the vector collection
* @param {import('../users').UserDirectoryList} directories - User directories
* @param {string} collectionId - The collection ID
* @param {string} source - The source of the vector
* @param {boolean} create - Whether to create the index if it doesn't exist
* @returns {Promise<vectra.LocalIndex>} - The index for the collection
*/
async function getIndex(collectionId, source, create = true) {
const store = new vectra.LocalIndex(path.join(process.cwd(), 'vectors', sanitize(source), sanitize(collectionId)));
async function getIndex(directories, collectionId, source, create = true) {
const pathToFile = path.join(directories.vectors, sanitize(source), sanitize(collectionId));
const store = new vectra.LocalIndex(pathToFile);
if (create && !await store.isIndexCreated()) {
await store.createIndex();
@ -91,17 +95,18 @@ async function getIndex(collectionId, source, create = true) {
/**
* Inserts items into the vector collection
* @param {import('../users').UserDirectoryList} directories - User directories
* @param {string} collectionId - The collection ID
* @param {string} source - The source of the vector
* @param {Object} sourceSettings - Settings for the source, if it needs any
* @param {{ hash: number; text: string; index: number; }[]} items - The items to insert
*/
async function insertVectorItems(collectionId, source, sourceSettings, items) {
const store = await getIndex(collectionId, source);
async function insertVectorItems(directories, collectionId, source, sourceSettings, items) {
const store = await getIndex(directories, collectionId, source);
await store.beginUpdate();
const vectors = await getBatchVector(source, sourceSettings, items.map(x => x.text));
const vectors = await getBatchVector(source, sourceSettings, items.map(x => x.text), directories);
for (let i = 0; i < items.length; i++) {
const item = items[i];
@ -114,12 +119,13 @@ async function insertVectorItems(collectionId, source, sourceSettings, items) {
/**
* Gets the hashes of the items in the vector collection
* @param {import('../users').UserDirectoryList} directories - User directories
* @param {string} collectionId - The collection ID
* @param {string} source - The source of the vector
* @returns {Promise<number[]>} - The hashes of the items in the collection
*/
async function getSavedHashes(collectionId, source) {
const store = await getIndex(collectionId, source);
async function getSavedHashes(directories, collectionId, source) {
const store = await getIndex(directories, collectionId, source);
const items = await store.listItems();
const hashes = items.map(x => Number(x.metadata.hash));
@ -129,12 +135,13 @@ async function getSavedHashes(collectionId, source) {
/**
* Deletes items from the vector collection by hash
* @param {import('../users').UserDirectoryList} directories - User directories
* @param {string} collectionId - The collection ID
* @param {string} source - The source of the vector
* @param {number[]} hashes - The hashes of the items to delete
*/
async function deleteVectorItems(collectionId, source, hashes) {
const store = await getIndex(collectionId, source);
async function deleteVectorItems(directories, collectionId, source, hashes) {
const store = await getIndex(directories, collectionId, source);
const items = await store.listItemsByMetadata({ hash: { '$in': hashes } });
await store.beginUpdate();
@ -155,9 +162,9 @@ async function deleteVectorItems(collectionId, source, hashes) {
* @param {number} topK - The number of results to return
* @returns {Promise<{hashes: number[], metadata: object[]}>} - The metadata of the items that match the search text
*/
async function queryCollection(collectionId, source, sourceSettings, searchText, topK) {
const store = await getIndex(collectionId, source);
const vector = await getVector(source, sourceSettings, searchText);
async function queryCollection(directories, collectionId, source, sourceSettings, searchText, topK) {
const store = await getIndex(directories, collectionId, source);
const vector = await getVector(source, sourceSettings, searchText, directories);
const result = await store.queryItems(vector, topK);
const metadata = result.map(x => x.item.metadata);
@ -214,7 +221,7 @@ router.post('/query', jsonParser, async (req, res) => {
const source = String(req.body.source) || 'transformers';
const sourceSettings = getSourceSettings(source, req);
const results = await queryCollection(collectionId, source, sourceSettings, searchText, topK);
const results = await queryCollection(req.user.directories, collectionId, source, sourceSettings, searchText, topK);
return res.json(results);
} catch (error) {
console.error(error);
@ -233,7 +240,7 @@ router.post('/insert', jsonParser, async (req, res) => {
const source = String(req.body.source) || 'transformers';
const sourceSettings = getSourceSettings(source, req);
await insertVectorItems(collectionId, source, sourceSettings, items);
await insertVectorItems(req.user.directories, collectionId, source, sourceSettings, items);
return res.sendStatus(200);
} catch (error) {
console.error(error);
@ -250,7 +257,7 @@ router.post('/list', jsonParser, async (req, res) => {
const collectionId = String(req.body.collectionId);
const source = String(req.body.source) || 'transformers';
const hashes = await getSavedHashes(collectionId, source);
const hashes = await getSavedHashes(req.user.directories, collectionId, source);
return res.json(hashes);
} catch (error) {
console.error(error);
@ -268,7 +275,7 @@ router.post('/delete', jsonParser, async (req, res) => {
const hashes = req.body.hashes.map(x => Number(x));
const source = String(req.body.source) || 'transformers';
await deleteVectorItems(collectionId, source, hashes);
await deleteVectorItems(req.user.directories, collectionId, source, hashes);
return res.sendStatus(200);
} catch (error) {
console.error(error);
@ -285,7 +292,7 @@ router.post('/purge', jsonParser, async (req, res) => {
const collectionId = String(req.body.collectionId);
for (const source of SOURCES) {
const index = await getIndex(collectionId, source, false);
const index = await getIndex(req.user.directories, collectionId, source, false);
const exists = await index.isIndexCreated();

15
src/endpoints/worldinfo.js
View File

@ -5,15 +5,16 @@ const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser, urlencodedParser } = require('../express-common');
const { DIRECTORIES, UPLOADS_PATH } = require('../constants');
const { UPLOADS_PATH } = require('../constants');
/**
* Reads a World Info file and returns its contents
* @param {import('../users').UserDirectoryList} directories User directories
* @param {string} worldInfoName Name of the World Info file
* @param {boolean} allowDummy If true, returns an empty object if the file doesn't exist
* @returns {object} World Info file contents
*/
function readWorldInfoFile(worldInfoName, allowDummy) {
function readWorldInfoFile(directories, worldInfoName, allowDummy) {
const dummyObject = allowDummy ? { entries: {} } : null;
if (!worldInfoName) {
@ -21,7 +22,7 @@ function readWorldInfoFile(worldInfoName, allowDummy) {
}
const filename = `${worldInfoName}.json`;
const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename);
const pathToWorldInfo = path.join(directories.worlds, filename);
if (!fs.existsSync(pathToWorldInfo)) {
console.log(`World info file ${filename} doesn't exist.`);
@ -40,7 +41,7 @@ router.post('/get', jsonParser, (request, response) => {
return response.sendStatus(400);
}
const file = readWorldInfoFile(request.body.name, true);
const file = readWorldInfoFile(request.user.directories, request.body.name, true);
return response.send(file);
});
@ -52,7 +53,7 @@ router.post('/delete', jsonParser, (request, response) => {
const worldInfoName = request.body.name;
const filename = sanitize(`${worldInfoName}.json`);
const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename);
const pathToWorldInfo = path.join(request.user.directories.worlds, filename);
if (!fs.existsSync(pathToWorldInfo)) {
throw new Error(`World info file ${filename} doesn't exist.`);
@ -87,7 +88,7 @@ router.post('/import', urlencodedParser, (request, response) => {
return response.status(400).send('Is not a valid world info file');
}
const pathToNewFile = path.join(DIRECTORIES.worlds, filename);
const pathToNewFile = path.join(request.user.directories.worlds, filename);
const worldName = path.parse(pathToNewFile).name;
if (!worldName) {
@ -116,7 +117,7 @@ router.post('/edit', jsonParser, (request, response) => {
}
const filename = `${sanitize(request.body.name)}.json`;
const pathToFile = path.join(DIRECTORIES.worlds, filename);
const pathToFile = path.join(request.user.directories.worlds, filename);
writeFileAtomicSync(pathToFile, JSON.stringify(request.body.data, null, 4));

10
src/makersuite-vectors.js
View File

@ -4,10 +4,11 @@ const { SECRET_KEYS, readSecret } = require('./endpoints/secrets');
/**
* Gets the vector for the given text from gecko model
* @param {string[]} texts - The array of texts to get the vector for
* @param {import('./users').UserDirectoryList} directories - The directories object for the user
* @returns {Promise<number[][]>} - The array of vectors for the texts
*/
async function getMakerSuiteBatchVector(texts) {
const promises = texts.map(text => getMakerSuiteVector(text));
async function getMakerSuiteBatchVector(texts, directories) {
const promises = texts.map(text => getMakerSuiteVector(text, directories));
const vectors = await Promise.all(promises);
return vectors;
}
@ -15,10 +16,11 @@ async function getMakerSuiteBatchVector(texts) {
/**
* Gets the vector for the given text from PaLM gecko model
* @param {string} text - The text to get the vector for
* @param {import('./users').UserDirectoryList} directories - The directories object for the user
* @returns {Promise<number[]>} - The vector for the text
*/
async function getMakerSuiteVector(text) {
const key = readSecret(SECRET_KEYS.MAKERSUITE);
async function getMakerSuiteVector(text, directories) {
const key = readSecret(directories, SECRET_KEYS.MAKERSUITE);
if (!key) {
console.log('No MakerSuite key found');

10
src/nomicai-vectors.js
View File

@ -13,9 +13,10 @@ const SOURCES = {
* Gets the vector for the given text batch from an OpenAI compatible endpoint.
* @param {string[]} texts - The array of texts to get the vector for
* @param {string} source - The source of the vector
* @param {import('./users').UserDirectoryList} directories - The directories object for the user
* @returns {Promise<number[][]>} - The array of vectors for the texts
*/
async function getNomicAIBatchVector(texts, source) {
async function getNomicAIBatchVector(texts, source, directories) {
const config = SOURCES[source];
if (!config) {
@ -23,7 +24,7 @@ async function getNomicAIBatchVector(texts, source) {
throw new Error('Unknown source');
}
const key = readSecret(config.secretKey);
const key = readSecret(directories, config.secretKey);
if (!key) {
console.log('No API key found');
@ -63,10 +64,11 @@ async function getNomicAIBatchVector(texts, source) {
* Gets the vector for the given text from an OpenAI compatible endpoint.
* @param {string} text - The text to get the vector for
* @param {string} source - The source of the vector
* @param {import('./users').UserDirectoryList} directories - The directories object for the user
* @returns {Promise<number[]>} - The vector for the text
*/
async function getNomicAIVector(text, source) {
const vectors = await getNomicAIBatchVector([text], source);
async function getNomicAIVector(text, source, directories) {
const vectors = await getNomicAIBatchVector([text], source, directories);
return vectors[0];
}

12
src/openai-vectors.js
View File

@ -23,10 +23,11 @@ const SOURCES = {
* Gets the vector for the given text batch from an OpenAI compatible endpoint.
* @param {string[]} texts - The array of texts to get the vector for
* @param {string} source - The source of the vector
* @param {import('./users').UserDirectoryList} directories - The directories object for the user
* @param {string} model - The model to use for the embedding
* @returns {Promise<number[][]>} - The array of vectors for the texts
*/
async function getOpenAIBatchVector(texts, source, model = '') {
async function getOpenAIBatchVector(texts, source, directories, model = '') {
const config = SOURCES[source];
if (!config) {
@ -34,7 +35,7 @@ async function getOpenAIBatchVector(texts, source, model = '') {
throw new Error('Unknown source');
}
const key = readSecret(config.secretKey);
const key = readSecret(directories, config.secretKey);
if (!key) {
console.log('No API key found');
@ -78,11 +79,12 @@ async function getOpenAIBatchVector(texts, source, model = '') {
* Gets the vector for the given text from an OpenAI compatible endpoint.
* @param {string} text - The text to get the vector for
* @param {string} source - The source of the vector
* @param model
* @param {import('./users').UserDirectoryList} directories - The directories object for the user
* @param {string} model - The model to use for the embedding
* @returns {Promise<number[]>} - The vector for the text
*/
async function getOpenAIVector(text, source, model = '') {
const vectors = await getOpenAIBatchVector([text], source, model);
async function getOpenAIVector(text, source, directories, model = '') {
const vectors = await getOpenAIBatchVector([text], source, directories, model);
return vectors[0];
}

2
src/polyfill.js
View File

@ -6,3 +6,5 @@ if (!Array.prototype.findLastIndex) {
return -1;
};
}
module.exports = {};

76
src/users.js
View File

@ -1,7 +1,7 @@
const fsPromises = require('fs').promises;
const path = require('path');
const { USER_DIRECTORY_TEMPLATE, DEFAULT_USER } = require('./constants');
const { getConfigValue } = require('./util');
const express = require('express');
const DATA_ROOT = getConfigValue('dataRoot', './data');
@ -42,6 +42,7 @@ const DATA_ROOT = getConfigValue('dataRoot', './data');
* @property {string} assets - The directory where the assets are stored
* @property {string} comfyWorkflows - The directory where the ComfyUI workflows are stored
* @property {string} files - The directory where the uploaded files are stored
* @property {string} vectors - The directory where the vectors are stored
*/
/**
@ -94,43 +95,9 @@ function getUserDirectories(handle) {
/**
* Middleware to add user data to the request object.
* @param {import('express').Application} app - The express app
* @returns {import('express').RequestHandler}
*/
function userDataMiddleware(app) {
app.use('/backgrounds/:path', async (req, res) => {
try {
const filePath = path.join(process.cwd(), req.user.directories.backgrounds, decodeURIComponent(req.params.path));
const data = await fsPromises.readFile(filePath);
return res.send(data);
}
catch {
return res.sendStatus(404);
}
});
app.use('/characters/:path', async (req, res) => {
try {
const filePath = path.join(process.cwd(), req.user.directories.characters, decodeURIComponent(req.params.path));
const data = await fsPromises.readFile(filePath);
return res.send(data);
}
catch {
return res.sendStatus(404);
}
});
app.use('/User Avatars/:path', async (req, res) => {
try {
const filePath = path.join(process.cwd(), req.user.directories.avatars, decodeURIComponent(req.params.path));
const data = await fsPromises.readFile(filePath);
return res.send(data);
}
catch {
return res.sendStatus(404);
}
});
function userDataMiddleware() {
/**
* Middleware to add user data to the request object.
* @param {import('express').Request} req Request object
@ -139,12 +106,44 @@ function userDataMiddleware(app) {
*/
return async (req, res, next) => {
const directories = await getCurrentUserDirectories(req);
req.user.profile = DEFAULT_USER;
req.user.directories = directories;
req.user = {
profile: DEFAULT_USER,
directories: directories,
};
next();
};
}
/**
* Creates a route handler for serving files from a specific directory.
* @param {(req: import('express').Request) => string} directoryFn A function that returns the directory path to serve files from
* @returns {import('express').RequestHandler}
*/
function createRouteHandler(directoryFn) {
return async (req, res) => {
try {
const directory = directoryFn(req);
const filePath = decodeURIComponent(req.params[0]);
return res.sendFile(filePath, { root: directory });
} catch (error) {
console.error(error);
return res.sendStatus(404);
}
};
}
/**
* Express router for serving files from the user's directories.
*/
const router = express.Router();
router.use('/backgrounds/*', createRouteHandler(req => req.user.directories.backgrounds));
router.use('/characters/*', createRouteHandler(req => req.user.directories.characters));
router.use('/User Avatars/*', createRouteHandler(req => req.user.directories.avatars));
router.use('/assets/*', createRouteHandler(req => req.user.directories.assets));
router.use('/user/images/*', createRouteHandler(req => req.user.directories.userImages));
router.use('/user/files/*', createRouteHandler(req => req.user.directories.files));
router.use('/scripts/extensions/third-party/*', createRouteHandler(req => req.user.directories.extensions));
module.exports = {
initUserStorage,
getCurrentUserDirectories,
@ -152,4 +151,5 @@ module.exports = {
getAllUserHandles,
getUserDirectories,
userDataMiddleware,
router,
};

9
src/util.js
View File

@ -321,11 +321,16 @@ function tryParse(str) {
/**
* Takes a path to a client-accessible file in the `public` folder and converts it to a relative URL segment that the
* client can fetch it from. This involves stripping the `public/` prefix and always using `/` as the separator.
* @param {string} root The root directory of the public folder.
* @param {string} inputPath The path to be converted.
* @returns The relative URL path from which the client can access the file.
*/
function clientRelativePath(inputPath) {
return path.normalize(inputPath).split(path.sep).slice(1).join('/');
function clientRelativePath(root, inputPath) {
if (!inputPath.startsWith(root)) {
throw new Error('Input path does not start with the root directory');
}
return inputPath.slice(root.length).split(path.sep).join('/');
}
/**