rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'staging' into feat/slash-commands-missing-args

This commit is contained in:
Cohee 2023-12-07 13:19:15 +02:00
parent d6d0371459 969c1ffefb
commit 25db4b6fde
12 changed files with 803 additions and 731 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
public/script.js
View File

@ -1249,7 +1249,7 @@ async function getCharacters() {
}
async function delChat(chatfile) {
const response = await fetch('/delchat', {
const response = await fetch('/api/chats/delete', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
@ -3883,7 +3883,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
toastr.error(data.response, 'API Error');
}
}
console.debug('/savechat called by /Generate');
console.debug('/api/chats/save called by /Generate');
await saveChatConditional();
is_send_press = false;
@ -4903,7 +4903,7 @@ async function renamePastChats(newAvatar, newValue) {
for (const { file_name } of pastChats) {
try {
const fileNameWithoutExtension = file_name.replace('.jsonl', '');
const getChatResponse = await fetch('/getchat', {
const getChatResponse = await fetch('/api/chats/get', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
@ -4927,7 +4927,7 @@ async function renamePastChats(newAvatar, newValue) {
}
}
const saveChatResponse = await fetch('/savechat', {
const saveChatResponse = await fetch('/api/chats/save', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
@ -5018,7 +5018,7 @@ async function saveChat(chat_name, withMetadata, mesId) {
];
return jQuery.ajax({
type: 'POST',
url: '/savechat',
url: '/api/chats/save',
data: JSON.stringify({
ch_name: characters[this_chid].name,
file_name: file_name,
@ -5110,11 +5110,11 @@ function getThumbnailUrl(type, file) {
}
async function getChat() {
//console.log('/getchat -- entered for -- ' + characters[this_chid].name);
//console.log('/api/chats/get -- entered for -- ' + characters[this_chid].name);
try {
const response = await $.ajax({
type: 'POST',
url: '/getchat',
url: '/api/chats/get',
data: JSON.stringify({
ch_name: characters[this_chid].name,
file_name: characters[this_chid].chat,
@ -5875,7 +5875,7 @@ export async function getChatsFromFiles(data, isGroupChat) {
let chat_promise = chat_list.map(({ file_name }) => {
return new Promise(async (res, rej) => {
try {
const endpoint = isGroupChat ? '/getgroupchat' : '/getchat';
const endpoint = isGroupChat ? '/api/chats/group/get' : '/api/chats/get';
const requestBody = isGroupChat
? JSON.stringify({ id: file_name })
: JSON.stringify({
@ -6562,7 +6562,7 @@ export async function saveChatConditional() {
async function importCharacterChat(formData) {
await jQuery.ajax({
type: 'POST',
url: '/importchat',
url: '/api/chats/import',
data: formData,
beforeSend: function () {
},
@ -8161,7 +8161,8 @@ jQuery(async function () {
};
try {
const response = await fetch('/renamechat', {
showLoader();
const response = await fetch('/api/chats/rename', {
method: 'POST',
body: JSON.stringify(body),
headers: getRequestHeaders(),
@ -8193,8 +8194,11 @@ jQuery(async function () {
$('#option_select_chat').trigger('click');
$('#options').hide();
} catch {
hideLoader();
await delay(500);
await callPopup('An error has occurred. Chat was not renamed.', 'text');
} finally {
hideLoader();
}
});
@ -8215,7 +8219,7 @@ jQuery(async function () {
};
console.log(body);
try {
const response = await fetch('/exportchat', {
const response = await fetch('/api/chats/export', {
method: 'POST',
body: JSON.stringify(body),
headers: getRequestHeaders(),

4
public/scripts/bookmarks.js
View File

@ -251,7 +251,7 @@ async function convertSoloToGroupChat() {
const metadata = Object.assign({}, chat_metadata);
delete metadata.main_chat;
const createGroupResponse = await fetch('/creategroup', {
const createGroupResponse = await fetch('/api/groups/create', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
@ -320,7 +320,7 @@ async function convertSoloToGroupChat() {
}
// Save group chat
const createChatResponse = await fetch('/savegroupchat', {
const createChatResponse = await fetch('/api/chats/group/save', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ id: chatName, chat: groupChat }),

20
public/scripts/group-chats.js
View File

@ -116,7 +116,7 @@ setInterval(groupChatAutoModeWorker, 5000);
const saveGroupDebounced = debounce(async (group, reload) => await _save(group, reload), 500);
async function _save(group, reload = true) {
await fetch('/editgroup', {
await fetch('/api/groups/edit', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify(group),
@ -152,7 +152,7 @@ async function regenerateGroup() {
}
async function loadGroupChat(chatId) {
const response = await fetch('/getgroupchat', {
const response = await fetch('/api/chats/group/get', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ id: chatId }),
@ -401,7 +401,7 @@ async function saveGroupChat(groupId, shouldSaveGroup) {
const group = groups.find(x => x.id == groupId);
const chat_id = group.chat_id;
group['date_last_chat'] = Date.now();
const response = await fetch('/savegroupchat', {
const response = await fetch('/api/chats/group/save', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ id: chat_id, chat: [...chat] }),
@ -455,7 +455,7 @@ export async function renameGroupMember(oldAvatar, newAvatar, newName) {
}
if (hadChanges) {
const saveChatResponse = await fetch('/savegroupchat', {
const saveChatResponse = await fetch('/api/chats/group/save', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ id: chatId, chat: [...messages] }),
@ -476,7 +476,7 @@ export async function renameGroupMember(oldAvatar, newAvatar, newName) {
}
async function getGroups() {
const response = await fetch('/getgroups', {
const response = await fetch('/api/groups/all', {
method: 'POST',
headers: getRequestHeaders(),
});
@ -968,7 +968,7 @@ function activateNaturalOrder(members, input, lastMessage, allowSelfResponses, i
async function deleteGroup(id) {
const group = groups.find((x) => x.id === id);
const response = await fetch('/deletegroup', {
const response = await fetch('/api/groups/delete', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ id: id }),
@ -1521,7 +1521,7 @@ async function createGroup() {
const chatName = humanizedDateTime();
const chats = [chatName];
const createGroupResponse = await fetch('/creategroup', {
const createGroupResponse = await fetch('/api/groups/create', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
@ -1659,7 +1659,7 @@ export async function deleteGroupChat(groupId, chatId) {
delete group.past_metadata[chatId];
updateChatMetadata(group.chat_metadata, true);
const response = await fetch('/deletegroupchat', {
const response = await fetch('/api/chats/group/delete', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ id: chatId }),
@ -1679,7 +1679,7 @@ export async function deleteGroupChat(groupId, chatId) {
export async function importGroupChat(formData) {
await jQuery.ajax({
type: 'POST',
url: '/importgroupchat',
url: '/api/chats/group/import',
data: formData,
beforeSend: function () {
},
@ -1720,7 +1720,7 @@ export async function saveGroupBookmarkChat(groupId, name, metadata, mesId) {
await editGroup(groupId, true, false);
await fetch('/savegroupchat', {
await fetch('/api/chats/group/save', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ id: name, chat: [...trimmed_chat] }),

1
public/scripts/power-user.js
View File

@ -813,6 +813,7 @@ async function CreateZenSliders(elmnt) {
if (numVal === offVal) { handle.text('Off').css('color', 'rgba(128,128,128,0.5'); }
else if (numVal === allVal) { handle.text('All'); }
else { handle.css('color', ''); }
numVal = steps[stepNumber];
}
//everything else uses the flat slider value
//also note: the above sliders are not custom inputtable due to the array aliasing

8
public/scripts/world-info.js
View File

@ -424,7 +424,7 @@ async function loadWorldInfoData(name) {
return worldInfoCache[name];
}
const response = await fetch('/getworldinfo', {
const response = await fetch('/api/worldinfo/get', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ name: name }),
@ -1402,7 +1402,7 @@ function createWorldInfoEntry(name, data, fromSlashCommand = false) {
}
async function _save(name, data) {
await fetch('/editworldinfo', {
await fetch('/api/worldinfo/edit', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ name: name, data: data }),
@ -1464,7 +1464,7 @@ async function deleteWorldInfo(worldInfoName) {
return;
}
const response = await fetch('/deleteworldinfo', {
const response = await fetch('/api/worldinfo/delete', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ name: worldInfoName }),
@ -2269,7 +2269,7 @@ export async function importWorldInfo(file) {
jQuery.ajax({
type: 'POST',
url: '/importworldinfo',
url: '/api/worldinfo/import',
data: formData,
beforeSend: () => { },
cache: false,

669
server.js
View File

@ -6,7 +6,6 @@ const fs = require('fs');
const http = require('http');
const https = require('https');
const path = require('path');
const readline = require('readline');
const util = require('util');
const { Readable } = require('stream');
@ -48,11 +47,10 @@ const { jsonParser, urlencodedParser } = require('./src/express-common.js');
const contentManager = require('./src/endpoints/content-manager');
const statsHelpers = require('./statsHelpers.js');
const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets');
const { delay, getVersion, getConfigValue, color, uuidv4, humanizedISO8601DateTime, tryParse, clientRelativePath, removeFileExtension } = require('./src/util');
const { delay, getVersion, getConfigValue, color, uuidv4, tryParse, clientRelativePath, removeFileExtension, generateTimestamp, removeOldBackups } = require('./src/util');
const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails');
const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers');
const { convertClaudePrompt } = require('./src/chat-completion');
const { readWorldInfoFile } = require('./src/worldinfo');
// Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0.
// https://github.com/nodejs/node/issues/47822#issuecomment-1564708870
@ -723,55 +721,6 @@ app.post('/api/textgenerationwebui/generate', jsonParser, async function (reques
}
});
app.post('/savechat', jsonParser, function (request, response) {
try {
var dir_name = String(request.body.avatar_url).replace('.png', '');
let chat_data = request.body.chat;
let jsonlData = chat_data.map(JSON.stringify).join('\n');
writeFileAtomicSync(`${DIRECTORIES.chats + sanitize(dir_name)}/${sanitize(String(request.body.file_name))}.jsonl`, jsonlData, 'utf8');
backupChat(dir_name, jsonlData);
return response.send({ result: 'ok' });
} catch (error) {
response.send(error);
return console.log(error);
}
});
app.post('/getchat', jsonParser, function (request, response) {
try {
const dirName = String(request.body.avatar_url).replace('.png', '');
const chatDirExists = fs.existsSync(DIRECTORIES.chats + dirName);
//if no chat dir for the character is found, make one with the character name
if (!chatDirExists) {
fs.mkdirSync(DIRECTORIES.chats + dirName);
return response.send({});
}
if (!request.body.file_name) {
return response.send({});
}
const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.file_name))}.jsonl`;
const chatFileExists = fs.existsSync(fileName);
if (!chatFileExists) {
return response.send({});
}
const data = fs.readFileSync(fileName, 'utf8');
const lines = data.split('\n');
// Iterate through the array of strings and parse each line as JSON
const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x);
return response.send(jsonData);
} catch (error) {
console.error(error);
return response.send({});
}
});
// Only called for kobold
app.post('/getstatus', jsonParser, async function (request, response) {
if (!request.body) return response.sendStatus(400);
@ -829,30 +778,6 @@ app.post('/getstatus', jsonParser, async function (request, response) {
}
});
app.post('/renamechat', jsonParser, async function (request, response) {
if (!request.body || !request.body.original_file || !request.body.renamed_file) {
return response.sendStatus(400);
}
const pathToFolder = request.body.is_group
? DIRECTORIES.groupChats
: path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', ''));
const pathToOriginalFile = path.join(pathToFolder, request.body.original_file);
const pathToRenamedFile = path.join(pathToFolder, request.body.renamed_file);
console.log('Old chat name', pathToOriginalFile);
console.log('New chat name', pathToRenamedFile);
if (!fs.existsSync(pathToOriginalFile) || fs.existsSync(pathToRenamedFile)) {
console.log('Either Source or Destination files are not available');
return response.status(400).send({ error: true });
}
console.log('Successfully renamed.');
fs.renameSync(pathToOriginalFile, pathToRenamedFile);
return response.send({ ok: true });
});
/**
* Handle a POST request to get the stats object
*
@ -967,37 +892,6 @@ app.post('/delbackground', jsonParser, function (request, response) {
return response.send('ok');
});
app.post('/delchat', jsonParser, function (request, response) {
console.log('/delchat entered');
if (!request.body) {
console.log('no request body seen');
return response.sendStatus(400);
}
if (request.body.chatfile !== sanitize(request.body.chatfile)) {
console.error('Malicious chat name prevented');
return response.sendStatus(403);
}
const dirName = String(request.body.avatar_url).replace('.png', '');
const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.chatfile))}`;
const chatFileExists = fs.existsSync(fileName);
if (!chatFileExists) {
console.log(`Chat file not found '${fileName}'`);
return response.sendStatus(400);
} else {
console.log('found the chat file: ' + fileName);
/* fs.unlinkSync(fileName); */
fs.rmSync(fileName);
console.log('deleted chat file: ' + fileName);
}
return response.send('ok');
});
app.post('/renamebackground', jsonParser, function (request, response) {
if (!request.body) return response.sendStatus(400);
@ -1165,34 +1059,6 @@ app.post('/getsettings', jsonParser, (request, response) => {
});
});
app.post('/getworldinfo', jsonParser, (request, response) => {
if (!request.body?.name) {
return response.sendStatus(400);
}
const file = readWorldInfoFile(request.body.name);
return response.send(file);
});
app.post('/deleteworldinfo', jsonParser, (request, response) => {
if (!request.body?.name) {
return response.sendStatus(400);
}
const worldInfoName = request.body.name;
const filename = sanitize(`${worldInfoName}.json`);
const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename);
if (!fs.existsSync(pathToWorldInfo)) {
throw new Error(`World info file ${filename} doesn't exist.`);
}
fs.rmSync(pathToWorldInfo);
return response.sendStatus(200);
});
app.post('/savetheme', jsonParser, (request, response) => {
if (!request.body || !request.body.name) {
return response.sendStatus(400);
@ -1237,275 +1103,6 @@ function getImages(path) {
.sort(Intl.Collator().compare);
}
app.post('/exportchat', jsonParser, async function (request, response) {
if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) {
return response.sendStatus(400);
}
const pathToFolder = request.body.is_group
? DIRECTORIES.groupChats
: path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', ''));
let filename = path.join(pathToFolder, request.body.file);
let exportfilename = request.body.exportfilename;
if (!fs.existsSync(filename)) {
const errorMessage = {
message: `Could not find JSONL file to export. Source chat file: ${filename}.`,
};
console.log(errorMessage.message);
return response.status(404).json(errorMessage);
}
try {
// Short path for JSONL files
if (request.body.format == 'jsonl') {
try {
const rawFile = fs.readFileSync(filename, 'utf8');
const successMessage = {
message: `Chat saved to ${exportfilename}`,
result: rawFile,
};
console.log(`Chat exported as ${exportfilename}`);
return response.status(200).json(successMessage);
}
catch (err) {
console.error(err);
const errorMessage = {
message: `Could not read JSONL file to export. Source chat file: ${filename}.`,
};
console.log(errorMessage.message);
return response.status(500).json(errorMessage);
}
}
const readStream = fs.createReadStream(filename);
const rl = readline.createInterface({
input: readStream,
});
let buffer = '';
rl.on('line', (line) => {
const data = JSON.parse(line);
if (data.mes) {
const name = data.name;
const message = (data?.extra?.display_text || data?.mes || '').replace(/\r?\n/g, '\n');
buffer += (`${name}: ${message}\n\n`);
}
});
rl.on('close', () => {
const successMessage = {
message: `Chat saved to ${exportfilename}`,
result: buffer,
};
console.log(`Chat exported as ${exportfilename}`);
return response.status(200).json(successMessage);
});
}
catch (err) {
console.log('chat export failed.');
console.log(err);
return response.sendStatus(400);
}
});
app.post('/importgroupchat', urlencodedParser, function (request, response) {
try {
const filedata = request.file;
if (!filedata) {
return response.sendStatus(400);
}
const chatname = humanizedISO8601DateTime();
const pathToUpload = path.join(UPLOADS_PATH, filedata.filename);
const pathToNewFile = path.join(DIRECTORIES.groupChats, `${chatname}.jsonl`);
fs.copyFileSync(pathToUpload, pathToNewFile);
fs.unlinkSync(pathToUpload);
return response.send({ res: chatname });
} catch (error) {
console.error(error);
return response.send({ error: true });
}
});
app.post('/importchat', urlencodedParser, function (request, response) {
if (!request.body) return response.sendStatus(400);
var format = request.body.file_type;
let filedata = request.file;
let avatar_url = (request.body.avatar_url).replace('.png', '');
let ch_name = request.body.character_name;
let user_name = request.body.user_name || 'You';
if (!filedata) {
return response.sendStatus(400);
}
try {
const data = fs.readFileSync(path.join(UPLOADS_PATH, filedata.filename), 'utf8');
if (format === 'json') {
const jsonData = JSON.parse(data);
if (jsonData.histories !== undefined) {
//console.log('/importchat confirms JSON histories are defined');
const chat = {
from(history) {
return [
{
user_name: user_name,
character_name: ch_name,
create_date: humanizedISO8601DateTime(),
},
...history.msgs.map(
(message) => ({
name: message.src.is_human ? user_name : ch_name,
is_user: message.src.is_human,
send_date: humanizedISO8601DateTime(),
mes: message.text,
}),
)];
},
};
const newChats = [];
(jsonData.histories.histories ?? []).forEach((history) => {
newChats.push(chat.from(history));
});
const errors = [];
for (const chat of newChats) {
const filePath = `${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`;
const fileContent = chat.map(tryParse).filter(x => x).join('\n');
try {
writeFileAtomicSync(filePath, fileContent, 'utf8');
} catch (err) {
errors.push(err);
}
}
if (0 < errors.length) {
response.send('Errors occurred while writing character files. Errors: ' + JSON.stringify(errors));
}
response.send({ res: true });
} else if (Array.isArray(jsonData.data_visible)) {
// oobabooga's format
/** @type {object[]} */
const chat = [{
user_name: user_name,
character_name: ch_name,
create_date: humanizedISO8601DateTime(),
}];
for (const arr of jsonData.data_visible) {
if (arr[0]) {
const userMessage = {
name: user_name,
is_user: true,
send_date: humanizedISO8601DateTime(),
mes: arr[0],
};
chat.push(userMessage);
}
if (arr[1]) {
const charMessage = {
name: ch_name,
is_user: false,
send_date: humanizedISO8601DateTime(),
mes: arr[1],
};
chat.push(charMessage);
}
}
const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');
writeFileAtomicSync(`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chatContent, 'utf8');
response.send({ res: true });
} else {
console.log('Incorrect chat format .json');
return response.send({ error: true });
}
}
if (format === 'jsonl') {
const line = data.split('\n')[0];
let jsonData = JSON.parse(line);
if (jsonData.user_name !== undefined || jsonData.name !== undefined) {
fs.copyFileSync(path.join(UPLOADS_PATH, filedata.filename), (`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()}.jsonl`));
response.send({ res: true });
} else {
console.log('Incorrect chat format .jsonl');
return response.send({ error: true });
}
}
} catch (error) {
console.error(error);
return response.send({ error: true });
}
});
app.post('/importworldinfo', urlencodedParser, (request, response) => {
if (!request.file) return response.sendStatus(400);
const filename = `${path.parse(sanitize(request.file.originalname)).name}.json`;
let fileContents = null;
if (request.body.convertedData) {
fileContents = request.body.convertedData;
} else {
const pathToUpload = path.join(UPLOADS_PATH, request.file.filename);
fileContents = fs.readFileSync(pathToUpload, 'utf8');
fs.unlinkSync(pathToUpload);
}
try {
const worldContent = JSON.parse(fileContents);
if (!('entries' in worldContent)) {
throw new Error('File must contain a world info entries list');
}
} catch (err) {
return response.status(400).send('Is not a valid world info file');
}
const pathToNewFile = path.join(DIRECTORIES.worlds, filename);
const worldName = path.parse(pathToNewFile).name;
if (!worldName) {
return response.status(400).send('World file must have a name');
}
writeFileAtomicSync(pathToNewFile, fileContents);
return response.send({ name: worldName });
});
app.post('/editworldinfo', jsonParser, (request, response) => {
if (!request.body) {
return response.sendStatus(400);
}
if (!request.body.name) {
return response.status(400).send('World file must have a name');
}
try {
if (!('entries' in request.body.data)) {
throw new Error('World info must contain an entries list');
}
} catch (err) {
return response.status(400).send('Is not a valid world info file');
}
const filename = `${sanitize(request.body.name)}.json`;
const pathToFile = path.join(DIRECTORIES.worlds, filename);
writeFileAtomicSync(pathToFile, JSON.stringify(request.body.data, null, 4));
return response.send({ ok: true });
});
app.post('/uploaduseravatar', urlencodedParser, async (request, response) => {
if (!request.file) return response.sendStatus(400);
@ -1614,181 +1211,6 @@ app.post('/listimgfiles/:folder', (req, res) => {
});
app.post('/getgroups', jsonParser, (_, response) => {
const groups = [];
if (!fs.existsSync(DIRECTORIES.groups)) {
fs.mkdirSync(DIRECTORIES.groups);
}
const files = fs.readdirSync(DIRECTORIES.groups).filter(x => path.extname(x) === '.json');
const chats = fs.readdirSync(DIRECTORIES.groupChats).filter(x => path.extname(x) === '.jsonl');
files.forEach(function (file) {
try {
const filePath = path.join(DIRECTORIES.groups, file);
const fileContents = fs.readFileSync(filePath, 'utf8');
const group = JSON.parse(fileContents);
const groupStat = fs.statSync(filePath);
group['date_added'] = groupStat.birthtimeMs;
group['create_date'] = humanizedISO8601DateTime(groupStat.birthtimeMs);
let chat_size = 0;
let date_last_chat = 0;
if (Array.isArray(group.chats) && Array.isArray(chats)) {
for (const chat of chats) {
if (group.chats.includes(path.parse(chat).name)) {
const chatStat = fs.statSync(path.join(DIRECTORIES.groupChats, chat));
chat_size += chatStat.size;
date_last_chat = Math.max(date_last_chat, chatStat.mtimeMs);
}
}
}
group['date_last_chat'] = date_last_chat;
group['chat_size'] = chat_size;
groups.push(group);
}
catch (error) {
console.error(error);
}
});
return response.send(groups);
});
app.post('/creategroup', jsonParser, (request, response) => {
if (!request.body) {
return response.sendStatus(400);
}
const id = String(Date.now());
const groupMetadata = {
id: id,
name: request.body.name ?? 'New Group',
members: request.body.members ?? [],
avatar_url: request.body.avatar_url,
allow_self_responses: !!request.body.allow_self_responses,
activation_strategy: request.body.activation_strategy ?? 0,
generation_mode: request.body.generation_mode ?? 0,
disabled_members: request.body.disabled_members ?? [],
chat_metadata: request.body.chat_metadata ?? {},
fav: request.body.fav,
chat_id: request.body.chat_id ?? id,
chats: request.body.chats ?? [id],
};
const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`);
const fileData = JSON.stringify(groupMetadata);
if (!fs.existsSync(DIRECTORIES.groups)) {
fs.mkdirSync(DIRECTORIES.groups);
}
writeFileAtomicSync(pathToFile, fileData);
return response.send(groupMetadata);
});
app.post('/editgroup', jsonParser, (request, response) => {
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`);
const fileData = JSON.stringify(request.body);
writeFileAtomicSync(pathToFile, fileData);
return response.send({ ok: true });
});
app.post('/getgroupchat', jsonParser, (request, response) => {
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
if (fs.existsSync(pathToFile)) {
const data = fs.readFileSync(pathToFile, 'utf8');
const lines = data.split('\n');
// Iterate through the array of strings and parse each line as JSON
const jsonData = lines.map(line => tryParse(line)).filter(x => x);
return response.send(jsonData);
} else {
return response.send([]);
}
});
app.post('/deletegroupchat', jsonParser, (request, response) => {
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
if (fs.existsSync(pathToFile)) {
fs.rmSync(pathToFile);
return response.send({ ok: true });
}
return response.send({ error: true });
});
app.post('/savegroupchat', jsonParser, (request, response) => {
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
if (!fs.existsSync(DIRECTORIES.groupChats)) {
fs.mkdirSync(DIRECTORIES.groupChats);
}
let chat_data = request.body.chat;
let jsonlData = chat_data.map(JSON.stringify).join('\n');
writeFileAtomicSync(pathToFile, jsonlData, 'utf8');
backupChat(String(id), jsonlData);
return response.send({ ok: true });
});
app.post('/deletegroup', jsonParser, async (request, response) => {
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
const id = request.body.id;
const pathToGroup = path.join(DIRECTORIES.groups, sanitize(`${id}.json`));
try {
// Delete group chats
const group = JSON.parse(fs.readFileSync(pathToGroup, 'utf8'));
if (group && Array.isArray(group.chats)) {
for (const chat of group.chats) {
console.log('Deleting group chat', chat);
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
if (fs.existsSync(pathToFile)) {
fs.rmSync(pathToFile);
}
}
}
} catch (error) {
console.error('Could not delete group chats. Clean them up manually.', error);
}
if (fs.existsSync(pathToGroup)) {
fs.rmSync(pathToGroup);
}
return response.send({ ok: true });
});
function cleanUploads() {
try {
if (fs.existsSync(UPLOADS_PATH)) {
@ -2605,6 +2027,30 @@ redirect('/importcharacter', '/api/characters/import');
redirect('/dupecharacter', '/api/characters/duplicate');
redirect('/exportcharacter', '/api/characters/export');
// Redirect deprecated chat API endpoints
redirect('/savechat', '/api/chats/save');
redirect('/getchat', '/api/chats/get');
redirect('/renamechat', '/api/chats/rename');
redirect('/delchat', '/api/chats/delete');
redirect('/exportchat', '/api/chats/export');
redirect('/importgroupchat', '/api/chats/group/import');
redirect('/importchat', '/api/chats/import');
redirect('/getgroupchat', '/api/chats/group/get');
redirect('/deletegroupchat', '/api/chats/group/delete');
redirect('/savegroupchat', '/api/chats/group/save');
// Redirect deprecated group API endpoints
redirect('/getgroups', '/api/groups/all');
redirect('/creategroup', '/api/groups/create');
redirect('/editgroup', '/api/groups/edit');
redirect('/deletegroup', '/api/groups/delete');
// Redirect deprecated worldinfo API endpoints
redirect('/getworldinfo', '/api/worldinfo/get');
redirect('/deleteworldinfo', '/api/worldinfo/delete');
redirect('/importworldinfo', '/api/worldinfo/import');
redirect('/editworldinfo', '/api/worldinfo/edit');
// ** REST CLIENT ASYNC WRAPPERS **
/**
@ -2656,6 +2102,15 @@ app.use('/api/files', require('./src/endpoints/files').router);
// Character management
app.use('/api/characters', require('./src/endpoints/characters').router);
// Chat management
app.use('/api/chats', require('./src/endpoints/chats').router);
// Group management
app.use('/api/groups', require('./src/endpoints/groups').router);
// World info management
app.use('/api/worldinfo', require('./src/endpoints/worldinfo').router);
// Character sprite management
app.use('/api/sprites', require('./src/endpoints/sprites').router);
@ -2760,47 +2215,6 @@ if (cliArguments.ssl) {
);
}
function generateTimestamp() {
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0');
const day = String(now.getDate()).padStart(2, '0');
const hours = String(now.getHours()).padStart(2, '0');
const minutes = String(now.getMinutes()).padStart(2, '0');
const seconds = String(now.getSeconds()).padStart(2, '0');
return `${year}${month}${day}-${hours}${minutes}${seconds}`;
}
/**
*
* @param {string} name
* @param {string} chat
*/
function backupChat(name, chat) {
try {
const isBackupDisabled = getConfigValue('disableChatBackup', false);
if (isBackupDisabled) {
return;
}
if (!fs.existsSync(DIRECTORIES.backups)) {
fs.mkdirSync(DIRECTORIES.backups);
}
// replace non-alphanumeric characters with underscores
name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();
const backupFile = path.join(DIRECTORIES.backups, `chat_${name}_${generateTimestamp()}.jsonl`);
writeFileAtomicSync(backupFile, chat, 'utf-8');
removeOldBackups(`chat_${name}_`);
} catch (err) {
console.log(`Could not backup chat for ${name}`, err);
}
}
function backupSettings() {
try {
if (!fs.existsSync(DIRECTORIES.backups)) {
@ -2816,21 +2230,6 @@ function backupSettings() {
}
}
/**
* @param {string} prefix
*/
function removeOldBackups(prefix) {
const MAX_BACKUPS = 25;
let files = fs.readdirSync(DIRECTORIES.backups).filter(f => f.startsWith(prefix));
if (files.length > MAX_BACKUPS) {
files = files.map(f => path.join(DIRECTORIES.backups, f));
files.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs);
fs.rmSync(files[0]);
}
}
function ensurePublicDirectoriesExist() {
for (const dir of Object.values(DIRECTORIES)) {
if (!fs.existsSync(dir)) {

42
src/endpoints/characters.js
View File

@ -16,7 +16,7 @@ const { jsonParser, urlencodedParser } = require('../express-common');
const { deepMerge, humanizedISO8601DateTime, tryParse } = require('../util');
const { TavernCardValidator } = require('../validator/TavernCardValidator');
const characterCardParser = require('../character-card-parser.js');
const { readWorldInfoFile, convertWorldInfoToCharacterBook } = require('../worldinfo');
const { readWorldInfoFile } = require('./worldinfo');
const { invalidateThumbnail } = require('./thumbnails');
const { importRisuSprites } = require('./sprites');
@ -330,6 +330,46 @@ function charaFormatData(data) {
return char;
}
/**
* @param {string} name Name of World Info file
* @param {object} entries Entries object
*/
function convertWorldInfoToCharacterBook(name, entries) {
/** @type {{ entries: object[]; name: string }} */
const result = { entries: [], name };
for (const index in entries) {
const entry = entries[index];
const originalEntry = {
id: entry.uid,
keys: entry.key,
secondary_keys: entry.keysecondary,
comment: entry.comment,
content: entry.content,
constant: entry.constant,
selective: entry.selective,
insertion_order: entry.order,
enabled: !entry.disable,
position: entry.position == 0 ? 'before_char' : 'after_char',
extensions: {
position: entry.position,
exclude_recursion: entry.excludeRecursion,
display_index: entry.displayIndex,
probability: entry.probability ?? null,
useProbability: entry.useProbability ?? false,
depth: entry.depth ?? 4,
selectiveLogic: entry.selectiveLogic ?? 0,
group: entry.group ?? '',
},
};
result.entries.push(originalEntry);
}
return result;
}
const router = express.Router();
router.post('/create', urlencodedParser, async function (request, response) {

411
src/endpoints/chats.js Normal file
View File

@ -0,0 +1,411 @@
const fs = require('fs');
const path = require('path');
const readline = require('readline');
const express = require('express');
const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser, urlencodedParser } = require('../express-common');
const { DIRECTORIES, UPLOADS_PATH } = require('../constants');
const { getConfigValue, humanizedISO8601DateTime, tryParse, generateTimestamp, removeOldBackups } = require('../util');
/**
*
* @param {string} name
* @param {string} chat
*/
function backupChat(name, chat) {
try {
const isBackupDisabled = getConfigValue('disableChatBackup', false);
if (isBackupDisabled) {
return;
}
if (!fs.existsSync(DIRECTORIES.backups)) {
fs.mkdirSync(DIRECTORIES.backups);
}
// replace non-alphanumeric characters with underscores
name = sanitize(name).replace(/[^a-z0-9]/gi, '_').toLowerCase();
const backupFile = path.join(DIRECTORIES.backups, `chat_${name}_${generateTimestamp()}.jsonl`);
writeFileAtomicSync(backupFile, chat, 'utf-8');
removeOldBackups(`chat_${name}_`);
} catch (err) {
console.log(`Could not backup chat for ${name}`, err);
}
}
const router = express.Router();
router.post('/save', jsonParser, function (request, response) {
try {
var dir_name = String(request.body.avatar_url).replace('.png', '');
let chat_data = request.body.chat;
let jsonlData = chat_data.map(JSON.stringify).join('\n');
writeFileAtomicSync(`${DIRECTORIES.chats + sanitize(dir_name)}/${sanitize(String(request.body.file_name))}.jsonl`, jsonlData, 'utf8');
backupChat(dir_name, jsonlData);
return response.send({ result: 'ok' });
} catch (error) {
response.send(error);
return console.log(error);
}
});
router.post('/get', jsonParser, function (request, response) {
try {
const dirName = String(request.body.avatar_url).replace('.png', '');
const chatDirExists = fs.existsSync(DIRECTORIES.chats + dirName);
//if no chat dir for the character is found, make one with the character name
if (!chatDirExists) {
fs.mkdirSync(DIRECTORIES.chats + dirName);
return response.send({});
}
if (!request.body.file_name) {
return response.send({});
}
const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.file_name))}.jsonl`;
const chatFileExists = fs.existsSync(fileName);
if (!chatFileExists) {
return response.send({});
}
const data = fs.readFileSync(fileName, 'utf8');
const lines = data.split('\n');
// Iterate through the array of strings and parse each line as JSON
const jsonData = lines.map((l) => { try { return JSON.parse(l); } catch (_) { return; } }).filter(x => x);
return response.send(jsonData);
} catch (error) {
console.error(error);
return response.send({});
}
});
router.post('/rename', jsonParser, async function (request, response) {
if (!request.body || !request.body.original_file || !request.body.renamed_file) {
return response.sendStatus(400);
}
const pathToFolder = request.body.is_group
? DIRECTORIES.groupChats
: path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', ''));
const pathToOriginalFile = path.join(pathToFolder, request.body.original_file);
const pathToRenamedFile = path.join(pathToFolder, request.body.renamed_file);
console.log('Old chat name', pathToOriginalFile);
console.log('New chat name', pathToRenamedFile);
if (!fs.existsSync(pathToOriginalFile) || fs.existsSync(pathToRenamedFile)) {
console.log('Either Source or Destination files are not available');
return response.status(400).send({ error: true });
}
console.log('Successfully renamed.');
fs.renameSync(pathToOriginalFile, pathToRenamedFile);
return response.send({ ok: true });
});
router.post('/delete', jsonParser, function (request, response) {
console.log('/api/chats/delete entered');
if (!request.body) {
console.log('no request body seen');
return response.sendStatus(400);
}
if (request.body.chatfile !== sanitize(request.body.chatfile)) {
console.error('Malicious chat name prevented');
return response.sendStatus(403);
}
const dirName = String(request.body.avatar_url).replace('.png', '');
const fileName = `${DIRECTORIES.chats + dirName}/${sanitize(String(request.body.chatfile))}`;
const chatFileExists = fs.existsSync(fileName);
if (!chatFileExists) {
console.log(`Chat file not found '${fileName}'`);
return response.sendStatus(400);
} else {
console.log('found the chat file: ' + fileName);
/* fs.unlinkSync(fileName); */
fs.rmSync(fileName);
console.log('deleted chat file: ' + fileName);
}
return response.send('ok');
});
router.post('/export', jsonParser, async function (request, response) {
if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) {
return response.sendStatus(400);
}
const pathToFolder = request.body.is_group
? DIRECTORIES.groupChats
: path.join(DIRECTORIES.chats, String(request.body.avatar_url).replace('.png', ''));
let filename = path.join(pathToFolder, request.body.file);
let exportfilename = request.body.exportfilename;
if (!fs.existsSync(filename)) {
const errorMessage = {
message: `Could not find JSONL file to export. Source chat file: ${filename}.`,
};
console.log(errorMessage.message);
return response.status(404).json(errorMessage);
}
try {
// Short path for JSONL files
if (request.body.format == 'jsonl') {
try {
const rawFile = fs.readFileSync(filename, 'utf8');
const successMessage = {
message: `Chat saved to ${exportfilename}`,
result: rawFile,
};
console.log(`Chat exported as ${exportfilename}`);
return response.status(200).json(successMessage);
}
catch (err) {
console.error(err);
const errorMessage = {
message: `Could not read JSONL file to export. Source chat file: ${filename}.`,
};
console.log(errorMessage.message);
return response.status(500).json(errorMessage);
}
}
const readStream = fs.createReadStream(filename);
const rl = readline.createInterface({
input: readStream,
});
let buffer = '';
rl.on('line', (line) => {
const data = JSON.parse(line);
if (data.mes) {
const name = data.name;
const message = (data?.extra?.display_text || data?.mes || '').replace(/\r?\n/g, '\n');
buffer += (`${name}: ${message}\n\n`);
}
});
rl.on('close', () => {
const successMessage = {
message: `Chat saved to ${exportfilename}`,
result: buffer,
};
console.log(`Chat exported as ${exportfilename}`);
return response.status(200).json(successMessage);
});
}
catch (err) {
console.log('chat export failed.');
console.log(err);
return response.sendStatus(400);
}
});
router.post('/group/import', urlencodedParser, function (request, response) {
try {
const filedata = request.file;
if (!filedata) {
return response.sendStatus(400);
}
const chatname = humanizedISO8601DateTime();
const pathToUpload = path.join(UPLOADS_PATH, filedata.filename);
const pathToNewFile = path.join(DIRECTORIES.groupChats, `${chatname}.jsonl`);
fs.copyFileSync(pathToUpload, pathToNewFile);
fs.unlinkSync(pathToUpload);
return response.send({ res: chatname });
} catch (error) {
console.error(error);
return response.send({ error: true });
}
});
router.post('/import', urlencodedParser, function (request, response) {
if (!request.body) return response.sendStatus(400);
var format = request.body.file_type;
let filedata = request.file;
let avatar_url = (request.body.avatar_url).replace('.png', '');
let ch_name = request.body.character_name;
let user_name = request.body.user_name || 'You';
if (!filedata) {
return response.sendStatus(400);
}
try {
const data = fs.readFileSync(path.join(UPLOADS_PATH, filedata.filename), 'utf8');
if (format === 'json') {
const jsonData = JSON.parse(data);
if (jsonData.histories !== undefined) {
//console.log('/api/chats/import confirms JSON histories are defined');
const chat = {
from(history) {
return [
{
user_name: user_name,
character_name: ch_name,
create_date: humanizedISO8601DateTime(),
},
...history.msgs.map(
(message) => ({
name: message.src.is_human ? user_name : ch_name,
is_user: message.src.is_human,
send_date: humanizedISO8601DateTime(),
mes: message.text,
}),
)];
},
};
const newChats = [];
(jsonData.histories.histories ?? []).forEach((history) => {
newChats.push(chat.from(history));
});
const errors = [];
for (const chat of newChats) {
const filePath = `${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`;
const fileContent = chat.map(tryParse).filter(x => x).join('\n');
try {
writeFileAtomicSync(filePath, fileContent, 'utf8');
} catch (err) {
errors.push(err);
}
}
if (0 < errors.length) {
response.send('Errors occurred while writing character files. Errors: ' + JSON.stringify(errors));
}
response.send({ res: true });
} else if (Array.isArray(jsonData.data_visible)) {
// oobabooga's format
/** @type {object[]} */
const chat = [{
user_name: user_name,
character_name: ch_name,
create_date: humanizedISO8601DateTime(),
}];
for (const arr of jsonData.data_visible) {
if (arr[0]) {
const userMessage = {
name: user_name,
is_user: true,
send_date: humanizedISO8601DateTime(),
mes: arr[0],
};
chat.push(userMessage);
}
if (arr[1]) {
const charMessage = {
name: ch_name,
is_user: false,
send_date: humanizedISO8601DateTime(),
mes: arr[1],
};
chat.push(charMessage);
}
}
const chatContent = chat.map(obj => JSON.stringify(obj)).join('\n');
writeFileAtomicSync(`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chatContent, 'utf8');
response.send({ res: true });
} else {
console.log('Incorrect chat format .json');
return response.send({ error: true });
}
}
if (format === 'jsonl') {
const line = data.split('\n')[0];
let jsonData = JSON.parse(line);
if (jsonData.user_name !== undefined || jsonData.name !== undefined) {
fs.copyFileSync(path.join(UPLOADS_PATH, filedata.filename), (`${DIRECTORIES.chats + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()}.jsonl`));
response.send({ res: true });
} else {
console.log('Incorrect chat format .jsonl');
return response.send({ error: true });
}
}
} catch (error) {
console.error(error);
return response.send({ error: true });
}
});
router.post('/group/get', jsonParser, (request, response) => {
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
if (fs.existsSync(pathToFile)) {
const data = fs.readFileSync(pathToFile, 'utf8');
const lines = data.split('\n');
// Iterate through the array of strings and parse each line as JSON
const jsonData = lines.map(line => tryParse(line)).filter(x => x);
return response.send(jsonData);
} else {
return response.send([]);
}
});
router.post('/group/delete', jsonParser, (request, response) => {
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
if (fs.existsSync(pathToFile)) {
fs.rmSync(pathToFile);
return response.send({ ok: true });
}
return response.send({ error: true });
});
router.post('/group/save', jsonParser, (request, response) => {
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
if (!fs.existsSync(DIRECTORIES.groupChats)) {
fs.mkdirSync(DIRECTORIES.groupChats);
}
let chat_data = request.body.chat;
let jsonlData = chat_data.map(JSON.stringify).join('\n');
writeFileAtomicSync(pathToFile, jsonlData, 'utf8');
backupChat(String(id), jsonlData);
return response.send({ ok: true });
});
module.exports = { router };

133
src/endpoints/groups.js Normal file
View File

@ -0,0 +1,133 @@
const fs = require('fs');
const path = require('path');
const express = require('express');
const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
const { humanizedISO8601DateTime } = require('../util');
const router = express.Router();
router.post('/all', jsonParser, (_, response) => {
const groups = [];
if (!fs.existsSync(DIRECTORIES.groups)) {
fs.mkdirSync(DIRECTORIES.groups);
}
const files = fs.readdirSync(DIRECTORIES.groups).filter(x => path.extname(x) === '.json');
const chats = fs.readdirSync(DIRECTORIES.groupChats).filter(x => path.extname(x) === '.jsonl');
files.forEach(function (file) {
try {
const filePath = path.join(DIRECTORIES.groups, file);
const fileContents = fs.readFileSync(filePath, 'utf8');
const group = JSON.parse(fileContents);
const groupStat = fs.statSync(filePath);
group['date_added'] = groupStat.birthtimeMs;
group['create_date'] = humanizedISO8601DateTime(groupStat.birthtimeMs);
let chat_size = 0;
let date_last_chat = 0;
if (Array.isArray(group.chats) && Array.isArray(chats)) {
for (const chat of chats) {
if (group.chats.includes(path.parse(chat).name)) {
const chatStat = fs.statSync(path.join(DIRECTORIES.groupChats, chat));
chat_size += chatStat.size;
date_last_chat = Math.max(date_last_chat, chatStat.mtimeMs);
}
}
}
group['date_last_chat'] = date_last_chat;
group['chat_size'] = chat_size;
groups.push(group);
}
catch (error) {
console.error(error);
}
});
return response.send(groups);
});
router.post('/create', jsonParser, (request, response) => {
if (!request.body) {
return response.sendStatus(400);
}
const id = String(Date.now());
const groupMetadata = {
id: id,
name: request.body.name ?? 'New Group',
members: request.body.members ?? [],
avatar_url: request.body.avatar_url,
allow_self_responses: !!request.body.allow_self_responses,
activation_strategy: request.body.activation_strategy ?? 0,
generation_mode: request.body.generation_mode ?? 0,
disabled_members: request.body.disabled_members ?? [],
chat_metadata: request.body.chat_metadata ?? {},
fav: request.body.fav,
chat_id: request.body.chat_id ?? id,
chats: request.body.chats ?? [id],
};
const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`);
const fileData = JSON.stringify(groupMetadata);
if (!fs.existsSync(DIRECTORIES.groups)) {
fs.mkdirSync(DIRECTORIES.groups);
}
writeFileAtomicSync(pathToFile, fileData);
return response.send(groupMetadata);
});
router.post('/edit', jsonParser, (request, response) => {
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
const id = request.body.id;
const pathToFile = path.join(DIRECTORIES.groups, `${id}.json`);
const fileData = JSON.stringify(request.body);
writeFileAtomicSync(pathToFile, fileData);
return response.send({ ok: true });
});
router.post('/delete', jsonParser, async (request, response) => {
if (!request.body || !request.body.id) {
return response.sendStatus(400);
}
const id = request.body.id;
const pathToGroup = path.join(DIRECTORIES.groups, sanitize(`${id}.json`));
try {
// Delete group chats
const group = JSON.parse(fs.readFileSync(pathToGroup, 'utf8'));
if (group && Array.isArray(group.chats)) {
for (const chat of group.chats) {
console.log('Deleting group chat', chat);
const pathToFile = path.join(DIRECTORIES.groupChats, `${id}.jsonl`);
if (fs.existsSync(pathToFile)) {
fs.rmSync(pathToFile);
}
}
}
} catch (error) {
console.error('Could not delete group chats. Clean them up manually.', error);
}
if (fs.existsSync(pathToGroup)) {
fs.rmSync(pathToGroup);
}
return response.send({ ok: true });
});
module.exports = { router };

120
src/endpoints/worldinfo.js Normal file
View File

@ -0,0 +1,120 @@
const fs = require('fs');
const path = require('path');
const express = require('express');
const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { jsonParser, urlencodedParser } = require('../express-common');
const { DIRECTORIES, UPLOADS_PATH } = require('../constants');
function readWorldInfoFile(worldInfoName) {
const dummyObject = { entries: {} };
if (!worldInfoName) {
return dummyObject;
}
const filename = `${worldInfoName}.json`;
const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename);
if (!fs.existsSync(pathToWorldInfo)) {
console.log(`World info file ${filename} doesn't exist.`);
return dummyObject;
}
const worldInfoText = fs.readFileSync(pathToWorldInfo, 'utf8');
const worldInfo = JSON.parse(worldInfoText);
return worldInfo;
}
const router = express.Router();
router.post('/get', jsonParser, (request, response) => {
if (!request.body?.name) {
return response.sendStatus(400);
}
const file = readWorldInfoFile(request.body.name);
return response.send(file);
});
router.post('/delete', jsonParser, (request, response) => {
if (!request.body?.name) {
return response.sendStatus(400);
}
const worldInfoName = request.body.name;
const filename = sanitize(`${worldInfoName}.json`);
const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename);
if (!fs.existsSync(pathToWorldInfo)) {
throw new Error(`World info file ${filename} doesn't exist.`);
}
fs.rmSync(pathToWorldInfo);
return response.sendStatus(200);
});
router.post('/import', urlencodedParser, (request, response) => {
if (!request.file) return response.sendStatus(400);
const filename = `${path.parse(sanitize(request.file.originalname)).name}.json`;
let fileContents = null;
if (request.body.convertedData) {
fileContents = request.body.convertedData;
} else {
const pathToUpload = path.join(UPLOADS_PATH, request.file.filename);
fileContents = fs.readFileSync(pathToUpload, 'utf8');
fs.unlinkSync(pathToUpload);
}
try {
const worldContent = JSON.parse(fileContents);
if (!('entries' in worldContent)) {
throw new Error('File must contain a world info entries list');
}
} catch (err) {
return response.status(400).send('Is not a valid world info file');
}
const pathToNewFile = path.join(DIRECTORIES.worlds, filename);
const worldName = path.parse(pathToNewFile).name;
if (!worldName) {
return response.status(400).send('World file must have a name');
}
writeFileAtomicSync(pathToNewFile, fileContents);
return response.send({ name: worldName });
});
router.post('/edit', jsonParser, (request, response) => {
if (!request.body) {
return response.sendStatus(400);
}
if (!request.body.name) {
return response.status(400).send('World file must have a name');
}
try {
if (!('entries' in request.body.data)) {
throw new Error('World info must contain an entries list');
}
} catch (err) {
return response.status(400).send('Is not a valid world info file');
}
const filename = `${sanitize(request.body.name)}.json`;
const pathToFile = path.join(DIRECTORIES.worlds, filename);
writeFileAtomicSync(pathToFile, JSON.stringify(request.body.data, null, 4));
return response.send({ ok: true });
});
module.exports = { router, readWorldInfoFile };

32
src/util.js
View File

@ -7,6 +7,8 @@ const mime = require('mime-types');
const yaml = require('yaml');
const { default: simpleGit } = require('simple-git');
const { DIRECTORIES } = require('./constants');
/**
* Returns the config object from the config.yaml file.
* @returns {object} Config object
@ -307,6 +309,34 @@ function removeFileExtension(filename) {
return filename.replace(/\.[^.]+$/, '');
}
function generateTimestamp() {
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0');
const day = String(now.getDate()).padStart(2, '0');
const hours = String(now.getHours()).padStart(2, '0');
const minutes = String(now.getMinutes()).padStart(2, '0');
const seconds = String(now.getSeconds()).padStart(2, '0');
return `${year}${month}${day}-${hours}${minutes}${seconds}`;
}
/**
* @param {string} prefix
*/
function removeOldBackups(prefix) {
const MAX_BACKUPS = 25;
let files = fs.readdirSync(DIRECTORIES.backups).filter(f => f.startsWith(prefix));
if (files.length > MAX_BACKUPS) {
files = files.map(f => path.join(DIRECTORIES.backups, f));
files.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs);
fs.rmSync(files[0]);
}
}
module.exports = {
getConfig,
getConfigValue,
@ -323,4 +353,6 @@ module.exports = {
tryParse,
clientRelativePath,
removeFileExtension,
generateTimestamp,
removeOldBackups,
};

68
src/worldinfo.js
View File

@ -1,68 +0,0 @@
const path = require('path');
const fs = require('fs');
const { DIRECTORIES } = require('./constants');
function readWorldInfoFile(worldInfoName) {
const dummyObject = { entries: {} };
if (!worldInfoName) {
return dummyObject;
}
const filename = `${worldInfoName}.json`;
const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename);
if (!fs.existsSync(pathToWorldInfo)) {
console.log(`World info file ${filename} doesn't exist.`);
return dummyObject;
}
const worldInfoText = fs.readFileSync(pathToWorldInfo, 'utf8');
const worldInfo = JSON.parse(worldInfoText);
return worldInfo;
}
/**
* @param {string} name Name of World Info file
* @param {object} entries Entries object
*/
function convertWorldInfoToCharacterBook(name, entries) {
/** @type {{ entries: object[]; name: string }} */
const result = { entries: [], name };
for (const index in entries) {
const entry = entries[index];
const originalEntry = {
id: entry.uid,
keys: entry.key,
secondary_keys: entry.keysecondary,
comment: entry.comment,
content: entry.content,
constant: entry.constant,
selective: entry.selective,
insertion_order: entry.order,
enabled: !entry.disable,
position: entry.position == 0 ? 'before_char' : 'after_char',
extensions: {
position: entry.position,
exclude_recursion: entry.excludeRecursion,
display_index: entry.displayIndex,
probability: entry.probability ?? null,
useProbability: entry.useProbability ?? false,
depth: entry.depth ?? 4,
selectiveLogic: entry.selectiveLogic ?? 0,
group: entry.group ?? '',
},
};
result.entries.push(originalEntry);
}
return result;
}
module.exports = {
readWorldInfoFile,
convertWorldInfoToCharacterBook,
};