From a457484c2dd3ddb01fb9587b4ff4431cac331b1d Mon Sep 17 00:00:00 2001 From: valadaptive Date: Mon, 4 Dec 2023 07:56:42 -0500 Subject: [PATCH] Move character endpoints into their own module --- server.js | 985 +----------------------------------- src/constants.js | 5 + src/endpoints/characters.js | 910 +++++++++++++++++++++++++++++++++ src/util.js | 23 + src/worldinfo.js | 68 +++ 5 files changed, 1012 insertions(+), 979 deletions(-) create mode 100644 src/endpoints/characters.js create mode 100644 src/worldinfo.js diff --git a/server.js b/server.js index 80667fd1d..95b275676 100644 --- a/server.js +++ b/server.js @@ -35,14 +35,8 @@ const ipaddr = require('ipaddr.js'); const ipMatching = require('ip-matching'); // image processing related library imports -const encode = require('png-chunks-encode'); -const extract = require('png-chunks-extract'); const jimp = require('jimp'); const mime = require('mime-types'); -const PNGtext = require('png-chunk-text'); - -// misc/other imports -const _ = require('lodash'); // Unrestrict console logs display limit util.inspect.defaultOptions.maxArrayLength = null; @@ -51,14 +45,14 @@ util.inspect.defaultOptions.maxStringLength = null; // local library imports const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware'); const { jsonParser, urlencodedParser } = require('./src/express-common.js'); -const characterCardParser = require('./src/character-card-parser.js'); const contentManager = require('./src/endpoints/content-manager'); const statsHelpers = require('./statsHelpers.js'); const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets'); -const { delay, getVersion, deepMerge, getConfigValue, color, uuidv4 } = require('./src/util'); +const { delay, getVersion, getConfigValue, color, uuidv4, humanizedISO8601DateTime, tryParse } = require('./src/util'); const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails'); const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers'); const { convertClaudePrompt } = require('./src/chat-completion'); +const { readWorldInfoFile } = require('./src/worldinfo'); // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. // https://github.com/nodejs/node/issues/47822#issuecomment-1564708870 @@ -131,7 +125,6 @@ const listen = getConfigValue('listen', false); const API_OPENAI = 'https://api.openai.com/v1'; const API_CLAUDE = 'https://api.anthropic.com/v1'; -let characters = {}; let response_dw_bg; function getMancerHeaders() { @@ -198,24 +191,8 @@ function setAdditionalHeaders(request, args, server) { Object.assign(args.headers, headers); } -function humanizedISO8601DateTime(date) { - let baseDate = typeof date === 'number' ? new Date(date) : new Date(); - let humanYear = baseDate.getFullYear(); - let humanMonth = (baseDate.getMonth() + 1); - let humanDate = baseDate.getDate(); - let humanHour = (baseDate.getHours() < 10 ? '0' : '') + baseDate.getHours(); - let humanMinute = (baseDate.getMinutes() < 10 ? '0' : '') + baseDate.getMinutes(); - let humanSecond = (baseDate.getSeconds() < 10 ? '0' : '') + baseDate.getSeconds(); - let humanMillisecond = (baseDate.getMilliseconds() < 10 ? '0' : '') + baseDate.getMilliseconds(); - let HumanizedDateTime = (humanYear + '-' + humanMonth + '-' + humanDate + ' @' + humanHour + 'h ' + humanMinute + 'm ' + humanSecond + 's ' + humanMillisecond + 'ms'); - return HumanizedDateTime; -} - const SETTINGS_FILE = './public/settings.json'; -const AVATAR_WIDTH = 400; -const AVATAR_HEIGHT = 600; -const { DIRECTORIES, UPLOADS_PATH, PALM_SAFETY, TEXTGEN_TYPES, CHAT_COMPLETION_SOURCES } = require('./src/constants'); -const { TavernCardValidator } = require('./src/validator/TavernCardValidator'); +const { DIRECTORIES, UPLOADS_PATH, PALM_SAFETY, TEXTGEN_TYPES, CHAT_COMPLETION_SOURCES, AVATAR_WIDTH, AVATAR_HEIGHT } = require('./src/constants'); // CSRF Protection // if (cliArguments.disableCsrf === false) { @@ -853,201 +830,6 @@ app.post('/getstatus', jsonParser, async function (request, response) { }); -function tryParse(str) { - try { - return JSON.parse(str); - } catch { - return undefined; - } -} - -function convertToV2(char) { - // Simulate incoming data from frontend form - const result = charaFormatData({ - json_data: JSON.stringify(char), - ch_name: char.name, - description: char.description, - personality: char.personality, - scenario: char.scenario, - first_mes: char.first_mes, - mes_example: char.mes_example, - creator_notes: char.creatorcomment, - talkativeness: char.talkativeness, - fav: char.fav, - creator: char.creator, - tags: char.tags, - depth_prompt_prompt: char.depth_prompt_prompt, - depth_prompt_response: char.depth_prompt_response, - }); - - result.chat = char.chat ?? humanizedISO8601DateTime(); - result.create_date = char.create_date ?? humanizedISO8601DateTime(); - return result; -} - - -function unsetFavFlag(char) { - _.set(char, 'fav', false); - _.set(char, 'data.extensions.fav', false); -} - -function readFromV2(char) { - if (_.isUndefined(char.data)) { - console.warn(`Char ${char['name']} has Spec v2 data missing`); - return char; - } - - const fieldMappings = { - name: 'name', - description: 'description', - personality: 'personality', - scenario: 'scenario', - first_mes: 'first_mes', - mes_example: 'mes_example', - talkativeness: 'extensions.talkativeness', - fav: 'extensions.fav', - tags: 'tags', - }; - - _.forEach(fieldMappings, (v2Path, charField) => { - //console.log(`Migrating field: ${charField} from ${v2Path}`); - const v2Value = _.get(char.data, v2Path); - if (_.isUndefined(v2Value)) { - let defaultValue = undefined; - - // Backfill default values for missing ST extension fields - if (v2Path === 'extensions.talkativeness') { - defaultValue = 0.5; - } - - if (v2Path === 'extensions.fav') { - defaultValue = false; - } - - if (!_.isUndefined(defaultValue)) { - //console.debug(`Spec v2 extension data missing for field: ${charField}, using default value: ${defaultValue}`); - char[charField] = defaultValue; - } else { - console.debug(`Char ${char['name']} has Spec v2 data missing for unknown field: ${charField}`); - return; - } - } - if (!_.isUndefined(char[charField]) && !_.isUndefined(v2Value) && String(char[charField]) !== String(v2Value)) { - console.debug(`Char ${char['name']} has Spec v2 data mismatch with Spec v1 for field: ${charField}`, char[charField], v2Value); - } - char[charField] = v2Value; - }); - - char['chat'] = char['chat'] ?? humanizedISO8601DateTime(); - - return char; -} - -//***************** Main functions -function charaFormatData(data) { - // This is supposed to save all the foreign keys that ST doesn't care about - const char = tryParse(data.json_data) || {}; - - // Checks if data.alternate_greetings is an array, a string, or neither, and acts accordingly. (expected to be an array of strings) - const getAlternateGreetings = data => { - if (Array.isArray(data.alternate_greetings)) return data.alternate_greetings; - if (typeof data.alternate_greetings === 'string') return [data.alternate_greetings]; - return []; - }; - - // Spec V1 fields - _.set(char, 'name', data.ch_name); - _.set(char, 'description', data.description || ''); - _.set(char, 'personality', data.personality || ''); - _.set(char, 'scenario', data.scenario || ''); - _.set(char, 'first_mes', data.first_mes || ''); - _.set(char, 'mes_example', data.mes_example || ''); - - // Old ST extension fields (for backward compatibility, will be deprecated) - _.set(char, 'creatorcomment', data.creator_notes); - _.set(char, 'avatar', 'none'); - _.set(char, 'chat', data.ch_name + ' - ' + humanizedISO8601DateTime()); - _.set(char, 'talkativeness', data.talkativeness); - _.set(char, 'fav', data.fav == 'true'); - - // Spec V2 fields - _.set(char, 'spec', 'chara_card_v2'); - _.set(char, 'spec_version', '2.0'); - _.set(char, 'data.name', data.ch_name); - _.set(char, 'data.description', data.description || ''); - _.set(char, 'data.personality', data.personality || ''); - _.set(char, 'data.scenario', data.scenario || ''); - _.set(char, 'data.first_mes', data.first_mes || ''); - _.set(char, 'data.mes_example', data.mes_example || ''); - - // New V2 fields - _.set(char, 'data.creator_notes', data.creator_notes || ''); - _.set(char, 'data.system_prompt', data.system_prompt || ''); - _.set(char, 'data.post_history_instructions', data.post_history_instructions || ''); - _.set(char, 'data.tags', typeof data.tags == 'string' ? (data.tags.split(',').map(x => x.trim()).filter(x => x)) : data.tags || []); - _.set(char, 'data.creator', data.creator || ''); - _.set(char, 'data.character_version', data.character_version || ''); - _.set(char, 'data.alternate_greetings', getAlternateGreetings(data)); - - // ST extension fields to V2 object - _.set(char, 'data.extensions.talkativeness', data.talkativeness); - _.set(char, 'data.extensions.fav', data.fav == 'true'); - _.set(char, 'data.extensions.world', data.world || ''); - - // Spec extension: depth prompt - const depth_default = 4; - const depth_value = !isNaN(Number(data.depth_prompt_depth)) ? Number(data.depth_prompt_depth) : depth_default; - _.set(char, 'data.extensions.depth_prompt.prompt', data.depth_prompt_prompt ?? ''); - _.set(char, 'data.extensions.depth_prompt.depth', depth_value); - //_.set(char, 'data.extensions.create_date', humanizedISO8601DateTime()); - //_.set(char, 'data.extensions.avatar', 'none'); - //_.set(char, 'data.extensions.chat', data.ch_name + ' - ' + humanizedISO8601DateTime()); - - if (data.world) { - try { - const file = readWorldInfoFile(data.world); - - // File was imported - save it to the character book - if (file && file.originalData) { - _.set(char, 'data.character_book', file.originalData); - } - - // File was not imported - convert the world info to the character book - if (file && file.entries) { - _.set(char, 'data.character_book', convertWorldInfoToCharacterBook(data.world, file.entries)); - } - - } catch { - console.debug(`Failed to read world info file: ${data.world}. Character book will not be available.`); - } - } - - return char; -} - -app.post('/api/characters/create', urlencodedParser, async function (request, response) { - if (!request.body) return response.sendStatus(400); - - request.body.ch_name = sanitize(request.body.ch_name); - - const char = JSON.stringify(charaFormatData(request.body)); - const internalName = getPngName(request.body.ch_name); - const avatarName = `${internalName}.png`; - const defaultAvatar = './public/img/ai4.png'; - const chatsPath = DIRECTORIES.chats + internalName; //path.join(chatsPath, internalName); - - if (!fs.existsSync(chatsPath)) fs.mkdirSync(chatsPath); - - if (!request.file) { - charaWrite(defaultAvatar, char, internalName, response, avatarName); - } else { - const crop = tryParse(request.query.crop); - const uploadPath = path.join(UPLOADS_PATH, request.file.filename); - await charaWrite(uploadPath, char, internalName, response, avatarName, crop); - fs.unlinkSync(uploadPath); - } -}); - app.post('/renamechat', jsonParser, async function (request, response) { if (!request.body || !request.body.original_file || !request.body.renamed_file) { return response.sendStatus(400); @@ -1071,397 +853,6 @@ app.post('/renamechat', jsonParser, async function (request, response) { return response.send({ ok: true }); }); -app.post('/api/characters/rename', jsonParser, async function (request, response) { - if (!request.body.avatar_url || !request.body.new_name) { - return response.sendStatus(400); - } - - const oldAvatarName = request.body.avatar_url; - const newName = sanitize(request.body.new_name); - const oldInternalName = path.parse(request.body.avatar_url).name; - const newInternalName = getPngName(newName); - const newAvatarName = `${newInternalName}.png`; - - const oldAvatarPath = path.join(DIRECTORIES.characters, oldAvatarName); - - const oldChatsPath = path.join(DIRECTORIES.chats, oldInternalName); - const newChatsPath = path.join(DIRECTORIES.chats, newInternalName); - - try { - // Read old file, replace name int it - const rawOldData = await charaRead(oldAvatarPath); - if (rawOldData === undefined) throw new Error('Failed to read character file'); - - const oldData = getCharaCardV2(JSON.parse(rawOldData)); - _.set(oldData, 'data.name', newName); - _.set(oldData, 'name', newName); - const newData = JSON.stringify(oldData); - - // Write data to new location - await charaWrite(oldAvatarPath, newData, newInternalName); - - // Rename chats folder - if (fs.existsSync(oldChatsPath) && !fs.existsSync(newChatsPath)) { - fs.renameSync(oldChatsPath, newChatsPath); - } - - // Remove the old character file - fs.rmSync(oldAvatarPath); - - // Return new avatar name to ST - return response.send({ 'avatar': newAvatarName }); - } - catch (err) { - console.error(err); - return response.sendStatus(500); - } -}); - -app.post('/api/characters/edit', urlencodedParser, async function (request, response) { - if (!request.body) { - console.error('Error: no response body detected'); - response.status(400).send('Error: no response body detected'); - return; - } - - if (request.body.ch_name === '' || request.body.ch_name === undefined || request.body.ch_name === '.') { - console.error('Error: invalid name.'); - response.status(400).send('Error: invalid name.'); - return; - } - - let char = charaFormatData(request.body); - char.chat = request.body.chat; - char.create_date = request.body.create_date; - char = JSON.stringify(char); - let target_img = (request.body.avatar_url).replace('.png', ''); - - try { - if (!request.file) { - const avatarPath = path.join(DIRECTORIES.characters, request.body.avatar_url); - await charaWrite(avatarPath, char, target_img, response, 'Character saved'); - } else { - const crop = tryParse(request.query.crop); - const newAvatarPath = path.join(UPLOADS_PATH, request.file.filename); - invalidateThumbnail('avatar', request.body.avatar_url); - await charaWrite(newAvatarPath, char, target_img, response, 'Character saved', crop); - fs.unlinkSync(newAvatarPath); - } - } - catch { - console.error('An error occured, character edit invalidated.'); - } -}); - - -/** - * Handle a POST request to edit a character attribute. - * - * This function reads the character data from a file, updates the specified attribute, - * and writes the updated data back to the file. - * - * @param {Object} request - The HTTP request object. - * @param {Object} response - The HTTP response object. - * @returns {void} - */ -app.post('/api/characters/edit-attribute', jsonParser, async function (request, response) { - console.log(request.body); - if (!request.body) { - console.error('Error: no response body detected'); - response.status(400).send('Error: no response body detected'); - return; - } - - if (request.body.ch_name === '' || request.body.ch_name === undefined || request.body.ch_name === '.') { - console.error('Error: invalid name.'); - response.status(400).send('Error: invalid name.'); - return; - } - - try { - const avatarPath = path.join(DIRECTORIES.characters, request.body.avatar_url); - let charJSON = await charaRead(avatarPath); - if (typeof charJSON !== 'string') throw new Error('Failed to read character file'); - - let char = JSON.parse(charJSON); - //check if the field exists - if (char[request.body.field] === undefined && char.data[request.body.field] === undefined) { - console.error('Error: invalid field.'); - response.status(400).send('Error: invalid field.'); - return; - } - char[request.body.field] = request.body.value; - char.data[request.body.field] = request.body.value; - let newCharJSON = JSON.stringify(char); - await charaWrite(avatarPath, newCharJSON, (request.body.avatar_url).replace('.png', ''), response, 'Character saved'); - } catch (err) { - console.error('An error occured, character edit invalidated.', err); - } -}); - -/** - * Handle a POST request to edit character properties. - * - * Merges the request body with the selected character and - * validates the result against TavernCard V2 specification. - * - * @param {Object} request - The HTTP request object. - * @param {Object} response - The HTTP response object. - * - * @returns {void} - * */ -app.post('/api/characters/merge-attributes', jsonParser, async function (request, response) { - const update = request.body; - const avatarPath = path.join(DIRECTORIES.characters, update.avatar); - - try { - let character = JSON.parse(await charaRead(avatarPath)); - character = deepMerge(character, update); - - const validator = new TavernCardValidator(character); - - //Accept either V1 or V2. - if (validator.validate()) { - await charaWrite( - avatarPath, - JSON.stringify(character), - (update.avatar).replace('.png', ''), - response, - 'Character saved', - ); - } else { - console.log(validator.lastValidationError); - response.status(400).send({ message: `Validation failed for ${character.name}`, error: validator.lastValidationError }); - } - } catch (exception) { - response.status(500).send({ message: 'Unexpected error while saving character.', error: exception.toString() }); - } -}); - -app.post('/api/characters/delete', jsonParser, async function (request, response) { - if (!request.body || !request.body.avatar_url) { - return response.sendStatus(400); - } - - if (request.body.avatar_url !== sanitize(request.body.avatar_url)) { - console.error('Malicious filename prevented'); - return response.sendStatus(403); - } - - const avatarPath = DIRECTORIES.characters + request.body.avatar_url; - if (!fs.existsSync(avatarPath)) { - return response.sendStatus(400); - } - - fs.rmSync(avatarPath); - invalidateThumbnail('avatar', request.body.avatar_url); - let dir_name = (request.body.avatar_url.replace('.png', '')); - - if (!dir_name.length) { - console.error('Malicious dirname prevented'); - return response.sendStatus(403); - } - - if (request.body.delete_chats == true) { - try { - await fs.promises.rm(path.join(DIRECTORIES.chats, sanitize(dir_name)), { recursive: true, force: true }); - } catch (err) { - console.error(err); - return response.sendStatus(500); - } - } - - return response.sendStatus(200); -}); - -/** - * @param {express.Response | undefined} response - * @param {{file_name: string} | string} mes - */ -async function charaWrite(img_url, data, target_img, response = undefined, mes = 'ok', crop = undefined) { - try { - // Read the image, resize, and save it as a PNG into the buffer - const image = await tryReadImage(img_url, crop); - - // Get the chunks - const chunks = extract(image); - const tEXtChunks = chunks.filter(chunk => chunk.name === 'tEXt'); - - // Remove all existing tEXt chunks - for (let tEXtChunk of tEXtChunks) { - chunks.splice(chunks.indexOf(tEXtChunk), 1); - } - // Add new chunks before the IEND chunk - const base64EncodedData = Buffer.from(data, 'utf8').toString('base64'); - chunks.splice(-1, 0, PNGtext.encode('chara', base64EncodedData)); - //chunks.splice(-1, 0, text.encode('lorem', 'ipsum')); - - writeFileAtomicSync(DIRECTORIES.characters + target_img + '.png', Buffer.from(encode(chunks))); - if (response !== undefined) response.send(mes); - return true; - } catch (err) { - console.log(err); - if (response !== undefined) response.status(500).send(err); - return false; - } -} - -async function tryReadImage(img_url, crop) { - try { - let rawImg = await jimp.read(img_url); - let final_width = rawImg.bitmap.width, final_height = rawImg.bitmap.height; - - // Apply crop if defined - if (typeof crop == 'object' && [crop.x, crop.y, crop.width, crop.height].every(x => typeof x === 'number')) { - rawImg = rawImg.crop(crop.x, crop.y, crop.width, crop.height); - // Apply standard resize if requested - if (crop.want_resize) { - final_width = AVATAR_WIDTH; - final_height = AVATAR_HEIGHT; - } else { - final_width = crop.width; - final_height = crop.height; - } - } - - const image = await rawImg.cover(final_width, final_height).getBufferAsync(jimp.MIME_PNG); - return image; - } - // If it's an unsupported type of image (APNG) - just read the file as buffer - catch { - return fs.readFileSync(img_url); - } -} - -async function charaRead(img_url, input_format) { - return characterCardParser.parse(img_url, input_format); -} - -/** - * calculateChatSize - Calculates the total chat size for a given character. - * - * @param {string} charDir The directory where the chats are stored. - * @return { {chatSize: number, dateLastChat: number} } The total chat size. - */ -const calculateChatSize = (charDir) => { - let chatSize = 0; - let dateLastChat = 0; - - if (fs.existsSync(charDir)) { - const chats = fs.readdirSync(charDir); - if (Array.isArray(chats) && chats.length) { - for (const chat of chats) { - const chatStat = fs.statSync(path.join(charDir, chat)); - chatSize += chatStat.size; - dateLastChat = Math.max(dateLastChat, chatStat.mtimeMs); - } - } - } - - return { chatSize, dateLastChat }; -}; - -// Calculate the total string length of the data object -const calculateDataSize = (data) => { - return typeof data === 'object' ? Object.values(data).reduce((acc, val) => acc + new String(val).length, 0) : 0; -}; - -/** - * processCharacter - Process a given character, read its data and calculate its statistics. - * - * @param {string} item The name of the character. - * @param {number} i The index of the character in the characters list. - * @return {Promise} A Promise that resolves when the character processing is done. - */ -const processCharacter = async (item, i) => { - try { - const img_data = await charaRead(DIRECTORIES.characters + item); - if (img_data === undefined) throw new Error('Failed to read character file'); - - let jsonObject = getCharaCardV2(JSON.parse(img_data)); - jsonObject.avatar = item; - characters[i] = jsonObject; - characters[i]['json_data'] = img_data; - const charStat = fs.statSync(path.join(DIRECTORIES.characters, item)); - characters[i]['date_added'] = charStat.birthtimeMs; - characters[i]['create_date'] = jsonObject['create_date'] || humanizedISO8601DateTime(charStat.birthtimeMs); - const char_dir = path.join(DIRECTORIES.chats, item.replace('.png', '')); - - const { chatSize, dateLastChat } = calculateChatSize(char_dir); - characters[i]['chat_size'] = chatSize; - characters[i]['date_last_chat'] = dateLastChat; - characters[i]['data_size'] = calculateDataSize(jsonObject?.data); - } - catch (err) { - characters[i] = { - date_added: 0, - date_last_chat: 0, - chat_size: 0, - }; - - console.log(`Could not process character: ${item}`); - - if (err instanceof SyntaxError) { - console.log('String [' + i + '] is not valid JSON!'); - } else { - console.log('An unexpected error occurred: ', err); - } - } -}; - - -/** - * HTTP POST endpoint for the "/api/characters/all" route. - * - * This endpoint is responsible for reading character files from the `DIRECTORIES.characters` directory, - * parsing character data, calculating stats for each character and responding with the data. - * Stats are calculated only on the first run, on subsequent runs the stats are fetched from - * the `charStats` variable. - * The stats are calculated by the `calculateStats` function. - * The characters are processed by the `processCharacter` function. - * - * @param {object} request The HTTP request object. - * @param {object} response The HTTP response object. - * @return {undefined} Does not return a value. - */ -app.post('/api/characters/all', jsonParser, function (request, response) { - fs.readdir(DIRECTORIES.characters, async (err, files) => { - if (err) { - console.error(err); - return; - } - - const pngFiles = files.filter(file => file.endsWith('.png')); - characters = {}; - - let processingPromises = pngFiles.map((file, index) => processCharacter(file, index)); - await Promise.all(processingPromises); performance.mark('B'); - - // Filter out invalid/broken characters - characters = Object.values(characters).filter(x => x?.name).reduce((acc, val, index) => { - acc[index] = val; - return acc; - }, {}); - - response.send(JSON.stringify(characters)); - }); -}); - -app.post('/api/characters/get', jsonParser, async function (request, response) { - if (!request.body) return response.sendStatus(400); - const item = request.body.avatar_url; - const filePath = path.join(DIRECTORIES.characters, item); - - if (!fs.existsSync(filePath)) { - return response.sendStatus(404); - } - - characters = {}; - await processCharacter(item, 0); - - return response.send(characters[0]); -}); - /** * Handle a POST request to get the stats object * @@ -1655,15 +1046,6 @@ app.post('/savesettings', jsonParser, function (request, response) { } }); -function getCharaCardV2(jsonObject) { - if (jsonObject.spec === undefined) { - jsonObject = convertToV2(jsonObject); - } else { - jsonObject = readFromV2(jsonObject); - } - return jsonObject; -} - function readAndParseFromDirectory(directoryPath, fileExtension = '.json') { const files = fs .readdirSync(directoryPath) @@ -1843,66 +1225,6 @@ app.post('/savequickreply', jsonParser, (request, response) => { return response.sendStatus(200); }); -/** - * @param {string} name Name of World Info file - * @param {object} entries Entries object - */ -function convertWorldInfoToCharacterBook(name, entries) { - /** @type {{ entries: object[]; name: string }} */ - const result = { entries: [], name }; - - for (const index in entries) { - const entry = entries[index]; - - const originalEntry = { - id: entry.uid, - keys: entry.key, - secondary_keys: entry.keysecondary, - comment: entry.comment, - content: entry.content, - constant: entry.constant, - selective: entry.selective, - insertion_order: entry.order, - enabled: !entry.disable, - position: entry.position == 0 ? 'before_char' : 'after_char', - extensions: { - position: entry.position, - exclude_recursion: entry.excludeRecursion, - display_index: entry.displayIndex, - probability: entry.probability ?? null, - useProbability: entry.useProbability ?? false, - depth: entry.depth ?? 4, - selectiveLogic: entry.selectiveLogic ?? 0, - group: entry.group ?? '', - }, - }; - - result.entries.push(originalEntry); - } - - return result; -} - -function readWorldInfoFile(worldInfoName) { - const dummyObject = { entries: {} }; - - if (!worldInfoName) { - return dummyObject; - } - - const filename = `${worldInfoName}.json`; - const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename); - - if (!fs.existsSync(pathToWorldInfo)) { - console.log(`World info file ${filename} doesn't exist.`); - return dummyObject; - } - - const worldInfoText = fs.readFileSync(pathToWorldInfo, 'utf8'); - const worldInfo = JSON.parse(worldInfoText); - return worldInfo; -} - function getImages(path) { return fs @@ -1914,274 +1236,6 @@ function getImages(path) { .sort(Intl.Collator().compare); } -app.post('/api/characters/chats', jsonParser, async function (request, response) { - if (!request.body) return response.sendStatus(400); - - const characterDirectory = (request.body.avatar_url).replace('.png', ''); - - try { - const chatsDirectory = path.join(DIRECTORIES.chats, characterDirectory); - const files = fs.readdirSync(chatsDirectory); - const jsonFiles = files.filter(file => path.extname(file) === '.jsonl'); - - if (jsonFiles.length === 0) { - response.send({ error: true }); - return; - } - - const jsonFilesPromise = jsonFiles.map((file) => { - return new Promise(async (res) => { - const pathToFile = path.join(DIRECTORIES.chats, characterDirectory, file); - const fileStream = fs.createReadStream(pathToFile); - const stats = fs.statSync(pathToFile); - const fileSizeInKB = `${(stats.size / 1024).toFixed(2)}kb`; - - const rl = readline.createInterface({ - input: fileStream, - crlfDelay: Infinity, - }); - - let lastLine; - let itemCounter = 0; - rl.on('line', (line) => { - itemCounter++; - lastLine = line; - }); - rl.on('close', () => { - rl.close(); - - if (lastLine) { - const jsonData = tryParse(lastLine); - if (jsonData && (jsonData.name || jsonData.character_name)) { - const chatData = {}; - - chatData['file_name'] = file; - chatData['file_size'] = fileSizeInKB; - chatData['chat_items'] = itemCounter - 1; - chatData['mes'] = jsonData['mes'] || '[The chat is empty]'; - chatData['last_mes'] = jsonData['send_date'] || Date.now(); - - res(chatData); - } else { - console.log('Found an invalid or corrupted chat file:', pathToFile); - res({}); - } - } - }); - }); - }); - - const chatData = await Promise.all(jsonFilesPromise); - const validFiles = chatData.filter(i => i.file_name); - - return response.send(validFiles); - } catch (error) { - console.log(error); - return response.send({ error: true }); - } -}); - -function getPngName(file) { - let i = 1; - let base_name = file; - while (fs.existsSync(DIRECTORIES.characters + file + '.png')) { - file = base_name + i; - i++; - } - return file; -} - -app.post('/api/characters/import', urlencodedParser, async function (request, response) { - - if (!request.body || request.file === undefined) return response.sendStatus(400); - - let png_name = ''; - let filedata = request.file; - let uploadPath = path.join(UPLOADS_PATH, filedata.filename); - var format = request.body.file_type; - const defaultAvatarPath = './public/img/ai4.png'; - const { importRisuSprites } = require('./src/endpoints/sprites'); - //console.log(format); - if (filedata) { - if (format == 'json') { - fs.readFile(uploadPath, 'utf8', async (err, data) => { - fs.unlinkSync(uploadPath); - - if (err) { - console.log(err); - response.send({ error: true }); - } - - let jsonData = JSON.parse(data); - - if (jsonData.spec !== undefined) { - console.log('importing from v2 json'); - importRisuSprites(jsonData); - unsetFavFlag(jsonData); - jsonData = readFromV2(jsonData); - jsonData['create_date'] = humanizedISO8601DateTime(); - png_name = getPngName(jsonData.data?.name || jsonData.name); - let char = JSON.stringify(jsonData); - charaWrite(defaultAvatarPath, char, png_name, response, { file_name: png_name }); - } else if (jsonData.name !== undefined) { - console.log('importing from v1 json'); - jsonData.name = sanitize(jsonData.name); - if (jsonData.creator_notes) { - jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', ''); - } - png_name = getPngName(jsonData.name); - let char = { - 'name': jsonData.name, - 'description': jsonData.description ?? '', - 'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '', - 'personality': jsonData.personality ?? '', - 'first_mes': jsonData.first_mes ?? '', - 'avatar': 'none', - 'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(), - 'mes_example': jsonData.mes_example ?? '', - 'scenario': jsonData.scenario ?? '', - 'create_date': humanizedISO8601DateTime(), - 'talkativeness': jsonData.talkativeness ?? 0.5, - 'creator': jsonData.creator ?? '', - 'tags': jsonData.tags ?? '', - }; - char = convertToV2(char); - let charJSON = JSON.stringify(char); - charaWrite(defaultAvatarPath, charJSON, png_name, response, { file_name: png_name }); - } else if (jsonData.char_name !== undefined) {//json Pygmalion notepad - console.log('importing from gradio json'); - jsonData.char_name = sanitize(jsonData.char_name); - if (jsonData.creator_notes) { - jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', ''); - } - png_name = getPngName(jsonData.char_name); - let char = { - 'name': jsonData.char_name, - 'description': jsonData.char_persona ?? '', - 'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '', - 'personality': '', - 'first_mes': jsonData.char_greeting ?? '', - 'avatar': 'none', - 'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(), - 'mes_example': jsonData.example_dialogue ?? '', - 'scenario': jsonData.world_scenario ?? '', - 'create_date': humanizedISO8601DateTime(), - 'talkativeness': jsonData.talkativeness ?? 0.5, - 'creator': jsonData.creator ?? '', - 'tags': jsonData.tags ?? '', - }; - char = convertToV2(char); - let charJSON = JSON.stringify(char); - charaWrite(defaultAvatarPath, charJSON, png_name, response, { file_name: png_name }); - } else { - console.log('Incorrect character format .json'); - response.send({ error: true }); - } - }); - } else { - try { - var img_data = await charaRead(uploadPath, format); - if (img_data === undefined) throw new Error('Failed to read character data'); - - let jsonData = JSON.parse(img_data); - - jsonData.name = sanitize(jsonData.data?.name || jsonData.name); - png_name = getPngName(jsonData.name); - - if (jsonData.spec !== undefined) { - console.log('Found a v2 character file.'); - importRisuSprites(jsonData); - unsetFavFlag(jsonData); - jsonData = readFromV2(jsonData); - jsonData['create_date'] = humanizedISO8601DateTime(); - const char = JSON.stringify(jsonData); - await charaWrite(uploadPath, char, png_name, response, { file_name: png_name }); - fs.unlinkSync(uploadPath); - } else if (jsonData.name !== undefined) { - console.log('Found a v1 character file.'); - - if (jsonData.creator_notes) { - jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', ''); - } - - let char = { - 'name': jsonData.name, - 'description': jsonData.description ?? '', - 'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '', - 'personality': jsonData.personality ?? '', - 'first_mes': jsonData.first_mes ?? '', - 'avatar': 'none', - 'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(), - 'mes_example': jsonData.mes_example ?? '', - 'scenario': jsonData.scenario ?? '', - 'create_date': humanizedISO8601DateTime(), - 'talkativeness': jsonData.talkativeness ?? 0.5, - 'creator': jsonData.creator ?? '', - 'tags': jsonData.tags ?? '', - }; - char = convertToV2(char); - const charJSON = JSON.stringify(char); - await charaWrite(uploadPath, charJSON, png_name, response, { file_name: png_name }); - fs.unlinkSync(uploadPath); - } else { - console.log('Unknown character card format'); - response.send({ error: true }); - } - } catch (err) { - console.log(err); - response.send({ error: true }); - } - } - } -}); - -app.post('/api/characters/duplicate', jsonParser, async function (request, response) { - try { - if (!request.body.avatar_url) { - console.log('avatar URL not found in request body'); - console.log(request.body); - return response.sendStatus(400); - } - let filename = path.join(DIRECTORIES.characters, sanitize(request.body.avatar_url)); - if (!fs.existsSync(filename)) { - console.log('file for dupe not found'); - console.log(filename); - return response.sendStatus(404); - } - let suffix = 1; - let newFilename = filename; - - // If filename ends with a _number, increment the number - const nameParts = path.basename(filename, path.extname(filename)).split('_'); - const lastPart = nameParts[nameParts.length - 1]; - - let baseName; - - if (!isNaN(Number(lastPart)) && nameParts.length > 1) { - suffix = parseInt(lastPart) + 1; - baseName = nameParts.slice(0, -1).join('_'); // construct baseName without suffix - } else { - baseName = nameParts.join('_'); // original filename is completely the baseName - } - - newFilename = path.join(DIRECTORIES.characters, `${baseName}_${suffix}${path.extname(filename)}`); - - while (fs.existsSync(newFilename)) { - let suffixStr = '_' + suffix; - newFilename = path.join(DIRECTORIES.characters, `${baseName}${suffixStr}${path.extname(filename)}`); - suffix++; - } - - fs.copyFileSync(filename, newFilename); - console.log(`${filename} was copied to ${newFilename}`); - response.sendStatus(200); - } - catch (error) { - console.error(error); - return response.send({ error: true }); - } -}); - app.post('/exportchat', jsonParser, async function (request, response) { if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) { return response.sendStatus(400); @@ -2250,36 +1304,6 @@ app.post('/exportchat', jsonParser, async function (request, response) { } }); -app.post('/api/characters/export', jsonParser, async function (request, response) { - if (!request.body.format || !request.body.avatar_url) { - return response.sendStatus(400); - } - - let filename = path.join(DIRECTORIES.characters, sanitize(request.body.avatar_url)); - - if (!fs.existsSync(filename)) { - return response.sendStatus(404); - } - - switch (request.body.format) { - case 'png': - return response.sendFile(filename, { root: process.cwd() }); - case 'json': { - try { - let json = await charaRead(filename); - if (json === undefined) return response.sendStatus(400); - let jsonObject = getCharaCardV2(JSON.parse(json)); - return response.type('json').send(jsonObject); - } - catch { - return response.sendStatus(400); - } - } - } - - return response.sendStatus(400); -}); - app.post('/importgroupchat', urlencodedParser, function (request, response) { try { const filedata = request.file; @@ -3600,6 +2624,9 @@ app.use('/api/assets', require('./src/endpoints/assets').router); // File management app.use('/api/files', require('./src/endpoints/files').router); +// Character management +app.use('/api/characters', require('./src/endpoints/characters').router); + // Character sprite management app.use('/api/sprites', require('./src/endpoints/sprites').router); diff --git a/src/constants.js b/src/constants.js index eac932d69..32ea6fad5 100644 --- a/src/constants.js +++ b/src/constants.js @@ -153,6 +153,9 @@ const TEXTGEN_TYPES = { KOBOLDCPP: 'koboldcpp', }; +const AVATAR_WIDTH = 400; +const AVATAR_HEIGHT = 600; + module.exports = { DIRECTORIES, UNSAFE_EXTENSIONS, @@ -160,4 +163,6 @@ module.exports = { PALM_SAFETY, TEXTGEN_TYPES, CHAT_COMPLETION_SOURCES, + AVATAR_WIDTH, + AVATAR_HEIGHT, }; diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js new file mode 100644 index 000000000..b67aeb6cb --- /dev/null +++ b/src/endpoints/characters.js @@ -0,0 +1,910 @@ +const path = require('path'); +const fs = require('fs'); +const readline = require('readline'); +const express = require('express'); +const sanitize = require('sanitize-filename'); +const writeFileAtomicSync = require('write-file-atomic').sync; +const _ = require('lodash'); + +const encode = require('png-chunks-encode'); +const extract = require('png-chunks-extract'); +const PNGtext = require('png-chunk-text'); +const jimp = require('jimp'); + +const { DIRECTORIES, UPLOADS_PATH, AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants'); +const { jsonParser, urlencodedParser } = require('../express-common'); +const { deepMerge, humanizedISO8601DateTime, tryParse } = require('../util'); +const { TavernCardValidator } = require('../validator/TavernCardValidator'); +const characterCardParser = require('../character-card-parser.js'); +const { readWorldInfoFile, convertWorldInfoToCharacterBook } = require('../worldinfo'); +const { invalidateThumbnail } = require('./thumbnails'); + +let characters = {}; + +async function charaRead(img_url, input_format) { + return characterCardParser.parse(img_url, input_format); +} + +/** + * @param {express.Response | undefined} response + * @param {{file_name: string} | string} mes + */ +async function charaWrite(img_url, data, target_img, response = undefined, mes = 'ok', crop = undefined) { + try { + // Read the image, resize, and save it as a PNG into the buffer + const image = await tryReadImage(img_url, crop); + + // Get the chunks + const chunks = extract(image); + const tEXtChunks = chunks.filter(chunk => chunk.name === 'tEXt'); + + // Remove all existing tEXt chunks + for (let tEXtChunk of tEXtChunks) { + chunks.splice(chunks.indexOf(tEXtChunk), 1); + } + // Add new chunks before the IEND chunk + const base64EncodedData = Buffer.from(data, 'utf8').toString('base64'); + chunks.splice(-1, 0, PNGtext.encode('chara', base64EncodedData)); + //chunks.splice(-1, 0, text.encode('lorem', 'ipsum')); + + writeFileAtomicSync(DIRECTORIES.characters + target_img + '.png', Buffer.from(encode(chunks))); + if (response !== undefined) response.send(mes); + return true; + } catch (err) { + console.log(err); + if (response !== undefined) response.status(500).send(err); + return false; + } +} + +async function tryReadImage(img_url, crop) { + try { + let rawImg = await jimp.read(img_url); + let final_width = rawImg.bitmap.width, final_height = rawImg.bitmap.height; + + // Apply crop if defined + if (typeof crop == 'object' && [crop.x, crop.y, crop.width, crop.height].every(x => typeof x === 'number')) { + rawImg = rawImg.crop(crop.x, crop.y, crop.width, crop.height); + // Apply standard resize if requested + if (crop.want_resize) { + final_width = AVATAR_WIDTH; + final_height = AVATAR_HEIGHT; + } else { + final_width = crop.width; + final_height = crop.height; + } + } + + const image = await rawImg.cover(final_width, final_height).getBufferAsync(jimp.MIME_PNG); + return image; + } + // If it's an unsupported type of image (APNG) - just read the file as buffer + catch { + return fs.readFileSync(img_url); + } +} + +/** + * calculateChatSize - Calculates the total chat size for a given character. + * + * @param {string} charDir The directory where the chats are stored. + * @return { {chatSize: number, dateLastChat: number} } The total chat size. + */ +const calculateChatSize = (charDir) => { + let chatSize = 0; + let dateLastChat = 0; + + if (fs.existsSync(charDir)) { + const chats = fs.readdirSync(charDir); + if (Array.isArray(chats) && chats.length) { + for (const chat of chats) { + const chatStat = fs.statSync(path.join(charDir, chat)); + chatSize += chatStat.size; + dateLastChat = Math.max(dateLastChat, chatStat.mtimeMs); + } + } + } + + return { chatSize, dateLastChat }; +}; + +// Calculate the total string length of the data object +const calculateDataSize = (data) => { + return typeof data === 'object' ? Object.values(data).reduce((acc, val) => acc + new String(val).length, 0) : 0; +}; + +/** + * processCharacter - Process a given character, read its data and calculate its statistics. + * + * @param {string} item The name of the character. + * @param {number} i The index of the character in the characters list. + * @return {Promise} A Promise that resolves when the character processing is done. + */ +const processCharacter = async (item, i) => { + try { + const img_data = await charaRead(DIRECTORIES.characters + item); + if (img_data === undefined) throw new Error('Failed to read character file'); + + let jsonObject = getCharaCardV2(JSON.parse(img_data)); + jsonObject.avatar = item; + characters[i] = jsonObject; + characters[i]['json_data'] = img_data; + const charStat = fs.statSync(path.join(DIRECTORIES.characters, item)); + characters[i]['date_added'] = charStat.birthtimeMs; + characters[i]['create_date'] = jsonObject['create_date'] || humanizedISO8601DateTime(charStat.birthtimeMs); + const char_dir = path.join(DIRECTORIES.chats, item.replace('.png', '')); + + const { chatSize, dateLastChat } = calculateChatSize(char_dir); + characters[i]['chat_size'] = chatSize; + characters[i]['date_last_chat'] = dateLastChat; + characters[i]['data_size'] = calculateDataSize(jsonObject?.data); + } + catch (err) { + characters[i] = { + date_added: 0, + date_last_chat: 0, + chat_size: 0, + }; + + console.log(`Could not process character: ${item}`); + + if (err instanceof SyntaxError) { + console.log('String [' + i + '] is not valid JSON!'); + } else { + console.log('An unexpected error occurred: ', err); + } + } +}; + +function getCharaCardV2(jsonObject) { + if (jsonObject.spec === undefined) { + jsonObject = convertToV2(jsonObject); + } else { + jsonObject = readFromV2(jsonObject); + } + return jsonObject; +} + +function convertToV2(char) { + // Simulate incoming data from frontend form + const result = charaFormatData({ + json_data: JSON.stringify(char), + ch_name: char.name, + description: char.description, + personality: char.personality, + scenario: char.scenario, + first_mes: char.first_mes, + mes_example: char.mes_example, + creator_notes: char.creatorcomment, + talkativeness: char.talkativeness, + fav: char.fav, + creator: char.creator, + tags: char.tags, + depth_prompt_prompt: char.depth_prompt_prompt, + depth_prompt_response: char.depth_prompt_response, + }); + + result.chat = char.chat ?? humanizedISO8601DateTime(); + result.create_date = char.create_date ?? humanizedISO8601DateTime(); + return result; +} + + +function unsetFavFlag(char) { + _.set(char, 'fav', false); + _.set(char, 'data.extensions.fav', false); +} + +function readFromV2(char) { + if (_.isUndefined(char.data)) { + console.warn(`Char ${char['name']} has Spec v2 data missing`); + return char; + } + + const fieldMappings = { + name: 'name', + description: 'description', + personality: 'personality', + scenario: 'scenario', + first_mes: 'first_mes', + mes_example: 'mes_example', + talkativeness: 'extensions.talkativeness', + fav: 'extensions.fav', + tags: 'tags', + }; + + _.forEach(fieldMappings, (v2Path, charField) => { + //console.log(`Migrating field: ${charField} from ${v2Path}`); + const v2Value = _.get(char.data, v2Path); + if (_.isUndefined(v2Value)) { + let defaultValue = undefined; + + // Backfill default values for missing ST extension fields + if (v2Path === 'extensions.talkativeness') { + defaultValue = 0.5; + } + + if (v2Path === 'extensions.fav') { + defaultValue = false; + } + + if (!_.isUndefined(defaultValue)) { + //console.debug(`Spec v2 extension data missing for field: ${charField}, using default value: ${defaultValue}`); + char[charField] = defaultValue; + } else { + console.debug(`Char ${char['name']} has Spec v2 data missing for unknown field: ${charField}`); + return; + } + } + if (!_.isUndefined(char[charField]) && !_.isUndefined(v2Value) && String(char[charField]) !== String(v2Value)) { + console.debug(`Char ${char['name']} has Spec v2 data mismatch with Spec v1 for field: ${charField}`, char[charField], v2Value); + } + char[charField] = v2Value; + }); + + char['chat'] = char['chat'] ?? humanizedISO8601DateTime(); + + return char; +} + +//***************** Main functions +function charaFormatData(data) { + // This is supposed to save all the foreign keys that ST doesn't care about + const char = tryParse(data.json_data) || {}; + + // Checks if data.alternate_greetings is an array, a string, or neither, and acts accordingly. (expected to be an array of strings) + const getAlternateGreetings = data => { + if (Array.isArray(data.alternate_greetings)) return data.alternate_greetings; + if (typeof data.alternate_greetings === 'string') return [data.alternate_greetings]; + return []; + }; + + // Spec V1 fields + _.set(char, 'name', data.ch_name); + _.set(char, 'description', data.description || ''); + _.set(char, 'personality', data.personality || ''); + _.set(char, 'scenario', data.scenario || ''); + _.set(char, 'first_mes', data.first_mes || ''); + _.set(char, 'mes_example', data.mes_example || ''); + + // Old ST extension fields (for backward compatibility, will be deprecated) + _.set(char, 'creatorcomment', data.creator_notes); + _.set(char, 'avatar', 'none'); + _.set(char, 'chat', data.ch_name + ' - ' + humanizedISO8601DateTime()); + _.set(char, 'talkativeness', data.talkativeness); + _.set(char, 'fav', data.fav == 'true'); + + // Spec V2 fields + _.set(char, 'spec', 'chara_card_v2'); + _.set(char, 'spec_version', '2.0'); + _.set(char, 'data.name', data.ch_name); + _.set(char, 'data.description', data.description || ''); + _.set(char, 'data.personality', data.personality || ''); + _.set(char, 'data.scenario', data.scenario || ''); + _.set(char, 'data.first_mes', data.first_mes || ''); + _.set(char, 'data.mes_example', data.mes_example || ''); + + // New V2 fields + _.set(char, 'data.creator_notes', data.creator_notes || ''); + _.set(char, 'data.system_prompt', data.system_prompt || ''); + _.set(char, 'data.post_history_instructions', data.post_history_instructions || ''); + _.set(char, 'data.tags', typeof data.tags == 'string' ? (data.tags.split(',').map(x => x.trim()).filter(x => x)) : data.tags || []); + _.set(char, 'data.creator', data.creator || ''); + _.set(char, 'data.character_version', data.character_version || ''); + _.set(char, 'data.alternate_greetings', getAlternateGreetings(data)); + + // ST extension fields to V2 object + _.set(char, 'data.extensions.talkativeness', data.talkativeness); + _.set(char, 'data.extensions.fav', data.fav == 'true'); + _.set(char, 'data.extensions.world', data.world || ''); + + // Spec extension: depth prompt + const depth_default = 4; + const depth_value = !isNaN(Number(data.depth_prompt_depth)) ? Number(data.depth_prompt_depth) : depth_default; + _.set(char, 'data.extensions.depth_prompt.prompt', data.depth_prompt_prompt ?? ''); + _.set(char, 'data.extensions.depth_prompt.depth', depth_value); + //_.set(char, 'data.extensions.create_date', humanizedISO8601DateTime()); + //_.set(char, 'data.extensions.avatar', 'none'); + //_.set(char, 'data.extensions.chat', data.ch_name + ' - ' + humanizedISO8601DateTime()); + + if (data.world) { + try { + const file = readWorldInfoFile(data.world); + + // File was imported - save it to the character book + if (file && file.originalData) { + _.set(char, 'data.character_book', file.originalData); + } + + // File was not imported - convert the world info to the character book + if (file && file.entries) { + _.set(char, 'data.character_book', convertWorldInfoToCharacterBook(data.world, file.entries)); + } + + } catch { + console.debug(`Failed to read world info file: ${data.world}. Character book will not be available.`); + } + } + + return char; +} + +const router = express.Router(); + +router.post('/create', urlencodedParser, async function (request, response) { + if (!request.body) return response.sendStatus(400); + + request.body.ch_name = sanitize(request.body.ch_name); + + const char = JSON.stringify(charaFormatData(request.body)); + const internalName = getPngName(request.body.ch_name); + const avatarName = `${internalName}.png`; + const defaultAvatar = './public/img/ai4.png'; + const chatsPath = DIRECTORIES.chats + internalName; //path.join(chatsPath, internalName); + + if (!fs.existsSync(chatsPath)) fs.mkdirSync(chatsPath); + + if (!request.file) { + charaWrite(defaultAvatar, char, internalName, response, avatarName); + } else { + const crop = tryParse(request.query.crop); + const uploadPath = path.join(UPLOADS_PATH, request.file.filename); + await charaWrite(uploadPath, char, internalName, response, avatarName, crop); + fs.unlinkSync(uploadPath); + } +}); + +router.post('/rename', jsonParser, async function (request, response) { + if (!request.body.avatar_url || !request.body.new_name) { + return response.sendStatus(400); + } + + const oldAvatarName = request.body.avatar_url; + const newName = sanitize(request.body.new_name); + const oldInternalName = path.parse(request.body.avatar_url).name; + const newInternalName = getPngName(newName); + const newAvatarName = `${newInternalName}.png`; + + const oldAvatarPath = path.join(DIRECTORIES.characters, oldAvatarName); + + const oldChatsPath = path.join(DIRECTORIES.chats, oldInternalName); + const newChatsPath = path.join(DIRECTORIES.chats, newInternalName); + + try { + // Read old file, replace name int it + const rawOldData = await charaRead(oldAvatarPath); + if (rawOldData === undefined) throw new Error('Failed to read character file'); + + const oldData = getCharaCardV2(JSON.parse(rawOldData)); + _.set(oldData, 'data.name', newName); + _.set(oldData, 'name', newName); + const newData = JSON.stringify(oldData); + + // Write data to new location + await charaWrite(oldAvatarPath, newData, newInternalName); + + // Rename chats folder + if (fs.existsSync(oldChatsPath) && !fs.existsSync(newChatsPath)) { + fs.renameSync(oldChatsPath, newChatsPath); + } + + // Remove the old character file + fs.rmSync(oldAvatarPath); + + // Return new avatar name to ST + return response.send({ 'avatar': newAvatarName }); + } + catch (err) { + console.error(err); + return response.sendStatus(500); + } +}); + +router.post('/edit', urlencodedParser, async function (request, response) { + if (!request.body) { + console.error('Error: no response body detected'); + response.status(400).send('Error: no response body detected'); + return; + } + + if (request.body.ch_name === '' || request.body.ch_name === undefined || request.body.ch_name === '.') { + console.error('Error: invalid name.'); + response.status(400).send('Error: invalid name.'); + return; + } + + let char = charaFormatData(request.body); + char.chat = request.body.chat; + char.create_date = request.body.create_date; + char = JSON.stringify(char); + let target_img = (request.body.avatar_url).replace('.png', ''); + + try { + if (!request.file) { + const avatarPath = path.join(DIRECTORIES.characters, request.body.avatar_url); + await charaWrite(avatarPath, char, target_img, response, 'Character saved'); + } else { + const crop = tryParse(request.query.crop); + const newAvatarPath = path.join(UPLOADS_PATH, request.file.filename); + invalidateThumbnail('avatar', request.body.avatar_url); + await charaWrite(newAvatarPath, char, target_img, response, 'Character saved', crop); + fs.unlinkSync(newAvatarPath); + } + } + catch { + console.error('An error occured, character edit invalidated.'); + } +}); + + +/** + * Handle a POST request to edit a character attribute. + * + * This function reads the character data from a file, updates the specified attribute, + * and writes the updated data back to the file. + * + * @param {Object} request - The HTTP request object. + * @param {Object} response - The HTTP response object. + * @returns {void} + */ +router.post('/edit-attribute', jsonParser, async function (request, response) { + console.log(request.body); + if (!request.body) { + console.error('Error: no response body detected'); + response.status(400).send('Error: no response body detected'); + return; + } + + if (request.body.ch_name === '' || request.body.ch_name === undefined || request.body.ch_name === '.') { + console.error('Error: invalid name.'); + response.status(400).send('Error: invalid name.'); + return; + } + + try { + const avatarPath = path.join(DIRECTORIES.characters, request.body.avatar_url); + let charJSON = await charaRead(avatarPath); + if (typeof charJSON !== 'string') throw new Error('Failed to read character file'); + + let char = JSON.parse(charJSON); + //check if the field exists + if (char[request.body.field] === undefined && char.data[request.body.field] === undefined) { + console.error('Error: invalid field.'); + response.status(400).send('Error: invalid field.'); + return; + } + char[request.body.field] = request.body.value; + char.data[request.body.field] = request.body.value; + let newCharJSON = JSON.stringify(char); + await charaWrite(avatarPath, newCharJSON, (request.body.avatar_url).replace('.png', ''), response, 'Character saved'); + } catch (err) { + console.error('An error occured, character edit invalidated.', err); + } +}); + +/** + * Handle a POST request to edit character properties. + * + * Merges the request body with the selected character and + * validates the result against TavernCard V2 specification. + * + * @param {Object} request - The HTTP request object. + * @param {Object} response - The HTTP response object. + * + * @returns {void} + * */ +router.post('/merge-attributes', jsonParser, async function (request, response) { + const update = request.body; + const avatarPath = path.join(DIRECTORIES.characters, update.avatar); + + try { + let character = JSON.parse(await charaRead(avatarPath)); + character = deepMerge(character, update); + + const validator = new TavernCardValidator(character); + + //Accept either V1 or V2. + if (validator.validate()) { + await charaWrite( + avatarPath, + JSON.stringify(character), + (update.avatar).replace('.png', ''), + response, + 'Character saved', + ); + } else { + console.log(validator.lastValidationError); + response.status(400).send({ message: `Validation failed for ${character.name}`, error: validator.lastValidationError }); + } + } catch (exception) { + response.status(500).send({ message: 'Unexpected error while saving character.', error: exception.toString() }); + } +}); + +router.post('/delete', jsonParser, async function (request, response) { + if (!request.body || !request.body.avatar_url) { + return response.sendStatus(400); + } + + if (request.body.avatar_url !== sanitize(request.body.avatar_url)) { + console.error('Malicious filename prevented'); + return response.sendStatus(403); + } + + const avatarPath = DIRECTORIES.characters + request.body.avatar_url; + if (!fs.existsSync(avatarPath)) { + return response.sendStatus(400); + } + + fs.rmSync(avatarPath); + invalidateThumbnail('avatar', request.body.avatar_url); + let dir_name = (request.body.avatar_url.replace('.png', '')); + + if (!dir_name.length) { + console.error('Malicious dirname prevented'); + return response.sendStatus(403); + } + + if (request.body.delete_chats == true) { + try { + await fs.promises.rm(path.join(DIRECTORIES.chats, sanitize(dir_name)), { recursive: true, force: true }); + } catch (err) { + console.error(err); + return response.sendStatus(500); + } + } + + return response.sendStatus(200); +}); + +/** + * HTTP POST endpoint for the "/api/characters/all" route. + * + * This endpoint is responsible for reading character files from the `charactersPath` directory, + * parsing character data, calculating stats for each character and responding with the data. + * Stats are calculated only on the first run, on subsequent runs the stats are fetched from + * the `charStats` variable. + * The stats are calculated by the `calculateStats` function. + * The characters are processed by the `processCharacter` function. + * + * @param {object} request The HTTP request object. + * @param {object} response The HTTP response object. + * @return {undefined} Does not return a value. + */ +router.post('/all', jsonParser, function (request, response) { + fs.readdir(DIRECTORIES.characters, async (err, files) => { + if (err) { + console.error(err); + return; + } + + const pngFiles = files.filter(file => file.endsWith('.png')); + characters = {}; + + let processingPromises = pngFiles.map((file, index) => processCharacter(file, index)); + await Promise.all(processingPromises); performance.mark('B'); + + // Filter out invalid/broken characters + characters = Object.values(characters).filter(x => x?.name).reduce((acc, val, index) => { + acc[index] = val; + return acc; + }, {}); + + response.send(JSON.stringify(characters)); + }); +}); + +router.post('/get', jsonParser, async function (request, response) { + if (!request.body) return response.sendStatus(400); + const item = request.body.avatar_url; + const filePath = path.join(DIRECTORIES.characters, item); + + if (!fs.existsSync(filePath)) { + return response.sendStatus(404); + } + + characters = {}; + await processCharacter(item, 0); + + return response.send(characters[0]); +}); + +router.post('/chats', jsonParser, async function (request, response) { + if (!request.body) return response.sendStatus(400); + + const characterDirectory = (request.body.avatar_url).replace('.png', ''); + + try { + const chatsDirectory = path.join(DIRECTORIES.chats, characterDirectory); + const files = fs.readdirSync(chatsDirectory); + const jsonFiles = files.filter(file => path.extname(file) === '.jsonl'); + + if (jsonFiles.length === 0) { + response.send({ error: true }); + return; + } + + const jsonFilesPromise = jsonFiles.map((file) => { + return new Promise(async (res) => { + const pathToFile = path.join(DIRECTORIES.chats, characterDirectory, file); + const fileStream = fs.createReadStream(pathToFile); + const stats = fs.statSync(pathToFile); + const fileSizeInKB = `${(stats.size / 1024).toFixed(2)}kb`; + + const rl = readline.createInterface({ + input: fileStream, + crlfDelay: Infinity, + }); + + let lastLine; + let itemCounter = 0; + rl.on('line', (line) => { + itemCounter++; + lastLine = line; + }); + rl.on('close', () => { + rl.close(); + + if (lastLine) { + const jsonData = tryParse(lastLine); + if (jsonData && (jsonData.name || jsonData.character_name)) { + const chatData = {}; + + chatData['file_name'] = file; + chatData['file_size'] = fileSizeInKB; + chatData['chat_items'] = itemCounter - 1; + chatData['mes'] = jsonData['mes'] || '[The chat is empty]'; + chatData['last_mes'] = jsonData['send_date'] || Date.now(); + + res(chatData); + } else { + console.log('Found an invalid or corrupted chat file:', pathToFile); + res({}); + } + } + }); + }); + }); + + const chatData = await Promise.all(jsonFilesPromise); + const validFiles = chatData.filter(i => i.file_name); + + return response.send(validFiles); + } catch (error) { + console.log(error); + return response.send({ error: true }); + } +}); + +function getPngName(file) { + let i = 1; + let base_name = file; + while (fs.existsSync(DIRECTORIES.characters + file + '.png')) { + file = base_name + i; + i++; + } + return file; +} + +router.post('/import', urlencodedParser, async function (request, response) { + + if (!request.body || request.file === undefined) return response.sendStatus(400); + + let png_name = ''; + let filedata = request.file; + let uploadPath = path.join(UPLOADS_PATH, filedata.filename); + var format = request.body.file_type; + const defaultAvatarPath = './public/img/ai4.png'; + const { importRisuSprites } = require('./src/endpoints/sprites'); + //console.log(format); + if (filedata) { + if (format == 'json') { + fs.readFile(uploadPath, 'utf8', async (err, data) => { + fs.unlinkSync(uploadPath); + + if (err) { + console.log(err); + response.send({ error: true }); + } + + let jsonData = JSON.parse(data); + + if (jsonData.spec !== undefined) { + console.log('importing from v2 json'); + importRisuSprites(jsonData); + unsetFavFlag(jsonData); + jsonData = readFromV2(jsonData); + jsonData['create_date'] = humanizedISO8601DateTime(); + png_name = getPngName(jsonData.data?.name || jsonData.name); + let char = JSON.stringify(jsonData); + charaWrite(defaultAvatarPath, char, png_name, response, { file_name: png_name }); + } else if (jsonData.name !== undefined) { + console.log('importing from v1 json'); + jsonData.name = sanitize(jsonData.name); + if (jsonData.creator_notes) { + jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', ''); + } + png_name = getPngName(jsonData.name); + let char = { + 'name': jsonData.name, + 'description': jsonData.description ?? '', + 'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '', + 'personality': jsonData.personality ?? '', + 'first_mes': jsonData.first_mes ?? '', + 'avatar': 'none', + 'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(), + 'mes_example': jsonData.mes_example ?? '', + 'scenario': jsonData.scenario ?? '', + 'create_date': humanizedISO8601DateTime(), + 'talkativeness': jsonData.talkativeness ?? 0.5, + 'creator': jsonData.creator ?? '', + 'tags': jsonData.tags ?? '', + }; + char = convertToV2(char); + let charJSON = JSON.stringify(char); + charaWrite(defaultAvatarPath, charJSON, png_name, response, { file_name: png_name }); + } else if (jsonData.char_name !== undefined) {//json Pygmalion notepad + console.log('importing from gradio json'); + jsonData.char_name = sanitize(jsonData.char_name); + if (jsonData.creator_notes) { + jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', ''); + } + png_name = getPngName(jsonData.char_name); + let char = { + 'name': jsonData.char_name, + 'description': jsonData.char_persona ?? '', + 'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '', + 'personality': '', + 'first_mes': jsonData.char_greeting ?? '', + 'avatar': 'none', + 'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(), + 'mes_example': jsonData.example_dialogue ?? '', + 'scenario': jsonData.world_scenario ?? '', + 'create_date': humanizedISO8601DateTime(), + 'talkativeness': jsonData.talkativeness ?? 0.5, + 'creator': jsonData.creator ?? '', + 'tags': jsonData.tags ?? '', + }; + char = convertToV2(char); + let charJSON = JSON.stringify(char); + charaWrite(defaultAvatarPath, charJSON, png_name, response, { file_name: png_name }); + } else { + console.log('Incorrect character format .json'); + response.send({ error: true }); + } + }); + } else { + try { + var img_data = await charaRead(uploadPath, format); + if (img_data === undefined) throw new Error('Failed to read character data'); + + let jsonData = JSON.parse(img_data); + + jsonData.name = sanitize(jsonData.data?.name || jsonData.name); + png_name = getPngName(jsonData.name); + + if (jsonData.spec !== undefined) { + console.log('Found a v2 character file.'); + importRisuSprites(jsonData); + unsetFavFlag(jsonData); + jsonData = readFromV2(jsonData); + jsonData['create_date'] = humanizedISO8601DateTime(); + const char = JSON.stringify(jsonData); + await charaWrite(uploadPath, char, png_name, response, { file_name: png_name }); + fs.unlinkSync(uploadPath); + } else if (jsonData.name !== undefined) { + console.log('Found a v1 character file.'); + + if (jsonData.creator_notes) { + jsonData.creator_notes = jsonData.creator_notes.replace('Creator\'s notes go here.', ''); + } + + let char = { + 'name': jsonData.name, + 'description': jsonData.description ?? '', + 'creatorcomment': jsonData.creatorcomment ?? jsonData.creator_notes ?? '', + 'personality': jsonData.personality ?? '', + 'first_mes': jsonData.first_mes ?? '', + 'avatar': 'none', + 'chat': jsonData.name + ' - ' + humanizedISO8601DateTime(), + 'mes_example': jsonData.mes_example ?? '', + 'scenario': jsonData.scenario ?? '', + 'create_date': humanizedISO8601DateTime(), + 'talkativeness': jsonData.talkativeness ?? 0.5, + 'creator': jsonData.creator ?? '', + 'tags': jsonData.tags ?? '', + }; + char = convertToV2(char); + const charJSON = JSON.stringify(char); + await charaWrite(uploadPath, charJSON, png_name, response, { file_name: png_name }); + fs.unlinkSync(uploadPath); + } else { + console.log('Unknown character card format'); + response.send({ error: true }); + } + } catch (err) { + console.log(err); + response.send({ error: true }); + } + } + } +}); + +router.post('/duplicate', jsonParser, async function (request, response) { + try { + if (!request.body.avatar_url) { + console.log('avatar URL not found in request body'); + console.log(request.body); + return response.sendStatus(400); + } + let filename = path.join(DIRECTORIES.characters, sanitize(request.body.avatar_url)); + if (!fs.existsSync(filename)) { + console.log('file for dupe not found'); + console.log(filename); + return response.sendStatus(404); + } + let suffix = 1; + let newFilename = filename; + + // If filename ends with a _number, increment the number + const nameParts = path.basename(filename, path.extname(filename)).split('_'); + const lastPart = nameParts[nameParts.length - 1]; + + let baseName; + + if (!isNaN(Number(lastPart)) && nameParts.length > 1) { + suffix = parseInt(lastPart) + 1; + baseName = nameParts.slice(0, -1).join('_'); // construct baseName without suffix + } else { + baseName = nameParts.join('_'); // original filename is completely the baseName + } + + newFilename = path.join(DIRECTORIES.characters, `${baseName}_${suffix}${path.extname(filename)}`); + + while (fs.existsSync(newFilename)) { + let suffixStr = '_' + suffix; + newFilename = path.join(DIRECTORIES.characters, `${baseName}${suffixStr}${path.extname(filename)}`); + suffix++; + } + + fs.copyFileSync(filename, newFilename); + console.log(`${filename} was copied to ${newFilename}`); + response.sendStatus(200); + } + catch (error) { + console.error(error); + return response.send({ error: true }); + } +}); + +router.post('/export', jsonParser, async function (request, response) { + if (!request.body.format || !request.body.avatar_url) { + return response.sendStatus(400); + } + + let filename = path.join(DIRECTORIES.characters, sanitize(request.body.avatar_url)); + + if (!fs.existsSync(filename)) { + return response.sendStatus(404); + } + + switch (request.body.format) { + case 'png': + return response.sendFile(filename, { root: process.cwd() }); + case 'json': { + try { + let json = await charaRead(filename); + if (json === undefined) return response.sendStatus(400); + let jsonObject = getCharaCardV2(JSON.parse(json)); + return response.type('json').send(jsonObject); + } + catch { + return response.sendStatus(400); + } + } + } + + return response.sendStatus(400); +}); + +module.exports = { router }; diff --git a/src/util.js b/src/util.js index ff2dadb1c..b01bdb615 100644 --- a/src/util.js +++ b/src/util.js @@ -267,6 +267,27 @@ function uuidv4() { }); } +function humanizedISO8601DateTime(date) { + let baseDate = typeof date === 'number' ? new Date(date) : new Date(); + let humanYear = baseDate.getFullYear(); + let humanMonth = (baseDate.getMonth() + 1); + let humanDate = baseDate.getDate(); + let humanHour = (baseDate.getHours() < 10 ? '0' : '') + baseDate.getHours(); + let humanMinute = (baseDate.getMinutes() < 10 ? '0' : '') + baseDate.getMinutes(); + let humanSecond = (baseDate.getSeconds() < 10 ? '0' : '') + baseDate.getSeconds(); + let humanMillisecond = (baseDate.getMilliseconds() < 10 ? '0' : '') + baseDate.getMilliseconds(); + let HumanizedDateTime = (humanYear + '-' + humanMonth + '-' + humanDate + ' @' + humanHour + 'h ' + humanMinute + 'm ' + humanSecond + 's ' + humanMillisecond + 'ms'); + return HumanizedDateTime; +} + +function tryParse(str) { + try { + return JSON.parse(str); + } catch { + return undefined; + } +} + module.exports = { getConfig, getConfigValue, @@ -279,4 +300,6 @@ module.exports = { deepMerge, color, uuidv4, + humanizedISO8601DateTime, + tryParse, }; diff --git a/src/worldinfo.js b/src/worldinfo.js new file mode 100644 index 000000000..31fdd21e4 --- /dev/null +++ b/src/worldinfo.js @@ -0,0 +1,68 @@ +const path = require('path'); +const fs = require('fs'); +const { DIRECTORIES } = require('./constants'); + +function readWorldInfoFile(worldInfoName) { + const dummyObject = { entries: {} }; + + if (!worldInfoName) { + return dummyObject; + } + + const filename = `${worldInfoName}.json`; + const pathToWorldInfo = path.join(DIRECTORIES.worlds, filename); + + if (!fs.existsSync(pathToWorldInfo)) { + console.log(`World info file ${filename} doesn't exist.`); + return dummyObject; + } + + const worldInfoText = fs.readFileSync(pathToWorldInfo, 'utf8'); + const worldInfo = JSON.parse(worldInfoText); + return worldInfo; +} + +/** + * @param {string} name Name of World Info file + * @param {object} entries Entries object + */ +function convertWorldInfoToCharacterBook(name, entries) { + /** @type {{ entries: object[]; name: string }} */ + const result = { entries: [], name }; + + for (const index in entries) { + const entry = entries[index]; + + const originalEntry = { + id: entry.uid, + keys: entry.key, + secondary_keys: entry.keysecondary, + comment: entry.comment, + content: entry.content, + constant: entry.constant, + selective: entry.selective, + insertion_order: entry.order, + enabled: !entry.disable, + position: entry.position == 0 ? 'before_char' : 'after_char', + extensions: { + position: entry.position, + exclude_recursion: entry.excludeRecursion, + display_index: entry.displayIndex, + probability: entry.probability ?? null, + useProbability: entry.useProbability ?? false, + depth: entry.depth ?? 4, + selectiveLogic: entry.selectiveLogic ?? 0, + group: entry.group ?? '', + }, + }; + + result.entries.push(originalEntry); + } + + return result; +} + +module.exports = { + readWorldInfoFile, + convertWorldInfoToCharacterBook, +};