diff --git a/package-lock.json b/package-lock.json index 7573bc7cf..929d67a14 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,7 @@ "simple-git": "^3.19.1", "uniqolor": "^1.1.0", "webp-converter": "2.3.2", + "write-file-atomic": "^5.0.1", "ws": "^8.13.0", "yargs": "^17.7.1", "yauzl": "^2.10.0" @@ -1805,6 +1806,14 @@ "@types/node": "16.9.1" } }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "engines": { + "node": ">=0.8.19" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -3073,6 +3082,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/simple-concat": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", @@ -3523,6 +3543,18 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "dev": true }, + "node_modules/write-file-atomic": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", + "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/ws": { "version": "8.13.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", diff --git a/package.json b/package.json index 12c2a4738..f54b0f507 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,8 @@ { "dependencies": { - "@dqbd/tiktoken": "^1.0.2", + "@agnai/sentencepiece-js": "^1.1.1", "@agnai/web-tokenizers": "^0.1.3", + "@dqbd/tiktoken": "^1.0.2", "axios": "^1.4.0", "command-exists": "^1.2.9", "compression": "^1", @@ -30,10 +31,10 @@ "png-chunks-extract": "^1.0.0", "response-time": "^2.3.2", "sanitize-filename": "^1.6.3", - "@agnai/sentencepiece-js": "^1.1.1", "simple-git": "^3.19.1", "uniqolor": "^1.1.0", "webp-converter": "2.3.2", + "write-file-atomic": "^5.0.1", "ws": "^8.13.0", "yargs": "^17.7.1", "yauzl": "^2.10.0" diff --git a/server.js b/server.js index e6e9440d5..f45be5cbd 100644 --- a/server.js +++ b/server.js @@ -68,6 +68,8 @@ app.use(compression()); app.use(responseTime()); const fs = require('fs'); +const writeFileAtomicSync = require('write-file-atomic').sync; +const writeFileAtomic = require('write-file-atomic'); const readline = require('readline'); const open = require('open'); @@ -692,7 +694,7 @@ app.post("/savechat", jsonParser, function (request, response) { var dir_name = String(request.body.avatar_url).replace('.png', ''); let chat_data = request.body.chat; let jsonlData = chat_data.map(JSON.stringify).join('\n'); - fs.writeFileSync(`${chatsPath + sanitize(dir_name)}/${sanitize(String(request.body.file_name))}.jsonl`, jsonlData, 'utf8'); + writeFileAtomicSync(`${chatsPath + sanitize(dir_name)}/${sanitize(String(request.body.file_name))}.jsonl`, jsonlData, 'utf8'); return response.send({ result: "ok" }); } catch (error) { response.send(error); @@ -1217,7 +1219,7 @@ async function charaWrite(img_url, data, target_img, response = undefined, mes = chunks.splice(-1, 0, PNGtext.encode('chara', base64EncodedData)); //chunks.splice(-1, 0, text.encode('lorem', 'ipsum')); - fs.writeFileSync(charactersPath + target_img + '.png', new Buffer.from(encode(chunks))); + writeFileAtomicSync(charactersPath + target_img + '.png', new Buffer.from(encode(chunks))); if (response !== undefined) response.send(mes); return true; } catch (err) { @@ -1429,7 +1431,7 @@ app.post('/deleteuseravatar', jsonParser, function (request, response) { app.post("/setbackground", jsonParser, function (request, response) { var bg = "#bg1 {background-image: url('../backgrounds/" + request.body.bg + "');}"; - fs.writeFile('public/css/bg_load.css', bg, 'utf8', function (err) { + writeFileAtomic('public/css/bg_load.css', bg, 'utf8', function (err) { if (err) { response.send(err); return console.log(err); @@ -1530,7 +1532,7 @@ app.post("/downloadbackground", urlencodedParser, function (request, response) { }); app.post("/savesettings", jsonParser, function (request, response) { - fs.writeFile('public/settings.json', JSON.stringify(request.body, null, 4), 'utf8', function (err) { + writeFileAtomic('public/settings.json', JSON.stringify(request.body, null, 4), 'utf8', function (err) { if (err) { response.send(err); console.log(err); @@ -1538,17 +1540,6 @@ app.post("/savesettings", jsonParser, function (request, response) { response.send({ result: "ok" }); } }); - - /*fs.writeFile('public/settings.json', JSON.stringify(request.body), 'utf8', function (err) { - if (err) { - response.send(err); - return console.log(err); - //response.send(err); - } else { - //response.redirect("/"); - response.send({ result: "ok" }); - } - });*/ }); function getCharaCardV2(jsonObject) { @@ -1714,7 +1705,7 @@ app.post('/savetheme', jsonParser, (request, response) => { } const filename = path.join(directories.themes, sanitize(request.body.name) + '.json'); - fs.writeFileSync(filename, JSON.stringify(request.body, null, 4), 'utf8'); + writeFileAtomicSync(filename, JSON.stringify(request.body, null, 4), 'utf8'); return response.sendStatus(200); }); @@ -1725,7 +1716,7 @@ app.post('/savemovingui', jsonParser, (request, response) => { } const filename = path.join(directories.movingUI, sanitize(request.body.name) + '.json'); - fs.writeFileSync(filename, JSON.stringify(request.body, null, 4), 'utf8'); + writeFileAtomicSync(filename, JSON.stringify(request.body, null, 4), 'utf8'); return response.sendStatus(200); }); @@ -1736,7 +1727,7 @@ app.post('/savequickreply', jsonParser, (request, response) => { } const filename = path.join(directories.quickreplies, sanitize(request.body.name) + '.json'); - fs.writeFileSync(filename, JSON.stringify(request.body, null, 4), 'utf8'); + writeFileAtomicSync(filename, JSON.stringify(request.body, null, 4), 'utf8'); return response.sendStatus(200); }); @@ -2330,7 +2321,7 @@ app.post("/exportcharacter", jsonParser, async function (request, response) { }, }; const exifString = exif.dump(metadata); - fs.writeFileSync(metadataPath, exifString, 'binary'); + writeFileAtomicSync(metadataPath, exifString, 'binary'); await webp.cwebp(filename, inputWebpPath, '-q 95'); await webp.webpmux_add(inputWebpPath, outputWebpPath, metadataPath, 'exif'); @@ -2412,9 +2403,9 @@ app.post("/importchat", urlencodedParser, function (request, response) { }); const errors = []; - newChats.forEach(chat => fs.writeFile( + newChats.forEach(chat => writeFileAtomic( `${chatsPath + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, - chat.map(JSON.stringify).join('\n'), + chat.map(tryParse).filter(x => x).join('\n'), 'utf8', (err) => err ?? errors.push(err) ) @@ -2456,7 +2447,7 @@ app.post("/importchat", urlencodedParser, function (request, response) { } } - fs.writeFileSync(`${chatsPath + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chat.map(JSON.stringify).join('\n'), 'utf8'); + writeFileAtomicSync(`${chatsPath + avatar_url}/${ch_name} - ${humanizedISO8601DateTime()} imported.jsonl`, chat.map(JSON.stringify).join('\n'), 'utf8'); response.send({ res: true }); } else { @@ -2525,7 +2516,7 @@ app.post('/importworldinfo', urlencodedParser, (request, response) => { return response.status(400).send('World file must have a name'); } - fs.writeFileSync(pathToNewFile, fileContents); + writeFileAtomicSync(pathToNewFile, fileContents); return response.send({ name: worldName }); }); @@ -2549,7 +2540,7 @@ app.post('/editworldinfo', jsonParser, (request, response) => { const filename = `${sanitize(request.body.name)}.json`; const pathToFile = path.join(directories.worlds, filename); - fs.writeFileSync(pathToFile, JSON.stringify(request.body.data, null, 4)); + writeFileAtomicSync(pathToFile, JSON.stringify(request.body.data, null, 4)); return response.send({ ok: true }); }); @@ -2570,7 +2561,7 @@ app.post('/uploaduseravatar', urlencodedParser, async (request, response) => { const filename = request.body.overwrite_name || `${Date.now()}.png`; const pathToNewFile = path.join(directories.avatars, filename); - fs.writeFileSync(pathToNewFile, image); + writeFileAtomicSync(pathToNewFile, image); fs.rmSync(pathToUpload); return response.send({ path: filename }); } catch (err) { @@ -2647,7 +2638,7 @@ app.post('/creategroup', jsonParser, (request, response) => { fs.mkdirSync(directories.groups); } - fs.writeFileSync(pathToFile, fileData); + writeFileAtomicSync(pathToFile, fileData); return response.send(groupMetadata); }); @@ -2659,7 +2650,7 @@ app.post('/editgroup', jsonParser, (request, response) => { const pathToFile = path.join(directories.groups, `${id}.json`); const fileData = JSON.stringify(request.body); - fs.writeFileSync(pathToFile, fileData); + writeFileAtomicSync(pathToFile, fileData); return response.send({ ok: true }); }); @@ -2713,7 +2704,7 @@ app.post('/savegroupchat', jsonParser, (request, response) => { let chat_data = request.body.chat; let jsonlData = chat_data.map(JSON.stringify).join('\n'); - fs.writeFileSync(pathToFile, jsonlData, 'utf8'); + writeFileAtomicSync(pathToFile, jsonlData, 'utf8'); return response.send({ ok: true }); }); @@ -2912,7 +2903,7 @@ async function generateThumbnail(type, file) { buffer = fs.readFileSync(pathToOriginalFile); } - fs.writeFileSync(pathToCachedFile, buffer); + writeFileAtomicSync(pathToCachedFile, buffer); } catch (outer) { return null; @@ -3473,7 +3464,7 @@ app.post("/save_preset", jsonParser, function (request, response) { } const fullpath = path.join(directory, filename); - fs.writeFileSync(fullpath, JSON.stringify(request.body.preset, null, 4), 'utf-8'); + writeFileAtomicSync(fullpath, JSON.stringify(request.body.preset, null, 4), 'utf-8'); return response.send({ name }); }); @@ -3508,7 +3499,7 @@ app.post("/savepreset_openai", jsonParser, function (request, response) { const filename = `${name}.settings`; const fullpath = path.join(directories.openAI_Settings, filename); - fs.writeFileSync(fullpath, JSON.stringify(request.body, null, 4), 'utf-8'); + writeFileAtomicSync(fullpath, JSON.stringify(request.body, null, 4), 'utf-8'); return response.send({ name }); }); @@ -3810,7 +3801,7 @@ function migrateSecrets() { if (modified) { console.log('Writing updated settings.json...'); const settingsContent = JSON.stringify(settings); - fs.writeFileSync(SETTINGS_FILE, settingsContent, "utf-8"); + writeFileAtomicSync(SETTINGS_FILE, settingsContent, "utf-8"); } } catch (error) { @@ -4181,7 +4172,7 @@ app.post('/upload_sprite_pack', urlencodedParser, async (request, response) => { // Write sprite buffer to disk const pathToSprite = path.join(spritesPath, filename); - fs.writeFileSync(pathToSprite, buffer); + writeFileAtomicSync(pathToSprite, buffer); } // Remove uploaded ZIP file @@ -4406,7 +4397,7 @@ function importRisuSprites(data) { const filename = label + '.png'; const pathToFile = path.join(spritesPath, filename); - fs.writeFileSync(pathToFile, fileBase64, { encoding: 'base64' }); + writeFileAtomicSync(pathToFile, fileBase64, { encoding: 'base64' }); } // Remove additionalAssets and emotions from data (they are now in the sprites folder) @@ -4420,13 +4411,13 @@ function importRisuSprites(data) { function writeSecret(key, value) { if (!fs.existsSync(SECRETS_FILE)) { const emptyFile = JSON.stringify({}); - fs.writeFileSync(SECRETS_FILE, emptyFile, "utf-8"); + writeFileAtomicSync(SECRETS_FILE, emptyFile, "utf-8"); } const fileContents = fs.readFileSync(SECRETS_FILE); const secrets = JSON.parse(fileContents); secrets[key] = value; - fs.writeFileSync(SECRETS_FILE, JSON.stringify(secrets), "utf-8"); + writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets), "utf-8"); } function readSecret(key) {