Merge branch 'release' into staging

.github/readme-zh_cn.md

@@ -41,8 +41,6 @@ SillyTavern 本身并无用处，因为它只是一个用户聊天界面。你
 
 <https://rentry.org/STAI-Termux>
 
-Termux 不支持**.Webp 字符卡的导入/导出。请使用 JSON 或 PNG 格式**。
-
 ## 有问题或建议？
 
 ### 我们现在有了 Discord 社区

.github/readme.md

@@ -41,8 +41,6 @@ Since Tavern is only a user interface, it has tiny hardware requirements, it wil
 
 <https://rentry.org/STAI-Termux>
 
-**.webp character cards import/export is not supported in Termux. Use either JSON or PNG formats instead.**
-
 ## Questions or suggestions?
 
 ### We now have a community Discord server
@@ -71,7 +69,6 @@ Get in touch with the developers directly:
 * [Oobabooga's TextGen WebUI](https://github.com/oobabooga/text-generation-webui) API connection
 * [AI Horde](https://horde.koboldai.net/) connection
 * Prompt generation formatting tweaking
-* webp character card interoperability (PNG is still an internal format)
 
 ## Extensions
 

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "sillytavern",
-    "version": "1.10.3",
+    "version": "1.10.4",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "sillytavern",
-            "version": "1.10.3",
+            "version": "1.10.4",
             "hasInstallScript": true,
             "license": "AGPL-3.0",
             "dependencies": {
@@ -21,7 +21,6 @@
                 "cors": "^2.8.5",
                 "csrf-csrf": "^2.2.3",
                 "device-detector-js": "^3.0.3",
-                "exifreader": "^4.12.0",
                 "express": "^4.18.2",
                 "google-translate-api-browser": "^3.0.1",
                 "gpt3-tokenizer": "^1.1.5",
@@ -35,7 +34,6 @@
                 "multer": "^1.4.5-lts.1",
                 "node-fetch": "^2.6.11",
                 "open": "^8.4.2",
-                "piexifjs": "^1.0.6",
                 "png-chunk-text": "^1.0.0",
                 "png-chunks-encode": "^1.0.0",
                 "png-chunks-extract": "^1.0.0",
@@ -45,7 +43,6 @@
                 "simple-git": "^3.19.1",
                 "uniqolor": "^1.1.0",
                 "vectra": "^0.2.2",
-                "webp-converter": "2.3.2",
                 "write-file-atomic": "^5.0.1",
                 "ws": "^8.13.0",
                 "yargs": "^17.7.1",
@@ -968,15 +965,6 @@
             "integrity": "sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A==",
             "peer": true
         },
-        "node_modules/@xmldom/xmldom": {
-            "version": "0.8.9",
-            "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.9.tgz",
-            "integrity": "sha512-4VSbbcMoxc4KLjb1gs96SRmi7w4h1SF+fCoiK0XaQX62buCc1G5d0DC5bJ9xJBNPDSVCmIrcl8BiYxzjrqaaJA==",
-            "optional": true,
-            "engines": {
-                "node": ">=10.0.0"
-            }
-        },
         "node_modules/accepts": {
             "version": "1.3.8",
             "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@@ -1793,15 +1781,6 @@
             "resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz",
             "integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw=="
         },
-        "node_modules/exifreader": {
-            "version": "4.13.0",
-            "resolved": "https://registry.npmjs.org/exifreader/-/exifreader-4.13.0.tgz",
-            "integrity": "sha512-IhJBpyXDLbCdgzVHkthadOvrMiZOR2XS7POVp0b5JoVfScRoCJ6YazZ+stTkbDTE5TRTP44bE5RKsujckAs45Q==",
-            "hasInstallScript": true,
-            "optionalDependencies": {
-                "@xmldom/xmldom": "^0.8.8"
-            }
-        },
         "node_modules/expand-template": {
             "version": "2.0.3",
             "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
@@ -3097,11 +3076,6 @@
                 "url": "https://github.com/sponsors/jonschlinkert"
             }
         },
-        "node_modules/piexifjs": {
-            "version": "1.0.6",
-            "resolved": "https://registry.npmjs.org/piexifjs/-/piexifjs-1.0.6.tgz",
-            "integrity": "sha512-0wVyH0cKohzBQ5Gi2V1BuxYpxWfxF3cSqfFXfPIpl5tl9XLS5z4ogqhUCD20AbHi0h9aJkqXNJnkVev6gwh2ag=="
-        },
         "node_modules/pixelmatch": {
             "version": "4.0.2",
             "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-4.0.2.tgz",
@@ -4183,11 +4157,6 @@
             "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
             "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
         },
-        "node_modules/webp-converter": {
-            "version": "2.3.2",
-            "resolved": "https://registry.npmjs.org/webp-converter/-/webp-converter-2.3.2.tgz",
-            "integrity": "sha512-9kQ9Q/MPzUV2mye8Tv7vA6vDIPk77rI4AWWm2vSaCyGAEsxqyVZYeVU2MSJY5fLkf6u7G5K343vLxKubOxz16Q=="
-        },
         "node_modules/whatwg-fetch": {
             "version": "3.6.18",
             "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.18.tgz",

package.json

@@ -11,7 +11,6 @@
         "cors": "^2.8.5",
         "csrf-csrf": "^2.2.3",
         "device-detector-js": "^3.0.3",
-        "exifreader": "^4.12.0",
         "express": "^4.18.2",
         "google-translate-api-browser": "^3.0.1",
         "gpt3-tokenizer": "^1.1.5",
@@ -25,7 +24,6 @@
         "multer": "^1.4.5-lts.1",
         "node-fetch": "^2.6.11",
         "open": "^8.4.2",
-        "piexifjs": "^1.0.6",
         "png-chunk-text": "^1.0.0",
         "png-chunks-encode": "^1.0.0",
         "png-chunks-extract": "^1.0.0",
@@ -35,7 +33,6 @@
         "simple-git": "^3.19.1",
         "uniqolor": "^1.1.0",
         "vectra": "^0.2.2",
-        "webp-converter": "2.3.2",
         "write-file-atomic": "^5.0.1",
         "ws": "^8.13.0",
         "yargs": "^17.7.1",
@@ -53,7 +50,7 @@
         "type": "git",
         "url": "https://github.com/SillyTavern/SillyTavern.git"
     },
-    "version": "1.10.3",
+    "version": "1.10.4",
     "scripts": {
         "start": "node server.js",
         "start-multi": "node server.js --disableCsrf",

public/i18n.json

@@ -284,7 +284,6 @@
         "Regenerate": "重新生成",
         "PNG": "PNG",
         "JSON": "JSON",
-        "WEBP": "WEBP",
         "presets": "预设",
         "Message Sound": "AI 消息提示音",
         "Author's Note": "作者注释",
@@ -836,7 +835,6 @@
         "Regenerate": "再生成",
         "PNG": "PNG",
         "JSON": "JSON",
-        "WEBP": "WEBP",
         "presets": "プリセット",
         "Message Sound": "メッセージ音",
         "Author's Note": "作者の注記",
@@ -1392,7 +1390,6 @@
         "Regenerate": "재생성",
         "PNG": "PNG",
         "JSON": "JSON",
-        "WEBP": "WEBP",
         "presets": "기본설정",
         "Message Sound": "메시지 효과음",
         "Author's Note": "글쓴이 쪽지",
@@ -2016,7 +2013,6 @@
         "Regenerate": "Повторная генерация",
         "PNG": "PNG",
         "JSON": "JSON",
-        "WEBP": "WEBP",
         "presets": "Предустановки",
         "Message Sound": "Звук сообщения",
         "Author's Note": "Авторские заметки",
@@ -2580,7 +2576,6 @@
         "Regenerate": "Rigenera",
         "PNG": "PNG",
         "JSON": "JSON",
-        "WEBP": "WEBP",
         "presets": "preset",
         "Message Sound": "Suono del messaggio",
         "Author's Note": "Note d'autore",
@@ -3259,7 +3254,6 @@
         "Regenerate": "Regenereren",
         "PNG": "PNG",
         "JSON": "JSON",
-        "WEBP": "WEBP",
         "presets": "sjablonen",
         "Message Sound": "Berichtgeluid",
         "Author's Note": "Notitie van auteur",

public/index.html

@@ -3499,7 +3499,7 @@
 
                 <div id="rm_character_import" class="right_menu" style="display: none;">
                     <form id="form_import" action="javascript:void(null);" method="post" enctype="multipart/form-data">
-                        <input multiple type="file" id="character_import_file" accept=".json, image/png, image/webp" name="avatar">
+                        <input multiple type="file" id="character_import_file" accept=".json, image/png" name="avatar">
                         <input id="character_import_file_type" name="file_type" class="text_pole" maxlength="999" size="2" value="" autocomplete="off">
                     </form>
                 </div>
@@ -4462,7 +4462,6 @@
     <div id="export_format_popup" class="list-group">
         <div class="export_format list-group-item" data-format="png">PNG</div>
         <div class="export_format list-group-item" data-format="json">JSON</div>
-        <div class="export_format list-group-item" data-format="webp">WEBP</div>
     </div>
 
     <div id="zoomed_avatar_template" class="template_element">

public/script.js

@@ -6819,7 +6819,6 @@ export function processDroppedFiles(files) {
     const allowedMimeTypes = [
         'application/json',
         'image/png',
-        'image/webp',
     ];
 
     for (const file of files) {
@@ -6835,7 +6834,7 @@ function importCharacter(file) {
     const ext = file.name.match(/\.(\w+)$/);
     if (
         !ext ||
-        (ext[1].toLowerCase() != "json" && ext[1].toLowerCase() != "png" && ext[1] != "webp")
+        (ext[1].toLowerCase() != "json" && ext[1].toLowerCase() != "png")
     ) {
         return;
     }

server.js

@@ -40,13 +40,11 @@ const json5 = require('json5');
 const WebSocket = require('ws');
 
 // image processing related library imports
-const exif = require('piexifjs');
 const encode = require('png-chunks-encode');
 const extract = require('png-chunks-extract');
 const jimp = require('jimp');
 const mime = require('mime-types');
 const PNGtext = require('png-chunk-text');
-const webp = require('webp-converter');
 const yauzl = require('yauzl');
 
 // tokenizing related library imports
@@ -1097,7 +1095,7 @@ app.post("/renamecharacter", jsonParser, async function (request, response) {
     try {
         // Read old file, replace name int it
         const rawOldData = await charaRead(oldAvatarPath);
-        if (rawOldData === false || rawOldData === undefined) throw new Error("Failed to read character file");
+        if (rawOldData === undefined) throw new Error("Failed to read character file");
 
         const oldData = getCharaCardV2(json5.parse(rawOldData));
         _.set(oldData, 'data.name', newName);
@@ -1344,7 +1342,7 @@ const calculateDataSize = (data) => {
 const processCharacter = async (item, i) => {
     try {
         const img_data = await charaRead(charactersPath + item);
-        if (img_data === false || img_data === undefined) throw new Error("Failed to read character file");
+        if (img_data === undefined) throw new Error("Failed to read character file");
 
         let jsonObject = getCharaCardV2(json5.parse(img_data));
         jsonObject.avatar = item;
@@ -2182,26 +2180,13 @@ app.post("/importcharacter", urlencodedParser, async function (request, response
         } else {
             try {
                 var img_data = await charaRead(uploadPath, format);
-                if (img_data === false || img_data === undefined) throw new Error('Failed to read character data');
+                if (img_data === undefined) throw new Error('Failed to read character data');
 
                 let jsonData = json5.parse(img_data);
 
                 jsonData.name = sanitize(jsonData.data?.name || jsonData.name);
                 png_name = getPngName(jsonData.name);
 
-                if (format == 'webp') {
-                    try {
-                        let convertedPath = path.join(UPLOADS_PATH, path.basename(uploadPath, ".webp") + ".png")
-                        await webp.dwebp(uploadPath, convertedPath, "-o");
-                        fs.unlinkSync(uploadPath);
-                        uploadPath = convertedPath;
-                    }
-                    catch {
-                        console.error('WEBP image conversion failed. Using the default character image.');
-                        uploadPath = defaultAvatarPath;
-                    }
-                }
-
                 if (jsonData.spec !== undefined) {
                     console.log('Found a v2 character file.');
                     importRisuSprites(jsonData);
@@ -2381,7 +2366,7 @@ app.post("/exportcharacter", jsonParser, async function (request, response) {
         case 'json': {
             try {
                 let json = await charaRead(filename);
-                if (json === false || json === undefined) return response.sendStatus(400);
+                if (json === undefined) return response.sendStatus(400);
                 let jsonObject = getCharaCardV2(json5.parse(json));
                 return response.type('json').send(jsonObject)
             }
@@ -2389,39 +2374,6 @@ app.post("/exportcharacter", jsonParser, async function (request, response) {
                 return response.sendStatus(400);
             }
         }
-        case 'webp': {
-            try {
-                let json = await charaRead(filename);
-                if (json === false || json === undefined) return response.sendStatus(400);
-                let stringByteArray = utf8Encode.encode(json).toString();
-                let inputWebpPath = path.join(UPLOADS_PATH, `${Date.now()}_input.webp`);
-                let outputWebpPath = path.join(UPLOADS_PATH, `${Date.now()}_output.webp`);
-                let metadataPath = path.join(UPLOADS_PATH, `${Date.now()}_metadata.exif`);
-                let metadata =
-                {
-                    "Exif": {
-                        [exif.ExifIFD.UserComment]: stringByteArray,
-                    },
-                };
-                const exifString = exif.dump(metadata);
-                writeFileAtomicSync(metadataPath, exifString, 'binary');
-
-                await webp.cwebp(filename, inputWebpPath, '-q 95');
-                await webp.webpmux_add(inputWebpPath, outputWebpPath, metadataPath, 'exif');
-
-                response.sendFile(outputWebpPath, { root: process.cwd() }, () => {
-                    fs.rmSync(inputWebpPath);
-                    fs.rmSync(metadataPath);
-                    fs.rmSync(outputWebpPath);
-                });
-
-                return;
-            }
-            catch (err) {
-                console.log(err);
-                return response.sendStatus(400);
-            }
-        }
     }
 
     return response.sendStatus(400);
@@ -4080,8 +4032,6 @@ const setupTasks = async function () {
     contentManager.checkForNewContent();
     cleanUploads();
 
-    await convertWebp();
-
     [spp_llama, spp_nerd, spp_nerd_v2, claude_tokenizer] = await Promise.all([
         loadSentencepieceTokenizer('src/sentencepiece/tokenizer.model'),
         loadSentencepieceTokenizer('src/sentencepiece/nerdstash.model'),
@@ -4141,47 +4091,6 @@ if (true === cliArguments.ssl) {
     );
 }
 
-async function convertWebp() {
-    const files = fs.readdirSync(directories.characters).filter(e => e.endsWith(".webp"));
-
-    if (!files.length) {
-        return;
-    }
-
-    console.log(`${files.length} WEBP files will be automatically converted.`);
-
-    for (const file of files) {
-        try {
-            const source = path.join(directories.characters, file);
-            const dest = path.join(directories.characters, path.basename(file, ".webp") + ".png");
-
-            if (fs.existsSync(dest)) {
-                console.log(`${dest} already exists. Delete ${source} manually`);
-                continue;
-            }
-
-            console.log(`Read... ${source}`);
-            const data = await charaRead(source);
-
-            console.log(`Convert... ${source} -> ${dest}`);
-            await webp.dwebp(source, dest, "-o");
-
-            console.log(`Write... ${dest}`);
-            const success = await charaWrite(dest, data, path.parse(dest).name);
-
-            if (!success) {
-                console.log(`Failure on ${source} -> ${dest}`);
-                continue;
-            }
-
-            console.log(`Remove... ${source}`);
-            fs.rmSync(source);
-        } catch (err) {
-            console.log(err);
-        }
-    }
-}
-
 function backupSettings() {
     const MAX_BACKUPS = 25;
 

src/character-card-parser.js

@@ -1,56 +1,12 @@
 const fs = require('fs');
-const json5 = require('json5');
-const ExifReader = require('exifreader');
 
 const extract = require('png-chunks-extract');
 const PNGtext = require('png-chunk-text');
 
-const utf8Decode = new TextDecoder('utf-8', { ignoreBOM: true });
-
 const parse = async (cardUrl, format) => {
-    let fileFormat;
+    let fileFormat = format === undefined ? 'png' : format;
-    if (format === undefined) {
-        if (cardUrl.indexOf('.webp') !== -1)
-            fileFormat = 'webp';
-        else
-            fileFormat = 'png';
-    }
-    else
-        fileFormat = format;
 
     switch (fileFormat) {
-        case 'webp':
-            try {
-                const exif_data = await ExifReader.load(fs.readFileSync(cardUrl));
-                let char_data;
-
-                if (exif_data['UserComment']['description']) {
-                    let description = exif_data['UserComment']['description'];
-                    if (description === 'Undefined' && exif_data['UserComment'].value && exif_data['UserComment'].value.length === 1) {
-                        description = exif_data['UserComment'].value[0];
-                    }
-
-                    try {
-                        json5.parse(description);
-                        char_data = description;
-                    } catch {
-                        const byteArr = description.split(",").map(Number);
-                        const uint8Array = new Uint8Array(byteArr);
-                        const char_data_string = utf8Decode.decode(uint8Array);
-                        char_data = char_data_string;
-                    }
-                }
-                else {
-                    console.log('No description found in EXIF data.');
-                    return false;
-                }
-
-                return char_data;
-            }
-            catch (err) {
-                console.log(err);
-                return false;
-            }
         case 'png':
             const buffer = fs.readFileSync(cardUrl);
             const chunks = extract(buffer);

tools/charaverter/main.mjs

@@ -1,117 +0,0 @@
-import fs from 'fs';
-import jimp from 'jimp';
-import extract from 'png-chunks-extract';
-import encode from 'png-chunks-encode';
-import PNGtext from 'png-chunk-text';
-import ExifReader from 'exifreader';
-import webp from 'webp-converter';
-import path from 'path';
-
-async function charaRead(img_url, input_format){
-    let format;
-    if(input_format === undefined){
-        if(img_url.indexOf('.webp') !== -1){
-            format = 'webp';
-        }else{
-            format = 'png';
-        }
-    }else{
-        format = input_format;
-    }
-    
-    switch(format){
-        case 'webp':
-            const exif_data = await ExifReader.load(fs.readFileSync(img_url));
-            const char_data = exif_data['UserComment']['description'];
-            if (char_data === 'Undefined' && exif_data['UserComment'].value && exif_data['UserComment'].value.length === 1) {
-                return exif_data['UserComment'].value[0];
-            }
-            return char_data;
-        case 'png':
-            const buffer = fs.readFileSync(img_url);
-            const chunks = extract(buffer);
-             
-            const textChunks = chunks.filter(function (chunk) {
-              return chunk.name === 'tEXt';
-            }).map(function (chunk) {
-                //console.log(text.decode(chunk.data));
-              return PNGtext.decode(chunk.data);
-            });
-            var base64DecodedData = Buffer.from(textChunks[0].text, 'base64').toString('utf8');
-            return base64DecodedData;//textChunks[0].text;
-            //console.log(textChunks[0].keyword); // 'hello'
-            //console.log(textChunks[0].text);    // 'world'
-        default:
-            break;
-    }                   
-
-}
-
-async function charaWrite(img_url, data, target_img, response = undefined, mes = 'ok') {
-    try {
-        // Read the image, resize, and save it as a PNG into the buffer
-
-        webp
-
-        const rawImg = await jimp.read(img_url);
-        const image = await rawImg.cover(400, 600).getBufferAsync(jimp.MIME_PNG);
-
-        // Get the chunks
-        const chunks = extract(image);
-        const tEXtChunks = chunks.filter(chunk => chunk.create_date === 'tEXt');
-
-        // Remove all existing tEXt chunks
-        for (let tEXtChunk of tEXtChunks) {
-            chunks.splice(chunks.indexOf(tEXtChunk), 1);
-        }
-        // Add new chunks before the IEND chunk
-        const base64EncodedData = Buffer.from(data, 'utf8').toString('base64');
-        chunks.splice(-1, 0, PNGtext.encode('chara', base64EncodedData));
-        //chunks.splice(-1, 0, text.encode('lorem', 'ipsum'));
-
-        fs.writeFileSync(target_img, new Buffer.from(encode(chunks)));
-        if (response !== undefined) response.send(mes);
-        return true;
-
-
-    } catch (err) {
-        console.log(err);
-        if (response !== undefined) response.status(500).send(err);
-        return false;
-    }
-}
-
-
-(async function() {
-    const spath = process.argv[2]
-    const dpath = process.argv[3] || spath
-    const files = fs.readdirSync(spath).filter(e => e.endsWith(".webp"))
-    if (!files.length) {
-        console.log("Nothing to convert.")
-        return
-    }
-
-    try { fs.mkdirSync(dpath) } catch {}
-
-    for(const f of files) {
-        const source = path.join(spath, f),
-            dest = path.join(dpath, path.basename(f, ".webp") + ".png")
-
-        console.log(`Read... ${source}`)
-        const data = await charaRead(source)
-
-        console.log(`Convert... ${source} -> ${dest}`)
-        await webp.dwebp(source, dest, "-o")
-
-        console.log(`Write... ${dest}`)
-        const success = await charaWrite(dest, data, path.parse(dest).name);
-
-        if (!success) {
-            console.log(`Failure on ${source} -> ${dest}`);
-            continue;
-        }
-
-        console.log(`Remove... ${source}`)
-        fs.rmSync(source)
-    }
-})()

tools/charaverter/package.json

@@ -1,10 +0,0 @@
-{
-    "dependencies": {
-        "exifreader": "^4.12.0",
-        "jimp": "^0.22.7",
-        "png-chunk-text": "^1.0.0",
-        "png-chunks-encode": "^1.0.0",
-        "png-chunks-extract": "^1.0.0",
-        "webp-converter": "^2.3.3"
-    }
-}