Update Jimp and add WASM plugins (#3784)

* Update jimp, use WASM format plugins

* Fix Jimp import path in thumbnails endpoint

* Fix size variable

* Add fetch patch to handle file URLs

* Fix JPEG thumbnailing

* Enhance fetch patch to validate file paths and support specific extensions

* Add default msBmp format

* Update jsconfig

* Update JPEG color space in thumbnail generation to YCbCr

* Install jimp plugins explicitly

* Refactor fetch patch utility functions

jsconfig.json

@@ -2,7 +2,7 @@
     "compilerOptions": {
         "module": "ESNext",
         "target": "ES2023",
-        "moduleResolution": "Node",
+        "moduleResolution": "Bundler",
         "strictNullChecks": true,
         "strictFunctionTypes": true,
         "checkJs": true,

package.json

@@ -4,6 +4,26 @@
         "@agnai/sentencepiece-js": "^1.1.1",
         "@agnai/web-tokenizers": "^0.1.3",
         "@iconfu/svg-inject": "^1.2.3",
+        "@jimp/core": "^1.6.0",
+        "@jimp/js-bmp": "^1.6.0",
+        "@jimp/js-gif": "^1.6.0",
+        "@jimp/js-tiff": "^1.6.0",
+        "@jimp/plugin-circle": "^1.6.0",
+        "@jimp/plugin-color": "^1.6.0",
+        "@jimp/plugin-contain": "^1.6.0",
+        "@jimp/plugin-cover": "^1.6.0",
+        "@jimp/plugin-crop": "^1.6.0",
+        "@jimp/plugin-displace": "^1.6.0",
+        "@jimp/plugin-fisheye": "^1.6.0",
+        "@jimp/plugin-flip": "^1.6.0",
+        "@jimp/plugin-mask": "^1.6.0",
+        "@jimp/plugin-quantize": "^1.6.0",
+        "@jimp/plugin-rotate": "^1.6.0",
+        "@jimp/plugin-threshold": "^1.6.0",
+        "@jimp/wasm-avif": "^1.6.0",
+        "@jimp/wasm-jpeg": "^1.6.0",
+        "@jimp/wasm-png": "^1.6.0",
+        "@jimp/wasm-webp": "^1.6.0",
         "@mozilla/readability": "^0.6.0",
         "@popperjs/core": "^2.11.8",
         "@zeldafan0225/ai_horde": "^5.2.0",
@@ -36,7 +56,6 @@
         "ip-regex": "^5.0.0",
         "ipaddr.js": "^2.2.0",
         "is-docker": "^3.0.0",
-        "jimp": "^0.22.10",
         "localforage": "^1.10.0",
         "lodash": "^4.17.21",
         "mime-types": "^2.1.35",

public/scripts/utils.js

@@ -1437,6 +1437,8 @@ export async function ensureImageFormatSupported(file) {
         'image/tiff',
         'image/gif',
         'image/apng',
+        'image/webp',
+        'image/avif',
     ];
 
     if (supportedTypes.includes(file.type) || !file.type.startsWith('image/')) {

server.js

@@ -20,6 +20,7 @@ import bodyParser from 'body-parser';
 import open from 'open';
 
 // local library imports
+import './src/fetch-patch.js';
 import { serverEvents, EVENT_NAMES } from './src/server-events.js';
 import { CommandLineParser } from './src/command-line.js';
 import { loadPlugins } from './src/plugin-loader.js';

src/endpoints/avatars.js

@@ -3,7 +3,7 @@ import fs from 'node:fs';
 
 import express from 'express';
 import sanitize from 'sanitize-filename';
-import jimp from 'jimp';
+import { Jimp, JimpMime } from '../jimp.js';
 import { sync as writeFileAtomicSync } from 'write-file-atomic';
 
 import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js';
@@ -41,13 +41,14 @@ router.post('/upload', async (request, response) => {
     try {
         const pathToUpload = path.join(request.file.destination, request.file.filename);
         const crop = tryParse(request.query.crop);
-        let rawImg = await jimp.read(pathToUpload);
+        const rawImg = await Jimp.read(pathToUpload);
 
         if (typeof crop == 'object' && [crop.x, crop.y, crop.width, crop.height].every(x => typeof x === 'number')) {
-            rawImg = rawImg.crop(crop.x, crop.y, crop.width, crop.height);
+            rawImg.crop({ w: crop.width, h: crop.height, x: crop.x, y: crop.y });
         }
 
-        const image = await rawImg.cover(AVATAR_WIDTH, AVATAR_HEIGHT).getBufferAsync(jimp.MIME_PNG);
+        rawImg.cover({ w: AVATAR_WIDTH, h: AVATAR_HEIGHT });
+        const image = await rawImg.getBuffer(JimpMime.png);
 
         const filename = request.body.overwrite_name || `${Date.now()}.png`;
         const pathToNewFile = path.join(request.user.directories.avatars, filename);

src/endpoints/characters.js

@@ -10,7 +10,7 @@ import { sync as writeFileAtomicSync } from 'write-file-atomic';
 import yaml from 'yaml';
 import _ from 'lodash';
 import mime from 'mime-types';
-import jimp from 'jimp';
+import { Jimp, JimpMime } from '../jimp.js';
 import storage from 'node-persist';
 
 import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js';
@@ -277,12 +277,12 @@ async function writeCharacterData(inputFile, data, outputFile, request, crop = u
  * @returns {Promise<Buffer>} Image buffer
  */
 async function parseImageBuffer(buffer, crop) {
-    const image = await jimp.read(buffer);
+    const image = await Jimp.fromBuffer(buffer);
     let finalWidth = image.bitmap.width, finalHeight = image.bitmap.height;
 
     // Apply crop if defined
     if (typeof crop == 'object' && [crop.x, crop.y, crop.width, crop.height].every(x => typeof x === 'number')) {
-        image.crop(crop.x, crop.y, crop.width, crop.height);
+        image.crop({ x: crop.x, y: crop.y, w: crop.width, h: crop.height });
         // Apply standard resize if requested
         if (crop.want_resize) {
             finalWidth = AVATAR_WIDTH;
@@ -293,7 +293,8 @@ async function parseImageBuffer(buffer, crop) {
         }
     }
 
-    return image.cover(finalWidth, finalHeight).getBufferAsync(jimp.MIME_PNG);
+    image.cover({ w: finalWidth, h: finalHeight });
+    return await image.getBuffer(JimpMime.png);
 }
 
 /**
@@ -304,12 +305,12 @@ async function parseImageBuffer(buffer, crop) {
  */
 async function tryReadImage(imgPath, crop) {
     try {
-        let rawImg = await jimp.read(imgPath);
+        const rawImg = await Jimp.read(imgPath);
         let finalWidth = rawImg.bitmap.width, finalHeight = rawImg.bitmap.height;
 
         // Apply crop if defined
         if (typeof crop == 'object' && [crop.x, crop.y, crop.width, crop.height].every(x => typeof x === 'number')) {
-            rawImg = rawImg.crop(crop.x, crop.y, crop.width, crop.height);
+            rawImg.crop({ x: crop.x, y: crop.y, w: crop.width, h: crop.height });
             // Apply standard resize if requested
             if (crop.want_resize) {
                 finalWidth = AVATAR_WIDTH;
@@ -320,8 +321,8 @@ async function tryReadImage(imgPath, crop) {
             }
         }
 
-        const image = await rawImg.cover(finalWidth, finalHeight).getBufferAsync(jimp.MIME_PNG);
-        return image;
+        rawImg.cover({ w: finalWidth, h: finalHeight });
+        return await rawImg.getBuffer(JimpMime.png);
     }
     // If it's an unsupported type of image (APNG) - just read the file as buffer
     catch (error) {

src/endpoints/thumbnails.js

@@ -5,7 +5,7 @@ import path from 'node:path';
 import mime from 'mime-types';
 import express from 'express';
 import sanitize from 'sanitize-filename';
-import jimp from 'jimp';
+import { Jimp, JimpMime } from '../jimp.js';
 import { sync as writeFileAtomicSync } from 'write-file-atomic';
 
 import { getConfigValue } from '../util.js';
@@ -122,14 +122,16 @@ async function generateThumbnail(directories, type, file) {
 
         try {
             const size = dimensions[type];
-            const image = await jimp.read(pathToOriginalFile);
-            const imgType = type == 'avatar' && pngFormat ? 'image/png' : 'image/jpeg';
+            const image = await Jimp.read(pathToOriginalFile);
             const width = !isNaN(size?.[0]) && size?.[0] > 0 ? size[0] : image.bitmap.width;
             const height = !isNaN(size?.[1]) && size?.[1] > 0 ? size[1] : image.bitmap.height;
-            buffer = await image.cover(width, height).quality(quality).getBufferAsync(imgType);
+            image.cover({ w: width, h: height });
+            buffer = pngFormat
+                ? await image.getBuffer(JimpMime.png)
+                : await image.getBuffer(JimpMime.jpeg, { quality: quality, jpegColorSpace: 'ycbcr' });
         }
         catch (inner) {
-            console.warn(`Thumbnailer can not process the image: ${pathToOriginalFile}. Using original size`);
+            console.warn(`Thumbnailer can not process the image: ${pathToOriginalFile}. Using original size`, inner);
             buffer = fs.readFileSync(pathToOriginalFile);
         }
 

src/fetch-patch.js

@@ -0,0 +1,91 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import mime from 'mime-types';
+
+const originalFetch = globalThis.fetch;
+
+const ALLOWED_EXTENSIONS = [
+    '.wasm',
+];
+
+/**
+ * Checks if a child path is under a parent path.
+ * @param {string} parentPath Parent path
+ * @param {string} childPath Child path
+ * @returns {boolean} Returns true if the child path is under the parent path, false otherwise
+ */
+function isPathUnderParent(parentPath, childPath) {
+    const normalizedParent = path.normalize(parentPath);
+    const normalizedChild = path.normalize(childPath);
+
+    const relativePath = path.relative(normalizedParent, normalizedChild);
+
+    return !relativePath.startsWith('..') && !path.isAbsolute(relativePath);
+}
+
+/**
+ * Checks if the given request is a file URL.
+ * @param {string | URL | Request} request The request to check
+ * @return {boolean} Returns true if the request is a file URL, false otherwise
+ */
+function isFileURL(request) {
+    if (typeof request === 'string') {
+        return request.startsWith('file://');
+    }
+    if (request instanceof URL) {
+        return request.protocol === 'file:';
+    }
+    if (request instanceof Request) {
+        return request.url.startsWith('file://');
+    }
+    return false;
+}
+
+/**
+ * Gets the URL from the request.
+ * @param {string | URL | Request} request The request to get the URL from
+ * @return {string} The URL of the request
+ */
+function getRequestURL(request) {
+    if (typeof request === 'string') {
+        return request;
+    }
+    if (request instanceof URL) {
+        return request.href;
+    }
+    if (request instanceof Request) {
+        return request.url;
+    }
+    throw new TypeError('Invalid request type');
+}
+
+// Patched fetch function that handles file URLs
+globalThis.fetch = async (/** @type {string | URL | Request} */ request, /** @type {RequestInit | undefined} */ options) => {
+    if (!isFileURL(request)) {
+        return originalFetch(request, options);
+    }
+    const url = getRequestURL(request);
+    const filePath = path.resolve(fileURLToPath(url));
+    const cwd = path.resolve(process.cwd()) + path.sep;
+    const isUnderCwd = isPathUnderParent(cwd, filePath);
+    if (!isUnderCwd) {
+        throw new Error('Requested file path is outside of the current working directory.');
+    }
+    const parsedPath = path.parse(filePath);
+    if (!ALLOWED_EXTENSIONS.includes(parsedPath.ext)) {
+        throw new Error('Unsupported file extension.');
+    }
+    const fileName = parsedPath.base;
+    const buffer = await fs.promises.readFile(filePath);
+    const blob = new Blob([buffer]);
+    const response = new Response(blob, {
+        status: 200,
+        statusText: 'OK',
+        headers: {
+            'Content-Type': mime.lookup(fileName) || 'application/octet-stream',
+            'Content-Length': buffer.length.toString(),
+        },
+    });
+    return response;
+};

src/jimp.js

@@ -0,0 +1,63 @@
+import { createJimp } from '@jimp/core';
+
+// Optimized image formats
+import webp from '@jimp/wasm-webp';
+import png from '@jimp/wasm-png';
+import jpeg from '@jimp/wasm-jpeg';
+import avif from '@jimp/wasm-avif';
+
+// Other image formats
+import bmp, { msBmp } from '@jimp/js-bmp';
+import gif from '@jimp/js-gif';
+import tiff from '@jimp/js-tiff';
+
+// Plugins
+import * as blit from '@jimp/plugin-blit';
+import * as circle from '@jimp/plugin-circle';
+import * as color from '@jimp/plugin-color';
+import * as contain from '@jimp/plugin-contain';
+import * as cover from '@jimp/plugin-cover';
+import * as crop from '@jimp/plugin-crop';
+import * as displace from '@jimp/plugin-displace';
+import * as fisheye from '@jimp/plugin-fisheye';
+import * as flip from '@jimp/plugin-flip';
+import * as mask from '@jimp/plugin-mask';
+import * as resize from '@jimp/plugin-resize';
+import * as rotate from '@jimp/plugin-rotate';
+import * as threshold from '@jimp/plugin-threshold';
+import * as quantize from '@jimp/plugin-quantize';
+
+const defaultPlugins = [
+    blit.methods,
+    circle.methods,
+    color.methods,
+    contain.methods,
+    cover.methods,
+    crop.methods,
+    displace.methods,
+    fisheye.methods,
+    flip.methods,
+    mask.methods,
+    resize.methods,
+    rotate.methods,
+    threshold.methods,
+    quantize.methods,
+];
+
+// A custom jimp that uses WASM for optimized formats and JS for the rest
+const Jimp = createJimp({
+    formats: [webp, png, jpeg, avif, bmp, msBmp, gif, tiff],
+    plugins: [...defaultPlugins],
+});
+
+const JimpMime = {
+    bmp: bmp().mime,
+    gif: gif().mime,
+    jpeg: jpeg().mime,
+    png: png().mime,
+    tiff: tiff().mime,
+};
+
+export default Jimp;
+
+export { Jimp, JimpMime };