Merge pull request #3088 from ceruleandeep/feature/comfyGgufModels

Feature/comfy gguf models
2025-06-05 21:59:27 +02:00 · 2024-11-18 20:52:52 +02:00
parent 4c0da00f5b 5478b37b22
commit dde9f93d77
6 changed files with 184 additions and 20 deletions
--- a/default/content/Char_Avatar_Comfy_Workflow.json
+++ b/default/content/Char_Avatar_Comfy_Workflow.json
@@ -0,0 +1,137 @@
+{
+  "3": {
+    "inputs": {
+      "seed": "%seed%",
+      "steps": "%steps%",
+      "cfg": "%scale%",
+      "sampler_name": "%sampler%",
+      "scheduler": "%scheduler%",
+      "denoise": "%denoise%",
+      "model": [
+        "4",
+        0
+      ],
+      "positive": [
+        "6",
+        0
+      ],
+      "negative": [
+        "7",
+        0
+      ],
+      "latent_image": [
+        "12",
+        0
+      ]
+    },
+    "class_type": "KSampler",
+    "_meta": {
+      "title": "KSampler"
+    }
+  },
+  "4": {
+    "inputs": {
+      "ckpt_name": "%model%"
+    },
+    "class_type": "CheckpointLoaderSimple",
+    "_meta": {
+      "title": "Load Checkpoint"
+    }
+  },
+  "6": {
+    "inputs": {
+      "text": "%prompt%",
+      "clip": [
+        "4",
+        1
+      ]
+    },
+    "class_type": "CLIPTextEncode",
+    "_meta": {
+      "title": "CLIP Text Encode (Prompt)"
+    }
+  },
+  "7": {
+    "inputs": {
+      "text": "%negative_prompt%",
+      "clip": [
+        "4",
+        1
+      ]
+    },
+    "class_type": "CLIPTextEncode",
+    "_meta": {
+      "title": "CLIP Text Encode (Negative Prompt)"
+    }
+  },
+  "8": {
+    "inputs": {
+      "samples": [
+        "3",
+        0
+      ],
+      "vae": [
+        "4",
+        2
+      ]
+    },
+    "class_type": "VAEDecode",
+    "_meta": {
+      "title": "VAE Decode"
+    }
+  },
+  "9": {
+    "inputs": {
+      "filename_prefix": "SillyTavern",
+      "images": [
+        "8",
+        0
+      ]
+    },
+    "class_type": "SaveImage",
+    "_meta": {
+      "title": "Save Image"
+    }
+  },
+  "10": {
+    "inputs": {
+      "image": "%char_avatar%"
+    },
+    "class_type": "ETN_LoadImageBase64",
+    "_meta": {
+      "title": "Load Image (Base64) [https://github.com/Acly/comfyui-tooling-nodes]"
+    }
+  },
+  "12": {
+    "inputs": {
+      "pixels": [
+        "13",
+        0
+      ],
+      "vae": [
+        "4",
+        2
+      ]
+    },
+    "class_type": "VAEEncode",
+    "_meta": {
+      "title": "VAE Encode"
+    }
+  },
+  "13": {
+    "inputs": {
+      "upscale_method": "bicubic",
+      "width": "%width%",
+      "height": "%height%",
+      "crop": "center",
+      "image": [
+        "10",
+        0
+      ]
+    },
+    "class_type": "ImageScale",
+    "_meta": {
+      "title": "Upscale Image"
+    }
+  }
+}
--- a/default/content/index.json
+++ b/default/content/index.json
@@ -135,6 +135,10 @@
        "filename": "Default_Comfy_Workflow.json",
        "type": "workflow"
    },
+    {
+        "filename": "Char_Avatar_Comfy_Workflow.json",
+        "type": "workflow"
+    },
    {
        "filename": "presets/kobold/Ace of Spades.json",
        "type": "kobold_preset"
--- a/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html
+++ b/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html
@@ -17,6 +17,7 @@
                    <li data-placeholder="scheduler" class="sd_comfy_workflow_editor_not_found">"%scheduler%"</li>
                    <li data-placeholder="steps" class="sd_comfy_workflow_editor_not_found">"%steps%"</li>
                    <li data-placeholder="scale" class="sd_comfy_workflow_editor_not_found">"%scale%"</li>
+                    <li data-placeholder="denoise" class="sd_comfy_workflow_editor_not_found">"%denoise%"</li>
                    <li data-placeholder="clip_skip" class="sd_comfy_workflow_editor_not_found">"%clip_skip%"</li>
                    <li data-placeholder="width" class="sd_comfy_workflow_editor_not_found">"%width%"</li>
                    <li data-placeholder="height" class="sd_comfy_workflow_editor_not_found">"%height%"</li>
--- a/public/scripts/extensions/stable-diffusion/index.js
+++ b/public/scripts/extensions/stable-diffusion/index.js
@@ -3269,6 +3269,10 @@ async function generateComfyImage(prompt, negativePrompt, signal) {

    const seed = extension_settings.sd.seed >= 0 ? extension_settings.sd.seed : Math.round(Math.random() * Number.MAX_SAFE_INTEGER);
    workflow = workflow.replaceAll('"%seed%"', JSON.stringify(seed));
+
+    const denoising_strength = extension_settings.sd.denoising_strength === undefined ? 1.0 : extension_settings.sd.denoising_strength;
+    workflow = workflow.replaceAll('"%denoise%"', JSON.stringify(denoising_strength));
+
    placeholders.forEach(ph => {
        workflow = workflow.replaceAll(`"%${ph}%"`, JSON.stringify(extension_settings.sd[ph]));
    });
@@ -3279,7 +3283,8 @@ async function generateComfyImage(prompt, negativePrompt, signal) {
        const response = await fetch(getUserAvatarUrl());
        if (response.ok) {
            const avatarBlob = await response.blob();
-            const avatarBase64 = await getBase64Async(avatarBlob);
+            const avatarBase64DataUrl = await getBase64Async(avatarBlob);
+            const avatarBase64 = avatarBase64DataUrl.split(',')[1];
            workflow = workflow.replaceAll('"%user_avatar%"', JSON.stringify(avatarBase64));
        } else {
            workflow = workflow.replaceAll('"%user_avatar%"', JSON.stringify(PNG_PIXEL));
@@ -3289,7 +3294,8 @@ async function generateComfyImage(prompt, negativePrompt, signal) {
        const response = await fetch(getCharacterAvatarUrl());
        if (response.ok) {
            const avatarBlob = await response.blob();
-            const avatarBase64 = await getBase64Async(avatarBlob);
+            const avatarBase64DataUrl = await getBase64Async(avatarBlob);
+            const avatarBase64 = avatarBase64DataUrl.split(',')[1];
            workflow = workflow.replaceAll('"%char_avatar%"', JSON.stringify(avatarBase64));
        } else {
            workflow = workflow.replaceAll('"%char_avatar%"', JSON.stringify(PNG_PIXEL));
--- a/public/scripts/extensions/stable-diffusion/settings.html
+++ b/public/scripts/extensions/stable-diffusion/settings.html
@@ -319,7 +319,7 @@
                    <input class="neo-range-input" type="number" id="sd_hr_scale_value" data-for="sd_hr_scale" min="{{hr_scale_min}}" max="{{hr_scale_max}}" step="{{hr_scale_step}}" value="{{hr_scale}}" >
                </div>

-                <div class="alignitemscenter flex-container flexFlowColumn flexGrow flexShrink gap0 flexBasis48p" data-sd-source="auto,vlad">
+                <div class="alignitemscenter flex-container flexFlowColumn flexGrow flexShrink gap0 flexBasis48p" data-sd-source="auto,vlad,comfy">
                    <small>
                        <span data-i18n="Denoising strength">Denoising strength</span>
                    </small>
--- a/src/endpoints/stable-diffusion.js
+++ b/src/endpoints/stable-diffusion.js
@@ -7,7 +7,7 @@ import sanitize from 'sanitize-filename';
 import { sync as writeFileAtomicSync } from 'write-file-atomic';
 import FormData from 'form-data';

-import { getBasicAuthHeader, delay } from '../util.js';
+import { delay, getBasicAuthHeader, tryParse } from '../util.js';
 import { jsonParser } from '../express-common.js';
 import { readSecret, SECRET_KEYS } from './secrets.js';

@@ -19,7 +19,7 @@ import { readSecret, SECRET_KEYS } from './secrets.js';
 function getComfyWorkflows(directories) {
    return fs
        .readdirSync(directories.comfyWorkflows)
-        .filter(file => file[0] != '.' && file.toLowerCase().endsWith('.json'))
+        .filter(file => file[0] !== '.' && file.toLowerCase().endsWith('.json'))
        .sort(Intl.Collator().compare);
 }

@@ -67,8 +67,7 @@ router.post('/upscalers', jsonParser, async (request, response) => {

            /** @type {any} */
            const data = await result.json();
-            const names = data.map(x => x.name);
-            return names;
+            return data.map(x => x.name);
        }

        async function getLatentUpscalers() {
@@ -88,8 +87,7 @@ router.post('/upscalers', jsonParser, async (request, response) => {

            /** @type {any} */
            const data = await result.json();
-            const names = data.map(x => x.name);
-            return names;
+            return data.map(x => x.name);
        }

        const [upscalers, latentUpscalers] = await Promise.all([getUpscalerModels(), getLatentUpscalers()]);
@@ -241,8 +239,7 @@ router.post('/set-model', jsonParser, async (request, response) => {
                    'Authorization': getBasicAuthHeader(request.body.auth),
                },
            });
-            const data = await result.json();
-            return data;
+            return await result.json();
        }

        const url = new URL(request.body.url);
@@ -274,7 +271,7 @@ router.post('/set-model', jsonParser, async (request, response) => {

            const progress = progressState['progress'];
            const jobCount = progressState['state']['job_count'];
-            if (progress == 0.0 && jobCount === 0) {
+            if (progress === 0.0 && jobCount === 0) {
                break;
            }

@@ -412,8 +409,19 @@ comfy.post('/models', jsonParser, async (request, response) => {
        }
        /** @type {any} */
        const data = await result.json();
-        return response.send(data.CheckpointLoaderSimple.input.required.ckpt_name[0].map(it => ({ value: it, text: it })));
-    } catch (error) {
+
+        const ckpts = data.CheckpointLoaderSimple.input.required.ckpt_name[0].map(it => ({ value: it, text: it })) || [];
+        const unets = data.UNETLoader.input.required.unet_name[0].map(it => ({ value: it, text: `UNet: ${it}` })) || [];
+
+        // load list of GGUF unets from diffusion_models if the loader node is available
+        const ggufs = data.UnetLoaderGGUF?.input.required.unet_name[0].map(it => ({ value: it, text: `GGUF: ${it}` })) || [];
+        const models = [...ckpts, ...unets, ...ggufs];
+
+        // make the display names of the models somewhat presentable
+        models.forEach(it => it.text = it.text.replace(/\.[^.]*$/, '').replace(/_/g, ' '));
+
+        return response.send(models);
+    } catch (error)     {
        console.log(error);
        return response.sendStatus(500);
    }
@@ -527,7 +535,8 @@ comfy.post('/generate', jsonParser, async (request, response) => {
            body: request.body.prompt,
        });
        if (!promptResult.ok) {
-            throw new Error('ComfyUI returned an error.');
+            const text = await promptResult.text();
+            throw new Error('ComfyUI returned an error.', { cause: tryParse(text) });
        }

        /** @type {any} */
@@ -550,7 +559,13 @@ comfy.post('/generate', jsonParser, async (request, response) => {
            await delay(100);
        }
        if (item.status.status_str === 'error') {
-            throw new Error('ComfyUI generation did not succeed.');
+            // Report node tracebacks if available
+            const errorMessages = item.status?.messages
+                ?.filter(it => it[0] === 'execution_error')
+                .map(it => it[1])
+                .map(it => `${it.node_type} [${it.node_id}] ${it.exception_type}: ${it.exception_message}`)
+                .join('\n') || '';
+            throw new Error(`ComfyUI generation did not succeed.\n\n${errorMessages}`.trim());
        }
        const imgInfo = Object.keys(item.outputs).map(it => item.outputs[it].images).flat()[0];
        const imgUrl = new URL(request.body.url);
@@ -560,11 +575,12 @@ comfy.post('/generate', jsonParser, async (request, response) => {
        if (!imgResponse.ok) {
            throw new Error('ComfyUI returned an error.');
        }
-        const imgBuffer = await imgResponse.buffer();
-        return response.send(imgBuffer.toString('base64'));
+        const imgBuffer = await imgResponse.arrayBuffer();
+        return response.send(Buffer.from(imgBuffer).toString('base64'));
    } catch (error) {
-        console.log(error);
-        return response.sendStatus(500);
+        console.log('ComfyUI error:', error);
+        response.status(500).send(error.message);
+        return response;
    }
 });