From 5992117904ed9be5af20c2e8582a979a37235cdd Mon Sep 17 00:00:00 2001 From: ceruleandeep Date: Mon, 18 Nov 2024 17:55:30 +1100 Subject: [PATCH 1/6] Add GGUF models and denoise parameter for ComfyUI --- .../stable-diffusion/comfyWorkflowEditor.html | 1 + .../extensions/stable-diffusion/index.js | 4 ++ .../extensions/stable-diffusion/settings.html | 2 +- src/endpoints/stable-diffusion.js | 42 ++++++++++++------- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html b/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html index 2427fa6fb..5ac02209b 100644 --- a/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html +++ b/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html @@ -17,6 +17,7 @@
  • "%scheduler%"
  • "%steps%"
  • "%scale%"
    • +
  • "%denoise%"
  • "%clip_skip%"
  • "%width%"
  • "%height%"
    • diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index 8bffdaafe..a176f2e6e 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -3269,6 +3269,10 @@ async function generateComfyImage(prompt, negativePrompt, signal) { const seed = extension_settings.sd.seed >= 0 ? extension_settings.sd.seed : Math.round(Math.random() * Number.MAX_SAFE_INTEGER); workflow = workflow.replaceAll('"%seed%"', JSON.stringify(seed)); + + const denoising_strength = extension_settings.sd.denoising_strength === undefined ? 1.0 : extension_settings.sd.denoising_strength; + workflow = workflow.replaceAll('"%denoise%"', JSON.stringify(denoising_strength)); + placeholders.forEach(ph => { workflow = workflow.replaceAll(`"%${ph}%"`, JSON.stringify(extension_settings.sd[ph])); }); diff --git a/public/scripts/extensions/stable-diffusion/settings.html b/public/scripts/extensions/stable-diffusion/settings.html index d07504dca..e3ee1eea9 100644 --- a/public/scripts/extensions/stable-diffusion/settings.html +++ b/public/scripts/extensions/stable-diffusion/settings.html @@ -319,7 +319,7 @@ -
    +
    Denoising strength diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index 115b540f2..3679d9ea1 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -7,7 +7,7 @@ import sanitize from 'sanitize-filename'; import { sync as writeFileAtomicSync } from 'write-file-atomic'; import FormData from 'form-data'; -import { getBasicAuthHeader, delay } from '../util.js'; +import { delay, getBasicAuthHeader } from '../util.js'; import { jsonParser } from '../express-common.js'; import { readSecret, SECRET_KEYS } from './secrets.js'; @@ -19,7 +19,7 @@ import { readSecret, SECRET_KEYS } from './secrets.js'; function getComfyWorkflows(directories) { return fs .readdirSync(directories.comfyWorkflows) - .filter(file => file[0] != '.' && file.toLowerCase().endsWith('.json')) + .filter(file => file[0] !== '.' && file.toLowerCase().endsWith('.json')) .sort(Intl.Collator().compare); } @@ -67,8 +67,7 @@ router.post('/upscalers', jsonParser, async (request, response) => { /** @type {any} */ const data = await result.json(); - const names = data.map(x => x.name); - return names; + return data.map(x => x.name); } async function getLatentUpscalers() { @@ -88,8 +87,7 @@ router.post('/upscalers', jsonParser, async (request, response) => { /** @type {any} */ const data = await result.json(); - const names = data.map(x => x.name); - return names; + return data.map(x => x.name); } const [upscalers, latentUpscalers] = await Promise.all([getUpscalerModels(), getLatentUpscalers()]); @@ -241,8 +239,7 @@ router.post('/set-model', jsonParser, async (request, response) => { 'Authorization': getBasicAuthHeader(request.body.auth), }, }); - const data = await result.json(); - return data; + return await result.json(); } const url = new URL(request.body.url); @@ -274,7 +271,7 @@ router.post('/set-model', jsonParser, async (request, response) => { const progress = progressState['progress']; const jobCount = progressState['state']['job_count']; - if (progress == 0.0 && jobCount === 0) { + if (progress === 0.0 && jobCount === 0) { break; } @@ -412,8 +409,18 @@ comfy.post('/models', jsonParser, async (request, response) => { } /** @type {any} */ const data = await result.json(); - return response.send(data.CheckpointLoaderSimple.input.required.ckpt_name[0].map(it => ({ value: it, text: it }))); - } catch (error) { + + const ckpts = data.CheckpointLoaderSimple.input.required.ckpt_name[0].map(it => ({ value: it, text: it })) || []; + + // load list of GGUF unets from diffusion_models if the loader node is available + const ggufs = data.UnetLoaderGGUF?.input.required.unet_name[0].map(it => ({ value: it, text: `GGUF: ${it}` })) || []; + const models = ckpts.concat(ggufs); + + // make the display names of the models somewhat presentable + models.forEach(it => it.text = it.text.replace(/\.[^.]*$/, '').replace(/_/g, ' ')); + + return response.send(models); + } catch (error) { console.log(error); return response.sendStatus(500); } @@ -550,7 +557,13 @@ comfy.post('/generate', jsonParser, async (request, response) => { await delay(100); } if (item.status.status_str === 'error') { - throw new Error('ComfyUI generation did not succeed.'); + // Report node tracebacks if available + const errorMessages = item.status?.messages + ?.filter(it => it[0] === 'execution_error') + .map(it => it[1]) + .map(it => `${it.node_type} [${it.node_id}] ${it.exception_type}: ${it.exception_message}`) + .join('\n') || ''; + throw new Error(`ComfyUI generation did not succeed.\n\n${errorMessages}`.trim()); } const imgInfo = Object.keys(item.outputs).map(it => item.outputs[it].images).flat()[0]; const imgUrl = new URL(request.body.url); @@ -563,8 +576,9 @@ comfy.post('/generate', jsonParser, async (request, response) => { const imgBuffer = await imgResponse.buffer(); return response.send(imgBuffer.toString('base64')); } catch (error) { - console.log(error); - return response.sendStatus(500); + console.log('ComfyUI error:', error); + response.status(500).send(`${error.message}`); + return response; } }); From 8927de45dd0bd437c9ff4824d6fff78e4bc31334 Mon Sep 17 00:00:00 2001 From: ceruleandeep Date: Mon, 18 Nov 2024 18:30:31 +1100 Subject: [PATCH 2/6] Convert data URL to plain base64-encoded image data --- public/scripts/extensions/stable-diffusion/index.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index a176f2e6e..2283d219f 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -3283,7 +3283,8 @@ async function generateComfyImage(prompt, negativePrompt, signal) { const response = await fetch(getUserAvatarUrl()); if (response.ok) { const avatarBlob = await response.blob(); - const avatarBase64 = await getBase64Async(avatarBlob); + const avatarBase64DataUrl = await getBase64Async(avatarBlob); + const avatarBase64 = avatarBase64DataUrl.split(',')[1]; workflow = workflow.replaceAll('"%user_avatar%"', JSON.stringify(avatarBase64)); } else { workflow = workflow.replaceAll('"%user_avatar%"', JSON.stringify(PNG_PIXEL)); @@ -3293,7 +3294,8 @@ async function generateComfyImage(prompt, negativePrompt, signal) { const response = await fetch(getCharacterAvatarUrl()); if (response.ok) { const avatarBlob = await response.blob(); - const avatarBase64 = await getBase64Async(avatarBlob); + const avatarBase64DataUrl = await getBase64Async(avatarBlob); + const avatarBase64 = avatarBase64DataUrl.split(',')[1]; workflow = workflow.replaceAll('"%char_avatar%"', JSON.stringify(avatarBase64)); } else { workflow = workflow.replaceAll('"%char_avatar%"', JSON.stringify(PNG_PIXEL)); From b46dae0d6e54b44b4bcea55db51de2f4e0836187 Mon Sep 17 00:00:00 2001 From: ceruleandeep Date: Mon, 18 Nov 2024 18:49:38 +1100 Subject: [PATCH 3/6] Add sample workflow for ComfyUI img2img --- .../content/Char_Avatar_Comfy_Workflow.json | 137 ++++++++++++++++++ default/content/index.json | 4 + 2 files changed, 141 insertions(+) create mode 100644 default/content/Char_Avatar_Comfy_Workflow.json diff --git a/default/content/Char_Avatar_Comfy_Workflow.json b/default/content/Char_Avatar_Comfy_Workflow.json new file mode 100644 index 000000000..c40bd8fb8 --- /dev/null +++ b/default/content/Char_Avatar_Comfy_Workflow.json @@ -0,0 +1,137 @@ +{ + "3": { + "inputs": { + "seed": "%seed%", + "steps": "%steps%", + "cfg": "%scale%", + "sampler_name": "%sampler%", + "scheduler": "%scheduler%", + "denoise": "%denoise%", + "model": [ + "4", + 0 + ], + "positive": [ + "6", + 0 + ], + "negative": [ + "7", + 0 + ], + "latent_image": [ + "12", + 0 + ] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "4": { + "inputs": { + "ckpt_name": "%model%" + }, + "class_type": "CheckpointLoaderSimple", + "_meta": { + "title": "Load Checkpoint" + } + }, + "6": { + "inputs": { + "text": "%prompt%", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "7": { + "inputs": { + "text": "%negative_prompt%", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Negative Prompt)" + } + }, + "8": { + "inputs": { + "samples": [ + "3", + 0 + ], + "vae": [ + "4", + 2 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "9": { + "inputs": { + "filename_prefix": "SillyTavern", + "images": [ + "8", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "10": { + "inputs": { + "image": "%char_avatar%" + }, + "class_type": "ETN_LoadImageBase64", + "_meta": { + "title": "Load Image (Base64) [https://github.com/Acly/comfyui-tooling-nodes]" + } + }, + "12": { + "inputs": { + "pixels": [ + "13", + 0 + ], + "vae": [ + "4", + 2 + ] + }, + "class_type": "VAEEncode", + "_meta": { + "title": "VAE Encode" + } + }, + "13": { + "inputs": { + "upscale_method": "bicubic", + "width": "%width%", + "height": "%height%", + "crop": "center", + "image": [ + "10", + 0 + ] + }, + "class_type": "ImageScale", + "_meta": { + "title": "Upscale Image" + } + } +} diff --git a/default/content/index.json b/default/content/index.json index 8c2ca66be..24e73e5c9 100644 --- a/default/content/index.json +++ b/default/content/index.json @@ -135,6 +135,10 @@ "filename": "Default_Comfy_Workflow.json", "type": "workflow" }, + { + "filename": "Char_Avatar_Comfy_Workflow.json", + "type": "workflow" + }, { "filename": "presets/kobold/Ace of Spades.json", "type": "kobold_preset" From bcf127e38717f93569677034bc69f0377c3fa9ee Mon Sep 17 00:00:00 2001 From: ceruleandeep Date: Mon, 18 Nov 2024 22:17:19 +1100 Subject: [PATCH 4/6] Add diffusion unets to ComfyUI models list rm string interpolation --- src/endpoints/stable-diffusion.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index 3679d9ea1..567dadf13 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -411,10 +411,11 @@ comfy.post('/models', jsonParser, async (request, response) => { const data = await result.json(); const ckpts = data.CheckpointLoaderSimple.input.required.ckpt_name[0].map(it => ({ value: it, text: it })) || []; + const unets = data.UNETLoader.input.required.unet_name[0].map(it => ({ value: it, text: `UNet: ${it}` })) || []; // load list of GGUF unets from diffusion_models if the loader node is available const ggufs = data.UnetLoaderGGUF?.input.required.unet_name[0].map(it => ({ value: it, text: `GGUF: ${it}` })) || []; - const models = ckpts.concat(ggufs); + const models = [...ckpts, ...unets, ...ggufs]; // make the display names of the models somewhat presentable models.forEach(it => it.text = it.text.replace(/\.[^.]*$/, '').replace(/_/g, ' ')); @@ -577,7 +578,7 @@ comfy.post('/generate', jsonParser, async (request, response) => { return response.send(imgBuffer.toString('base64')); } catch (error) { console.log('ComfyUI error:', error); - response.status(500).send(`${error.message}`); + response.status(500).send(error.message); return response; } }); From b2b5c1f10cb04054c60e1337052a3252fb4d8568 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 18 Nov 2024 20:35:48 +0200 Subject: [PATCH 5/6] ComfyUI: Log error details on prompt submission --- src/endpoints/stable-diffusion.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index 567dadf13..5151d3eeb 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -7,7 +7,7 @@ import sanitize from 'sanitize-filename'; import { sync as writeFileAtomicSync } from 'write-file-atomic'; import FormData from 'form-data'; -import { delay, getBasicAuthHeader } from '../util.js'; +import { delay, getBasicAuthHeader, tryParse } from '../util.js'; import { jsonParser } from '../express-common.js'; import { readSecret, SECRET_KEYS } from './secrets.js'; @@ -535,7 +535,8 @@ comfy.post('/generate', jsonParser, async (request, response) => { body: request.body.prompt, }); if (!promptResult.ok) { - throw new Error('ComfyUI returned an error.'); + const text = await promptResult.text(); + throw new Error('ComfyUI returned an error.', { cause: tryParse(text) }); } /** @type {any} */ From 5478b37b22f1469730c7833c89f6e943f80d21ae Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 18 Nov 2024 20:39:44 +0200 Subject: [PATCH 6/6] Don't use deprecated fetch method. --- src/endpoints/stable-diffusion.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index 5151d3eeb..479ed97fd 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -575,8 +575,8 @@ comfy.post('/generate', jsonParser, async (request, response) => { if (!imgResponse.ok) { throw new Error('ComfyUI returned an error.'); } - const imgBuffer = await imgResponse.buffer(); - return response.send(imgBuffer.toString('base64')); + const imgBuffer = await imgResponse.arrayBuffer(); + return response.send(Buffer.from(imgBuffer).toString('base64')); } catch (error) { console.log('ComfyUI error:', error); response.status(500).send(error.message);