From 2aeaf43c2895ef1dcebb0e0b6822bb93a923cd2d Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 29 May 2024 02:14:08 +0300 Subject: [PATCH] Add CLIP skip control for SD --- .../stable-diffusion/comfyWorkflowEditor.html | 1 + .../extensions/stable-diffusion/index.js | 26 +++++++++++++++++-- .../extensions/stable-diffusion/settings.html | 4 +++ src/endpoints/horde.js | 1 + 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html b/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html index b510f29d9..2427fa6fb 100644 --- a/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html +++ b/public/scripts/extensions/stable-diffusion/comfyWorkflowEditor.html @@ -17,6 +17,7 @@
  • "%scheduler%"
  • "%steps%"
  • "%scale%"
    • +
  • "%clip_skip%"
  • "%width%"
  • "%height%"
  • "%user_avatar%"
    • diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index db7f3b92f..4fae374e5 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -229,6 +229,12 @@ const defaultSettings = { hr_second_pass_steps_max: 150, hr_second_pass_steps_step: 1, + // CLIP skip + clip_skip_min: 1, + clip_skip_max: 12, + clip_skip_step: 1, + clip_skip: 1, + // NovelAI settings novel_upscale_ratio_min: 1.0, novel_upscale_ratio_max: 4.0, @@ -404,6 +410,8 @@ async function loadSettings() { $('#sd_comfy_url').val(extension_settings.sd.comfy_url); $('#sd_comfy_prompt').val(extension_settings.sd.comfy_prompt); $('#sd_snap').prop('checked', extension_settings.sd.snap); + $('#sd_clip_skip').val(extension_settings.sd.clip_skip); + $('#sd_clip_skip_value').text(extension_settings.sd.clip_skip); for (const style of extension_settings.sd.styles) { const option = document.createElement('option'); @@ -465,7 +473,7 @@ function addPromptTemplates() { const label = $('') .text(modeLabels[name]) .attr('for', `sd_prompt_${name}`) - .attr('data-i18n', `sd_prompt_${name}`); + .attr('data-i18n', `sd_prompt_${name}`); const textarea = $('') .addClass('textarea_compact text_pole') .attr('id', `sd_prompt_${name}`) @@ -477,7 +485,7 @@ function addPromptTemplates() { const button = $('') .addClass('menu_button fa-solid fa-undo') .attr('title', 'Restore default') - .attr('data-i18n', 'Restore default') + .attr('data-i18n', 'Restore default') .on('click', () => { textarea.val(promptTemplates[name]); extension_settings.sd.prompts[name] = promptTemplates[name]; @@ -691,6 +699,12 @@ function onRefineModeInput() { saveSettingsDebounced(); } +function onClipSkipInput() { + extension_settings.sd.clip_skip = Number($('#sd_clip_skip').val()); + $('#sd_clip_skip_value').text(extension_settings.sd.clip_skip); + saveSettingsDebounced(); +} + function onScaleInput() { extension_settings.sd.scale = Number($('#sd_scale').val()); $('#sd_scale_value').text(extension_settings.sd.scale.toFixed(1)); @@ -2371,6 +2385,7 @@ async function generateHordeImage(prompt, negativePrompt) { restore_faces: !!extension_settings.sd.restore_faces, enable_hr: !!extension_settings.sd.enable_hr, sanitize: !!extension_settings.sd.horde_sanitize, + clip_skip: extension_settings.sd.clip_skip, }), }); @@ -2409,6 +2424,10 @@ async function generateAutoImage(prompt, negativePrompt) { hr_scale: extension_settings.sd.hr_scale, denoising_strength: extension_settings.sd.denoising_strength, hr_second_pass_steps: extension_settings.sd.hr_second_pass_steps, + override_settings: { + CLIP_stop_at_last_layers: extension_settings.sd.clip_skip, + }, + override_settings_restore_afterwards: true, // Ensure generated img is saved to disk save_images: true, send_images: true, @@ -2449,6 +2468,7 @@ async function generateDrawthingsImage(prompt, negativePrompt) { restore_faces: !!extension_settings.sd.restore_faces, enable_hr: !!extension_settings.sd.enable_hr, denoising_strength: extension_settings.sd.denoising_strength, + clip_skip: extension_settings.sd.clip_skip, // TODO: advanced API parameters: hr, upscaler }), }); @@ -2633,6 +2653,7 @@ async function generateComfyImage(prompt, negativePrompt) { 'scale', 'width', 'height', + 'clip_skip', ]; const workflowResponse = await fetch('/api/sd/comfy/workflow', { @@ -3144,6 +3165,7 @@ jQuery(async () => { $('#sd_openai_quality').on('change', onOpenAiQualitySelect); $('#sd_multimodal_captioning').on('input', onMultimodalCaptioningInput); $('#sd_snap').on('input', onSnapInput); + $('#sd_clip_skip').on('input', onClipSkipInput); $('.sd_settings .inline-drawer-toggle').on('click', function () { initScrollHeight($('#sd_prompt_prefix')); diff --git a/public/scripts/extensions/stable-diffusion/settings.html b/public/scripts/extensions/stable-diffusion/settings.html index 417e99c5e..6fac4c41a 100644 --- a/public/scripts/extensions/stable-diffusion/settings.html +++ b/public/scripts/extensions/stable-diffusion/settings.html @@ -232,6 +232,10 @@ Hires. Fix +
    + + +
    diff --git a/src/endpoints/horde.js b/src/endpoints/horde.js index 32cb08f7b..565c94f95 100644 --- a/src/endpoints/horde.js +++ b/src/endpoints/horde.js @@ -325,6 +325,7 @@ router.post('/generate-image', jsonParser, async (request, response) => { width: request.body.width, height: request.body.height, karras: Boolean(request.body.karras), + clip_skip: request.body.clip_skip, n: 1, }, r2: false,