From b445d549dbdc79935e2f569f12552e3e709babe8 Mon Sep 17 00:00:00 2001 From: majick <125657+majick@users.noreply.github.com> Date: Fri, 10 Nov 2023 18:49:53 -0800 Subject: [PATCH 01/16] Increase kobold max temp to 4.0 Not only is a higher temp accepted by the back end, but it's just about necessary in order to make use of Minimum P sampling. --- public/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/index.html b/public/index.html index 9c0f396cd..4832476cd 100644 --- a/public/index.html +++ b/public/index.html @@ -715,8 +715,8 @@ Temperature
- - + +
From 303026e01fb193ae250331d551caf52995ef525c Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 11 Nov 2023 14:53:08 +0200 Subject: [PATCH 02/16] Nested bogus folders + back button --- public/index.html | 14 +++- public/script.js | 124 +++++++++++++++++++++++++++-------- public/scripts/filters.js | 27 +++++--- public/scripts/power-user.js | 12 +++- public/scripts/tags.js | 7 +- 5 files changed, 141 insertions(+), 43 deletions(-) diff --git a/public/index.html b/public/index.html index 9c0f396cd..b9a28240f 100644 --- a/public/index.html +++ b/public/index.html @@ -4167,7 +4167,7 @@
- +
@@ -4180,6 +4180,18 @@
+
+
+
+ +
+
+
+ Go back +
+
+
+
diff --git a/public/script.js b/public/script.js index c38d423aa..59aba505a 100644 --- a/public/script.js +++ b/public/script.js @@ -989,16 +989,28 @@ export async function selectCharacterById(id) { } } -function getTagBlock(item) { - const count = Object.values(tag_map).flat().filter(x => x == item.id).length; +function getTagBlock(item, entities) { + let count = 0; + + for (const entity of entities) { + if (entitiesFilter.isElementTagged(entity, item.id)) { + count++; + } + } + const template = $('#bogus_folder_template .bogus_folder_select').clone(); template.attr({ 'tagid': item.id, 'id': `BogusFolder${item.id}` }); - template.find('.avatar').css({'background-color': item.color, 'color': item.color2 }); + template.find('.avatar').css({ 'background-color': item.color, 'color': item.color2 }); template.find('.ch_name').text(item.name); template.find('.bogus_folder_counter').text(count); return template; } +function getBackBlock() { + const template = $('#bogus_folder_back_template .bogus_folder_select').clone(); + return template; +} + function getEmptyBlock() { const icons = ['fa-dragon', 'fa-otter', 'fa-kiwi-bird', 'fa-crow', 'fa-frog']; const texts = ['Here be dragons', 'Otterly empty', 'Kiwibunga', 'Pump-a-Rum', 'Croak it']; @@ -1060,10 +1072,9 @@ async function printCharacters(fullRefresh = false) { saveCharactersPage = 0; printTagFilters(tag_filter_types.character); printTagFilters(tag_filter_types.group_member); - const isBogusFolderOpen = !!entitiesFilter.getFilterData(FILTER_TYPES.TAG)?.bogus; // Return to main list - if (isBogusFolderOpen) { + if (isBogusFolderOpen()) { entitiesFilter.setFilterData(FILTER_TYPES.TAG, { excluded: [], selected: [] }); } @@ -1072,8 +1083,11 @@ async function printCharacters(fullRefresh = false) { } const storageKey = 'Characters_PerPage'; + const listId = '#rm_print_characters_block'; + const entities = getEntitiesList({ doFilter: true }); + $("#rm_print_characters_pagination").pagination({ - dataSource: getEntitiesList({ doFilter: true }), + dataSource: entities, pageSize: Number(localStorage.getItem(storageKey)) || per_page_default, sizeChangerOptions: [10, 25, 50, 100, 250, 500, 1000], pageRange: 1, @@ -1086,20 +1100,25 @@ async function printCharacters(fullRefresh = false) { formatNavigator: PAGINATION_TEMPLATE, showNavigator: true, callback: function (data) { - $("#rm_print_characters_block").empty(); - for (const i of data) { - if (i.type === 'character') { - $("#rm_print_characters_block").append(getCharacterBlock(i.item, i.id)); - } - if (i.type === 'group') { - $("#rm_print_characters_block").append(getGroupBlock(i.item)); - } - if (i.type === 'tag') { - $("#rm_print_characters_block").append(getTagBlock(i.item)); - } + $(listId).empty(); + if (isBogusFolderOpen()) { + $(listId).append(getBackBlock()); } if (!data.length) { - $("#rm_print_characters_block").append(getEmptyBlock()); + $(listId).append(getEmptyBlock()); + } + for (const i of data) { + switch (i.type) { + case 'character': + $(listId).append(getCharacterBlock(i.item, i.id)); + break; + case 'group': + $(listId).append(getGroupBlock(i.item)); + break; + case 'tag': + $(listId).append(getTagBlock(i.item, entities)); + break; + } } eventSource.emit(event_types.CHARACTER_PAGE_LOADED); }, @@ -1110,26 +1129,60 @@ async function printCharacters(fullRefresh = false) { saveCharactersPage = e; }, afterRender: function () { - $('#rm_print_characters_block').scrollTop(0); + $(listId).scrollTop(0); }, }); favsToHotswap(); } -export function getEntitiesList({ doFilter } = {}) { - let entities = []; - entities.push(...characters.map((item, index) => ({ item, id: index, type: 'character' }))); - entities.push(...groups.map((item) => ({ item, id: item.id, type: 'group' }))); +/** + * Indicates whether a user is currently in a bogus folder. + * @returns {boolean} If currently viewing a folder + */ +function isBogusFolderOpen() { + return !!entitiesFilter.getFilterData(FILTER_TYPES.TAG)?.bogus; +} - if (power_user.bogus_folders) { - entities.push(...tags.map((item) => ({ item, id: item.id, type: 'tag' }))); +export function getEntitiesList({ doFilter } = {}) { + function characterToEntity(character, id) { + return { item: character, id, type: 'character' }; } + function groupToEntity(group) { + return { item: group, id: group.id, type: 'group' }; + } + + function tagToEntity(tag) { + return { item: structuredClone(tag), id: tag.id, type: 'tag' }; + } + + let entities = [ + ...characters.map((item, index) => characterToEntity(item, index)), + ...groups.map(item => groupToEntity(item)), + ...(power_user.bogus_folders ? tags.map(item => tagToEntity(item)) : []), + ]; + if (doFilter) { entities = entitiesFilter.applyFilters(entities); } + if (isBogusFolderOpen()) { + // Get tags of entities within the bogus folder + const filterData = structuredClone(entitiesFilter.getFilterData(FILTER_TYPES.TAG)); + entities = entities.filter(x => x.type !== 'tag'); + const otherTags = tags.filter(x => !filterData.selected.includes(x.id)); + const bogusTags = []; + for (const entity of entities) { + for (const tag of otherTags) { + if (!bogusTags.includes(tag) && entitiesFilter.isElementTagged(entity, tag.id)) { + bogusTags.push(tag); + } + } + } + entities.push(...bogusTags.map(item => tagToEntity(item))); + } + sortEntitiesList(entities); return entities; } @@ -7523,8 +7576,25 @@ jQuery(async function () { $(document).on("click", ".bogus_folder_select", function () { const tagId = $(this).attr('tagid'); console.log('Bogus folder clicked', tagId); - entitiesFilter.setFilterData(FILTER_TYPES.TAG, { excluded: [], selected: [tagId], bogus: true, }); - }) + + const filterData = structuredClone(entitiesFilter.getFilterData(FILTER_TYPES.TAG)); + + if (!Array.isArray(filterData.selected)) { + filterData.selected = []; + filterData.excluded = []; + filterData.bogus = false; + } + + if (tagId === 'back') { + filterData.selected.pop(); + filterData.bogus = filterData.selected.length > 0; + } else { + filterData.selected.push(tagId); + filterData.bogus = true; + } + + entitiesFilter.setFilterData(FILTER_TYPES.TAG, filterData); + }); $(document).on("input", ".edit_textarea", function () { scroll_holder = $("#chat").scrollTop(); diff --git a/public/scripts/filters.js b/public/scripts/filters.js index f382845cc..ca2e5a103 100644 --- a/public/scripts/filters.js +++ b/public/scripts/filters.js @@ -69,6 +69,20 @@ export class FilterHelper { return data.filter(entity => fuzzySearchResults.includes(entity.uid)); } + /** + * Checks if the given entity is tagged with the given tag ID. + * @param {object} entity Searchable entity + * @param {string} tagId Tag ID to check + * @returns {boolean} Whether the entity is tagged with the given tag ID + */ + isElementTagged(entity, tagId) { + const isCharacter = entity.type === 'character'; + const lookupValue = isCharacter ? entity.item.avatar : String(entity.id); + const isTagged = Array.isArray(tag_map[lookupValue]) && tag_map[lookupValue].includes(tagId); + + return isTagged; + } + /** * Applies a tag filter to the data. * @param {any[]} data The data to filter. @@ -82,19 +96,12 @@ export class FilterHelper { return data; } - function isElementTagged(entity, tagId) { - const isCharacter = entity.type === 'character'; - const lookupValue = isCharacter ? entity.item.avatar : String(entity.id); - const isTagged = Array.isArray(tag_map[lookupValue]) && tag_map[lookupValue].includes(tagId); - return isTagged; - } - - function getIsTagged(entity) { - const tagFlags = selected.map(tagId => isElementTagged(entity, tagId)); + const getIsTagged = (entity) => { + const tagFlags = selected.map(tagId => this.isElementTagged(entity, tagId)); const trueFlags = tagFlags.filter(x => x); const isTagged = TAG_LOGIC_AND ? tagFlags.length === trueFlags.length : trueFlags.length > 0; - const excludedTagFlags = excluded.map(tagId => isElementTagged(entity, tagId)); + const excludedTagFlags = excluded.map(tagId => this.isElementTagged(entity, tagId)); const isExcluded = excludedTagFlags.includes(true); if (isExcluded) { diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index efc4c9e14..51f1e1c09 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -1649,7 +1649,17 @@ function sortEntitiesList(entities) { return; } - entities.sort((a, b) => sortFunc(a.item, b.item)); + entities.sort((a, b) => { + if (a.type === 'tag' && b.type !== 'tag') { + return -1; + } + + if (a.type !== 'tag' && b.type === 'tag') { + return 1; + } + + return sortFunc(a.item, b.item); + }); } async function saveTheme() { diff --git a/public/scripts/tags.js b/public/scripts/tags.js index 02425f644..a746bb7b7 100644 --- a/public/scripts/tags.js +++ b/public/scripts/tags.js @@ -38,14 +38,13 @@ export const tag_filter_types = { }; const ACTIONABLE_TAGS = { - FAV: { id: 1, name: 'Show only favorites', color: 'rgba(255, 255, 0, 0.5)', action: applyFavFilter, icon: 'fa-solid fa-star', class: 'filterByFavorites' }, GROUP: { id: 0, name: 'Show only groups', color: 'rgba(100, 100, 100, 0.5)', action: filterByGroups, icon: 'fa-solid fa-users', class: 'filterByGroups' }, + VIEW: { id: 2, name: 'Manage tags', color: 'rgba(150, 100, 100, 0.5)', action: onViewTagsListClick, icon: 'fa-solid fa-gear', class: 'manageTags' }, HINT: { id: 3, name: 'Show Tag List', color: 'rgba(150, 100, 100, 0.5)', action: onTagListHintClick, icon: 'fa-solid fa-tags', class: 'showTagList' }, } const InListActionable = { - VIEW: { id: 2, name: 'Manage tags', color: 'rgba(150, 100, 100, 0.5)', action: onViewTagsListClick, icon: 'fa-solid fa-gear' }, } const DEFAULT_TAGS = [ @@ -321,9 +320,9 @@ function appendTagToList(listElement, tag, { removable, selectable, action, isGe tagElement.on('click', () => action.bind(tagElement)(filter)); tagElement.addClass('actionable'); } - if (action && tag.id === 2) { + /*if (action && tag.id === 2) { tagElement.addClass('innerActionable hidden'); - } + }*/ $(listElement).append(tagElement); } From a02504381a5c1e133dd793d040a4ff69819d6439 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 11 Nov 2023 16:12:02 +0200 Subject: [PATCH 03/16] Forbid trigger command while the group is generating --- public/scripts/slash-commands.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index bbc782c70..b1af99469 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -26,7 +26,7 @@ import { setCharacterName, } from "../script.js"; import { getMessageTimeStamp } from "./RossAscends-mods.js"; -import { groups, resetSelectedGroup, selected_group } from "./group-chats.js"; +import { groups, is_group_generating, resetSelectedGroup, selected_group } from "./group-chats.js"; import { getRegexedString, regex_placement } from "./extensions/regex/engine.js"; import { chat_styles, power_user } from "./power-user.js"; import { autoSelectPersona } from "./personas.js"; @@ -270,7 +270,12 @@ async function unhideMessageCallback(_, arg) { async function triggerGroupMessageCallback(_, arg) { if (!selected_group) { - toastr.warning("Cannot run this command outside of a group chat."); + toastr.warning("Cannot run trigger command outside of a group chat."); + return; + } + + if (is_group_generating) { + toastr.warning("Cannot run trigger command while the group reply is generating."); return; } From 28bb5db04f369359be5836a4fd8d065ececcc4a2 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 11 Nov 2023 16:21:20 +0200 Subject: [PATCH 04/16] Add new settings to default/settings.json. --- default/settings.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/default/settings.json b/default/settings.json index fd4edfda7..f8c4b54bf 100644 --- a/default/settings.json +++ b/default/settings.json @@ -163,6 +163,8 @@ "custom_stopping_strings_macro": true, "fuzzy_search": true, "encode_tags": false, + "enableLabMode": false, + "enableZenSliders": false, "ui_mode": 1 }, "extension_settings": { From 8a8880fca12f0e4edf656defdd8e759b6121cf49 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 11 Nov 2023 16:31:13 +0200 Subject: [PATCH 05/16] Visual touch-up --- public/css/tags.css | 2 +- public/scripts/RossAscends-mods.js | 2 +- public/scripts/templates/macros.html | 46 ++++++++++++++-------------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/public/css/tags.css b/public/css/tags.css index 788e7dc05..3ad18c468 100644 --- a/public/css/tags.css +++ b/public/css/tags.css @@ -13,7 +13,7 @@ .tag_view_item { display: flex; flex-direction: row; - align-items: baseline; + align-items: center; gap: 10px; margin-bottom: 5px; } diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index b40e5f85a..35f7b1c2c 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -371,7 +371,7 @@ function RA_checkOnlineStatus() { connection_made = false; } else { if (online_status !== undefined && online_status !== "no_connection") { - $("#send_textarea").attr("placeholder", `Type a message, or /? for command list`); //on connect, placeholder tells user to type message + $("#send_textarea").attr("placeholder", `Type a message, or /? for help`); //on connect, placeholder tells user to type message $('#send_form').removeClass("no-connection"); $("#API-status-top").removeClass("fa-plug-circle-exclamation redOverlayGlow"); $("#API-status-top").addClass("fa-plug"); diff --git a/public/scripts/templates/macros.html b/public/scripts/templates/macros.html index bd58ae1fe..dc2844439 100644 --- a/public/scripts/templates/macros.html +++ b/public/scripts/templates/macros.html @@ -1,26 +1,26 @@ System-wide Replacement Macros (in order of evaluation):
    -
  • {{original}} - global prompts defined in API settings. Only valid in Advanced Definitions prompt overrides.
    • -
  • {{input}} - the user input
    • -
  • {{description}} - the Character's Description
    • -
  • {{personality}} - the Character's Personality
    • -
  • {{scenario}} - the Character's Scenario
    • -
  • {{persona}} - your current Persona Description
    • -
  • {{mesExamples}} - the Character's Dialogue Examples
    • -
  • {{user}} - your current Persona username
    • -
  • {{char}} - the Character's name
    • -
  • {{lastMessageId}} - index # of the latest chat message. Useful for slash command batching.
    • -
  • {{// (note)}} - you can leave a note here, and the macro will be replaced with blank content. Not visible for the AI.
    • -
  • {{time}} - the current time
    • -
  • {{date}} - the current date
    • -
  • {{weekday}} - the current weekday
    • -
  • {{isotime}} - the current ISO date (YYYY-MM-DD)
    • -
  • {{isodate}} - the current ISO time (24-hour clock)
    • -
  • {{datetimeformat …}} - the current date/time in the specified format, e. g. for German date/time: {{datetimeformat DD.MM.YYYY HH:mm}}
    • -
  • {{time_UTC±#}} - the current time in the specified UTC time zone offset, e.g. UTC-4 or UTC+2
    • -
  • {{idle_duration}} - the time since the last user message was sent
    • -
  • {{bias "text here"}} - sets a behavioral bias for the AI until the next user input. Quotes around the text are important.
    • -
  • {{random:(args)}} - returns a random item from the list. (ex: {{random:1,2,3,4}} will return 1 of the 4 numbers at random. Works with text lists too.
    • -
  • {{roll:(formula)}} - rolls a dice. (ex: {{roll:1d6}} will roll a 6-sided dice and return a number between 1 and 6)
    • -
  • {{banned "text here"}} - dynamically add text in the quotes to banned words sequences, if Text Generation WebUI backend used. Do nothing for others backends. Can be used anywhere (Character description, WI, AN, etc.) Quotes around the text are important.
    • +
  • {{original}} – global prompts defined in API settings. Only valid in Advanced Definitions prompt overrides.
    • +
  • {{input}} – the user input
    • +
  • {{description}} – the Character's Description
    • +
  • {{personality}} – the Character's Personality
    • +
  • {{scenario}} – the Character's Scenario
    • +
  • {{persona}} – your current Persona Description
    • +
  • {{mesExamples}} – the Character's Dialogue Examples
    • +
  • {{user}} – your current Persona username
    • +
  • {{char}} – the Character's name
    • +
  • {{lastMessageId}} – index # of the latest chat message. Useful for slash command batching.
    • +
  • {{// (note)}} – you can leave a note here, and the macro will be replaced with blank content. Not visible for the AI.
    • +
  • {{time}} – the current time
    • +
  • {{date}} – the current date
    • +
  • {{weekday}} – the current weekday
    • +
  • {{isotime}} – the current ISO date (YYYY-MM-DD)
    • +
  • {{isodate}} – the current ISO time (24-hour clock)
    • +
  • {{datetimeformat …}} – the current date/time in the specified format, e. g. for German date/time: {{datetimeformat DD.MM.YYYY HH:mm}}
    • +
  • {{time_UTC±#}} – the current time in the specified UTC time zone offset, e.g. UTC-4 or UTC+2
    • +
  • {{idle_duration}} – the time since the last user message was sent
    • +
  • {{bias "text here"}} – sets a behavioral bias for the AI until the next user input. Quotes around the text are important.
    • +
  • {{random:(args)}} – returns a random item from the list. (ex: {{random:1,2,3,4}} will return 1 of the 4 numbers at random. Works with text lists too.
    • +
  • {{roll:(formula)}} – rolls a dice. (ex: {{roll:1d6}} will roll a 6- sided dice and return a number between 1 and 6)
    • +
  • {{banned "text here"}} – dynamically add text in the quotes to banned words sequences, if Text Generation WebUI backend used. Do nothing for others backends. Can be used anywhere (Character description, WI, AN, etc.) Quotes around the text are important.
From f1d0e39d39e868a4129de1e34aa3605ed658bb91 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 11 Nov 2023 16:39:54 +0200 Subject: [PATCH 06/16] Require a name for dummy personas --- public/index.html | 2 +- public/scripts/personas.js | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/public/index.html b/public/index.html index 575d7d2e5..205ae4691 100644 --- a/public/index.html +++ b/public/index.html @@ -3088,7 +3088,7 @@
diff --git a/public/scripts/personas.js b/public/scripts/personas.js index fc91e8008..8a3f0341d 100644 --- a/public/scripts/personas.js +++ b/public/scripts/personas.js @@ -39,7 +39,23 @@ async function uploadUserAvatar(url, name) { } async function createDummyPersona() { - await uploadUserAvatar(default_avatar); + const personaName = await callPopup('

Enter a name for this persona:

', 'input', ''); + + if (!personaName) { + console.debug('User cancelled creating dummy persona'); + return; + } + + // Date + name (only ASCII) to make it unique + const avatarId = `${Date.now()}-${personaName.replace(/[^a-zA-Z0-9]/g, '')}.png`; + power_user.personas[avatarId] = personaName; + power_user.persona_descriptions[avatarId] = { + description: '', + position: persona_description_positions.IN_PROMPT, + }; + + await uploadUserAvatar(default_avatar, avatarId); + saveSettingsDebounced(); } export async function convertCharacterToPersona(characterId = null) { From 91a1cc81a0b5708d7ab76cc41a624091127cb42a Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sat, 11 Nov 2023 18:25:43 +0200 Subject: [PATCH 07/16] #1242 Add aux field selector --- public/index.html | 9 ++++++++- public/script.js | 7 ++++--- public/scripts/power-user.js | 9 +++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/public/index.html b/public/index.html index 9c0f396cd..909af8577 100644 --- a/public/index.html +++ b/public/index.html @@ -2680,7 +2680,7 @@
-
+

Theme Toggles

Miscellaneous

+
+ + +
diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 7eee1fc8c..9b9a6670e 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -587,7 +587,7 @@ function getWorldEntry(name, data, entry) { setOriginalDataValue(data, uid, "keys", data.entries[uid].key); saveWorldInfo(name, data); }); - keyInput.val(entry.key.join(",")).trigger("input"); + keyInput.val(entry.key.join(", ")).trigger("input"); //initScrollHeight(keyInput); // logic AND/NOT @@ -708,7 +708,7 @@ function getWorldEntry(name, data, entry) { saveWorldInfo(name, data); }); - keySecondaryInput.val(entry.keysecondary.join(",")).trigger("input"); + keySecondaryInput.val(entry.keysecondary.join(", ")).trigger("input"); initScrollHeight(keySecondaryInput); // comment From ab5b0cb1db4d65ff78e90672fa70e3b73a646d0c Mon Sep 17 00:00:00 2001 From: ThisIsPIRI Date: Sun, 12 Nov 2023 06:32:04 +0900 Subject: [PATCH 11/16] Fix context size counter when size is unlocked --- public/scripts/power-user.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index 0f24fe0e0..ab019e67c 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -1357,7 +1357,7 @@ function loadMaxContextUnlocked() { } function switchMaxContextSize() { - const elements = [$('#max_context'), $('#rep_pen_range'), $('#rep_pen_range_textgenerationwebui')]; + const elements = [$('#max_context'), $('#max_context_counter'), $('#rep_pen_range'), $('#rep_pen_range_textgenerationwebui')]; const maxValue = power_user.max_context_unlocked ? MAX_CONTEXT_UNLOCKED : MAX_CONTEXT_DEFAULT; const minValue = power_user.max_context_unlocked ? maxContextMin : maxContextMin; const steps = power_user.max_context_unlocked ? unlockedMaxContextStep : maxContextStep; @@ -1366,7 +1366,7 @@ function switchMaxContextSize() { element.attr('max', maxValue); element.attr('step', steps); - if (element.attr('id') == 'max_context') { + if (element.attr('id').indexOf('max_context') !== -1) { element.attr('min', minValue); } const value = Number(element.val()); From 2c4f53e7b5a79c824f49a26e99ebab331419e1a3 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 12 Nov 2023 00:09:48 +0200 Subject: [PATCH 12/16] Add native GPT-4V image inlining --- public/css/st-tailwind.css | 8 ++ public/css/toggle-dependent.css | 8 ++ public/index.html | 13 +++ public/script.js | 2 +- public/scripts/extensions/caption/index.js | 91 ++++++++++++++- public/scripts/openai.js | 124 ++++++++++++++++++--- 6 files changed, 229 insertions(+), 17 deletions(-) diff --git a/public/css/st-tailwind.css b/public/css/st-tailwind.css index 701d03890..d86d53f49 100644 --- a/public/css/st-tailwind.css +++ b/public/css/st-tailwind.css @@ -229,6 +229,10 @@ display: flex; } +.flexBasis100p { + flex-basis: 100%; +} + .flexBasis50p { flex-basis: 50% } @@ -263,6 +267,10 @@ flex-shrink: 1 } +.flexWrap { + flex-wrap: wrap; +} + .flexnowrap { flex-wrap: nowrap; } diff --git a/public/css/toggle-dependent.css b/public/css/toggle-dependent.css index 4bf01922e..692ebd233 100644 --- a/public/css/toggle-dependent.css +++ b/public/css/toggle-dependent.css @@ -358,3 +358,11 @@ body.expandMessageActions .mes .mes_buttons .extraMesButtons { body.expandMessageActions .mes .mes_buttons .extraMesButtonsHint { display: none !important; } + +#openai_image_inlining:not(:checked) ~ #image_inlining_hint { + display: none; +} + +#openai_image_inlining:checked ~ #image_inlining_hint { + display: block; +} diff --git a/public/index.html b/public/index.html index 25fec85c5..3188f7e12 100644 --- a/public/index.html +++ b/public/index.html @@ -1723,6 +1723,7 @@ + @@ -1745,6 +1746,17 @@ Show "External" models (provided by API) +
@@ -4057,6 +4069,7 @@
+
diff --git a/public/script.js b/public/script.js index 4d7e5d0d3..914613071 100644 --- a/public/script.js +++ b/public/script.js @@ -3409,7 +3409,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject, generate_data = getNovelGenerationData(finalPrompt, presetSettings, maxLength, isImpersonate, cfgValues); } else if (main_api == 'openai') { - let [prompt, counts] = prepareOpenAIMessages({ + let [prompt, counts] = await prepareOpenAIMessages({ name2: name2, charDescription: description, charPersonality: personality, diff --git a/public/scripts/extensions/caption/index.js b/public/scripts/extensions/caption/index.js index 05f70dff4..4d83d1419 100644 --- a/public/scripts/extensions/caption/index.js +++ b/public/scripts/extensions/caption/index.js @@ -1,8 +1,9 @@ import { getBase64Async, saveBase64AsFile } from "../../utils.js"; import { getContext, getApiUrl, doExtrasFetch, extension_settings, modules } from "../../extensions.js"; -import { callPopup, getRequestHeaders, saveSettingsDebounced, substituteParams } from "../../../script.js"; +import { appendImageToMessage, callPopup, getRequestHeaders, saveSettingsDebounced, substituteParams } from "../../../script.js"; import { getMessageTimeStamp } from "../../RossAscends-mods.js"; import { SECRET_KEYS, secret_state } from "../../secrets.js"; +import { isImageInliningSupported } from "../../openai.js"; export { MODULE_NAME }; const MODULE_NAME = 'caption'; @@ -223,6 +224,83 @@ function onRefineModeInput() { saveSettingsDebounced(); } +async function sendEmbeddedImage(e) { + const file = e.target.files[0]; + + if (!file || !(file instanceof File)) { + return; + } + + try { + const context = getContext(); + const fileData = await getBase64Async(file); + const base64Format = fileData.split(',')[0].split(';')[0].split('/')[1]; + const base64Data = fileData.split(',')[1]; + const caption = await callPopup('

Enter a comment or question (optional)

', 'input', 'What is this?', { okButton: 'Send', rows: 2 }); + const imagePath = await saveBase64AsFile(base64Data, context.name2, '', base64Format); + const message = { + name: context.name1, + is_user: true, + send_date: getMessageTimeStamp(), + mes: caption || `[${context.name1} sends ${context.name2} a picture]`, + extra: { + image: imagePath, + inline_image: !!caption, + title: caption || '', + }, + }; + context.chat.push(message); + context.addOneMessage(message); + await context.generate('caption'); + } + catch (error) { + console.log(error); + } + finally { + e.target.form.reset(); + setImageIcon(); + } +} + +function onImageEmbedClicked() { + const context = getContext(); + const messageElement = $(this).closest('.mes'); + const messageId = messageElement.attr('mesid'); + const message = context.chat[messageId]; + + if (!message) { + console.warn('Failed to find message with id', messageId); + return; + } + + $('#embed_img_file') + .off('change') + .on('change', parseAndUploadEmbed) + .trigger('click'); + + async function parseAndUploadEmbed(e) { + const file = e.target.files[0]; + + if (!file || !(file instanceof File)) { + return; + } + const fileData = await getBase64Async(file); + const base64Data = fileData.split(',')[1]; + const base64Format = fileData.split(',')[0].split(';')[0].split('/')[1]; + const imagePath = await saveBase64AsFile(base64Data, context.name2, '', base64Format); + + if (!message.extra) { + message.extra = {}; + } + + message.extra.image = imagePath; + message.extra.inline_image = true; + message.extra.title = ''; + appendImageToMessage(message, messageElement); + await context.saveChat(); + } +} + jQuery(function () { function addSendPictureButton() { const sendButton = $(` @@ -234,6 +312,12 @@ jQuery(function () { $('#extensionsMenu').prepend(sendButton); $(sendButton).hide(); $(sendButton).on('click', () => { + if (isImageInliningSupported()) { + console.log('Native image inlining is supported. Skipping captioning.'); + $('#embed_img_file').off('change').on('change', sendEmbeddedImage).trigger('click'); + return; + } + const hasCaptionModule = (modules.includes('caption') && extension_settings.caption.source === 'extras') || (extension_settings.caption.source === 'openai' && secret_state[SECRET_KEYS.OPENAI]) || @@ -249,10 +333,12 @@ jQuery(function () { }); } function addPictureSendForm() { - const inputHtml = ``; + const inputHtml = ``; + const embedInputHtml = ``; const imgForm = document.createElement('form'); imgForm.id = 'img_form'; $(imgForm).append(inputHtml); + $(imgForm).append(embedInputHtml); $(imgForm).hide(); $('#form_sheld').append(imgForm); $('#img_file').on('change', onSelectImage); @@ -312,5 +398,6 @@ jQuery(function () { extension_settings.caption.template = String($('#caption_template').val()); saveSettingsDebounced(); }); + $(document).on('click', '.mes_embed', onImageEmbedClicked); setInterval(moduleWorker, UPDATE_INTERVAL); }); diff --git a/public/scripts/openai.js b/public/scripts/openai.js index ec5e712bf..c98860888 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -54,7 +54,9 @@ import { import { delay, download, + getBase64Async, getFileText, getSortableDelay, + isDataURL, parseJsonFile, resetScrollHeight, stringFormat, @@ -70,7 +72,6 @@ export { setOpenAIMessages, setOpenAIMessageExamples, setupChatCompletionPromptManager, - prepareOpenAIMessages, sendOpenAIRequest, getChatCompletionModel, TokenHandler, @@ -221,6 +222,7 @@ const default_settings = { exclude_assistant: false, use_alt_scale: false, squash_system_messages: false, + image_inlining: false, }; const oai_settings = { @@ -267,6 +269,7 @@ const oai_settings = { exclude_assistant: false, use_alt_scale: false, squash_system_messages: false, + image_inlining: false, }; let openai_setting_names; @@ -409,7 +412,8 @@ function setOpenAIMessages(chat) { // Apply the "wrap in quotes" option if (role == 'user' && oai_settings.wrap_in_quotes) content = `"${content}"`; const name = chat[j]['name']; - openai_msgs[i] = { "role": role, "content": content, name: name }; + const image = chat[j]?.extra?.image; + openai_msgs[i] = { "role": role, "content": content, name: name, "image": image }; j++; } } @@ -592,7 +596,7 @@ export function isOpenRouterWithInstruct() { * @param type * @param cyclePrompt */ -function populateChatHistory(prompts, chatCompletion, type = null, cyclePrompt = null) { +async function populateChatHistory(prompts, chatCompletion, type = null, cyclePrompt = null) { chatCompletion.add(new MessageCollection('chatHistory'), prompts.index('chatHistory')); let names = (selected_group && groups.find(x => x.id === selected_group)?.members.map(member => characters.find(c => c.avatar === member)?.name).filter(Boolean).join(', ')) || ''; @@ -629,8 +633,13 @@ function populateChatHistory(prompts, chatCompletion, type = null, cyclePrompt = chatCompletion.insert(message, 'chatHistory'); } + const imageInlining = isImageInliningSupported(); + // Insert chat messages as long as there is budget available - [...openai_msgs].reverse().every((chatPrompt, index) => { + const chatPool = [...openai_msgs].reverse(); + for (let index = 0; index < chatPool.length; index++) { + const chatPrompt = chatPool[index]; + // We do not want to mutate the prompt const prompt = new Prompt(chatPrompt); prompt.identifier = `chatHistory-${openai_msgs.length - index}`; @@ -641,10 +650,16 @@ function populateChatHistory(prompts, chatCompletion, type = null, cyclePrompt = chatMessage.setName(messageName); } - if (chatCompletion.canAfford(chatMessage)) chatCompletion.insertAtStart(chatMessage, 'chatHistory'); - else return false; - return true; - }); + if (imageInlining && chatPrompt.image) { + await chatMessage.addImage(chatPrompt.image); + } + + if (chatCompletion.canAfford(chatMessage)) { + chatCompletion.insertAtStart(chatMessage, 'chatHistory'); + } else { + break; + } + } // Insert and free new chat chatCompletion.freeBudget(newChatMessage); @@ -724,7 +739,7 @@ function getPromptPosition(position) { * @param {string} options.quietPrompt - Instruction prompt for extras * @param {string} options.type - The type of the chat, can be 'impersonate'. */ -function populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, type, cyclePrompt } = {}) { +async function populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, type, cyclePrompt } = {}) { // Helper function for preparing a prompt, that already exists within the prompt collection, for completion const addToChatCompletion = (source, target = null) => { // We need the prompts array to determine a position for the source. @@ -825,9 +840,9 @@ function populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, ty // Decide whether dialogue examples should always be added if (power_user.pin_examples) { populateDialogueExamples(prompts, chatCompletion); - populateChatHistory(prompts, chatCompletion, type, cyclePrompt); + await populateChatHistory(prompts, chatCompletion, type, cyclePrompt); } else { - populateChatHistory(prompts, chatCompletion, type, cyclePrompt); + await populateChatHistory(prompts, chatCompletion, type, cyclePrompt); populateDialogueExamples(prompts, chatCompletion); } @@ -969,7 +984,7 @@ function preparePromptsForChatCompletion({ Scenario, charPersonality, name2, wor * @param dryRun - Whether this is a live call or not. * @returns {(*[]|boolean)[]} An array where the first element is the prepared chat and the second element is a boolean flag. */ -function prepareOpenAIMessages({ +export async function prepareOpenAIMessages({ name2, charDescription, charPersonality, @@ -1012,7 +1027,7 @@ function prepareOpenAIMessages({ }); // Fill the chat completion with as much context as the budget allows - populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, type, cyclePrompt }); + await populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, type, cyclePrompt }); } catch (error) { if (error instanceof TokenBudgetExceededError) { toastr.error('An error occurred while counting tokens: Token budget exceeded.') @@ -1372,6 +1387,16 @@ async function sendOpenAIRequest(type, openai_msgs_tosend, signal) { "stop": getCustomStoppingStrings(openai_max_stop_strings), }; + // Empty array will produce a validation error + if (!Array.isArray(generate_data.stop) || !generate_data.stop.length) { + delete generate_data.stop; + } + + // Vision models don't support logit bias + if (isImageInliningSupported()) { + delete generate_data.logit_bias; + } + // Proxy is only supported for Claude and OpenAI if (oai_settings.reverse_proxy && [chat_completion_sources.CLAUDE, chat_completion_sources.OPENAI].includes(oai_settings.chat_completion_source)) { validateReverseProxy(); @@ -1640,7 +1665,18 @@ class InvalidCharacterNameError extends Error { * Used for creating, managing, and interacting with a specific message object. */ class Message { - tokens; identifier; role; content; name; + static tokensPerImage = 85; + + /** @type {number} */ + tokens; + /** @type {string} */ + identifier; + /** @type {string} */ + role; + /** @type {string|any[]} */ + content; + /** @type {string} */ + name; /** * @constructor @@ -1665,6 +1701,30 @@ class Message { this.tokens = tokenHandler.count({ role: this.role, content: this.content, name: this.name }); } + async addImage(image) { + const textContent = this.content; + const isDataUrl = isDataURL(image); + + if (!isDataUrl) { + try { + const response = await fetch(image, { method: 'GET', cache: 'force-cache' }); + if (!response.ok) throw new Error('Failed to fetch image'); + const blob = await response.blob(); + image = await getBase64Async(blob); + } catch (error) { + console.error('Image adding skipped', error); + return; + } + } + + this.content = [ + { type: "text", text: textContent }, + { type: "image_url", image_url: { "url": image, "detail": "low" } }, + ]; + + this.tokens += Message.tokensPerImage; + } + /** * Create a new Message instance from a prompt. * @static @@ -2148,6 +2208,7 @@ function loadOpenAISettings(data, settings) { oai_settings.show_external_models = settings.show_external_models ?? default_settings.show_external_models; oai_settings.proxy_password = settings.proxy_password ?? default_settings.proxy_password; oai_settings.assistant_prefill = settings.assistant_prefill ?? default_settings.assistant_prefill; + oai_settings.image_inlining = settings.image_inlining ?? default_settings.image_inlining; oai_settings.prompts = settings.prompts ?? default_settings.prompts; oai_settings.prompt_order = settings.prompt_order ?? default_settings.prompt_order; @@ -2168,6 +2229,7 @@ function loadOpenAISettings(data, settings) { $('#api_url_scale').val(oai_settings.api_url_scale); $('#openai_proxy_password').val(oai_settings.proxy_password); $('#claude_assistant_prefill').val(oai_settings.assistant_prefill); + $('#openai_image_inlining').prop('checked', oai_settings.image_inlining); $('#model_openai_select').val(oai_settings.openai_model); $(`#model_openai_select option[value="${oai_settings.openai_model}"`).attr('selected', true); @@ -2388,6 +2450,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) { exclude_assistant: settings.exclude_assistant, use_alt_scale: settings.use_alt_scale, squash_system_messages: settings.squash_system_messages, + image_inlining: settings.image_inlining, }; const savePresetSettings = await fetch(`/api/presets/save-openai?name=${name}`, { @@ -2741,6 +2804,7 @@ function onSettingsPresetChange() { exclude_assistant: ['#exclude_assistant', 'exclude_assistant', true], use_alt_scale: ['#use_alt_scale', 'use_alt_scale', true], squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true], + image_inlining: ['#openai_image_inlining', 'image_inlining', true], }; const presetName = $('#settings_preset_openai').find(":selected").text(); @@ -2785,6 +2849,9 @@ function getMaxContextOpenAI(value) { else if (value.includes('gpt-4-1106')) { return max_128k; } + else if (value.includes('gpt-4-vision')) { + return max_128k; + } else if (value.includes('gpt-3.5-turbo-1106')) { return max_16k; } @@ -2831,6 +2898,9 @@ function getMaxContextWindowAI(value) { else if (value.includes('gpt-4-1106')) { return max_128k; } + else if (value.includes('gpt-4-vision')) { + return max_128k; + } else if (value.includes('gpt-4-32k')) { return max_32k; } @@ -3217,6 +3287,27 @@ function updateScaleForm() { } } +/** + * Check if the model supports image inlining + * @returns {boolean} True if the model supports image inlining + */ +export function isImageInliningSupported() { + const modelId = 'gpt-4-vision'; + + if (!oai_settings.image_inlining) { + return false; + } + + switch (oai_settings.chat_completion_source) { + case chat_completion_sources.OPENAI: + return oai_settings.openai_model.includes(modelId); + case chat_completion_sources.OPENROUTER: + return oai_settings.openrouter_model.includes(modelId); + default: + return false; + } +} + $(document).ready(async function () { $('#test_api_button').on('click', testApiConnection); @@ -3463,6 +3554,11 @@ $(document).ready(async function () { saveSettingsDebounced(); }); + $('#openai_image_inlining').on('input', function () { + oai_settings.image_inlining = !!$(this).prop('checked'); + saveSettingsDebounced(); + }); + $(document).on('input', '#openai_settings .autoSetHeight', function () { resetScrollHeight($(this)); }); From 879502c1e7a35e53079ccc89ce4ade1dac656fa0 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 12 Nov 2023 00:13:30 +0200 Subject: [PATCH 13/16] Only allow inlining if OAI is the selected API --- public/scripts/openai.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index c98860888..7242e2cd3 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -3292,6 +3292,10 @@ function updateScaleForm() { * @returns {boolean} True if the model supports image inlining */ export function isImageInliningSupported() { + if (main_api !== 'openai') { + return false; + } + const modelId = 'gpt-4-vision'; if (!oai_settings.image_inlining) { From a42c1fc581cd6bddaca6b808e4f5a0a6cc1cb4d5 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 12 Nov 2023 01:12:14 +0200 Subject: [PATCH 14/16] Hide MinP under a feature flag. Send Miro to Horde --- public/scripts/kai-settings.js | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/public/scripts/kai-settings.js b/public/scripts/kai-settings.js index f2bc7ef73..8cab041fd 100644 --- a/public/scripts/kai-settings.js +++ b/public/scripts/kai-settings.js @@ -31,6 +31,11 @@ export const kai_settings = { seed: -1, }; +/** + * Stable version of KoboldAI has a nasty payload validation. + * It will reject any payload that has a key that is not in the whitelist. + * @typedef {Object.} kai_flags + */ export const kai_flags = { can_use_tokenization: false, can_use_stop_sequence: false, @@ -38,6 +43,7 @@ export const kai_flags = { can_use_default_badwordsids: false, can_use_mirostat: false, can_use_grammar: false, + can_use_min_p: false, }; const defaultValues = Object.freeze(structuredClone(kai_settings)); @@ -48,6 +54,7 @@ const MIN_STREAMING_KCPPVERSION = '1.30'; const MIN_TOKENIZATION_KCPPVERSION = '1.41'; const MIN_MIROSTAT_KCPPVERSION = '1.35'; const MIN_GRAMMAR_KCPPVERSION = '1.44'; +const MIN_MIN_P_KCPPVERSION = '1.48'; const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5]; export function formatKoboldUrl(value) { @@ -114,7 +121,7 @@ export function getKoboldGenerationData(finalPrompt, settings, maxLength, maxCon top_a: kai_settings.top_a, top_k: kai_settings.top_k, top_p: kai_settings.top_p, - min_p: kai_settings.min_p, + min_p: (kai_flags.can_use_min_p || isHorde) ? kai_settings.min_p : undefined, typical: kai_settings.typical, s1: sampler_order[0], s2: sampler_order[1], @@ -128,11 +135,11 @@ export function getKoboldGenerationData(finalPrompt, settings, maxLength, maxCon stop_sequence: (kai_flags.can_use_stop_sequence || isHorde) ? getStoppingStrings(isImpersonate) : undefined, streaming: kai_settings.streaming_kobold && kai_flags.can_use_streaming && type !== 'quiet', can_abort: kai_flags.can_use_streaming, - mirostat: kai_flags.can_use_mirostat ? kai_settings.mirostat : undefined, - mirostat_tau: kai_flags.can_use_mirostat ? kai_settings.mirostat_tau : undefined, - mirostat_eta: kai_flags.can_use_mirostat ? kai_settings.mirostat_eta : undefined, - use_default_badwordsids: kai_flags.can_use_default_badwordsids ? kai_settings.use_default_badwordsids : undefined, - grammar: kai_flags.can_use_grammar ? substituteParams(kai_settings.grammar) : undefined, + mirostat: (kai_flags.can_use_mirostat || isHorde) ? kai_settings.mirostat : undefined, + mirostat_tau: (kai_flags.can_use_mirostat || isHorde) ? kai_settings.mirostat_tau : undefined, + mirostat_eta: (kai_flags.can_use_mirostat || isHorde) ? kai_settings.mirostat_eta : undefined, + use_default_badwordsids: (kai_flags.can_use_default_badwordsids || isHorde) ? kai_settings.use_default_badwordsids : undefined, + grammar: (kai_flags.can_use_grammar || isHorde) ? substituteParams(kai_settings.grammar) : undefined, sampler_seed: kai_settings.seed >= 0 ? kai_settings.seed : undefined, }; return generate_data; @@ -302,6 +309,7 @@ export function setKoboldFlags(version, koboldVersion) { kai_flags.can_use_default_badwordsids = canUseDefaultBadwordIds(version); kai_flags.can_use_mirostat = canUseMirostat(koboldVersion); kai_flags.can_use_grammar = canUseGrammar(koboldVersion); + kai_flags.can_use_min_p = canUseMinP(koboldVersion); } /** @@ -366,6 +374,17 @@ function canUseGrammar(koboldVersion) { } else return false; } +/** + * Determines if the Kobold min_p can be used with the given version. + * @param {{result:string, version:string;}} koboldVersion KoboldAI version object. + * @returns {boolean} True if the Kobold min_p can be used, false otherwise. + */ +function canUseMinP(koboldVersion) { + if (koboldVersion && koboldVersion.result == 'KoboldCpp') { + return (koboldVersion.version || '0.0').localeCompare(MIN_MIN_P_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1; + } else return false; +} + /** * Sorts the sampler items by the given order. * @param {any[]} orderArray Sampler order array. From 6f061adc1ead8097b20bf2fbb3e85b19f51dec3d Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 12 Nov 2023 02:28:03 +0200 Subject: [PATCH 15/16] Add OpenAI TTS provider --- public/scripts/extensions/tts/elevenlabs.js | 4 + public/scripts/extensions/tts/index.js | 2 + public/scripts/extensions/tts/openai.js | 148 ++++++++++++++++++++ src/openai.js | 39 ++++++ 4 files changed, 193 insertions(+) create mode 100644 public/scripts/extensions/tts/openai.js diff --git a/public/scripts/extensions/tts/elevenlabs.js b/public/scripts/extensions/tts/elevenlabs.js index 3e0c90fe2..4cb7813fe 100644 --- a/public/scripts/extensions/tts/elevenlabs.js +++ b/public/scripts/extensions/tts/elevenlabs.js @@ -45,6 +45,8 @@ class ElevenLabsTtsProvider { this.settings.stability = $('#elevenlabs_tts_stability').val() this.settings.similarity_boost = $('#elevenlabs_tts_similarity_boost').val() this.settings.model = $('#elevenlabs_tts_model').find(':selected').val() + $('#elevenlabs_tts_stability_output').text(this.settings.stability); + $('#elevenlabs_tts_similarity_boost_output').text(this.settings.similarity_boost); saveTtsProviderSettings() } @@ -79,6 +81,8 @@ class ElevenLabsTtsProvider { $('#elevenlabs_tts_similarity_boost').on('input', this.onSettingsChange.bind(this)) $('#elevenlabs_tts_stability').on('input', this.onSettingsChange.bind(this)) $('#elevenlabs_tts_model').on('change', this.onSettingsChange.bind(this)) + $('#elevenlabs_tts_stability_output').text(this.settings.stability); + $('#elevenlabs_tts_similarity_boost_output').text(this.settings.similarity_boost); try { await this.checkReady() diff --git a/public/scripts/extensions/tts/index.js b/public/scripts/extensions/tts/index.js index 0b22753d1..2ff01190a 100644 --- a/public/scripts/extensions/tts/index.js +++ b/public/scripts/extensions/tts/index.js @@ -9,6 +9,7 @@ import { SystemTtsProvider } from './system.js' import { NovelTtsProvider } from './novel.js' import { power_user } from '../../power-user.js' import { registerSlashCommand } from '../../slash-commands.js' +import { OpenAITtsProvider } from './openai.js' export { talkingAnimation }; const UPDATE_INTERVAL = 1000 @@ -73,6 +74,7 @@ let ttsProviders = { Coqui: CoquiTtsProvider, Edge: EdgeTtsProvider, Novel: NovelTtsProvider, + OpenAI: OpenAITtsProvider, } let ttsProvider let ttsProviderName diff --git a/public/scripts/extensions/tts/openai.js b/public/scripts/extensions/tts/openai.js new file mode 100644 index 000000000..393a82940 --- /dev/null +++ b/public/scripts/extensions/tts/openai.js @@ -0,0 +1,148 @@ +import { getRequestHeaders } from "../../../script.js" +import { saveTtsProviderSettings } from "./index.js"; + +export { OpenAITtsProvider } + +class OpenAITtsProvider { + static voices = [ + { name: 'Alloy', voice_id: 'alloy', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/alloy.wav' }, + { name: 'Echo', voice_id: 'echo', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/echo.wav' }, + { name: 'Fable', voice_id: 'fable', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/fable.wav' }, + { name: 'Onyx', voice_id: 'onyx', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/onyx.wav' }, + { name: 'Nova', voice_id: 'nova', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/nova.wav' }, + { name: 'Shimmer', voice_id: 'shimmer', lang: 'en-US', preview_url: 'https://cdn.openai.com/API/docs/audio/shimmer.wav' }, + ]; + + settings + voices = [] + separator = ' . ' + audioElement = document.createElement('audio') + + defaultSettings = { + voiceMap: {}, + customVoices: [], + model: 'tts-1', + speed: 1, + } + + get settingsHtml() { + let html = ` +
Use OpenAI's TTS engine.
+ Hint: Save an API key in the OpenAI API settings to use it here. +
+ + +
+
+ + +
`; + return html; + } + + async loadSettings(settings) { + // Populate Provider UI given input settings + if (Object.keys(settings).length == 0) { + console.info("Using default TTS Provider settings") + } + + // Only accept keys defined in defaultSettings + this.settings = this.defaultSettings; + + for (const key in settings) { + if (key in this.settings) { + this.settings[key] = settings[key]; + } else { + throw `Invalid setting passed to TTS Provider: ${key}`; + } + } + + $('#openai-tts-model').val(this.settings.model); + $('#openai-tts-model').on('change', () => { + this.onSettingsChange(); + }); + + $('#openai-tts-speed').val(this.settings.speed); + $('#openai-tts-speed').on('input', () => { + this.onSettingsChange(); + }); + + $('#openai-tts-speed-output').text(this.settings.speed); + + await this.checkReady(); + console.debug("OpenAI TTS: Settings loaded"); + } + + onSettingsChange() { + // Update dynamically + this.settings.model = String($('#openai-tts-model').find(':selected').val()); + this.settings.speed = Number($('#openai-tts-speed').val()); + $('#openai-tts-speed-output').text(this.settings.speed); + saveTtsProviderSettings(); + } + + async checkReady() { + await this.fetchTtsVoiceObjects(); + } + + async onRefreshClick() { + return; + } + + async getVoice(voiceName) { + if (!voiceName) { + throw `TTS Voice name not provided` + } + + const voice = OpenAITtsProvider.voices.find(voice => voice.voice_id === voiceName || voice.name === voiceName); + + if (!voice) { + throw `TTS Voice not found: ${voiceName}` + } + + return voice; + } + + async generateTts(text, voiceId) { + const response = await this.fetchTtsGeneration(text, voiceId) + return response + } + + async fetchTtsVoiceObjects() { + return OpenAITtsProvider.voices; + } + + async previewTtsVoice(_) { + return; + } + + async fetchTtsGeneration(inputText, voiceId) { + console.info(`Generating new TTS for voice_id ${voiceId}`) + const response = await fetch(`/api/openai/generate-voice`, { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ + "text": inputText, + "voice": voiceId, + "model": this.settings.model, + "speed": this.settings.speed, + }), + }); + + if (!response.ok) { + toastr.error(response.statusText, 'TTS Generation Failed'); + throw new Error(`HTTP ${response.status}: ${await response.text()}`); + } + + return response; + } +} diff --git a/src/openai.js b/src/openai.js index aa2445ada..857c0198f 100644 --- a/src/openai.js +++ b/src/openai.js @@ -63,6 +63,45 @@ function registerEndpoints(app, jsonParser) { } }); + app.post('/api/openai/generate-voice', jsonParser, async (request, response) => { + try { + const key = readSecret(SECRET_KEYS.OPENAI); + + if (!key) { + console.log('No OpenAI key found'); + return response.sendStatus(401); + } + + const result = await fetch('https://api.openai.com/v1/audio/speech', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${key}`, + }, + body: JSON.stringify({ + input: request.body.text, + response_format: 'mp3', + voice: request.body.voice ?? 'alloy', + speed: request.body.speed ?? 1, + model: request.body.model ?? 'tts-1', + }), + }); + + if (!result.ok) { + const text = await result.text(); + console.log('OpenAI request failed', result.statusText, text); + return response.status(500).send(text); + } + + const buffer = await result.arrayBuffer(); + response.setHeader('Content-Type', 'audio/mpeg'); + return response.send(Buffer.from(buffer)); + } catch (error) { + console.error('OpenAI TTS generation failed', error); + response.status(500).send('Internal server error'); + } + }); + app.post('/api/openai/generate-image', jsonParser, async (request, response) => { try { const key = readSecret(SECRET_KEYS.OPENAI); From 35c5d4e528d76253838f692f21d82250f46d64eb Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 12 Nov 2023 02:35:37 +0200 Subject: [PATCH 16/16] SD interactive mode fixes --- public/scripts/extensions/stable-diffusion/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index dd964b55c..c859fc250 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -71,7 +71,7 @@ const triggerWords = { } const messageTrigger = { - activationRegex: /\b(send|mail|imagine|generate|make|create|draw|paint|render)\b.*\b(pic|picture|image|drawing|painting|photo|photograph)\b(?:\s+of)?(?:\s+(?:a|an|the)?)?(.+)/i, + activationRegex: /\b(send|mail|imagine|generate|make|create|draw|paint|render)\b.*\b(pic|picture|image|drawing|painting|photo|photograph)\b(?:\s+of)?(?:\s+(?:a|an|the|this|that|those)?)?(.+)/i, specialCases: { [generationMode.CHARACTER]: ['you', 'yourself'], [generationMode.USER]: ['me', 'myself'], @@ -251,12 +251,12 @@ function processTriggers(chat, _, abort) { console.log(`SD: Triggered by "${message}", detected subject: ${subject}"`); - for (const [specialMode, triggers] of Object.entries(messageTrigger.specialCases)) { + outer: for (const [specialMode, triggers] of Object.entries(messageTrigger.specialCases)) { for (const trigger of triggers) { if (subject === trigger) { subject = triggerWords[specialMode][0]; console.log(`SD: Detected special case "${trigger}", switching to mode ${specialMode}`); - break; + break outer; } } }