Remove legacy Pygmalion formatting, part 2

default/settings.json

@@ -68,7 +68,6 @@
         "tokenizer": 99,
         "token_padding": 64,
         "collapse_newlines": false,
-        "pygmalion_formatting": 0,
         "pin_examples": false,
         "strip_examples": false,
         "trim_sentences": false,

public/i18n.json

@@ -157,7 +157,6 @@
         "Always add character's name to prompt": "始终将角色名称添加到提示符中",
         "Keep Example Messages in Prompt": "保持示例消息提示",
         "Remove Empty New Lines from Output": "从输出中删除空的新行",
-        "Pygmalion Formatting": "Pygmalion 格式",
         "Disabled for all models": "对所有模型禁用",
         "Automatic (based on model name)": "自动（基于型号名称）",
         "Enabled for all models": "所有模型启用",
@@ -713,7 +712,6 @@
         "Always add character's name to prompt": "常にキャラクター名をプロンプトに追加",
         "Keep Example Messages in Prompt": "プロンプトに例示メッセージを保持",
         "Remove Empty New Lines from Output": "出力から空の改行を削除",
-        "Pygmalion Formatting": "ピグマリオンフォーマット",
         "Disabled for all models": "すべてのモデルで無効",
         "Automatic (based on model name)": "自動（モデル名に基づく）",
         "Enabled for all models": "すべてのモデルで有効",
@@ -1271,7 +1269,6 @@
         "Always add character's name to prompt": "프롬프트에 항상 캐릭터 이름 삽입",
         "Keep Example Messages in Prompt": "예사 답변을 프롬프트에 유지",
         "Remove Empty New Lines from Output": "출력에서 빈줄 삭제",
-        "Pygmalion Formatting": "Pygmalion 서식",
         "Disabled for all models": "모든 모델에 비활성화",
         "Automatic (based on model name)": "모델 서식 자동탐지",
         "Enabled for all models": "모든 모델에 활성화",
@@ -1884,7 +1881,6 @@
         "Always add character's name to prompt": "Всегда добавлять имя персонажа в инструкции",
         "Keep Example Messages in Prompt": "Сохранять примеры сообщений в инструкции",
         "Remove Empty New Lines from Output": "Удалять пустые строчки из вывода",
-        "Pygmalion Formatting": "Форматирование Pygmalion",
         "Disabled for all models": "Выключено для всех моделей",
         "Automatic (based on model name)": "Автоматически (выбор по названию модели)",
         "Enabled for all models": "Включить для всех моделей",
@@ -2465,7 +2461,6 @@
         "Always add character's name to prompt": "Aggiungi sempre il nome del personaggio al prompt",
         "Keep Example Messages in Prompt": "Mantieni i messaggi d'esempio nel Prompt",
         "Remove Empty New Lines from Output": "Rimuovi le linee di testo vuote dall'output",
-        "Pygmalion Formatting": "Formattazione Pygmalion",
         "Disabled for all models": "Disabilita per tutti i modelli",
         "Automatic (based on model name)": "Automatico (basato sul nome del modello)",
         "Enabled for all models": "Abilita per tutti i modelli",
@@ -3147,7 +3142,6 @@
         "Always add character's name to prompt": "Voeg altijd de naam van het personage toe aan de prompt",
         "Keep Example Messages in Prompt": "Behoud voorbeeldberichten in de prompt",
         "Remove Empty New Lines from Output": "Verwijder lege regels uit de uitvoer",
-        "Pygmalion Formatting": "Pygmalion-opmaak",
         "Disabled for all models": "Uitgeschakeld voor alle modellen",
         "Automatic (based on model name)": "Automatisch (op basis van modelnaam)",
         "Enabled for all models": "Ingeschakeld voor alle modellen",

public/index.html

@@ -2468,22 +2468,6 @@
                                     Replace Macro in Custom Stopping Strings
                                 </span>
                             </label>
-                            <h4>
-                                <span data-i18n="Pygmalion Formatting">
-                                    Pygmalion Formatting
-                                </span>
-                            </h4>
-                            <select id="pygmalion_formatting">
-                                <option value="-1" data-i18n="Disabled for all models">
-                                    Disabled for all models
-                                </option>
-                                <option value="0" data-i18n="Automatic (based on model name)">
-                                    Automatic (based on model name)
-                                </option>
-                                <option value="1" data-i18n="Enabled for all models">
-                                    Enabled for all models
-                                </option>
-                            </select>
                         </div>
                         <div data-newbie-hidden>
                             <h4>

public/script.js

@@ -62,7 +62,6 @@ import {
     playMessageSound,
     fixMarkdown,
     power_user,
-    pygmalion_options,
     persona_description_positions,
     loadMovingUIState,
     getCustomStoppingStrings,
@@ -135,6 +134,7 @@ import {
     waitUntilCondition,
     escapeRegex,
     resetScrollHeight,
+    onlyUnique,
 } from "./scripts/utils.js";
 
 import { extension_settings, getContext, loadExtensionSettings, processExtensionHelpers, registerExtensionHelper, runGenerationInterceptors, saveMetadataDebounced } from "./scripts/extensions.js";
@@ -1794,7 +1794,7 @@ function diceRollReplace(input, invalidRollPlaceholder = '') {
     });
 }
 
-function getStoppingStrings(isImpersonate, addSpace) {
+function getStoppingStrings(isImpersonate) {
     const charString = `\n${name2}:`;
     const userString = `\n${name1}:`;
     const result = isImpersonate ? [charString] : [userString];
@@ -1821,7 +1821,7 @@ function getStoppingStrings(isImpersonate, addSpace) {
         result.push(...customStoppingStrings);
     }
 
-    return addSpace ? result.map(x => `${x} `) : result;
+    return result.filter(onlyUnique);
 }
 
 
@@ -3097,7 +3097,6 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
                     let title = extractTitleFromData(data);
                     kobold_horde_model = title;
 
-                    //Pygmalion run again
                     // to make it continue generating so long as it's under max_amount and hasn't signaled
                     // an end to the character's response via typing "You:" or adding "<endoftext>"
                     if (isMultigenEnabled() && type !== 'quiet') {
@@ -3870,7 +3869,7 @@ function cleanUpMessage(getMessage, isImpersonate, isContinue, displayIncomplete
         getMessage = getMessage.trim();
     }
 
-    const stoppingStrings = getStoppingStrings(isImpersonate, false);
+    const stoppingStrings = getStoppingStrings(isImpersonate);
 
     for (const stoppingString of stoppingStrings) {
         if (stoppingString.length) {

public/scripts/kai-settings.js

@@ -112,7 +112,7 @@ export function getKoboldGenerationData(finalPrompt, this_settings, this_amount_
         s7: sampler_order[6],
         use_world_info: false,
         singleline: kai_settings.single_line,
-        stop_sequence: kai_flags.can_use_stop_sequence ? getStoppingStrings(isImpersonate, false) : undefined,
+        stop_sequence: kai_flags.can_use_stop_sequence ? getStoppingStrings(isImpersonate) : undefined,
         streaming: kai_settings.streaming_kobold && kai_flags.can_use_streaming && type !== 'quiet',
         can_abort: kai_flags.can_use_streaming,
         mirostat: kai_flags.can_use_mirostat ?  kai_settings.mirostat : undefined,

public/scripts/nai-settings.js

@@ -417,7 +417,7 @@ export function getNovelGenerationData(finalPrompt, this_settings, this_amount_g
 
     const tokenizerType = kayra ? tokenizers.NERD2 : (clio ? tokenizers.NERD : tokenizers.NONE);
     const stopSequences = (tokenizerType !== tokenizers.NONE)
-        ? getStoppingStrings(isImpersonate, false)
+        ? getStoppingStrings(isImpersonate)
             .map(t => getTextTokens(tokenizerType, t))
         : undefined;
 

public/scripts/power-user.js

@@ -40,7 +40,6 @@ export {
     sortEntitiesList,
     fixMarkdown,
     power_user,
-    pygmalion_options,
     send_on_enter_options,
 };
 
@@ -67,12 +66,6 @@ export const chat_styles = {
     DOCUMENT: 2,
 }
 
-const pygmalion_options = {
-    DISABLED: -1,
-    AUTO: 0,
-    ENABLED: 1,
-}
-
 const send_on_enter_options = {
     DISABLED: -1,
     AUTO: 0,
@@ -93,7 +86,6 @@ let power_user = {
     tokenizer: tokenizers.BEST_MATCH,
     token_padding: 64,
     collapse_newlines: false,
-    pygmalion_formatting: pygmalion_options.AUTO,
     pin_examples: false,
     strip_examples: false,
     trim_sentences: false,
@@ -817,7 +809,6 @@ function loadPowerUserSettings(settings, data) {
     $('#auto_fix_generated_markdown').prop("checked", power_user.auto_fix_generated_markdown);
     $('#auto_scroll_chat_to_bottom').prop("checked", power_user.auto_scroll_chat_to_bottom);
     $(`#tokenizer option[value="${power_user.tokenizer}"]`).attr('selected', true);
-    $(`#pygmalion_formatting option[value=${power_user.pygmalion_formatting}]`).attr("selected", true);
     $(`#send_on_enter option[value=${power_user.send_on_enter}]`).attr("selected", true);
     $("#import_card_tags").prop("checked", power_user.import_card_tags);
     $("#confirm_message_delete").prop("checked", power_user.confirm_message_delete !== undefined ? !!power_user.confirm_message_delete : true);
@@ -1739,12 +1730,6 @@ $(document).ready(() => {
         saveSettingsDebounced();
     });
 
-    $("#pygmalion_formatting").change(function (e) {
-        power_user.pygmalion_formatting = Number($(this).find(":selected").val());
-        getStatus();
-        saveSettingsDebounced();
-    });
-
     $("#pin-examples-checkbox").change(function () {
         if ($(this).prop("checked")) {
             $("#remove-examples-checkbox").prop("checked", false).prop("disabled", true);

public/scripts/textgen-settings.js

@@ -255,7 +255,7 @@ export function getTextGenGenerationData(finalPrompt, this_amount_gen, isImperso
         'negative_prompt': cfgValues?.negativePrompt ?? textgenerationwebui_settings.negative_prompt ?? '',
         'seed': textgenerationwebui_settings.seed,
         'add_bos_token': textgenerationwebui_settings.add_bos_token,
-        'stopping_strings': getStoppingStrings(isImpersonate, false),
+        'stopping_strings': getStoppingStrings(isImpersonate),
         'truncation_length': max_context,
         'ban_eos_token': textgenerationwebui_settings.ban_eos_token,
         'skip_special_tokens': textgenerationwebui_settings.skip_special_tokens,