Custom --grammar support [for koboldcpp] (#1161)

* Basic kobold grammar implementation

This is probably jank as all hell, I don't write js, but I think I got it to work.

* No value by default

* Visual cleanup

+ reworded it a bit

* Conditionally enable Grammar based on version flag. Fix layout

---------

Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

public/index.html

@@ -277,6 +277,7 @@
                                         </div>
                                     </div>
                                 </div>
+
                                 <div class="range-block">
                                     <div class="range-block-title" data-i18n="temperature">
                                         Temperature
@@ -1061,6 +1062,18 @@
                                         </div>
                                     </div>
                                     <hr>
+                                    <div id="grammar_block">
+                                        <h4 data-i18n="Grammar">Grammar</h4>
+                                        <div class="range-block">
+                                            <textarea id="grammar" rows="2" class="text_pole textarea_compact monospace"></textarea>
+                                            <div class="toggle-description justifyLeft">
+                                                <span data-i18n="Type in the desired custom grammar (GBNF).">
+                                                    Type in the desired custom grammar (<a href="https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md" target="_blank">GBNF</a>).
+                                                </span>
+                                            </div>
+                                        </div>
+                                    </div>
+                                    <hr>
                                     <div class="range-block flexFlowColumn">
                                         <div class="range-block-title">
                                             <span data-i18n="Samplers Order">Samplers Order</span>

public/scripts/kai-settings.js

@@ -26,6 +26,7 @@ export const kai_settings = {
     mirostat_tau: 5.0,
     mirostat_eta: 0.1,
     use_default_badwordsids: true,
+    grammar: "",
 };
 
 export const kai_flags = {
@@ -34,6 +35,7 @@ export const kai_flags = {
     can_use_streaming: false,
     can_use_default_badwordsids: false,
     can_use_mirostat: false,
+    can_use_grammar: false,
 };
 
 const defaultValues = Object.freeze(structuredClone(kai_settings));
@@ -43,6 +45,7 @@ const MIN_UNBAN_VERSION = '1.2.4';
 const MIN_STREAMING_KCPPVERSION = '1.30';
 const MIN_TOKENIZATION_KCPPVERSION = '1.41';
 const MIN_MIROSTAT_KCPPVERSION = '1.35';
+const MIN_GRAMMAR_KCPPVERSION = '1.44';
 const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];
 
 export function formatKoboldUrl(value) {
@@ -88,6 +91,7 @@ export function loadKoboldSettings(preset) {
 
 export function getKoboldGenerationData(finalPrompt, this_settings, this_amount_gen, this_max_context, isImpersonate, type) {
     const sampler_order = kai_settings.sampler_order || this_settings.sampler_order;
+
     let generate_data = {
         prompt: finalPrompt,
         gui_settings: false,
@@ -119,6 +123,7 @@ export function getKoboldGenerationData(finalPrompt, this_settings, this_amount_
         mirostat_tau: kai_flags.can_use_mirostat ? kai_settings.mirostat_tau : undefined,
         mirostat_eta: kai_flags.can_use_mirostat ? kai_settings.mirostat_eta : undefined,
         use_default_badwordsids: kai_flags.can_use_default_badwordsids ? kai_settings.use_default_badwordsids : undefined,
+        grammar: kai_flags.can_use_grammar ? kai_settings.grammar : undefined,
     };
     return generate_data;
 }
@@ -257,6 +262,13 @@ const sliders = [
         format: (val) => val,
         setValue: (val) => { kai_settings.mirostat_eta = Number(val); },
     },
+    {
+        name: "grammar",
+        sliderId: "#grammar",
+        counterId: "#grammar_counter_kobold",
+        format: (val) => val,
+        setValue: (val) => { kai_settings.grammar = val; },
+    },
 ];
 
 export function setKoboldFlags(version, koboldVersion) {
@@ -265,6 +277,7 @@ export function setKoboldFlags(version, koboldVersion) {
     kai_flags.can_use_tokenization = canUseKoboldTokenization(koboldVersion);
     kai_flags.can_use_default_badwordsids = canUseDefaultBadwordIds(version);
     kai_flags.can_use_mirostat = canUseMirostat(koboldVersion);
+    kai_flags.can_use_grammar = canUseGrammar(koboldVersion);
 }
 
 /**
@@ -307,12 +320,28 @@ function canUseKoboldTokenization(koboldVersion) {
     } else return false;
 }
 
+/**
+ * Determines if the Kobold mirostat can be used with the given version.
+ * @param {{result: string; version: string;}} koboldVersion KoboldAI version object.
+ * @returns {boolean} True if the Kobold mirostat API can be used, false otherwise.
+ */
 function canUseMirostat(koboldVersion) {
     if (koboldVersion && koboldVersion.result == 'KoboldCpp') {
         return (koboldVersion.version || '0.0').localeCompare(MIN_MIROSTAT_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
     } else return false;
 }
 
+/**
+ * Determines if the Kobold grammar can be used with the given version.
+ * @param {{result: string; version:string;}} koboldVersion KoboldAI version object.
+ * @returns {boolean} True if the Kobold grammar can be used, false otherwise.
+ */
+function canUseGrammar(koboldVersion) {
+    if (koboldVersion && koboldVersion.result == 'KoboldCpp') {
+        return (koboldVersion.version || '0.0').localeCompare(MIN_GRAMMAR_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
+    } else return false;
+}
+
 /**
  * Sorts the sampler items by the given order.
  * @param {any[]} orderArray Sampler order array.
@@ -334,7 +363,7 @@ jQuery(function () {
             const value = $(this).val();
             const formattedValue = slider.format(value);
             slider.setValue(value);
-            $(slider.counterId).html(formattedValue);
+            $(slider.counterId).text(formattedValue);
             saveSettingsDebounced();
         });
     });

server.js

@@ -375,6 +375,7 @@ app.post("/generate", jsonParser, async function (request, response_generate) {
             mirostat: request.body.mirostat,
             mirostat_eta: request.body.mirostat_eta,
             mirostat_tau: request.body.mirostat_tau,
+            grammar: request.body.grammar,
         };
         if (!!request.body.stop_sequence) {
             this_settings['stop_sequence'] = request.body.stop_sequence;