From a12df762a091678ca811986d9c329a0df285f93b Mon Sep 17 00:00:00 2001
From: kingbri <bdashore3@proton.me>
Date: Tue, 21 May 2024 23:35:29 -0400
Subject: [PATCH] Textgen: Add speculative_ngram for TabbyAPI

Speculative ngram allows for a different method of speculative
decoding. Using a draft model is still preferred.

Signed-off-by: kingbri <bdashore3@proton.me>
---
 public/index.html                  | 7 +++++++
 public/scripts/textgen-settings.js | 4 ++++
 2 files changed, 11 insertions(+)

diff --git a/public/index.html b/public/index.html
index d67c78a71..016147616 100644
--- a/public/index.html
+++ b/public/index.html
@@ -1405,6 +1405,13 @@
                                                     <div class="fa-solid fa-circle-info opacity50p " data-i18n="[title]Use the temperature sampler last" title="Use the temperature sampler last. This is almost always the sensible thing to do.&#13;When enabled: sample the set of plausible tokens first, then apply temperature to adjust their relative probabilities (technically, logits).&#13;When disabled: apply temperature to adjust the relative probabilities of ALL tokens first, then sample plausible tokens from that.&#13;Disabling Temperature Last boosts the probabilities in the tail of the distribution, which tends to amplify the chances of getting an incoherent response."></div>
                                                 </label>
                                             </label>
+                                            <label data-tg-type="tabby" class="checkbox_label  flexGrow flexShrink" for="speculative_ngram_textgenerationwebui">
+                                                <input type="checkbox" id="speculative_ngram_textgenerationwebui" />
+                                                <label>
+                                                    <small data-i18n="Speculative Ngram">Speculative Ngram</small>
+                                                    <div class="fa-solid fa-circle-info opacity50p " data-i18n="[title]Use a different speculative decoding method without a draft model" title="Use a different speculative decoding method without a draft model.&#13;Using a draft model is preferred. Speculative ngram is not as effective."></div>
+                                                </label>
+                                            </label>
 
                                             <label data-tg-type="vllm, aphrodite" class="checkbox_label" for="spaces_between_special_tokens_textgenerationwebui">
                                                 <input type="checkbox" id="spaces_between_special_tokens_textgenerationwebui" />
diff --git a/public/scripts/textgen-settings.js b/public/scripts/textgen-settings.js
index 8b7a438d6..569597f60 100644
--- a/public/scripts/textgen-settings.js
+++ b/public/scripts/textgen-settings.js
@@ -139,6 +139,7 @@ const settings = {
     //best_of_aphrodite: 1,
     ignore_eos_token: false,
     spaces_between_special_tokens: true,
+    speculative_ngram: false,
     //logits_processors_aphrodite: [],
     //log_probs_aphrodite: 0,
     //prompt_log_probs_aphrodite: 0,
@@ -214,6 +215,7 @@ export const setting_names = [
     //'best_of_aphrodite',
     'ignore_eos_token',
     'spaces_between_special_tokens',
+    'speculative_ngram',
     //'logits_processors_aphrodite',
     //'log_probs_aphrodite',
     //'prompt_log_probs_aphrodite'
@@ -657,6 +659,7 @@ jQuery(function () {
             'freq_pen_textgenerationwebui': 0,
             'presence_pen_textgenerationwebui': 0,
             'no_repeat_ngram_size_textgenerationwebui': 0,
+            'speculative_ngram_textgenerationwebui': false,
             'min_length_textgenerationwebui': 0,
             'num_beams_textgenerationwebui': 1,
             'length_penalty_textgenerationwebui': 1,
@@ -1060,6 +1063,7 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate,
         'no_repeat_ngram_size': settings.type === OOBA ? settings.no_repeat_ngram_size : undefined,
         'penalty_alpha': settings.type === OOBA ? settings.penalty_alpha : undefined,
         'temperature_last': (settings.type === OOBA || settings.type === APHRODITE || settings.type == TABBY) ? settings.temperature_last : undefined,
+        'speculative_ngram': settings.type === TABBY ? settings.speculative_ngram : undefined,
         'do_sample': settings.type === OOBA ? settings.do_sample : undefined,
         'seed': settings.seed,
         'guidance_scale': cfgValues?.guidanceScale?.value ?? settings.guidance_scale ?? 1,