rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Textgen: Add speculative_ngram for TabbyAPI

Browse Source 
Speculative ngram allows for a different method of speculative
decoding. Using a draft model is still preferred.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
2024-05-21 23:35:29 -04:00
parent f5fccc0387
commit a12df762a0
2 changed files with 11 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
public/scripts/textgen-settings.js
View File

@@ -139,6 +139,7 @@ const settings = {
//best_of_aphrodite: 1,
ignore_eos_token: false,
spaces_between_special_tokens: true,
speculative_ngram: false,
//logits_processors_aphrodite: [],
//log_probs_aphrodite: 0,
//prompt_log_probs_aphrodite: 0,
@@ -214,6 +215,7 @@ export const setting_names = [
//'best_of_aphrodite',
'ignore_eos_token',
'spaces_between_special_tokens',
'speculative_ngram',
//'logits_processors_aphrodite',
//'log_probs_aphrodite',
//'prompt_log_probs_aphrodite'
@@ -657,6 +659,7 @@ jQuery(function () {
'freq_pen_textgenerationwebui': 0,
'presence_pen_textgenerationwebui': 0,
'no_repeat_ngram_size_textgenerationwebui': 0,
'speculative_ngram_textgenerationwebui': false,
'min_length_textgenerationwebui': 0,
'num_beams_textgenerationwebui': 1,
'length_penalty_textgenerationwebui': 1,
@@ -1060,6 +1063,7 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate,
'no_repeat_ngram_size': settings.type === OOBA ? settings.no_repeat_ngram_size : undefined,
'penalty_alpha': settings.type === OOBA ? settings.penalty_alpha : undefined,
'temperature_last': (settings.type === OOBA || settings.type === APHRODITE || settings.type == TABBY) ? settings.temperature_last : undefined,
'speculative_ngram': settings.type === TABBY ? settings.speculative_ngram : undefined,
'do_sample': settings.type === OOBA ? settings.do_sample : undefined,
'seed': settings.seed,
'guidance_scale': cfgValues?.guidanceScale?.value ?? settings.guidance_scale ?? 1,