Custom --grammar support [for koboldcpp] (#1161)
* Basic kobold grammar implementation This is probably jank as all hell, I don't write js, but I think I got it to work. * No value by default * Visual cleanup + reworded it a bit * Conditionally enable Grammar based on version flag. Fix layout --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>
This commit is contained in:
kalomaze
GitHub
parent
49c26f3810
commit
ad9382a98c
|
|
@ -277,6 +277,7 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="range-block">
|
||||
<div class="range-block-title" data-i18n="temperature">
|
||||
Temperature
|
||||
|
|
@ -1061,6 +1062,18 @@
|
|||
</div>
|
||||
</div>
|
||||
<hr>
|
||||
<div id="grammar_block">
|
||||
<h4 data-i18n="Grammar">Grammar</h4>
|
||||
<div class="range-block">
|
||||
<textarea id="grammar" rows="2" class="text_pole textarea_compact monospace"></textarea>
|
||||
<div class="toggle-description justifyLeft">
|
||||
<span data-i18n="Type in the desired custom grammar (GBNF).">
|
||||
Type in the desired custom grammar (<a href="https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md" target="_blank">GBNF</a>).
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<hr>
|
||||
<div class="range-block flexFlowColumn">
|
||||
<div class="range-block-title">
|
||||
<span data-i18n="Samplers Order">Samplers Order</span>
|
||||
|
|
@ -4544,4 +4557,4 @@
|
|||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ export const kai_settings = {
|
|||
mirostat_tau: 5.0,
|
||||
mirostat_eta: 0.1,
|
||||
use_default_badwordsids: true,
|
||||
grammar: "",
|
||||
};
|
||||
|
||||
export const kai_flags = {
|
||||
|
|
@ -34,6 +35,7 @@ export const kai_flags = {
|
|||
can_use_streaming: false,
|
||||
can_use_default_badwordsids: false,
|
||||
can_use_mirostat: false,
|
||||
can_use_grammar: false,
|
||||
};
|
||||
|
||||
const defaultValues = Object.freeze(structuredClone(kai_settings));
|
||||
|
|
@ -43,6 +45,7 @@ const MIN_UNBAN_VERSION = '1.2.4';
|
|||
const MIN_STREAMING_KCPPVERSION = '1.30';
|
||||
const MIN_TOKENIZATION_KCPPVERSION = '1.41';
|
||||
const MIN_MIROSTAT_KCPPVERSION = '1.35';
|
||||
const MIN_GRAMMAR_KCPPVERSION = '1.44';
|
||||
const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];
|
||||
|
||||
export function formatKoboldUrl(value) {
|
||||
|
|
@ -88,6 +91,7 @@ export function loadKoboldSettings(preset) {
|
|||
|
||||
export function getKoboldGenerationData(finalPrompt, this_settings, this_amount_gen, this_max_context, isImpersonate, type) {
|
||||
const sampler_order = kai_settings.sampler_order || this_settings.sampler_order;
|
||||
|
||||
let generate_data = {
|
||||
prompt: finalPrompt,
|
||||
gui_settings: false,
|
||||
|
|
@ -115,10 +119,11 @@ export function getKoboldGenerationData(finalPrompt, this_settings, this_amount_
|
|||
stop_sequence: kai_flags.can_use_stop_sequence ? getStoppingStrings(isImpersonate) : undefined,
|
||||
streaming: kai_settings.streaming_kobold && kai_flags.can_use_streaming && type !== 'quiet',
|
||||
can_abort: kai_flags.can_use_streaming,
|
||||
mirostat: kai_flags.can_use_mirostat ? kai_settings.mirostat : undefined,
|
||||
mirostat: kai_flags.can_use_mirostat ? kai_settings.mirostat : undefined,
|
||||
mirostat_tau: kai_flags.can_use_mirostat ? kai_settings.mirostat_tau : undefined,
|
||||
mirostat_eta: kai_flags.can_use_mirostat ? kai_settings.mirostat_eta : undefined,
|
||||
use_default_badwordsids: kai_flags.can_use_default_badwordsids ? kai_settings.use_default_badwordsids : undefined,
|
||||
grammar: kai_flags.can_use_grammar ? kai_settings.grammar : undefined,
|
||||
};
|
||||
return generate_data;
|
||||
}
|
||||
|
|
@ -257,6 +262,13 @@ const sliders = [
|
|||
format: (val) => val,
|
||||
setValue: (val) => { kai_settings.mirostat_eta = Number(val); },
|
||||
},
|
||||
{
|
||||
name: "grammar",
|
||||
sliderId: "#grammar",
|
||||
counterId: "#grammar_counter_kobold",
|
||||
format: (val) => val,
|
||||
setValue: (val) => { kai_settings.grammar = val; },
|
||||
},
|
||||
];
|
||||
|
||||
export function setKoboldFlags(version, koboldVersion) {
|
||||
|
|
@ -265,6 +277,7 @@ export function setKoboldFlags(version, koboldVersion) {
|
|||
kai_flags.can_use_tokenization = canUseKoboldTokenization(koboldVersion);
|
||||
kai_flags.can_use_default_badwordsids = canUseDefaultBadwordIds(version);
|
||||
kai_flags.can_use_mirostat = canUseMirostat(koboldVersion);
|
||||
kai_flags.can_use_grammar = canUseGrammar(koboldVersion);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -307,12 +320,28 @@ function canUseKoboldTokenization(koboldVersion) {
|
|||
} else return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the Kobold mirostat can be used with the given version.
|
||||
* @param {{result: string; version: string;}} koboldVersion KoboldAI version object.
|
||||
* @returns {boolean} True if the Kobold mirostat API can be used, false otherwise.
|
||||
*/
|
||||
function canUseMirostat(koboldVersion) {
|
||||
if (koboldVersion && koboldVersion.result == 'KoboldCpp') {
|
||||
return (koboldVersion.version || '0.0').localeCompare(MIN_MIROSTAT_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
|
||||
} else return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the Kobold grammar can be used with the given version.
|
||||
* @param {{result: string; version:string;}} koboldVersion KoboldAI version object.
|
||||
* @returns {boolean} True if the Kobold grammar can be used, false otherwise.
|
||||
*/
|
||||
function canUseGrammar(koboldVersion) {
|
||||
if (koboldVersion && koboldVersion.result == 'KoboldCpp') {
|
||||
return (koboldVersion.version || '0.0').localeCompare(MIN_GRAMMAR_KCPPVERSION, undefined, { numeric: true, sensitivity: 'base' }) > -1;
|
||||
} else return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sorts the sampler items by the given order.
|
||||
* @param {any[]} orderArray Sampler order array.
|
||||
|
|
@ -334,7 +363,7 @@ jQuery(function () {
|
|||
const value = $(this).val();
|
||||
const formattedValue = slider.format(value);
|
||||
slider.setValue(value);
|
||||
$(slider.counterId).html(formattedValue);
|
||||
$(slider.counterId).text(formattedValue);
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -375,6 +375,7 @@ app.post("/generate", jsonParser, async function (request, response_generate) {
|
|||
mirostat: request.body.mirostat,
|
||||
mirostat_eta: request.body.mirostat_eta,
|
||||
mirostat_tau: request.body.mirostat_tau,
|
||||
grammar: request.body.grammar,
|
||||
};
|
||||
if (!!request.body.stop_sequence) {
|
||||
this_settings['stop_sequence'] = request.body.stop_sequence;
|
||||
|
|
|
|||
Loading…
Reference in New Issue