rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

New AI21 Jamba + tokenizer

Browse Source
This commit is contained in:
Cohee
2024-08-26 12:07:36 +03:00
parent ff834efde3
commit 5fc16a2474
10 changed files with 188 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
public/scripts/openai.js
View File

@ -132,13 +132,9 @@ const max_2mil = 2000 * 1000;
const scale_max = 8191;
const claude_max = 9000; // We have a proper tokenizer, so theoretically could be larger (up to 9k)
const claude_100k_max = 99000;
let ai21_max = 9200; //can easily fit 9k gpt tokens because j2's tokenizer is efficient af
const unlocked_max = max_200k;
const oai_max_temp = 2.0;
const claude_max_temp = 1.0; //same as j2
const j2_max_topk = 10.0;
const j2_max_freq = 5.0;
const j2_max_pres = 5.0;
const claude_max_temp = 1.0;
const openrouter_website_model = 'OR_Website';
const openai_max_stop_strings = 4;
@ -218,25 +214,11 @@ const sensitiveFields = [
'custom_include_headers',
];
function getPrefixMap() {
return selected_group ? {
assistant: '',
user: '',
system: 'OOC: ',
}
: {
assistant: '{{char}}:',
user: '{{user}}:',
system: '',
};
}
const default_settings = {
preset_settings_openai: 'Default',
temp_openai: 1.0,
freq_pen_openai: 0,
pres_pen_openai: 0,
count_pen: 0.0,
top_p_openai: 1.0,
top_k_openai: 0,
min_p_openai: 0,
@ -264,7 +246,7 @@ const default_settings = {
openai_model: 'gpt-4-turbo',
claude_model: 'claude-3-5-sonnet-20240620',
google_model: 'gemini-1.5-pro',
ai21_model: 'j2-ultra',
ai21_model: 'jamba-1.5-mini',
mistralai_model: 'mistral-large-latest',
cohere_model: 'command-r-plus',
perplexity_model: 'llama-3.1-70b-instruct',
@ -294,8 +276,6 @@ const default_settings = {
assistant_prefill: '',
assistant_impersonation: '',
human_sysprompt_message: default_claude_human_sysprompt_message,
use_ai21_tokenizer: false,
use_google_tokenizer: false,
claude_use_sysprompt: false,
use_makersuite_sysprompt: true,
use_alt_scale: false,
@ -317,7 +297,6 @@ const oai_settings = {
temp_openai: 1.0,
freq_pen_openai: 0,
pres_pen_openai: 0,
count_pen: 0.0,
top_p_openai: 1.0,
top_k_openai: 0,
min_p_openai: 0,
@ -345,7 +324,7 @@ const oai_settings = {
openai_model: 'gpt-4-turbo',
claude_model: 'claude-3-5-sonnet-20240620',
google_model: 'gemini-1.5-pro',
ai21_model: 'j2-ultra',
ai21_model: 'jamba-1.5-mini',
mistralai_model: 'mistral-large-latest',
cohere_model: 'command-r-plus',
perplexity_model: 'llama-3.1-70b-instruct',
@ -375,8 +354,6 @@ const oai_settings = {
assistant_prefill: '',
assistant_impersonation: '',
human_sysprompt_message: default_claude_human_sysprompt_message,
use_ai21_tokenizer: false,
use_google_tokenizer: false,
claude_use_sysprompt: false,
use_makersuite_sysprompt: true,
use_alt_scale: false,
@ -1802,7 +1779,6 @@ async function sendOpenAIRequest(type, messages, signal) {
const isClaude = oai_settings.chat_completion_source == chat_completion_sources.CLAUDE;
const isOpenRouter = oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER;
const isScale = oai_settings.chat_completion_source == chat_completion_sources.SCALE;
const isAI21 = oai_settings.chat_completion_source == chat_completion_sources.AI21;
const isGoogle = oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE;
const isOAI = oai_settings.chat_completion_source == chat_completion_sources.OPENAI;
const isMistral = oai_settings.chat_completion_source == chat_completion_sources.MISTRALAI;
@ -1815,7 +1791,7 @@ async function sendOpenAIRequest(type, messages, signal) {
const isQuiet = type === 'quiet';
const isImpersonate = type === 'impersonate';
const isContinue = type === 'continue';
const stream = oai_settings.stream_openai && !isQuiet && !isScale && !isAI21 && !(isGoogle && oai_settings.google_model.includes('bison'));
const stream = oai_settings.stream_openai && !isQuiet && !isScale && !(isGoogle && oai_settings.google_model.includes('bison'));
const useLogprobs = !!power_user.request_token_probabilities;
const canMultiSwipe = oai_settings.n > 1 && !isContinue && !isImpersonate && !isQuiet && (isOAI || isCustom);
@ -1824,15 +1800,6 @@ async function sendOpenAIRequest(type, messages, signal) {
replaceItemizedPromptText(messageId, messages);
}
if (isAI21) {
const joinedMsgs = messages.reduce((acc, obj) => {
const prefix = getPrefixMap()[obj.role];
return acc + (prefix ? (selected_group ? '\n' : prefix + ' ') : '') + obj.content + '\n';
}, '');
messages = substituteParams(joinedMsgs) + (isImpersonate ? `${name1}:` : `${name2}:`);
replaceItemizedPromptText(messageId, messages);
}
// If we're using the window.ai extension, use that instead
// Doesn't support logit bias yet
if (oai_settings.chat_completion_source == chat_completion_sources.WINDOWAI) {
@ -1931,12 +1898,6 @@ async function sendOpenAIRequest(type, messages, signal) {
generate_data['use_makersuite_sysprompt'] = oai_settings.use_makersuite_sysprompt;
}
if (isAI21) {
generate_data['top_k'] = Number(oai_settings.top_k_openai);
generate_data['count_pen'] = Number(oai_settings.count_pen);
generate_data['stop_tokens'] = [name1 + ':', substituteParams(oai_settings.new_chat_prompt), substituteParams(oai_settings.new_group_chat_prompt)];
}
if (isMistral) {
generate_data['safe_prompt'] = false; // already defaults to false, but just incase they change that in the future.
}
@ -3008,7 +2969,6 @@ function loadOpenAISettings(data, settings) {
oai_settings.temp_openai = settings.temp_openai ?? default_settings.temp_openai;
oai_settings.freq_pen_openai = settings.freq_pen_openai ?? default_settings.freq_pen_openai;
oai_settings.pres_pen_openai = settings.pres_pen_openai ?? default_settings.pres_pen_openai;
oai_settings.count_pen = settings.count_pen ?? default_settings.count_pen;
oai_settings.top_p_openai = settings.top_p_openai ?? default_settings.top_p_openai;
oai_settings.top_k_openai = settings.top_k_openai ?? default_settings.top_k_openai;
oai_settings.top_a_openai = settings.top_a_openai ?? default_settings.top_a_openai;
@ -3080,10 +3040,12 @@ function loadOpenAISettings(data, settings) {
oai_settings.names_behavior = character_names_behavior.COMPLETION;
}
if (oai_settings.ai21_model.startsWith('j2-')) {
oai_settings.ai21_model = 'jamba-1.5-mini';
}
if (settings.wrap_in_quotes !== undefined) oai_settings.wrap_in_quotes = !!settings.wrap_in_quotes;
if (settings.openai_model !== undefined) oai_settings.openai_model = settings.openai_model;
if (settings.use_ai21_tokenizer !== undefined) { oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer; oai_settings.use_ai21_tokenizer ? ai21_max = 8191 : ai21_max = 9200; }
if (settings.use_google_tokenizer !== undefined) oai_settings.use_google_tokenizer = !!settings.use_google_tokenizer;
if (settings.claude_use_sysprompt !== undefined) oai_settings.claude_use_sysprompt = !!settings.claude_use_sysprompt;
if (settings.use_makersuite_sysprompt !== undefined) oai_settings.use_makersuite_sysprompt = !!settings.use_makersuite_sysprompt;
if (settings.use_alt_scale !== undefined) { oai_settings.use_alt_scale = !!settings.use_alt_scale; updateScaleForm(); }
@ -3133,8 +3095,6 @@ function loadOpenAISettings(data, settings) {
$('#jailbreak_system').prop('checked', oai_settings.jailbreak_system);
$('#openai_show_external_models').prop('checked', oai_settings.show_external_models);
$('#openai_external_category').toggle(oai_settings.show_external_models);
$('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer);
$('#use_google_tokenizer').prop('checked', oai_settings.use_google_tokenizer);
$('#claude_use_sysprompt').prop('checked', oai_settings.claude_use_sysprompt);
$('#use_makersuite_sysprompt').prop('checked', oai_settings.use_makersuite_sysprompt);
$('#scale-alt').prop('checked', oai_settings.use_alt_scale);
@ -3170,9 +3130,6 @@ function loadOpenAISettings(data, settings) {
$('#pres_pen_openai').val(oai_settings.pres_pen_openai);
$('#pres_pen_counter_openai').val(Number(oai_settings.pres_pen_openai).toFixed(2));
$('#count_pen').val(oai_settings.count_pen);
$('#count_pen_counter').val(Number(oai_settings.count_pen).toFixed(2));
$('#top_p_openai').val(oai_settings.top_p_openai);
$('#top_p_counter_openai').val(Number(oai_settings.top_p_openai).toFixed(2));
@ -3392,7 +3349,6 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
temperature: settings.temp_openai,
frequency_penalty: settings.freq_pen_openai,
presence_penalty: settings.pres_pen_openai,
count_penalty: settings.count_pen,
top_p: settings.top_p_openai,
top_k: settings.top_k_openai,
top_a: settings.top_a_openai,
@ -3427,8 +3383,6 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
assistant_prefill: settings.assistant_prefill,
assistant_impersonation: settings.assistant_impersonation,
human_sysprompt_message: settings.human_sysprompt_message,
use_ai21_tokenizer: settings.use_ai21_tokenizer,
use_google_tokenizer: settings.use_google_tokenizer,
claude_use_sysprompt: settings.claude_use_sysprompt,
use_makersuite_sysprompt: settings.use_makersuite_sysprompt,
use_alt_scale: settings.use_alt_scale,
@ -3799,7 +3753,6 @@ function onSettingsPresetChange() {
temperature: ['#temp_openai', 'temp_openai', false],
frequency_penalty: ['#freq_pen_openai', 'freq_pen_openai', false],
presence_penalty: ['#pres_pen_openai', 'pres_pen_openai', false],
count_penalty: ['#count_pen', 'count_pen', false],
top_p: ['#top_p_openai', 'top_p_openai', false],
top_k: ['#top_k_openai', 'top_k_openai', false],
top_a: ['#top_a_openai', 'top_a_openai', false],
@ -3856,8 +3809,6 @@ function onSettingsPresetChange() {
assistant_prefill: ['#claude_assistant_prefill', 'assistant_prefill', false],
assistant_impersonation: ['#claude_assistant_impersonation', 'assistant_impersonation', false],
human_sysprompt_message: ['#claude_human_sysprompt_textarea', 'human_sysprompt_message', false],
use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', true],
use_google_tokenizer: ['#use_google_tokenizer', 'use_google_tokenizer', true],
claude_use_sysprompt: ['#claude_use_sysprompt', 'claude_use_sysprompt', true],
use_makersuite_sysprompt: ['#use_makersuite_sysprompt', 'use_makersuite_sysprompt', true],
use_alt_scale: ['#use_alt_scale', 'use_alt_scale', true],
@ -4305,33 +4256,13 @@ async function onModelChange() {
if (oai_settings.chat_completion_source == chat_completion_sources.AI21) {
if (oai_settings.max_context_unlocked) {
$('#openai_max_context').attr('max', unlocked_max);
} else {
$('#openai_max_context').attr('max', ai21_max);
} else if (oai_settings.ai21_model.includes('jamba-1.5') || oai_settings.ai21_model.includes('jamba-instruct')) {
$('#openai_max_context').attr('max', max_256k);
}
oai_settings.openai_max_context = Math.min(oai_settings.openai_max_context, Number($('#openai_max_context').attr('max')));
oai_settings.openai_max_context = Math.min(Number($('#openai_max_context').attr('max')), oai_settings.openai_max_context);
$('#openai_max_context').val(oai_settings.openai_max_context).trigger('input');
oai_settings.temp_openai = Math.min(claude_max_temp, oai_settings.temp_openai);
$('#temp_openai').attr('max', claude_max_temp).val(oai_settings.temp_openai).trigger('input');
oai_settings.freq_pen_openai = Math.min(j2_max_freq, oai_settings.freq_pen_openai < 0 ? 0 : oai_settings.freq_pen_openai);
$('#freq_pen_openai').attr('min', 0).attr('max', j2_max_freq).val(oai_settings.freq_pen_openai).trigger('input');
oai_settings.pres_pen_openai = Math.min(j2_max_pres, oai_settings.pres_pen_openai < 0 ? 0 : oai_settings.pres_pen_openai);
$('#pres_pen_openai').attr('min', 0).attr('max', j2_max_pres).val(oai_settings.pres_pen_openai).trigger('input');
oai_settings.top_k_openai = Math.min(j2_max_topk, oai_settings.top_k_openai);
$('#top_k_openai').attr('max', j2_max_topk).val(oai_settings.top_k_openai).trigger('input');
} else if (oai_settings.chat_completion_source != chat_completion_sources.AI21) {
oai_settings.freq_pen_openai = Math.min(2.0, oai_settings.freq_pen_openai);
$('#freq_pen_openai').attr('min', -2.0).attr('max', 2.0).val(oai_settings.freq_pen_openai).trigger('input');
oai_settings.pres_pen_openai = Math.min(2.0, oai_settings.pres_pen_openai);
$('#pres_pen_openai').attr('min', -2.0).attr('max', 2.0).val(oai_settings.pres_pen_openai).trigger('input');
oai_settings.top_k_openai = Math.min(200, oai_settings.top_k_openai);
$('#top_k_openai').attr('max', 200).val(oai_settings.top_k_openai).trigger('input');
$('#temp_openai').attr('max', oai_max_temp).val(oai_settings.temp_openai).trigger('input');
}
if (oai_settings.chat_completion_source == chat_completion_sources.CUSTOM) {
@ -4953,12 +4884,6 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#count_pen').on('input', function () {
oai_settings.count_pen = Number($(this).val());
$('#count_pen_counter').val(Number($(this).val()).toFixed(2));
saveSettingsDebounced();
});
$('#top_p_openai').on('input', function () {
oai_settings.top_p_openai = Number($(this).val());
$('#top_p_counter_openai').val(Number($(this).val()).toFixed(2));
@ -5017,20 +4942,6 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#use_ai21_tokenizer').on('change', function () {
oai_settings.use_ai21_tokenizer = !!$('#use_ai21_tokenizer').prop('checked');
oai_settings.use_ai21_tokenizer ? ai21_max = 8191 : ai21_max = 9200;
oai_settings.openai_max_context = Math.min(ai21_max, oai_settings.openai_max_context);
$('#openai_max_context').attr('max', ai21_max).val(oai_settings.openai_max_context).trigger('input');
$('#openai_max_context_counter').attr('max', Number($('#openai_max_context').attr('max')));
saveSettingsDebounced();
});
$('#use_google_tokenizer').on('change', function () {
oai_settings.use_google_tokenizer = !!$('#use_google_tokenizer').prop('checked');
saveSettingsDebounced();
});
$('#claude_use_sysprompt').on('change', function () {
oai_settings.claude_use_sysprompt = !!$('#claude_use_sysprompt').prop('checked');
$('#claude_human_sysprompt_message_block').toggle(oai_settings.claude_use_sysprompt);