AI21 Adapter + Tokenization implementation
This commit is contained in:
based
parent
991ff98eaa
commit
0f21eabb6e
|
|
@ -654,7 +654,7 @@
|
|||
Max prompt cost: <span id="openrouter_max_prompt_cost">Unknown</span>
|
||||
</div>
|
||||
<hr>
|
||||
<div class="range-block" data-source="openai,claude,windowai,openrouter">
|
||||
<div class="range-block" data-source="openai,claude,windowai,openrouter,ai21">
|
||||
<div class="range-block-title" data-i18n="Temperature">
|
||||
Temperature
|
||||
</div>
|
||||
|
|
@ -669,7 +669,7 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="range-block" data-source="openai,openrouter">
|
||||
<div class="range-block" data-source="openai,openrouter,ai21">
|
||||
<div class="range-block-title" data-i18n="Frequency Penalty">
|
||||
Frequency Penalty
|
||||
</div>
|
||||
|
|
@ -684,7 +684,7 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="range-block" data-source="openai,openrouter">
|
||||
<div class="range-block" data-source="openai,openrouter,ai21">
|
||||
<div class="range-block-title" data-i18n="Presence Penalty">
|
||||
Presence Penalty
|
||||
</div>
|
||||
|
|
@ -699,7 +699,22 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="range-block" data-source="claude,openrouter">
|
||||
<div class="range-block" data-source="ai21">
|
||||
<div class="range-block-title" data-i18n="Count Penalty">
|
||||
Count Penalty
|
||||
</div>
|
||||
<div class="range-block-range-and-counter">
|
||||
<div class="range-block-range">
|
||||
<input type="range" id="count_pen" name="volume" min="0" max="1" step="0.01">
|
||||
</div>
|
||||
<div class="range-block-counter">
|
||||
<div contenteditable="true" data-for="count_pen" id="count_pen_counter">
|
||||
select
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="range-block" data-source="claude,openrouter,ai21">
|
||||
<div class="range-block-title" data-i18n="Top K">
|
||||
Top K
|
||||
</div>
|
||||
|
|
@ -714,7 +729,7 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="range-block" data-source="openai,claude,openrouter">
|
||||
<div class="range-block" data-source="openai,claude,openrouter,ai21">
|
||||
<div class="range-block-title" data-i18n="Top-p">
|
||||
Top P
|
||||
</div>
|
||||
|
|
@ -1418,6 +1433,14 @@
|
|||
<span data-i18n="May help the model to understand context. Names must only contain letters or numbers.">Helps the model to associate messages in group chats. Names must only contain letters or numbers without whitespaces.</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="range-block" data-source="ai21">
|
||||
<label for="use_ai21_tokenizer" title="Use AI21 Tokenizer" class="checkbox_label widthFreeExpand">
|
||||
<input id="use_ai21_tokenizer" type="checkbox" /><span data-i18n="Use AI21 Tokenizer">Use AI21 Tokenizer</span>
|
||||
</label>
|
||||
<div class="toggle-description justifyLeft">
|
||||
<span data-i18n="Use the appropriate tokenizer for Jurassic models, which is more efficient than GPT's.">Use the appropriate tokenizer for Jurassic models, which is more efficient than GPT's.</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="inline-drawer m-t-1 wide100p">
|
||||
<div class="inline-drawer-toggle inline-drawer-header">
|
||||
<b data-i18n="Quick Edit">Quick Edit</b>
|
||||
|
|
@ -1618,7 +1641,7 @@
|
|||
<option value="koboldhorde"><span data-i18n="KoboldAI Horde">KoboldAI Horde</span></option>
|
||||
<option value="textgenerationwebui"><span data-i18n="Text Gen WebUI (ooba/Mancer)">Text Gen WebUI (ooba/Mancer)</span></option>
|
||||
<option value="novel"><span data-i18n="NovelAI">NovelAI</span></option>
|
||||
<option value="openai"><span data-i18n="Chat Completion (OpenAI, Claude, Window/OpenRouter, Scale)">Chat Completion (OpenAI, Claude, Window, OpenRouter, Scale)</span></option>
|
||||
<option value="openai"><span data-i18n="Chat Completion (OpenAI, Claude, Window/OpenRouter, Scale, AI21)">Chat Completion (OpenAI, Claude, Window, OpenRouter, Scale, AI21)</span></option>
|
||||
</select>
|
||||
</div>
|
||||
<div id="kobold_horde" style="position: relative;"> <!-- shows the kobold settings -->
|
||||
|
|
@ -1804,6 +1827,7 @@
|
|||
<option value="openrouter">OpenRouter</option>
|
||||
<option value="claude">Claude</option>
|
||||
<option value="scale">Scale</option>
|
||||
<option value="ai21">AI21</option>
|
||||
</select>
|
||||
<form id="openai_form" data-source="openai" action="javascript:void(null);" method="post" enctype="multipart/form-data">
|
||||
<h4><span data-i18n="OpenAI API key">OpenAI API key</span></h4>
|
||||
|
|
@ -1972,6 +1996,27 @@
|
|||
<select id="model_scale_select" class="displayNone"></select>
|
||||
</form>
|
||||
|
||||
<form id="ai21_form" data-source="ai21" action="javascript:void(null);" method="post" enctype="multipart/form-data">
|
||||
<h4>AI21 API Key</h4>
|
||||
<div class="flex-container">
|
||||
<input id="api_key_ai21" name="api_key_ai21" class="text_pole flex1" maxlength="500" value="" type="text" autocomplete="off">
|
||||
<div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_ai21"></div>
|
||||
</div>
|
||||
<div data-for="api_key_ai21" class="neutral_warning">
|
||||
For privacy reasons, your API key will be hidden after you reload the page.
|
||||
</div>
|
||||
<div>
|
||||
<h4 data-i18n="AI21 Model">AI21 Model</h4>
|
||||
<select id="model_ai21_select">
|
||||
<optgroup label="Latest">
|
||||
<option value="j2-ultra">j2-ultra</option>
|
||||
<option value="j2-mid">j2-mid</option>
|
||||
<option value="j2-light">j2-light</option>
|
||||
</optgroup>
|
||||
</select>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div class="flex-container flex">
|
||||
<input id="api_button_openai" class="menu_button" type="submit" value="Connect">
|
||||
<input data-source="openrouter" id="openrouter_authorize" class="menu_button" type="button" value="Authorize" title="Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai" data-i18n="[title]Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai">
|
||||
|
|
|
|||
|
|
@ -2131,7 +2131,7 @@ function baseChatReplace(value, name1, name2) {
|
|||
}
|
||||
|
||||
function isStreamingEnabled() {
|
||||
return ((main_api == 'openai' && oai_settings.stream_openai && oai_settings.chat_completion_source !== chat_completion_sources.SCALE)
|
||||
return ((main_api == 'openai' && oai_settings.stream_openai && oai_settings.chat_completion_source !== chat_completion_sources.SCALE && oai_settings.chat_completion_source !== chat_completion_sources.AI21)
|
||||
|| (main_api == 'kobold' && kai_settings.streaming_kobold && kai_settings.can_use_streaming)
|
||||
|| (main_api == 'novel' && nai_settings.streaming_novel)
|
||||
|| (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming))
|
||||
|
|
@ -4738,6 +4738,7 @@ function changeMainAPI() {
|
|||
case chat_completion_sources.WINDOWAI:
|
||||
case chat_completion_sources.CLAUDE:
|
||||
case chat_completion_sources.OPENAI:
|
||||
case chat_completion_sources.AI21:
|
||||
default:
|
||||
setupChatCompletionPromptManager(oai_settings);
|
||||
break;
|
||||
|
|
@ -7178,6 +7179,11 @@ function connectAPISlash(_, text) {
|
|||
source: 'openrouter',
|
||||
button: '#api_button_openai',
|
||||
},
|
||||
'ai21': {
|
||||
selected: 'openai',
|
||||
source: 'ai21',
|
||||
button: '#api_button_openai',
|
||||
}
|
||||
};
|
||||
|
||||
const apiConfig = apiMap[text];
|
||||
|
|
@ -7396,7 +7402,7 @@ $(document).ready(function () {
|
|||
}
|
||||
|
||||
registerSlashCommand('dupe', DupeChar, [], "– duplicates the currently selected character", true, true);
|
||||
registerSlashCommand('api', connectAPISlash, [], "(kobold, horde, novel, ooba, oai, claude, windowai) – connect to an API", true, true);
|
||||
registerSlashCommand('api', connectAPISlash, [], "(kobold, horde, novel, ooba, oai, claude, windowai, ai21) – connect to an API", true, true);
|
||||
registerSlashCommand('impersonate', doImpersonate, ['imp'], "- calls an impersonation response", true, true);
|
||||
registerSlashCommand('delchat', doDeleteChat, [], "- deletes the current chat", true, true);
|
||||
registerSlashCommand('closechat', doCloseChat, [], "- closes the current chat", true, true);
|
||||
|
|
|
|||
|
|
@ -478,6 +478,7 @@ function RA_autoconnect(PrevApi) {
|
|||
|| (secret_state[SECRET_KEYS.SCALE] && oai_settings.chat_completion_source == chat_completion_sources.SCALE)
|
||||
|| (oai_settings.chat_completion_source == chat_completion_sources.WINDOWAI)
|
||||
|| (secret_state[SECRET_KEYS.OPENROUTER] && oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER)
|
||||
|| (secret_state[SECRET_KEYS.AI21] && oai_settings.chat_completion_source == chat_completion_sources.AI21)
|
||||
) {
|
||||
$("#api_button_openai").click();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -113,9 +113,13 @@ const scale_max = 7900; // Probably more. Save some for the system prompt define
|
|||
const claude_max = 8000; // We have a proper tokenizer, so theoretically could be larger (up to 9k)
|
||||
const palm2_max = 7500; // The real context window is 8192, spare some for padding due to using turbo tokenizer
|
||||
const claude_100k_max = 99000;
|
||||
let ai21_max = 9200; //can easily fit 9k gpt tokens because j2's tokenizer is efficient af
|
||||
const unlocked_max = 100 * 1024;
|
||||
const oai_max_temp = 2.0;
|
||||
const claude_max_temp = 1.0;
|
||||
const claude_max_temp = 1.0; //same as j2
|
||||
const j2_max_topk = 10.0;
|
||||
const j2_max_freq = 5.0;
|
||||
const j2_max_pres = 5.0;
|
||||
const openrouter_website_model = 'OR_Website';
|
||||
|
||||
let biasCache = undefined;
|
||||
|
|
@ -161,13 +165,26 @@ export const chat_completion_sources = {
|
|||
CLAUDE: 'claude',
|
||||
SCALE: 'scale',
|
||||
OPENROUTER: 'openrouter',
|
||||
AI21: 'ai21',
|
||||
};
|
||||
|
||||
const prefixMap = selected_group ? {
|
||||
assistant: "",
|
||||
user: "",
|
||||
system: "OOC: "
|
||||
}
|
||||
: {
|
||||
assistant: "{{char}}:",
|
||||
user: "{{user}}:",
|
||||
system: ""
|
||||
};
|
||||
|
||||
const default_settings = {
|
||||
preset_settings_openai: 'Default',
|
||||
temp_openai: 0.9,
|
||||
freq_pen_openai: 0.7,
|
||||
pres_pen_openai: 0.7,
|
||||
count_pen: 0.0,
|
||||
top_p_openai: 1.0,
|
||||
top_k_openai: 0,
|
||||
stream_openai: false,
|
||||
|
|
@ -188,6 +205,7 @@ const default_settings = {
|
|||
wi_format: default_wi_format,
|
||||
openai_model: 'gpt-3.5-turbo',
|
||||
claude_model: 'claude-instant-v1',
|
||||
ai21_model: 'j2-ultra',
|
||||
windowai_model: '',
|
||||
openrouter_model: openrouter_website_model,
|
||||
jailbreak_system: false,
|
||||
|
|
@ -199,6 +217,7 @@ const default_settings = {
|
|||
show_external_models: false,
|
||||
proxy_password: '',
|
||||
assistant_prefill: '',
|
||||
use_ai21_tokenizer: false,
|
||||
};
|
||||
|
||||
const oai_settings = {
|
||||
|
|
@ -206,6 +225,7 @@ const oai_settings = {
|
|||
temp_openai: 1.0,
|
||||
freq_pen_openai: 0,
|
||||
pres_pen_openai: 0,
|
||||
count_pen: 0.0,
|
||||
top_p_openai: 1.0,
|
||||
top_k_openai: 0,
|
||||
stream_openai: false,
|
||||
|
|
@ -226,6 +246,7 @@ const oai_settings = {
|
|||
wi_format: default_wi_format,
|
||||
openai_model: 'gpt-3.5-turbo',
|
||||
claude_model: 'claude-instant-v1',
|
||||
ai21_model: 'j2-ultra',
|
||||
windowai_model: '',
|
||||
openrouter_model: openrouter_website_model,
|
||||
jailbreak_system: false,
|
||||
|
|
@ -237,6 +258,7 @@ const oai_settings = {
|
|||
show_external_models: false,
|
||||
proxy_password: '',
|
||||
assistant_prefill: '',
|
||||
use_ai21_tokenizer: false,
|
||||
};
|
||||
|
||||
let openai_setting_names;
|
||||
|
|
@ -967,6 +989,8 @@ function getChatCompletionModel() {
|
|||
return '';
|
||||
case chat_completion_sources.OPENROUTER:
|
||||
return oai_settings.openrouter_model !== openrouter_website_model ? oai_settings.openrouter_model : null;
|
||||
case chat_completion_sources.AI21:
|
||||
return oai_settings.ai21_model;
|
||||
default:
|
||||
throw new Error(`Unknown chat completion source: ${oai_settings.chat_completion_source}`);
|
||||
}
|
||||
|
|
@ -1048,10 +1072,19 @@ async function sendOpenAIRequest(type, openai_msgs_tosend, signal) {
|
|||
const isClaude = oai_settings.chat_completion_source == chat_completion_sources.CLAUDE;
|
||||
const isOpenRouter = oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER;
|
||||
const isScale = oai_settings.chat_completion_source == chat_completion_sources.SCALE;
|
||||
const isAI21 = oai_settings.chat_completion_source == chat_completion_sources.AI21;
|
||||
const isTextCompletion = oai_settings.chat_completion_source == chat_completion_sources.OPENAI && (oai_settings.openai_model.startsWith('text-') || oai_settings.openai_model.startsWith('code-'));
|
||||
const stream = type !== 'quiet' && oai_settings.stream_openai && !isScale;
|
||||
const stream = type !== 'quiet' && oai_settings.stream_openai && !isScale && !isAI21;
|
||||
const isQuiet = type === 'quiet';
|
||||
|
||||
if(isAI21) {
|
||||
const joinedMsgs = openai_msgs_tosend.reduce((acc, obj) => {
|
||||
const prefix = prefixMap[obj.role];
|
||||
return acc + (prefix ? (selected_group ? "\n" : prefix + " ") : "") + obj.content + "\n";
|
||||
}, "");
|
||||
openai_msgs_tosend = substituteParams(joinedMsgs);
|
||||
}
|
||||
|
||||
// If we're using the window.ai extension, use that instead
|
||||
// Doesn't support logit bias yet
|
||||
if (oai_settings.chat_completion_source == chat_completion_sources.WINDOWAI) {
|
||||
|
|
@ -1107,6 +1140,13 @@ async function sendOpenAIRequest(type, openai_msgs_tosend, signal) {
|
|||
generate_data['api_url_scale'] = oai_settings.api_url_scale;
|
||||
}
|
||||
|
||||
if (isAI21) {
|
||||
generate_data['use_ai21'] = true;
|
||||
generate_data['top_k'] = parseFloat(oai_settings.top_k_openai);
|
||||
generate_data['count_pen'] = parseFloat(oai_settings.count_pen);
|
||||
generate_data['stop_tokens'] = [name1 + ':', 'prompt: [Start a new chat]'];
|
||||
}
|
||||
|
||||
const generate_url = '/generate_openai';
|
||||
const response = await fetch(generate_url, {
|
||||
method: 'POST',
|
||||
|
|
@ -1297,6 +1337,7 @@ class TokenHandler {
|
|||
}
|
||||
|
||||
function countTokens(messages, full = false) {
|
||||
let shouldTokenizeAI21 = oai_settings.chat_completion_source === chat_completion_sources.AI21 && oai_settings.use_ai21_tokenizer;
|
||||
let chatId = 'undefined';
|
||||
|
||||
try {
|
||||
|
|
@ -1329,12 +1370,13 @@ function countTokens(messages, full = false) {
|
|||
if (typeof cachedCount === 'number') {
|
||||
token_count += cachedCount;
|
||||
}
|
||||
else {
|
||||
|
||||
else {
|
||||
console.log(JSON.stringify([message]));
|
||||
jQuery.ajax({
|
||||
async: false,
|
||||
type: 'POST', //
|
||||
url: `/tokenize_openai?model=${model}`,
|
||||
url: shouldTokenizeAI21 ? '/tokenize_ai21' : `/tokenize_openai?model=${model}`,
|
||||
data: JSON.stringify([message]),
|
||||
dataType: "json",
|
||||
contentType: "application/json",
|
||||
|
|
@ -1850,6 +1892,7 @@ function loadOpenAISettings(data, settings) {
|
|||
oai_settings.temp_openai = settings.temp_openai ?? default_settings.temp_openai;
|
||||
oai_settings.freq_pen_openai = settings.freq_pen_openai ?? default_settings.freq_pen_openai;
|
||||
oai_settings.pres_pen_openai = settings.pres_pen_openai ?? default_settings.pres_pen_openai;
|
||||
oai_settings.count_pen = settings.count_pen ?? default_settings.count_pen;
|
||||
oai_settings.top_p_openai = settings.top_p_openai ?? default_settings.top_p_openai;
|
||||
oai_settings.top_k_openai = settings.top_k_openai ?? default_settings.top_k_openai;
|
||||
oai_settings.stream_openai = settings.stream_openai ?? default_settings.stream_openai;
|
||||
|
|
@ -1865,6 +1908,7 @@ function loadOpenAISettings(data, settings) {
|
|||
oai_settings.claude_model = settings.claude_model ?? default_settings.claude_model;
|
||||
oai_settings.windowai_model = settings.windowai_model ?? default_settings.windowai_model;
|
||||
oai_settings.openrouter_model = settings.openrouter_model ?? default_settings.openrouter_model;
|
||||
oai_settings.ai21_model = settings.ai21_model ?? default_settings.ai21_model;
|
||||
oai_settings.chat_completion_source = settings.chat_completion_source ?? default_settings.chat_completion_source;
|
||||
oai_settings.api_url_scale = settings.api_url_scale ?? default_settings.api_url_scale;
|
||||
oai_settings.show_external_models = settings.show_external_models ?? default_settings.show_external_models;
|
||||
|
|
@ -1883,7 +1927,7 @@ function loadOpenAISettings(data, settings) {
|
|||
if (settings.wrap_in_quotes !== undefined) oai_settings.wrap_in_quotes = !!settings.wrap_in_quotes;
|
||||
if (settings.names_in_completion !== undefined) oai_settings.names_in_completion = !!settings.names_in_completion;
|
||||
if (settings.openai_model !== undefined) oai_settings.openai_model = settings.openai_model;
|
||||
|
||||
if (settings.use_ai21_tokenizer !== undefined) oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer;
|
||||
$('#stream_toggle').prop('checked', oai_settings.stream_openai);
|
||||
$('#api_url_scale').val(oai_settings.api_url_scale);
|
||||
$('#openai_proxy_password').val(oai_settings.proxy_password);
|
||||
|
|
@ -1895,6 +1939,8 @@ function loadOpenAISettings(data, settings) {
|
|||
$(`#model_claude_select option[value="${oai_settings.claude_model}"`).attr('selected', true);
|
||||
$('#model_windowai_select').val(oai_settings.windowai_model);
|
||||
$(`#model_windowai_select option[value="${oai_settings.windowai_model}"`).attr('selected', true);
|
||||
$('#model_ai21_select').val(oai_settings.ai21_model);
|
||||
$(`#model_ai21_select option[value="${oai_settings.ai21_model}"`).attr('selected', true);
|
||||
$('#openai_max_context').val(oai_settings.openai_max_context);
|
||||
$('#openai_max_context_counter').text(`${oai_settings.openai_max_context}`);
|
||||
$('#model_openrouter_select').val(oai_settings.openrouter_model);
|
||||
|
|
@ -1910,7 +1956,7 @@ function loadOpenAISettings(data, settings) {
|
|||
$('#legacy_streaming').prop('checked', oai_settings.legacy_streaming);
|
||||
$('#openai_show_external_models').prop('checked', oai_settings.show_external_models);
|
||||
$('#openai_external_category').toggle(oai_settings.show_external_models);
|
||||
|
||||
$('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer);
|
||||
if (settings.impersonation_prompt !== undefined) oai_settings.impersonation_prompt = settings.impersonation_prompt;
|
||||
|
||||
$('#impersonation_prompt_textarea').val(oai_settings.impersonation_prompt);
|
||||
|
|
@ -1933,6 +1979,9 @@ function loadOpenAISettings(data, settings) {
|
|||
$('#pres_pen_openai').val(oai_settings.pres_pen_openai);
|
||||
$('#pres_pen_counter_openai').text(Number(oai_settings.pres_pen_openai).toFixed(2));
|
||||
|
||||
$('#count_pen').val(oai_settings.count_pen);
|
||||
$('#count_pen_counter').text(Number(oai_settings.count_pen).toFixed(2));
|
||||
|
||||
$('#top_p_openai').val(oai_settings.top_p_openai);
|
||||
$('#top_p_counter_openai').text(Number(oai_settings.top_p_openai).toFixed(2));
|
||||
|
||||
|
|
@ -1975,7 +2024,7 @@ async function getStatusOpen() {
|
|||
return resultCheckStatusOpen();
|
||||
}
|
||||
|
||||
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE || oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {
|
||||
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE || oai_settings.chat_completion_source == chat_completion_sources.CLAUDE || oai_settings.chat_completion_source == chat_completion_sources.AI21) {
|
||||
let status = 'Unable to verify key; press "Test Message" to validate.';
|
||||
setOnlineStatus(status);
|
||||
return resultCheckStatusOpen();
|
||||
|
|
@ -2072,9 +2121,11 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
|
|||
claude_model: settings.claude_model,
|
||||
windowai_model: settings.windowai_model,
|
||||
openrouter_model: settings.openrouter_model,
|
||||
ai21_model: settings.ai21_model,
|
||||
temperature: settings.temp_openai,
|
||||
frequency_penalty: settings.freq_pen_openai,
|
||||
presence_penalty: settings.pres_pen_openai,
|
||||
count_penalty: settings.count_pen,
|
||||
top_p: settings.top_p_openai,
|
||||
top_k: settings.top_k_openai,
|
||||
openai_max_context: settings.openai_max_context,
|
||||
|
|
@ -2102,6 +2153,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
|
|||
api_url_scale: settings.api_url_scale,
|
||||
show_external_models: settings.show_external_models,
|
||||
assistant_prefill: settings.assistant_prefill,
|
||||
use_ai21_tokenizer: settings.use_ai21_tokenizer,
|
||||
};
|
||||
|
||||
const savePresetSettings = await fetch(`/savepreset_openai?name=${name}`, {
|
||||
|
|
@ -2401,6 +2453,7 @@ function onSettingsPresetChange() {
|
|||
temperature: ['#temp_openai', 'temp_openai', false],
|
||||
frequency_penalty: ['#freq_pen_openai', 'freq_pen_openai', false],
|
||||
presence_penalty: ['#pres_pen_openai', 'pres_pen_openai', false],
|
||||
count_penalty: ['#count_pen', 'count_pen', false],
|
||||
top_p: ['#top_p_openai', 'top_p_openai', false],
|
||||
top_k: ['#top_k_openai', 'top_k_openai', false],
|
||||
max_context_unlocked: ['#oai_max_context_unlocked', 'max_context_unlocked', true],
|
||||
|
|
@ -2408,6 +2461,7 @@ function onSettingsPresetChange() {
|
|||
claude_model: ['#model_claude_select', 'claude_model', false],
|
||||
windowai_model: ['#model_windowai_select', 'windowai_model', false],
|
||||
openrouter_model: ['#model_openrouter_select', 'openrouter_model', false],
|
||||
ai21_model: ['#model_ai21_select', 'ai21_model', false],
|
||||
openai_max_context: ['#openai_max_context', 'openai_max_context', false],
|
||||
openai_max_tokens: ['#openai_max_tokens', 'openai_max_tokens', false],
|
||||
wrap_in_quotes: ['#wrap_in_quotes', 'wrap_in_quotes', true],
|
||||
|
|
@ -2431,6 +2485,7 @@ function onSettingsPresetChange() {
|
|||
show_external_models: ['#openai_show_external_models', 'show_external_models', true],
|
||||
proxy_password: ['#openai_proxy_password', 'proxy_password', false],
|
||||
assistant_prefill: ['#claude_assistant_prefill', 'assistant_prefill', false],
|
||||
use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', false],
|
||||
};
|
||||
|
||||
const presetName = $('#settings_perset_openai').find(":selected").text();
|
||||
|
|
@ -2555,6 +2610,11 @@ async function onModelChange() {
|
|||
oai_settings.openrouter_model = value;
|
||||
}
|
||||
|
||||
if ($(this).is('#model_ai21_select')) {
|
||||
console.log('AI21 model changed to', value);
|
||||
oai_settings.ai21_model = value;
|
||||
}
|
||||
|
||||
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
|
||||
if (oai_settings.max_context_unlocked) {
|
||||
$('#openai_max_context').attr('max', unlocked_max);
|
||||
|
|
@ -2641,6 +2701,38 @@ async function onModelChange() {
|
|||
$('#temp_openai').attr('max', oai_max_temp).val(oai_settings.temp_openai).trigger('input');
|
||||
}
|
||||
|
||||
if (oai_settings.chat_completion_source == chat_completion_sources.AI21) {
|
||||
if (oai_settings.max_context_unlocked) {
|
||||
$('#openai_max_context').attr('max', unlocked_max);
|
||||
} else {
|
||||
$('#openai_max_context').attr('max', ai21_max);
|
||||
}
|
||||
|
||||
oai_settings.openai_max_context = Math.min(oai_settings.openai_max_context, Number($('#openai_max_context').attr('max')));
|
||||
$('#openai_max_context').val(oai_settings.openai_max_context).trigger('input');
|
||||
|
||||
oai_settings.temp_openai = Math.min(claude_max_temp, oai_settings.temp_openai);
|
||||
$('#temp_openai').attr('max', claude_max_temp).val(oai_settings.temp_openai).trigger('input');
|
||||
|
||||
oai_settings.freq_pen_openai = Math.min(j2_max_freq, oai_settings.freq_pen_openai < 0 ? 0 : oai_settings.freq_pen_openai);
|
||||
$('#freq_pen_openai').attr('min', 0).attr('max', j2_max_freq).val(oai_settings.freq_pen_openai).trigger('input');
|
||||
|
||||
oai_settings.pres_pen_openai = Math.min(j2_max_pres, oai_settings.pres_pen_openai < 0 ? 0 : oai_settings.pres_pen_openai);
|
||||
$('#pres_pen_openai').attr('min', 0).attr('max', j2_max_pres).val(oai_settings.pres_pen_openai).trigger('input');
|
||||
|
||||
oai_settings.top_k_openai = Math.min(j2_max_topk, oai_settings.top_k_openai);
|
||||
$('#top_k_openai').attr('max', j2_max_topk).val(oai_settings.top_k_openai).trigger('input');
|
||||
} else if (oai_settings.chat_completion_source != chat_completion_sources.AI21) {
|
||||
oai_settings.freq_pen_openai = Math.min(2.0, oai_settings.freq_pen_openai);
|
||||
$('#freq_pen_openai').attr('min', -2.0).attr('max', 2.0).val(oai_settings.freq_pen_openai).trigger('input');
|
||||
|
||||
oai_settings.freq_pen_openai = Math.min(2.0, oai_settings.pres_pen_openai);
|
||||
$('#pres_pen_openai').attr('min', -2.0).attr('max', 2.0).val(oai_settings.freq_pen_openai).trigger('input');
|
||||
|
||||
oai_settings.top_k_openai = Math.min(200, oai_settings.top_k_openai);
|
||||
$('#top_k_openai').attr('max', 200).val(oai_settings.top_k_openai).trigger('input');
|
||||
}
|
||||
|
||||
saveSettingsDebounced();
|
||||
eventSource.emit(event_types.CHATCOMPLETION_MODEL_CHANGED, value);
|
||||
}
|
||||
|
|
@ -2731,6 +2823,19 @@ async function onConnectButtonClick(e) {
|
|||
}
|
||||
}
|
||||
|
||||
if (oai_settings.chat_completion_source == chat_completion_sources.AI21) {
|
||||
const api_key_ai21 = $('#api_key_ai21').val().trim();
|
||||
|
||||
if (api_key_ai21.length) {
|
||||
await writeSecret(SECRET_KEYS.AI21, api_key_ai21);
|
||||
}
|
||||
|
||||
if (!secret_state[SECRET_KEYS.AI21] && !oai_settings.reverse_proxy) {
|
||||
console.log('No secret key saved for Claude');
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
$("#api_loading_openai").css("display", 'inline-block');
|
||||
$("#api_button_openai").css("display", 'none');
|
||||
saveSettingsDebounced();
|
||||
|
|
@ -2760,7 +2865,9 @@ function toggleChatCompletionForms() {
|
|||
else if (oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER) {
|
||||
$('#model_openrouter_select').trigger('change');
|
||||
}
|
||||
|
||||
else if (oai_settings.chat_completion_source == chat_completion_sources.AI21) {
|
||||
$('#model_ai21_select').trigger('change');
|
||||
}
|
||||
$('[data-source]').each(function () {
|
||||
const validSources = $(this).data('source').split(',');
|
||||
$(this).toggle(validSources.includes(oai_settings.chat_completion_source));
|
||||
|
|
@ -2818,7 +2925,12 @@ $(document).ready(async function () {
|
|||
oai_settings.pres_pen_openai = $(this).val();
|
||||
$('#pres_pen_counter_openai').text(Number($(this).val()).toFixed(2));
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
$(document).on('input', '#count_pen', function () {
|
||||
oai_settings.count_pen = $(this).val();
|
||||
$('#count_pen_counter').text(Number($(this).val()).toFixed(2));
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
$(document).on('input', '#top_p_openai', function () {
|
||||
|
|
@ -2856,6 +2968,14 @@ $(document).ready(async function () {
|
|||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
$('#use_ai21_tokenizer').on('change', function () {
|
||||
oai_settings.use_ai21_tokenizer = !!$('#use_ai21_tokenizer').prop('checked');
|
||||
oai_settings.use_ai21_tokenizer ? ai21_max = 8191: ai21_max = 9200;
|
||||
oai_settings.openai_max_context = Math.min(ai21_max, oai_settings.openai_max_context);
|
||||
$('#openai_max_context').attr('max', ai21_max).val(oai_settings.openai_max_context).trigger('input');
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
$('#names_in_completion').on('change', function () {
|
||||
oai_settings.names_in_completion = !!$('#names_in_completion').prop('checked');
|
||||
saveSettingsDebounced();
|
||||
|
|
@ -3023,6 +3143,7 @@ $(document).ready(async function () {
|
|||
$("#model_windowai_select").on("change", onModelChange);
|
||||
$("#model_scale_select").on("change", onModelChange);
|
||||
$("#model_openrouter_select").on("change", onModelChange);
|
||||
$("#model_ai21_select").on("change", onModelChange);
|
||||
$("#settings_perset_openai").on("change", onSettingsPresetChange);
|
||||
$("#new_oai_preset").on("click", onNewPresetClick);
|
||||
$("#delete_oai_preset").on("click", onDeletePresetClick);
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ export const SECRET_KEYS = {
|
|||
CLAUDE: 'api_key_claude',
|
||||
OPENROUTER: 'api_key_openrouter',
|
||||
SCALE: 'api_key_scale',
|
||||
AI21: 'api_key_ai21',
|
||||
}
|
||||
|
||||
const INPUT_MAP = {
|
||||
|
|
@ -18,6 +19,7 @@ const INPUT_MAP = {
|
|||
[SECRET_KEYS.CLAUDE]: '#api_key_claude',
|
||||
[SECRET_KEYS.OPENROUTER]: '#api_key_openrouter',
|
||||
[SECRET_KEYS.SCALE]: '#api_key_scale',
|
||||
[SECRET_KEYS.AI21]: '#api_key_ai21',
|
||||
}
|
||||
|
||||
async function clearSecret() {
|
||||
|
|
|
|||
95
server.js
95
server.js
|
|
@ -3290,6 +3290,10 @@ app.post("/generate_openai", jsonParser, function (request, response_generate_op
|
|||
return sendScaleRequest(request, response_generate_openai);
|
||||
}
|
||||
|
||||
if (request.body.use_ai21) {
|
||||
return sendAI21Request(request, response_generate_openai);
|
||||
}
|
||||
|
||||
let api_url;
|
||||
let api_key_openai;
|
||||
let headers;
|
||||
|
|
@ -3479,6 +3483,96 @@ app.post("/tokenize_openai", jsonParser, function (request, response_tokenize_op
|
|||
response_tokenize_openai.send({ "token_count": num_tokens });
|
||||
});
|
||||
|
||||
async function sendAI21Request(request, response) {
|
||||
if (!request.body) return response.sendStatus(400);
|
||||
const controller = new AbortController();
|
||||
console.log(request.body.messages)
|
||||
request.socket.removeAllListeners('close');
|
||||
request.socket.on('close', function () {
|
||||
controller.abort();
|
||||
});
|
||||
//console.log(request.body)
|
||||
const options = {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
accept: 'application/json',
|
||||
'content-type': 'application/json',
|
||||
Authorization: `Bearer ${readSecret(SECRET_KEYS.AI21)}`
|
||||
},
|
||||
body: JSON.stringify({
|
||||
numResults: 1,
|
||||
maxTokens: request.body.max_tokens,
|
||||
minTokens: 0,
|
||||
temperature: request.body.temperature,
|
||||
topP: request.body.top_p,
|
||||
stopSequences: request.body.stop_tokens,
|
||||
topKReturn: request.body.top_k,
|
||||
frequencyPenalty: {
|
||||
scale: request.body.frequency_penalty * 100,
|
||||
applyToWhitespaces: false,
|
||||
applyToPunctuations: false,
|
||||
applyToNumbers: false,
|
||||
applyToStopwords: false,
|
||||
applyToEmojis: false
|
||||
},
|
||||
presencePenalty: {
|
||||
scale: request.body.presence_penalty,
|
||||
applyToWhitespaces: false,
|
||||
applyToPunctuations: false,
|
||||
applyToNumbers: false,
|
||||
applyToStopwords: false,
|
||||
applyToEmojis: false
|
||||
},
|
||||
countPenalty: {
|
||||
scale: request.body.count_pen,
|
||||
applyToWhitespaces: false,
|
||||
applyToPunctuations: false,
|
||||
applyToNumbers: false,
|
||||
applyToStopwords: false,
|
||||
applyToEmojis: false
|
||||
},
|
||||
prompt: request.body.messages
|
||||
}),
|
||||
signal: controller.signal,
|
||||
};
|
||||
|
||||
fetch(`https://api.ai21.com/studio/v1/${request.body.model}/complete`, options)
|
||||
.then(r => r.json())
|
||||
.then(r => {
|
||||
if (r.completions === undefined) {
|
||||
console.log(r)
|
||||
} else {
|
||||
console.log(r.completions[0].data.text)
|
||||
}
|
||||
const reply = { choices: [{ "message": { "content": r.completions[0].data.text, } }] };
|
||||
return response.send(reply)
|
||||
})
|
||||
.catch(err => {
|
||||
console.error(err)
|
||||
return response.send({error: true})
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
app.post("/tokenize_ai21", jsonParser, function (request, response_tokenize_ai21 = response) {
|
||||
if (!request.body) return response_tokenize_ai21.sendStatus(400);
|
||||
console.log(request.body[0].content)
|
||||
const options = {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
accept: 'application/json',
|
||||
'content-type': 'application/json',
|
||||
Authorization: `Bearer ${readSecret(SECRET_KEYS.AI21)}`
|
||||
},
|
||||
body: JSON.stringify({text: request.body[0].content})
|
||||
};
|
||||
|
||||
fetch('https://api.ai21.com/studio/v1/tokenize', options)
|
||||
.then(response => response.json())
|
||||
.then(response => response_tokenize_ai21.send({"token_count": response.tokens.length}))
|
||||
.catch(err => console.error(err));
|
||||
});
|
||||
|
||||
app.post("/save_preset", jsonParser, function (request, response) {
|
||||
const name = sanitize(request.body.name);
|
||||
if (!request.body.preset || !name) {
|
||||
|
|
@ -3791,6 +3885,7 @@ const SECRET_KEYS = {
|
|||
DEEPL: 'deepl',
|
||||
OPENROUTER: 'api_key_openrouter',
|
||||
SCALE: 'api_key_scale',
|
||||
AI21: 'api_key_ai21'
|
||||
}
|
||||
|
||||
function migrateSecrets() {
|
||||
|
|
|
|||
Loading…
Reference in New Issue