rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #971 from kingbased/jurassic 

AI21 Adapter
This commit is contained in:
Cohee 2023-08-19 18:48:15 +03:00 committed by GitHub
parent 37fffa9930 0f21eabb6e
commit 438c78574f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 286 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
public/index.html
View File

@ -654,7 +654,7 @@
Max prompt cost: <span id="openrouter_max_prompt_cost">Unknown</span>
</div>
<hr>
<div class="range-block" data-source="openai,claude,windowai,openrouter">
<div class="range-block" data-source="openai,claude,windowai,openrouter,ai21">
<div class="range-block-title" data-i18n="Temperature">
Temperature
</div>
@ -669,7 +669,7 @@
</div>
</div>
</div>
<div class="range-block" data-source="openai,openrouter">
<div class="range-block" data-source="openai,openrouter,ai21">
<div class="range-block-title" data-i18n="Frequency Penalty">
Frequency Penalty
</div>
@ -684,7 +684,7 @@
</div>
</div>
</div>
<div class="range-block" data-source="openai,openrouter">
<div class="range-block" data-source="openai,openrouter,ai21">
<div class="range-block-title" data-i18n="Presence Penalty">
Presence Penalty
</div>
@ -699,7 +699,22 @@
</div>
</div>
</div>
<div class="range-block" data-source="claude,openrouter">
<div class="range-block" data-source="ai21">
<div class="range-block-title" data-i18n="Count Penalty">
Count Penalty
</div>
<div class="range-block-range-and-counter">
<div class="range-block-range">
<input type="range" id="count_pen" name="volume" min="0" max="1" step="0.01">
</div>
<div class="range-block-counter">
<div contenteditable="true" data-for="count_pen" id="count_pen_counter">
select
</div>
</div>
</div>
</div>
<div class="range-block" data-source="claude,openrouter,ai21">
<div class="range-block-title" data-i18n="Top K">
Top K
</div>
@ -714,7 +729,7 @@
</div>
</div>
</div>
<div class="range-block" data-source="openai,claude,openrouter">
<div class="range-block" data-source="openai,claude,openrouter,ai21">
<div class="range-block-title" data-i18n="Top-p">
Top P
</div>
@ -1418,6 +1433,14 @@
<span data-i18n="May help the model to understand context. Names must only contain letters or numbers.">Helps the model to associate messages in group chats. Names must only contain letters or numbers without whitespaces.</span>
</div>
</div>
<div class="range-block" data-source="ai21">
<label for="use_ai21_tokenizer" title="Use AI21 Tokenizer" class="checkbox_label widthFreeExpand">
<input id="use_ai21_tokenizer" type="checkbox" /><span data-i18n="Use AI21 Tokenizer">Use AI21 Tokenizer</span>
</label>
<div class="toggle-description justifyLeft">
<span data-i18n="Use the appropriate tokenizer for Jurassic models, which is more efficient than GPT's.">Use the appropriate tokenizer for Jurassic models, which is more efficient than GPT's.</span>
</div>
</div>
<div class="inline-drawer m-t-1 wide100p">
<div class="inline-drawer-toggle inline-drawer-header">
<b data-i18n="Quick Edit">Quick Edit</b>
@ -1618,7 +1641,7 @@
<option value="koboldhorde"><span data-i18n="KoboldAI Horde">KoboldAI Horde</span></option>
<option value="textgenerationwebui"><span data-i18n="Text Gen WebUI (ooba/Mancer)">Text Gen WebUI (ooba/Mancer)</span></option>
<option value="novel"><span data-i18n="NovelAI">NovelAI</span></option>
<option value="openai"><span data-i18n="Chat Completion (OpenAI, Claude, Window/OpenRouter, Scale)">Chat Completion (OpenAI, Claude, Window, OpenRouter, Scale)</span></option>
<option value="openai"><span data-i18n="Chat Completion (OpenAI, Claude, Window/OpenRouter, Scale, AI21)">Chat Completion (OpenAI, Claude, Window, OpenRouter, Scale, AI21)</span></option>
</select>
</div>
<div id="kobold_horde" style="position: relative;"> <!-- shows the kobold settings -->
@ -1804,6 +1827,7 @@
<option value="openrouter">OpenRouter</option>
<option value="claude">Claude</option>
<option value="scale">Scale</option>
<option value="ai21">AI21</option>
</select>
<form id="openai_form" data-source="openai" action="javascript:void(null);" method="post" enctype="multipart/form-data">
<h4><span data-i18n="OpenAI API key">OpenAI API key</span></h4>
@ -1972,6 +1996,27 @@
<select id="model_scale_select" class="displayNone"></select>
</form>
<form id="ai21_form" data-source="ai21" action="javascript:void(null);" method="post" enctype="multipart/form-data">
<h4>AI21 API Key</h4>
<div class="flex-container">
<input id="api_key_ai21" name="api_key_ai21" class="text_pole flex1" maxlength="500" value="" type="text" autocomplete="off">
<div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_ai21"></div>
</div>
<div data-for="api_key_ai21" class="neutral_warning">
For privacy reasons, your API key will be hidden after you reload the page.
</div>
<div>
<h4 data-i18n="AI21 Model">AI21 Model</h4>
<select id="model_ai21_select">
<optgroup label="Latest">
<option value="j2-ultra">j2-ultra</option>
<option value="j2-mid">j2-mid</option>
<option value="j2-light">j2-light</option>
</optgroup>
</select>
</div>
</form>
<div class="flex-container flex">
<input id="api_button_openai" class="menu_button" type="submit" value="Connect">
<input data-source="openrouter" id="openrouter_authorize" class="menu_button" type="button" value="Authorize" title="Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai" data-i18n="[title]Get your OpenRouter API token using OAuth flow. You will be redirected to openrouter.ai">

10
public/script.js
View File

@ -2131,7 +2131,7 @@ function baseChatReplace(value, name1, name2) {
}
function isStreamingEnabled() {
return ((main_api == 'openai' && oai_settings.stream_openai && oai_settings.chat_completion_source !== chat_completion_sources.SCALE)
return ((main_api == 'openai' && oai_settings.stream_openai && oai_settings.chat_completion_source !== chat_completion_sources.SCALE && oai_settings.chat_completion_source !== chat_completion_sources.AI21)
|| (main_api == 'kobold' && kai_settings.streaming_kobold && kai_settings.can_use_streaming)
|| (main_api == 'novel' && nai_settings.streaming_novel)
|| (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming))
@ -4738,6 +4738,7 @@ function changeMainAPI() {
case chat_completion_sources.WINDOWAI:
case chat_completion_sources.CLAUDE:
case chat_completion_sources.OPENAI:
case chat_completion_sources.AI21:
default:
setupChatCompletionPromptManager(oai_settings);
break;
@ -7178,6 +7179,11 @@ function connectAPISlash(_, text) {
source: 'openrouter',
button: '#api_button_openai',
},
'ai21': {
selected: 'openai',
source: 'ai21',
button: '#api_button_openai',
}
};
const apiConfig = apiMap[text];
@ -7396,7 +7402,7 @@ $(document).ready(function () {
}
registerSlashCommand('dupe', DupeChar, [], "– duplicates the currently selected character", true, true);
registerSlashCommand('api', connectAPISlash, [], "(kobold, horde, novel, ooba, oai, claude, windowai) – connect to an API", true, true);
registerSlashCommand('api', connectAPISlash, [], "(kobold, horde, novel, ooba, oai, claude, windowai, ai21) – connect to an API", true, true);
registerSlashCommand('impersonate', doImpersonate, ['imp'], "- calls an impersonation response", true, true);
registerSlashCommand('delchat', doDeleteChat, [], "- deletes the current chat", true, true);
registerSlashCommand('closechat', doCloseChat, [], "- closes the current chat", true, true);

1
public/scripts/RossAscends-mods.js
View File

@ -478,6 +478,7 @@ function RA_autoconnect(PrevApi) {
|| (secret_state[SECRET_KEYS.SCALE] && oai_settings.chat_completion_source == chat_completion_sources.SCALE)
|| (oai_settings.chat_completion_source == chat_completion_sources.WINDOWAI)
|| (secret_state[SECRET_KEYS.OPENROUTER] && oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER)
|| (secret_state[SECRET_KEYS.AI21] && oai_settings.chat_completion_source == chat_completion_sources.AI21)
) {
$("#api_button_openai").click();
}

137
public/scripts/openai.js
View File

@ -113,9 +113,13 @@ const scale_max = 7900; // Probably more. Save some for the system prompt define
const claude_max = 8000; // We have a proper tokenizer, so theoretically could be larger (up to 9k)
const palm2_max = 7500; // The real context window is 8192, spare some for padding due to using turbo tokenizer
const claude_100k_max = 99000;
let ai21_max = 9200; //can easily fit 9k gpt tokens because j2's tokenizer is efficient af
const unlocked_max = 100 * 1024;
const oai_max_temp = 2.0;
const claude_max_temp = 1.0;
const claude_max_temp = 1.0; //same as j2
const j2_max_topk = 10.0;
const j2_max_freq = 5.0;
const j2_max_pres = 5.0;
const openrouter_website_model = 'OR_Website';
let biasCache = undefined;
@ -161,13 +165,26 @@ export const chat_completion_sources = {
CLAUDE: 'claude',
SCALE: 'scale',
OPENROUTER: 'openrouter',
AI21: 'ai21',
};
const prefixMap = selected_group ? {
assistant: "",
user: "",
system: "OOC: "
}
: {
assistant: "{{char}}:",
user: "{{user}}:",
system: ""
};
const default_settings = {
preset_settings_openai: 'Default',
temp_openai: 0.9,
freq_pen_openai: 0.7,
pres_pen_openai: 0.7,
count_pen: 0.0,
top_p_openai: 1.0,
top_k_openai: 0,
stream_openai: false,
@ -188,6 +205,7 @@ const default_settings = {
wi_format: default_wi_format,
openai_model: 'gpt-3.5-turbo',
claude_model: 'claude-instant-v1',
ai21_model: 'j2-ultra',
windowai_model: '',
openrouter_model: openrouter_website_model,
jailbreak_system: false,
@ -199,6 +217,7 @@ const default_settings = {
show_external_models: false,
proxy_password: '',
assistant_prefill: '',
use_ai21_tokenizer: false,
};
const oai_settings = {
@ -206,6 +225,7 @@ const oai_settings = {
temp_openai: 1.0,
freq_pen_openai: 0,
pres_pen_openai: 0,
count_pen: 0.0,
top_p_openai: 1.0,
top_k_openai: 0,
stream_openai: false,
@ -226,6 +246,7 @@ const oai_settings = {
wi_format: default_wi_format,
openai_model: 'gpt-3.5-turbo',
claude_model: 'claude-instant-v1',
ai21_model: 'j2-ultra',
windowai_model: '',
openrouter_model: openrouter_website_model,
jailbreak_system: false,
@ -237,6 +258,7 @@ const oai_settings = {
show_external_models: false,
proxy_password: '',
assistant_prefill: '',
use_ai21_tokenizer: false,
};
let openai_setting_names;
@ -973,6 +995,8 @@ function getChatCompletionModel() {
return '';
case chat_completion_sources.OPENROUTER:
return oai_settings.openrouter_model !== openrouter_website_model ? oai_settings.openrouter_model : null;
case chat_completion_sources.AI21:
return oai_settings.ai21_model;
default:
throw new Error(`Unknown chat completion source: ${oai_settings.chat_completion_source}`);
}
@ -1054,10 +1078,19 @@ async function sendOpenAIRequest(type, openai_msgs_tosend, signal) {
const isClaude = oai_settings.chat_completion_source == chat_completion_sources.CLAUDE;
const isOpenRouter = oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER;
const isScale = oai_settings.chat_completion_source == chat_completion_sources.SCALE;
const isAI21 = oai_settings.chat_completion_source == chat_completion_sources.AI21;
const isTextCompletion = oai_settings.chat_completion_source == chat_completion_sources.OPENAI && (oai_settings.openai_model.startsWith('text-') || oai_settings.openai_model.startsWith('code-'));
const stream = type !== 'quiet' && oai_settings.stream_openai && !isScale;
const stream = type !== 'quiet' && oai_settings.stream_openai && !isScale && !isAI21;
const isQuiet = type === 'quiet';
if(isAI21) {
const joinedMsgs = openai_msgs_tosend.reduce((acc, obj) => {
const prefix = prefixMap[obj.role];
return acc + (prefix ? (selected_group ? "\n" : prefix + " ") : "") + obj.content + "\n";
}, "");
openai_msgs_tosend = substituteParams(joinedMsgs);
}
// If we're using the window.ai extension, use that instead
// Doesn't support logit bias yet
if (oai_settings.chat_completion_source == chat_completion_sources.WINDOWAI) {
@ -1113,6 +1146,13 @@ async function sendOpenAIRequest(type, openai_msgs_tosend, signal) {
generate_data['api_url_scale'] = oai_settings.api_url_scale;
}
if (isAI21) {
generate_data['use_ai21'] = true;
generate_data['top_k'] = parseFloat(oai_settings.top_k_openai);
generate_data['count_pen'] = parseFloat(oai_settings.count_pen);
generate_data['stop_tokens'] = [name1 + ':', 'prompt: [Start a new chat]'];
}
const generate_url = '/generate_openai';
const response = await fetch(generate_url, {
method: 'POST',
@ -1303,6 +1343,7 @@ class TokenHandler {
}
function countTokens(messages, full = false) {
let shouldTokenizeAI21 = oai_settings.chat_completion_source === chat_completion_sources.AI21 && oai_settings.use_ai21_tokenizer;
let chatId = 'undefined';
try {
@ -1335,12 +1376,13 @@ function countTokens(messages, full = false) {
if (typeof cachedCount === 'number') {
token_count += cachedCount;
}
else {
else {
console.log(JSON.stringify([message]));
jQuery.ajax({
async: false,
type: 'POST', //
url: `/tokenize_openai?model=${model}`,
url: shouldTokenizeAI21 ? '/tokenize_ai21' : `/tokenize_openai?model=${model}`,
data: JSON.stringify([message]),
dataType: "json",
contentType: "application/json",
@ -1856,6 +1898,7 @@ function loadOpenAISettings(data, settings) {
oai_settings.temp_openai = settings.temp_openai ?? default_settings.temp_openai;
oai_settings.freq_pen_openai = settings.freq_pen_openai ?? default_settings.freq_pen_openai;
oai_settings.pres_pen_openai = settings.pres_pen_openai ?? default_settings.pres_pen_openai;
oai_settings.count_pen = settings.count_pen ?? default_settings.count_pen;
oai_settings.top_p_openai = settings.top_p_openai ?? default_settings.top_p_openai;
oai_settings.top_k_openai = settings.top_k_openai ?? default_settings.top_k_openai;
oai_settings.stream_openai = settings.stream_openai ?? default_settings.stream_openai;
@ -1871,6 +1914,7 @@ function loadOpenAISettings(data, settings) {
oai_settings.claude_model = settings.claude_model ?? default_settings.claude_model;
oai_settings.windowai_model = settings.windowai_model ?? default_settings.windowai_model;
oai_settings.openrouter_model = settings.openrouter_model ?? default_settings.openrouter_model;
oai_settings.ai21_model = settings.ai21_model ?? default_settings.ai21_model;
oai_settings.chat_completion_source = settings.chat_completion_source ?? default_settings.chat_completion_source;
oai_settings.api_url_scale = settings.api_url_scale ?? default_settings.api_url_scale;
oai_settings.show_external_models = settings.show_external_models ?? default_settings.show_external_models;
@ -1889,7 +1933,7 @@ function loadOpenAISettings(data, settings) {
if (settings.wrap_in_quotes !== undefined) oai_settings.wrap_in_quotes = !!settings.wrap_in_quotes;
if (settings.names_in_completion !== undefined) oai_settings.names_in_completion = !!settings.names_in_completion;
if (settings.openai_model !== undefined) oai_settings.openai_model = settings.openai_model;
if (settings.use_ai21_tokenizer !== undefined) oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer;
$('#stream_toggle').prop('checked', oai_settings.stream_openai);
$('#api_url_scale').val(oai_settings.api_url_scale);
$('#openai_proxy_password').val(oai_settings.proxy_password);
@ -1901,6 +1945,8 @@ function loadOpenAISettings(data, settings) {
$(`#model_claude_select option[value="${oai_settings.claude_model}"`).attr('selected', true);
$('#model_windowai_select').val(oai_settings.windowai_model);
$(`#model_windowai_select option[value="${oai_settings.windowai_model}"`).attr('selected', true);
$('#model_ai21_select').val(oai_settings.ai21_model);
$(`#model_ai21_select option[value="${oai_settings.ai21_model}"`).attr('selected', true);
$('#openai_max_context').val(oai_settings.openai_max_context);
$('#openai_max_context_counter').text(`${oai_settings.openai_max_context}`);
$('#model_openrouter_select').val(oai_settings.openrouter_model);
@ -1916,7 +1962,7 @@ function loadOpenAISettings(data, settings) {
$('#legacy_streaming').prop('checked', oai_settings.legacy_streaming);
$('#openai_show_external_models').prop('checked', oai_settings.show_external_models);
$('#openai_external_category').toggle(oai_settings.show_external_models);
$('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer);
if (settings.impersonation_prompt !== undefined) oai_settings.impersonation_prompt = settings.impersonation_prompt;
$('#impersonation_prompt_textarea').val(oai_settings.impersonation_prompt);
@ -1939,6 +1985,9 @@ function loadOpenAISettings(data, settings) {
$('#pres_pen_openai').val(oai_settings.pres_pen_openai);
$('#pres_pen_counter_openai').text(Number(oai_settings.pres_pen_openai).toFixed(2));
$('#count_pen').val(oai_settings.count_pen);
$('#count_pen_counter').text(Number(oai_settings.count_pen).toFixed(2));
$('#top_p_openai').val(oai_settings.top_p_openai);
$('#top_p_counter_openai').text(Number(oai_settings.top_p_openai).toFixed(2));
@ -1981,7 +2030,7 @@ async function getStatusOpen() {
return resultCheckStatusOpen();
}
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE || oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) {
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE || oai_settings.chat_completion_source == chat_completion_sources.CLAUDE || oai_settings.chat_completion_source == chat_completion_sources.AI21) {
let status = 'Unable to verify key; press "Test Message" to validate.';
setOnlineStatus(status);
return resultCheckStatusOpen();
@ -2078,9 +2127,11 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
claude_model: settings.claude_model,
windowai_model: settings.windowai_model,
openrouter_model: settings.openrouter_model,
ai21_model: settings.ai21_model,
temperature: settings.temp_openai,
frequency_penalty: settings.freq_pen_openai,
presence_penalty: settings.pres_pen_openai,
count_penalty: settings.count_pen,
top_p: settings.top_p_openai,
top_k: settings.top_k_openai,
openai_max_context: settings.openai_max_context,
@ -2108,6 +2159,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
api_url_scale: settings.api_url_scale,
show_external_models: settings.show_external_models,
assistant_prefill: settings.assistant_prefill,
use_ai21_tokenizer: settings.use_ai21_tokenizer,
};
const savePresetSettings = await fetch(`/savepreset_openai?name=${name}`, {
@ -2407,6 +2459,7 @@ function onSettingsPresetChange() {
temperature: ['#temp_openai', 'temp_openai', false],
frequency_penalty: ['#freq_pen_openai', 'freq_pen_openai', false],
presence_penalty: ['#pres_pen_openai', 'pres_pen_openai', false],
count_penalty: ['#count_pen', 'count_pen', false],
top_p: ['#top_p_openai', 'top_p_openai', false],
top_k: ['#top_k_openai', 'top_k_openai', false],
max_context_unlocked: ['#oai_max_context_unlocked', 'max_context_unlocked', true],
@ -2414,6 +2467,7 @@ function onSettingsPresetChange() {
claude_model: ['#model_claude_select', 'claude_model', false],
windowai_model: ['#model_windowai_select', 'windowai_model', false],
openrouter_model: ['#model_openrouter_select', 'openrouter_model', false],
ai21_model: ['#model_ai21_select', 'ai21_model', false],
openai_max_context: ['#openai_max_context', 'openai_max_context', false],
openai_max_tokens: ['#openai_max_tokens', 'openai_max_tokens', false],
wrap_in_quotes: ['#wrap_in_quotes', 'wrap_in_quotes', true],
@ -2437,6 +2491,7 @@ function onSettingsPresetChange() {
show_external_models: ['#openai_show_external_models', 'show_external_models', true],
proxy_password: ['#openai_proxy_password', 'proxy_password', false],
assistant_prefill: ['#claude_assistant_prefill', 'assistant_prefill', false],
use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', false],
};
const presetName = $('#settings_perset_openai').find(":selected").text();
@ -2561,6 +2616,11 @@ async function onModelChange() {
oai_settings.openrouter_model = value;
}
if ($(this).is('#model_ai21_select')) {
console.log('AI21 model changed to', value);
oai_settings.ai21_model = value;
}
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
if (oai_settings.max_context_unlocked) {
$('#openai_max_context').attr('max', unlocked_max);
@ -2647,6 +2707,38 @@ async function onModelChange() {
$('#temp_openai').attr('max', oai_max_temp).val(oai_settings.temp_openai).trigger('input');
}
if (oai_settings.chat_completion_source == chat_completion_sources.AI21) {
if (oai_settings.max_context_unlocked) {
$('#openai_max_context').attr('max', unlocked_max);
} else {
$('#openai_max_context').attr('max', ai21_max);
}
oai_settings.openai_max_context = Math.min(oai_settings.openai_max_context, Number($('#openai_max_context').attr('max')));
$('#openai_max_context').val(oai_settings.openai_max_context).trigger('input');
oai_settings.temp_openai = Math.min(claude_max_temp, oai_settings.temp_openai);
$('#temp_openai').attr('max', claude_max_temp).val(oai_settings.temp_openai).trigger('input');
oai_settings.freq_pen_openai = Math.min(j2_max_freq, oai_settings.freq_pen_openai < 0 ? 0 : oai_settings.freq_pen_openai);
$('#freq_pen_openai').attr('min', 0).attr('max', j2_max_freq).val(oai_settings.freq_pen_openai).trigger('input');
oai_settings.pres_pen_openai = Math.min(j2_max_pres, oai_settings.pres_pen_openai < 0 ? 0 : oai_settings.pres_pen_openai);
$('#pres_pen_openai').attr('min', 0).attr('max', j2_max_pres).val(oai_settings.pres_pen_openai).trigger('input');
oai_settings.top_k_openai = Math.min(j2_max_topk, oai_settings.top_k_openai);
$('#top_k_openai').attr('max', j2_max_topk).val(oai_settings.top_k_openai).trigger('input');
} else if (oai_settings.chat_completion_source != chat_completion_sources.AI21) {
oai_settings.freq_pen_openai = Math.min(2.0, oai_settings.freq_pen_openai);
$('#freq_pen_openai').attr('min', -2.0).attr('max', 2.0).val(oai_settings.freq_pen_openai).trigger('input');
oai_settings.freq_pen_openai = Math.min(2.0, oai_settings.pres_pen_openai);
$('#pres_pen_openai').attr('min', -2.0).attr('max', 2.0).val(oai_settings.freq_pen_openai).trigger('input');
oai_settings.top_k_openai = Math.min(200, oai_settings.top_k_openai);
$('#top_k_openai').attr('max', 200).val(oai_settings.top_k_openai).trigger('input');
}
saveSettingsDebounced();
eventSource.emit(event_types.CHATCOMPLETION_MODEL_CHANGED, value);
}
@ -2737,6 +2829,19 @@ async function onConnectButtonClick(e) {
}
}
if (oai_settings.chat_completion_source == chat_completion_sources.AI21) {
const api_key_ai21 = $('#api_key_ai21').val().trim();
if (api_key_ai21.length) {
await writeSecret(SECRET_KEYS.AI21, api_key_ai21);
}
if (!secret_state[SECRET_KEYS.AI21] && !oai_settings.reverse_proxy) {
console.log('No secret key saved for Claude');
return;
}
}
$("#api_loading_openai").css("display", 'inline-block');
$("#api_button_openai").css("display", 'none');
saveSettingsDebounced();
@ -2766,7 +2871,9 @@ function toggleChatCompletionForms() {
else if (oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER) {
$('#model_openrouter_select').trigger('change');
}
else if (oai_settings.chat_completion_source == chat_completion_sources.AI21) {
$('#model_ai21_select').trigger('change');
}
$('[data-source]').each(function () {
const validSources = $(this).data('source').split(',');
$(this).toggle(validSources.includes(oai_settings.chat_completion_source));
@ -2824,7 +2931,12 @@ $(document).ready(async function () {
oai_settings.pres_pen_openai = $(this).val();
$('#pres_pen_counter_openai').text(Number($(this).val()).toFixed(2));
saveSettingsDebounced();
});
$(document).on('input', '#count_pen', function () {
oai_settings.count_pen = $(this).val();
$('#count_pen_counter').text(Number($(this).val()).toFixed(2));
saveSettingsDebounced();
});
$(document).on('input', '#top_p_openai', function () {
@ -2862,6 +2974,14 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#use_ai21_tokenizer').on('change', function () {
oai_settings.use_ai21_tokenizer = !!$('#use_ai21_tokenizer').prop('checked');
oai_settings.use_ai21_tokenizer ? ai21_max = 8191: ai21_max = 9200;
oai_settings.openai_max_context = Math.min(ai21_max, oai_settings.openai_max_context);
$('#openai_max_context').attr('max', ai21_max).val(oai_settings.openai_max_context).trigger('input');
saveSettingsDebounced();
});
$('#names_in_completion').on('change', function () {
oai_settings.names_in_completion = !!$('#names_in_completion').prop('checked');
saveSettingsDebounced();
@ -3029,6 +3149,7 @@ $(document).ready(async function () {
$("#model_windowai_select").on("change", onModelChange);
$("#model_scale_select").on("change", onModelChange);
$("#model_openrouter_select").on("change", onModelChange);
$("#model_ai21_select").on("change", onModelChange);
$("#settings_perset_openai").on("change", onSettingsPresetChange);
$("#new_oai_preset").on("click", onNewPresetClick);
$("#delete_oai_preset").on("click", onDeletePresetClick);

2
public/scripts/secrets.js
View File

@ -8,6 +8,7 @@ export const SECRET_KEYS = {
CLAUDE: 'api_key_claude',
OPENROUTER: 'api_key_openrouter',
SCALE: 'api_key_scale',
AI21: 'api_key_ai21',
}
const INPUT_MAP = {
@ -18,6 +19,7 @@ const INPUT_MAP = {
[SECRET_KEYS.CLAUDE]: '#api_key_claude',
[SECRET_KEYS.OPENROUTER]: '#api_key_openrouter',
[SECRET_KEYS.SCALE]: '#api_key_scale',
[SECRET_KEYS.AI21]: '#api_key_ai21',
}
async function clearSecret() {

95
server.js
View File

@ -3323,6 +3323,10 @@ app.post("/generate_openai", jsonParser, function (request, response_generate_op
return sendScaleRequest(request, response_generate_openai);
}
if (request.body.use_ai21) {
return sendAI21Request(request, response_generate_openai);
}
let api_url;
let api_key_openai;
let headers;
@ -3512,6 +3516,96 @@ app.post("/tokenize_openai", jsonParser, function (request, response_tokenize_op
response_tokenize_openai.send({ "token_count": num_tokens });
});
async function sendAI21Request(request, response) {
if (!request.body) return response.sendStatus(400);
const controller = new AbortController();
console.log(request.body.messages)
request.socket.removeAllListeners('close');
request.socket.on('close', function () {
controller.abort();
});
//console.log(request.body)
const options = {
method: 'POST',
headers: {
accept: 'application/json',
'content-type': 'application/json',
Authorization: `Bearer ${readSecret(SECRET_KEYS.AI21)}`
},
body: JSON.stringify({
numResults: 1,
maxTokens: request.body.max_tokens,
minTokens: 0,
temperature: request.body.temperature,
topP: request.body.top_p,
stopSequences: request.body.stop_tokens,
topKReturn: request.body.top_k,
frequencyPenalty: {
scale: request.body.frequency_penalty * 100,
applyToWhitespaces: false,
applyToPunctuations: false,
applyToNumbers: false,
applyToStopwords: false,
applyToEmojis: false
},
presencePenalty: {
scale: request.body.presence_penalty,
applyToWhitespaces: false,
applyToPunctuations: false,
applyToNumbers: false,
applyToStopwords: false,
applyToEmojis: false
},
countPenalty: {
scale: request.body.count_pen,
applyToWhitespaces: false,
applyToPunctuations: false,
applyToNumbers: false,
applyToStopwords: false,
applyToEmojis: false
},
prompt: request.body.messages
}),
signal: controller.signal,
};
fetch(`https://api.ai21.com/studio/v1/${request.body.model}/complete`, options)
.then(r => r.json())
.then(r => {
if (r.completions === undefined) {
console.log(r)
} else {
console.log(r.completions[0].data.text)
}
const reply = { choices: [{ "message": { "content": r.completions[0].data.text, } }] };
return response.send(reply)
})
.catch(err => {
console.error(err)
return response.send({error: true})
});
}
app.post("/tokenize_ai21", jsonParser, function (request, response_tokenize_ai21 = response) {
if (!request.body) return response_tokenize_ai21.sendStatus(400);
console.log(request.body[0].content)
const options = {
method: 'POST',
headers: {
accept: 'application/json',
'content-type': 'application/json',
Authorization: `Bearer ${readSecret(SECRET_KEYS.AI21)}`
},
body: JSON.stringify({text: request.body[0].content})
};
fetch('https://api.ai21.com/studio/v1/tokenize', options)
.then(response => response.json())
.then(response => response_tokenize_ai21.send({"token_count": response.tokens.length}))
.catch(err => console.error(err));
});
app.post("/save_preset", jsonParser, function (request, response) {
const name = sanitize(request.body.name);
if (!request.body.preset || !name) {
@ -3825,6 +3919,7 @@ const SECRET_KEYS = {
DEEPL: 'deepl',
OPENROUTER: 'api_key_openrouter',
SCALE: 'api_key_scale',
AI21: 'api_key_ai21'
}
function migrateSecrets() {