rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

bedrock claude 2 - non streaming mode

Browse Source
This commit is contained in:
Chen188
2024-03-18 03:46:29 +00:00
parent 2d152d2705
commit b2e5a31f9e
11 changed files with 1696 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
public/scripts/openai.js
View File

@@ -169,6 +169,7 @@ export const chat_completion_sources = {
MAKERSUITE: 'makersuite',
MISTRALAI: 'mistralai',
CUSTOM: 'custom',
BEDROCK: 'bedrock',
};
const prefixMap = selected_group ? {
@@ -286,6 +287,8 @@ const oai_settings = {
google_model: 'gemini-pro',
ai21_model: 'j2-ultra',
mistralai_model: 'mistral-medium',
bedrock_model: 'anthropic.claude-instant-v1',
bedrock_region: 'us-west-2',
custom_model: '',
custom_url: '',
custom_include_body: '',
@@ -1323,6 +1326,8 @@ function getChatCompletionModel() {
return oai_settings.mistralai_model;
case chat_completion_sources.CUSTOM:
return oai_settings.custom_model;
case chat_completion_sources.BEDROCK:
return oai_settings.bedrock_model;
default:
throw new Error(`Unknown chat completion source: ${oai_settings.chat_completion_source}`);
}
@@ -1535,6 +1540,7 @@ async function sendOpenAIRequest(type, messages, signal) {
let logit_bias = {};
const messageId = getNextMessageId(type);
const isClaude = oai_settings.chat_completion_source == chat_completion_sources.CLAUDE;
const isBedrock = oai_settings.chat_completion_source == chat_completion_sources.BEDROCK;
const isOpenRouter = oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER;
const isScale = oai_settings.chat_completion_source == chat_completion_sources.SCALE;
const isAI21 = oai_settings.chat_completion_source == chat_completion_sources.AI21;
@@ -1635,6 +1641,19 @@ async function sendOpenAIRequest(type, messages, signal) {
}
}
if (isBedrock) {
generate_data['top_k'] = Number(oai_settings.top_k_openai);
generate_data['exclude_assistant'] = oai_settings.exclude_assistant;
generate_data['claude_use_sysprompt'] = oai_settings.claude_use_sysprompt;
generate_data['claude_exclude_prefixes'] = oai_settings.claude_exclude_prefixes;
generate_data['stop'] = getCustomStoppingStrings(); // Claude shouldn't have limits on stop strings.
generate_data['human_sysprompt_message'] = substituteParams(oai_settings.human_sysprompt_message);
// Don't add a prefill on quiet gens (summarization)
if (!isQuiet && !oai_settings.exclude_assistant) {
generate_data['assistant_prefill'] = substituteParams(oai_settings.assistant_prefill);
}
}
if (isOpenRouter) {
generate_data['top_k'] = Number(oai_settings.top_k_openai);
generate_data['min_p'] = Number(oai_settings.min_p_openai);
@@ -2702,6 +2721,10 @@ async function getStatusOpen() {
data.custom_include_headers = oai_settings.custom_include_headers;
}
if (oai_settings.chat_completion_source === chat_completion_sources.BEDROCK) {
data.bedrock_region = oai_settings.bedrock_region;
}
const canBypass = (oai_settings.chat_completion_source === chat_completion_sources.OPENAI && oai_settings.bypass_status_check) || oai_settings.chat_completion_source === chat_completion_sources.CUSTOM;
if (canBypass) {
setOnlineStatus('Status check bypassed');
@@ -3360,6 +3383,11 @@ async function onModelChange() {
$('#custom_model_id').val(value).trigger('input');
}
if ($(this).is('#model_bedrock_select')) {
console.log('Bedrock model changed to', value);
oai_settings.bedrock_model = value;
}
if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) {
if (oai_settings.max_context_unlocked) {
$('#openai_max_context').attr('max', unlocked_max);
@@ -3513,6 +3541,29 @@ async function onModelChange() {
$('#openai_max_context').val(oai_settings.openai_max_context).trigger('input');
}
if (oai_settings.chat_completion_source == chat_completion_sources.BEDROCK) {
oai_settings.bedrock_region = String($('#aws_region_select').val());
if (oai_settings.max_context_unlocked) {
$('#openai_max_context').attr('max', max_200k);
}
else if (value == 'anthropic.claude-2:1') {
$('#openai_max_context').attr('max', max_200k);
}
else if (value.endsWith('100k') || value === 'anthropic.claude-instant-v1') {
$('#openai_max_context').attr('max', claude_100k_max);
}
else {
$('#openai_max_context').attr('max', claude_max);
}
oai_settings.openai_max_context = Math.min(oai_settings.openai_max_context, Number($('#openai_max_context').attr('max')));
$('#openai_max_context').val(oai_settings.openai_max_context).trigger('input');
oai_settings.temp_openai = Math.min(claude_max_temp, oai_settings.temp_openai);
$('#temp_openai').attr('max', claude_max_temp).val(oai_settings.temp_openai).trigger('input');
}
$('#openai_max_context_counter').attr('max', Number($('#openai_max_context').attr('max')));
saveSettingsDebounced();
@@ -3668,6 +3719,19 @@ async function onConnectButtonClick(e) {
}
}
if (oai_settings.chat_completion_source == chat_completion_sources.BEDROCK) {
const access_key_aws = String($('#access_key_aws').val()).trim();
const secret_key_aws = String($('#secret_key_aws').val()).trim();
if (access_key_aws.length && secret_key_aws.length) {
await writeSecret(SECRET_KEYS.BEDROCK, [access_key_aws, secret_key_aws]);
}
if (!secret_state[SECRET_KEYS.BEDROCK]) {
console.log('No secret key saved for Amazon Bedrock');
return;
}
}
startStatusLoading();
saveSettingsDebounced();
await getStatusOpen();
@@ -3706,6 +3770,9 @@ function toggleChatCompletionForms() {
else if (oai_settings.chat_completion_source == chat_completion_sources.CUSTOM) {
$('#model_custom_select').trigger('change');
}
else if (oai_settings.chat_completion_source == chat_completion_sources.BEDROCK) {
$('#model_bedrock_select').trigger('change');
}
$('[data-source]').each(function () {
const validSources = $(this).data('source').split(',');
$(this).toggle(validSources.includes(oai_settings.chat_completion_source));
@@ -3730,6 +3797,7 @@ async function testApiConnection() {
toastr.success('API connection successful!');
}
catch (err) {
console.log(err);
toastr.error('Could not get a reply from API. Check your connection settings / API key and try again.');
}
}
@@ -4320,6 +4388,8 @@ $(document).ready(async function () {
$('#model_ai21_select').on('change', onModelChange);
$('#model_mistralai_select').on('change', onModelChange);
$('#model_custom_select').on('change', onModelChange);
$('#model_bedrock_select').on('change', onModelChange);
$('#aws_region_select').on('change', onModelChange);
$('#settings_preset_openai').on('change', onSettingsPresetChange);
$('#new_oai_preset').on('click', onNewPresetClick);
$('#delete_oai_preset').on('click', onDeletePresetClick);