From 69e24c96866f7ec9e7dbd9ce68c7c0523bca5fbe Mon Sep 17 00:00:00 2001 From: based Date: Thu, 14 Dec 2023 11:14:41 +1000 Subject: [PATCH 01/21] change palm naming in UI --- public/index.html | 27 ++++++++++++++++++--------- public/script.js | 8 ++++---- public/scripts/RossAscends-mods.js | 2 +- public/scripts/openai.js | 8 ++++---- public/scripts/secrets.js | 4 ++-- server.js | 6 +++--- src/endpoints/secrets.js | 2 +- 7 files changed, 33 insertions(+), 24 deletions(-) diff --git a/public/index.html b/public/index.html index 348a47fa3..939440aaa 100644 --- a/public/index.html +++ b/public/index.html @@ -444,7 +444,7 @@ complete. -
+
Temperature
@@ -496,7 +496,7 @@
-
+
Top K
@@ -509,7 +509,7 @@
-
+
Top P
@@ -1585,7 +1585,7 @@ - +
@@ -1833,7 +1833,7 @@ - +

OpenAI API key

@@ -2100,14 +2100,23 @@
-

PaLM API Key

+

MakerSuite API Key

- -
+ +
-
+
For privacy reasons, your API key will be hidden after you reload the page.
+
+

Google Model

+ +
diff --git a/public/script.js b/public/script.js index c9d45baf7..cefd9d50b 100644 --- a/public/script.js +++ b/public/script.js @@ -2557,7 +2557,7 @@ function getCharacterCardFields() { } function isStreamingEnabled() { - const noStreamSources = [chat_completion_sources.SCALE, chat_completion_sources.AI21, chat_completion_sources.PALM]; + const noStreamSources = [chat_completion_sources.SCALE, chat_completion_sources.AI21, chat_completion_sources.MAKERSUITE]; return ((main_api == 'openai' && oai_settings.stream_openai && !noStreamSources.includes(oai_settings.chat_completion_source)) || (main_api == 'kobold' && kai_settings.streaming_kobold && kai_flags.can_use_streaming) || (main_api == 'novel' && nai_settings.streaming_novel) @@ -5395,7 +5395,7 @@ function changeMainAPI() { case chat_completion_sources.CLAUDE: case chat_completion_sources.OPENAI: case chat_completion_sources.AI21: - case chat_completion_sources.PALM: + case chat_completion_sources.MAKERSUITE: default: setupChatCompletionPromptManager(oai_settings); break; @@ -7535,9 +7535,9 @@ async function connectAPISlash(_, text) { source: 'ai21', button: '#api_button_openai', }, - 'palm': { + 'makersuite': { selected: 'openai', - source: 'palm', + source: 'makersuite', button: '#api_button_openai', }, }; diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index f5bad628b..7235763b6 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -415,7 +415,7 @@ function RA_autoconnect(PrevApi) { || (oai_settings.chat_completion_source == chat_completion_sources.WINDOWAI) || (secret_state[SECRET_KEYS.OPENROUTER] && oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER) || (secret_state[SECRET_KEYS.AI21] && oai_settings.chat_completion_source == chat_completion_sources.AI21) - || (secret_state[SECRET_KEYS.PALM] && oai_settings.chat_completion_source == chat_completion_sources.PALM) + || (secret_state[SECRET_KEYS.MAKERSUITE] && oai_settings.chat_completion_source == chat_completion_sources.PALM) ) { $('#api_button_openai').trigger('click'); } diff --git a/public/scripts/openai.js b/public/scripts/openai.js index a9bc5e304..7741d6c9d 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -164,7 +164,7 @@ export const chat_completion_sources = { SCALE: 'scale', OPENROUTER: 'openrouter', AI21: 'ai21', - PALM: 'palm', + MAKERSUITE: 'makersuite', }; const prefixMap = selected_group ? { @@ -3255,10 +3255,10 @@ async function onConnectButtonClick(e) { } if (oai_settings.chat_completion_source == chat_completion_sources.PALM) { - const api_key_palm = String($('#api_key_palm').val()).trim(); + const api_key_makersuite = String($('#api_key_makersuite').val()).trim(); - if (api_key_palm.length) { - await writeSecret(SECRET_KEYS.PALM, api_key_palm); + if (api_key_makersuite.length) { + await writeSecret(SECRET_KEYS.PALM, api_key_makersuite); } if (!secret_state[SECRET_KEYS.PALM]) { diff --git a/public/scripts/secrets.js b/public/scripts/secrets.js index 84279641d..6afb538f1 100644 --- a/public/scripts/secrets.js +++ b/public/scripts/secrets.js @@ -12,7 +12,7 @@ export const SECRET_KEYS = { SCALE: 'api_key_scale', AI21: 'api_key_ai21', SCALE_COOKIE: 'scale_cookie', - PALM: 'api_key_palm', + MAKERSUITE: 'api_key_makersuite', SERPAPI: 'api_key_serpapi', }; @@ -26,7 +26,7 @@ const INPUT_MAP = { [SECRET_KEYS.SCALE]: '#api_key_scale', [SECRET_KEYS.AI21]: '#api_key_ai21', [SECRET_KEYS.SCALE_COOKIE]: '#scale_cookie', - [SECRET_KEYS.PALM]: '#api_key_palm', + [SECRET_KEYS.MAKERSUITE]: '#api_key_makersuite', [SECRET_KEYS.APHRODITE]: '#api_key_aphrodite', [SECRET_KEYS.TABBY]: '#api_key_tabby', }; diff --git a/server.js b/server.js index 2374df5a2..6da29c278 100644 --- a/server.js +++ b/server.js @@ -995,9 +995,9 @@ async function sendClaudeRequest(request, response) { * @param {express.Response} response */ async function sendPalmRequest(request, response) { - const api_key_palm = readSecret(SECRET_KEYS.PALM); + const api_key_makersuite = readSecret(SECRET_KEYS.PALM); - if (!api_key_palm) { + if (!api_key_makersuite) { console.log('Palm API key is missing.'); return response.status(400).send({ error: true }); } @@ -1024,7 +1024,7 @@ async function sendPalmRequest(request, response) { controller.abort(); }); - const generateResponse = await fetch(`https://generativelanguage.googleapis.com/v1beta2/models/text-bison-001:generateText?key=${api_key_palm}`, { + const generateResponse = await fetch(`https://generativelanguage.googleapis.com/v1beta2/models/text-bison-001:generateText?key=${api_key_makersuite}`, { body: JSON.stringify(body), method: 'POST', headers: { diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index 54687cbeb..e4705706f 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -23,7 +23,7 @@ const SECRET_KEYS = { SCALE_COOKIE: 'scale_cookie', ONERING_URL: 'oneringtranslator_url', DEEPLX_URL: 'deeplx_url', - PALM: 'api_key_palm', + PALM: 'api_key_makersuite', SERPAPI: 'api_key_serpapi', }; From be396991de5f35e93cd16ec1c5e2ea7b77842962 Mon Sep 17 00:00:00 2001 From: based Date: Thu, 14 Dec 2023 11:53:26 +1000 Subject: [PATCH 02/21] finish implementing ui changes for google models --- public/index.html | 4 +-- public/script.js | 2 +- public/scripts/RossAscends-mods.js | 2 +- public/scripts/openai.js | 51 +++++++++++++++++++----------- src/endpoints/secrets.js | 2 +- 5 files changed, 37 insertions(+), 24 deletions(-) diff --git a/public/index.html b/public/index.html index 939440aaa..474d605de 100644 --- a/public/index.html +++ b/public/index.html @@ -2099,7 +2099,7 @@
-
+

MakerSuite API Key

@@ -2117,8 +2117,6 @@
- -
diff --git a/public/script.js b/public/script.js index cefd9d50b..5ed6f7c51 100644 --- a/public/script.js +++ b/public/script.js @@ -7826,7 +7826,7 @@ jQuery(async function () { } registerSlashCommand('dupe', DupeChar, [], '– duplicates the currently selected character', true, true); - registerSlashCommand('api', connectAPISlash, [], '(kobold, horde, novel, ooba, tabby, mancer, aphrodite, kcpp, oai, claude, windowai, openrouter, scale, ai21, palm) – connect to an API', true, true); + registerSlashCommand('api', connectAPISlash, [], '(kobold, horde, novel, ooba, tabby, mancer, aphrodite, kcpp, oai, claude, windowai, openrouter, scale, ai21, makersuite) – connect to an API', true, true); registerSlashCommand('impersonate', doImpersonate, ['imp'], '– calls an impersonation response', true, true); registerSlashCommand('delchat', doDeleteChat, [], '– deletes the current chat', true, true); registerSlashCommand('closechat', doCloseChat, [], '– closes the current chat', true, true); diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index 7235763b6..0cf4fa5c3 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -415,7 +415,7 @@ function RA_autoconnect(PrevApi) { || (oai_settings.chat_completion_source == chat_completion_sources.WINDOWAI) || (secret_state[SECRET_KEYS.OPENROUTER] && oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER) || (secret_state[SECRET_KEYS.AI21] && oai_settings.chat_completion_source == chat_completion_sources.AI21) - || (secret_state[SECRET_KEYS.MAKERSUITE] && oai_settings.chat_completion_source == chat_completion_sources.PALM) + || (secret_state[SECRET_KEYS.MAKERSUITE] && oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) ) { $('#api_button_openai').trigger('click'); } diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 7741d6c9d..644f4f195 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -114,7 +114,6 @@ const max_128k = 128 * 1000; const max_200k = 200 * 1000; const scale_max = 8191; const claude_max = 9000; // We have a proper tokenizer, so theoretically could be larger (up to 9k) -const palm2_max = 7400; // The real context window is 8192, spare some for padding due to using turbo tokenizer const claude_100k_max = 99000; let ai21_max = 9200; //can easily fit 9k gpt tokens because j2's tokenizer is efficient af const unlocked_max = 100 * 1024; @@ -207,6 +206,7 @@ const default_settings = { personality_format: default_personality_format, openai_model: 'gpt-3.5-turbo', claude_model: 'claude-instant-v1', + google_model: 'gemini-pro', ai21_model: 'j2-ultra', windowai_model: '', openrouter_model: openrouter_website_model, @@ -260,6 +260,7 @@ const oai_settings = { personality_format: default_personality_format, openai_model: 'gpt-3.5-turbo', claude_model: 'claude-instant-v1', + google_model: 'gemini-pro', ai21_model: 'j2-ultra', windowai_model: '', openrouter_model: openrouter_website_model, @@ -1252,8 +1253,8 @@ function getChatCompletionModel() { return oai_settings.windowai_model; case chat_completion_sources.SCALE: return ''; - case chat_completion_sources.PALM: - return ''; + case chat_completion_sources.MAKERSUITE: + return oai_settings.google_model; case chat_completion_sources.OPENROUTER: return oai_settings.openrouter_model !== openrouter_website_model ? oai_settings.openrouter_model : null; case chat_completion_sources.AI21: @@ -1443,20 +1444,20 @@ async function sendOpenAIRequest(type, messages, signal) { const isOpenRouter = oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER; const isScale = oai_settings.chat_completion_source == chat_completion_sources.SCALE; const isAI21 = oai_settings.chat_completion_source == chat_completion_sources.AI21; - const isPalm = oai_settings.chat_completion_source == chat_completion_sources.PALM; + const isGoogle = oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE; const isOAI = oai_settings.chat_completion_source == chat_completion_sources.OPENAI; const isTextCompletion = (isOAI && textCompletionModels.includes(oai_settings.openai_model)) || (isOpenRouter && oai_settings.openrouter_force_instruct && power_user.instruct.enabled); const isQuiet = type === 'quiet'; const isImpersonate = type === 'impersonate'; const isContinue = type === 'continue'; - const stream = oai_settings.stream_openai && !isQuiet && !isScale && !isAI21 && !isPalm; + const stream = oai_settings.stream_openai && !isQuiet && !isScale && !isAI21 && !isGoogle; if (isTextCompletion && isOpenRouter) { messages = convertChatCompletionToInstruct(messages, type); replaceItemizedPromptText(messageId, messages); } - if (isAI21 || isPalm) { + if (isAI21 || isGoogle) { const joinedMsgs = messages.reduce((acc, obj) => { const prefix = prefixMap[obj.role]; return acc + (prefix ? (selected_group ? '\n' : prefix + ' ') : '') + obj.content + '\n'; @@ -1539,7 +1540,7 @@ async function sendOpenAIRequest(type, messages, signal) { generate_data['api_url_scale'] = oai_settings.api_url_scale; } - if (isPalm) { + if (isGoogle) { const nameStopString = isImpersonate ? `\n${name2}:` : `\n${name1}:`; const stopStringsLimit = 3; // 5 - 2 (nameStopString and new_chat_prompt) generate_data['top_k'] = Number(oai_settings.top_k_openai); @@ -2290,6 +2291,7 @@ function loadOpenAISettings(data, settings) { oai_settings.openrouter_use_fallback = settings.openrouter_use_fallback ?? default_settings.openrouter_use_fallback; oai_settings.openrouter_force_instruct = settings.openrouter_force_instruct ?? default_settings.openrouter_force_instruct; oai_settings.ai21_model = settings.ai21_model ?? default_settings.ai21_model; + oai_settings.google_model = settings.google_model ?? default_settings.google_model; oai_settings.chat_completion_source = settings.chat_completion_source ?? default_settings.chat_completion_source; oai_settings.api_url_scale = settings.api_url_scale ?? default_settings.api_url_scale; oai_settings.show_external_models = settings.show_external_models ?? default_settings.show_external_models; @@ -2326,6 +2328,8 @@ function loadOpenAISettings(data, settings) { $(`#model_claude_select option[value="${oai_settings.claude_model}"`).attr('selected', true); $('#model_windowai_select').val(oai_settings.windowai_model); $(`#model_windowai_select option[value="${oai_settings.windowai_model}"`).attr('selected', true); + $('#model_google_select').val(oai_settings.google_model); + $(`#model_google_select option[value="${oai_settings.google_model}"`).attr('selected', true); $('#model_ai21_select').val(oai_settings.ai21_model); $(`#model_ai21_select option[value="${oai_settings.ai21_model}"`).attr('selected', true); $('#openai_max_context').val(oai_settings.openai_max_context); @@ -2416,7 +2420,7 @@ async function getStatusOpen() { return resultCheckStatus(); } - const noValidateSources = [chat_completion_sources.SCALE, chat_completion_sources.CLAUDE, chat_completion_sources.AI21, chat_completion_sources.PALM]; + const noValidateSources = [chat_completion_sources.SCALE, chat_completion_sources.CLAUDE, chat_completion_sources.AI21, chat_completion_sources.MAKERSUITE]; if (noValidateSources.includes(oai_settings.chat_completion_source)) { let status = 'Unable to verify key; press "Test Message" to validate.'; setOnlineStatus(status); @@ -2499,6 +2503,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) { openrouter_group_models: settings.openrouter_group_models, openrouter_sort_models: settings.openrouter_sort_models, ai21_model: settings.ai21_model, + google_model: settings.google_model, temperature: settings.temp_openai, frequency_penalty: settings.freq_pen_openai, presence_penalty: settings.pres_pen_openai, @@ -2868,6 +2873,7 @@ function onSettingsPresetChange() { openrouter_group_models: ['#openrouter_group_models', 'openrouter_group_models', false], openrouter_sort_models: ['#openrouter_sort_models', 'openrouter_sort_models', false], ai21_model: ['#model_ai21_select', 'ai21_model', false], + google_model: ['#model_google_select', 'google_model', false], openai_max_context: ['#openai_max_context', 'openai_max_context', false], openai_max_tokens: ['#openai_max_tokens', 'openai_max_tokens', false], wrap_in_quotes: ['#wrap_in_quotes', 'wrap_in_quotes', true], @@ -3000,7 +3006,7 @@ function getMaxContextWindowAI(value) { return max_8k; } else if (value.includes('palm-2')) { - return palm2_max; + return max_8k; } else if (value.includes('GPT-NeoXT')) { return max_2k; @@ -3045,6 +3051,11 @@ async function onModelChange() { oai_settings.ai21_model = value; } + if ($(this).is('#model_google_select')) { + console.log('Google model changed to', value); + oai_settings.google_model = value; + } + if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) { if (oai_settings.max_context_unlocked) { $('#openai_max_context').attr('max', unlocked_max); @@ -3055,11 +3066,15 @@ async function onModelChange() { $('#openai_max_context').val(oai_settings.openai_max_context).trigger('input'); } - if (oai_settings.chat_completion_source == chat_completion_sources.PALM) { + if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) { if (oai_settings.max_context_unlocked) { $('#openai_max_context').attr('max', unlocked_max); + } else if (value === 'gemini-pro') { + $('#openai_max_context').attr('max', max_32k); + } else if (value === 'gemini-pro-vision') { + $('#openai_max_context').attr('max', max_16k); } else { - $('#openai_max_context').attr('max', palm2_max); + $('#openai_max_context').attr('max', max_8k); } oai_settings.openai_max_context = Math.min(Number($('#openai_max_context').attr('max')), oai_settings.openai_max_context); @@ -3254,15 +3269,15 @@ async function onConnectButtonClick(e) { } } - if (oai_settings.chat_completion_source == chat_completion_sources.PALM) { + if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) { const api_key_makersuite = String($('#api_key_makersuite').val()).trim(); if (api_key_makersuite.length) { - await writeSecret(SECRET_KEYS.PALM, api_key_makersuite); + await writeSecret(SECRET_KEYS.MAKERSUITE, api_key_makersuite); } - if (!secret_state[SECRET_KEYS.PALM]) { - console.log('No secret key saved for PALM'); + if (!secret_state[SECRET_KEYS.MAKERSUITE]) { + console.log('No secret key saved for MakerSuite'); return; } } @@ -3329,8 +3344,8 @@ function toggleChatCompletionForms() { else if (oai_settings.chat_completion_source == chat_completion_sources.SCALE) { $('#model_scale_select').trigger('change'); } - else if (oai_settings.chat_completion_source == chat_completion_sources.PALM) { - $('#model_palm_select').trigger('change'); + else if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) { + $('#model_google_select').trigger('change'); } else if (oai_settings.chat_completion_source == chat_completion_sources.OPENROUTER) { $('#model_openrouter_select').trigger('change'); @@ -3702,7 +3717,7 @@ $(document).ready(async function () { $('#model_claude_select').on('change', onModelChange); $('#model_windowai_select').on('change', onModelChange); $('#model_scale_select').on('change', onModelChange); - $('#model_palm_select').on('change', onModelChange); + $('#model_google_select').on('change', onModelChange); $('#model_openrouter_select').on('change', onModelChange); $('#openrouter_group_models').on('change', onOpenrouterModelSortChange); $('#openrouter_sort_models').on('change', onOpenrouterModelSortChange); diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index e4705706f..c997e5efe 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -23,7 +23,7 @@ const SECRET_KEYS = { SCALE_COOKIE: 'scale_cookie', ONERING_URL: 'oneringtranslator_url', DEEPLX_URL: 'deeplx_url', - PALM: 'api_key_makersuite', + MAKERSUITE: 'api_key_makersuite', SERPAPI: 'api_key_serpapi', }; From e26159c00d8190ece95e9778009e1f763f205321 Mon Sep 17 00:00:00 2001 From: based Date: Thu, 14 Dec 2023 15:49:50 +1000 Subject: [PATCH 03/21] refactor and rework palm request to work with the 'content' format and added an endpoint for googles tokenizer --- public/img/{palm.svg => makersuite.svg} | 0 public/scripts/openai.js | 2 +- public/scripts/tokenizers.js | 17 ++++++- server.js | 60 ++++++++++++++----------- src/chat-completion.js | 30 +++++++++++++ src/constants.js | 22 +++------ src/endpoints/tokenizers.js | 21 +++++++++ src/palm-vectors.js | 2 +- 8 files changed, 108 insertions(+), 46 deletions(-) rename public/img/{palm.svg => makersuite.svg} (100%) diff --git a/public/img/palm.svg b/public/img/makersuite.svg similarity index 100% rename from public/img/palm.svg rename to public/img/makersuite.svg diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 644f4f195..015dd66a6 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -1457,7 +1457,7 @@ async function sendOpenAIRequest(type, messages, signal) { replaceItemizedPromptText(messageId, messages); } - if (isAI21 || isGoogle) { + if (isAI21) { const joinedMsgs = messages.reduce((acc, obj) => { const prefix = prefixMap[obj.role]; return acc + (prefix ? (selected_group ? '\n' : prefix + ' ') : '') + obj.content + '\n'; diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index decd0f919..7600a0909 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -376,6 +376,10 @@ export function getTokenizerModel() { } } + if(oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) { + return oai_settings.google_model; + } + if (oai_settings.chat_completion_source == chat_completion_sources.CLAUDE) { return claudeTokenizer; } @@ -389,6 +393,15 @@ export function getTokenizerModel() { */ export function countTokensOpenAI(messages, full = false) { const shouldTokenizeAI21 = oai_settings.chat_completion_source === chat_completion_sources.AI21 && oai_settings.use_ai21_tokenizer; + const shouldTokenizeGoogle = oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE; + let tokenizerEndpoint = ''; + if(shouldTokenizeAI21) { + tokenizerEndpoint = '/api/tokenizers/ai21/count'; + } else if (shouldTokenizeGoogle) { + tokenizerEndpoint = `/api/tokenizers/google/count?model=${getTokenizerModel()}`; + } else { + tokenizerEndpoint = `/api/tokenizers/openai/count?model=${getTokenizerModel()}`; + } const cacheObject = getTokenCacheObject(); if (!Array.isArray(messages)) { @@ -400,7 +413,7 @@ export function countTokensOpenAI(messages, full = false) { for (const message of messages) { const model = getTokenizerModel(); - if (model === 'claude' || shouldTokenizeAI21) { + if (model === 'claude' || shouldTokenizeAI21 || shouldTokenizeGoogle) { full = true; } @@ -416,7 +429,7 @@ export function countTokensOpenAI(messages, full = false) { jQuery.ajax({ async: false, type: 'POST', // - url: shouldTokenizeAI21 ? '/api/tokenizers/ai21/count' : `/api/tokenizers/openai/count?model=${model}`, + url: tokenizerEndpoint, data: JSON.stringify([message]), dataType: 'json', contentType: 'application/json', diff --git a/server.js b/server.js index 6da29c278..ac56f54fb 100644 --- a/server.js +++ b/server.js @@ -59,7 +59,7 @@ const { } = require('./src/util'); const { ensureThumbnailCache } = require('./src/endpoints/thumbnails'); const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers'); -const { convertClaudePrompt } = require('./src/chat-completion'); +const { convertClaudePrompt, convertGooglePrompt } = require('./src/chat-completion'); // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. // https://github.com/nodejs/node/issues/47822#issuecomment-1564708870 @@ -131,7 +131,7 @@ const API_OPENAI = 'https://api.openai.com/v1'; const API_CLAUDE = 'https://api.anthropic.com/v1'; const SETTINGS_FILE = './public/settings.json'; -const { DIRECTORIES, UPLOADS_PATH, PALM_SAFETY, CHAT_COMPLETION_SOURCES, AVATAR_WIDTH, AVATAR_HEIGHT } = require('./src/constants'); +const { DIRECTORIES, UPLOADS_PATH, MAKERSUITE_SAFETY, CHAT_COMPLETION_SOURCES, AVATAR_WIDTH, AVATAR_HEIGHT } = require('./src/constants'); // CORS Settings // const CORS = cors({ @@ -994,29 +994,30 @@ async function sendClaudeRequest(request, response) { * @param {express.Request} request * @param {express.Response} response */ -async function sendPalmRequest(request, response) { - const api_key_makersuite = readSecret(SECRET_KEYS.PALM); +async function sendMakerSuiteRequest(request, response) { + const api_key_makersuite = readSecret(SECRET_KEYS.MAKERSUITE); if (!api_key_makersuite) { - console.log('Palm API key is missing.'); + console.log('MakerSuite API key is missing.'); return response.status(400).send({ error: true }); } - const body = { - prompt: { - text: request.body.messages, - }, + const generationConfig = { stopSequences: request.body.stop, - safetySettings: PALM_SAFETY, + candidateCount: 1, + maxOutputTokens: request.body.max_tokens, temperature: request.body.temperature, topP: request.body.top_p, topK: request.body.top_k || undefined, - maxOutputTokens: request.body.max_tokens, - candidate_count: 1, }; - console.log('Palm request:', body); + const body = { + contents: convertGooglePrompt(request.body.messages), + safetySettings: MAKERSUITE_SAFETY, + generationConfig: generationConfig, + }; + const google_model = request.body.model; try { const controller = new AbortController(); request.socket.removeAllListeners('close'); @@ -1024,7 +1025,7 @@ async function sendPalmRequest(request, response) { controller.abort(); }); - const generateResponse = await fetch(`https://generativelanguage.googleapis.com/v1beta2/models/text-bison-001:generateText?key=${api_key_makersuite}`, { + const generateResponse = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${google_model}:generateContent?key=${api_key_makersuite}`, { body: JSON.stringify(body), method: 'POST', headers: { @@ -1035,32 +1036,37 @@ async function sendPalmRequest(request, response) { }); if (!generateResponse.ok) { - console.log(`Palm API returned error: ${generateResponse.status} ${generateResponse.statusText} ${await generateResponse.text()}`); + console.log(`MakerSuite API returned error: ${generateResponse.status} ${generateResponse.statusText} ${await generateResponse.text()}`); return response.status(generateResponse.status).send({ error: true }); } const generateResponseJson = await generateResponse.json(); - const responseText = generateResponseJson?.candidates[0]?.output; - if (!responseText) { - console.log('Palm API returned no response', generateResponseJson); - let message = `Palm API returned no response: ${JSON.stringify(generateResponseJson)}`; - - // Check for filters - if (generateResponseJson?.filters[0]?.message) { - message = `Palm filter triggered: ${generateResponseJson.filters[0].message}`; + const candidates = generateResponseJson?.candidates; + if (!candidates || candidates.length === 0) { + let message = 'MakerSuite API returned no candidate'; + console.log(message, generateResponseJson); + if (generateResponseJson?.promptFeedback?.blockReason) { + message += `\nPrompt was blocked due to : ${generateResponseJson.promptFeedback.blockReason}`; } - return response.send({ error: { message } }); } - console.log('Palm response:', responseText); + const responseContent = candidates[0].content; + const responseText = responseContent.parts[0].text; + if (!responseText) { + let message = 'MakerSuite Candidate text empty'; + console.log(message, generateResponseJson); + return response.send({ error: { message } }); + } + + console.log('MakerSuite response:', responseText); // Wrap it back to OAI format const reply = { choices: [{ 'message': { 'content': responseText } }] }; return response.send(reply); } catch (error) { - console.log('Error communicating with Palm API: ', error); + console.log('Error communicating with MakerSuite API: ', error); if (!response.headersSent) { return response.status(500).send({ error: true }); } @@ -1074,7 +1080,7 @@ app.post('/generate_openai', jsonParser, function (request, response_generate_op case CHAT_COMPLETION_SOURCES.CLAUDE: return sendClaudeRequest(request, response_generate_openai); case CHAT_COMPLETION_SOURCES.SCALE: return sendScaleRequest(request, response_generate_openai); case CHAT_COMPLETION_SOURCES.AI21: return sendAI21Request(request, response_generate_openai); - case CHAT_COMPLETION_SOURCES.PALM: return sendPalmRequest(request, response_generate_openai); + case CHAT_COMPLETION_SOURCES.MAKERSUITE: return sendMakerSuiteRequest(request, response_generate_openai); } let api_url; diff --git a/src/chat-completion.js b/src/chat-completion.js index 4fc21a550..d1f97f8a3 100644 --- a/src/chat-completion.js +++ b/src/chat-completion.js @@ -72,6 +72,36 @@ function convertClaudePrompt(messages, addHumanPrefix, addAssistantPostfix, with return requestPrompt; } +function convertGooglePrompt(messages) { + const contents = []; + let lastRole = ''; + let currentText = ''; + + messages.forEach((message, index) => { + const role = message.role === 'assistant' ? 'model' : 'user'; + if (lastRole === role) { + currentText += '\n\n' + message.content; + } else { + if (currentText !== '') { + contents.push({ + parts: [{ text: currentText.trim() }], + role: lastRole, + }); + } + currentText = message.content; + lastRole = role; + } + if (index === messages.length - 1) { + contents.push({ + parts: [{ text: currentText.trim() }], + role: lastRole, + }); + } + }); + return contents; +} + module.exports = { convertClaudePrompt, + convertGooglePrompt, }; diff --git a/src/constants.js b/src/constants.js index 32ea6fad5..7151ada24 100644 --- a/src/constants.js +++ b/src/constants.js @@ -105,29 +105,21 @@ const UNSAFE_EXTENSIONS = [ '.ws', ]; -const PALM_SAFETY = [ +const MAKERSUITE_SAFETY = [ { - category: 'HARM_CATEGORY_DEROGATORY', + category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_NONE', }, { - category: 'HARM_CATEGORY_TOXICITY', + category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_NONE', }, { - category: 'HARM_CATEGORY_VIOLENCE', + category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_NONE', }, { - category: 'HARM_CATEGORY_SEXUAL', - threshold: 'BLOCK_NONE', - }, - { - category: 'HARM_CATEGORY_MEDICAL', - threshold: 'BLOCK_NONE', - }, - { - category: 'HARM_CATEGORY_DANGEROUS', + category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_NONE', }, ]; @@ -139,7 +131,7 @@ const CHAT_COMPLETION_SOURCES = { SCALE: 'scale', OPENROUTER: 'openrouter', AI21: 'ai21', - PALM: 'palm', + MAKERSUITE: 'makersuite', }; const UPLOADS_PATH = './uploads'; @@ -160,7 +152,7 @@ module.exports = { DIRECTORIES, UNSAFE_EXTENSIONS, UPLOADS_PATH, - PALM_SAFETY, + MAKERSUITE_SAFETY, TEXTGEN_TYPES, CHAT_COMPLETION_SOURCES, AVATAR_WIDTH, diff --git a/src/endpoints/tokenizers.js b/src/endpoints/tokenizers.js index a81779d97..096b0f093 100644 --- a/src/endpoints/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -387,6 +387,27 @@ router.post('/ai21/count', jsonParser, async function (req, res) { } }); +router.post('/google/count', jsonParser, async function (req, res) { + if (!req.body) return res.sendStatus(400); + const options = { + method: 'POST', + headers: { + accept: 'application/json', + 'content-type': 'application/json', + }, + body: JSON.stringify({ prompt: { text: req.body[0].content } }), + }; + try { + const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${req.query.model}:countTextTokens?key=${readSecret(SECRET_KEYS.MAKERSUITE)}`, options); + const data = await response.json(); + console.log(data) + return res.send({ 'token_count': data?.tokenCount || 0 }); + } catch (err) { + console.error(err); + return res.send({ 'token_count': 0 }); + } +}); + router.post('/llama/encode', jsonParser, createSentencepieceEncodingHandler(spp_llama)); router.post('/nerdstash/encode', jsonParser, createSentencepieceEncodingHandler(spp_nerd)); router.post('/nerdstash_v2/encode', jsonParser, createSentencepieceEncodingHandler(spp_nerd_v2)); diff --git a/src/palm-vectors.js b/src/palm-vectors.js index 788b474cd..b4e6a68bd 100644 --- a/src/palm-vectors.js +++ b/src/palm-vectors.js @@ -14,7 +14,7 @@ async function getPaLMVector(text) { throw new Error('No PaLM key found'); } - const response = await fetch(`https://generativelanguage.googleapis.com/v1beta2/models/embedding-gecko-001:embedText?key=${key}`, { + const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/embedding-gecko-001:embedText?key=${key}`, { method: 'POST', headers: { 'Content-Type': 'application/json', From 3e82a7d4399d6824afa4a60fa4a0965fd8ee980b Mon Sep 17 00:00:00 2001 From: based Date: Thu, 14 Dec 2023 16:31:08 +1000 Subject: [PATCH 04/21] tokenizer changes and fixes. + a toggle --- public/index.html | 8 ++++++++ public/scripts/openai.js | 11 +++++++++++ public/scripts/tokenizers.js | 2 +- src/chat-completion.js | 1 - src/endpoints/tokenizers.js | 9 ++++----- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/public/index.html b/public/index.html index 474d605de..67bb1b96d 100644 --- a/public/index.html +++ b/public/index.html @@ -1516,6 +1516,14 @@ Use the appropriate tokenizer for Jurassic models, which is more efficient than GPT's.
+
+ +
+ Use the appropriate tokenizer for Google models via their API. Slower prompt processing, but offers much more accurate token counting. +
+
diff --git a/public/scripts/extensions/shared.js b/public/scripts/extensions/shared.js index 1eb4cd905..681daac0b 100644 --- a/public/scripts/extensions/shared.js +++ b/public/scripts/extensions/shared.js @@ -1,7 +1,7 @@ -import { getRequestHeaders } from '../../script.js'; -import { extension_settings } from '../extensions.js'; -import { SECRET_KEYS, secret_state } from '../secrets.js'; -import { createThumbnail } from '../utils.js'; +import {getRequestHeaders} from '../../script.js'; +import {extension_settings} from '../extensions.js'; +import {SECRET_KEYS, secret_state} from '../secrets.js'; +import {createThumbnail} from '../utils.js'; /** * Generates a caption for an image using a multimodal model. @@ -18,6 +18,10 @@ export async function getMultimodalCaption(base64Img, prompt) { throw new Error('OpenRouter API key is not set.'); } + if (extension_settings.caption.multimodal_api === 'google' && !secret_state[SECRET_KEYS.MAKERSUITE]) { + throw new Error('MakerSuite API key is not set.'); + } + // OpenRouter has a payload limit of ~2MB const base64Bytes = base64Img.length * 0.75; const compressionLimit = 2 * 1024 * 1024; @@ -26,16 +30,25 @@ export async function getMultimodalCaption(base64Img, prompt) { base64Img = await createThumbnail(base64Img, maxSide, maxSide, 'image/jpeg'); } - const apiResult = await fetch('/api/openai/caption-image', { - method: 'POST', - headers: getRequestHeaders(), - body: JSON.stringify({ - image: base64Img, - prompt: prompt, - api: extension_settings.caption.multimodal_api || 'openai', - model: extension_settings.caption.multimodal_model || 'gpt-4-vision-preview', - }), - }); + const apiResult = extension_settings.caption.multimodal_api === 'google' ? + await fetch('/api/google/caption-image', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ + image: base64Img, + prompt: prompt, + }), + }) + : await fetch('/api/openai/caption-image', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ + image: base64Img, + prompt: prompt, + api: extension_settings.caption.multimodal_api || 'openai', + model: extension_settings.caption.multimodal_model || 'gpt-4-vision-preview', + }), + }); if (!apiResult.ok) { throw new Error('Failed to caption image via OpenAI.'); diff --git a/public/scripts/openai.js b/public/scripts/openai.js index d5cedc384..878254b21 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -3424,6 +3424,7 @@ export function isImageInliningSupported() { } const gpt4v = 'gpt-4-vision'; + const geminiProV = 'gemini-pro-vision'; const llava13b = 'llava-13b'; if (!oai_settings.image_inlining) { @@ -3433,6 +3434,8 @@ export function isImageInliningSupported() { switch (oai_settings.chat_completion_source) { case chat_completion_sources.OPENAI: return oai_settings.openai_model.includes(gpt4v); + case chat_completion_sources.MAKERSUITE: + return oai_settings.openai_model.includes(geminiProV); case chat_completion_sources.OPENROUTER: return oai_settings.openrouter_model.includes(gpt4v) || oai_settings.openrouter_model.includes(llava13b); default: diff --git a/server.js b/server.js index 6a9fe9171..4a19f3cd6 100644 --- a/server.js +++ b/server.js @@ -1412,6 +1412,9 @@ redirect('/downloadbackground', '/api/backgrounds/upload'); // yes, the download // OpenAI API app.use('/api/openai', require('./src/endpoints/openai').router); +//Google API +app.use('/api/google', require('./src/endpoints/google').router); + // Tokenizers app.use('/api/tokenizers', require('./src/endpoints/tokenizers').router); diff --git a/src/endpoints/google.js b/src/endpoints/google.js new file mode 100644 index 000000000..e2c5c0c2d --- /dev/null +++ b/src/endpoints/google.js @@ -0,0 +1,56 @@ +const { readSecret, SECRET_KEYS } = require('./secrets'); +const fetch = require('node-fetch').default; +const express = require('express'); +const { jsonParser } = require('../express-common'); +const { MAKERSUITE_SAFETY } = require('../constants'); + +const router = express.Router(); + +router.post('/caption-image', jsonParser, async (request, response) => { + try { + const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-pro-vision:generateContent?key=${readSecret(SECRET_KEYS.MAKERSUITE)}`; + const body = { + contents: [{ + parts: [ + { text: request.body.prompt }, + { inlineData: { + mimeType: 'image/png', + data: request.body.image, + }, + }], + }], + safetySettings: MAKERSUITE_SAFETY, + generationConfig: { maxOutputTokens: 1000 }, + }; + + const result = await fetch(url, { + body: JSON.stringify(body), + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + timeout: 0, + }); + + console.log('Multimodal captioning request', body); + + if (!result.ok) { + console.log(`MakerSuite API returned error: ${result.status} ${result.statusText} ${await result.text()}`); + return response.status(result.status).send({ error: true }); + } + + const data = await result.json(); + + const caption = data?.candidates[0].content.parts[0].text; + if (!caption) { + return response.status(500).send('No caption found'); + } + + return response.json({ caption }); + } catch (error) { + console.error(error); + response.status(500).send('Internal server error'); + } +}); + +module.exports = { router }; From 178b07f4148a8e6fc2c50be5e434a797edb61fd4 Mon Sep 17 00:00:00 2001 From: based Date: Thu, 14 Dec 2023 23:18:56 +1000 Subject: [PATCH 07/21] cleaned up a little --- public/scripts/extensions/shared.js | 34 +++++++++++++---------------- src/endpoints/google.js | 10 +++++++-- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/public/scripts/extensions/shared.js b/public/scripts/extensions/shared.js index 681daac0b..fa2105082 100644 --- a/public/scripts/extensions/shared.js +++ b/public/scripts/extensions/shared.js @@ -30,25 +30,21 @@ export async function getMultimodalCaption(base64Img, prompt) { base64Img = await createThumbnail(base64Img, maxSide, maxSide, 'image/jpeg'); } - const apiResult = extension_settings.caption.multimodal_api === 'google' ? - await fetch('/api/google/caption-image', { - method: 'POST', - headers: getRequestHeaders(), - body: JSON.stringify({ - image: base64Img, - prompt: prompt, - }), - }) - : await fetch('/api/openai/caption-image', { - method: 'POST', - headers: getRequestHeaders(), - body: JSON.stringify({ - image: base64Img, - prompt: prompt, - api: extension_settings.caption.multimodal_api || 'openai', - model: extension_settings.caption.multimodal_model || 'gpt-4-vision-preview', - }), - }); + const isGoogle = extension_settings.caption.multimodal_api === 'google'; + const apiResult = await fetch(`/api/${isGoogle ? 'google' : 'openai'}/caption-image`, { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ + image: base64Img, + prompt: prompt, + ...(isGoogle + ? {} + : { + api: extension_settings.caption.multimodal_api || 'openai', + model: extension_settings.caption.multimodal_model || 'gpt-4-vision-preview', + }), + }), + }); if (!apiResult.ok) { throw new Error('Failed to caption image via OpenAI.'); diff --git a/src/endpoints/google.js b/src/endpoints/google.js index e2c5c0c2d..df675c253 100644 --- a/src/endpoints/google.js +++ b/src/endpoints/google.js @@ -14,7 +14,7 @@ router.post('/caption-image', jsonParser, async (request, response) => { parts: [ { text: request.body.prompt }, { inlineData: { - mimeType: 'image/png', + mimeType: 'image/png', //jpg images seem to work fine even with this mimetype set? data: request.body.image, }, }], @@ -40,8 +40,14 @@ router.post('/caption-image', jsonParser, async (request, response) => { } const data = await result.json(); + console.log('Multimodal captioning response', data); - const caption = data?.candidates[0].content.parts[0].text; + const candidates = data?.candidates; + if(!candidates) { + return response.status(500).send('No candidates found, image was most likely filtered.'); + } + + const caption = candidates[0].content.parts[0].text; if (!caption) { return response.status(500).send('No caption found'); } From d5bcd96eefc9741a343e35181787519f8d541401 Mon Sep 17 00:00:00 2001 From: based Date: Fri, 15 Dec 2023 01:28:54 +1000 Subject: [PATCH 08/21] message inlining vision support --- public/scripts/openai.js | 19 +++++++------ server.js | 7 +++-- src/chat-completion.js | 59 ++++++++++++++++++++++++++++------------ 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 878254b21..1d22348ef 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -31,18 +31,18 @@ import { system_message_types, this_chid, } from '../script.js'; -import { groups, selected_group } from './group-chats.js'; +import {groups, selected_group} from './group-chats.js'; import { chatCompletionDefaultPrompts, INJECTION_POSITION, Prompt, - promptManagerDefaultPromptOrders, PromptManager, + promptManagerDefaultPromptOrders, } from './PromptManager.js'; -import { getCustomStoppingStrings, persona_description_positions, power_user } from './power-user.js'; -import { SECRET_KEYS, secret_state, writeSecret } from './secrets.js'; +import {getCustomStoppingStrings, persona_description_positions, power_user} from './power-user.js'; +import {SECRET_KEYS, secret_state, writeSecret} from './secrets.js'; import EventSourceStream from './sse-stream.js'; import { @@ -56,7 +56,7 @@ import { resetScrollHeight, stringFormat, } from './utils.js'; -import { countTokensOpenAI, getTokenizerModel } from './tokenizers.js'; +import {countTokensOpenAI, getTokenizerModel} from './tokenizers.js'; import { formatInstructModeChat, formatInstructModeExamples, @@ -1795,13 +1795,15 @@ class Message { async addImage(image) { const textContent = this.content; const isDataUrl = isDataURL(image); - if (!isDataUrl) { try { const response = await fetch(image, { method: 'GET', cache: 'force-cache' }); if (!response.ok) throw new Error('Failed to fetch image'); const blob = await response.blob(); image = await getBase64Async(blob); + if (oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE) { + image = image.split(',')[1]; + } } catch (error) { console.error('Image adding skipped', error); return; @@ -3087,7 +3089,8 @@ async function onModelChange() { } else { $('#openai_max_context').attr('max', max_8k); } - + oai_settings.temp_openai = Math.min(claude_max_temp, oai_settings.temp_openai); + $('#temp_openai').attr('max', claude_max_temp).val(oai_settings.temp_openai).trigger('input'); oai_settings.openai_max_context = Math.min(Number($('#openai_max_context').attr('max')), oai_settings.openai_max_context); $('#openai_max_context').val(oai_settings.openai_max_context).trigger('input'); } @@ -3435,7 +3438,7 @@ export function isImageInliningSupported() { case chat_completion_sources.OPENAI: return oai_settings.openai_model.includes(gpt4v); case chat_completion_sources.MAKERSUITE: - return oai_settings.openai_model.includes(geminiProV); + return oai_settings.google_model.includes(geminiProV); case chat_completion_sources.OPENROUTER: return oai_settings.openrouter_model.includes(gpt4v) || oai_settings.openrouter_model.includes(llava13b); default: diff --git a/server.js b/server.js index 4a19f3cd6..57264347d 100644 --- a/server.js +++ b/server.js @@ -1002,6 +1002,9 @@ async function sendMakerSuiteRequest(request, response) { return response.status(400).send({ error: true }); } + const google_model = request.body.model; + const should_stream = request.body.stream; + const generationConfig = { stopSequences: request.body.stop, candidateCount: 1, @@ -1012,13 +1015,11 @@ async function sendMakerSuiteRequest(request, response) { }; const body = { - contents: convertGooglePrompt(request.body.messages), + contents: convertGooglePrompt(request.body.messages, google_model), safetySettings: MAKERSUITE_SAFETY, generationConfig: generationConfig, }; - const google_model = request.body.model; - const should_stream = request.body.stream; try { const controller = new AbortController(); request.socket.removeAllListeners('close'); diff --git a/src/chat-completion.js b/src/chat-completion.js index cbed76abf..268a044e5 100644 --- a/src/chat-completion.js +++ b/src/chat-completion.js @@ -72,31 +72,56 @@ function convertClaudePrompt(messages, addHumanPrefix, addAssistantPostfix, with return requestPrompt; } -function convertGooglePrompt(messages) { +function convertGooglePrompt(messages, type) { const contents = []; let lastRole = ''; let currentText = ''; - messages.forEach((message, index) => { - const role = message.role === 'assistant' ? 'model' : 'user'; - if (lastRole === role) { - currentText += '\n\n' + message.content; - } else { - if (currentText !== '') { + + const isMultimodal = type === 'gemini-pro-vision'; + + if (isMultimodal) { + const combinedText = messages.map((message) => { + const role = message.role === 'assistant' ? 'MODEL: ' : 'USER: '; + return role + message.content; + }).join('\n\n').trim(); + + const imageEntry = messages.find((message) => message.content[1]?.image_url); + contents.push({ + parts: [ + { text: combinedText }, + { + inlineData: { + mimeType: 'image/png', + data: imageEntry.content[1].image_url.url ?? '', + }, + }, + ], + role: 'user', + }); + } else { + messages.forEach((message, index) => { + const role = message.role === 'assistant' ? 'model' : 'user'; + if (lastRole === role) { + currentText += '\n\n' + message.content; + } else { + if (currentText !== '') { + contents.push({ + parts: [{ text: currentText.trim() }], + role: lastRole, + }); + } + currentText = message.content; + lastRole = role; + } + if (index === messages.length - 1) { contents.push({ parts: [{ text: currentText.trim() }], role: lastRole, }); } - currentText = message.content; - lastRole = role; - } - if (index === messages.length - 1) { - contents.push({ - parts: [{ text: currentText.trim() }], - role: lastRole, - }); - } - }); + }); + } + return contents; } From 60880cfd4d5add7257fc8fa38ed36fe965a71d58 Mon Sep 17 00:00:00 2001 From: based Date: Fri, 15 Dec 2023 01:39:12 +1000 Subject: [PATCH 09/21] merge --- public/scripts/openai.js | 6 +- src/endpoints/backends/chat-completions.js | 128 +++++++++++++++------ 2 files changed, 93 insertions(+), 41 deletions(-) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 08c1cd386..b5b505e3f 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -31,7 +31,7 @@ import { system_message_types, this_chid, } from '../script.js'; -import {groups, selected_group} from './group-chats.js'; +import { groups, selected_group } from './group-chats.js'; import { chatCompletionDefaultPrompts, @@ -41,8 +41,8 @@ import { promptManagerDefaultPromptOrders, } from './PromptManager.js'; -import {getCustomStoppingStrings, persona_description_positions, power_user} from './power-user.js'; -import {SECRET_KEYS, secret_state, writeSecret} from './secrets.js'; +import { getCustomStoppingStrings, persona_description_positions, power_user } from './power-user.js'; +import { SECRET_KEYS, secret_state, writeSecret } from './secrets.js'; import EventSourceStream from './sse-stream.js'; import { diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index af463bd21..16b87ecd6 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -2,9 +2,9 @@ const express = require('express'); const fetch = require('node-fetch').default; const { jsonParser } = require('../../express-common'); -const { CHAT_COMPLETION_SOURCES, PALM_SAFETY } = require('../../constants'); +const { CHAT_COMPLETION_SOURCES, MAKERSUITE_SAFETY } = require('../../constants'); const { forwardFetchResponse, getConfigValue, tryParse, uuidv4 } = require('../../util'); -const { convertClaudePrompt, convertTextCompletionPrompt } = require('../prompt-converters'); +const { convertClaudePrompt, convertGooglePrompt, convertTextCompletionPrompt } = require('../prompt-converters'); const { readSecret, SECRET_KEYS } = require('../secrets'); const { getTokenizerModel, getSentencepiceTokenizer, getTiktokenTokenizer, sentencepieceTokenizers, TEXT_COMPLETION_MODELS } = require('../tokenizers'); @@ -151,28 +151,35 @@ async function sendScaleRequest(request, response) { * @param {express.Request} request Express request * @param {express.Response} response Express response */ -async function sendPalmRequest(request, response) { - const api_key_palm = readSecret(SECRET_KEYS.PALM); +/** + * @param {express.Request} request + * @param {express.Response} response + */ +async function sendMakerSuiteRequest(request, response) { + const api_key_makersuite = readSecret(SECRET_KEYS.MAKERSUITE); - if (!api_key_palm) { - console.log('Palm API key is missing.'); + if (!api_key_makersuite) { + console.log('MakerSuite API key is missing.'); return response.status(400).send({ error: true }); } - const body = { - prompt: { - text: request.body.messages, - }, + const google_model = request.body.model; + const should_stream = request.body.stream; + + const generationConfig = { stopSequences: request.body.stop, - safetySettings: PALM_SAFETY, + candidateCount: 1, + maxOutputTokens: request.body.max_tokens, temperature: request.body.temperature, topP: request.body.top_p, topK: request.body.top_k || undefined, - maxOutputTokens: request.body.max_tokens, - candidate_count: 1, }; - console.log('Palm request:', body); + const body = { + contents: convertGooglePrompt(request.body.messages, google_model), + safetySettings: MAKERSUITE_SAFETY, + generationConfig: generationConfig, + }; try { const controller = new AbortController(); @@ -181,7 +188,7 @@ async function sendPalmRequest(request, response) { controller.abort(); }); - const generateResponse = await fetch(`https://generativelanguage.googleapis.com/v1beta2/models/text-bison-001:generateText?key=${api_key_palm}`, { + const generateResponse = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${google_model}:${should_stream ? 'streamGenerateContent' : 'generateContent'}?key=${api_key_makersuite}`, { body: JSON.stringify(body), method: 'POST', headers: { @@ -190,34 +197,79 @@ async function sendPalmRequest(request, response) { signal: controller.signal, timeout: 0, }); + // have to do this because of their busted ass streaming endpoint + if (should_stream) { + try { + let partialData = ''; + generateResponse.body.on('data', (data) => { + const chunk = data.toString(); + if (chunk.startsWith(',') || chunk.endsWith(',') || chunk.startsWith('[') || chunk.endsWith(']')) { + partialData = chunk.slice(1); + } else { + partialData += chunk; + } + while (true) { + let json; + try { + json = JSON.parse(partialData); + } catch (e) { + break; + } + response.write(JSON.stringify(json)); + partialData = ''; + } + }); - if (!generateResponse.ok) { - console.log(`Palm API returned error: ${generateResponse.status} ${generateResponse.statusText} ${await generateResponse.text()}`); - return response.status(generateResponse.status).send({ error: true }); - } + request.socket.on('close', function () { + generateResponse.body.destroy(); + response.end(); + }); - const generateResponseJson = await generateResponse.json(); - const responseText = generateResponseJson?.candidates?.[0]?.output; + generateResponse.body.on('end', () => { + console.log('Streaming request finished'); + response.end(); + }); - if (!responseText) { - console.log('Palm API returned no response', generateResponseJson); - let message = `Palm API returned no response: ${JSON.stringify(generateResponseJson)}`; - - // Check for filters - if (generateResponseJson?.filters?.[0]?.reason) { - message = `Palm filter triggered: ${generateResponseJson.filters[0].reason}`; + } catch (error) { + console.log('Error forwarding streaming response:', error); + if (!response.headersSent) { + return response.status(500).send({ error: true }); + } + } + } else { + if (!generateResponse.ok) { + console.log(`MakerSuite API returned error: ${generateResponse.status} ${generateResponse.statusText} ${await generateResponse.text()}`); + return response.status(generateResponse.status).send({ error: true }); } - return response.send({ error: { message } }); + const generateResponseJson = await generateResponse.json(); + + const candidates = generateResponseJson?.candidates; + if (!candidates || candidates.length === 0) { + let message = 'MakerSuite API returned no candidate'; + console.log(message, generateResponseJson); + if (generateResponseJson?.promptFeedback?.blockReason) { + message += `\nPrompt was blocked due to : ${generateResponseJson.promptFeedback.blockReason}`; + } + return response.send({ error: { message } }); + } + + const responseContent = candidates[0].content; + const responseText = responseContent.parts[0].text; + if (!responseText) { + let message = 'MakerSuite Candidate text empty'; + console.log(message, generateResponseJson); + return response.send({ error: { message } }); + } + + console.log('MakerSuite response:', responseText); + + // Wrap it back to OAI format + const reply = { choices: [{ 'message': { 'content': responseText } }] }; + return response.send(reply); } - - console.log('Palm response:', responseText); - - // Wrap it back to OAI format - const reply = { choices: [{ 'message': { 'content': responseText } }] }; - return response.send(reply); } catch (error) { - console.log('Error communicating with Palm API: ', error); + console.log('Error communicating with MakerSuite API: ', error); if (!response.headersSent) { return response.status(500).send({ error: true }); } @@ -225,7 +277,7 @@ async function sendPalmRequest(request, response) { } /** - * Sends a request to Google AI API. + * Sends a request to AI21 API. * @param {express.Request} request Express request * @param {express.Response} response Express response */ @@ -457,7 +509,7 @@ router.post('/generate', jsonParser, function (request, response) { case CHAT_COMPLETION_SOURCES.CLAUDE: return sendClaudeRequest(request, response); case CHAT_COMPLETION_SOURCES.SCALE: return sendScaleRequest(request, response); case CHAT_COMPLETION_SOURCES.AI21: return sendAI21Request(request, response); - case CHAT_COMPLETION_SOURCES.PALM: return sendPalmRequest(request, response); + case CHAT_COMPLETION_SOURCES.MAKERSUITE: return sendMakerSuiteRequest(request, response); } let apiUrl; From 5071b9a3697c6499113b72f001e04f5d49c203ca Mon Sep 17 00:00:00 2001 From: based Date: Fri, 15 Dec 2023 02:01:42 +1000 Subject: [PATCH 10/21] webstorm moment --- public/scripts/extensions/shared.js | 8 ++++---- public/scripts/openai.js | 2 +- src/endpoints/prompt-converters.js | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/public/scripts/extensions/shared.js b/public/scripts/extensions/shared.js index fa2105082..5d9a10150 100644 --- a/public/scripts/extensions/shared.js +++ b/public/scripts/extensions/shared.js @@ -1,7 +1,7 @@ -import {getRequestHeaders} from '../../script.js'; -import {extension_settings} from '../extensions.js'; -import {SECRET_KEYS, secret_state} from '../secrets.js'; -import {createThumbnail} from '../utils.js'; +import { getRequestHeaders } from '../../script.js'; +import { extension_settings } from '../extensions.js'; +import { SECRET_KEYS, secret_state } from '../secrets.js'; +import { createThumbnail } from '../utils.js'; /** * Generates a caption for an image using a multimodal model. diff --git a/public/scripts/openai.js b/public/scripts/openai.js index b5b505e3f..7977c8ff9 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -56,7 +56,7 @@ import { resetScrollHeight, stringFormat, } from './utils.js'; -import {countTokensOpenAI, getTokenizerModel} from './tokenizers.js'; +import { countTokensOpenAI, getTokenizerModel } from './tokenizers.js'; import { formatInstructModeChat, formatInstructModeExamples, diff --git a/src/endpoints/prompt-converters.js b/src/endpoints/prompt-converters.js index e793acc52..4b5e1bc37 100644 --- a/src/endpoints/prompt-converters.js +++ b/src/endpoints/prompt-converters.js @@ -72,12 +72,12 @@ function convertClaudePrompt(messages, addHumanPrefix, addAssistantPostfix, with return requestPrompt; } -function convertGooglePrompt(messages, type) { +function convertGooglePrompt(messages, model) { const contents = []; let lastRole = ''; let currentText = ''; - const isMultimodal = type === 'gemini-pro-vision'; + const isMultimodal = model === 'gemini-pro-vision'; if (isMultimodal) { const combinedText = messages.map((message) => { From d4f96020f28774e6f9d8bb6219941fb08a23a6df Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 14 Dec 2023 19:33:23 +0200 Subject: [PATCH 11/21] Migrate Palm secret key, fix vector source key access --- src/endpoints/secrets.js | 19 +++++++++++++++++++ src/endpoints/vectors.js | 4 ++-- ...{palm-vectors.js => makersuite-vectors.js} | 14 +++++++------- 3 files changed, 28 insertions(+), 9 deletions(-) rename src/{palm-vectors.js => makersuite-vectors.js} (71%) diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index c997e5efe..585bee7ee 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -44,6 +44,17 @@ function writeSecret(key, value) { writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets, null, 4), 'utf-8'); } +function deleteSecret(key) { + if (!fs.existsSync(SECRETS_FILE)) { + return; + } + + const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8'); + const secrets = JSON.parse(fileContents); + delete secrets[key]; + writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets, null, 4), 'utf-8'); +} + /** * Reads a secret from the secrets file * @param {string} key Secret key @@ -119,6 +130,14 @@ function migrateSecrets(settingsFile) { modified = true; } + const palmKey = readSecret('api_key_palm'); + if (palmKey) { + console.log('Migrating Palm key...'); + writeSecret(SECRET_KEYS.MAKERSUITE, palmKey); + deleteSecret('api_key_palm'); + modified = true; + } + if (modified) { console.log('Writing updated settings.json...'); const settingsContent = JSON.stringify(settings, null, 4); diff --git a/src/endpoints/vectors.js b/src/endpoints/vectors.js index 387803ccb..e49d157fa 100644 --- a/src/endpoints/vectors.js +++ b/src/endpoints/vectors.js @@ -17,7 +17,7 @@ async function getVector(source, text) { case 'transformers': return require('../embedding').getTransformersVector(text); case 'palm': - return require('../palm-vectors').getPaLMVector(text); + return require('../makersuite-vectors').getMakerSuiteVector(text); } throw new Error(`Unknown vector source ${source}`); @@ -196,7 +196,7 @@ router.post('/purge', jsonParser, async (req, res) => { const collectionId = String(req.body.collectionId); - const sources = ['transformers', 'openai']; + const sources = ['transformers', 'openai', 'palm']; for (const source of sources) { const index = await getIndex(collectionId, source, false); diff --git a/src/palm-vectors.js b/src/makersuite-vectors.js similarity index 71% rename from src/palm-vectors.js rename to src/makersuite-vectors.js index b4e6a68bd..66d1a6fd8 100644 --- a/src/palm-vectors.js +++ b/src/makersuite-vectors.js @@ -6,12 +6,12 @@ const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); * @param {string} text - The text to get the vector for * @returns {Promise} - The vector for the text */ -async function getPaLMVector(text) { - const key = readSecret(SECRET_KEYS.PALM); +async function getMakerSuiteVector(text) { + const key = readSecret(SECRET_KEYS.MAKERSUITE); if (!key) { - console.log('No PaLM key found'); - throw new Error('No PaLM key found'); + console.log('No MakerSuite key found'); + throw new Error('No MakerSuite key found'); } const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/embedding-gecko-001:embedText?key=${key}`, { @@ -26,8 +26,8 @@ async function getPaLMVector(text) { if (!response.ok) { const text = await response.text(); - console.log('PaLM request failed', response.statusText, text); - throw new Error('PaLM request failed'); + console.log('MakerSuite request failed', response.statusText, text); + throw new Error('MakerSuite request failed'); } const data = await response.json(); @@ -39,5 +39,5 @@ async function getPaLMVector(text) { } module.exports = { - getPaLMVector, + getMakerSuiteVector, }; From 6bb894286e65e40e80d246b508d578bf6e6389fa Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 14 Dec 2023 19:54:31 +0200 Subject: [PATCH 12/21] Migrate palm source to makersuite --- public/scripts/openai.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 7977c8ff9..db0d7c06d 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -2411,6 +2411,11 @@ function loadOpenAISettings(data, settings) { } $('#openai_logit_bias_preset').trigger('change'); + // Upgrade Palm to Makersuite + if (oai_settings.chat_completion_source === 'palm') { + oai_settings.chat_completion_source = chat_completion_sources.MAKERSUITE; + } + $('#chat_completion_source').val(oai_settings.chat_completion_source).trigger('change'); $('#oai_max_context_unlocked').prop('checked', oai_settings.max_context_unlocked); } From a6bb75456705af53d8a4de7d93ce4bf49992ea5d Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 14 Dec 2023 19:56:43 +0200 Subject: [PATCH 13/21] Fix API key access --- public/scripts/extensions/vectors/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index e02501f2e..214b1d887 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -394,7 +394,7 @@ async function getSavedHashes(collectionId) { */ async function insertVectorItems(collectionId, items) { if (settings.source === 'openai' && !secret_state[SECRET_KEYS.OPENAI] || - settings.source === 'palm' && !secret_state[SECRET_KEYS.PALM]) { + settings.source === 'palm' && !secret_state[SECRET_KEYS.MAKERSUITE]) { throw new Error('Vectors: API key missing', { cause: 'api_key_missing' }); } From 47c71a62f35a378d4f4435ca4dc87579c22176d0 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 14 Dec 2023 19:58:27 +0200 Subject: [PATCH 14/21] Don't rewrite settings if just moving the key --- src/endpoints/secrets.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index 585bee7ee..5da0cc730 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -96,6 +96,13 @@ function readSecretState() { * @returns {void} */ function migrateSecrets(settingsFile) { + const palmKey = readSecret('api_key_palm'); + if (palmKey) { + console.log('Migrating Palm key...'); + writeSecret(SECRET_KEYS.MAKERSUITE, palmKey); + deleteSecret('api_key_palm'); + } + if (!fs.existsSync(settingsFile)) { console.log('Settings file does not exist'); return; @@ -130,14 +137,6 @@ function migrateSecrets(settingsFile) { modified = true; } - const palmKey = readSecret('api_key_palm'); - if (palmKey) { - console.log('Migrating Palm key...'); - writeSecret(SECRET_KEYS.MAKERSUITE, palmKey); - deleteSecret('api_key_palm'); - modified = true; - } - if (modified) { console.log('Writing updated settings.json...'); const settingsContent = JSON.stringify(settings, null, 4); From d1be9d534716ee0fa6f2f9cb364f749cc6a5d676 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 14 Dec 2023 20:05:27 +0200 Subject: [PATCH 15/21] Fix JSDoc + lint + readability --- src/endpoints/backends/chat-completions.js | 21 +++++++++------------ src/endpoints/prompt-converters.js | 6 ++++++ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index 16b87ecd6..bd8969ac2 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -1,5 +1,6 @@ const express = require('express'); const fetch = require('node-fetch').default; +const { Readable } = require('stream'); const { jsonParser } = require('../../express-common'); const { CHAT_COMPLETION_SOURCES, MAKERSUITE_SAFETY } = require('../../constants'); @@ -151,20 +152,16 @@ async function sendScaleRequest(request, response) { * @param {express.Request} request Express request * @param {express.Response} response Express response */ -/** - * @param {express.Request} request - * @param {express.Response} response - */ async function sendMakerSuiteRequest(request, response) { - const api_key_makersuite = readSecret(SECRET_KEYS.MAKERSUITE); + const apiKey = readSecret(SECRET_KEYS.MAKERSUITE); - if (!api_key_makersuite) { + if (!apiKey) { console.log('MakerSuite API key is missing.'); return response.status(400).send({ error: true }); } - const google_model = request.body.model; - const should_stream = request.body.stream; + const model = request.body.model; + const stream = request.body.stream; const generationConfig = { stopSequences: request.body.stop, @@ -176,7 +173,7 @@ async function sendMakerSuiteRequest(request, response) { }; const body = { - contents: convertGooglePrompt(request.body.messages, google_model), + contents: convertGooglePrompt(request.body.messages, model), safetySettings: MAKERSUITE_SAFETY, generationConfig: generationConfig, }; @@ -188,7 +185,7 @@ async function sendMakerSuiteRequest(request, response) { controller.abort(); }); - const generateResponse = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${google_model}:${should_stream ? 'streamGenerateContent' : 'generateContent'}?key=${api_key_makersuite}`, { + const generateResponse = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${model}:${stream ? 'streamGenerateContent' : 'generateContent'}?key=${apiKey}`, { body: JSON.stringify(body), method: 'POST', headers: { @@ -198,7 +195,7 @@ async function sendMakerSuiteRequest(request, response) { timeout: 0, }); // have to do this because of their busted ass streaming endpoint - if (should_stream) { + if (stream) { try { let partialData = ''; generateResponse.body.on('data', (data) => { @@ -221,7 +218,7 @@ async function sendMakerSuiteRequest(request, response) { }); request.socket.on('close', function () { - generateResponse.body.destroy(); + if (generateResponse.body instanceof Readable) generateResponse.body.destroy(); response.end(); }); diff --git a/src/endpoints/prompt-converters.js b/src/endpoints/prompt-converters.js index 4b5e1bc37..a31a9be47 100644 --- a/src/endpoints/prompt-converters.js +++ b/src/endpoints/prompt-converters.js @@ -72,6 +72,12 @@ function convertClaudePrompt(messages, addHumanPrefix, addAssistantPostfix, with return requestPrompt; } +/** + * Convert a prompt from the ChatML objects to the format used by Google MakerSuite models. + * @param {object[]} messages Array of messages + * @param {string} model Model name + * @returns {object[]} Prompt for Google MakerSuite models + */ function convertGooglePrompt(messages, model) { const contents = []; let lastRole = ''; From bb8b8f9386ebecbecef5764e64688aa96802413d Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 14 Dec 2023 20:36:31 +0200 Subject: [PATCH 16/21] Fix sending PNG/WEBP to Google captioning --- .../extensions/stable-diffusion/index.js | 26 ++++++++++++------- src/endpoints/google.js | 20 ++++++++------ 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index 9935d43e4..204e268c2 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -1756,22 +1756,28 @@ async function generateMultimodalPrompt(generationType, quietPrompt) { } } - const response = await fetch(avatarUrl); + try { + const response = await fetch(avatarUrl); - if (!response.ok) { - throw new Error('Could not fetch avatar image.'); - } + if (!response.ok) { + throw new Error('Could not fetch avatar image.'); + } - const avatarBlob = await response.blob(); - const avatarBase64 = await getBase64Async(avatarBlob); + const avatarBlob = await response.blob(); + const avatarBase64 = await getBase64Async(avatarBlob); - const caption = await getMultimodalCaption(avatarBase64, quietPrompt); + const caption = await getMultimodalCaption(avatarBase64, quietPrompt); - if (!caption) { + if (!caption) { + throw new Error('No caption returned from the API.'); + } + + return caption; + } catch (error) { + console.error(error); + toastr.error('Multimodal captioning failed. Please try again.', 'Image Generation'); throw new Error('Multimodal captioning failed.'); } - - return caption; } /** diff --git a/src/endpoints/google.js b/src/endpoints/google.js index df675c253..1e74f71c7 100644 --- a/src/endpoints/google.js +++ b/src/endpoints/google.js @@ -8,21 +8,26 @@ const router = express.Router(); router.post('/caption-image', jsonParser, async (request, response) => { try { + const mimeType = request.body.image.split(';')[0].split(':')[1]; + const base64Data = request.body.image.split(',')[1]; const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-pro-vision:generateContent?key=${readSecret(SECRET_KEYS.MAKERSUITE)}`; const body = { contents: [{ parts: [ { text: request.body.prompt }, - { inlineData: { - mimeType: 'image/png', //jpg images seem to work fine even with this mimetype set? - data: request.body.image, - }, + { + inlineData: { + mimeType: 'image/png', // It needs to specify a MIME type in data if it's not a PNG + data: mimeType === 'image/png' ? base64Data : request.body.image, + }, }], }], safetySettings: MAKERSUITE_SAFETY, generationConfig: { maxOutputTokens: 1000 }, }; + console.log('Multimodal captioning request', body); + const result = await fetch(url, { body: JSON.stringify(body), method: 'POST', @@ -32,10 +37,9 @@ router.post('/caption-image', jsonParser, async (request, response) => { timeout: 0, }); - console.log('Multimodal captioning request', body); - if (!result.ok) { - console.log(`MakerSuite API returned error: ${result.status} ${result.statusText} ${await result.text()}`); + const error = await result.json(); + console.log(`MakerSuite API returned error: ${result.status} ${result.statusText}`, error); return response.status(result.status).send({ error: true }); } @@ -43,7 +47,7 @@ router.post('/caption-image', jsonParser, async (request, response) => { console.log('Multimodal captioning response', data); const candidates = data?.candidates; - if(!candidates) { + if (!candidates) { return response.status(500).send('No candidates found, image was most likely filtered.'); } From 2045e414d1938610eccf52586214bc4843aef70b Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 14 Dec 2023 20:57:43 +0200 Subject: [PATCH 17/21] lint: format fix --- public/scripts/tokenizers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index 83842d73d..196f3ec9c 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -376,7 +376,7 @@ export function getTokenizerModel() { } } - if(oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) { + if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) { return oai_settings.google_model; } @@ -395,7 +395,7 @@ export function countTokensOpenAI(messages, full = false) { const shouldTokenizeAI21 = oai_settings.chat_completion_source === chat_completion_sources.AI21 && oai_settings.use_ai21_tokenizer; const shouldTokenizeGoogle = oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE && oai_settings.use_google_tokenizer; let tokenizerEndpoint = ''; - if(shouldTokenizeAI21) { + if (shouldTokenizeAI21) { tokenizerEndpoint = '/api/tokenizers/ai21/count'; } else if (shouldTokenizeGoogle) { tokenizerEndpoint = `/api/tokenizers/google/count?model=${getTokenizerModel()}`; From eec28469f8ad492a5f3cc331b93d9e7223aa56f8 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 14 Dec 2023 21:21:37 +0200 Subject: [PATCH 18/21] Fix server crash if multimodal prompt contains no image --- public/index.html | 2 +- src/endpoints/prompt-converters.js | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/public/index.html b/public/index.html index 67bb1b96d..bd12da858 100644 --- a/public/index.html +++ b/public/index.html @@ -1497,7 +1497,7 @@
-
+