Add reverse proxy support to Google MakerSuite to allow some Google MakerSuite URLs to no longer be hardcoded with domain names. (#2307)

* Add reverse proxy support to Google MakerSuite.

* Remove hardcoded URLs for some Google MakerSuite API calls.

* Don't send real key to alt.endpoint

* Fix for image captioning

* Fix key validation

* +fix key check for mistral

* Fix caption key validation

* Fix tokenization endpoint use

---------

Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>

public/index.html

@@ -2353,7 +2353,7 @@
                                 <option value="windowai">Window AI</option>
                             </optgroup>
                         </select>
-                        <div data-newbie-hidden class="inline-drawer wide100p" data-source="openai,claude,mistralai">
+                        <div data-newbie-hidden class="inline-drawer wide100p" data-source="openai,claude,mistralai,makersuite">
                             <div class="inline-drawer-toggle inline-drawer-header">
                                 <b data-i18n="Reverse Proxy">Reverse Proxy</b>
                                 <div class="fa-solid fa-circle-chevron-down inline-drawer-icon down"></div>

public/scripts/extensions/caption/index.js

@@ -348,8 +348,8 @@ jQuery(function () {
                 (modules.includes('caption') && extension_settings.caption.source === 'extras') ||
                 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openai' && (secret_state[SECRET_KEYS.OPENAI] || extension_settings.caption.allow_reverse_proxy)) ||
                 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openrouter' && secret_state[SECRET_KEYS.OPENROUTER]) ||
-                (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && secret_state[SECRET_KEYS.MAKERSUITE]) ||
+                (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && (secret_state[SECRET_KEYS.MAKERSUITE] || extension_settings.caption.allow_reverse_proxy)) ||
-                (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'anthropic' && secret_state[SECRET_KEYS.CLAUDE]) ||
+                (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'anthropic' && (secret_state[SECRET_KEYS.CLAUDE] || extension_settings.caption.allow_reverse_proxy)) ||
                 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ollama' && textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) ||
                 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'llamacpp' && textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP]) ||
                 (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ooba' && textgenerationwebui_settings.server_urls[textgen_types.OOBA]) ||
@@ -465,7 +465,7 @@ jQuery(function () {
                                 <option data-type="custom" value="custom_current">[Currently selected]</option>
                             </select>
                         </div>
-                        <label data-type="openai,anthropic" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
+                        <label data-type="openai,anthropic,google" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
                             <input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">
                             Allow reverse proxy
                         </label>

public/scripts/extensions/shared.js

@@ -12,7 +12,13 @@ import { createThumbnail, isValidUrl } from '../utils.js';
  * @returns {Promise<string>} Generated caption
  */
 export async function getMultimodalCaption(base64Img, prompt) {
-    throwIfInvalidModel();
+    const useReverseProxy =
+        (['openai', 'anthropic', 'google'].includes(extension_settings.caption.multimodal_api))
+        && extension_settings.caption.allow_reverse_proxy
+        && oai_settings.reverse_proxy
+        && isValidUrl(oai_settings.reverse_proxy);
+
+    throwIfInvalidModel(useReverseProxy);
 
     const noPrefix = ['google', 'ollama', 'llamacpp'].includes(extension_settings.caption.multimodal_api);
 
@@ -39,27 +45,18 @@ export async function getMultimodalCaption(base64Img, prompt) {
         }
     }
 
-    const useReverseProxy =
-        (extension_settings.caption.multimodal_api === 'openai' || extension_settings.caption.multimodal_api === 'anthropic')
-        && extension_settings.caption.allow_reverse_proxy
-        && oai_settings.reverse_proxy
-        && isValidUrl(oai_settings.reverse_proxy);
-
     const proxyUrl = useReverseProxy ? oai_settings.reverse_proxy : '';
     const proxyPassword = useReverseProxy ? oai_settings.proxy_password : '';
 
     const requestBody = {
         image: base64Img,
         prompt: prompt,
+        reverse_proxy: proxyUrl,
+        proxy_password: proxyPassword,
+        api: extension_settings.caption.multimodal_api || 'openai',
+        model: extension_settings.caption.multimodal_model || 'gpt-4-turbo',
     };
 
-    if (!isGoogle) {
-        requestBody.api = extension_settings.caption.multimodal_api || 'openai';
-        requestBody.model = extension_settings.caption.multimodal_model || 'gpt-4-turbo';
-        requestBody.reverse_proxy = proxyUrl;
-        requestBody.proxy_password = proxyPassword;
-    }
-
     if (isOllama) {
         if (extension_settings.caption.multimodal_model === 'ollama_current') {
             requestBody.model = textgenerationwebui_settings.ollama_model;
@@ -117,8 +114,8 @@ export async function getMultimodalCaption(base64Img, prompt) {
     return String(caption).trim();
 }
 
-function throwIfInvalidModel() {
+function throwIfInvalidModel(useReverseProxy) {
-    if (extension_settings.caption.multimodal_api === 'openai' && !secret_state[SECRET_KEYS.OPENAI]) {
+    if (extension_settings.caption.multimodal_api === 'openai' && !secret_state[SECRET_KEYS.OPENAI] && !useReverseProxy) {
         throw new Error('OpenAI API key is not set.');
     }
 
@@ -126,7 +123,11 @@ function throwIfInvalidModel() {
         throw new Error('OpenRouter API key is not set.');
     }
 
-    if (extension_settings.caption.multimodal_api === 'google' && !secret_state[SECRET_KEYS.MAKERSUITE]) {
+    if (extension_settings.caption.multimodal_api === 'anthropic' && !secret_state[SECRET_KEYS.CLAUDE] && !useReverseProxy) {
+        throw new Error('Anthropic (Claude) API key is not set.');
+    }
+
+    if (extension_settings.caption.multimodal_api === 'google' && !secret_state[SECRET_KEYS.MAKERSUITE] && !useReverseProxy) {
         throw new Error('MakerSuite API key is not set.');
     }
 

public/scripts/openai.js

@@ -1743,8 +1743,8 @@ async function sendOpenAIRequest(type, messages, signal) {
         delete generate_data.stop;
     }
 
-    // Proxy is only supported for Claude, OpenAI and Mistral
+    // Proxy is only supported for Claude, OpenAI, Mistral, and Google MakerSuite
-    if (oai_settings.reverse_proxy && [chat_completion_sources.CLAUDE, chat_completion_sources.OPENAI, chat_completion_sources.MISTRALAI].includes(oai_settings.chat_completion_source)) {
+    if (oai_settings.reverse_proxy && [chat_completion_sources.CLAUDE, chat_completion_sources.OPENAI, chat_completion_sources.MISTRALAI, chat_completion_sources.MAKERSUITE].includes(oai_settings.chat_completion_source)) {
         validateReverseProxy();
         generate_data['reverse_proxy'] = oai_settings.reverse_proxy;
         generate_data['proxy_password'] = oai_settings.proxy_password;
@@ -4038,7 +4038,7 @@ async function onConnectButtonClick(e) {
             await writeSecret(SECRET_KEYS.MAKERSUITE, api_key_makersuite);
         }
 
-        if (!secret_state[SECRET_KEYS.MAKERSUITE]) {
+        if (!secret_state[SECRET_KEYS.MAKERSUITE] && !oai_settings.reverse_proxy) {
             console.log('No secret key saved for MakerSuite');
             return;
         }
@@ -4090,7 +4090,7 @@ async function onConnectButtonClick(e) {
             await writeSecret(SECRET_KEYS.MISTRALAI, api_key_mistralai);
         }
 
-        if (!secret_state[SECRET_KEYS.MISTRALAI]) {
+        if (!secret_state[SECRET_KEYS.MISTRALAI] && !oai_settings.reverse_proxy) {
             console.log('No secret key saved for MistralAI');
             return;
         }

public/scripts/tokenizers.js

@@ -560,7 +560,7 @@ export function countTokensOpenAI(messages, full = false) {
     if (shouldTokenizeAI21) {
         tokenizerEndpoint = '/api/tokenizers/ai21/count';
     } else if (shouldTokenizeGoogle) {
-        tokenizerEndpoint = `/api/tokenizers/google/count?model=${getTokenizerModel()}`;
+        tokenizerEndpoint = `/api/tokenizers/google/count?model=${getTokenizerModel()}&reverse_proxy=${oai_settings.reverse_proxy}&proxy_password=${oai_settings.proxy_password}`;
     } else {
         tokenizerEndpoint = `/api/tokenizers/openai/count?model=${getTokenizerModel()}`;
     }

src/endpoints/backends/chat-completions.js

@@ -16,6 +16,7 @@ const API_MISTRAL = 'https://api.mistral.ai/v1';
 const API_COHERE = 'https://api.cohere.ai/v1';
 const API_PERPLEXITY = 'https://api.perplexity.ai';
 const API_GROQ = 'https://api.groq.com/openai/v1';
+const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
 
 /**
  * Applies a post-processing step to the generated messages.
@@ -232,9 +233,10 @@ async function sendScaleRequest(request, response) {
  * @param {express.Response} response Express response
  */
 async function sendMakerSuiteRequest(request, response) {
-    const apiKey = readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
+    const apiUrl = new URL(request.body.reverse_proxy || API_MAKERSUITE);
+    const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
 
-    if (!apiKey) {
+    if (!request.body.reverse_proxy && !apiKey) {
         console.log('MakerSuite API key is missing.');
         return response.status(400).send({ error: true });
     }
@@ -316,7 +318,7 @@ async function sendMakerSuiteRequest(request, response) {
             ? (stream ? 'streamGenerateContent' : 'generateContent')
             : (isText ? 'generateText' : 'generateMessage');
 
-        const generateResponse = await fetch(`https://generativelanguage.googleapis.com/${apiVersion}/models/${model}:${responseType}?key=${apiKey}${stream ? '&alt=sse' : ''}`, {
+        const generateResponse = await fetch(`${apiUrl.origin}/${apiVersion}/models/${model}:${responseType}?key=${apiKey}${stream ? '&alt=sse' : ''}`, {
             body: JSON.stringify(body),
             method: 'POST',
             headers: {

src/endpoints/google.js

@@ -4,14 +4,18 @@ const express = require('express');
 const { jsonParser } = require('../express-common');
 const { GEMINI_SAFETY } = require('../constants');
 
+const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
+
 const router = express.Router();
 
 router.post('/caption-image', jsonParser, async (request, response) => {
     try {
         const mimeType = request.body.image.split(';')[0].split(':')[1];
         const base64Data = request.body.image.split(',')[1];
-        const key = readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
+        const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
-        const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-pro-vision:generateContent?key=${key}`;
+        const apiUrl = new URL(request.body.reverse_proxy || API_MAKERSUITE);
+        const model = request.body.model || 'gemini-pro-vision';
+        const url = `${apiUrl.origin}/v1beta/models/${model}:generateContent?key=${apiKey}`;
         const body = {
             contents: [{
                 parts: [
@@ -27,7 +31,7 @@ router.post('/caption-image', jsonParser, async (request, response) => {
             generationConfig: { maxOutputTokens: 1000 },
         };
 
-        console.log('Multimodal captioning request', body);
+        console.log('Multimodal captioning request', model, body);
 
         const result = await fetch(url, {
             body: JSON.stringify(body),

src/endpoints/tokenizers.js

@@ -10,6 +10,8 @@ const { TEXTGEN_TYPES } = require('../constants');
 const { jsonParser } = require('../express-common');
 const { setAdditionalHeaders } = require('../additional-headers');
 
+const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
+
 /**
  * @typedef { (req: import('express').Request, res: import('express').Response) => Promise<any> } TokenizationHandler
  */
@@ -555,8 +557,11 @@ router.post('/google/count', jsonParser, async function (req, res) {
         body: JSON.stringify({ contents: convertGooglePrompt(req.body, String(req.query.model)).contents }),
     };
     try {
-        const key = readSecret(req.user.directories, SECRET_KEYS.MAKERSUITE);
+        const reverseProxy = req.query.reverse_proxy?.toString() || '';
-        const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${req.query.model}:countTokens?key=${key}`, options);
+        const proxyPassword = req.query.proxy_password?.toString() || '';
+        const apiKey = reverseProxy ? proxyPassword : readSecret(req.user.directories, SECRET_KEYS.MAKERSUITE);
+        const apiUrl = new URL(reverseProxy || API_MAKERSUITE);
+        const response = await fetch(`${apiUrl.origin}/v1beta/models/${req.query.model}:countTokens?key=${apiKey}`, options);
         const data = await response.json();
         return res.send({ 'token_count': data?.totalTokens || 0 });
     } catch (err) {