mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-02-03 04:37:40 +01:00
Add reverse proxy support to Google MakerSuite to allow some Google MakerSuite URLs to no longer be hardcoded with domain names. (#2307)
* Add reverse proxy support to Google MakerSuite. * Remove hardcoded URLs for some Google MakerSuite API calls. * Don't send real key to alt.endpoint * Fix for image captioning * Fix key validation * +fix key check for mistral * Fix caption key validation * Fix tokenization endpoint use --------- Co-authored-by: Cohee <18619528+Cohee1207@users.noreply.github.com>
This commit is contained in:
daiaji
GitHub
parent
e1dfbc0bea
commit
66454bb711
@ -2353,7 +2353,7 @@
|
||||
<option value="windowai">Window AI</option>
|
||||
</optgroup>
|
||||
</select>
|
||||
<div data-newbie-hidden class="inline-drawer wide100p" data-source="openai,claude,mistralai">
|
||||
<div data-newbie-hidden class="inline-drawer wide100p" data-source="openai,claude,mistralai,makersuite">
|
||||
<div class="inline-drawer-toggle inline-drawer-header">
|
||||
<b data-i18n="Reverse Proxy">Reverse Proxy</b>
|
||||
<div class="fa-solid fa-circle-chevron-down inline-drawer-icon down"></div>
|
||||
|
||||
@ -348,8 +348,8 @@ jQuery(function () {
|
||||
(modules.includes('caption') && extension_settings.caption.source === 'extras') ||
|
||||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openai' && (secret_state[SECRET_KEYS.OPENAI] || extension_settings.caption.allow_reverse_proxy)) ||
|
||||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openrouter' && secret_state[SECRET_KEYS.OPENROUTER]) ||
|
||||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && secret_state[SECRET_KEYS.MAKERSUITE]) ||
|
||||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'anthropic' && secret_state[SECRET_KEYS.CLAUDE]) ||
|
||||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && (secret_state[SECRET_KEYS.MAKERSUITE] || extension_settings.caption.allow_reverse_proxy)) ||
|
||||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'anthropic' && (secret_state[SECRET_KEYS.CLAUDE] || extension_settings.caption.allow_reverse_proxy)) ||
|
||||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ollama' && textgenerationwebui_settings.server_urls[textgen_types.OLLAMA]) ||
|
||||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'llamacpp' && textgenerationwebui_settings.server_urls[textgen_types.LLAMACPP]) ||
|
||||
(extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'ooba' && textgenerationwebui_settings.server_urls[textgen_types.OOBA]) ||
|
||||
@ -465,7 +465,7 @@ jQuery(function () {
|
||||
<option data-type="custom" value="custom_current">[Currently selected]</option>
|
||||
</select>
|
||||
</div>
|
||||
<label data-type="openai,anthropic" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
|
||||
<label data-type="openai,anthropic,google" class="checkbox_label flexBasis100p" for="caption_allow_reverse_proxy" title="Allow using reverse proxy if defined and valid.">
|
||||
<input id="caption_allow_reverse_proxy" type="checkbox" class="checkbox">
|
||||
Allow reverse proxy
|
||||
</label>
|
||||
|
||||
@ -12,7 +12,13 @@ import { createThumbnail, isValidUrl } from '../utils.js';
|
||||
* @returns {Promise<string>} Generated caption
|
||||
*/
|
||||
export async function getMultimodalCaption(base64Img, prompt) {
|
||||
throwIfInvalidModel();
|
||||
const useReverseProxy =
|
||||
(['openai', 'anthropic', 'google'].includes(extension_settings.caption.multimodal_api))
|
||||
&& extension_settings.caption.allow_reverse_proxy
|
||||
&& oai_settings.reverse_proxy
|
||||
&& isValidUrl(oai_settings.reverse_proxy);
|
||||
|
||||
throwIfInvalidModel(useReverseProxy);
|
||||
|
||||
const noPrefix = ['google', 'ollama', 'llamacpp'].includes(extension_settings.caption.multimodal_api);
|
||||
|
||||
@ -39,27 +45,18 @@ export async function getMultimodalCaption(base64Img, prompt) {
|
||||
}
|
||||
}
|
||||
|
||||
const useReverseProxy =
|
||||
(extension_settings.caption.multimodal_api === 'openai' || extension_settings.caption.multimodal_api === 'anthropic')
|
||||
&& extension_settings.caption.allow_reverse_proxy
|
||||
&& oai_settings.reverse_proxy
|
||||
&& isValidUrl(oai_settings.reverse_proxy);
|
||||
|
||||
const proxyUrl = useReverseProxy ? oai_settings.reverse_proxy : '';
|
||||
const proxyPassword = useReverseProxy ? oai_settings.proxy_password : '';
|
||||
|
||||
const requestBody = {
|
||||
image: base64Img,
|
||||
prompt: prompt,
|
||||
reverse_proxy: proxyUrl,
|
||||
proxy_password: proxyPassword,
|
||||
api: extension_settings.caption.multimodal_api || 'openai',
|
||||
model: extension_settings.caption.multimodal_model || 'gpt-4-turbo',
|
||||
};
|
||||
|
||||
if (!isGoogle) {
|
||||
requestBody.api = extension_settings.caption.multimodal_api || 'openai';
|
||||
requestBody.model = extension_settings.caption.multimodal_model || 'gpt-4-turbo';
|
||||
requestBody.reverse_proxy = proxyUrl;
|
||||
requestBody.proxy_password = proxyPassword;
|
||||
}
|
||||
|
||||
if (isOllama) {
|
||||
if (extension_settings.caption.multimodal_model === 'ollama_current') {
|
||||
requestBody.model = textgenerationwebui_settings.ollama_model;
|
||||
@ -117,8 +114,8 @@ export async function getMultimodalCaption(base64Img, prompt) {
|
||||
return String(caption).trim();
|
||||
}
|
||||
|
||||
function throwIfInvalidModel() {
|
||||
if (extension_settings.caption.multimodal_api === 'openai' && !secret_state[SECRET_KEYS.OPENAI]) {
|
||||
function throwIfInvalidModel(useReverseProxy) {
|
||||
if (extension_settings.caption.multimodal_api === 'openai' && !secret_state[SECRET_KEYS.OPENAI] && !useReverseProxy) {
|
||||
throw new Error('OpenAI API key is not set.');
|
||||
}
|
||||
|
||||
@ -126,7 +123,11 @@ function throwIfInvalidModel() {
|
||||
throw new Error('OpenRouter API key is not set.');
|
||||
}
|
||||
|
||||
if (extension_settings.caption.multimodal_api === 'google' && !secret_state[SECRET_KEYS.MAKERSUITE]) {
|
||||
if (extension_settings.caption.multimodal_api === 'anthropic' && !secret_state[SECRET_KEYS.CLAUDE] && !useReverseProxy) {
|
||||
throw new Error('Anthropic (Claude) API key is not set.');
|
||||
}
|
||||
|
||||
if (extension_settings.caption.multimodal_api === 'google' && !secret_state[SECRET_KEYS.MAKERSUITE] && !useReverseProxy) {
|
||||
throw new Error('MakerSuite API key is not set.');
|
||||
}
|
||||
|
||||
|
||||
@ -1743,8 +1743,8 @@ async function sendOpenAIRequest(type, messages, signal) {
|
||||
delete generate_data.stop;
|
||||
}
|
||||
|
||||
// Proxy is only supported for Claude, OpenAI and Mistral
|
||||
if (oai_settings.reverse_proxy && [chat_completion_sources.CLAUDE, chat_completion_sources.OPENAI, chat_completion_sources.MISTRALAI].includes(oai_settings.chat_completion_source)) {
|
||||
// Proxy is only supported for Claude, OpenAI, Mistral, and Google MakerSuite
|
||||
if (oai_settings.reverse_proxy && [chat_completion_sources.CLAUDE, chat_completion_sources.OPENAI, chat_completion_sources.MISTRALAI, chat_completion_sources.MAKERSUITE].includes(oai_settings.chat_completion_source)) {
|
||||
validateReverseProxy();
|
||||
generate_data['reverse_proxy'] = oai_settings.reverse_proxy;
|
||||
generate_data['proxy_password'] = oai_settings.proxy_password;
|
||||
@ -4038,7 +4038,7 @@ async function onConnectButtonClick(e) {
|
||||
await writeSecret(SECRET_KEYS.MAKERSUITE, api_key_makersuite);
|
||||
}
|
||||
|
||||
if (!secret_state[SECRET_KEYS.MAKERSUITE]) {
|
||||
if (!secret_state[SECRET_KEYS.MAKERSUITE] && !oai_settings.reverse_proxy) {
|
||||
console.log('No secret key saved for MakerSuite');
|
||||
return;
|
||||
}
|
||||
@ -4090,7 +4090,7 @@ async function onConnectButtonClick(e) {
|
||||
await writeSecret(SECRET_KEYS.MISTRALAI, api_key_mistralai);
|
||||
}
|
||||
|
||||
if (!secret_state[SECRET_KEYS.MISTRALAI]) {
|
||||
if (!secret_state[SECRET_KEYS.MISTRALAI] && !oai_settings.reverse_proxy) {
|
||||
console.log('No secret key saved for MistralAI');
|
||||
return;
|
||||
}
|
||||
|
||||
@ -560,7 +560,7 @@ export function countTokensOpenAI(messages, full = false) {
|
||||
if (shouldTokenizeAI21) {
|
||||
tokenizerEndpoint = '/api/tokenizers/ai21/count';
|
||||
} else if (shouldTokenizeGoogle) {
|
||||
tokenizerEndpoint = `/api/tokenizers/google/count?model=${getTokenizerModel()}`;
|
||||
tokenizerEndpoint = `/api/tokenizers/google/count?model=${getTokenizerModel()}&reverse_proxy=${oai_settings.reverse_proxy}&proxy_password=${oai_settings.proxy_password}`;
|
||||
} else {
|
||||
tokenizerEndpoint = `/api/tokenizers/openai/count?model=${getTokenizerModel()}`;
|
||||
}
|
||||
|
||||
@ -16,6 +16,7 @@ const API_MISTRAL = 'https://api.mistral.ai/v1';
|
||||
const API_COHERE = 'https://api.cohere.ai/v1';
|
||||
const API_PERPLEXITY = 'https://api.perplexity.ai';
|
||||
const API_GROQ = 'https://api.groq.com/openai/v1';
|
||||
const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
|
||||
|
||||
/**
|
||||
* Applies a post-processing step to the generated messages.
|
||||
@ -232,9 +233,10 @@ async function sendScaleRequest(request, response) {
|
||||
* @param {express.Response} response Express response
|
||||
*/
|
||||
async function sendMakerSuiteRequest(request, response) {
|
||||
const apiKey = readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
|
||||
const apiUrl = new URL(request.body.reverse_proxy || API_MAKERSUITE);
|
||||
const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
|
||||
|
||||
if (!apiKey) {
|
||||
if (!request.body.reverse_proxy && !apiKey) {
|
||||
console.log('MakerSuite API key is missing.');
|
||||
return response.status(400).send({ error: true });
|
||||
}
|
||||
@ -316,7 +318,7 @@ async function sendMakerSuiteRequest(request, response) {
|
||||
? (stream ? 'streamGenerateContent' : 'generateContent')
|
||||
: (isText ? 'generateText' : 'generateMessage');
|
||||
|
||||
const generateResponse = await fetch(`https://generativelanguage.googleapis.com/${apiVersion}/models/${model}:${responseType}?key=${apiKey}${stream ? '&alt=sse' : ''}`, {
|
||||
const generateResponse = await fetch(`${apiUrl.origin}/${apiVersion}/models/${model}:${responseType}?key=${apiKey}${stream ? '&alt=sse' : ''}`, {
|
||||
body: JSON.stringify(body),
|
||||
method: 'POST',
|
||||
headers: {
|
||||
|
||||
@ -4,14 +4,18 @@ const express = require('express');
|
||||
const { jsonParser } = require('../express-common');
|
||||
const { GEMINI_SAFETY } = require('../constants');
|
||||
|
||||
const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.post('/caption-image', jsonParser, async (request, response) => {
|
||||
try {
|
||||
const mimeType = request.body.image.split(';')[0].split(':')[1];
|
||||
const base64Data = request.body.image.split(',')[1];
|
||||
const key = readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
|
||||
const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-pro-vision:generateContent?key=${key}`;
|
||||
const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE);
|
||||
const apiUrl = new URL(request.body.reverse_proxy || API_MAKERSUITE);
|
||||
const model = request.body.model || 'gemini-pro-vision';
|
||||
const url = `${apiUrl.origin}/v1beta/models/${model}:generateContent?key=${apiKey}`;
|
||||
const body = {
|
||||
contents: [{
|
||||
parts: [
|
||||
@ -27,7 +31,7 @@ router.post('/caption-image', jsonParser, async (request, response) => {
|
||||
generationConfig: { maxOutputTokens: 1000 },
|
||||
};
|
||||
|
||||
console.log('Multimodal captioning request', body);
|
||||
console.log('Multimodal captioning request', model, body);
|
||||
|
||||
const result = await fetch(url, {
|
||||
body: JSON.stringify(body),
|
||||
|
||||
@ -10,6 +10,8 @@ const { TEXTGEN_TYPES } = require('../constants');
|
||||
const { jsonParser } = require('../express-common');
|
||||
const { setAdditionalHeaders } = require('../additional-headers');
|
||||
|
||||
const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
|
||||
|
||||
/**
|
||||
* @typedef { (req: import('express').Request, res: import('express').Response) => Promise<any> } TokenizationHandler
|
||||
*/
|
||||
@ -555,8 +557,11 @@ router.post('/google/count', jsonParser, async function (req, res) {
|
||||
body: JSON.stringify({ contents: convertGooglePrompt(req.body, String(req.query.model)).contents }),
|
||||
};
|
||||
try {
|
||||
const key = readSecret(req.user.directories, SECRET_KEYS.MAKERSUITE);
|
||||
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${req.query.model}:countTokens?key=${key}`, options);
|
||||
const reverseProxy = req.query.reverse_proxy?.toString() || '';
|
||||
const proxyPassword = req.query.proxy_password?.toString() || '';
|
||||
const apiKey = reverseProxy ? proxyPassword : readSecret(req.user.directories, SECRET_KEYS.MAKERSUITE);
|
||||
const apiUrl = new URL(reverseProxy || API_MAKERSUITE);
|
||||
const response = await fetch(`${apiUrl.origin}/v1beta/models/${req.query.model}:countTokens?key=${apiKey}`, options);
|
||||
const data = await response.json();
|
||||
return res.send({ 'token_count': data?.totalTokens || 0 });
|
||||
} catch (err) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user