From 6706cce10de9cc02e2337d512120ec3a0e8d431c Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 3 Oct 2024 08:41:45 +0300 Subject: [PATCH 1/3] Groq: Add new models and multimodal captions --- public/index.html | 35 +++++++++++++------ public/scripts/extensions/caption/index.js | 1 + .../scripts/extensions/caption/settings.html | 4 +++ public/scripts/extensions/shared.js | 6 +++- public/scripts/openai.js | 5 ++- src/endpoints/openai.js | 8 +++++ 6 files changed, 47 insertions(+), 12 deletions(-) diff --git a/public/index.html b/public/index.html index c6117f665..b10a66d46 100644 --- a/public/index.html +++ b/public/index.html @@ -2954,16 +2954,31 @@

Groq Model

diff --git a/public/scripts/extensions/caption/index.js b/public/scripts/extensions/caption/index.js index 0f740cc9a..04812cead 100644 --- a/public/scripts/extensions/caption/index.js +++ b/public/scripts/extensions/caption/index.js @@ -403,6 +403,7 @@ jQuery(async function () { (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openai' && (secret_state[SECRET_KEYS.OPENAI] || extension_settings.caption.allow_reverse_proxy)) || (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'openrouter' && secret_state[SECRET_KEYS.OPENROUTER]) || (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'zerooneai' && secret_state[SECRET_KEYS.ZEROONEAI]) || + (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'groq' && secret_state[SECRET_KEYS.GROQ]) || (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'mistral' && (secret_state[SECRET_KEYS.MISTRALAI] || extension_settings.caption.allow_reverse_proxy)) || (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'google' && (secret_state[SECRET_KEYS.MAKERSUITE] || extension_settings.caption.allow_reverse_proxy)) || (extension_settings.caption.source === 'multimodal' && extension_settings.caption.multimodal_api === 'anthropic' && (secret_state[SECRET_KEYS.CLAUDE] || extension_settings.caption.allow_reverse_proxy)) || diff --git a/public/scripts/extensions/caption/settings.html b/public/scripts/extensions/caption/settings.html index 318577277..f6a6fc7a3 100644 --- a/public/scripts/extensions/caption/settings.html +++ b/public/scripts/extensions/caption/settings.html @@ -21,6 +21,7 @@ + @@ -60,6 +61,9 @@ + + + diff --git a/public/scripts/extensions/shared.js b/public/scripts/extensions/shared.js index 950105ce1..444231527 100644 --- a/public/scripts/extensions/shared.js +++ b/public/scripts/extensions/shared.js @@ -36,7 +36,7 @@ export async function getMultimodalCaption(base64Img, prompt) { const isVllm = extension_settings.caption.multimodal_api === 'vllm'; const base64Bytes = base64Img.length * 0.75; const compressionLimit = 2 * 1024 * 1024; - if ((['google', 'openrouter', 'mistral'].includes(extension_settings.caption.multimodal_api) && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) { + if ((['google', 'openrouter', 'mistral', 'groq'].includes(extension_settings.caption.multimodal_api) && base64Bytes > compressionLimit) || isOoba || isKoboldCpp) { const maxSide = 1024; base64Img = await createThumbnail(base64Img, maxSide, maxSide, 'image/jpeg'); } @@ -135,6 +135,10 @@ function throwIfInvalidModel(useReverseProxy) { throw new Error('01.AI API key is not set.'); } + if (extension_settings.caption.multimodal_api === 'groq' && !secret_state[SECRET_KEYS.GROQ]) { + throw new Error('Groq API key is not set.'); + } + if (extension_settings.caption.multimodal_api === 'google' && !secret_state[SECRET_KEYS.MAKERSUITE] && !useReverseProxy) { throw new Error('Google AI Studio API key is not set.'); } diff --git a/public/scripts/openai.js b/public/scripts/openai.js index 20c141316..670fd2968 100644 --- a/public/scripts/openai.js +++ b/public/scripts/openai.js @@ -4196,7 +4196,10 @@ async function onModelChange() { if (oai_settings.max_context_unlocked) { $('#openai_max_context').attr('max', unlocked_max); } - else if (oai_settings.groq_model.includes('llama-3.1')) { + else if (oai_settings.groq_model.includes('llama-3.2') && oai_settings.groq_model.includes('-preview')) { + $('#openai_max_context').attr('max', max_8k); + } + else if (oai_settings.groq_model.includes('llama-3.2') || oai_settings.groq_model.includes('llama-3.1')) { $('#openai_max_context').attr('max', max_128k); } else if (oai_settings.groq_model.includes('llama3-groq')) { diff --git a/src/endpoints/openai.js b/src/endpoints/openai.js index af1f107e2..e7760a8fb 100644 --- a/src/endpoints/openai.js +++ b/src/endpoints/openai.js @@ -55,6 +55,10 @@ router.post('/caption-image', jsonParser, async (request, response) => { key = readSecret(request.user.directories, SECRET_KEYS.MISTRALAI); } + if (request.body.api === 'groq') { + key = readSecret(request.user.directories, SECRET_KEYS.GROQ); + } + if (!key && !request.body.reverse_proxy && ['custom', 'ooba', 'koboldcpp', 'vllm'].includes(request.body.api) === false) { console.log('No key found for API', request.body.api); return response.sendStatus(400); @@ -111,6 +115,10 @@ router.post('/caption-image', jsonParser, async (request, response) => { apiUrl = 'https://api.01.ai/v1/chat/completions'; } + if (request.body.api === 'groq') { + apiUrl = 'https://api.groq.com/openai/v1/chat/completions'; + } + if (request.body.api === 'mistral') { apiUrl = 'https://api.mistral.ai/v1/chat/completions'; } From c59afd1b56f11df07bb137a9799ab0c505dd41c1 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 3 Oct 2024 09:51:38 +0300 Subject: [PATCH 2/3] Claude: console.log system before messages --- src/endpoints/backends/chat-completions.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/endpoints/backends/chat-completions.js b/src/endpoints/backends/chat-completions.js index ca86db051..cb4b77f65 100644 --- a/src/endpoints/backends/chat-completions.js +++ b/src/endpoints/backends/chat-completions.js @@ -107,6 +107,7 @@ async function sendClaudeRequest(request, response) { } const requestBody = { + /** @type {any} */ system: '', messages: convertedPrompt.messages, model: request.body.model, max_tokens: request.body.max_tokens, @@ -120,6 +121,8 @@ async function sendClaudeRequest(request, response) { requestBody.system = enableSystemPromptCache ? [{ type: 'text', text: convertedPrompt.systemPrompt, cache_control: { type: 'ephemeral' } }] : convertedPrompt.systemPrompt; + } else { + delete requestBody.system; } if (Array.isArray(request.body.tools) && request.body.tools.length > 0) { // Claude doesn't do prefills on function calls, and doesn't allow empty messages @@ -156,12 +159,13 @@ async function sendClaudeRequest(request, response) { forwardFetchResponse(generateResponse, response); } else { if (!generateResponse.ok) { - console.log(color.red(`Claude API returned error: ${generateResponse.status} ${generateResponse.statusText}\n${await generateResponse.text()}\n${divider}`)); + const generateResponseText = await generateResponse.text(); + console.log(color.red(`Claude API returned error: ${generateResponse.status} ${generateResponse.statusText}\n${generateResponseText}\n${divider}`)); return response.status(generateResponse.status).send({ error: true }); } const generateResponseJson = await generateResponse.json(); - const responseText = generateResponseJson.content[0].text; + const responseText = generateResponseJson?.content?.[0]?.text || ''; console.log('Claude response:', generateResponseJson); // Wrap it back to OAI format + save the original content From 1278b5c309b30d14f743a4720d453cd76f8f1f64 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Thu, 3 Oct 2024 22:13:37 +0300 Subject: [PATCH 3/3] Fix character attachment content saving into settings.json --- .../scripts/extensions/attachments/index.js | 21 +++++++++++++++++++ public/scripts/extensions/vectors/index.js | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/attachments/index.js b/public/scripts/extensions/attachments/index.js index 3ae36e46f..7db88dc4f 100644 --- a/public/scripts/extensions/attachments/index.js +++ b/public/scripts/extensions/attachments/index.js @@ -1,3 +1,4 @@ +import { event_types, eventSource, saveSettingsDebounced } from '../../../script.js'; import { deleteAttachment, getDataBankAttachments, getDataBankAttachmentsForSource, getFileAttachment, uploadFileAttachmentToServer } from '../../chats.js'; import { extension_settings, renderExtensionTemplateAsync } from '../../extensions.js'; import { SlashCommand } from '../../slash-commands/SlashCommand.js'; @@ -196,7 +197,27 @@ async function enableDataBankAttachment(args, value) { return ''; } +function cleanUpAttachments() { + let shouldSaveSettings = false; + if (extension_settings.character_attachments) { + Object.values(extension_settings.character_attachments).flat().filter(a => a.text).forEach(a => { + shouldSaveSettings = true; + delete a.text; + }); + } + if (Array.isArray(extension_settings.attachments)) { + extension_settings.attachments.filter(a => a.text).forEach(a => { + shouldSaveSettings = true; + delete a.text; + }); + } + if (shouldSaveSettings) { + saveSettingsDebounced(); + } +} + jQuery(async () => { + eventSource.on(event_types.APP_READY, cleanUpAttachments); const manageButton = await renderExtensionTemplateAsync('attachments', 'manage-button', {}); const attachButton = await renderExtensionTemplateAsync('attachments', 'attach-button', {}); $('#data_bank_wand_container').append(manageButton); diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 3d1a7a539..c78004917 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -466,13 +466,13 @@ async function ingestDataBankAttachments(source) { } // Download and process the file - file.text = await getFileAttachment(file.url); + const fileText = await getFileAttachment(file.url); console.log(`Vectors: Retrieved file ${file.name} from Data Bank`); // Convert kilobytes to string length const thresholdLength = settings.size_threshold_db * 1024; // Use chunk size from settings if file is larger than threshold const chunkSize = file.size > thresholdLength ? settings.chunk_size_db : -1; - await vectorizeFile(file.text, file.name, collectionId, chunkSize, settings.overlap_percent_db); + await vectorizeFile(fileText, file.name, collectionId, chunkSize, settings.overlap_percent_db); } return dataBankCollectionIds;