rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cleaned up a little

This commit is contained in:
based 2023-12-14 23:18:56 +10:00
parent 0b7c1a98cd
commit 178b07f414
2 changed files with 23 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
public/scripts/extensions/shared.js
View File

@ -30,25 +30,21 @@ export async function getMultimodalCaption(base64Img, prompt) {
base64Img = await createThumbnail(base64Img, maxSide, maxSide, 'image/jpeg');
}
const apiResult = extension_settings.caption.multimodal_api === 'google' ?
await fetch('/api/google/caption-image', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
image: base64Img,
prompt: prompt,
}),
})
: await fetch('/api/openai/caption-image', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
image: base64Img,
prompt: prompt,
api: extension_settings.caption.multimodal_api || 'openai',
model: extension_settings.caption.multimodal_model || 'gpt-4-vision-preview',
}),
});
const isGoogle = extension_settings.caption.multimodal_api === 'google';
const apiResult = await fetch(`/api/${isGoogle ? 'google' : 'openai'}/caption-image`, {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
image: base64Img,
prompt: prompt,
...(isGoogle
? {}
: {
api: extension_settings.caption.multimodal_api || 'openai',
model: extension_settings.caption.multimodal_model || 'gpt-4-vision-preview',
}),
}),
});
if (!apiResult.ok) {
throw new Error('Failed to caption image via OpenAI.');

10
src/endpoints/google.js
View File

@ -14,7 +14,7 @@ router.post('/caption-image', jsonParser, async (request, response) => {
parts: [
{ text: request.body.prompt },
{ inlineData: {
mimeType: 'image/png',
mimeType: 'image/png', //jpg images seem to work fine even with this mimetype set?
data: request.body.image,
},
}],
@ -40,8 +40,14 @@ router.post('/caption-image', jsonParser, async (request, response) => {
}
const data = await result.json();
console.log('Multimodal captioning response', data);
const caption = data?.candidates[0].content.parts[0].text;
const candidates = data?.candidates;
if(!candidates) {
return response.status(500).send('No candidates found, image was most likely filtered.');
}
const caption = candidates[0].content.parts[0].text;
if (!caption) {
return response.status(500).send('No caption found');
}