rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-02-09 08:38:53 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1973 from kingbased/goog

Browse Source 
Update Makersuite models + proper prompt conversion
This commit is contained in:
Cohee 2024-03-27 21:40:02 +02:00 committed by GitHub
commit 82d50e553d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 91 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
public/index.html
View File

@ -2604,11 +2604,20 @@
<div>
<h4 data-i18n="Google Model">Google Model</h4>
<select id="model_google_select">
<option value="gemini-1.5-pro">Gemini 1.5 Pro</option>
<option value="gemini-pro">Gemini Pro</option>
<option value="gemini-pro-vision">Gemini Pro Vision</option>
<option value="text-bison-001">Bison Text</option>
<option value="chat-bison-001">Bison Chat</option>
<optgroup label="Latest">
<!-- Points to 1.0, no default 1.5 endpoint -->
<option value="gemini-pro">Gemini Pro</option>
<option value="gemini-pro-vision">Gemini Pro Vision</option>
<option value="gemini-ultra">Gemini Ultra</option>
<option value="text-bison-001">Bison Text</option>
<option value="chat-bison-001">Bison Chat</option>
</optgroup>
<optgroup label="Sub-versions">
<option value="gemini-1.5-pro-latest">Gemini 1.5 Pro</option>
<option value="gemini-1.0-pro-latest">Gemini 1.0 Pro</option>
<option value="gemini-1.0-pro-vision-latest">Gemini 1.0 Pro Vision</option>
<option value="gemini-1.0-ultra-latest">Gemini 1.0 Ultra</option>
</optgroup>
</select>
</div>
</form>

25
public/scripts/openai.js
View File

@ -3507,11 +3507,11 @@ async function onModelChange() {
if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
if (oai_settings.max_context_unlocked) {
$('#openai_max_context').attr('max', unlocked_max);
} else if (value === 'gemini-1.5-pro') {
} else if (value === 'gemini-1.5-pro-latest') {
$('#openai_max_context').attr('max', max_1mil);
} else if (value === 'gemini-pro') {
} else if (value === 'gemini-ultra' || value === 'gemini-1.0-pro-latest' || value === 'gemini-pro' || value === 'gemini-1.0-ultra-latest') {
$('#openai_max_context').attr('max', max_32k);
} else if (value === 'gemini-pro-vision') {
} else if (value === 'gemini-1.0-pro-vision-latest' || value === 'gemini-pro-vision') {
$('#openai_max_context').attr('max', max_16k);
} else {
$('#openai_max_context').attr('max', max_8k);
@ -3939,21 +3939,26 @@ export function isImageInliningSupported() {
return false;
}
const gpt4v = 'gpt-4-vision';
const geminiProV = 'gemini-pro-vision';
const claude = 'claude-3';
if (!oai_settings.image_inlining) {
return false;
}
// gultra just isn't being offered as multimodal, thanks google.
const visionSupportedModels = [
'gpt-4-vision',
'gemini-1.0-pro-vision-latest',
'gemini-1.5-pro-latest',
'gemini-pro-vision',
'claude-3'
];
switch (oai_settings.chat_completion_source) {
case chat_completion_sources.OPENAI:
return oai_settings.openai_model.includes(gpt4v);
return visionSupportedModels.some(model => oai_settings.openai_model.includes(model));
case chat_completion_sources.MAKERSUITE:
return oai_settings.google_model.includes(geminiProV);
return visionSupportedModels.some(model => oai_settings.google_model.includes(model));
case chat_completion_sources.CLAUDE:
return oai_settings.claude_model.includes(claude);
return visionSupportedModels.some(model => oai_settings.claude_model.includes(model));
case chat_completion_sources.OPENROUTER:
return !oai_settings.openrouter_force_instruct;
case chat_completion_sources.CUSTOM:

104
src/prompt-converters.js
View File

@ -192,53 +192,73 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
function convertGooglePrompt(messages, model) {
// This is a 1x1 transparent PNG
const PNG_PIXEL = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';
const visionSupportedModels = [
'gemini-1.0-pro-vision-latest',
'gemini-1.5-pro-latest',
'gemini-pro-vision',
];
const isMultimodal = visionSupportedModels.includes(model);
let hasImage = false;
const contents = [];
let lastRole = '';
let currentText = '';
messages.forEach((message, index) => {
// fix the roles
if (message.role === 'system') {
message.role = 'user';
} else if (message.role === 'assistant') {
message.role = 'model';
}
const isMultimodal = model === 'gemini-pro-vision';
if (isMultimodal) {
const combinedText = messages.map((message) => {
const role = message.role === 'assistant' ? 'MODEL: ' : 'USER: ';
return role + message.content;
}).join('\n\n').trim();
const imageEntry = messages.find((message) => message.content?.[1]?.image_url);
const imageData = imageEntry?.content?.[1]?.image_url?.data ?? PNG_PIXEL;
contents.push({
parts: [
{ text: combinedText },
{
inlineData: {
mimeType: 'image/png',
data: imageData,
},
},
],
role: 'user',
});
} else {
messages.forEach((message, index) => {
const role = message.role === 'assistant' ? 'model' : 'user';
if (lastRole === role) {
currentText += '\n\n' + message.content;
// similar story as claude
if (message.name) {
if (Array.isArray(message.content)) {
message.content[0].text = `${message.name}: ${message.content[0].text}`;
} else {
if (currentText !== '') {
contents.push({
parts: [{ text: currentText.trim() }],
role: lastRole,
message.content = `${message.name}: ${message.content}`;
}
delete message.name;
}
//create the prompt parts
const parts = [];
if (typeof message.content === 'string') {
parts.push({ text: message.content });
} else if (Array.isArray(message.content)) {
message.content.forEach((part) => {
if (part.type === 'text') {
parts.push({ text: part.text });
} else if (part.type === 'image_url' && isMultimodal) {
parts.push({
inlineData: {
mimeType: 'image/png',
data: part.image_url.url,
},
});
hasImage = true;
}
currentText = message.content;
lastRole = role;
}
if (index === messages.length - 1) {
contents.push({
parts: [{ text: currentText.trim() }],
role: lastRole,
});
}
});
}
// merge consecutive messages with the same role
if (index > 0 && message.role === contents[contents.length - 1].role) {
contents[contents.length - 1].parts[0].text += '\n\n' + parts[0].text;
} else {
contents.push({
role: message.role,
parts: parts,
});
}
});
// pro 1.5 doesn't require a dummy image to be attached, other vision models do
if (isMultimodal && model !== 'gemini-1.5-pro-latest' && !hasImage) {
contents[0].parts.push({
inlineData: {
mimeType: 'image/png',
data: PNG_PIXEL,
},
});
}