rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'staging' into parser-v2

This commit is contained in:
LenAnderson 2024-03-29 08:43:05 -04:00
parent 21480862f5 7c99d87238
commit 84907aebf0
10 changed files with 192 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
public/index.html
View File

@ -2604,11 +2604,20 @@
<div>
<h4 data-i18n="Google Model">Google Model</h4>
<select id="model_google_select">
<option value="gemini-1.5-pro">Gemini 1.5 Pro</option>
<option value="gemini-pro">Gemini Pro</option>
<option value="gemini-pro-vision">Gemini Pro Vision</option>
<option value="text-bison-001">Bison Text</option>
<option value="chat-bison-001">Bison Chat</option>
<optgroup label="Latest">
<!-- Points to 1.0, no default 1.5 endpoint -->
<option value="gemini-pro">Gemini Pro</option>
<option value="gemini-pro-vision">Gemini Pro Vision</option>
<option value="gemini-ultra">Gemini Ultra</option>
<option value="text-bison-001">Bison Text</option>
<option value="chat-bison-001">Bison Chat</option>
</optgroup>
<optgroup label="Sub-versions">
<option value="gemini-1.5-pro-latest">Gemini 1.5 Pro</option>
<option value="gemini-1.0-pro-latest">Gemini 1.0 Pro</option>
<option value="gemini-1.0-pro-vision-latest">Gemini 1.0 Pro Vision</option>
<option value="gemini-1.0-ultra-latest">Gemini 1.0 Ultra</option>
</optgroup>
</select>
</div>
</form>
@ -4109,7 +4118,7 @@
</div>
<div id="tags_div">
<div class="tag_controls">
<input id="tagInput" class="text_pole tag_input wide100p margin0" data-i18n="[placeholder]Search / Create Tags" placeholder="Search / Create tags" maxlength="50" />
<input id="tagInput" class="text_pole textarea_compact tag_input wide100p margin0" data-i18n="[placeholder]Search / Create Tags" placeholder="Search / Create tags" maxlength="50" />
<div class="tags_view menu_button fa-solid fa-tags" title="View all tags" data-i18n="[title]View all tags"></div>
</div>
<div id="tagList" class="tags"></div>
@ -4129,7 +4138,7 @@
</div>
<div id="descriptionWrapper" class="flex-container flexFlowColumn flex1">
<hr>
<div id="description_div" class="marginBot5 flex-container alignitemscenter">
<div id="description_div" class="flex-container alignitemscenter">
<span data-i18n="Character Description">Description</span>
<i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="description_textarea" title="Expand the editor"></i>
<a href="https://docs.sillytavern.app/usage/core-concepts/characterdesign/#character-description" class="notes-link" target="_blank">
@ -4142,7 +4151,7 @@
</div>
</div>
<div id="firstMessageWrapper" class="flex-container flexFlowColumn flex1">
<div id="first_message_div" class="marginBot5 title_restorable">
<div id="first_message_div" class="title_restorable">
<div class="flex-container alignitemscenter flex1">
<span data-i18n="First message">First message</span>
<i class="editor_maximize fa-solid fa-maximize right_menu_button" data-for="firstmessage_textarea" title="Expand the editor"></i>
@ -4192,7 +4201,7 @@
</div>
<div id="group_tags_div" class="wide100p">
<div class="tag_controls">
<input id="groupTagInput" class="text_pole tag_input flex1 margin0" data-i18n="[placeholder]Search / Create Tags" placeholder="Search / Create tags" maxlength="50" />
<input id="groupTagInput" class="text_pole textarea_compact tag_input flex1 margin0" data-i18n="[placeholder]Search / Create Tags" placeholder="Search / Create tags" maxlength="50" />
<div class="tags_view menu_button fa-solid fa-tags margin0" title="View all tags" data-i18n="[title]View all tags"></div>
</div>
<div id="groupTagList" class="tags paddingTopBot5"></div>

23
public/script.js
View File

@ -172,6 +172,7 @@ import {
importTags,
tag_filter_types,
compareTagsForSort,
initTags,
} from './scripts/tags.js';
import {
SECRET_KEYS,
@ -412,6 +413,7 @@ export const event_types = {
CHARACTER_FIRST_MESSAGE_SELECTED: 'character_first_message_selected',
// TODO: Naming convention is inconsistent with other events
CHARACTER_DELETED: 'characterDeleted',
CHARACTER_DUPLICATED: 'character_duplicated',
};
export const eventSource = new EventEmitter();
@ -864,6 +866,7 @@ async function firstLoadInit() {
getSystemMessages();
sendSystemMessage(system_message_types.WELCOME);
initLocales();
initTags();
await getUserAvatars(true, user_avatar);
await getCharacters();
await getBackgrounds();
@ -1237,7 +1240,7 @@ function getCharacterBlock(item, id) {
const template = $('#character_template .character_select').clone();
template.attr({ 'chid': id, 'id': `CharID${id}` });
template.find('img').attr('src', this_avatar).attr('alt', item.name);
template.find('.avatar').attr('title', `[Character] ${item.name}`);
template.find('.avatar').attr('title', `[Character] ${item.name}\nFile: ${item.avatar}`);
template.find('.ch_name').text(item.name).attr('title', `[Character] ${item.name}`);
if (power_user.show_card_avatar_urls) {
template.find('.ch_avatar_url').text(item.avatar);
@ -1855,6 +1858,7 @@ function insertSVGIcon(mes, extra) {
function getMessageFromTemplate({
mesId,
swipeId,
characterName,
isUser,
avatarImg,
@ -1872,6 +1876,7 @@ function getMessageFromTemplate({
const mes = messageTemplate.clone();
mes.attr({
'mesid': mesId,
'swipeid': swipeId,
'ch_name': characterName,
'is_user': isUser,
'is_system': !!isSystem,
@ -2018,6 +2023,7 @@ function addOneMessage(mes, { type = 'normal', insertAfter = null, scroll = true
let params = {
mesId: forceId ?? chat.length - 1,
swipeId: mes.swipe_id ?? 0,
characterName: mes.name,
isUser: mes.is_user,
avatarImg: avatarImg,
@ -2727,9 +2733,7 @@ class StreamingProcessor {
const continueMsg = this.type === 'continue' ? this.messageAlreadyGenerated : undefined;
saveLogprobsForActiveMessage(this.messageLogprobs.filter(Boolean), continueMsg);
await saveChatConditional();
activateSendButtons();
showSwipeButtons();
setGenerationProgress(0);
unblockGeneration();
generatedPromptCache = '';
//console.log("Generated text size:", text.length, text)
@ -2772,11 +2776,8 @@ class StreamingProcessor {
this.isStopped = true;
this.hideMessageButtons(this.messageId);
$('#send_textarea').removeAttr('disabled');
is_send_press = false;
activateSendButtons();
setGenerationProgress(0);
showSwipeButtons();
generatedPromptCache = '';
unblockGeneration();
}
setFirstSwipe(messageId) {
@ -3967,6 +3968,8 @@ async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, qu
toastr.error(exception.error.message, 'Error', { timeOut: 10000, extendedTimeOut: 20000 });
}
generatedPromptCache = '';
unblockGeneration();
console.log(exception);
streamingProcessor = null;
@ -4320,6 +4323,8 @@ async function DupeChar() {
});
if (response.ok) {
toastr.success('Character Duplicated');
const data = await response.json();
await eventSource.emit(event_types.CHARACTER_DUPLICATED, { oldAvatar: body.avatar_url, newAvatar: data.path });
getCharacters();
}
}

14
public/scripts/BulkEditOverlay.js
View File

@ -48,16 +48,24 @@ class CharacterContextMenu {
* Duplicate one or more characters
*
* @param characterId
* @returns {Promise<Response>}
* @returns {Promise<any>}
*/
static duplicate = async (characterId) => {
const character = CharacterContextMenu.#getCharacter(characterId);
const body = { avatar_url: character.avatar };
return fetch('/api/characters/duplicate', {
const result = await fetch('/api/characters/duplicate', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ avatar_url: character.avatar }),
body: JSON.stringify(body),
});
if (!result.ok) {
throw new Error('Character not duplicated');
}
const data = await result.json();
await eventSource.emit(event_types.CHARACTER_DUPLICATED, { oldAvatar: body.avatar_url, newAvatar: data.path });
};
/**

69
public/scripts/RossAscends-mods.js
View File

@ -399,6 +399,7 @@ function saveUserInput() {
const userInput = String($('#send_textarea').val());
SaveLocal('userInput', userInput);
}
const saveUserInputDebounced = debounce(saveUserInput);
// Make the DIV element draggable:
@ -658,6 +659,30 @@ export async function initMovingUI() {
}
}
/**@type {HTMLTextAreaElement} */
const sendTextArea = document.querySelector('#send_textarea');
const chatBlock = document.getElementById('chat');
const isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1;
/**
* this makes the chat input text area resize vertically to match the text size (limited by CSS at 50% window height)
*/
function autoFitSendTextArea() {
const originalScrollBottom = chatBlock.scrollHeight - (chatBlock.scrollTop + chatBlock.offsetHeight);
if (sendTextArea.scrollHeight == sendTextArea.offsetHeight) {
// Needs to be pulled dynamically because it is affected by font size changes
const sendTextAreaMinHeight = window.getComputedStyle(sendTextArea).getPropertyValue('min-height');
sendTextArea.style.height = sendTextAreaMinHeight;
}
sendTextArea.style.height = sendTextArea.scrollHeight + 0.3 + 'px';
if (!isFirefox) {
const newScrollTop = Math.round(chatBlock.scrollHeight - (chatBlock.offsetHeight + originalScrollBottom));
chatBlock.scrollTop = newScrollTop;
}
}
export const autoFitSendTextAreaDebounced = debounce(autoFitSendTextArea);
// ---------------------------------------------------
export function initRossMods() {
@ -820,19 +845,13 @@ export function initRossMods() {
saveSettingsDebounced();
});
//this makes the chat input text area resize vertically to match the text size (limited by CSS at 50% window height)
$('#send_textarea').on('input', function () {
const isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1;
const chatBlock = $('#chat');
const originalScrollBottom = chatBlock[0].scrollHeight - (chatBlock.scrollTop() + chatBlock.outerHeight());
this.style.height = window.getComputedStyle(this).getPropertyValue('min-height');
this.style.height = this.scrollHeight + 0.3 + 'px';
if (!isFirefox) {
const newScrollTop = Math.round(chatBlock[0].scrollHeight - (chatBlock.outerHeight() + originalScrollBottom));
chatBlock.scrollTop(newScrollTop);
$(sendTextArea).on('input', () => {
if (sendTextArea.scrollHeight > sendTextArea.offsetHeight || sendTextArea.value === '') {
autoFitSendTextArea();
} else {
autoFitSendTextAreaDebounced();
}
saveUserInput();
saveUserInputDebounced();
});
restoreUserInput();
@ -887,23 +906,30 @@ export function initRossMods() {
processHotkeys(event.originalEvent);
});
const hotkeyTargets = {
'send_textarea': sendTextArea,
'dialogue_popup_input': document.querySelector('#dialogue_popup_input'),
};
//Additional hotkeys CTRL+ENTER and CTRL+UPARROW
/**
* @param {KeyboardEvent} event
*/
function processHotkeys(event) {
//Enter to send when send_textarea in focus
if ($(':focus').attr('id') === 'send_textarea') {
if (document.activeElement == hotkeyTargets['send_textarea']) {
const sendOnEnter = shouldSendOnEnter();
if (!event.shiftKey && !event.ctrlKey && !event.altKey && event.key == 'Enter' && sendOnEnter) {
event.preventDefault();
sendTextareaMessage();
return;
}
}
if ($(':focus').attr('id') === 'dialogue_popup_input' && !isMobile()) {
if (document.activeElement == hotkeyTargets['dialogue_popup_input'] && !isMobile()) {
if (!event.shiftKey && !event.ctrlKey && event.key == 'Enter') {
event.preventDefault();
$('#dialogue_popup_ok').trigger('click');
return;
}
}
//ctrl+shift+up to scroll to context line
@ -915,6 +941,7 @@ export function initRossMods() {
scrollTop: contextLine.offset().top - $('#chat').offset().top + $('#chat').scrollTop(),
}, 300);
} else { toastr.warning('Context line not found, send a message first!'); }
return;
}
//ctrl+shift+down to scroll to bottom of chat
if (event.shiftKey && event.ctrlKey && event.key == 'ArrowDown') {
@ -922,6 +949,7 @@ export function initRossMods() {
$('#chat').animate({
scrollTop: $('#chat').prop('scrollHeight'),
}, 300);
return;
}
// Alt+Enter or AltGr+Enter to Continue
@ -929,6 +957,7 @@ export function initRossMods() {
if (is_send_press == false) {
console.debug('Continuing with Alt+Enter');
$('#option_continue').trigger('click');
return;
}
}
@ -938,6 +967,7 @@ export function initRossMods() {
if (editMesDone.length > 0) {
console.debug('Accepting edits with Ctrl+Enter');
editMesDone.trigger('click');
return;
} else if (is_send_press == false) {
const skipConfirmKey = 'RegenerateWithCtrlEnter';
const skipConfirm = LoadLocalBool(skipConfirmKey);
@ -964,6 +994,7 @@ export function initRossMods() {
doRegenerate();
});
}
return;
} else {
console.debug('Ctrl+Enter ignored');
}
@ -972,7 +1003,7 @@ export function initRossMods() {
// Helper function to check if nanogallery2's lightbox is active
function isNanogallery2LightboxActive() {
// Check if the body has the 'nGY2On' class, adjust this based on actual behavior
return $('body').hasClass('nGY2_body_scrollbar');
return document.body.classList.contains('nGY2_body_scrollbar');
}
if (event.key == 'ArrowLeft') { //swipes left
@ -985,6 +1016,7 @@ export function initRossMods() {
!isInputElementInFocus()
) {
$('.swipe_left:last').click();
return;
}
}
if (event.key == 'ArrowRight') { //swipes right
@ -997,13 +1029,14 @@ export function initRossMods() {
!isInputElementInFocus()
) {
$('.swipe_right:last').click();
return;
}
}
if (event.ctrlKey && event.key == 'ArrowUp') { //edits last USER message if chatbar is empty and focused
if (
$('#send_textarea').val() === '' &&
hotkeyTargets['send_textarea'].value === '' &&
chatbarInFocus === true &&
($('.swipe_right:last').css('display') === 'flex' || $('.last_mes').attr('is_system') === 'true') &&
$('#character_popup').css('display') === 'none' &&
@ -1014,6 +1047,7 @@ export function initRossMods() {
const editMes = lastIsUserMes.querySelector('.mes_block .mes_edit');
if (editMes !== null) {
$(editMes).trigger('click');
return;
}
}
}
@ -1021,7 +1055,7 @@ export function initRossMods() {
if (event.key == 'ArrowUp') { //edits last message if chatbar is empty and focused
console.log('got uparrow input');
if (
$('#send_textarea').val() === '' &&
hotkeyTargets['send_textarea'].value === '' &&
chatbarInFocus === true &&
//$('.swipe_right:last').css('display') === 'flex' &&
$('.last_mes .mes_buttons').is(':visible') &&
@ -1032,6 +1066,7 @@ export function initRossMods() {
const editMes = lastMes.querySelector('.mes_block .mes_edit');
if (editMes !== null) {
$(editMes).click();
return;
}
}
}

25
public/scripts/openai.js
View File

@ -3507,11 +3507,11 @@ async function onModelChange() {
if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
if (oai_settings.max_context_unlocked) {
$('#openai_max_context').attr('max', unlocked_max);
} else if (value === 'gemini-1.5-pro') {
} else if (value === 'gemini-1.5-pro-latest') {
$('#openai_max_context').attr('max', max_1mil);
} else if (value === 'gemini-pro') {
} else if (value === 'gemini-ultra' || value === 'gemini-1.0-pro-latest' || value === 'gemini-pro' || value === 'gemini-1.0-ultra-latest') {
$('#openai_max_context').attr('max', max_32k);
} else if (value === 'gemini-pro-vision') {
} else if (value === 'gemini-1.0-pro-vision-latest' || value === 'gemini-pro-vision') {
$('#openai_max_context').attr('max', max_16k);
} else {
$('#openai_max_context').attr('max', max_8k);
@ -3939,21 +3939,26 @@ export function isImageInliningSupported() {
return false;
}
const gpt4v = 'gpt-4-vision';
const geminiProV = 'gemini-pro-vision';
const claude = 'claude-3';
if (!oai_settings.image_inlining) {
return false;
}
// gultra just isn't being offered as multimodal, thanks google.
const visionSupportedModels = [
'gpt-4-vision',
'gemini-1.0-pro-vision-latest',
'gemini-1.5-pro-latest',
'gemini-pro-vision',
'claude-3'
];
switch (oai_settings.chat_completion_source) {
case chat_completion_sources.OPENAI:
return oai_settings.openai_model.includes(gpt4v);
return visionSupportedModels.some(model => oai_settings.openai_model.includes(model));
case chat_completion_sources.MAKERSUITE:
return oai_settings.google_model.includes(geminiProV);
return visionSupportedModels.some(model => oai_settings.google_model.includes(model));
case chat_completion_sources.CLAUDE:
return oai_settings.claude_model.includes(claude);
return visionSupportedModels.some(model => oai_settings.claude_model.includes(model));
case chat_completion_sources.OPENROUTER:
return !oai_settings.openrouter_force_instruct;
case chat_completion_sources.CUSTOM:

22
public/scripts/tags.js
View File

@ -8,6 +8,8 @@ import {
entitiesFilter,
printCharacters,
buildAvatarList,
eventSource,
event_types,
} from '../script.js';
// eslint-disable-next-line no-unused-vars
import { FILTER_TYPES, FILTER_STATES, isFilterState, FilterHelper } from './filters.js';
@ -132,7 +134,7 @@ function filterByTagState(entities, { globalDisplayFilters = false, subForEntity
}
if (subForEntity !== undefined && subForEntity.type === 'tag') {
entities = filterTagSubEntities(subForEntity.item, entities, { filterHidden : filterHidden });
entities = filterTagSubEntities(subForEntity.item, entities, { filterHidden: filterHidden });
}
return entities;
@ -1140,7 +1142,7 @@ function onClearAllFiltersClick() {
// We have to manually go through the elements and unfilter by clicking...
// Thankfully nearly all filter controls are three-state-toggles
const filterTags = $('.rm_tag_controls .rm_tag_filter').find('.tag');
for(const tag of filterTags) {
for (const tag of filterTags) {
const toggleState = $(tag).attr('data-toggle-state');
if (toggleState !== undefined && !isFilterState(toggleState ?? FILTER_STATES.UNDEFINED, FILTER_STATES.UNDEFINED)) {
toggleTagThreeState($(tag), { stateOverride: FILTER_STATES.UNDEFINED, simulateClick: true });
@ -1151,7 +1153,17 @@ function onClearAllFiltersClick() {
$('#character_search_bar').val('').trigger('input');
}
jQuery(() => {
/**
* Copy tags from one character to another.
* @param {{oldAvatar: string, newAvatar: string}} data Event data
*/
function copyTags(data) {
const prevTagMap = tag_map[data.oldAvatar] || [];
const newTagMap = tag_map[data.newAvatar] || [];
tag_map[data.newAvatar] = Array.from(new Set([...prevTagMap, ...newTagMap]));
}
export function initTags() {
createTagInput('#tagInput', '#tagList');
createTagInput('#groupTagInput', '#groupTagList');
@ -1168,5 +1180,5 @@ jQuery(() => {
$(document).on('click', '.tag_view_create', onTagCreateClick);
$(document).on('click', '.tag_view_backup', onTagsBackupClick);
$(document).on('click', '.tag_view_restore', onBackupRestoreClick);
});
eventSource.on(event_types.CHARACTER_DUPLICATED, copyTags);
}

2
public/scripts/templates/macros.html
View File

@ -33,7 +33,7 @@
<li><tt>&lcub;&lcub;time_UTC±#&rcub;&rcub;</tt> – the current time in the specified UTC time zone offset, e.g. UTC-4 or UTC+2</li>
<li><tt>&lcub;&lcub;idle_duration&rcub;&rcub;</tt> – the time since the last user message was sent</li>
<li><tt>&lcub;&lcub;bias "text here"&rcub;&rcub;</tt> – sets a behavioral bias for the AI until the next user input. Quotes around the text are important.</li>
<li><tt>&lcub;&lcub;roll:(formula)&rcub;&rcub;</tt> – rolls a dice. (ex: <tt>>&lcub;&lcub;roll:1d6&rcub;&rcub</tt> will roll a 6-sided dice and return a number between 1 and 6)</li>
<li><tt>&lcub;&lcub;roll:(formula)&rcub;&rcub;</tt> – rolls a dice. (ex: <tt>>&lcub;&lcub;roll:1d6&rcub;&rcub;</tt> will roll a 6-sided dice and return a number between 1 and 6)</li>
<li><tt>&lcub;&lcub;random:(args)&rcub;&rcub;</tt> – returns a random item from the list. (ex: <tt>&lcub;&lcub;random:1,2,3,4&rcub;&rcub;</tt> will return 1 of the 4 numbers at random. Works with text lists too.</li>
<li><tt>&lcub;&lcub;random::(arg1)::(arg2)&rcub;&rcub;</tt> – alternative syntax for random that allows to use commas in the list items.</li>
<li><tt>&lcub;&lcub;banned "text here"&rcub;&rcub;</tt> – dynamically add text in the quotes to banned words sequences, if Text Generation WebUI backend used. Do nothing for others backends. Can be used anywhere (Character description, WI, AN, etc.) Quotes around the text are important.</li>

2
public/style.css
View File

@ -1445,7 +1445,7 @@ input[type="file"] {
}
.extension_token_counter {
font-size: calc(var(--mainFontSize) * 0.9);
font-size: calc(var(--mainFontSize) * 0.875);
width: 100%;
text-align: right;
margin-bottom: 5px;

2
src/endpoints/characters.js
View File

@ -1008,7 +1008,7 @@ router.post('/duplicate', jsonParser, async function (request, response) {
fs.copyFileSync(filename, newFilename);
console.log(`${filename} was copied to ${newFilename}`);
response.sendStatus(200);
response.send({ path: path.parse(newFilename).base });
}
catch (error) {
console.error(error);

104
src/prompt-converters.js
View File

@ -192,53 +192,73 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, humanMsgFi
function convertGooglePrompt(messages, model) {
// This is a 1x1 transparent PNG
const PNG_PIXEL = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';
const visionSupportedModels = [
'gemini-1.0-pro-vision-latest',
'gemini-1.5-pro-latest',
'gemini-pro-vision',
];
const isMultimodal = visionSupportedModels.includes(model);
let hasImage = false;
const contents = [];
let lastRole = '';
let currentText = '';
messages.forEach((message, index) => {
// fix the roles
if (message.role === 'system') {
message.role = 'user';
} else if (message.role === 'assistant') {
message.role = 'model';
}
const isMultimodal = model === 'gemini-pro-vision';
if (isMultimodal) {
const combinedText = messages.map((message) => {
const role = message.role === 'assistant' ? 'MODEL: ' : 'USER: ';
return role + message.content;
}).join('\n\n').trim();
const imageEntry = messages.find((message) => message.content?.[1]?.image_url);
const imageData = imageEntry?.content?.[1]?.image_url?.data ?? PNG_PIXEL;
contents.push({
parts: [
{ text: combinedText },
{
inlineData: {
mimeType: 'image/png',
data: imageData,
},
},
],
role: 'user',
});
} else {
messages.forEach((message, index) => {
const role = message.role === 'assistant' ? 'model' : 'user';
if (lastRole === role) {
currentText += '\n\n' + message.content;
// similar story as claude
if (message.name) {
if (Array.isArray(message.content)) {
message.content[0].text = `${message.name}: ${message.content[0].text}`;
} else {
if (currentText !== '') {
contents.push({
parts: [{ text: currentText.trim() }],
role: lastRole,
message.content = `${message.name}: ${message.content}`;
}
delete message.name;
}
//create the prompt parts
const parts = [];
if (typeof message.content === 'string') {
parts.push({ text: message.content });
} else if (Array.isArray(message.content)) {
message.content.forEach((part) => {
if (part.type === 'text') {
parts.push({ text: part.text });
} else if (part.type === 'image_url' && isMultimodal) {
parts.push({
inlineData: {
mimeType: 'image/png',
data: part.image_url.url,
},
});
hasImage = true;
}
currentText = message.content;
lastRole = role;
}
if (index === messages.length - 1) {
contents.push({
parts: [{ text: currentText.trim() }],
role: lastRole,
});
}
});
}
// merge consecutive messages with the same role
if (index > 0 && message.role === contents[contents.length - 1].role) {
contents[contents.length - 1].parts[0].text += '\n\n' + parts[0].text;
} else {
contents.push({
role: message.role,
parts: parts,
});
}
});
// pro 1.5 doesn't require a dummy image to be attached, other vision models do
if (isMultimodal && model !== 'gemini-1.5-pro-latest' && !hasImage) {
contents[0].parts.push({
inlineData: {
mimeType: 'image/png',
data: PNG_PIXEL,
},
});
}