rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

system prompt support for gemini 1.5

This commit is contained in:
based 2024-04-11 16:38:20 +10:00
parent afeaca0fe0
commit 34bca69950
5 changed files with 50 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
public/index.html
View File

@ -1761,6 +1761,17 @@
<div class="toggle-description justifyLeft">
<span data-i18n="Use the appropriate tokenizer for Google models via their API. Slower prompt processing, but offers much more accurate token counting.">Use the appropriate tokenizer for Google models via their API. Slower prompt processing, but offers much more accurate token counting.</span>
</div>
<label for="use_makersuite_sysprompt" class="checkbox_label widthFreeExpand">
<input id="use_makersuite_sysprompt" type="checkbox" />
<span data-i18n="Use system prompt (Gemini 1.5 pro+ only)">
Use system prompt (Gemini 1.5 pro+ only)
</span>
</label>
<div class="toggle-description justifyLeft marginBot5">
<span data-i18n="Merges all system messages up until the first message with a non system role, and sends them through google's system_instruction field instead of with the rest of the prompt contents.">
Merges all system messages up until the first message with a non system role, and sends them through google's system_instruction field instead of with the rest of the prompt contents.
</span>
</div>
</div>
<div data-newbie-hidden class="range-block" data-source="claude">
<div class="wide100p">

12
public/scripts/openai.js
View File

@ -260,6 +260,7 @@ const default_settings = {
use_ai21_tokenizer: false,
use_google_tokenizer: false,
claude_use_sysprompt: false,
use_makersuite_sysprompt: true,
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
@ -330,6 +331,7 @@ const oai_settings = {
use_ai21_tokenizer: false,
use_google_tokenizer: false,
claude_use_sysprompt: false,
use_makersuite_sysprompt: true,
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
@ -1733,6 +1735,7 @@ async function sendOpenAIRequest(type, messages, signal) {
const stopStringsLimit = 3; // 5 - 2 (nameStopString and new_chat_prompt)
generate_data['top_k'] = Number(oai_settings.top_k_openai);
generate_data['stop'] = [nameStopString, substituteParams(oai_settings.new_chat_prompt), ...getCustomStoppingStrings(stopStringsLimit)];
generate_data['use_makersuite_sysprompt'] = oai_settings.use_makersuite_sysprompt;
}
if (isAI21) {
@ -2668,6 +2671,7 @@ function loadOpenAISettings(data, settings) {
if (settings.use_ai21_tokenizer !== undefined) { oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer; oai_settings.use_ai21_tokenizer ? ai21_max = 8191 : ai21_max = 9200; }
if (settings.use_google_tokenizer !== undefined) oai_settings.use_google_tokenizer = !!settings.use_google_tokenizer;
if (settings.claude_use_sysprompt !== undefined) oai_settings.claude_use_sysprompt = !!settings.claude_use_sysprompt;
if (settings.use_makersuite_sysprompt !== undefined) oai_settings.use_makersuite_sysprompt = !!settings.use_makersuite_sysprompt;
if (settings.use_alt_scale !== undefined) { oai_settings.use_alt_scale = !!settings.use_alt_scale; updateScaleForm(); }
$('#stream_toggle').prop('checked', oai_settings.stream_openai);
$('#api_url_scale').val(oai_settings.api_url_scale);
@ -2707,6 +2711,7 @@ function loadOpenAISettings(data, settings) {
$('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer);
$('#use_google_tokenizer').prop('checked', oai_settings.use_google_tokenizer);
$('#claude_use_sysprompt').prop('checked', oai_settings.claude_use_sysprompt);
$('#use_makersuite_sysprompt').prop('checked', oai_settings.use_makersuite_sysprompt);
$('#scale-alt').prop('checked', oai_settings.use_alt_scale);
$('#openrouter_use_fallback').prop('checked', oai_settings.openrouter_use_fallback);
$('#openrouter_force_instruct').prop('checked', oai_settings.openrouter_force_instruct);
@ -2976,6 +2981,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
use_ai21_tokenizer: settings.use_ai21_tokenizer,
use_google_tokenizer: settings.use_google_tokenizer,
claude_use_sysprompt: settings.claude_use_sysprompt,
use_makersuite_sysprompt: settings.use_makersuite_sysprompt,
use_alt_scale: settings.use_alt_scale,
squash_system_messages: settings.squash_system_messages,
image_inlining: settings.image_inlining,
@ -3354,6 +3360,7 @@ function onSettingsPresetChange() {
use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', true],
use_google_tokenizer: ['#use_google_tokenizer', 'use_google_tokenizer', true],
claude_use_sysprompt: ['#claude_use_sysprompt', 'claude_use_sysprompt', true],
use_makersuite_sysprompt: ['#use_makersuite_sysprompt', 'use_makersuite_sysprompt', true],
use_alt_scale: ['#use_alt_scale', 'use_alt_scale', true],
squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true],
image_inlining: ['#openai_image_inlining', 'image_inlining', true],
@ -4290,6 +4297,11 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#use_makersuite_sysprompt').on('change', function () {
oai_settings.use_makersuite_sysprompt = !!$('#use_makersuite_sysprompt').prop('checked');
saveSettingsDebounced();
});
$('#send_if_empty_textarea').on('input', function () {
oai_settings.send_if_empty = String($('#send_if_empty_textarea').val());
saveSettingsDebounced();

16
src/endpoints/backends/chat-completions.js
View File

@ -252,17 +252,25 @@ async function sendMakerSuiteRequest(request, response) {
};
function getGeminiBody() {
return {
contents: convertGooglePrompt(request.body.messages, model),
let should_use_system_prompt = model === 'gemini-1.5-pro-latest' && request.body.use_makersuite_sysprompt;
let prompt = convertGooglePrompt(request.body.messages, model, should_use_system_prompt);
let body = {
contents: prompt.contents,
safetySettings: GEMINI_SAFETY,
generationConfig: generationConfig,
};
}
if (should_use_system_prompt) {
body.system_instruction = prompt.system_instruction;
}
return body;
}
function getBisonBody() {
const prompt = isText
? ({ text: convertTextCompletionPrompt(request.body.messages) })
: ({ messages: convertGooglePrompt(request.body.messages, model) });
: ({ messages: convertGooglePrompt(request.body.messages, model).contents });
/** @type {any} Shut the lint up */
const bisonBody = {

2
src/endpoints/tokenizers.js
View File

@ -398,7 +398,7 @@ router.post('/google/count', jsonParser, async function (req, res) {
accept: 'application/json',
'content-type': 'application/json',
},
body: JSON.stringify({ contents: convertGooglePrompt(req.body, String(req.query.model)) }),
body: JSON.stringify({ contents: convertGooglePrompt(req.body, String(req.query.model)).contents }),
};
try {
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${req.query.model}:countTokens?key=${readSecret(SECRET_KEYS.MAKERSUITE)}`, options);

17
src/prompt-converters.js
View File

@ -252,9 +252,10 @@ function convertCohereMessages(messages, charName = '', userName = '') {
* Convert a prompt from the ChatML objects to the format used by Google MakerSuite models.
* @param {object[]} messages Array of messages
* @param {string} model Model name
* @returns {object[]} Prompt for Google MakerSuite models
* @param {boolean} useSysPrompt Use system prompt
* @returns {{contents: *[], system_instruction: {parts: {text: string}}}} Prompt for Google MakerSuite models
*/
function convertGooglePrompt(messages, model) {
function convertGooglePrompt(messages, model, useSysPrompt = false) {
// This is a 1x1 transparent PNG
const PNG_PIXEL = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';
@ -267,6 +268,16 @@ function convertGooglePrompt(messages, model) {
const isMultimodal = visionSupportedModels.includes(model);
let hasImage = false;
let sys_prompt = '';
if (useSysPrompt) {
while (messages.length > 1 && messages[0].role === 'system') {
sys_prompt += `${messages[0].content}\n\n`;
messages.shift();
}
}
const system_instruction = { parts: { text: sys_prompt }};
const contents = [];
messages.forEach((message, index) => {
// fix the roles
@ -327,7 +338,7 @@ function convertGooglePrompt(messages, model) {
});
}
return contents;
return { contents: contents, system_instruction: system_instruction };
}
/**