rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2063 from kingbased/goog 

Gemini system prompt support
This commit is contained in:
Cohee 2024-04-11 19:49:20 +03:00 committed by GitHub
parent 6e377fdbea 58152a60c6
commit ee62f77e96
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 64 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
public/index.html
View File

@ -1762,6 +1762,19 @@
<span data-i18n="Use the appropriate tokenizer for Google models via their API. Slower prompt processing, but offers much more accurate token counting.">Use the appropriate tokenizer for Google models via their API. Slower prompt processing, but offers much more accurate token counting.</span>
</div>
</div>
<div class="range-block" data-source="makersuite">
<label for="use_makersuite_sysprompt" class="checkbox_label widthFreeExpand">
<input id="use_makersuite_sysprompt" type="checkbox" />
<span data-i18n="Use system prompt (Gemini 1.5 pro+ only)">
Use system prompt (Gemini 1.5 pro+ only)
</span>
</label>
<div class="toggle-description justifyLeft marginBot5">
<span data-i18n="Merges all system messages up until the first message with a non system role, and sends them through google's system_instruction field instead of with the rest of the prompt contents.">
Merges all system messages up until the first message with a non system role, and sends them through google's system_instruction field instead of with the rest of the prompt contents.
</span>
</div>
</div>
<div data-newbie-hidden class="range-block" data-source="claude">
<div class="wide100p">
<span id="claude_assistant_prefill_text" data-i18n="Assistant Prefill">Assistant Prefill</span>

12
public/scripts/openai.js
View File

@ -260,6 +260,7 @@ const default_settings = {
use_ai21_tokenizer: false,
use_google_tokenizer: false,
claude_use_sysprompt: false,
use_makersuite_sysprompt: true,
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
@ -330,6 +331,7 @@ const oai_settings = {
use_ai21_tokenizer: false,
use_google_tokenizer: false,
claude_use_sysprompt: false,
use_makersuite_sysprompt: true,
use_alt_scale: false,
squash_system_messages: false,
image_inlining: false,
@ -1733,6 +1735,7 @@ async function sendOpenAIRequest(type, messages, signal) {
const stopStringsLimit = 3; // 5 - 2 (nameStopString and new_chat_prompt)
generate_data['top_k'] = Number(oai_settings.top_k_openai);
generate_data['stop'] = [nameStopString, substituteParams(oai_settings.new_chat_prompt), ...getCustomStoppingStrings(stopStringsLimit)];
generate_data['use_makersuite_sysprompt'] = oai_settings.use_makersuite_sysprompt;
}
if (isAI21) {
@ -2668,6 +2671,7 @@ function loadOpenAISettings(data, settings) {
if (settings.use_ai21_tokenizer !== undefined) { oai_settings.use_ai21_tokenizer = !!settings.use_ai21_tokenizer; oai_settings.use_ai21_tokenizer ? ai21_max = 8191 : ai21_max = 9200; }
if (settings.use_google_tokenizer !== undefined) oai_settings.use_google_tokenizer = !!settings.use_google_tokenizer;
if (settings.claude_use_sysprompt !== undefined) oai_settings.claude_use_sysprompt = !!settings.claude_use_sysprompt;
if (settings.use_makersuite_sysprompt !== undefined) oai_settings.use_makersuite_sysprompt = !!settings.use_makersuite_sysprompt;
if (settings.use_alt_scale !== undefined) { oai_settings.use_alt_scale = !!settings.use_alt_scale; updateScaleForm(); }
$('#stream_toggle').prop('checked', oai_settings.stream_openai);
$('#api_url_scale').val(oai_settings.api_url_scale);
@ -2707,6 +2711,7 @@ function loadOpenAISettings(data, settings) {
$('#use_ai21_tokenizer').prop('checked', oai_settings.use_ai21_tokenizer);
$('#use_google_tokenizer').prop('checked', oai_settings.use_google_tokenizer);
$('#claude_use_sysprompt').prop('checked', oai_settings.claude_use_sysprompt);
$('#use_makersuite_sysprompt').prop('checked', oai_settings.use_makersuite_sysprompt);
$('#scale-alt').prop('checked', oai_settings.use_alt_scale);
$('#openrouter_use_fallback').prop('checked', oai_settings.openrouter_use_fallback);
$('#openrouter_force_instruct').prop('checked', oai_settings.openrouter_force_instruct);
@ -2976,6 +2981,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
use_ai21_tokenizer: settings.use_ai21_tokenizer,
use_google_tokenizer: settings.use_google_tokenizer,
claude_use_sysprompt: settings.claude_use_sysprompt,
use_makersuite_sysprompt: settings.use_makersuite_sysprompt,
use_alt_scale: settings.use_alt_scale,
squash_system_messages: settings.squash_system_messages,
image_inlining: settings.image_inlining,
@ -3354,6 +3360,7 @@ function onSettingsPresetChange() {
use_ai21_tokenizer: ['#use_ai21_tokenizer', 'use_ai21_tokenizer', true],
use_google_tokenizer: ['#use_google_tokenizer', 'use_google_tokenizer', true],
claude_use_sysprompt: ['#claude_use_sysprompt', 'claude_use_sysprompt', true],
use_makersuite_sysprompt: ['#use_makersuite_sysprompt', 'use_makersuite_sysprompt', true],
use_alt_scale: ['#use_alt_scale', 'use_alt_scale', true],
squash_system_messages: ['#squash_system_messages', 'squash_system_messages', true],
image_inlining: ['#openai_image_inlining', 'image_inlining', true],
@ -4290,6 +4297,11 @@ $(document).ready(async function () {
saveSettingsDebounced();
});
$('#use_makersuite_sysprompt').on('change', function () {
oai_settings.use_makersuite_sysprompt = !!$('#use_makersuite_sysprompt').prop('checked');
saveSettingsDebounced();
});
$('#send_if_empty_textarea').on('input', function () {
oai_settings.send_if_empty = String($('#send_if_empty_textarea').val());
saveSettingsDebounced();

14
src/endpoints/backends/chat-completions.js
View File

@ -252,17 +252,25 @@ async function sendMakerSuiteRequest(request, response) {
};
function getGeminiBody() {
return {
contents: convertGooglePrompt(request.body.messages, model),
const should_use_system_prompt = model === 'gemini-1.5-pro-latest' && request.body.use_makersuite_sysprompt;
const prompt = convertGooglePrompt(request.body.messages, model, should_use_system_prompt, request.body.char_name, request.body.user_name);
let body = {
contents: prompt.contents,
safetySettings: GEMINI_SAFETY,
generationConfig: generationConfig,
};
if (should_use_system_prompt) {
body.system_instruction = prompt.system_instruction;
}
return body;
}
function getBisonBody() {
const prompt = isText
? ({ text: convertTextCompletionPrompt(request.body.messages) })
: ({ messages: convertGooglePrompt(request.body.messages, model) });
: ({ messages: convertGooglePrompt(request.body.messages, model).contents });
/** @type {any} Shut the lint up */
const bisonBody = {

2
src/endpoints/tokenizers.js
View File

@ -398,7 +398,7 @@ router.post('/google/count', jsonParser, async function (req, res) {
accept: 'application/json',
'content-type': 'application/json',
},
body: JSON.stringify({ contents: convertGooglePrompt(req.body, String(req.query.model)) }),
body: JSON.stringify({ contents: convertGooglePrompt(req.body, String(req.query.model)).contents }),
};
try {
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${req.query.model}:countTokens?key=${readSecret(SECRET_KEYS.MAKERSUITE)}`, options);

30
src/prompt-converters.js
View File

@ -252,9 +252,12 @@ function convertCohereMessages(messages, charName = '', userName = '') {
* Convert a prompt from the ChatML objects to the format used by Google MakerSuite models.
* @param {object[]} messages Array of messages
* @param {string} model Model name
* @returns {object[]} Prompt for Google MakerSuite models
* @param {boolean} useSysPrompt Use system prompt
* @param {string} charName Character name
* @param {string} userName User name
* @returns {{contents: *[], system_instruction: {parts: {text: string}}}} Prompt for Google MakerSuite models
*/
function convertGooglePrompt(messages, model) {
function convertGooglePrompt(messages, model, useSysPrompt = false, charName = '', userName = '') {
// This is a 1x1 transparent PNG
const PNG_PIXEL = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';
@ -267,6 +270,27 @@ function convertGooglePrompt(messages, model) {
const isMultimodal = visionSupportedModels.includes(model);
let hasImage = false;
let sys_prompt = '';
if (useSysPrompt) {
while (messages.length > 1 && messages[0].role === 'system') {
// Append example names if not already done by the frontend (e.g. for group chats).
if (userName && messages[0].name === 'example_user') {
if (!messages[0].content.startsWith(`${userName}: `)) {
messages[0].content = `${userName}: ${messages[0].content}`;
}
}
if (charName && messages[0].name === 'example_assistant') {
if (!messages[0].content.startsWith(`${charName}: `)) {
messages[0].content = `${charName}: ${messages[0].content}`;
}
}
sys_prompt += `${messages[0].content}\n\n`;
messages.shift();
}
}
const system_instruction = { parts: { text: sys_prompt.trim() } };
const contents = [];
messages.forEach((message, index) => {
// fix the roles
@ -327,7 +351,7 @@ function convertGooglePrompt(messages, model) {
});
}
return contents;
return { contents: contents, system_instruction: system_instruction };
}
/**