rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#2557 Improve Mistral prompt conversion

This commit is contained in:
Cohee 2024-07-25 21:04:57 +03:00
parent d31e44dec8
commit ef59e8abae
2 changed files with 16 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/endpoints/backends/chat-completions.js
View File

@ -483,7 +483,7 @@ async function sendMistralAIRequest(request, response) {
} }
try { try {
const messages = convertMistralMessages(request.body.messages, request.body.model, request.body.char_name, request.body.user_name); const messages = convertMistralMessages(request.body.messages, request.body.char_name, request.body.user_name);
const controller = new AbortController(); const controller = new AbortController();
request.socket.removeAllListeners('close'); request.socket.removeAllListeners('close');
request.socket.on('close', function () { request.socket.on('close', function () {

46
src/prompt-converters.js
View File

@ -364,65 +364,49 @@ function convertGooglePrompt(messages, model, useSysPrompt = false, charName = '
/** /**
* Convert a prompt from the ChatML objects to the format used by MistralAI. * Convert a prompt from the ChatML objects to the format used by MistralAI.
* @param {object[]} messages Array of messages * @param {object[]} messages Array of messages
* @param {string} model Model name
* @param {string} charName Character name * @param {string} charName Character name
* @param {string} userName User name * @param {string} userName User name
*/ */
function convertMistralMessages(messages, model, charName = '', userName = '') { function convertMistralMessages(messages, charName = '', userName = '') {
if (!Array.isArray(messages)) { if (!Array.isArray(messages)) {
return []; return [];
} }
//large seems to be throwing a 500 error if we don't make the first message a user role, most likely a bug since the other models won't do this // Make the last assistant message a prefill
if (model.includes('large')) {
messages[0].role = 'user';
}
//must send a user role as last message
const lastMsg = messages[messages.length - 1]; const lastMsg = messages[messages.length - 1];
if (messages.length > 0 && lastMsg && (lastMsg.role === 'system' || lastMsg.role === 'assistant')) { if (messages.length > 0 && lastMsg && (lastMsg.role === 'assistant')) {
if (lastMsg.role === 'assistant' && lastMsg.name) { lastMsg.prefix = true;
lastMsg.content = lastMsg.name + ': ' + lastMsg.content;
} else if (lastMsg.role === 'system') {
lastMsg.content = '[INST] ' + lastMsg.content + ' [/INST]';
}
lastMsg.role = 'user';
} }
//system prompts can be stacked at the start, but any futher sys prompts after the first user/assistant message will break the model // Doesn't support completion names, so prepend if not already done by the frontend (e.g. for group chats).
let encounteredNonSystemMessage = false;
messages.forEach(msg => { messages.forEach(msg => {
if (msg.role === 'system' && msg.name === 'example_assistant') { if (msg.role === 'system' && msg.name === 'example_assistant') {
if (charName) { if (charName && !msg.content.startsWith(`${charName}: `)) {
msg.content = `${charName}: ${msg.content}`; msg.content = `${charName}: ${msg.content}`;
} }
delete msg.name; delete msg.name;
} }
if (msg.role === 'system' && msg.name === 'example_user') { if (msg.role === 'system' && msg.name === 'example_user') {
if (userName) { if (userName && !msg.content.startsWith(`${userName}: `)) {
msg.content = `${userName}: ${msg.content}`; msg.content = `${userName}: ${msg.content}`;
} }
delete msg.name; delete msg.name;
} }
if (msg.name) { if (msg.name && msg.role !== 'system' && !msg.content.startsWith(`${msg.name}: `)) {
msg.content = `${msg.name}: ${msg.content}`; msg.content = `${msg.name}: ${msg.content}`;
delete msg.name; delete msg.name;
} }
if ((msg.role === 'user' || msg.role === 'assistant') && !encounteredNonSystemMessage) {
encounteredNonSystemMessage = true;
}
if (encounteredNonSystemMessage && msg.role === 'system') {
msg.role = 'user';
//unsure if the instruct version is what they've deployed on their endpoints and if this will make a difference or not.
//it should be better than just sending the message as a user role without context though
msg.content = '[INST] ' + msg.content + ' [/INST]';
}
}); });
// If system role message immediately follows an assistant message, change its role to user
for (let i = 0; i < messages.length - 1; i++) {
if (messages[i].role === 'assistant' && messages[i + 1].role === 'system') {
messages[i + 1].role = 'user';
}
}
return messages; return messages;
} }