rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

More multigen settings

Browse Source
This commit is contained in:
SillyLossy
2023-04-15 19:15:25 +03:00
parent 7c9e424863
commit 06e941fd93
7 changed files with 162 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
public/index.html
View File

@ -968,46 +968,57 @@
<a href="/notes/advanced_formatting" class="notes-link" target="_blank"> <a href="/notes/advanced_formatting" class="notes-link" target="_blank">
<span class="note-link-span">?</span> <span class="note-link-span">?</span>
</a> </a>
</h3> </h3>
<div class="flex-container"> <div class="flex-container">
<div name="PygOverrides"> <div name="PygOverrides">
<h4>AutoFormat Overrides</h4> <h4>AutoFormat Overrides</h4>
<label class="checkbox_label" for="disable-description-formatting-checkbox"> <label class="checkbox_label" for="disable-description-formatting-checkbox">
<input id="disable-description-formatting-checkbox" type="checkbox" /> <input id="disable-description-formatting-checkbox" type="checkbox" />
Disable description formatting Disable description formatting
</label> </label>
<label class="checkbox_label" for="disable-scenario-formatting-checkbox"> <label class="checkbox_label" for="disable-scenario-formatting-checkbox">
<input id="disable-scenario-formatting-checkbox" type="checkbox" /> <input id="disable-scenario-formatting-checkbox" type="checkbox" />
Disable scenario formatting Disable scenario formatting
</label> </label>
<label class="checkbox_label" for="disable-personality-formatting-checkbox"> <label class="checkbox_label" for="disable-personality-formatting-checkbox">
<input id="disable-personality-formatting-checkbox" type="checkbox" /> <input id="disable-personality-formatting-checkbox" type="checkbox" />
Disable personality formatting Disable personality formatting
</label> </label>
<div>
<h4>
Custom Chat Separator
</h4>
<div> <div>
<input id="custom_chat_separator" class="text_pole" type="text" placeholder="&lt;START&gt;" /> <h4>
Custom Chat Separator
</h4>
<div>
<input id="custom_chat_separator" class="text_pole" type="text" placeholder="&lt;START&gt;" />
</div>
</div>
<div id="anchors-block">
<h4>
Anchors Order
<a href="/notes/9" class="notes-link" target="_blank">
<span class="note-link-span">?</span>
</a>
</h4>
<select id="anchor_order">
<option value="0">Character then Style</option>
<option value="1">Style then Character</option>
</select>
<div id="anchor_checkbox">
<label for="character_anchor"><input id="character_anchor" type="checkbox" />
Character Anchor
</label>
<label for="style_anchor"><input id="style_anchor" type="checkbox" />
Style Anchor
</label>
</div>
</div> </div>
</div> </div>
</div>
<div name="ContextFormatting"> <div name="ContextFormatting">
<h4>Context Formatting</h4> <h4>Context Formatting</h4>
<label class="checkbox_label" for="always-force-name2-checkbox"> <label class="checkbox_label" for="always-force-name2-checkbox">
<input id="always-force-name2-checkbox" type="checkbox" /> <input id="always-force-name2-checkbox" type="checkbox" />
Always add character's name to prompt Always add character's name to prompt
</label> </label>
<label class="checkbox_label" for="multigen">
<input id="multigen" type="checkbox" />
<span>
Multigen
<a href="/notes/multigen" class="notes-link" target="_blank">
<span class="note-link-span">?</span>
</a>
</span>
</label>
<label class="checkbox_label" for="pin-examples-checkbox"> <label class="checkbox_label" for="pin-examples-checkbox">
<input id="pin-examples-checkbox" type="checkbox" /> <input id="pin-examples-checkbox" type="checkbox" />
Keep Example Messages in Prompt Keep Example Messages in Prompt
@ -1023,29 +1034,31 @@
<option value="1">Enabled for all models</option> <option value="1">Enabled for all models</option>
</select> </select>
</div> </div>
</div> <div>
<div id="anchors-block"> <h4>
<h4> Multigen
Anchors Order <a href="/notes/multigen" class="notes-link" target="_blank">
<a href="/notes/9" class="notes-link" target="_blank"> <span class="note-link-span">?</span>
<span class="note-link-span">?</span> </a>
</a> </h4>
</h4> <label class="checkbox_label" for="multigen">
<select id="anchor_order"> <input id="multigen" type="checkbox" />
<option value="0">Character then Style</option> <span>
<option value="1">Style then Character</option> Enabled
</select> </span>
<div id="anchor_checkbox">
<label for="character_anchor"><input id="character_anchor" type="checkbox" />
Character Anchor
</label>
<label for="style_anchor"><input id="style_anchor" type="checkbox" />
Style Anchor
</label> </label>
<div class="multigen_settings_block">
<label for="multigen_1st_chunk">
<small>First chunk (tokens)</small>
<input id="multigen_first_chunk" type="number" class="text_pole" min="1" max="512" />
</label>
<label for="multigen_next_chunk">
<small>Next chunks (tokens)</small>
<input id="multigen_next_chunks" type="number" class="text_pole" min="1" max="512" />
</label>
</div>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
</div> </div>

11
public/notes/multigen.html
View File

@ -16,11 +16,14 @@
<div id="main"> <div id="main">
<div id="content"> <div id="content">
<h2>Multigen</h2> <h2>Multigen</h2>
<p>TavernAI tries to create longer responses by chaining the generation using smaller batches.</p> <p>TavernAI tries to create faster and longer responses by chaining the generation using smaller batches.</p>
<h3>Default settings:</h3>
<p>First batch = 50 tokens</p>
<p>Next batches = 30 tokens</p>
<h3>Algorithm:</h3> <h3>Algorithm:</h3>
<p>1. If amount of generation is more than 50 tokens, then generate first 50 tokens.</p> <p>1. Generate the first batch (if amount of generation setting is more than batch length).</p>
<p>2. Generate by 30 tokens until one of the stopping conditions is reached.</p> <p>2. Generate next batch of tokens until one of the stopping conditions is reached.</p>
<p>3. Append the generated batch to the next cycle's prompt.</p> <p>3. Append the generated text to the next cycle's prompt.</p>
<h3>Stopping conditions:</h3> <h3>Stopping conditions:</h3>
<p>1. Generated enough text.</p> <p>1. Generated enough text.</p>
<p>2. Character starts speaking for You.</p> <p>2. Character starts speaking for You.</p>

48
public/script.js
View File

@ -373,7 +373,6 @@ let padding_tokens = 64; // reserved tokens to prevent prompt overflow
var is_pygmalion = false; var is_pygmalion = false;
var tokens_already_generated = 0; var tokens_already_generated = 0;
var message_already_generated = ""; var message_already_generated = "";
const tokens_cycle_count = 30;
var cycle_count_generation = 0; var cycle_count_generation = 0;
var swipes = false; var swipes = false;
@ -1015,7 +1014,9 @@ function substituteParams(content, _name1, _name2) {
} }
function getStoppingStrings(isImpersonate) { function getStoppingStrings(isImpersonate) {
return isImpersonate ? [`\n${name2}: `] : [`\n${name1}:`]; const charString = [`\n${name2}: `];
const userString = is_pygmalion ? [`\nYou: `] : [`\n${name1}: `];
return isImpersonate ? charString : userString;
} }
function getSlashCommand(message, type) { function getSlashCommand(message, type) {
@ -1254,7 +1255,6 @@ class StreamingProcessor {
constructor(type, force_name2) { constructor(type, force_name2) {
this.result = ""; this.result = "";
this.prefix = "";
this.messageId = -1; this.messageId = -1;
this.type = type; this.type = type;
this.force_name2 = force_name2; this.force_name2 = force_name2;
@ -1269,11 +1269,6 @@ class StreamingProcessor {
await delay(1); // delay for message to be rendered await delay(1); // delay for message to be rendered
} }
// for multigen
if (this.result.length) {
this.prefix = this.result;
}
for await (const text of this.generator()) { for await (const text of this.generator()) {
if (this.isStopped) { if (this.isStopped) {
this.onStopStreaming(); this.onStopStreaming();
@ -1374,7 +1369,7 @@ async function Generate(type, automatic_trigger, force_name2) {
var storyString = ""; var storyString = "";
var userSendString = ""; var userSendString = "";
var finalPromt = ""; var finalPromt = "";
var postAnchorChar = "Elaborate speaker";//'Talk a lot with description what is going on around';// in asterisks var postAnchorChar = "Elaborate speaker";
var postAnchorStyle = "Writing style: very long messages";//"[Genre: roleplay chat][Tone: very long messages with descriptions]"; var postAnchorStyle = "Writing style: very long messages";//"[Genre: roleplay chat][Tone: very long messages with descriptions]";
var anchorTop = ''; var anchorTop = '';
var anchorBottom = ''; var anchorBottom = '';
@ -1812,25 +1807,32 @@ async function Generate(type, automatic_trigger, force_name2) {
finalPromt = collapseNewlines(finalPromt); finalPromt = collapseNewlines(finalPromt);
} }
//console.log('final prompt decided');
let this_amount_gen = parseInt(amount_gen); // how many tokens the AI will be requested to generate let this_amount_gen = parseInt(amount_gen); // how many tokens the AI will be requested to generate
let this_settings = koboldai_settings[koboldai_setting_names[preset_settings]]; let this_settings = koboldai_settings[koboldai_setting_names[preset_settings]];
if (isMultigenEnabled()) { if (isMultigenEnabled()) {
if (tokens_already_generated === 0) { // if nothing has been generated yet.. // if nothing has been generated yet..
if (parseInt(amount_gen) >= 50) { // if the max gen setting is > 50...( if (tokens_already_generated === 0) {
this_amount_gen = 50; // then only try to make 50 this cycle.. // if the max gen setting is > 50...(
if (parseInt(amount_gen) >= power_user.multigen_first_chunk) {
// then only try to make 50 this cycle..
this_amount_gen = power_user.multigen_first_chunk;
} }
else { else {
this_amount_gen = parseInt(amount_gen); // otherwise, make as much as the max amount request. // otherwise, make as much as the max amount request.
this_amount_gen = parseInt(amount_gen);
} }
} }
else { // if we already recieved some generated text... // if we already received some generated text...
if (parseInt(amount_gen) - tokens_already_generated < tokens_cycle_count) { // if the remaining tokens to be made is less than next potential cycle count else {
this_amount_gen = parseInt(amount_gen) - tokens_already_generated; // subtract already generated amount from the desired max gen amount // if the remaining tokens to be made is less than next potential cycle count
if (parseInt(amount_gen) - tokens_already_generated < power_user.multigen_next_chunks) {
// subtract already generated amount from the desired max gen amount
this_amount_gen = parseInt(amount_gen) - tokens_already_generated;
} }
else { else {
this_amount_gen = tokens_cycle_count; // otherwise make the standard cycle amont (frist 50, and 30 after that) // otherwise make the standard cycle amount (first 50, and 30 after that)
this_amount_gen = power_user.multigen_next_chunks;
} }
} }
} }
@ -1962,6 +1964,10 @@ async function Generate(type, automatic_trigger, force_name2) {
hideSwipeButtons(); hideSwipeButtons();
let getMessage = await streamingProcessor.generate(); let getMessage = await streamingProcessor.generate();
if (generatedPromtCache.length === 0) {
generatedPromtCache = message_already_generated;
}
if (isMultigenEnabled()) { if (isMultigenEnabled()) {
tokens_already_generated += this_amount_gen; // add new gen amt to any prev gen counter.. tokens_already_generated += this_amount_gen; // add new gen amt to any prev gen counter..
message_already_generated += getMessage; message_already_generated += getMessage;
@ -2074,7 +2080,8 @@ async function Generate(type, automatic_trigger, force_name2) {
} //generate ends } //generate ends
function shouldContinueMultigen(getMessage) { function shouldContinueMultigen(getMessage) {
return message_already_generated.indexOf('You:') === -1 && //if there is no 'You:' in the response msg const nameString = is_pygmalion ? 'You:' : `${name1}:`;
return message_already_generated.indexOf(nameString) === -1 && //if there is no 'You:' in the response msg
message_already_generated.indexOf('<|endoftext|>') === -1 && //if there is no <endoftext> stamp in the response msg message_already_generated.indexOf('<|endoftext|>') === -1 && //if there is no <endoftext> stamp in the response msg
tokens_already_generated < parseInt(amount_gen) && //if the gen'd msg is less than the max response length.. tokens_already_generated < parseInt(amount_gen) && //if the gen'd msg is less than the max response length..
getMessage.length > 0; //if we actually have gen'd text at all... getMessage.length > 0; //if we actually have gen'd text at all...
@ -3304,7 +3311,8 @@ function showSwipeButtons() {
if (swipeId !== undefined && swipeId != 0) { if (swipeId !== undefined && swipeId != 0) {
currentMessage.children('.swipe_left').css('display', 'flex'); currentMessage.children('.swipe_left').css('display', 'flex');
} }
if (is_send_press === false || chat[chat.length - 1].swipes.length >= swipeId) { //only show right when generate is off, or when next right swipe would not make a generate happen //only show right when generate is off, or when next right swipe would not make a generate happen
if (is_send_press === false || chat[chat.length - 1].swipes.length >= swipeId) {
currentMessage.children('.swipe_right').css('display', 'flex'); currentMessage.children('.swipe_right').css('display', 'flex');
currentMessage.children('.swipe_right').css('opacity', '0.3'); currentMessage.children('.swipe_right').css('opacity', '0.3');
} }

15
public/scripts/power-user.js
View File

@ -47,6 +47,8 @@ let power_user = {
disable_personality_formatting: false, disable_personality_formatting: false,
always_force_name2: false, always_force_name2: false,
multigen: false, multigen: false,
multigen_first_chunk: 50,
multigen_next_chunks: 30,
custom_chat_separator: '', custom_chat_separator: '',
fast_ui_mode: true, fast_ui_mode: true,
avatar_style: avatar_styles.ROUND, avatar_style: avatar_styles.ROUND,
@ -288,6 +290,8 @@ function loadPowerUserSettings(settings, data) {
$("#fast_ui_mode").prop("checked", power_user.fast_ui_mode); $("#fast_ui_mode").prop("checked", power_user.fast_ui_mode);
$("#waifuMode").prop("checked", power_user.waifuMode); $("#waifuMode").prop("checked", power_user.waifuMode);
$("#multigen").prop("checked", power_user.multigen); $("#multigen").prop("checked", power_user.multigen);
$("#multigen_first_chunk").val(power_user.multigen_first_chunk);
$("#multigen_next_chunks").val(power_user.multigen_next_chunks);
$("#play_message_sound").prop("checked", power_user.play_message_sound); $("#play_message_sound").prop("checked", power_user.play_message_sound);
$("#play_sound_unfocused").prop("checked", power_user.play_sound_unfocused); $("#play_sound_unfocused").prop("checked", power_user.play_sound_unfocused);
$(`input[name="avatar_style"][value="${power_user.avatar_style}"]`).prop("checked", true); $(`input[name="avatar_style"][value="${power_user.avatar_style}"]`).prop("checked", true);
@ -519,6 +523,17 @@ $(document).ready(() => {
saveSettingsDebounced(); saveSettingsDebounced();
}); });
$("#multigen_first_chunk").on('input', function () {
power_user.multigen_first_chunk = Number($(this).val());
saveSettingsDebounced();
});
$("#multigen_next_chunks").on('input', function () {
power_user.multigen_next_chunks = Number($(this).val());
saveSettingsDebounced();
});
$(window).on('focus', function () { $(window).on('focus', function () {
browser_has_focus = true; browser_has_focus = true;
}); });

10
public/scripts/textgen-settings.js
View File

@ -164,8 +164,16 @@ async function generateTextGenWithStreaming(generate_data) {
while (true) { while (true) {
const { done, value } = await reader.read(); const { done, value } = await reader.read();
let response = decoder.decode(value); let response = decoder.decode(value);
let delta = '';
getMessage += response; try {
delta = JSON.parse(response).delta;
}
catch {
delta = '';
}
getMessage += delta;
if (done) { if (done) {
return; return;

15
public/style.css
View File

@ -1740,7 +1740,7 @@ input[type='checkbox']:not(#nav-toggle):not(#rm_button_panel_pin):not(#lm_button
.range-block-counter { .range-block-counter {
width: max-content; width: max-content;
margin-left: 5px; margin-left: 5px;
font-size: calc(var(--mainFontSize) - 0.3rem); font-size: calc(var(--mainFontSize) - 0.2rem);
color: var(--white50a); color: var(--white50a);
} }
@ -1924,13 +1924,24 @@ input[type="range"]::-webkit-slider-thumb {
#anchor_checkbox label, #anchor_checkbox label,
#power-user-option-checkboxes label, #power-user-option-checkboxes label,
.checkbox_label { .checkbox_label,
.multigen_settings_block {
display: flex; display: flex;
flex-direction: row; flex-direction: row;
column-gap: 5px; column-gap: 5px;
align-items: center; align-items: center;
} }
.multigen_settings_block {
margin-top: 10px;
}
.multigen_settings_block label {
flex: 1;
display: flex;
flex-direction: column;
}
#shadow_character_popup { #shadow_character_popup {
backdrop-filter: blur(var(--SmartThemeBlurStrength)); backdrop-filter: blur(var(--SmartThemeBlurStrength));
background-color: var(--black70a); background-color: var(--black70a);

32
server.js
View File

@ -332,12 +332,12 @@ function textGenProcessStartedHandler(websocket, content, session, prompt, fn_in
case "process_starts": case "process_starts":
break; break;
case "process_generating": case "process_generating":
return content.output.data[0]; return { text: content.output.data[0], completed: false };
case "process_completed": case "process_completed":
return null; return { text: content.output.data[0], completed: true };
} }
return ''; return { text: '', completed: false };
} }
//************** Text generation web UI //************** Text generation web UI
@ -360,6 +360,7 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
const url = new URL(api_server); const url = new URL(api_server);
const websocket = new WebSocket(`ws://${url.host}/queue/join`, { perMessageDeflate: false }); const websocket = new WebSocket(`ws://${url.host}/queue/join`, { perMessageDeflate: false });
let text = ''; let text = '';
let completed = false;
websocket.on('open', async function () { websocket.on('open', async function () {
console.log('websocket open'); console.log('websocket open');
@ -378,7 +379,9 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
websocket.on('message', async (message) => { websocket.on('message', async (message) => {
const content = json5.parse(message); const content = json5.parse(message);
console.log(content); console.log(content);
text = textGenProcessStartedHandler(websocket, content, session, request.body, fn_index); let result = textGenProcessStartedHandler(websocket, content, session, request.body, fn_index);
text = result.text;
completed = result.completed;
}); });
while (true) { while (true) {
@ -386,17 +389,23 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
await delay(50); await delay(50);
yield text; yield text;
if (!text && typeof text !== 'string') { if (completed || (!text && typeof text !== 'string')) {
websocket.close(); websocket.close();
yield null;
break;
} }
} }
else { else {
break; break;
} }
} }
return null;
} }
let result = json5.parse(request.body.data)[0]; let result = json5.parse(request.body.data)[0];
let prompt = result;
let stopping_strings = json5.parse(request.body.data)[1].custom_stopping_strings;
try { try {
for await (const text of readWebsocket()) { for await (const text of readWebsocket()) {
@ -411,7 +420,18 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
} }
result = text; result = text;
response_generate.write(newText);
const generatedText = result.substring(prompt.length);
response_generate.write(JSON.stringify({ delta: newText }));
if (generatedText) {
for (const str of stopping_strings) {
if (generatedText.indexOf(str) !== -1) {
break;
}
}
}
} }
} }
finally { finally {