rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add poe streaming

Browse Source
This commit is contained in:
SillyLossy
2023-04-12 02:26:09 +03:00
parent 2040b8d3ff
commit e1b514ef5a
4 changed files with 79 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
public/index.html
View File

@ -347,6 +347,15 @@
</div>
</div>
<div id="range_block_poe">
<div class="range-block">
<label class="checkbox_label" for="poe_streaming">
<input id="poe_streaming" type="checkbox" />
Streaming
</label>
<div class="range-block-counter justifyLeft">
Displays the message text as soon as it is generated
</div>
</div>
<div class="range-block">
<label for="poe_auto_purge" class="checkbox_label">
<input id="poe_auto_purge" type="checkbox">

16
public/script.js
View File

@ -1114,7 +1114,7 @@ function appendToStoryString(value, prefix) {
}
function isStreamingEnabled() {
return (main_api == 'openai' && oai_settings.stream_openai);
return (main_api == 'openai' && oai_settings.stream_openai) || (main_api == 'poe' && poe_settings.streaming);
}
class StreamingProcessor {
@ -1811,8 +1811,6 @@ async function Generate(type, automatic_trigger, force_name2) {
if (isStreamingEnabled()) {
streamingProcessor.generator = await sendOpenAIRequest(prompt);
await streamingProcessor.generate();
streamingProcessor = null;
}
else {
sendOpenAIRequest(prompt).then(onSuccess).catch(onError);
@ -1822,7 +1820,12 @@ async function Generate(type, automatic_trigger, force_name2) {
generateHorde(finalPromt, generate_data).then(onSuccess).catch(onError);
}
else if (main_api == 'poe') {
generatePoe(finalPromt).then(onSuccess).catch(onError);
if (isStreamingEnabled()) {
streamingProcessor.generator = await generatePoe(finalPromt);
}
else {
generatePoe(finalPromt).then(onSuccess).catch(onError);
}
}
else {
jQuery.ajax({
@ -1840,6 +1843,11 @@ async function Generate(type, automatic_trigger, force_name2) {
}); //end of "if not data error"
}
if (isStreamingEnabled()) {
await streamingProcessor.generate();
streamingProcessor = null;
}
function onSuccess(data) {
tokens_already_generated += this_amount_gen; // add new gen amt to any prev gen counter..

34
public/scripts/poe.js
View File

@ -43,6 +43,7 @@ const poe_settings = {
auto_jailbreak: true,
character_nudge: true,
auto_purge: true,
streaming: false,
};
let auto_jailbroken = false;
@ -61,6 +62,7 @@ function loadPoeSettings(settings) {
$('#poe_character_nudge').prop('checked', poe_settings.character_nudge);
$('#poe_auto_jailbreak').prop('checked', poe_settings.auto_jailbreak);
$('#poe_auto_purge').prop('checked', poe_settings.auto_purge);
$('#poe_streaming').prop('checked', poe_settings.streaming);
$('#poe_token').val(poe_settings.token ?? '');
selectBot();
}
@ -94,7 +96,7 @@ async function generatePoe(finalPrompt) {
if (poe_settings.auto_jailbreak && !auto_jailbroken) {
for (let retryNumber = 0; retryNumber < MAX_RETRIES_FOR_ACTIVATION; retryNumber++) {
const reply = await sendMessage(poe_settings.jailbreak_message);
const reply = await sendMessage(poe_settings.jailbreak_message, false);
if (reply.toLowerCase().includes(poe_settings.jailbreak_response.toLowerCase())) {
auto_jailbroken = true;
@ -115,7 +117,7 @@ async function generatePoe(finalPrompt) {
finalPrompt += nudge;
}
const reply = await sendMessage(finalPrompt);
const reply = await sendMessage(finalPrompt, true);
got_reply = true;
return reply;
}
@ -139,10 +141,11 @@ async function purgeConversation(count = -1) {
return response.ok;
}
async function sendMessage(prompt) {
async function sendMessage(prompt, withStreaming) {
const body = JSON.stringify({
bot: poe_settings.bot,
token: poe_settings.token,
streaming: withStreaming && poe_settings.streaming,
prompt,
});
@ -155,6 +158,25 @@ async function sendMessage(prompt) {
method: 'POST',
});
if (withStreaming && poe_settings.streaming) {
return async function* streamData() {
const decoder = new TextDecoder();
const reader = response.body.getReader();
let getMessage = '';
while (true) {
const { done, value } = await reader.read();
let response = decoder.decode(value);
getMessage += response;
if (done) {
return;
}
yield getMessage;
}
}
}
try {
if (response.ok) {
const data = await response.json();
@ -261,6 +283,11 @@ function onCharacterNudgeMessageInput() {
saveSettingsDebounced();
}
function onStreamingInput() {
poe_settings.streaming = !!$(this).prop('checked');
saveSettingsDebounced();
}
$('document').ready(function () {
$('#poe_token').on('input', onTokenInput);
$('#poe_bots').on('change', onBotChange);
@ -271,4 +298,5 @@ $('document').ready(function () {
$('#poe_auto_jailbreak').on('input', onAutoJailbreakInput);
$('#poe_character_nudge').on('input', onCharacterNudgeInput);
$('#poe_nudge_text').on('input', onCharacterNudgeMessageInput);
$('#poe_streaming').on('input', onStreamingInput);
});

37
server.js
View File

@ -1776,22 +1776,39 @@ app.post('/generate_poe', jsonParser, async (request, response) => {
const token = request.body.token;
const prompt = request.body.prompt;
const bot = request.body.bot ?? POE_DEFAULT_BOT;
const streaming = request.body.streaming ?? false;
try {
const client = await getPoeClient(token);
let reply;
for await (const mes of client.send_message(bot, prompt)) {
reply = mes.text;
if (streaming) {
response.writeHead(200, {
'Transfer-Encoding': 'chunked',
'Cache-Control': 'no-transform',
});
let reply = '';
for await (const mes of client.send_message(bot, prompt)) {
let newText = mes.text.substring(reply.length);
reply = mes.text;
response.write(newText);
}
console.log(reply);
client.disconnect_ws();
response.end();
}
else {
let reply;
for await (const mes of client.send_message(bot, prompt)) {
reply = mes.text;
}
console.log(reply);
client.disconnect_ws();
return response.send({ 'reply': reply });
}
console.log(reply);
client.disconnect_ws();
return response.send({ 'reply': reply });
}
catch {
catch (error) {
console.error(error);
return response.sendStatus(500);
}
});