rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Implement support for koboldcpp SSE streaming.

Browse Source 
koboldcpp added SSE streaming to its API as of:
https://github.com/LostRuins/koboldcpp/pull/220

This has not yet been added to release, but I figued I'd try to
get the work done to make it ready.

Work mostly hijacked from:
10bbc97069
This commit is contained in:
Zixaphir
2023-06-11 13:24:07 -07:00
parent 4bce33b1a9
commit 698461c188
4 changed files with 107 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
public/script.js
View File

@@ -2,6 +2,7 @@ import { humanizedDateTime, favsToHotswap } from "./scripts/RossAscends-mods.js"
import { encode } from "../scripts/gpt-2-3-tokenizer/mod.js";
import { GPT3BrowserTokenizer } from "../scripts/gpt-3-tokenizer/gpt3-tokenizer.js";
import {
generateKoboldWithStreaming,
kai_settings,
loadKoboldSettings,
formatKoboldUrl,
@@ -1586,6 +1587,7 @@ function appendToStoryString(value, prefix) {
function isStreamingEnabled() {
return ((main_api == 'openai' && oai_settings.stream_openai)
|| (main_api == 'kobold' && kai_settings.streaming_kobold)
|| (main_api == 'novel' && nai_settings.streaming_novel)
|| (main_api == 'poe' && poe_settings.streaming)
|| (main_api == 'textgenerationwebui' && textgenerationwebui_settings.streaming))
@@ -2367,6 +2369,9 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
else if (main_api == 'novel' && isStreamingEnabled() && type !== 'quiet') {
streamingProcessor.generator = await generateNovelWithStreaming(generate_data, streamingProcessor.abortController.signal);
}
else if (main_api == 'kobold' && isStreamingEnabled() && type !== 'quiet') {
streamingProcessor.generator = await generateKoboldWithStreaming(generate_data, streamingProcessor.abortController.signal);
}
else {
try {
const response = await fetch(generate_url, {
@@ -6690,7 +6695,7 @@ $(document).ready(function () {
if (this_chid !== undefined || selected_group) {
// Previously system messages we're allowed to be edited
/*const message = $(this).closest(".mes");
if (message.data("isSystem")) {
return;
}*/