mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-02-26 17:07:46 +01:00
Split oobabooga/mancer sources. Add aphrodite support
This commit is contained in:
Cohee
parent
306cf51da4
commit
bb47712696
@ -1875,60 +1875,80 @@
|
||||
</div>
|
||||
<div id="textgenerationwebui_api" style="display: none;position: relative;">
|
||||
<form action="javascript:void(null);" method="post" enctype="multipart/form-data">
|
||||
<span data-i18n="If you are using:"> If you are using:</span>
|
||||
<div class="flex-container indent20p">
|
||||
<a href="https://github.com/oobabooga/text-generation-webui" target="_blank">
|
||||
oobabooga/text-generation-webui
|
||||
</a>
|
||||
<span data-i18n="Make sure you run it with">
|
||||
Make sure you run it with <tt>--api</tt> flag
|
||||
</span>
|
||||
</div>
|
||||
<div class="flex-container indent20p">
|
||||
<a href="https://mancer.tech/" target="_blank">
|
||||
Mancer AI
|
||||
</a>
|
||||
<label class="checkbox_label" for="use-mancer-api-checkbox">
|
||||
<span data-i18n="Use API key (Only required for Mancer)">
|
||||
Click this box (and add your API key!):
|
||||
</span>
|
||||
<input id="use-mancer-api-checkbox" type="checkbox" />
|
||||
</label>
|
||||
</div>
|
||||
<div>
|
||||
<div id="mancer_api_subpanel" class="flex-container flexFlowColumn" style="display:none;">
|
||||
<h4 data-i18n="Mancer API key">Mancer API key</h4>
|
||||
<div class="flex-container">
|
||||
<input id="api_key_mancer" name="api_key_mancer" class="text_pole flex1 wide100p" maxlength="500" size="35" type="text" autocomplete="off">
|
||||
<div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_mancer">
|
||||
</div>
|
||||
</div>
|
||||
<div data-for="api_key_mancer" class="neutral_warning" data-i18n="For privacy reasons, your API key will be hidden after you reload the page.">
|
||||
For privacy reasons, your API key will be hidden after you reload the page.
|
||||
</div>
|
||||
<div class="flex1">
|
||||
<h4>Mancer Model</h4>
|
||||
<select id="mancer_model"></select>
|
||||
<h4 data-i18n="Mancer API url">Mancer API URL</h4>
|
||||
<small data-i18n="Example: https://neuro.mancer.tech/webui/MODEL/api">Example: https://neuro.mancer.tech/webui/MODEL/api</small>
|
||||
<input id="mancer_api_url_text" name="mancer_api_url" class="text_pole wide100p" maxlength="500" value="" autocomplete="off">
|
||||
</div>
|
||||
</div>
|
||||
<div id="tgwebui_api_subpanel" class="flex-container flexFlowColumn">
|
||||
<div class="flex1">
|
||||
<h4 data-i18n="Blocking API url">Blocking API URL</h4>
|
||||
<small data-i18n="Example: http://127.0.0.1:5000/api ">Example: http://127.0.0.1:5000/api </small>
|
||||
<input id="textgenerationwebui_api_url_text" name="textgenerationwebui_api_url" class="text_pole wide100p" maxlength="500" value="" autocomplete="off" data-server-history="ooba_blocking">
|
||||
</div>
|
||||
<div class="flex1">
|
||||
<h4 data-i18n="Streaming API url">Streaming API URL</h4>
|
||||
<small data-i18n="Example: ws://127.0.0.1:5005/api/v1/stream">Example: ws://127.0.0.1:5005/api/v1/stream </small>
|
||||
<input id="streaming_url_textgenerationwebui" type="text" class="text_pole wide100p" maxlength="500" value="" autocomplete="off" data-server-history="ooba_streaming">
|
||||
</div>
|
||||
</div>
|
||||
<div id="api_button_textgenerationwebui" class="menu_button" type="submit" data-i18n="Connect" data-server-connect="ooba_blocking,ooba_streaming">Connect</div>
|
||||
<div id="api_loading_textgenerationwebui" class="api-load-icon fa-solid fa-hourglass fa-spin"></div>
|
||||
<h4>API Type</h4>
|
||||
<select id="textgen_type">
|
||||
<option value="ooba">Default (oobabooga)</option>
|
||||
<option value="mancer">Mancer</option>
|
||||
<option value="aphrodite">Aphrodite</option>
|
||||
</select>
|
||||
</div>
|
||||
<div data-tg-type="mancer" class="flex-container flexFlowColumn">
|
||||
<div class="flex-container flexFlowColumn">
|
||||
<a href="https://mancer.tech/" target="_blank">
|
||||
Mancer AI
|
||||
</a>
|
||||
</div>
|
||||
<h4 data-i18n="Mancer API key">Mancer API key</h4>
|
||||
<div class="flex-container">
|
||||
<input id="api_key_mancer" name="api_key_mancer" class="text_pole flex1 wide100p" maxlength="500" size="35" type="text" autocomplete="off">
|
||||
<div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_mancer">
|
||||
</div>
|
||||
</div>
|
||||
<div data-for="api_key_mancer" class="neutral_warning" data-i18n="For privacy reasons, your API key will be hidden after you reload the page.">
|
||||
For privacy reasons, your API key will be hidden after you reload the page.
|
||||
</div>
|
||||
<div class="flex1">
|
||||
<h4>Mancer Model</h4>
|
||||
<select id="mancer_model"></select>
|
||||
<h4 data-i18n="Mancer API url">Mancer API URL</h4>
|
||||
<small data-i18n="Example: https://neuro.mancer.tech/webui/MODEL/api">Example: https://neuro.mancer.tech/webui/MODEL/api</small>
|
||||
<input id="mancer_api_url_text" name="mancer_api_url" class="text_pole wide100p" maxlength="500" value="" autocomplete="off">
|
||||
</div>
|
||||
</div>
|
||||
<div data-tg-type="ooba" class="flex-container flexFlowColumn">
|
||||
<div class="flex-container flexFlowColumn">
|
||||
<a href="https://github.com/oobabooga/text-generation-webui" target="_blank">
|
||||
oobabooga/text-generation-webui
|
||||
</a>
|
||||
<span data-i18n="Make sure you run it with">
|
||||
Make sure you run it with <tt>--api</tt> flag
|
||||
</span>
|
||||
</div>
|
||||
<div class="flex1">
|
||||
<h4 data-i18n="Blocking API url">Blocking API URL</h4>
|
||||
<small data-i18n="Example: http://127.0.0.1:5000/api ">Example: http://127.0.0.1:5000/api </small>
|
||||
<input id="textgenerationwebui_api_url_text" name="textgenerationwebui_api_url" class="text_pole wide100p" maxlength="500" value="" autocomplete="off" data-server-history="ooba_blocking">
|
||||
</div>
|
||||
<div class="flex1">
|
||||
<h4 data-i18n="Streaming API url">Streaming API URL</h4>
|
||||
<small data-i18n="Example: ws://127.0.0.1:5005/api/v1/stream">Example: ws://127.0.0.1:5005/api/v1/stream </small>
|
||||
<input id="streaming_url_textgenerationwebui" type="text" class="text_pole wide100p" maxlength="500" value="" autocomplete="off" data-server-history="ooba_streaming">
|
||||
</div>
|
||||
</div>
|
||||
<div data-tg-type="aphrodite">
|
||||
<div class="flex-container flexFlowColumn">
|
||||
<a href="https://github.com/oobabooga/text-generation-webui" target="_blank">
|
||||
PygmalionAI/aphrodite-engine
|
||||
</a>
|
||||
</div>
|
||||
<h4 data-i18n="Aphrodite API key">Aphrodite API key</h4>
|
||||
<div class="flex-container">
|
||||
<input id="api_key_aphrodite" name="api_key_aphrodite" class="text_pole flex1 wide100p" maxlength="500" size="35" type="text" autocomplete="off">
|
||||
<div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_aphrodite">
|
||||
</div>
|
||||
</div>
|
||||
<div data-for="api_key_aphrodite" class="neutral_warning" data-i18n="For privacy reasons, your API key will be hidden after you reload the page.">
|
||||
For privacy reasons, your API key will be hidden after you reload the page.
|
||||
</div>
|
||||
<div class="flex1">
|
||||
<h4 data-i18n="API url">API URL</h4>
|
||||
<small data-i18n="Example: http://127.0.0.1:5000/api ">Example: http://127.0.0.1:5000/api </small>
|
||||
<input id="aphrodite_api_url_text" class="text_pole wide100p" maxlength="500" value="" autocomplete="off" data-server-history="aphrodite">
|
||||
</div>
|
||||
</div>
|
||||
<div id="api_button_textgenerationwebui" class="menu_button" type="submit" data-i18n="Connect" data-server-connect="ooba_blocking,ooba_streaming,aphrodite">Connect</div>
|
||||
<div id="api_loading_textgenerationwebui" class="api-load-icon fa-solid fa-hourglass fa-spin"></div>
|
||||
</form>
|
||||
<div class="online_status4">
|
||||
<div class="online_status_indicator4"></div>
|
||||
|
||||
@ -16,6 +16,9 @@ import {
|
||||
generateTextGenWithStreaming,
|
||||
getTextGenGenerationData,
|
||||
formatTextGenURL,
|
||||
getTextGenUrlSourceId,
|
||||
isMancer,
|
||||
isAphrodite,
|
||||
} from "./scripts/textgen-settings.js";
|
||||
|
||||
import {
|
||||
@ -178,7 +181,6 @@ import {
|
||||
import { applyLocale } from "./scripts/i18n.js";
|
||||
import { getTokenCount, getTokenizerModel, saveTokenCache } from "./scripts/tokenizers.js";
|
||||
import { initPersonas, selectCurrentPersona, setPersonaDescription } from "./scripts/personas.js";
|
||||
import { loadMancerModels } from "./scripts/mancer-settings.js";
|
||||
|
||||
//exporting functions and vars for mods
|
||||
export {
|
||||
@ -632,7 +634,6 @@ let is_get_status = false;
|
||||
let is_get_status_novel = false;
|
||||
let is_api_button_press = false;
|
||||
let is_api_button_press_novel = false;
|
||||
let api_use_mancer_webui = false;
|
||||
|
||||
let is_send_press = false; //Send generation
|
||||
|
||||
@ -771,7 +772,8 @@ async function getStatus() {
|
||||
data: JSON.stringify({
|
||||
api_server: main_api == "kobold" ? api_server : api_server_textgenerationwebui,
|
||||
main_api: main_api,
|
||||
use_mancer: main_api == "textgenerationwebui" ? api_use_mancer_webui : false,
|
||||
use_mancer: main_api == "textgenerationwebui" ? isMancer() : false,
|
||||
use_aphrodite: main_api == "textgenerationwebui" ? isAphrodite() : false,
|
||||
}),
|
||||
beforeSend: function () { },
|
||||
cache: false,
|
||||
@ -3010,7 +3012,8 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
|
||||
}
|
||||
else if (main_api == 'textgenerationwebui') {
|
||||
generate_data = getTextGenGenerationData(finalPrompt, this_amount_gen, isImpersonate, cfgValues);
|
||||
generate_data.use_mancer = api_use_mancer_webui;
|
||||
generate_data.use_mancer = isMancer();
|
||||
generate_data.use_aphrodite = isAphrodite();
|
||||
}
|
||||
else if (main_api == 'novel') {
|
||||
const this_settings = novelai_settings[novelai_setting_names[nai_settings.preset_settings_novel]];
|
||||
@ -3246,7 +3249,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
|
||||
//console.log('runGenerate calling showSwipeBtns');
|
||||
showSwipeButtons();
|
||||
|
||||
if (main_api == 'textgenerationwebui' && api_use_mancer_webui) {
|
||||
if (main_api == 'textgenerationwebui' && isMancer()) {
|
||||
const errorText = `<h3>Inferencer endpoint is unhappy!</h3>
|
||||
Returned status <tt>${data.status}</tt> with the reason:<br/>
|
||||
${data.response}`;
|
||||
@ -4979,9 +4982,7 @@ async function getSettings(type) {
|
||||
api_server_textgenerationwebui = settings.api_server_textgenerationwebui;
|
||||
$("#textgenerationwebui_api_url_text").val(api_server_textgenerationwebui);
|
||||
$("#mancer_api_url_text").val(api_server_textgenerationwebui);
|
||||
api_use_mancer_webui = settings.api_use_mancer_webui
|
||||
$('#use-mancer-api-checkbox').prop("checked", api_use_mancer_webui);
|
||||
$('#use-mancer-api-checkbox').trigger("change");
|
||||
$("#aphrodite_api_url_text").val(api_server_textgenerationwebui);
|
||||
|
||||
selected_button = settings.selected_button;
|
||||
|
||||
@ -5017,7 +5018,6 @@ async function saveSettings(type) {
|
||||
active_group: active_group,
|
||||
api_server: api_server,
|
||||
api_server_textgenerationwebui: api_server_textgenerationwebui,
|
||||
api_use_mancer_webui: api_use_mancer_webui,
|
||||
preset_settings: preset_settings,
|
||||
user_avatar: user_avatar,
|
||||
amount_gen: amount_gen,
|
||||
@ -7606,41 +7606,30 @@ jQuery(async function () {
|
||||
}
|
||||
});
|
||||
|
||||
$("#use-mancer-api-checkbox").on("change", function (e) {
|
||||
const enabled = $("#use-mancer-api-checkbox").prop("checked");
|
||||
$("#mancer_api_subpanel").toggle(enabled);
|
||||
$("#tgwebui_api_subpanel").toggle(!enabled);
|
||||
$("#api_button_textgenerationwebui").on('click', async function (e) {
|
||||
const urlSourceId = getTextGenUrlSourceId();
|
||||
|
||||
api_use_mancer_webui = enabled;
|
||||
saveSettingsDebounced();
|
||||
getStatus();
|
||||
|
||||
if (enabled) {
|
||||
loadMancerModels();
|
||||
}
|
||||
});
|
||||
|
||||
$("#api_button_textgenerationwebui").click(async function (e) {
|
||||
const url_source = api_use_mancer_webui ? "#mancer_api_url_text" : "#textgenerationwebui_api_url_text";
|
||||
if ($(url_source).val() != "") {
|
||||
let value = formatTextGenURL(String($(url_source).val()).trim(), api_use_mancer_webui);
|
||||
if ($(urlSourceId).val() != "") {
|
||||
let value = formatTextGenURL(String($(urlSourceId).val()).trim(), isMancer());
|
||||
if (!value) {
|
||||
callPopup("Please enter a valid URL.<br/>WebUI URLs should end with <tt>/api</tt><br/>Enable 'Relaxed API URLs' to allow other paths.", 'text');
|
||||
return;
|
||||
}
|
||||
|
||||
const mancer_key = String($("#api_key_mancer").val()).trim();
|
||||
if (mancer_key.length) {
|
||||
await writeSecret(SECRET_KEYS.MANCER, mancer_key);
|
||||
const mancerKey = String($("#api_key_mancer").val()).trim();
|
||||
if (mancerKey.length) {
|
||||
await writeSecret(SECRET_KEYS.MANCER, mancerKey);
|
||||
}
|
||||
|
||||
$(url_source).val(value);
|
||||
const aphroditeKey = String($("#api_key_aphrodite").val()).trim();
|
||||
if (aphroditeKey.length) {
|
||||
await writeSecret(SECRET_KEYS.APHRODITE, aphroditeKey);
|
||||
}
|
||||
|
||||
$(urlSourceId).val(value);
|
||||
$("#api_loading_textgenerationwebui").css("display", "inline-block");
|
||||
$("#api_button_textgenerationwebui").css("display", "none");
|
||||
|
||||
if (api_use_mancer_webui) {
|
||||
textgenerationwebui_settings.streaming_url = value.replace("http", "ws") + "/v1/stream";
|
||||
}
|
||||
api_server_textgenerationwebui = value;
|
||||
main_api = "textgenerationwebui";
|
||||
saveSettingsDebounced();
|
||||
|
||||
@ -3,6 +3,7 @@ import { callPopup, getRequestHeaders } from "../script.js";
|
||||
export const SECRET_KEYS = {
|
||||
HORDE: 'api_key_horde',
|
||||
MANCER: 'api_key_mancer',
|
||||
APHRODITE: 'api_key_aphrodite',
|
||||
OPENAI: 'api_key_openai',
|
||||
NOVEL: 'api_key_novel',
|
||||
CLAUDE: 'api_key_claude',
|
||||
@ -24,6 +25,7 @@ const INPUT_MAP = {
|
||||
[SECRET_KEYS.AI21]: '#api_key_ai21',
|
||||
[SECRET_KEYS.SCALE_COOKIE]: '#scale_cookie',
|
||||
[SECRET_KEYS.PALM]: '#api_key_palm',
|
||||
[SECRET_KEYS.APHRODITE]: '#api_key_aphrodite',
|
||||
}
|
||||
|
||||
async function clearSecret() {
|
||||
|
||||
@ -1,16 +1,18 @@
|
||||
import {
|
||||
api_server_textgenerationwebui,
|
||||
getRequestHeaders,
|
||||
getStoppingStrings,
|
||||
max_context,
|
||||
saveSettingsDebounced,
|
||||
setGenerationParamsFromPreset,
|
||||
} from "../script.js";
|
||||
import { loadMancerModels } from "./mancer-settings.js";
|
||||
|
||||
import {
|
||||
power_user,
|
||||
} from "./power-user.js";
|
||||
import { getTextTokens, tokenizers } from "./tokenizers.js";
|
||||
import { onlyUnique } from "./utils.js";
|
||||
import { delay, onlyUnique } from "./utils.js";
|
||||
|
||||
export {
|
||||
textgenerationwebui_settings,
|
||||
@ -19,6 +21,12 @@ export {
|
||||
formatTextGenURL,
|
||||
}
|
||||
|
||||
export const textgen_types = {
|
||||
OOBA: 'ooba',
|
||||
MANCER: 'mancer',
|
||||
APHRODITE: 'aphrodite',
|
||||
};
|
||||
|
||||
const textgenerationwebui_settings = {
|
||||
temp: 0.7,
|
||||
top_p: 0.5,
|
||||
@ -54,6 +62,7 @@ const textgenerationwebui_settings = {
|
||||
negative_prompt: '',
|
||||
grammar_string: '',
|
||||
banned_tokens: '',
|
||||
type: textgen_types.OOBA,
|
||||
};
|
||||
|
||||
export let textgenerationwebui_presets = [];
|
||||
@ -173,6 +182,10 @@ function loadTextGenSettings(data, settings) {
|
||||
textgenerationwebui_preset_names = data.textgenerationwebui_preset_names ?? [];
|
||||
Object.assign(textgenerationwebui_settings, settings.textgenerationwebui_settings ?? {});
|
||||
|
||||
if (settings.api_use_mancer_webui) {
|
||||
textgenerationwebui_settings.type = textgen_types.MANCER;
|
||||
}
|
||||
|
||||
for (const name of textgenerationwebui_preset_names) {
|
||||
const option = document.createElement('option');
|
||||
option.value = name;
|
||||
@ -188,9 +201,51 @@ function loadTextGenSettings(data, settings) {
|
||||
const value = textgenerationwebui_settings[i];
|
||||
setSettingByName(i, value);
|
||||
}
|
||||
|
||||
$('#textgen_type').val(textgenerationwebui_settings.type).trigger('change');
|
||||
}
|
||||
|
||||
$(document).ready(function () {
|
||||
export function isMancer() {
|
||||
return textgenerationwebui_settings.type === textgen_types.MANCER;
|
||||
}
|
||||
|
||||
export function isAphrodite() {
|
||||
return textgenerationwebui_settings.type === textgen_types.APHRODITE;
|
||||
}
|
||||
|
||||
export function getTextGenUrlSourceId() {
|
||||
switch (textgenerationwebui_settings.type) {
|
||||
case textgen_types.MANCER:
|
||||
return "#mancer_api_url_text";
|
||||
case textgen_types.OOBA:
|
||||
return "#textgenerationwebui_api_url_text";
|
||||
case textgen_types.APHRODITE:
|
||||
return "#aphrodite_api_url_text";
|
||||
}
|
||||
}
|
||||
|
||||
jQuery(function () {
|
||||
$('#textgen_type').on('change', function () {
|
||||
const type = String($(this).val());
|
||||
textgenerationwebui_settings.type = type;
|
||||
|
||||
$('[data-tg-type]').each(function () {
|
||||
const tgType = $(this).attr('data-tg-type');
|
||||
if (tgType == type) {
|
||||
$(this).show();
|
||||
} else {
|
||||
$(this).hide();
|
||||
}
|
||||
});
|
||||
|
||||
if (isMancer()) {
|
||||
loadMancerModels();
|
||||
}
|
||||
|
||||
saveSettingsDebounced();
|
||||
$('#api_button_textgenerationwebui').trigger('click');
|
||||
});
|
||||
|
||||
$('#settings_preset_textgenerationwebui').on('change', function () {
|
||||
const presetName = $(this).val();
|
||||
selectPreset(presetName);
|
||||
@ -248,11 +303,21 @@ function setSettingByName(i, value, trigger) {
|
||||
}
|
||||
|
||||
async function generateTextGenWithStreaming(generate_data, signal) {
|
||||
let streamingUrl = textgenerationwebui_settings.streaming_url;
|
||||
|
||||
if (isMancer()) {
|
||||
streamingUrl = api_server_textgenerationwebui.replace("http", "ws") + "/v1/stream";
|
||||
}
|
||||
|
||||
if (isAphrodite()){
|
||||
streamingUrl = api_server_textgenerationwebui;
|
||||
}
|
||||
|
||||
const response = await fetch('/generate_textgenerationwebui', {
|
||||
headers: {
|
||||
...getRequestHeaders(),
|
||||
'X-Response-Streaming': String(true),
|
||||
'X-Streaming-URL': textgenerationwebui_settings.streaming_url,
|
||||
'X-Streaming-URL': streamingUrl,
|
||||
},
|
||||
body: JSON.stringify(generate_data),
|
||||
method: 'POST',
|
||||
@ -266,13 +331,43 @@ async function generateTextGenWithStreaming(generate_data, signal) {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
let response = decoder.decode(value);
|
||||
getMessage += response;
|
||||
|
||||
if (done) {
|
||||
return;
|
||||
if (isAphrodite()) {
|
||||
const events = response.split('\n\n');
|
||||
|
||||
for (const event of events) {
|
||||
if (event.length == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const { results } = JSON.parse(event);
|
||||
|
||||
if (Array.isArray(results) && results.length > 0) {
|
||||
getMessage = results[0].text;
|
||||
yield getMessage;
|
||||
|
||||
// unhang UI thread
|
||||
await delay(1);
|
||||
}
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
|
||||
if (done) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
|
||||
getMessage += response;
|
||||
|
||||
if (done) {
|
||||
return;
|
||||
}
|
||||
|
||||
yield getMessage;
|
||||
}
|
||||
|
||||
yield getMessage;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
111
server.js
111
server.js
@ -148,9 +148,14 @@ let color = {
|
||||
white: (mess) => color.byNum(mess, 37)
|
||||
};
|
||||
|
||||
function get_mancer_headers() {
|
||||
const api_key_mancer = readSecret(SECRET_KEYS.MANCER);
|
||||
return api_key_mancer ? { "X-API-KEY": api_key_mancer } : {};
|
||||
function getMancerHeaders() {
|
||||
const apiKey = readSecret(SECRET_KEYS.MANCER);
|
||||
return apiKey ? { "X-API-KEY": apiKey } : {};
|
||||
}
|
||||
|
||||
function getAphroditeHeaders() {
|
||||
const apiKey = readSecret(SECRET_KEYS.APHRODITE);
|
||||
return apiKey ? { "X-API-KEY": apiKey } : {};
|
||||
}
|
||||
|
||||
function getOverrideHeaders(urlHost) {
|
||||
@ -162,6 +167,26 @@ function getOverrideHeaders(urlHost) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets additional headers for the request.
|
||||
* @param {object} request Original request body
|
||||
* @param {object} args New request arguments
|
||||
* @param {string|null} server API server for new request
|
||||
*/
|
||||
function setAdditionalHeaders(request, args, server) {
|
||||
let headers = {};
|
||||
|
||||
if (request.body.use_mancer) {
|
||||
headers = getMancerHeaders();
|
||||
} else if (request.body.use_aphrodite) {
|
||||
headers = getAphroditeHeaders();
|
||||
} else {
|
||||
headers = server ? getOverrideHeaders((new URL(server))?.host) : '';
|
||||
}
|
||||
|
||||
args.headers = Object.assign(args.headers, headers);
|
||||
}
|
||||
|
||||
function humanizedISO8601DateTime(date) {
|
||||
let baseDate = typeof date === 'number' ? new Date(date) : new Date();
|
||||
let humanYear = baseDate.getFullYear();
|
||||
@ -451,6 +476,52 @@ app.post("/generate", jsonParser, async function (request, response_generate) {
|
||||
return response_generate.send({ error: true });
|
||||
});
|
||||
|
||||
/**
|
||||
* @param {string} streamingUrlString Streaming URL
|
||||
* @param {import('express').Request} request Express request
|
||||
* @param {import('express').Response} response Express response
|
||||
* @param {AbortController} controller Abort controller
|
||||
* @returns
|
||||
*/
|
||||
async function sendAphroditeStreamingRequest(streamingUrlString, request, response, controller) {
|
||||
request.body['stream'] = true;
|
||||
|
||||
const args = {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(request.body),
|
||||
headers: { "Content-Type": "application/json" },
|
||||
signal: controller.signal,
|
||||
};
|
||||
|
||||
setAdditionalHeaders(request, args, streamingUrlString);
|
||||
|
||||
try {
|
||||
const generateResponse = await fetch(streamingUrlString + "/v1/generate", args);
|
||||
// Pipe remote SSE stream to Express response
|
||||
generateResponse.body.pipe(response);
|
||||
|
||||
request.socket.on('close', function () {
|
||||
if (generateResponse.body instanceof Readable) generateResponse.body.destroy(); // Close the remote stream
|
||||
response.end(); // End the Express response
|
||||
});
|
||||
|
||||
generateResponse.body.on('end', function () {
|
||||
console.log("Streaming request finished");
|
||||
response.end();
|
||||
});
|
||||
} catch (error) {
|
||||
let value = { error: true, status: error.status, response: error.statusText };
|
||||
console.log("Aphrodite endpoint error:", error);
|
||||
|
||||
if (!response.headersSent) {
|
||||
return response.send(value);
|
||||
} else {
|
||||
return response.end();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//************** Text generation web UI
|
||||
app.post("/generate_textgenerationwebui", jsonParser, async function (request, response_generate) {
|
||||
if (!request.body) return response_generate.sendStatus(400);
|
||||
@ -470,6 +541,10 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
|
||||
if (streamingUrlHeader === undefined) return response_generate.sendStatus(400);
|
||||
const streamingUrlString = streamingUrlHeader.replace("localhost", "127.0.0.1");
|
||||
|
||||
if (request.body.use_aphrodite) {
|
||||
return sendAphroditeStreamingRequest(streamingUrlString, request, response_generate, controller);
|
||||
}
|
||||
|
||||
response_generate.writeHead(200, {
|
||||
'Content-Type': 'text/plain;charset=utf-8',
|
||||
'Transfer-Encoding': 'chunked',
|
||||
@ -482,9 +557,20 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
|
||||
|
||||
websocket.on('open', async function () {
|
||||
console.log('WebSocket opened');
|
||||
|
||||
let headers = {};
|
||||
|
||||
if (request.body.use_mancer) {
|
||||
headers = getMancerHeaders();
|
||||
} else if (request.body.use_aphrodite) {
|
||||
headers = getAphroditeHeaders();
|
||||
} else {
|
||||
headers = getOverrideHeaders(streamingUrl?.host);
|
||||
}
|
||||
|
||||
const combined_args = Object.assign(
|
||||
{},
|
||||
request.body.use_mancer ? get_mancer_headers() : getOverrideHeaders(streamingUrl?.host),
|
||||
headers,
|
||||
request.body
|
||||
);
|
||||
console.log(combined_args);
|
||||
@ -568,11 +654,7 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
|
||||
signal: controller.signal,
|
||||
};
|
||||
|
||||
if (request.body.use_mancer) {
|
||||
args.headers = Object.assign(args.headers, get_mancer_headers());
|
||||
} else {
|
||||
args.headers = Object.assign(args.headers, getOverrideHeaders((new URL(api_server))?.host));
|
||||
}
|
||||
setAdditionalHeaders(request, args, api_server);
|
||||
|
||||
try {
|
||||
const data = await postAsync(api_server + "/v1/generate", args);
|
||||
@ -677,11 +759,7 @@ app.post("/getstatus", jsonParser, async function (request, response) {
|
||||
headers: { "Content-Type": "application/json" }
|
||||
};
|
||||
|
||||
if (main_api == 'textgenerationwebui' && request.body.use_mancer) {
|
||||
args.headers = Object.assign(args.headers, get_mancer_headers());
|
||||
} else {
|
||||
args.headers = Object.assign(args.headers, getOverrideHeaders((new URL(api_server))?.host));
|
||||
}
|
||||
setAdditionalHeaders(request, args, api_server);
|
||||
|
||||
const url = api_server + "/v1/model";
|
||||
let version = '';
|
||||
@ -3237,9 +3315,8 @@ app.post("/tokenize_via_api", jsonParser, async function (request, response) {
|
||||
};
|
||||
|
||||
if (main_api == 'textgenerationwebui') {
|
||||
if (request.body.use_mancer) {
|
||||
args.headers = Object.assign(args.headers, get_mancer_headers());
|
||||
}
|
||||
setAdditionalHeaders(request, args, null);
|
||||
|
||||
const data = await postAsync(api_server + "/v1/token-count", args);
|
||||
return response.send({ count: data['results'][0]['tokens'] });
|
||||
}
|
||||
|
||||
@ -7,6 +7,7 @@ const SECRETS_FILE = path.join(process.cwd(), './secrets.json');
|
||||
const SECRET_KEYS = {
|
||||
HORDE: 'api_key_horde',
|
||||
MANCER: 'api_key_mancer',
|
||||
APHRODITE: 'api_key_aphrodite',
|
||||
OPENAI: 'api_key_openai',
|
||||
NOVEL: 'api_key_novel',
|
||||
CLAUDE: 'api_key_claude',
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user