rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'staging' into presets-router

This commit is contained in:
Cohee 2023-12-04 23:51:08 +02:00
parent ba74288e4a 9a50036657
commit 6fc0e485cd
18 changed files with 1264 additions and 1259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintrc.js
View File

@ -58,6 +58,7 @@ module.exports = {
'comma-dangle': ['error', 'always-multiline'],
'eol-last': ['error', 'always'],
'no-trailing-spaces': 'error',
'object-curly-spacing': ['error', 'always'],
// These rules should eventually be enabled.
'no-async-promise-executor': 'off',

30
public/script.js
View File

@ -887,7 +887,7 @@ async function getStatus() {
api_type: textgen_settings.type,
legacy_api: main_api == 'textgenerationwebui' ?
textgen_settings.legacy_api &&
textgen_settings.type !== MANCER :
textgen_settings.type !== MANCER :
false,
}),
signal: abortStatusCheck.signal,
@ -2914,7 +2914,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
let textareaText;
if (type !== 'regenerate' && type !== 'swipe' && type !== 'quiet' && !isImpersonate && !dryRun) {
is_send_press = true;
textareaText = $('#send_textarea').val();
textareaText = String($('#send_textarea').val());
$('#send_textarea').val('').trigger('input');
} else {
textareaText = '';
@ -2960,7 +2960,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
//*********************************
//for normal messages sent from user..
if ((textareaText != '' || hasPendingFileAttachment()) && !automatic_trigger && type !== 'quiet') {
if ((textareaText != '' || hasPendingFileAttachment()) && !automatic_trigger && type !== 'quiet' && !dryRun) {
// If user message contains no text other than bias - send as a system message
if (messageBias && replaceBiasMarkup(textareaText).trim().length === 0) {
sendSystemMessage(system_message_types.GENERIC, ' ', { bias: messageBias });
@ -2969,7 +2969,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
await sendMessageAsUser(textareaText, messageBias);
}
}
else if (textareaText == '' && !automatic_trigger && type === undefined && main_api == 'openai' && oai_settings.send_if_empty.trim().length > 0) {
else if (textareaText == '' && !automatic_trigger && !dryRun && type === undefined && main_api == 'openai' && oai_settings.send_if_empty.trim().length > 0) {
// Use send_if_empty if set and the user message is empty. Only when sending messages normally
await sendMessageAsUser(oai_settings.send_if_empty.trim(), messageBias);
}
@ -4154,11 +4154,11 @@ async function DupeChar() {
return;
}
const confirm = await callPopup(`
<h3>Are you sure you want to duplicate this character?</h3>
<span>If you just want to start a new chat with the same character, use "Start new chat" option in the bottom-left options menu.</span><br><br>`,
'confirm',
);
const confirmMessage = `
<h3>Are you sure you want to duplicate this character?</h3>
<span>If you just want to start a new chat with the same character, use "Start new chat" option in the bottom-left options menu.</span><br><br>`;
const confirm = await callPopup(confirmMessage, 'confirm');
if (!confirm) {
console.log('User cancelled duplication');
@ -7631,10 +7631,7 @@ function doTogglePanels() {
}
function addDebugFunctions() {
registerDebugFunction('backfillTokenCounts', 'Backfill token counters',
`Recalculates token counts of all messages in the current chat to refresh the counters.
Useful when you switch between models that have different tokenizers.
This is a visual change only. Your chat will be reloaded.`, async () => {
const doBackfill = async () => {
for (const message of chat) {
// System messages are not counted
if (message.is_system) {
@ -7650,7 +7647,12 @@ function addDebugFunctions() {
await saveChatConditional();
await reloadCurrentChat();
});
};
registerDebugFunction('backfillTokenCounts', 'Backfill token counters',
`Recalculates token counts of all messages in the current chat to refresh the counters.
Useful when you switch between models that have different tokenizers.
This is a visual change only. Your chat will be reloaded.`, doBackfill);
registerDebugFunction('generationTest', 'Send a generation request', 'Generates text using the currently selected API.', async () => {
const text = prompt('Input text:', 'Hello');

2
public/scripts/bulk-edit.js
View File

@ -1,5 +1,5 @@
import { characters, getCharacters, handleDeleteCharacter, callPopup } from '../script.js';
import {BulkEditOverlay, BulkEditOverlayState} from './BulkEditOverlay.js';
import { BulkEditOverlay, BulkEditOverlayState } from './BulkEditOverlay.js';
let is_bulk_edit = false;

4
public/scripts/chats.js
View File

@ -152,7 +152,7 @@ export async function populateFileAttachment(message, inputId = 'file_form_input
*/
export async function uploadFileAttachment(fileName, base64Data) {
try {
const result = await fetch('/api/file/upload', {
const result = await fetch('/api/files/upload', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
@ -302,7 +302,7 @@ async function viewMessageFile(messageId) {
modalTemplate.addClass('file_modal');
addCopyToCodeBlocks(modalTemplate);
callPopup(modalTemplate, 'text');
callPopup(modalTemplate, 'text', '', { wide: true, large: true });
}
/**

2
public/scripts/extensions/translate/index.js
View File

@ -1,4 +1,4 @@
export {translate};
export { translate };
import {
callPopup,

2
public/scripts/extensions/tts/index.js
View File

@ -10,7 +10,7 @@ import { NovelTtsProvider } from './novel.js';
import { power_user } from '../../power-user.js';
import { registerSlashCommand } from '../../slash-commands.js';
import { OpenAITtsProvider } from './openai.js';
import {XTTSTtsProvider} from './xtts.js';
import { XTTSTtsProvider } from './xtts.js';
export { talkingAnimation };
const UPDATE_INTERVAL = 1000;

2
public/scripts/extensions/tts/novel.js
View File

@ -125,7 +125,7 @@ class NovelTtsProvider {
throw 'TTS Voice name not provided';
}
return { name: voiceName, voice_id: voiceName, lang: 'en-US', preview_url: false};
return { name: voiceName, voice_id: voiceName, lang: 'en-US', preview_url: false };
}
async generateTts(text, voiceId) {

8
public/scripts/power-user.js
View File

@ -666,7 +666,7 @@ async function CreateZenSliders(elmnt) {
min: sliderMin,
max: sliderMax,
create: async function () {
await delay(100)
await delay(100);
var handle = $(this).find('.ui-slider-handle');
var handleText, stepNumber, leftMargin;
@ -711,7 +711,7 @@ async function CreateZenSliders(elmnt) {
stepNumber = ((sliderValue - sliderMin) / stepScale);
leftMargin = (stepNumber / numSteps) * 50 * -1;
originalSlider.val(numVal)
.data('newSlider', newSlider)
.data('newSlider', newSlider);
//console.log(`${newSlider.attr('id')} sliderValue = ${sliderValue}, handleText:${handleText, numVal}, stepNum:${stepNumber}, numSteps:${numSteps}, left-margin:${leftMargin}`)
var isManualInput = false;
var valueBeforeManualInput;
@ -737,8 +737,8 @@ async function CreateZenSliders(elmnt) {
isManualInput = true;
//allow enter to trigger slider update
if (e.key === 'Enter') {
e.preventDefault
handle.trigger('blur')
e.preventDefault;
handle.trigger('blur');
}
})
//trigger slider changes when user clicks away

19
server.js
View File

@ -3576,7 +3576,7 @@ async function fetchJSON(url, args = {}) {
// ** END **
// OpenAI API
require('./src/endpoints/openai').registerEndpoints(app, jsonParser, urlencodedParser);
app.use('/api/openai', require('./src/endpoints/openai').router);
// Tokenizers
require('./src/endpoints/tokenizers').registerEndpoints(app, jsonParser);
@ -3591,25 +3591,28 @@ require('./src/endpoints/secrets').registerEndpoints(app, jsonParser);
require('./src/endpoints/thumbnails').registerEndpoints(app, jsonParser);
// NovelAI generation
require('./src/endpoints/novelai').registerEndpoints(app, jsonParser);
app.use('/api/novelai', require('./src/endpoints/novelai').router);
// Third-party extensions
require('./src/endpoints/extensions').registerEndpoints(app, jsonParser);
app.use('/api/extensions', require('./src/endpoints/extensions').router);
// Asset management
require('./src/endpoints/assets').registerEndpoints(app, jsonParser);
app.use('/api/assets', require('./src/endpoints/assets').router);
// File management
app.use('/api/files', require('./src/endpoints/files').router);
// Character sprite management
require('./src/endpoints/sprites').registerEndpoints(app, jsonParser, urlencodedParser);
// Custom content management
require('./src/endpoints/content-manager').registerEndpoints(app, jsonParser);
app.use('/api/content', require('./src/endpoints/content-manager').router);
// Stable Diffusion generation
require('./src/endpoints/stable-diffusion').registerEndpoints(app, jsonParser);
// LLM and SD Horde generation
require('./src/endpoints/horde').registerEndpoints(app, jsonParser);
app.use('/api/horde', require('./src/endpoints/horde').router);
// Vector storage DB
require('./src/endpoints/vectors').registerEndpoints(app, jsonParser);
@ -3618,10 +3621,10 @@ require('./src/endpoints/vectors').registerEndpoints(app, jsonParser);
require('./src/endpoints/translate').registerEndpoints(app, jsonParser);
// Emotion classification
require('./src/endpoints/classify').registerEndpoints(app, jsonParser);
app.use('/api/extra/classify', require('./src/endpoints/classify').router);
// Image captioning
require('./src/endpoints/caption').registerEndpoints(app, jsonParser);
app.use('/api/extra/caption', require('./src/endpoints/caption').router);
// Web search extension
require('./src/endpoints/serpapi').registerEndpoints(app, jsonParser);

497
src/endpoints/assets.js
View File

@ -1,10 +1,12 @@
const path = require('path');
const fs = require('fs');
const express = require('express');
const sanitize = require('sanitize-filename');
const fetch = require('node-fetch').default;
const { finished } = require('stream/promises');
const writeFileSyncAtomic = require('write-file-atomic').sync;
const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants');
const { jsonParser } = require('../express-common');
const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d'];
@ -57,273 +59,266 @@ function getFiles(dir, files = []) {
return files;
}
const router = express.Router();
/**
* Registers the endpoints for the asset management.
* @param {import('express').Express} app Express app
* @param {any} jsonParser JSON parser middleware
* HTTP POST handler function to retrieve name of all files of a given folder path.
*
* @param {Object} request - HTTP Request object. Require folder path in query
* @param {Object} response - HTTP Response object will contain a list of file path.
*
* @returns {void}
*/
function registerEndpoints(app, jsonParser) {
/**
* HTTP POST handler function to retrieve name of all files of a given folder path.
*
* @param {Object} request - HTTP Request object. Require folder path in query
* @param {Object} response - HTTP Response object will contain a list of file path.
*
* @returns {void}
*/
app.post('/api/assets/get', jsonParser, async (_, response) => {
const folderPath = path.join(DIRECTORIES.assets);
let output = {};
//console.info("Checking files into",folderPath);
router.post('/get', jsonParser, async (_, response) => {
const folderPath = path.join(DIRECTORIES.assets);
let output = {};
//console.info("Checking files into",folderPath);
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
const folders = fs.readdirSync(folderPath)
.filter(filename => {
return fs.statSync(path.join(folderPath, filename)).isDirectory();
});
for (const folder of folders) {
if (folder == 'temp')
continue;
// Live2d assets
if (folder == 'live2d') {
output[folder] = [];
const live2d_folder = path.normalize(path.join(folderPath, folder));
const files = getFiles(live2d_folder);
//console.debug("FILE FOUND:",files)
for (let file of files) {
file = path.normalize(file.replace('public' + path.sep, ''));
if (file.includes('model') && file.endsWith('.json')) {
//console.debug("Asset live2d model found:",file)
output[folder].push(path.normalize(path.join(file)));
}
}
continue;
}
// Other assets (bgm/ambient/blip)
const files = fs.readdirSync(path.join(folderPath, folder))
.filter(filename => {
return filename != '.placeholder';
});
output[folder] = [];
for (const file of files) {
output[folder].push(path.join('assets', folder, file));
}
}
}
}
catch (err) {
console.log(err);
}
return response.send(output);
});
/**
* HTTP POST handler function to download the requested asset.
*
* @param {Object} request - HTTP Request object, expects a url, a category and a filename.
* @param {Object} response - HTTP Response only gives status.
*
* @returns {void}
*/
app.post('/api/assets/download', jsonParser, async (request, response) => {
const url = request.body.url;
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const temp_path = path.join(DIRECTORIES.assets, 'temp', safe_input);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to download', url, 'to', file_path);
try {
// Download to temp
const res = await fetch(url);
if (!res.ok || res.body === null) {
throw new Error(`Unexpected response ${res.statusText}`);
}
const destination = path.resolve(temp_path);
// Delete if previous download failed
if (fs.existsSync(temp_path)) {
fs.unlink(temp_path, (err) => {
if (err) throw err;
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
const folders = fs.readdirSync(folderPath)
.filter(filename => {
return fs.statSync(path.join(folderPath, filename)).isDirectory();
});
}
const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
await finished(res.body.pipe(fileStream));
// Move into asset place
console.debug('Download finished, moving file from', temp_path, 'to', file_path);
fs.renameSync(temp_path, file_path);
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
/**
* HTTP POST handler function to delete the requested asset.
*
* @param {Object} request - HTTP Request object, expects a category and a filename
* @param {Object} response - HTTP Response only gives stats.
*
* @returns {void}
*/
app.post('/api/assets/delete', jsonParser, async (request, response) => {
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to delete', category, file_path);
try {
// Delete if previous download failed
if (fs.existsSync(file_path)) {
fs.unlink(file_path, (err) => {
if (err) throw err;
});
console.debug('Asset deleted.');
}
else {
console.debug('Asset not found.');
response.sendStatus(400);
}
// Move into asset place
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
///////////////////////////////
/**
* HTTP POST handler function to retrieve a character background music list.
*
* @param {Object} request - HTTP Request object, expects a character name in the query.
* @param {Object} response - HTTP Response object will contain a list of audio file path.
*
* @returns {void}
*/
app.post('/api/assets/character', jsonParser, async (request, response) => {
if (request.query.name === undefined) return response.sendStatus(400);
const name = sanitize(request.query.name.toString());
const inputCategory = request.query.category;
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
const folderPath = path.join(DIRECTORIES.characters, name, category);
let output = [];
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
for (const folder of folders) {
if (folder == 'temp')
continue;
// Live2d assets
if (category == 'live2d') {
const folders = fs.readdirSync(folderPath);
for (let modelFolder of folders) {
const live2dModelPath = path.join(folderPath, modelFolder);
if (fs.statSync(live2dModelPath).isDirectory()) {
for (let file of fs.readdirSync(live2dModelPath)) {
//console.debug("Character live2d model found:", file)
if (file.includes('model') && file.endsWith('.json'))
output.push(path.join('characters', name, category, modelFolder, file));
}
if (folder == 'live2d') {
output[folder] = [];
const live2d_folder = path.normalize(path.join(folderPath, folder));
const files = getFiles(live2d_folder);
//console.debug("FILE FOUND:",files)
for (let file of files) {
file = path.normalize(file.replace('public' + path.sep, ''));
if (file.includes('model') && file.endsWith('.json')) {
//console.debug("Asset live2d model found:",file)
output[folder].push(path.normalize(path.join(file)));
}
}
return response.send(output);
continue;
}
// Other assets
const files = fs.readdirSync(folderPath)
// Other assets (bgm/ambient/blip)
const files = fs.readdirSync(path.join(folderPath, folder))
.filter(filename => {
return filename != '.placeholder';
});
for (let i of files)
output.push(`/characters/${name}/${category}/${i}`);
output[folder] = [];
for (const file of files) {
output[folder].push(path.join('assets', folder, file));
}
}
return response.send(output);
}
catch (err) {
console.log(err);
return response.sendStatus(500);
}
catch (err) {
console.log(err);
}
return response.send(output);
});
/**
* HTTP POST handler function to download the requested asset.
*
* @param {Object} request - HTTP Request object, expects a url, a category and a filename.
* @param {Object} response - HTTP Response only gives status.
*
* @returns {void}
*/
router.post('/download', jsonParser, async (request, response) => {
const url = request.body.url;
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const temp_path = path.join(DIRECTORIES.assets, 'temp', safe_input);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to download', url, 'to', file_path);
try {
// Download to temp
const res = await fetch(url);
if (!res.ok || res.body === null) {
throw new Error(`Unexpected response ${res.statusText}`);
}
});
app.post('/api/file/upload', jsonParser, async (request, response) => {
try {
if (!request.body.name) {
return response.status(400).send('No upload name specified');
}
if (!request.body.data) {
return response.status(400).send('No upload data specified');
}
const safeInput = checkAssetFileName(request.body.name);
if (!safeInput) {
return response.status(400).send('Invalid upload name');
}
const pathToUpload = path.join(DIRECTORIES.files, safeInput);
writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
return response.send({ path: url });
} catch (error) {
console.log(error);
return response.sendStatus(500);
const destination = path.resolve(temp_path);
// Delete if previous download failed
if (fs.existsSync(temp_path)) {
fs.unlink(temp_path, (err) => {
if (err) throw err;
});
}
});
}
const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
await finished(res.body.pipe(fileStream));
module.exports = {
registerEndpoints,
};
// Move into asset place
console.debug('Download finished, moving file from', temp_path, 'to', file_path);
fs.renameSync(temp_path, file_path);
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
/**
* HTTP POST handler function to delete the requested asset.
*
* @param {Object} request - HTTP Request object, expects a category and a filename
* @param {Object} response - HTTP Response only gives stats.
*
* @returns {void}
*/
router.post('/delete', jsonParser, async (request, response) => {
const inputCategory = request.body.category;
const inputFilename = sanitize(request.body.filename);
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
// Sanitize filename
const safe_input = checkAssetFileName(inputFilename);
if (safe_input == '')
return response.sendStatus(400);
const file_path = path.join(DIRECTORIES.assets, category, safe_input);
console.debug('Request received to delete', category, file_path);
try {
// Delete if previous download failed
if (fs.existsSync(file_path)) {
fs.unlink(file_path, (err) => {
if (err) throw err;
});
console.debug('Asset deleted.');
}
else {
console.debug('Asset not found.');
response.sendStatus(400);
}
// Move into asset place
response.sendStatus(200);
}
catch (error) {
console.log(error);
response.sendStatus(500);
}
});
///////////////////////////////
/**
* HTTP POST handler function to retrieve a character background music list.
*
* @param {Object} request - HTTP Request object, expects a character name in the query.
* @param {Object} response - HTTP Response object will contain a list of audio file path.
*
* @returns {void}
*/
router.post('/character', jsonParser, async (request, response) => {
if (request.query.name === undefined) return response.sendStatus(400);
const name = sanitize(request.query.name.toString());
const inputCategory = request.query.category;
// Check category
let category = null;
for (let i of VALID_CATEGORIES)
if (i == inputCategory)
category = i;
if (category === null) {
console.debug('Bad request: unsuported asset category.');
return response.sendStatus(400);
}
const folderPath = path.join(DIRECTORIES.characters, name, category);
let output = [];
try {
if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
// Live2d assets
if (category == 'live2d') {
const folders = fs.readdirSync(folderPath);
for (let modelFolder of folders) {
const live2dModelPath = path.join(folderPath, modelFolder);
if (fs.statSync(live2dModelPath).isDirectory()) {
for (let file of fs.readdirSync(live2dModelPath)) {
//console.debug("Character live2d model found:", file)
if (file.includes('model') && file.endsWith('.json'))
output.push(path.join('characters', name, category, modelFolder, file));
}
}
}
return response.send(output);
}
// Other assets
const files = fs.readdirSync(folderPath)
.filter(filename => {
return filename != '.placeholder';
});
for (let i of files)
output.push(`/characters/${name}/${category}/${i}`);
}
return response.send(output);
}
catch (err) {
console.log(err);
return response.sendStatus(500);
}
});
router.post('/upload', jsonParser, async (request, response) => {
try {
if (!request.body.name) {
return response.status(400).send('No upload name specified');
}
if (!request.body.data) {
return response.status(400).send('No upload data specified');
}
const safeInput = checkAssetFileName(request.body.name);
if (!safeInput) {
return response.status(400).send('Invalid upload name');
}
const pathToUpload = path.join(DIRECTORIES.files, safeInput);
writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
return response.send({ path: url });
} catch (error) {
console.log(error);
return response.sendStatus(500);
}
});
module.exports = { router, checkAssetFileName };

53
src/endpoints/caption.js
View File

@ -1,35 +1,32 @@
const express = require('express');
const { jsonParser } = require('../express-common');
const TASK = 'image-to-text';
/**
* @param {import("express").Express} app
* @param {any} jsonParser
*/
function registerEndpoints(app, jsonParser) {
app.post('/api/extra/caption', jsonParser, async (req, res) => {
try {
const { image } = req.body;
const router = express.Router();
const module = await import('../transformers.mjs');
const rawImage = await module.default.getRawImage(image);
router.post('/', jsonParser, async (req, res) => {
try {
const { image } = req.body;
if (!rawImage) {
console.log('Failed to parse captioned image');
return res.sendStatus(400);
}
const module = await import('../transformers.mjs');
const rawImage = await module.default.getRawImage(image);
const pipe = await module.default.getPipeline(TASK);
const result = await pipe(rawImage);
const text = result[0].generated_text;
console.log('Image caption:', text);
return res.json({ caption: text });
} catch (error) {
console.error(error);
return res.sendStatus(500);
if (!rawImage) {
console.log('Failed to parse captioned image');
return res.sendStatus(400);
}
});
}
module.exports = {
registerEndpoints,
};
const pipe = await module.default.getPipeline(TASK);
const result = await pipe(rawImage);
const text = result[0].generated_text;
console.log('Image caption:', text);
return res.json({ caption: text });
} catch (error) {
console.error(error);
return res.sendStatus(500);
}
});
module.exports = { router };

87
src/endpoints/classify.js
View File

@ -1,53 +1,50 @@
const express = require('express');
const { jsonParser } = require('../express-common');
const TASK = 'text-classification';
/**
* @param {import("express").Express} app
* @param {any} jsonParser
*/
function registerEndpoints(app, jsonParser) {
const cacheObject = {};
const router = express.Router();
app.post('/api/extra/classify/labels', jsonParser, async (req, res) => {
try {
const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(TASK);
const result = Object.keys(pipe.model.config.label2id);
return res.json({ labels: result });
} catch (error) {
console.error(error);
return res.sendStatus(500);
}
});
const cacheObject = {};
app.post('/api/extra/classify', jsonParser, async (req, res) => {
try {
const { text } = req.body;
router.post('/labels', jsonParser, async (req, res) => {
try {
const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(TASK);
const result = Object.keys(pipe.model.config.label2id);
return res.json({ labels: result });
} catch (error) {
console.error(error);
return res.sendStatus(500);
}
});
async function getResult(text) {
if (Object.hasOwn(cacheObject, text)) {
return cacheObject[text];
} else {
const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(TASK);
const result = await pipe(text, { topk: 5 });
result.sort((a, b) => b.score - a.score);
cacheObject[text] = result;
return result;
}
router.post('/', jsonParser, async (req, res) => {
try {
const { text } = req.body;
async function getResult(text) {
if (Object.hasOwn(cacheObject, text)) {
return cacheObject[text];
} else {
const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(TASK);
const result = await pipe(text, { topk: 5 });
result.sort((a, b) => b.score - a.score);
cacheObject[text] = result;
return result;
}
console.log('Classify input:', text);
const result = await getResult(text);
console.log('Classify output:', result);
return res.json({ classification: result });
} catch (error) {
console.error(error);
return res.sendStatus(500);
}
});
}
module.exports = {
registerEndpoints,
};
console.log('Classify input:', text);
const result = await getResult(text);
console.log('Classify output:', result);
return res.json({ classification: result });
} catch (error) {
console.error(error);
return res.sendStatus(500);
}
});
module.exports = { router };

91
src/endpoints/content-manager.js
View File

@ -1,8 +1,10 @@
const fs = require('fs');
const path = require('path');
const express = require('express');
const fetch = require('node-fetch').default;
const sanitize = require('sanitize-filename');
const { getConfigValue } = require('../util');
const { jsonParser } = require('../express-common');
const contentDirectory = path.join(process.cwd(), 'default/content');
const contentLogPath = path.join(contentDirectory, 'content.log');
const contentIndexPath = path.join(contentDirectory, 'index.json');
@ -302,62 +304,57 @@ function parseJannyUrl(url) {
return uuid;
}
/**
* Registers endpoints for custom content management
* @param {import('express').Express} app Express app
* @param {any} jsonParser JSON parser middleware
*/
function registerEndpoints(app, jsonParser) {
app.post('/api/content/import', jsonParser, async (request, response) => {
if (!request.body.url) {
return response.sendStatus(400);
}
const router = express.Router();
try {
const url = request.body.url;
let result;
let type;
router.post('/import', jsonParser, async (request, response) => {
if (!request.body.url) {
return response.sendStatus(400);
}
const isJannnyContent = url.includes('janitorai');
if (isJannnyContent) {
const uuid = parseJannyUrl(url);
if (!uuid) {
return response.sendStatus(404);
}
try {
const url = request.body.url;
let result;
let type;
type = 'character';
result = await downloadJannyCharacter(uuid);
} else {
const chubParsed = parseChubUrl(url);
type = chubParsed?.type;
if (chubParsed?.type === 'character') {
console.log('Downloading chub character:', chubParsed.id);
result = await downloadChubCharacter(chubParsed.id);
}
else if (chubParsed?.type === 'lorebook') {
console.log('Downloading chub lorebook:', chubParsed.id);
result = await downloadChubLorebook(chubParsed.id);
}
else {
return response.sendStatus(404);
}
const isJannnyContent = url.includes('janitorai');
if (isJannnyContent) {
const uuid = parseJannyUrl(url);
if (!uuid) {
return response.sendStatus(404);
}
if (result.fileType) response.set('Content-Type', result.fileType);
response.set('Content-Disposition', `attachment; filename="${result.fileName}"`);
response.set('X-Custom-Content-Type', type);
return response.send(result.buffer);
} catch (error) {
console.log('Importing custom content failed', error);
return response.sendStatus(500);
type = 'character';
result = await downloadJannyCharacter(uuid);
} else {
const chubParsed = parseChubUrl(url);
type = chubParsed?.type;
if (chubParsed?.type === 'character') {
console.log('Downloading chub character:', chubParsed.id);
result = await downloadChubCharacter(chubParsed.id);
}
else if (chubParsed?.type === 'lorebook') {
console.log('Downloading chub lorebook:', chubParsed.id);
result = await downloadChubLorebook(chubParsed.id);
}
else {
return response.sendStatus(404);
}
}
});
}
if (result.fileType) response.set('Content-Type', result.fileType);
response.set('Content-Disposition', `attachment; filename="${result.fileName}"`);
response.set('X-Custom-Content-Type', type);
return response.send(result.buffer);
} catch (error) {
console.log('Importing custom content failed', error);
return response.sendStatus(500);
}
});
module.exports = {
checkForNewContent,
registerEndpoints,
getDefaultPresets,
getDefaultPresetFile,
router,
};

374
src/endpoints/extensions.js
View File

@ -1,8 +1,10 @@
const path = require('path');
const fs = require('fs');
const express = require('express');
const { default: simpleGit } = require('simple-git');
const sanitize = require('sanitize-filename');
const { DIRECTORIES } = require('../constants');
const { jsonParser } = require('../express-common');
/**
* This function extracts the extension information from the manifest file.
@ -45,206 +47,198 @@ async function checkIfRepoIsUpToDate(extensionPath) {
};
}
const router = express.Router();
/**
* Registers the endpoints for the third-party extensions API.
* @param {import('express').Express} app - Express app
* @param {any} jsonParser - JSON parser middleware
* HTTP POST handler function to clone a git repository from a provided URL, read the extension manifest,
* and return extension information and path.
*
* @param {Object} request - HTTP Request object, expects a JSON body with a 'url' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
function registerEndpoints(app, jsonParser) {
/**
* HTTP POST handler function to clone a git repository from a provided URL, read the extension manifest,
* and return extension information and path.
*
* @param {Object} request - HTTP Request object, expects a JSON body with a 'url' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/api/extensions/install', jsonParser, async (request, response) => {
if (!request.body.url) {
return response.status(400).send('Bad Request: URL is required in the request body.');
}
router.post('/install', jsonParser, async (request, response) => {
if (!request.body.url) {
return response.status(400).send('Bad Request: URL is required in the request body.');
}
try {
const git = simpleGit();
// make sure the third-party directory exists
if (!fs.existsSync(path.join(DIRECTORIES.extensions, 'third-party'))) {
fs.mkdirSync(path.join(DIRECTORIES.extensions, 'third-party'));
}
const url = request.body.url;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', path.basename(url, '.git'));
if (fs.existsSync(extensionPath)) {
return response.status(409).send(`Directory already exists at ${extensionPath}`);
}
await git.clone(url, extensionPath, { '--depth': 1 });
console.log(`Extension has been cloned at ${extensionPath}`);
const { version, author, display_name } = await getManifest(extensionPath);
return response.send({ version, author, display_name, extensionPath });
} catch (error) {
console.log('Importing custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to pull the latest updates from a git repository
* based on the extension name provided in the request body. It returns the latest commit hash,
* the path of the extension, the status of the repository (whether it's up-to-date or not),
* and the remote URL of the repository.
*
* @param {Object} request - HTTP Request object, expects a JSON body with an 'extensionName' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/api/extensions/update', jsonParser, async (request, response) => {
try {
const git = simpleGit();
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
const { isUpToDate, remoteUrl } = await checkIfRepoIsUpToDate(extensionPath);
const currentBranch = await git.cwd(extensionPath).branch();
if (!isUpToDate) {
await git.cwd(extensionPath).pull('origin', currentBranch.current);
console.log(`Extension has been updated at ${extensionPath}`);
} else {
console.log(`Extension is up to date at ${extensionPath}`);
}
await git.cwd(extensionPath).fetch('origin');
const fullCommitHash = await git.cwd(extensionPath).revparse(['HEAD']);
const shortCommitHash = fullCommitHash.slice(0, 7);
return response.send({ shortCommitHash, extensionPath, isUpToDate, remoteUrl });
} catch (error) {
console.log('Updating custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to get the current git commit hash and branch name for a given extension.
* It checks whether the repository is up-to-date with the remote, and returns the status along with
* the remote URL of the repository.
*
* @param {Object} request - HTTP Request object, expects a JSON body with an 'extensionName' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/api/extensions/version', jsonParser, async (request, response) => {
const git = simpleGit();
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
const currentBranch = await git.cwd(extensionPath).branch();
// get only the working branch
const currentBranchName = currentBranch.current;
await git.cwd(extensionPath).fetch('origin');
const currentCommitHash = await git.cwd(extensionPath).revparse(['HEAD']);
console.log(currentBranch, currentCommitHash);
const { isUpToDate, remoteUrl } = await checkIfRepoIsUpToDate(extensionPath);
return response.send({ currentBranchName, currentCommitHash, isUpToDate, remoteUrl });
} catch (error) {
console.log('Getting extension version failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to delete a git repository based on the extension name provided in the request body.
*
* @param {Object} request - HTTP Request object, expects a JSON body with a 'url' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
app.post('/api/extensions/delete', jsonParser, async (request, response) => {
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
// Sanatize the extension name to prevent directory traversal
const extensionName = sanitize(request.body.extensionName);
try {
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
await fs.promises.rmdir(extensionPath, { recursive: true });
console.log(`Extension has been deleted at ${extensionPath}`);
return response.send(`Extension has been deleted at ${extensionPath}`);
} catch (error) {
console.log('Deleting custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* Discover the extension folders
* If the folder is called third-party, search for subfolders instead
*/
app.get('/api/extensions/discover', jsonParser, function (_, response) {
// get all folders in the extensions folder, except third-party
const extensions = fs
.readdirSync(DIRECTORIES.extensions)
.filter(f => fs.statSync(path.join(DIRECTORIES.extensions, f)).isDirectory())
.filter(f => f !== 'third-party');
// get all folders in the third-party folder, if it exists
// make sure the third-party directory exists
if (!fs.existsSync(path.join(DIRECTORIES.extensions, 'third-party'))) {
return response.send(extensions);
fs.mkdirSync(path.join(DIRECTORIES.extensions, 'third-party'));
}
const thirdPartyExtensions = fs
.readdirSync(path.join(DIRECTORIES.extensions, 'third-party'))
.filter(f => fs.statSync(path.join(DIRECTORIES.extensions, 'third-party', f)).isDirectory());
const url = request.body.url;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', path.basename(url, '.git'));
// add the third-party extensions to the extensions array
extensions.push(...thirdPartyExtensions.map(f => `third-party/${f}`));
console.log(extensions);
if (fs.existsSync(extensionPath)) {
return response.status(409).send(`Directory already exists at ${extensionPath}`);
}
await git.clone(url, extensionPath, { '--depth': 1 });
console.log(`Extension has been cloned at ${extensionPath}`);
const { version, author, display_name } = await getManifest(extensionPath);
return response.send({ version, author, display_name, extensionPath });
} catch (error) {
console.log('Importing custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to pull the latest updates from a git repository
* based on the extension name provided in the request body. It returns the latest commit hash,
* the path of the extension, the status of the repository (whether it's up-to-date or not),
* and the remote URL of the repository.
*
* @param {Object} request - HTTP Request object, expects a JSON body with an 'extensionName' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
router.post('/update', jsonParser, async (request, response) => {
const git = simpleGit();
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
const { isUpToDate, remoteUrl } = await checkIfRepoIsUpToDate(extensionPath);
const currentBranch = await git.cwd(extensionPath).branch();
if (!isUpToDate) {
await git.cwd(extensionPath).pull('origin', currentBranch.current);
console.log(`Extension has been updated at ${extensionPath}`);
} else {
console.log(`Extension is up to date at ${extensionPath}`);
}
await git.cwd(extensionPath).fetch('origin');
const fullCommitHash = await git.cwd(extensionPath).revparse(['HEAD']);
const shortCommitHash = fullCommitHash.slice(0, 7);
return response.send({ shortCommitHash, extensionPath, isUpToDate, remoteUrl });
} catch (error) {
console.log('Updating custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to get the current git commit hash and branch name for a given extension.
* It checks whether the repository is up-to-date with the remote, and returns the status along with
* the remote URL of the repository.
*
* @param {Object} request - HTTP Request object, expects a JSON body with an 'extensionName' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
router.post('/version', jsonParser, async (request, response) => {
const git = simpleGit();
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
try {
const extensionName = request.body.extensionName;
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
const currentBranch = await git.cwd(extensionPath).branch();
// get only the working branch
const currentBranchName = currentBranch.current;
await git.cwd(extensionPath).fetch('origin');
const currentCommitHash = await git.cwd(extensionPath).revparse(['HEAD']);
console.log(currentBranch, currentCommitHash);
const { isUpToDate, remoteUrl } = await checkIfRepoIsUpToDate(extensionPath);
return response.send({ currentBranchName, currentCommitHash, isUpToDate, remoteUrl });
} catch (error) {
console.log('Getting extension version failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* HTTP POST handler function to delete a git repository based on the extension name provided in the request body.
*
* @param {Object} request - HTTP Request object, expects a JSON body with a 'url' property.
* @param {Object} response - HTTP Response object used to respond to the HTTP request.
*
* @returns {void}
*/
router.post('/delete', jsonParser, async (request, response) => {
if (!request.body.extensionName) {
return response.status(400).send('Bad Request: extensionName is required in the request body.');
}
// Sanatize the extension name to prevent directory traversal
const extensionName = sanitize(request.body.extensionName);
try {
const extensionPath = path.join(DIRECTORIES.extensions, 'third-party', extensionName);
if (!fs.existsSync(extensionPath)) {
return response.status(404).send(`Directory does not exist at ${extensionPath}`);
}
await fs.promises.rmdir(extensionPath, { recursive: true });
console.log(`Extension has been deleted at ${extensionPath}`);
return response.send(`Extension has been deleted at ${extensionPath}`);
} catch (error) {
console.log('Deleting custom content failed', error);
return response.status(500).send(`Server Error: ${error.message}`);
}
});
/**
* Discover the extension folders
* If the folder is called third-party, search for subfolders instead
*/
router.get('/discover', jsonParser, function (_, response) {
// get all folders in the extensions folder, except third-party
const extensions = fs
.readdirSync(DIRECTORIES.extensions)
.filter(f => fs.statSync(path.join(DIRECTORIES.extensions, f)).isDirectory())
.filter(f => f !== 'third-party');
// get all folders in the third-party folder, if it exists
if (!fs.existsSync(path.join(DIRECTORIES.extensions, 'third-party'))) {
return response.send(extensions);
});
}
}
module.exports = {
registerEndpoints,
};
const thirdPartyExtensions = fs
.readdirSync(path.join(DIRECTORIES.extensions, 'third-party'))
.filter(f => fs.statSync(path.join(DIRECTORIES.extensions, 'third-party', f)).isDirectory());
// add the third-party extensions to the extensions array
extensions.push(...thirdPartyExtensions.map(f => `third-party/${f}`));
console.log(extensions);
return response.send(extensions);
});
module.exports = { router };

35
src/endpoints/files.js Normal file
View File

@ -0,0 +1,35 @@
const path = require('path');
const writeFileSyncAtomic = require('write-file-atomic').sync;
const express = require('express');
const router = express.Router();
const { checkAssetFileName } = require('./assets');
const { jsonParser } = require('../express-common');
const { DIRECTORIES } = require('../constants');
router.post('/upload', jsonParser, async (request, response) => {
try {
if (!request.body.name) {
return response.status(400).send('No upload name specified');
}
if (!request.body.data) {
return response.status(400).send('No upload data specified');
}
const safeInput = checkAssetFileName(request.body.name);
if (!safeInput) {
return response.status(400).send('Invalid upload name');
}
const pathToUpload = path.join(DIRECTORIES.files, safeInput);
writeFileSyncAtomic(pathToUpload, request.body.data, 'base64');
const url = path.normalize(pathToUpload.replace('public' + path.sep, ''));
return response.send({ path: url });
} catch (error) {
console.log(error);
return response.sendStatus(500);
}
});
module.exports = { router };

399
src/endpoints/horde.js
View File

@ -1,7 +1,9 @@
const fetch = require('node-fetch').default;
const express = require('express');
const AIHorde = require('../ai_horde');
const { getVersion, delay } = require('../util');
const { readSecret, SECRET_KEYS } = require('./secrets');
const { jsonParser } = require('../express-common');
const ANONYMOUS_KEY = '0000000000';
@ -52,221 +54,214 @@ function sanitizeHordeImagePrompt(prompt) {
return prompt;
}
/**
*
* @param {import("express").Express} app
* @param {any} jsonParser
*/
function registerEndpoints(app, jsonParser) {
app.post('/api/horde/generate-text', jsonParser, async (request, response) => {
const router = express.Router();
router.post('/generate-text', jsonParser, async (request, response) => {
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
const url = 'https://horde.koboldai.net/api/v2/generate/text/async';
console.log(request.body);
try {
const result = await fetch(url, {
method: 'POST',
body: JSON.stringify(request.body),
headers: {
'Content-Type': 'application/json',
'apikey': api_key_horde,
'Client-Agent': String(request.header('Client-Agent')),
},
});
if (!result.ok) {
const message = await result.text();
console.log('Horde returned an error:', message);
return response.send({ error: { message } });
}
const data = await result.json();
return response.send(data);
} catch (error) {
console.log(error);
return response.send({ error: true });
}
});
router.post('/sd-samplers', jsonParser, async (_, response) => {
try {
const ai_horde = await getHordeClient();
const samplers = Object.values(ai_horde.ModelGenerationInputStableSamplers);
response.send(samplers);
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
router.post('/sd-models', jsonParser, async (_, response) => {
try {
const ai_horde = await getHordeClient();
const models = await ai_horde.getModels();
response.send(models);
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
router.post('/caption-image', jsonParser, async (request, response) => {
try {
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
const url = 'https://horde.koboldai.net/api/v2/generate/text/async';
const ai_horde = await getHordeClient();
const result = await ai_horde.postAsyncInterrogate({
source_image: request.body.image,
forms: [{ name: AIHorde.ModelInterrogationFormTypes.caption }],
}, { token: api_key_horde });
console.log(request.body);
try {
const result = await fetch(url, {
method: 'POST',
body: JSON.stringify(request.body),
headers: {
'Content-Type': 'application/json',
'apikey': api_key_horde,
'Client-Agent': String(request.header('Client-Agent')),
},
});
if (!result.ok) {
const message = await result.text();
console.log('Horde returned an error:', message);
return response.send({ error: { message } });
}
const data = await result.json();
return response.send(data);
} catch (error) {
console.log(error);
return response.send({ error: true });
}
});
app.post('/api/horde/sd-samplers', jsonParser, async (_, response) => {
try {
const ai_horde = await getHordeClient();
const samplers = Object.values(ai_horde.ModelGenerationInputStableSamplers);
response.send(samplers);
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
app.post('/api/horde/sd-models', jsonParser, async (_, response) => {
try {
const ai_horde = await getHordeClient();
const models = await ai_horde.getModels();
response.send(models);
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
app.post('/api/horde/caption-image', jsonParser, async (request, response) => {
try {
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
const ai_horde = await getHordeClient();
const result = await ai_horde.postAsyncInterrogate({
source_image: request.body.image,
forms: [{ name: AIHorde.ModelInterrogationFormTypes.caption }],
}, { token: api_key_horde });
if (!result.id) {
console.error('Image interrogation request is not satisfyable:', result.message || 'unknown error');
return response.sendStatus(400);
}
const MAX_ATTEMPTS = 200;
const CHECK_INTERVAL = 3000;
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
await delay(CHECK_INTERVAL);
const status = await ai_horde.getInterrogationStatus(result.id);
console.log(status);
if (status.state === AIHorde.HordeAsyncRequestStates.done) {
if (status.forms === undefined) {
console.error('Image interrogation request failed: no forms found.');
return response.sendStatus(500);
}
console.log('Image interrogation result:', status);
const caption = status?.forms[0]?.result?.caption || '';
if (!caption) {
console.error('Image interrogation request failed: no caption found.');
return response.sendStatus(500);
}
return response.send({ caption });
}
if (status.state === AIHorde.HordeAsyncRequestStates.faulted || status.state === AIHorde.HordeAsyncRequestStates.cancelled) {
console.log('Image interrogation request is not successful.');
return response.sendStatus(503);
}
}
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
app.post('/api/horde/user-info', jsonParser, async (_, response) => {
const api_key_horde = readSecret(SECRET_KEYS.HORDE);
if (!api_key_horde) {
return response.send({ anonymous: true });
}
try {
const ai_horde = await getHordeClient();
const user = await ai_horde.findUser({ token: api_key_horde });
return response.send(user);
} catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
app.post('/api/horde/generate-image', jsonParser, async (request, response) => {
if (!request.body.prompt) {
if (!result.id) {
console.error('Image interrogation request is not satisfyable:', result.message || 'unknown error');
return response.sendStatus(400);
}
const MAX_ATTEMPTS = 200;
const CHECK_INTERVAL = 3000;
const PROMPT_THRESHOLD = 5000;
try {
const maxLength = PROMPT_THRESHOLD - String(request.body.negative_prompt).length - 5;
if (String(request.body.prompt).length > maxLength) {
console.log('Stable Horde prompt is too long, truncating...');
request.body.prompt = String(request.body.prompt).substring(0, maxLength);
}
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
await delay(CHECK_INTERVAL);
const status = await ai_horde.getInterrogationStatus(result.id);
console.log(status);
// Sanitize prompt if requested
if (request.body.sanitize) {
const sanitized = sanitizeHordeImagePrompt(request.body.prompt);
if (status.state === AIHorde.HordeAsyncRequestStates.done) {
if (request.body.prompt !== sanitized) {
console.log('Stable Horde prompt was sanitized.');
}
request.body.prompt = sanitized;
}
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
console.log('Stable Horde request:', request.body);
const ai_horde = await getHordeClient();
const generation = await ai_horde.postAsyncImageGenerate(
{
prompt: `${request.body.prompt} ### ${request.body.negative_prompt}`,
params:
{
sampler_name: request.body.sampler,
hires_fix: request.body.enable_hr,
// @ts-ignore - use_gfpgan param is not in the type definition, need to update to new ai_horde @ https://github.com/ZeldaFan0225/ai_horde/blob/main/index.ts
use_gfpgan: request.body.restore_faces,
cfg_scale: request.body.scale,
steps: request.body.steps,
width: request.body.width,
height: request.body.height,
karras: Boolean(request.body.karras),
n: 1,
},
r2: false,
nsfw: request.body.nfsw,
models: [request.body.model],
},
{ token: api_key_horde });
if (!generation.id) {
console.error('Image generation request is not satisfyable:', generation.message || 'unknown error');
return response.sendStatus(400);
}
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
await delay(CHECK_INTERVAL);
const check = await ai_horde.getImageGenerationCheck(generation.id);
console.log(check);
if (check.done) {
const result = await ai_horde.getImageGenerationStatus(generation.id);
if (result.generations === undefined) return response.sendStatus(500);
return response.send(result.generations[0].img);
}
/*
if (!check.is_possible) {
return response.sendStatus(503);
}
*/
if (check.faulted) {
if (status.forms === undefined) {
console.error('Image interrogation request failed: no forms found.');
return response.sendStatus(500);
}
console.log('Image interrogation result:', status);
const caption = status?.forms[0]?.result?.caption || '';
if (!caption) {
console.error('Image interrogation request failed: no caption found.');
return response.sendStatus(500);
}
return response.send({ caption });
}
return response.sendStatus(504);
} catch (error) {
console.error(error);
return response.sendStatus(500);
if (status.state === AIHorde.HordeAsyncRequestStates.faulted || status.state === AIHorde.HordeAsyncRequestStates.cancelled) {
console.log('Image interrogation request is not successful.');
return response.sendStatus(503);
}
}
});
}
module.exports = {
registerEndpoints,
};
} catch (error) {
console.error(error);
response.sendStatus(500);
}
});
router.post('/user-info', jsonParser, async (_, response) => {
const api_key_horde = readSecret(SECRET_KEYS.HORDE);
if (!api_key_horde) {
return response.send({ anonymous: true });
}
try {
const ai_horde = await getHordeClient();
const user = await ai_horde.findUser({ token: api_key_horde });
return response.send(user);
} catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
router.post('/generate-image', jsonParser, async (request, response) => {
if (!request.body.prompt) {
return response.sendStatus(400);
}
const MAX_ATTEMPTS = 200;
const CHECK_INTERVAL = 3000;
const PROMPT_THRESHOLD = 5000;
try {
const maxLength = PROMPT_THRESHOLD - String(request.body.negative_prompt).length - 5;
if (String(request.body.prompt).length > maxLength) {
console.log('Stable Horde prompt is too long, truncating...');
request.body.prompt = String(request.body.prompt).substring(0, maxLength);
}
// Sanitize prompt if requested
if (request.body.sanitize) {
const sanitized = sanitizeHordeImagePrompt(request.body.prompt);
if (request.body.prompt !== sanitized) {
console.log('Stable Horde prompt was sanitized.');
}
request.body.prompt = sanitized;
}
const api_key_horde = readSecret(SECRET_KEYS.HORDE) || ANONYMOUS_KEY;
console.log('Stable Horde request:', request.body);
const ai_horde = await getHordeClient();
const generation = await ai_horde.postAsyncImageGenerate(
{
prompt: `${request.body.prompt} ### ${request.body.negative_prompt}`,
params:
{
sampler_name: request.body.sampler,
hires_fix: request.body.enable_hr,
// @ts-ignore - use_gfpgan param is not in the type definition, need to update to new ai_horde @ https://github.com/ZeldaFan0225/ai_horde/blob/main/index.ts
use_gfpgan: request.body.restore_faces,
cfg_scale: request.body.scale,
steps: request.body.steps,
width: request.body.width,
height: request.body.height,
karras: Boolean(request.body.karras),
n: 1,
},
r2: false,
nsfw: request.body.nfsw,
models: [request.body.model],
},
{ token: api_key_horde });
if (!generation.id) {
console.error('Image generation request is not satisfyable:', generation.message || 'unknown error');
return response.sendStatus(400);
}
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
await delay(CHECK_INTERVAL);
const check = await ai_horde.getImageGenerationCheck(generation.id);
console.log(check);
if (check.done) {
const result = await ai_horde.getImageGenerationStatus(generation.id);
if (result.generations === undefined) return response.sendStatus(500);
return response.send(result.generations[0].img);
}
/*
if (!check.is_possible) {
return response.sendStatus(503);
}
*/
if (check.faulted) {
return response.sendStatus(500);
}
}
return response.sendStatus(504);
} catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
module.exports = { router };

545
src/endpoints/novelai.js
View File

@ -1,8 +1,10 @@
const fetch = require('node-fetch').default;
const express = require('express');
const util = require('util');
const { Readable } = require('stream');
const { readSecret, SECRET_KEYS } = require('./secrets');
const { readAllChunks, extractFileFromZipBuffer } = require('../util');
const { jsonParser } = require('../express-common');
const API_NOVELAI = 'https://api.novelai.net';
@ -60,312 +62,305 @@ function getBadWordsList(model) {
return list.slice();
}
/**
* Registers NovelAI API endpoints.
* @param {import('express').Express} app - Express app
* @param {any} jsonParser - JSON parser middleware
*/
function registerEndpoints(app, jsonParser) {
app.post('/api/novelai/status', jsonParser, async function (req, res) {
if (!req.body) return res.sendStatus(400);
const api_key_novel = readSecret(SECRET_KEYS.NOVEL);
const router = express.Router();
if (!api_key_novel) {
return res.sendStatus(401);
}
router.post('/status', jsonParser, async function (req, res) {
if (!req.body) return res.sendStatus(400);
const api_key_novel = readSecret(SECRET_KEYS.NOVEL);
try {
const response = await fetch(API_NOVELAI + '/user/subscription', {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Authorization': 'Bearer ' + api_key_novel,
},
});
if (!api_key_novel) {
return res.sendStatus(401);
}
if (response.ok) {
const data = await response.json();
return res.send(data);
} else if (response.status == 401) {
console.log('NovelAI Access Token is incorrect.');
return res.send({ error: true });
}
else {
console.log('NovelAI returned an error:', response.statusText);
return res.send({ error: true });
}
} catch (error) {
console.log(error);
return res.send({ error: true });
}
});
app.post('/api/novelai/generate', jsonParser, async function (req, res) {
if (!req.body) return res.sendStatus(400);
const api_key_novel = readSecret(SECRET_KEYS.NOVEL);
if (!api_key_novel) {
return res.sendStatus(401);
}
const controller = new AbortController();
req.socket.removeAllListeners('close');
req.socket.on('close', function () {
controller.abort();
try {
const response = await fetch(API_NOVELAI + '/user/subscription', {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Authorization': 'Bearer ' + api_key_novel,
},
});
const isNewModel = (req.body.model.includes('clio') || req.body.model.includes('kayra'));
const badWordsList = getBadWordsList(req.body.model);
// Add customized bad words for Clio and Kayra
if (isNewModel && Array.isArray(req.body.bad_words_ids)) {
for (const badWord of req.body.bad_words_ids) {
if (Array.isArray(badWord) && badWord.every(x => Number.isInteger(x))) {
badWordsList.push(badWord);
}
}
}
// Remove empty arrays from bad words list
for (const badWord of badWordsList) {
if (badWord.length === 0) {
badWordsList.splice(badWordsList.indexOf(badWord), 1);
}
}
// Add default biases for dinkus and asterism
const logit_bias_exp = isNewModel ? logitBiasExp.slice() : [];
if (Array.isArray(logit_bias_exp) && Array.isArray(req.body.logit_bias_exp)) {
logit_bias_exp.push(...req.body.logit_bias_exp);
}
const data = {
'input': req.body.input,
'model': req.body.model,
'parameters': {
'use_string': req.body.use_string ?? true,
'temperature': req.body.temperature,
'max_length': req.body.max_length,
'min_length': req.body.min_length,
'tail_free_sampling': req.body.tail_free_sampling,
'repetition_penalty': req.body.repetition_penalty,
'repetition_penalty_range': req.body.repetition_penalty_range,
'repetition_penalty_slope': req.body.repetition_penalty_slope,
'repetition_penalty_frequency': req.body.repetition_penalty_frequency,
'repetition_penalty_presence': req.body.repetition_penalty_presence,
'repetition_penalty_whitelist': isNewModel ? repPenaltyAllowList : null,
'top_a': req.body.top_a,
'top_p': req.body.top_p,
'top_k': req.body.top_k,
'typical_p': req.body.typical_p,
'mirostat_lr': req.body.mirostat_lr,
'mirostat_tau': req.body.mirostat_tau,
'cfg_scale': req.body.cfg_scale,
'cfg_uc': req.body.cfg_uc,
'phrase_rep_pen': req.body.phrase_rep_pen,
'stop_sequences': req.body.stop_sequences,
'bad_words_ids': badWordsList.length ? badWordsList : null,
'logit_bias_exp': logit_bias_exp,
'generate_until_sentence': req.body.generate_until_sentence,
'use_cache': req.body.use_cache,
'return_full_text': req.body.return_full_text,
'prefix': req.body.prefix,
'order': req.body.order,
},
};
console.log(util.inspect(data, { depth: 4 }));
const args = {
body: JSON.stringify(data),
headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + api_key_novel },
signal: controller.signal,
};
try {
const url = req.body.streaming ? `${API_NOVELAI}/ai/generate-stream` : `${API_NOVELAI}/ai/generate`;
const response = await fetch(url, { method: 'POST', timeout: 0, ...args });
if (req.body.streaming) {
// Pipe remote SSE stream to Express response
response.body.pipe(res);
req.socket.on('close', function () {
if (response.body instanceof Readable) response.body.destroy(); // Close the remote stream
res.end(); // End the Express response
});
response.body.on('end', function () {
console.log('Streaming request finished');
res.end();
});
} else {
if (!response.ok) {
const text = await response.text();
let message = text;
console.log(`Novel API returned error: ${response.status} ${response.statusText} ${text}`);
try {
const data = JSON.parse(text);
message = data.message;
}
catch {
// ignore
}
return res.status(response.status).send({ error: { message } });
}
const data = await response.json();
console.log(data);
return res.send(data);
}
} catch (error) {
if (response.ok) {
const data = await response.json();
return res.send(data);
} else if (response.status == 401) {
console.log('NovelAI Access Token is incorrect.');
return res.send({ error: true });
}
else {
console.log('NovelAI returned an error:', response.statusText);
return res.send({ error: true });
}
} catch (error) {
console.log(error);
return res.send({ error: true });
}
});
router.post('/generate', jsonParser, async function (req, res) {
if (!req.body) return res.sendStatus(400);
const api_key_novel = readSecret(SECRET_KEYS.NOVEL);
if (!api_key_novel) {
return res.sendStatus(401);
}
const controller = new AbortController();
req.socket.removeAllListeners('close');
req.socket.on('close', function () {
controller.abort();
});
app.post('/api/novelai/generate-image', jsonParser, async (request, response) => {
if (!request.body) {
return response.sendStatus(400);
const isNewModel = (req.body.model.includes('clio') || req.body.model.includes('kayra'));
const badWordsList = getBadWordsList(req.body.model);
// Add customized bad words for Clio and Kayra
if (isNewModel && Array.isArray(req.body.bad_words_ids)) {
for (const badWord of req.body.bad_words_ids) {
if (Array.isArray(badWord) && badWord.every(x => Number.isInteger(x))) {
badWordsList.push(badWord);
}
}
}
// Remove empty arrays from bad words list
for (const badWord of badWordsList) {
if (badWord.length === 0) {
badWordsList.splice(badWordsList.indexOf(badWord), 1);
}
}
// Add default biases for dinkus and asterism
const logit_bias_exp = isNewModel ? logitBiasExp.slice() : [];
if (Array.isArray(logit_bias_exp) && Array.isArray(req.body.logit_bias_exp)) {
logit_bias_exp.push(...req.body.logit_bias_exp);
}
const data = {
'input': req.body.input,
'model': req.body.model,
'parameters': {
'use_string': req.body.use_string ?? true,
'temperature': req.body.temperature,
'max_length': req.body.max_length,
'min_length': req.body.min_length,
'tail_free_sampling': req.body.tail_free_sampling,
'repetition_penalty': req.body.repetition_penalty,
'repetition_penalty_range': req.body.repetition_penalty_range,
'repetition_penalty_slope': req.body.repetition_penalty_slope,
'repetition_penalty_frequency': req.body.repetition_penalty_frequency,
'repetition_penalty_presence': req.body.repetition_penalty_presence,
'repetition_penalty_whitelist': isNewModel ? repPenaltyAllowList : null,
'top_a': req.body.top_a,
'top_p': req.body.top_p,
'top_k': req.body.top_k,
'typical_p': req.body.typical_p,
'mirostat_lr': req.body.mirostat_lr,
'mirostat_tau': req.body.mirostat_tau,
'cfg_scale': req.body.cfg_scale,
'cfg_uc': req.body.cfg_uc,
'phrase_rep_pen': req.body.phrase_rep_pen,
'stop_sequences': req.body.stop_sequences,
'bad_words_ids': badWordsList.length ? badWordsList : null,
'logit_bias_exp': logit_bias_exp,
'generate_until_sentence': req.body.generate_until_sentence,
'use_cache': req.body.use_cache,
'return_full_text': req.body.return_full_text,
'prefix': req.body.prefix,
'order': req.body.order,
},
};
console.log(util.inspect(data, { depth: 4 }));
const args = {
body: JSON.stringify(data),
headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + api_key_novel },
signal: controller.signal,
};
try {
const url = req.body.streaming ? `${API_NOVELAI}/ai/generate-stream` : `${API_NOVELAI}/ai/generate`;
const response = await fetch(url, { method: 'POST', timeout: 0, ...args });
if (req.body.streaming) {
// Pipe remote SSE stream to Express response
response.body.pipe(res);
req.socket.on('close', function () {
if (response.body instanceof Readable) response.body.destroy(); // Close the remote stream
res.end(); // End the Express response
});
response.body.on('end', function () {
console.log('Streaming request finished');
res.end();
});
} else {
if (!response.ok) {
const text = await response.text();
let message = text;
console.log(`Novel API returned error: ${response.status} ${response.statusText} ${text}`);
try {
const data = JSON.parse(text);
message = data.message;
}
catch {
// ignore
}
return res.status(response.status).send({ error: { message } });
}
const data = await response.json();
console.log(data);
return res.send(data);
}
} catch (error) {
return res.send({ error: true });
}
});
router.post('/generate-image', jsonParser, async (request, response) => {
if (!request.body) {
return response.sendStatus(400);
}
const key = readSecret(SECRET_KEYS.NOVEL);
if (!key) {
return response.sendStatus(401);
}
try {
console.log('NAI Diffusion request:', request.body);
const generateUrl = `${API_NOVELAI}/ai/generate-image`;
const generateResult = await fetch(generateUrl, {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
action: 'generate',
input: request.body.prompt,
model: request.body.model ?? 'nai-diffusion',
parameters: {
negative_prompt: request.body.negative_prompt ?? '',
height: request.body.height ?? 512,
width: request.body.width ?? 512,
scale: request.body.scale ?? 9,
seed: Math.floor(Math.random() * 9999999999),
sampler: request.body.sampler ?? 'k_dpmpp_2m',
steps: request.body.steps ?? 28,
n_samples: 1,
// NAI handholding for prompts
ucPreset: 0,
qualityToggle: false,
add_original_image: false,
controlnet_strength: 1,
dynamic_thresholding: false,
legacy: false,
sm: false,
sm_dyn: false,
uncond_scale: 1,
},
}),
});
if (!generateResult.ok) {
const text = await generateResult.text();
console.log('NovelAI returned an error.', generateResult.statusText, text);
return response.sendStatus(500);
}
const key = readSecret(SECRET_KEYS.NOVEL);
const archiveBuffer = await generateResult.arrayBuffer();
const imageBuffer = await extractFileFromZipBuffer(archiveBuffer, '.png');
const originalBase64 = imageBuffer.toString('base64');
if (!key) {
return response.sendStatus(401);
// No upscaling
if (isNaN(request.body.upscale_ratio) || request.body.upscale_ratio <= 1) {
return response.send(originalBase64);
}
try {
console.log('NAI Diffusion request:', request.body);
const generateUrl = `${API_NOVELAI}/ai/generate-image`;
const generateResult = await fetch(generateUrl, {
console.debug('Upscaling image...');
const upscaleUrl = `${API_NOVELAI}/ai/upscale`;
const upscaleResult = await fetch(upscaleUrl, {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
action: 'generate',
input: request.body.prompt,
model: request.body.model ?? 'nai-diffusion',
parameters: {
negative_prompt: request.body.negative_prompt ?? '',
height: request.body.height ?? 512,
width: request.body.width ?? 512,
scale: request.body.scale ?? 9,
seed: Math.floor(Math.random() * 9999999999),
sampler: request.body.sampler ?? 'k_dpmpp_2m',
steps: request.body.steps ?? 28,
n_samples: 1,
// NAI handholding for prompts
ucPreset: 0,
qualityToggle: false,
add_original_image: false,
controlnet_strength: 1,
dynamic_thresholding: false,
legacy: false,
sm: false,
sm_dyn: false,
uncond_scale: 1,
},
image: originalBase64,
height: request.body.height,
width: request.body.width,
scale: request.body.upscale_ratio,
}),
});
if (!generateResult.ok) {
const text = await generateResult.text();
console.log('NovelAI returned an error.', generateResult.statusText, text);
return response.sendStatus(500);
if (!upscaleResult.ok) {
throw new Error('NovelAI returned an error.');
}
const archiveBuffer = await generateResult.arrayBuffer();
const imageBuffer = await extractFileFromZipBuffer(archiveBuffer, '.png');
const originalBase64 = imageBuffer.toString('base64');
const upscaledArchiveBuffer = await upscaleResult.arrayBuffer();
const upscaledImageBuffer = await extractFileFromZipBuffer(upscaledArchiveBuffer, '.png');
const upscaledBase64 = upscaledImageBuffer.toString('base64');
// No upscaling
if (isNaN(request.body.upscale_ratio) || request.body.upscale_ratio <= 1) {
return response.send(originalBase64);
}
try {
console.debug('Upscaling image...');
const upscaleUrl = `${API_NOVELAI}/ai/upscale`;
const upscaleResult = await fetch(upscaleUrl, {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
image: originalBase64,
height: request.body.height,
width: request.body.width,
scale: request.body.upscale_ratio,
}),
});
if (!upscaleResult.ok) {
throw new Error('NovelAI returned an error.');
}
const upscaledArchiveBuffer = await upscaleResult.arrayBuffer();
const upscaledImageBuffer = await extractFileFromZipBuffer(upscaledArchiveBuffer, '.png');
const upscaledBase64 = upscaledImageBuffer.toString('base64');
return response.send(upscaledBase64);
} catch (error) {
console.warn('NovelAI generated an image, but upscaling failed. Returning original image.');
return response.send(originalBase64);
}
return response.send(upscaledBase64);
} catch (error) {
console.log(error);
return response.sendStatus(500);
console.warn('NovelAI generated an image, but upscaling failed. Returning original image.');
return response.send(originalBase64);
}
});
} catch (error) {
console.log(error);
return response.sendStatus(500);
}
});
app.post('/api/novelai/generate-voice', jsonParser, async (request, response) => {
const token = readSecret(SECRET_KEYS.NOVEL);
router.post('/generate-voice', jsonParser, async (request, response) => {
const token = readSecret(SECRET_KEYS.NOVEL);
if (!token) {
return response.sendStatus(401);
if (!token) {
return response.sendStatus(401);
}
const text = request.body.text;
const voice = request.body.voice;
if (!text || !voice) {
return response.sendStatus(400);
}
try {
const url = `${API_NOVELAI}/ai/generate-voice?text=${encodeURIComponent(text)}&voice=-1&seed=${encodeURIComponent(voice)}&opus=false&version=v2`;
const result = await fetch(url, {
method: 'GET',
headers: {
'Authorization': `Bearer ${token}`,
'Accept': 'audio/mpeg',
},
timeout: 0,
});
if (!result.ok) {
return response.sendStatus(result.status);
}
const text = request.body.text;
const voice = request.body.voice;
const chunks = await readAllChunks(result.body);
const buffer = Buffer.concat(chunks);
response.setHeader('Content-Type', 'audio/mpeg');
return response.send(buffer);
}
catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
if (!text || !voice) {
return response.sendStatus(400);
}
try {
const url = `${API_NOVELAI}/ai/generate-voice?text=${encodeURIComponent(text)}&voice=-1&seed=${encodeURIComponent(voice)}&opus=false&version=v2`;
const result = await fetch(url, {
method: 'GET',
headers: {
'Authorization': `Bearer ${token}`,
'Accept': 'audio/mpeg',
},
timeout: 0,
});
if (!result.ok) {
return response.sendStatus(result.status);
}
const chunks = await readAllChunks(result.body);
const buffer = Buffer.concat(chunks);
response.setHeader('Content-Type', 'audio/mpeg');
return response.send(buffer);
}
catch (error) {
console.error(error);
return response.sendStatus(500);
}
});
}
module.exports = {
registerEndpoints,
};
module.exports = { router };

372
src/endpoints/openai.js
View File

@ -1,216 +1,210 @@
const { readSecret, SECRET_KEYS } = require('./secrets');
const fetch = require('node-fetch').default;
const express = require('express');
const FormData = require('form-data');
const fs = require('fs');
const { jsonParser, urlencodedParser } = require('../express-common');
/**
* Registers the OpenAI endpoints.
* @param {import("express").Express} app Express app
* @param {any} jsonParser JSON parser
* @param {any} urlencodedParser Form data parser
*/
function registerEndpoints(app, jsonParser, urlencodedParser) {
app.post('/api/openai/caption-image', jsonParser, async (request, response) => {
try {
let key = '';
const router = express.Router();
if (request.body.api === 'openai') {
key = readSecret(SECRET_KEYS.OPENAI);
}
router.post('/caption-image', jsonParser, async (request, response) => {
try {
let key = '';
if (request.body.api === 'openrouter') {
key = readSecret(SECRET_KEYS.OPENROUTER);
}
if (request.body.api === 'openai') {
key = readSecret(SECRET_KEYS.OPENAI);
}
if (!key) {
console.log('No key found for API', request.body.api);
return response.sendStatus(401);
}
if (request.body.api === 'openrouter') {
key = readSecret(SECRET_KEYS.OPENROUTER);
}
const body = {
model: request.body.model,
messages: [
{
role: 'user',
content: [
{ type: 'text', text: request.body.prompt },
{ type: 'image_url', image_url: { 'url': request.body.image } },
],
},
],
max_tokens: 500,
};
if (!key) {
console.log('No key found for API', request.body.api);
return response.sendStatus(401);
}
console.log('Multimodal captioning request', body);
let apiUrl = '';
let headers = {};
if (request.body.api === 'openrouter') {
apiUrl = 'https://openrouter.ai/api/v1/chat/completions';
headers['HTTP-Referer'] = request.headers.referer;
}
if (request.body.api === 'openai') {
apiUrl = 'https://api.openai.com/v1/chat/completions';
}
const result = await fetch(apiUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${key}`,
...headers,
const body = {
model: request.body.model,
messages: [
{
role: 'user',
content: [
{ type: 'text', text: request.body.prompt },
{ type: 'image_url', image_url: { 'url': request.body.image } },
],
},
body: JSON.stringify(body),
timeout: 0,
});
],
max_tokens: 500,
};
if (!result.ok) {
const text = await result.text();
console.log('Multimodal captioning request failed', result.statusText, text);
return response.status(500).send(text);
}
console.log('Multimodal captioning request', body);
const data = await result.json();
console.log('Multimodal captioning response', data);
const caption = data?.choices[0]?.message?.content;
let apiUrl = '';
let headers = {};
if (!caption) {
return response.status(500).send('No caption found');
}
return response.json({ caption });
if (request.body.api === 'openrouter') {
apiUrl = 'https://openrouter.ai/api/v1/chat/completions';
headers['HTTP-Referer'] = request.headers.referer;
}
catch (error) {
console.error(error);
response.status(500).send('Internal server error');
if (request.body.api === 'openai') {
apiUrl = 'https://api.openai.com/v1/chat/completions';
}
});
app.post('/api/openai/transcribe-audio', urlencodedParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.OPENAI);
const result = await fetch(apiUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${key}`,
...headers,
},
body: JSON.stringify(body),
timeout: 0,
});
if (!key) {
console.log('No OpenAI key found');
return response.sendStatus(401);
}
if (!request.file) {
console.log('No audio file found');
return response.sendStatus(400);
}
const formData = new FormData();
console.log('Processing audio file', request.file.path);
formData.append('file', fs.createReadStream(request.file.path), { filename: 'audio.wav', contentType: 'audio/wav' });
formData.append('model', request.body.model);
if (request.body.language) {
formData.append('language', request.body.language);
}
const result = await fetch('https://api.openai.com/v1/audio/transcriptions', {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
...formData.getHeaders(),
},
body: formData,
});
if (!result.ok) {
const text = await result.text();
console.log('OpenAI request failed', result.statusText, text);
return response.status(500).send(text);
}
fs.rmSync(request.file.path);
const data = await result.json();
console.log('OpenAI transcription response', data);
return response.json(data);
} catch (error) {
console.error('OpenAI transcription failed', error);
response.status(500).send('Internal server error');
if (!result.ok) {
const text = await result.text();
console.log('Multimodal captioning request failed', result.statusText, text);
return response.status(500).send(text);
}
});
app.post('/api/openai/generate-voice', jsonParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.OPENAI);
const data = await result.json();
console.log('Multimodal captioning response', data);
const caption = data?.choices[0]?.message?.content;
if (!key) {
console.log('No OpenAI key found');
return response.sendStatus(401);
}
const result = await fetch('https://api.openai.com/v1/audio/speech', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${key}`,
},
body: JSON.stringify({
input: request.body.text,
response_format: 'mp3',
voice: request.body.voice ?? 'alloy',
speed: request.body.speed ?? 1,
model: request.body.model ?? 'tts-1',
}),
});
if (!result.ok) {
const text = await result.text();
console.log('OpenAI request failed', result.statusText, text);
return response.status(500).send(text);
}
const buffer = await result.arrayBuffer();
response.setHeader('Content-Type', 'audio/mpeg');
return response.send(Buffer.from(buffer));
} catch (error) {
console.error('OpenAI TTS generation failed', error);
response.status(500).send('Internal server error');
if (!caption) {
return response.status(500).send('No caption found');
}
});
app.post('/api/openai/generate-image', jsonParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.OPENAI);
return response.json({ caption });
}
catch (error) {
console.error(error);
response.status(500).send('Internal server error');
}
});
if (!key) {
console.log('No OpenAI key found');
return response.sendStatus(401);
}
router.post('/transcribe-audio', urlencodedParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.OPENAI);
console.log('OpenAI request', request.body);
const result = await fetch('https://api.openai.com/v1/images/generations', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${key}`,
},
body: JSON.stringify(request.body),
timeout: 0,
});
if (!result.ok) {
const text = await result.text();
console.log('OpenAI request failed', result.statusText, text);
return response.status(500).send(text);
}
const data = await result.json();
return response.json(data);
} catch (error) {
console.error(error);
response.status(500).send('Internal server error');
if (!key) {
console.log('No OpenAI key found');
return response.sendStatus(401);
}
});
}
module.exports = {
registerEndpoints,
};
if (!request.file) {
console.log('No audio file found');
return response.sendStatus(400);
}
const formData = new FormData();
console.log('Processing audio file', request.file.path);
formData.append('file', fs.createReadStream(request.file.path), { filename: 'audio.wav', contentType: 'audio/wav' });
formData.append('model', request.body.model);
if (request.body.language) {
formData.append('language', request.body.language);
}
const result = await fetch('https://api.openai.com/v1/audio/transcriptions', {
method: 'POST',
headers: {
'Authorization': `Bearer ${key}`,
...formData.getHeaders(),
},
body: formData,
});
if (!result.ok) {
const text = await result.text();
console.log('OpenAI request failed', result.statusText, text);
return response.status(500).send(text);
}
fs.rmSync(request.file.path);
const data = await result.json();
console.log('OpenAI transcription response', data);
return response.json(data);
} catch (error) {
console.error('OpenAI transcription failed', error);
response.status(500).send('Internal server error');
}
});
router.post('/generate-voice', jsonParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.OPENAI);
if (!key) {
console.log('No OpenAI key found');
return response.sendStatus(401);
}
const result = await fetch('https://api.openai.com/v1/audio/speech', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${key}`,
},
body: JSON.stringify({
input: request.body.text,
response_format: 'mp3',
voice: request.body.voice ?? 'alloy',
speed: request.body.speed ?? 1,
model: request.body.model ?? 'tts-1',
}),
});
if (!result.ok) {
const text = await result.text();
console.log('OpenAI request failed', result.statusText, text);
return response.status(500).send(text);
}
const buffer = await result.arrayBuffer();
response.setHeader('Content-Type', 'audio/mpeg');
return response.send(Buffer.from(buffer));
} catch (error) {
console.error('OpenAI TTS generation failed', error);
response.status(500).send('Internal server error');
}
});
router.post('/generate-image', jsonParser, async (request, response) => {
try {
const key = readSecret(SECRET_KEYS.OPENAI);
if (!key) {
console.log('No OpenAI key found');
return response.sendStatus(401);
}
console.log('OpenAI request', request.body);
const result = await fetch('https://api.openai.com/v1/images/generations', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${key}`,
},
body: JSON.stringify(request.body),
timeout: 0,
});
if (!result.ok) {
const text = await result.text();
console.log('OpenAI request failed', result.statusText, text);
return response.status(500).send(text);
}
const data = await result.json();
return response.json(data);
} catch (error) {
console.error(error);
response.status(500).send('Internal server error');
}
});
module.exports = { router };