mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-06-05 21:59:27 +02:00
Merge branch 'staging' into feature/exorcism
This commit is contained in:
Cohee
382
server.js
382
server.js
@ -1,10 +1,77 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// native node modules
|
||||
const child_process = require('child_process')
|
||||
const crypto = require('crypto');
|
||||
const fs = require('fs');
|
||||
const http = require("http");
|
||||
const https = require('https');
|
||||
const path = require('path');
|
||||
const readline = require('readline');
|
||||
const util = require('util');
|
||||
const { Readable } = require('stream');
|
||||
const { finished } = require('stream/promises');
|
||||
const { TextEncoder, TextDecoder } = require('util');
|
||||
|
||||
// cli/fs related library imports
|
||||
const commandExistsSync = require('command-exists').sync;
|
||||
const open = require('open');
|
||||
const sanitize = require('sanitize-filename');
|
||||
const simpleGit = require('simple-git');
|
||||
const writeFileAtomicSync = require('write-file-atomic').sync;
|
||||
const yargs = require('yargs/yargs');
|
||||
const { hideBin } = require('yargs/helpers');
|
||||
|
||||
// express/server related library imports
|
||||
const cors = require('cors');
|
||||
const doubleCsrf = require('csrf-csrf').doubleCsrf;
|
||||
const express = require('express');
|
||||
const compression = require('compression');
|
||||
const cookieParser = require('cookie-parser');
|
||||
const multer = require("multer");
|
||||
const responseTime = require('response-time');
|
||||
|
||||
// net related library imports
|
||||
const axios = require('axios');
|
||||
const DeviceDetector = require("device-detector-js");
|
||||
const fetch = require('node-fetch').default;
|
||||
const ipaddr = require('ipaddr.js');
|
||||
const ipMatching = require('ip-matching');
|
||||
const json5 = require('json5');
|
||||
const RESTClient = require('node-rest-client').Client;
|
||||
const WebSocket = require('ws');
|
||||
|
||||
// image processing related library imports
|
||||
const exif = require('piexifjs');
|
||||
const encode = require('png-chunks-encode');
|
||||
const extract = require('png-chunks-extract');
|
||||
const jimp = require('jimp');
|
||||
const mime = require('mime-types');
|
||||
const PNGtext = require('png-chunk-text');
|
||||
const webp = require('webp-converter');
|
||||
const yauzl = require('yauzl');
|
||||
|
||||
// tokenizing related library imports
|
||||
const { SentencePieceProcessor } = require("@agnai/sentencepiece-js");
|
||||
const tiktoken = require('@dqbd/tiktoken');
|
||||
const { Tokenizer } = require('@agnai/web-tokenizers');
|
||||
|
||||
// misc/other imports
|
||||
const _ = require('lodash');
|
||||
const { generateRequestUrl, normaliseResponse } = require('google-translate-api-browser');
|
||||
|
||||
// Create files before running anything else
|
||||
createDefaultFiles();
|
||||
|
||||
// local library imports
|
||||
const AIHorde = require("./src/horde");
|
||||
const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware');
|
||||
const characterCardParser = require('./src/character-card-parser.js');
|
||||
const contentManager = require('./src/content-manager');
|
||||
const novelai = require('./src/novelai');
|
||||
const statsHelpers = require('./statsHelpers.js');
|
||||
|
||||
function createDefaultFiles() {
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const files = {
|
||||
settings: 'public/settings.json',
|
||||
bg_load: 'public/css/bg_load.css',
|
||||
@ -24,13 +91,9 @@ function createDefaultFiles() {
|
||||
}
|
||||
}
|
||||
|
||||
const yargs = require('yargs/yargs');
|
||||
const { hideBin } = require('yargs/helpers');
|
||||
const net = require("net");
|
||||
// work around a node v20 bug: https://github.com/nodejs/node/issues/47822#issuecomment-1564708870
|
||||
if (net.setDefaultAutoSelectFamily) {
|
||||
net.setDefaultAutoSelectFamily(false);
|
||||
}
|
||||
// @ts-ignore work around a node v20 bug: https://github.com/nodejs/node/issues/47822#issuecomment-1564708870
|
||||
if (net.setDefaultAutoSelectFamily) net.setDefaultAutoSelectFamily(false);
|
||||
|
||||
const cliArguments = yargs(hideBin(process.argv))
|
||||
.option('disableCsrf', {
|
||||
@ -49,57 +112,21 @@ const cliArguments = yargs(hideBin(process.argv))
|
||||
type: 'string',
|
||||
default: 'certs/privkey.pem',
|
||||
describe: 'Path to your private key file.'
|
||||
}).argv;
|
||||
}).parseSync();
|
||||
|
||||
// change all relative paths
|
||||
const path = require('path');
|
||||
const directory = process.pkg ? path.dirname(process.execPath) : __dirname;
|
||||
console.log(process.pkg ? 'Running from binary' : 'Running from source');
|
||||
const directory = process['pkg'] ? path.dirname(process.execPath) : __dirname;
|
||||
console.log(process['pkg'] ? 'Running from binary' : 'Running from source');
|
||||
process.chdir(directory);
|
||||
|
||||
const express = require('express');
|
||||
const compression = require('compression');
|
||||
const app = express();
|
||||
const responseTime = require('response-time');
|
||||
const simpleGit = require('simple-git');
|
||||
|
||||
app.use(compression());
|
||||
app.use(responseTime());
|
||||
|
||||
const fs = require('fs');
|
||||
const writeFileAtomicSync = require('write-file-atomic').sync;
|
||||
const readline = require('readline');
|
||||
const open = require('open');
|
||||
|
||||
const multer = require("multer");
|
||||
const http = require("http");
|
||||
const https = require('https');
|
||||
const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware');
|
||||
const contentManager = require('./src/content-manager');
|
||||
const extract = require('png-chunks-extract');
|
||||
const encode = require('png-chunks-encode');
|
||||
const PNGtext = require('png-chunk-text');
|
||||
|
||||
const jimp = require('jimp');
|
||||
const sanitize = require('sanitize-filename');
|
||||
const mime = require('mime-types');
|
||||
|
||||
const cookieParser = require('cookie-parser');
|
||||
const crypto = require('crypto');
|
||||
const ipaddr = require('ipaddr.js');
|
||||
const json5 = require('json5');
|
||||
|
||||
const exif = require('piexifjs');
|
||||
const webp = require('webp-converter');
|
||||
const DeviceDetector = require("device-detector-js");
|
||||
const { TextEncoder, TextDecoder } = require('util');
|
||||
const utf8Encode = new TextEncoder();
|
||||
const commandExistsSync = require('command-exists').sync;
|
||||
|
||||
// impoort from statsHelpers.js
|
||||
const statsHelpers = require('./statsHelpers.js');
|
||||
|
||||
const characterCardParser = require('./src/character-card-parser.js');
|
||||
const config = require(path.join(process.cwd(), './config.conf'));
|
||||
|
||||
const server_port = process.env.SILLY_TAVERN_PORT || config.port;
|
||||
@ -120,25 +147,16 @@ const enableExtensions = config.enableExtensions;
|
||||
const listen = config.listen;
|
||||
const allowKeysExposure = config.allowKeysExposure;
|
||||
|
||||
const axios = require('axios');
|
||||
const tiktoken = require('@dqbd/tiktoken');
|
||||
const WebSocket = require('ws');
|
||||
|
||||
function getHordeClient() {
|
||||
const AIHorde = require("./src/horde");
|
||||
const ai_horde = new AIHorde({
|
||||
client_agent: getVersion()?.agent || 'SillyTavern:UNKNOWN:Cohee#1207',
|
||||
});
|
||||
return ai_horde;
|
||||
}
|
||||
|
||||
const ipMatching = require('ip-matching');
|
||||
const yauzl = require('yauzl');
|
||||
const restClient = new RESTClient();
|
||||
|
||||
const Client = require('node-rest-client').Client;
|
||||
const client = new Client();
|
||||
|
||||
client.on('error', (err) => {
|
||||
restClient.on('error', (err) => {
|
||||
console.error('An error occurred:', err);
|
||||
});
|
||||
|
||||
@ -183,8 +201,6 @@ function get_mancer_headers() {
|
||||
|
||||
const delay = ms => new Promise(resolve => setTimeout(resolve, ms))
|
||||
|
||||
const { SentencePieceProcessor } = require("@agnai/sentencepiece-js");
|
||||
const { Tokenizer } = require('@agnai/web-tokenizers');
|
||||
const CHARS_PER_TOKEN = 3.35;
|
||||
|
||||
let spp_llama;
|
||||
@ -333,8 +349,6 @@ const directories = {
|
||||
|
||||
// CSRF Protection //
|
||||
if (cliArguments.disableCsrf === false) {
|
||||
const doubleCsrf = require('csrf-csrf').doubleCsrf;
|
||||
|
||||
const CSRF_SECRET = crypto.randomBytes(8).toString('hex');
|
||||
const COOKIES_SECRET = crypto.randomBytes(8).toString('hex');
|
||||
|
||||
@ -352,7 +366,7 @@ if (cliArguments.disableCsrf === false) {
|
||||
|
||||
app.get("/csrf-token", (req, res) => {
|
||||
res.json({
|
||||
"token": generateToken(res)
|
||||
"token": generateToken(res, req)
|
||||
});
|
||||
});
|
||||
|
||||
@ -368,7 +382,6 @@ if (cliArguments.disableCsrf === false) {
|
||||
}
|
||||
|
||||
// CORS Settings //
|
||||
const cors = require('cors');
|
||||
const CORS = cors({
|
||||
origin: 'null',
|
||||
methods: ['OPTIONS']
|
||||
@ -385,7 +398,7 @@ function getIpFromRequest(req) {
|
||||
let clientIp = req.connection.remoteAddress;
|
||||
let ip = ipaddr.parse(clientIp);
|
||||
// Check if the IP address is IPv4-mapped IPv6 address
|
||||
if (ip.kind() === 'ipv6' && ip.isIPv4MappedAddress()) {
|
||||
if (ip.kind() === 'ipv6' && ip instanceof ipaddr.IPv6 && ip.isIPv4MappedAddress()) {
|
||||
const ipv4 = ip.toIPv4Address().toString();
|
||||
clientIp = ipv4;
|
||||
} else {
|
||||
@ -422,7 +435,7 @@ app.use(function (req, res, next) {
|
||||
});
|
||||
|
||||
|
||||
app.use(express.static(process.cwd() + "/public", { refresh: true }));
|
||||
app.use(express.static(process.cwd() + "/public", {}));
|
||||
|
||||
app.use('/backgrounds', (req, res) => {
|
||||
const filePath = decodeURIComponent(path.join(process.cwd(), 'public/backgrounds', req.url.replace(/%20/g, ' ')));
|
||||
@ -456,7 +469,7 @@ app.get("/notes/*", function (request, response) {
|
||||
app.get('/deviceinfo', function (request, response) {
|
||||
const userAgent = request.header('user-agent');
|
||||
const deviceDetector = new DeviceDetector();
|
||||
const deviceInfo = deviceDetector.parse(userAgent);
|
||||
const deviceInfo = deviceDetector.parse(userAgent || "");
|
||||
return response.send(deviceInfo);
|
||||
});
|
||||
app.get('/version', function (_, response) {
|
||||
@ -465,7 +478,7 @@ app.get('/version', function (_, response) {
|
||||
})
|
||||
|
||||
//**************Kobold api
|
||||
app.post("/generate", jsonParser, async function (request, response_generate = response) {
|
||||
app.post("/generate", jsonParser, async function (request, response_generate) {
|
||||
if (!request.body) return response_generate.sendStatus(400);
|
||||
|
||||
const request_prompt = request.body.prompt;
|
||||
@ -536,10 +549,9 @@ app.post("/generate", jsonParser, async function (request, response_generate = r
|
||||
|
||||
const MAX_RETRIES = 50;
|
||||
const delayAmount = 2500;
|
||||
let fetch, url, response;
|
||||
let url, response;
|
||||
for (let i = 0; i < MAX_RETRIES; i++) {
|
||||
try {
|
||||
fetch = require('node-fetch').default;
|
||||
url = request.body.streaming ? `${api_server}/extra/generate/stream` : `${api_server}/v1/generate`;
|
||||
response = await fetch(url, { method: 'POST', timeout: 0, ...args });
|
||||
|
||||
@ -596,7 +608,7 @@ app.post("/generate", jsonParser, async function (request, response_generate = r
|
||||
});
|
||||
|
||||
//************** Text generation web UI
|
||||
app.post("/generate_textgenerationwebui", jsonParser, async function (request, response_generate = response) {
|
||||
app.post("/generate_textgenerationwebui", jsonParser, async function (request, response_generate) {
|
||||
if (!request.body) return response_generate.sendStatus(400);
|
||||
|
||||
console.log(request.body);
|
||||
@ -610,6 +622,10 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
|
||||
});
|
||||
|
||||
if (request.header('X-Response-Streaming')) {
|
||||
const streamingUrlHeader = request.header('X-Streaming-URL');
|
||||
if (streamingUrlHeader === undefined) return response_generate.sendStatus(400);
|
||||
const streamingUrl = streamingUrlHeader.replace("localhost", "127.0.0.1");
|
||||
|
||||
response_generate.writeHead(200, {
|
||||
'Content-Type': 'text/plain;charset=utf-8',
|
||||
'Transfer-Encoding': 'chunked',
|
||||
@ -617,7 +633,6 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
|
||||
});
|
||||
|
||||
async function* readWebsocket() {
|
||||
const streamingUrl = request.header('X-Streaming-URL').replace("localhost", "127.0.0.1");
|
||||
const websocket = new WebSocket(streamingUrl);
|
||||
|
||||
websocket.on('open', async function () {
|
||||
@ -648,7 +663,7 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
|
||||
websocket.once('error', reject);
|
||||
websocket.once('message', (data, isBinary) => {
|
||||
websocket.removeListener('error', reject);
|
||||
resolve(data, isBinary);
|
||||
resolve(data);
|
||||
});
|
||||
});
|
||||
} catch (err) {
|
||||
@ -711,7 +726,7 @@ app.post("/generate_textgenerationwebui", jsonParser, async function (request, r
|
||||
console.log("Endpoint response:", data);
|
||||
return response_generate.send(data);
|
||||
} catch (error) {
|
||||
retval = { error: true, status: error.status, response: error.statusText };
|
||||
let retval = { error: true, status: error.status, response: error.statusText };
|
||||
console.log("Endpoint error:", error);
|
||||
try {
|
||||
retval.response = await error.json();
|
||||
@ -837,12 +852,12 @@ function getVersion() {
|
||||
try {
|
||||
const pkgJson = require('./package.json');
|
||||
pkgVersion = pkgJson.version;
|
||||
if (!process.pkg && commandExistsSync('git')) {
|
||||
gitRevision = require('child_process')
|
||||
if (!process['pkg'] && commandExistsSync('git')) {
|
||||
gitRevision = child_process
|
||||
.execSync('git rev-parse --short HEAD', { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'ignore'] })
|
||||
.toString().trim();
|
||||
|
||||
gitBranch = require('child_process')
|
||||
gitBranch = child_process
|
||||
.execSync('git rev-parse --abbrev-ref HEAD', { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'ignore'] })
|
||||
.toString().trim();
|
||||
}
|
||||
@ -887,13 +902,11 @@ function convertToV2(char) {
|
||||
|
||||
|
||||
function unsetFavFlag(char) {
|
||||
const _ = require('lodash');
|
||||
_.set(char, 'fav', false);
|
||||
_.set(char, 'data.extensions.fav', false);
|
||||
}
|
||||
|
||||
function readFromV2(char) {
|
||||
const _ = require('lodash');
|
||||
if (_.isUndefined(char.data)) {
|
||||
console.warn('Spec v2 data missing');
|
||||
return char;
|
||||
@ -948,16 +961,14 @@ function readFromV2(char) {
|
||||
//***************** Main functions
|
||||
function charaFormatData(data) {
|
||||
// This is supposed to save all the foreign keys that ST doesn't care about
|
||||
const _ = require('lodash');
|
||||
const char = tryParse(data.json_data) || {};
|
||||
|
||||
// This function uses _.cond() to create a series of conditional checks that return the desired output based on the input data.
|
||||
// It checks if data.alternate_greetings is an array, a string, or neither, and acts accordingly.
|
||||
const getAlternateGreetings = data => _.cond([
|
||||
[d => Array.isArray(d.alternate_greetings), d => d.alternate_greetings],
|
||||
[d => typeof d.alternate_greetings === 'string', d => [d.alternate_greetings]],
|
||||
[_.stubTrue, _.constant([])]
|
||||
])(data);
|
||||
// Checks if data.alternate_greetings is an array, a string, or neither, and acts accordingly. (expected to be an array of strings)
|
||||
const getAlternateGreetings = data => {
|
||||
if (Array.isArray(data.alternate_greetings)) return data.alternate_greetings
|
||||
if (typeof data.alternate_greetings === 'string') return [data.alternate_greetings]
|
||||
return []
|
||||
}
|
||||
|
||||
// Spec V1 fields
|
||||
_.set(char, 'name', data.ch_name);
|
||||
@ -1087,9 +1098,10 @@ app.post("/renamecharacter", jsonParser, async function (request, response) {
|
||||
const newChatsPath = path.join(chatsPath, newInternalName);
|
||||
|
||||
try {
|
||||
const _ = require('lodash');
|
||||
// Read old file, replace name int it
|
||||
const rawOldData = await charaRead(oldAvatarPath);
|
||||
if (rawOldData === false || rawOldData === undefined) throw new Error("Failed to read character file");
|
||||
|
||||
const oldData = getCharaCardV2(json5.parse(rawOldData));
|
||||
_.set(oldData, 'data.name', newName);
|
||||
_.set(oldData, 'name', newName);
|
||||
@ -1178,26 +1190,22 @@ app.post("/editcharacterattribute", jsonParser, async function (request, respons
|
||||
|
||||
try {
|
||||
const avatarPath = path.join(charactersPath, request.body.avatar_url);
|
||||
charaRead(avatarPath).then((char) => {
|
||||
char = JSON.parse(char);
|
||||
//check if the field exists
|
||||
if (char[request.body.field] === undefined && char.data[request.body.field] === undefined) {
|
||||
console.error('Error: invalid field.');
|
||||
response.status(400).send('Error: invalid field.');
|
||||
return;
|
||||
}
|
||||
char[request.body.field] = request.body.value;
|
||||
char.data[request.body.field] = request.body.value;
|
||||
char = JSON.stringify(char);
|
||||
return { char };
|
||||
}).then(({ char }) => {
|
||||
charaWrite(avatarPath, char, (request.body.avatar_url).replace('.png', ''), response, 'Character saved');
|
||||
}).catch((err) => {
|
||||
console.error('An error occured, character edit invalidated.', err);
|
||||
});
|
||||
}
|
||||
catch {
|
||||
console.error('An error occured, character edit invalidated.');
|
||||
let charJSON = await charaRead(avatarPath);
|
||||
if (typeof charJSON !== 'string') throw new Error("Failed to read character file");
|
||||
|
||||
let char = JSON.parse(charJSON)
|
||||
//check if the field exists
|
||||
if (char[request.body.field] === undefined && char.data[request.body.field] === undefined) {
|
||||
console.error('Error: invalid field.');
|
||||
response.status(400).send('Error: invalid field.');
|
||||
return;
|
||||
}
|
||||
char[request.body.field] = request.body.value;
|
||||
char.data[request.body.field] = request.body.value;
|
||||
let newCharJSON = JSON.stringify(char);
|
||||
await charaWrite(avatarPath, newCharJSON, (request.body.avatar_url).replace('.png', ''), response, 'Character saved');
|
||||
} catch (err) {
|
||||
console.error('An error occured, character edit invalidated.', err);
|
||||
}
|
||||
});
|
||||
|
||||
@ -1237,6 +1245,10 @@ app.post("/deletecharacter", jsonParser, async function (request, response) {
|
||||
return response.sendStatus(200);
|
||||
});
|
||||
|
||||
/**
|
||||
* @param {express.Response | undefined} response
|
||||
* @param {{file_name: string} | string} mes
|
||||
*/
|
||||
async function charaWrite(img_url, data, target_img, response = undefined, mes = 'ok', crop = undefined) {
|
||||
try {
|
||||
// Read the image, resize, and save it as a PNG into the buffer
|
||||
@ -1841,8 +1853,7 @@ function getImages(path) {
|
||||
|
||||
//***********Novel.ai API
|
||||
|
||||
app.post("/getstatus_novelai", jsonParser, function (request, response_getstatus_novel = response) {
|
||||
|
||||
app.post("/getstatus_novelai", jsonParser, async function (request, response_getstatus_novel) {
|
||||
if (!request.body) return response_getstatus_novel.sendStatus(400);
|
||||
const api_key_novel = readSecret(SECRET_KEYS.NOVEL);
|
||||
|
||||
@ -1850,27 +1861,30 @@ app.post("/getstatus_novelai", jsonParser, function (request, response_getstatus
|
||||
return response_getstatus_novel.sendStatus(401);
|
||||
}
|
||||
|
||||
var data = {};
|
||||
var args = {
|
||||
data: data,
|
||||
headers: { "Content-Type": "application/json", "Authorization": "Bearer " + api_key_novel }
|
||||
};
|
||||
client.get(api_novelai + "/user/subscription", args, function (data, response) {
|
||||
if (response.statusCode == 200) {
|
||||
//console.log(data);
|
||||
response_getstatus_novel.send(data);//data);
|
||||
try {
|
||||
const response = await fetch(api_novelai + "/user/subscription", {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': "Bearer " + api_key_novel,
|
||||
},
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
return response_getstatus_novel.send(data);
|
||||
} else if (response.status == 401) {
|
||||
console.log('NovelAI Access Token is incorrect.');
|
||||
return response_getstatus_novel.send({ error: true });
|
||||
}
|
||||
else {
|
||||
if (response.statusCode == 401) {
|
||||
console.log('Access Token is incorrect.');
|
||||
}
|
||||
|
||||
console.log(data);
|
||||
response_getstatus_novel.send({ error: true });
|
||||
console.log('NovelAI returned an error:', response.statusText);
|
||||
return response_getstatus_novel.send({ error: true });
|
||||
}
|
||||
}).on('error', function () {
|
||||
response_getstatus_novel.send({ error: true });
|
||||
});
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
return response_getstatus_novel.send({ error: true });
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/generate_novelai", jsonParser, async function (request, response_generate_novel = response) {
|
||||
@ -1888,7 +1902,6 @@ app.post("/generate_novelai", jsonParser, async function (request, response_gene
|
||||
controller.abort();
|
||||
});
|
||||
|
||||
const novelai = require('./src/novelai');
|
||||
const isNewModel = (request.body.model.includes('clio') || request.body.model.includes('kayra'));
|
||||
const badWordsList = novelai.getBadWordsList(request.body.model);
|
||||
|
||||
@ -1943,7 +1956,7 @@ app.post("/generate_novelai", jsonParser, async function (request, response_gene
|
||||
"order": request.body.order
|
||||
}
|
||||
};
|
||||
const util = require('util');
|
||||
|
||||
console.log(util.inspect(data, { depth: 4 }))
|
||||
|
||||
const args = {
|
||||
@ -1953,7 +1966,6 @@ app.post("/generate_novelai", jsonParser, async function (request, response_gene
|
||||
};
|
||||
|
||||
try {
|
||||
const fetch = require('node-fetch').default;
|
||||
const url = request.body.streaming ? `${api_novelai}/ai/generate-stream` : `${api_novelai}/ai/generate`;
|
||||
const response = await fetch(url, { method: 'POST', timeout: 0, ...args });
|
||||
|
||||
@ -2304,7 +2316,6 @@ app.post("/exportchat", jsonParser, async function (request, response) {
|
||||
}
|
||||
}
|
||||
|
||||
const readline = require('readline');
|
||||
const readStream = fs.createReadStream(filename);
|
||||
const rl = readline.createInterface({
|
||||
input: readStream,
|
||||
@ -3105,7 +3116,7 @@ app.get('/thumbnail', jsonParser, async function (request, response) {
|
||||
});
|
||||
|
||||
/* OpenAI */
|
||||
app.post("/getstatus_openai", jsonParser, function (request, response_getstatus_openai = response) {
|
||||
app.post("/getstatus_openai", jsonParser, function (request, response_getstatus_openai) {
|
||||
if (!request.body) return response_getstatus_openai.sendStatus(400);
|
||||
|
||||
let api_url;
|
||||
@ -3133,14 +3144,14 @@ app.post("/getstatus_openai", jsonParser, function (request, response_getstatus_
|
||||
...headers,
|
||||
},
|
||||
};
|
||||
client.get(api_url + "/models", args, function (data, response) {
|
||||
restClient.get(api_url + "/models", args, function (data, response) {
|
||||
if (response.statusCode == 200) {
|
||||
response_getstatus_openai.send(data);
|
||||
if (request.body.use_openrouter) {
|
||||
let models = [];
|
||||
data.data.forEach(model => {
|
||||
const context_length = model.context_length;
|
||||
const tokens_dollar = parseFloat(1 / (1000 * model.pricing.prompt));
|
||||
const tokens_dollar = Number(1 / (1000 * model.pricing.prompt));
|
||||
const tokens_rounded = (Math.round(tokens_dollar * 1000) / 1000).toFixed(0);
|
||||
models[model.id] = {
|
||||
tokens_per_dollar: tokens_rounded + 'k',
|
||||
@ -3283,7 +3294,6 @@ function convertClaudePrompt(messages, addHumanPrefix, addAssistantPostfix) {
|
||||
}
|
||||
|
||||
async function sendScaleRequest(request, response) {
|
||||
const fetch = require('node-fetch').default;
|
||||
|
||||
const api_url = new URL(request.body.api_url_scale).toString();
|
||||
const api_key_scale = readSecret(SECRET_KEYS.SCALE);
|
||||
@ -3397,7 +3407,6 @@ app.post("/generate_altscale", jsonParser, function (request, response_generate_
|
||||
});
|
||||
|
||||
async function sendClaudeRequest(request, response) {
|
||||
const fetch = require('node-fetch').default;
|
||||
|
||||
const api_url = new URL(request.body.reverse_proxy || api_claude).toString();
|
||||
const api_key_claude = request.body.reverse_proxy ? request.body.proxy_password : readSecret(SECRET_KEYS.CLAUDE);
|
||||
@ -3658,7 +3667,7 @@ app.post("/generate_openai", jsonParser, function (request, response_generate_op
|
||||
makeRequest(config, response_generate_openai, request);
|
||||
});
|
||||
|
||||
app.post("/tokenize_openai", jsonParser, function (request, response_tokenize_openai = response) {
|
||||
app.post("/tokenize_openai", jsonParser, function (request, response_tokenize_openai) {
|
||||
if (!request.body) return response_tokenize_openai.sendStatus(400);
|
||||
|
||||
let num_tokens = 0;
|
||||
@ -3766,7 +3775,7 @@ async function sendAI21Request(request, response) {
|
||||
|
||||
}
|
||||
|
||||
app.post("/tokenize_ai21", jsonParser, function (request, response_tokenize_ai21 = response) {
|
||||
app.post("/tokenize_ai21", jsonParser, function (request, response_tokenize_ai21) {
|
||||
if (!request.body) return response_tokenize_ai21.sendStatus(400);
|
||||
const options = {
|
||||
method: 'POST',
|
||||
@ -3973,7 +3982,6 @@ app.post("/tokenize_via_api", jsonParser, async function (request, response) {
|
||||
// ** REST CLIENT ASYNC WRAPPERS **
|
||||
|
||||
async function postAsync(url, args) {
|
||||
const fetch = require('node-fetch').default;
|
||||
const response = await fetch(url, { method: 'POST', timeout: 0, ...args });
|
||||
|
||||
if (response.ok) {
|
||||
@ -3986,7 +3994,7 @@ async function postAsync(url, args) {
|
||||
|
||||
function getAsync(url, args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
client.get(url, args, (data, response) => {
|
||||
restClient.get(url, args, (data, response) => {
|
||||
if (response.statusCode >= 400) {
|
||||
reject(data);
|
||||
}
|
||||
@ -4063,23 +4071,24 @@ if (listen && !config.whitelistMode && !config.basicAuthMode) {
|
||||
}
|
||||
}
|
||||
|
||||
if (true === cliArguments.ssl)
|
||||
if (true === cliArguments.ssl) {
|
||||
https.createServer(
|
||||
{
|
||||
cert: fs.readFileSync(cliArguments.certPath),
|
||||
key: fs.readFileSync(cliArguments.keyPath)
|
||||
}, app)
|
||||
.listen(
|
||||
tavernUrl.port || 443,
|
||||
Number(tavernUrl.port) || 443,
|
||||
tavernUrl.hostname,
|
||||
setupTasks
|
||||
);
|
||||
else
|
||||
} else {
|
||||
http.createServer(app).listen(
|
||||
tavernUrl.port || 80,
|
||||
Number(tavernUrl.port) || 80,
|
||||
tavernUrl.hostname,
|
||||
setupTasks
|
||||
);
|
||||
}
|
||||
|
||||
async function convertWebp() {
|
||||
const files = fs.readdirSync(directories.characters).filter(e => e.endsWith(".webp"));
|
||||
@ -4190,7 +4199,7 @@ function migrateSecrets() {
|
||||
|
||||
try {
|
||||
let modified = false;
|
||||
const fileContents = fs.readFileSync(SETTINGS_FILE);
|
||||
const fileContents = fs.readFileSync(SETTINGS_FILE, 'utf8');
|
||||
const settings = JSON.parse(fileContents);
|
||||
const oaiKey = settings?.api_key_openai;
|
||||
const hordeKey = settings?.horde_settings?.api_key;
|
||||
@ -4242,7 +4251,7 @@ app.post('/readsecretstate', jsonParser, (_, response) => {
|
||||
}
|
||||
|
||||
try {
|
||||
const fileContents = fs.readFileSync(SECRETS_FILE);
|
||||
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf8');
|
||||
const secrets = JSON.parse(fileContents);
|
||||
const state = {};
|
||||
|
||||
@ -4301,7 +4310,7 @@ app.post('/viewsecrets', jsonParser, async (_, response) => {
|
||||
}
|
||||
|
||||
try {
|
||||
const fileContents = fs.readFileSync(SECRETS_FILE);
|
||||
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8');
|
||||
const secrets = JSON.parse(fileContents);
|
||||
return response.send(secrets);
|
||||
} catch (error) {
|
||||
@ -4364,6 +4373,7 @@ app.post('/horde_generateimage', jsonParser, async (request, response) => {
|
||||
{
|
||||
sampler_name: request.body.sampler,
|
||||
hires_fix: request.body.enable_hr,
|
||||
// @ts-ignore - use_gfpgan param is not in the type definition, need to update to new ai_horde @ https://github.com/ZeldaFan0225/ai_horde/blob/main/index.ts
|
||||
use_gfpgan: request.body.restore_faces,
|
||||
cfg_scale: request.body.scale,
|
||||
steps: request.body.steps,
|
||||
@ -4390,6 +4400,7 @@ app.post('/horde_generateimage', jsonParser, async (request, response) => {
|
||||
|
||||
if (check.done) {
|
||||
const result = await ai_horde.getImageGenerationStatus(generation.id);
|
||||
if (result.generations === undefined) return response.sendStatus(500);
|
||||
return response.send(result.generations[0].img);
|
||||
}
|
||||
|
||||
@ -4452,8 +4463,6 @@ app.post('/libre_translate', jsonParser, async (request, response) => {
|
||||
});
|
||||
|
||||
app.post('/google_translate', jsonParser, async (request, response) => {
|
||||
const { generateRequestUrl, normaliseResponse } = require('google-translate-api-browser');
|
||||
|
||||
const text = request.body.text;
|
||||
const lang = request.body.lang;
|
||||
|
||||
@ -4499,7 +4508,6 @@ app.post('/deepl_translate', jsonParser, async (request, response) => {
|
||||
|
||||
console.log('Input text: ' + text);
|
||||
|
||||
const fetch = require('node-fetch').default;
|
||||
const params = new URLSearchParams();
|
||||
params.append('text', text);
|
||||
params.append('target_lang', lang);
|
||||
@ -4545,7 +4553,6 @@ app.post('/novel_tts', jsonParser, async (request, response) => {
|
||||
}
|
||||
|
||||
try {
|
||||
const fetch = require('node-fetch').default;
|
||||
const url = `${api_novelai}/ai/generate-voice?text=${encodeURIComponent(text)}&voice=-1&seed=${encodeURIComponent(voice)}&opus=false&version=v2`;
|
||||
const result = await fetch(url, {
|
||||
method: 'GET',
|
||||
@ -4718,7 +4725,7 @@ app.post('/import_custom', jsonParser, async (request, response) => {
|
||||
return response.sendStatus(404);
|
||||
}
|
||||
|
||||
response.set('Content-Type', result.fileType);
|
||||
if (result.fileType) response.set('Content-Type', result.fileType)
|
||||
response.set('Content-Disposition', `attachment; filename="${result.fileName}"`);
|
||||
response.set('X-Custom-Content-Type', chubParsed?.type);
|
||||
return response.send(result.buffer);
|
||||
@ -4729,8 +4736,6 @@ app.post('/import_custom', jsonParser, async (request, response) => {
|
||||
});
|
||||
|
||||
async function downloadChubLorebook(id) {
|
||||
const fetch = require('node-fetch').default;
|
||||
|
||||
const result = await fetch('https://api.chub.ai/api/lorebooks/download', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@ -4754,8 +4759,6 @@ async function downloadChubLorebook(id) {
|
||||
}
|
||||
|
||||
async function downloadChubCharacter(id) {
|
||||
const fetch = require('node-fetch').default;
|
||||
|
||||
const result = await fetch('https://api.chub.ai/api/characters/download', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@ -4776,6 +4779,11 @@ async function downloadChubCharacter(id) {
|
||||
return { buffer, fileName, fileType };
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {String} str
|
||||
* @returns { { id: string, type: "character" | "lorebook" } | null }
|
||||
*/
|
||||
function parseChubUrl(str) {
|
||||
const splitStr = str.split('/');
|
||||
const length = splitStr.length;
|
||||
@ -4880,7 +4888,7 @@ function writeSecret(key, value) {
|
||||
writeFileAtomicSync(SECRETS_FILE, emptyFile, "utf-8");
|
||||
}
|
||||
|
||||
const fileContents = fs.readFileSync(SECRETS_FILE);
|
||||
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8');
|
||||
const secrets = JSON.parse(fileContents);
|
||||
secrets[key] = value;
|
||||
writeFileAtomicSync(SECRETS_FILE, JSON.stringify(secrets), "utf-8");
|
||||
@ -4891,7 +4899,7 @@ function readSecret(key) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const fileContents = fs.readFileSync(SECRETS_FILE);
|
||||
const fileContents = fs.readFileSync(SECRETS_FILE, 'utf-8');
|
||||
const secrets = JSON.parse(fileContents);
|
||||
return secrets[key];
|
||||
}
|
||||
@ -4972,7 +4980,7 @@ async function getImageBuffers(zipFilePath) {
|
||||
/**
|
||||
* This function extracts the extension information from the manifest file.
|
||||
* @param {string} extensionPath - The path of the extension folder
|
||||
* @returns {Object} - Returns the manifest data as an object
|
||||
* @returns {Promise<Object>} - Returns the manifest data as an object
|
||||
*/
|
||||
async function getManifest(extensionPath) {
|
||||
const manifestPath = path.join(extensionPath, 'manifest.json');
|
||||
@ -4987,6 +4995,7 @@ async function getManifest(extensionPath) {
|
||||
}
|
||||
|
||||
async function checkIfRepoIsUpToDate(extensionPath) {
|
||||
// @ts-ignore - simple-git types are incorrect, this is apparently callable but no call signature
|
||||
const git = simpleGit();
|
||||
await git.cwd(extensionPath).fetch('origin');
|
||||
const currentBranch = await git.cwd(extensionPath).branch();
|
||||
@ -5017,6 +5026,7 @@ async function checkIfRepoIsUpToDate(extensionPath) {
|
||||
* @returns {void}
|
||||
*/
|
||||
app.post('/get_extension', jsonParser, async (request, response) => {
|
||||
// @ts-ignore - simple-git types are incorrect, this is apparently callable but no call signature
|
||||
const git = simpleGit();
|
||||
if (!request.body.url) {
|
||||
return response.status(400).send('Bad Request: URL is required in the request body.');
|
||||
@ -5062,6 +5072,7 @@ app.post('/get_extension', jsonParser, async (request, response) => {
|
||||
* @returns {void}
|
||||
*/
|
||||
app.post('/update_extension', jsonParser, async (request, response) => {
|
||||
// @ts-ignore - simple-git types are incorrect, this is apparently callable but no call signature
|
||||
const git = simpleGit();
|
||||
if (!request.body.extensionName) {
|
||||
return response.status(400).send('Bad Request: extensionName is required in the request body.');
|
||||
@ -5107,6 +5118,7 @@ app.post('/update_extension', jsonParser, async (request, response) => {
|
||||
* @returns {void}
|
||||
*/
|
||||
app.post('/get_extension_version', jsonParser, async (request, response) => {
|
||||
// @ts-ignore - simple-git types are incorrect, this is apparently callable but no call signature
|
||||
const git = simpleGit();
|
||||
if (!request.body.extensionName) {
|
||||
return response.status(400).send('Bad Request: extensionName is required in the request body.');
|
||||
@ -5249,16 +5261,15 @@ function checkAssetFileName(inputFilename) {
|
||||
* @returns {void}
|
||||
*/
|
||||
app.post('/asset_download', jsonParser, async (request, response) => {
|
||||
const { Readable } = require('stream');
|
||||
const { finished } = require('stream/promises');
|
||||
const url = request.body.url;
|
||||
const inputCategory = request.body.category;
|
||||
const inputFilename = sanitize(request.body.filename);
|
||||
const validCategories = ["bgm", "ambient"];
|
||||
const fetch = require('node-fetch').default;
|
||||
|
||||
// Check category
|
||||
let category = null;
|
||||
for (i of validCategories)
|
||||
for (let i of validCategories)
|
||||
if (i == inputCategory)
|
||||
category = i;
|
||||
|
||||
@ -5270,7 +5281,7 @@ app.post('/asset_download', jsonParser, async (request, response) => {
|
||||
// Sanitize filename
|
||||
const safe_input = checkAssetFileName(inputFilename);
|
||||
if (safe_input == '')
|
||||
return response.sendFile(400);
|
||||
return response.sendStatus(400);
|
||||
|
||||
const temp_path = path.join(directories.assets, "temp", safe_input)
|
||||
const file_path = path.join(directories.assets, category, safe_input)
|
||||
@ -5278,23 +5289,19 @@ app.post('/asset_download', jsonParser, async (request, response) => {
|
||||
|
||||
try {
|
||||
// Download to temp
|
||||
const downloadFile = (async (url, temp_path) => {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) {
|
||||
throw new Error(`Unexpected response ${res.statusText}`);
|
||||
}
|
||||
const destination = path.resolve(temp_path);
|
||||
// Delete if previous download failed
|
||||
if (fs.existsSync(temp_path)) {
|
||||
fs.unlink(temp_path, (err) => {
|
||||
if (err) throw err;
|
||||
});
|
||||
}
|
||||
const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
|
||||
await finished(Readable.fromWeb(res.body).pipe(fileStream));
|
||||
});
|
||||
|
||||
await downloadFile(url, temp_path);
|
||||
const res = await fetch(url);
|
||||
if (!res.ok || res.body === null) {
|
||||
throw new Error(`Unexpected response ${res.statusText}`);
|
||||
}
|
||||
const destination = path.resolve(temp_path);
|
||||
// Delete if previous download failed
|
||||
if (fs.existsSync(temp_path)) {
|
||||
fs.unlink(temp_path, (err) => {
|
||||
if (err) throw err;
|
||||
});
|
||||
}
|
||||
const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
|
||||
await finished(res.body.pipe(fileStream));
|
||||
|
||||
// Move into asset place
|
||||
console.debug("Download finished, moving file from", temp_path, "to", file_path);
|
||||
@ -5316,15 +5323,13 @@ app.post('/asset_download', jsonParser, async (request, response) => {
|
||||
* @returns {void}
|
||||
*/
|
||||
app.post('/asset_delete', jsonParser, async (request, response) => {
|
||||
const { Readable } = require('stream');
|
||||
const { finished } = require('stream/promises');
|
||||
const inputCategory = request.body.category;
|
||||
const inputFilename = sanitize(request.body.filename);
|
||||
const validCategories = ["bgm", "ambient"];
|
||||
|
||||
// Check category
|
||||
let category = null;
|
||||
for (i of validCategories)
|
||||
for (let i of validCategories)
|
||||
if (i == inputCategory)
|
||||
category = i;
|
||||
|
||||
@ -5336,7 +5341,7 @@ app.post('/asset_delete', jsonParser, async (request, response) => {
|
||||
// Sanitize filename
|
||||
const safe_input = checkAssetFileName(inputFilename);
|
||||
if (safe_input == '')
|
||||
return response.sendFile(400);
|
||||
return response.sendStatus(400);
|
||||
|
||||
const file_path = path.join(directories.assets, category, safe_input)
|
||||
console.debug("Request received to delete", category, file_path);
|
||||
@ -5373,13 +5378,14 @@ app.post('/asset_delete', jsonParser, async (request, response) => {
|
||||
* @returns {void}
|
||||
*/
|
||||
app.post('/get_character_assets_list', jsonParser, async (request, response) => {
|
||||
const name = sanitize(request.query.name);
|
||||
if (request.query.name === undefined) return response.sendStatus(400);
|
||||
const name = sanitize(request.query.name.toString());
|
||||
const inputCategory = request.query.category;
|
||||
const validCategories = ["bgm", "ambient"]
|
||||
|
||||
// Check category
|
||||
let category = null
|
||||
for (i of validCategories)
|
||||
for (let i of validCategories)
|
||||
if (i == inputCategory)
|
||||
category = i
|
||||
|
||||
@ -5398,7 +5404,7 @@ app.post('/get_character_assets_list', jsonParser, async (request, response) =>
|
||||
return filename != ".placeholder";
|
||||
});
|
||||
|
||||
for (i of files)
|
||||
for (let i of files)
|
||||
output.push(`/characters/${name}/${category}/${i}`);
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user