diff --git a/default/config.yaml b/default/config.yaml index 6378d9aec..c05cc9371 100644 --- a/default/config.yaml +++ b/default/config.yaml @@ -28,6 +28,11 @@ port: 8000 # - Use -1 to use the server port. # - Specify a port to override the default. autorunPortOverride: -1 +# -- SSL options -- +ssl: + enabled: false + certPath: "./certs/cert.pem" + keyPath: "./certs/privkey.pem" # -- SECURITY CONFIGURATION -- # Toggle whitelist mode whitelistMode: true diff --git a/server.js b/server.js index 2e7e9e9c3..97e286fdd 100644 --- a/server.js +++ b/server.js @@ -1,10 +1,6 @@ #!/usr/bin/env node // native node modules -import fs from 'node:fs'; -import http from 'node:http'; -import https from 'node:https'; -import os from 'os'; import path from 'node:path'; import util from 'node:util'; import net from 'node:net'; @@ -12,12 +8,6 @@ import dns from 'node:dns'; import process from 'node:process'; import { fileURLToPath } from 'node:url'; -// cli/fs related library imports -import open from 'open'; -import yargs from 'yargs/yargs'; -import { hideBin } from 'yargs/helpers'; - -// express/server related library imports import cors from 'cors'; import { csrfSync } from 'csrf-sync'; import express from 'express'; @@ -27,17 +17,11 @@ import multer from 'multer'; import responseTime from 'response-time'; import helmet from 'helmet'; import bodyParser from 'body-parser'; - -// net related library imports +import open from 'open'; import fetch from 'node-fetch'; -import ipRegex from 'ip-regex'; - -// Unrestrict console logs display limit -util.inspect.defaultOptions.maxArrayLength = null; -util.inspect.defaultOptions.maxStringLength = null; -util.inspect.defaultOptions.depth = 4; // local library imports +import { CommandLineParser } from './src/command-line.js'; import { loadPlugins } from './src/plugin-loader.js'; import { initUserStorage, @@ -53,6 +37,8 @@ import { shouldRedirectToLogin, tryAutoLogin, router as userDataRouter, + cleanUploads, + getSessionCookieAge, } from './src/users.js'; import getWebpackServeMiddleware from './src/middleware/webpack-serve.js'; @@ -62,18 +48,16 @@ import accessLoggerMiddleware, { getAccessLogPath, migrateAccessLog } from './sr import multerMonkeyPatch from './src/middleware/multerMonkeyPatch.js'; import initRequestProxy from './src/request-proxy.js'; import getCacheBusterMiddleware from './src/middleware/cacheBuster.js'; +import corsProxyMiddleware from './src/middleware/corsProxy.js'; import { getVersion, getConfigValue, color, - forwardFetchResponse, removeColorFormatting, getSeparator, - stringToBool, - urlHostnameToIPv6, - canResolve, safeReadFileSync, setupLogLevel, + setWindowTitle, } from './src/util.js'; import { UPLOADS_DIRECTORY } from './src/constants.js'; import { ensureThumbnailCache } from './src/endpoints/thumbnails.js'; @@ -82,45 +66,15 @@ import { ensureThumbnailCache } from './src/endpoints/thumbnails.js'; import { router as usersPublicRouter } from './src/endpoints/users-public.js'; import { router as usersPrivateRouter } from './src/endpoints/users-private.js'; import { router as usersAdminRouter } from './src/endpoints/users-admin.js'; -import { router as movingUIRouter } from './src/endpoints/moving-ui.js'; -import { router as imagesRouter } from './src/endpoints/images.js'; -import { router as quickRepliesRouter } from './src/endpoints/quick-replies.js'; -import { router as avatarsRouter } from './src/endpoints/avatars.js'; -import { router as themesRouter } from './src/endpoints/themes.js'; -import { router as openAiRouter } from './src/endpoints/openai.js'; -import { router as googleRouter } from './src/endpoints/google.js'; -import { router as anthropicRouter } from './src/endpoints/anthropic.js'; -import { router as tokenizersRouter } from './src/endpoints/tokenizers.js'; -import { router as presetsRouter } from './src/endpoints/presets.js'; -import { router as secretsRouter } from './src/endpoints/secrets.js'; -import { router as thumbnailRouter } from './src/endpoints/thumbnails.js'; -import { router as novelAiRouter } from './src/endpoints/novelai.js'; -import { router as extensionsRouter } from './src/endpoints/extensions.js'; -import { router as assetsRouter } from './src/endpoints/assets.js'; -import { router as filesRouter } from './src/endpoints/files.js'; -import { router as charactersRouter } from './src/endpoints/characters.js'; -import { router as chatsRouter } from './src/endpoints/chats.js'; -import { router as groupsRouter } from './src/endpoints/groups.js'; -import { router as worldInfoRouter } from './src/endpoints/worldinfo.js'; -import { router as statsRouter, init as statsInit, onExit as statsOnExit } from './src/endpoints/stats.js'; -import { router as backgroundsRouter } from './src/endpoints/backgrounds.js'; -import { router as spritesRouter } from './src/endpoints/sprites.js'; -import { router as contentManagerRouter, checkForNewContent } from './src/endpoints/content-manager.js'; -import { router as settingsRouter, init as settingsInit } from './src/endpoints/settings.js'; -import { router as stableDiffusionRouter } from './src/endpoints/stable-diffusion.js'; -import { router as hordeRouter } from './src/endpoints/horde.js'; -import { router as vectorsRouter } from './src/endpoints/vectors.js'; -import { router as translateRouter } from './src/endpoints/translate.js'; -import { router as classifyRouter } from './src/endpoints/classify.js'; -import { router as captionRouter } from './src/endpoints/caption.js'; -import { router as searchRouter } from './src/endpoints/search.js'; -import { router as openRouterRouter } from './src/endpoints/openrouter.js'; -import { router as chatCompletionsRouter } from './src/endpoints/backends/chat-completions.js'; -import { router as koboldRouter } from './src/endpoints/backends/kobold.js'; -import { router as textCompletionsRouter } from './src/endpoints/backends/text-completions.js'; -import { router as scaleAltRouter } from './src/endpoints/backends/scale-alt.js'; -import { router as speechRouter } from './src/endpoints/speech.js'; -import { router as azureRouter } from './src/endpoints/azure.js'; +import { init as statsInit, onExit as statsOnExit } from './src/endpoints/stats.js'; +import { checkForNewContent } from './src/endpoints/content-manager.js'; +import { init as settingsInit } from './src/endpoints/settings.js'; +import { redirectDeprecatedEndpoints, ServerStartup, setupPrivateEndpoints } from './src/server-startup.js'; + +// Unrestrict console logs display limit +util.inspect.defaultOptions.maxArrayLength = null; +util.inspect.defaultOptions.maxStringLength = null; +util.inspect.defaultOptions.depth = 4; // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. // https://github.com/nodejs/node/issues/47822#issuecomment-1564708870 @@ -130,122 +84,14 @@ if (process.versions && process.versions.node && process.versions.node.match(/20 if (net.setDefaultAutoSelectFamily) net.setDefaultAutoSelectFamily(false); } -const DEFAULT_PORT = 8000; -const DEFAULT_AUTORUN = false; -const DEFAULT_LISTEN = false; -const DEFAULT_LISTEN_ADDRESS_IPV6 = '[::]'; -const DEFAULT_LISTEN_ADDRESS_IPV4 = '0.0.0.0'; -const DEFAULT_CORS_PROXY = false; -const DEFAULT_WHITELIST = true; -const DEFAULT_ACCOUNTS = false; -const DEFAULT_CSRF_DISABLED = false; -const DEFAULT_BASIC_AUTH = false; -const DEFAULT_PER_USER_BASIC_AUTH = false; +const cliParser = new CommandLineParser(); +const cliArgs = cliParser.parse(process.argv); +globalThis.DATA_ROOT = cliArgs.dataRoot; -const DEFAULT_ENABLE_IPV6 = false; -const DEFAULT_ENABLE_IPV4 = true; - -const DEFAULT_PREFER_IPV6 = false; - -const DEFAULT_AVOID_LOCALHOST = false; - -const DEFAULT_AUTORUN_HOSTNAME = 'auto'; -const DEFAULT_AUTORUN_PORT = -1; - -const DEFAULT_PROXY_ENABLED = false; -const DEFAULT_PROXY_URL = ''; -const DEFAULT_PROXY_BYPASS = []; - -const cliArguments = yargs(hideBin(process.argv)) - .usage('Usage: [options]') - .option('enableIPv6', { - type: 'string', - default: null, - describe: `Enables IPv6.\n[config default: ${DEFAULT_ENABLE_IPV6}]`, - }).option('enableIPv4', { - type: 'string', - default: null, - describe: `Enables IPv4.\n[config default: ${DEFAULT_ENABLE_IPV4}]`, - }).option('port', { - type: 'number', - default: null, - describe: `Sets the port under which SillyTavern will run.\nIf not provided falls back to yaml config 'port'.\n[config default: ${DEFAULT_PORT}]`, - }).option('dnsPreferIPv6', { - type: 'boolean', - default: null, - describe: `Prefers IPv6 for dns\nyou should probably have the enabled if you're on an IPv6 only network\nIf not provided falls back to yaml config 'preferIPv6'.\n[config default: ${DEFAULT_PREFER_IPV6}]`, - }).option('autorun', { - type: 'boolean', - default: null, - describe: `Automatically launch SillyTavern in the browser.\nAutorun is automatically disabled if --ssl is set to true.\nIf not provided falls back to yaml config 'autorun'.\n[config default: ${DEFAULT_AUTORUN}]`, - }).option('autorunHostname', { - type: 'string', - default: null, - describe: 'the autorun hostname, probably best left on \'auto\'.\nuse values like \'localhost\', \'st.example.com\'', - }).option('autorunPortOverride', { - type: 'string', - default: null, - describe: 'Overrides the port for autorun with open your browser with this port and ignore what port the server is running on. -1 is use server port', - }).option('listen', { - type: 'boolean', - default: null, - describe: `SillyTavern is listening on all network interfaces (Wi-Fi, LAN, localhost). If false, will limit it only to internal localhost (127.0.0.1).\nIf not provided falls back to yaml config 'listen'.\n[config default: ${DEFAULT_LISTEN}]`, - }).option('listenAddressIPv6', { - type: 'string', - default: null, - describe: 'Set SillyTavern to listen to a specific IPv6 address. If not set, it will fallback to listen to all.\n[config default: [::] ]', - }).option('listenAddressIPv4', { - type: 'string', - default: null, - describe: 'Set SillyTavern to listen to a specific IPv4 address. If not set, it will fallback to listen to all.\n[config default: 0.0.0.0 ]', - }).option('corsProxy', { - type: 'boolean', - default: null, - describe: `Enables CORS proxy\nIf not provided falls back to yaml config 'enableCorsProxy'.\n[config default: ${DEFAULT_CORS_PROXY}]`, - }).option('disableCsrf', { - type: 'boolean', - default: null, - describe: 'Disables CSRF protection', - }).option('ssl', { - type: 'boolean', - default: false, - describe: 'Enables SSL', - }).option('certPath', { - type: 'string', - default: 'certs/cert.pem', - describe: 'Path to your certificate file.', - }).option('keyPath', { - type: 'string', - default: 'certs/privkey.pem', - describe: 'Path to your private key file.', - }).option('whitelist', { - type: 'boolean', - default: null, - describe: 'Enables whitelist mode', - }).option('dataRoot', { - type: 'string', - default: null, - describe: 'Root directory for data storage', - }).option('avoidLocalhost', { - type: 'boolean', - default: null, - describe: 'Avoids using \'localhost\' for autorun in auto mode.\nuse if you don\'t have \'localhost\' in your hosts file', - }).option('basicAuthMode', { - type: 'boolean', - default: null, - describe: 'Enables basic authentication', - }).option('requestProxyEnabled', { - type: 'boolean', - default: null, - describe: 'Enables a use of proxy for outgoing requests', - }).option('requestProxyUrl', { - type: 'string', - default: null, - describe: 'Request proxy URL (HTTP or SOCKS protocols)', - }).option('requestProxyBypass', { - type: 'array', - describe: 'Request proxy bypass list (space separated list of hosts)', - }).parseSync(); +if (!cliArgs.enableIPv6 && !cliArgs.enableIPv4) { + console.error('error: You can\'t disable all internet protocols: at least IPv6 or IPv4 must be enabled.'); + process.exit(1); +} // change all relative paths const serverDirectory = import.meta.dirname ?? path.dirname(fileURLToPath(import.meta.url)); @@ -259,53 +105,10 @@ app.use(helmet({ app.use(compression()); app.use(responseTime()); - -/** @type {number} */ -const server_port = cliArguments.port ?? getConfigValue('port', DEFAULT_PORT, 'number'); /** @type {boolean} */ -const autorun = (cliArguments.autorun ?? getConfigValue('autorun', DEFAULT_AUTORUN, 'boolean')) && !cliArguments.ssl; -/** @type {boolean} */ -const listen = cliArguments.listen ?? getConfigValue('listen', DEFAULT_LISTEN, 'boolean'); -/** @type {string} */ -const listenAddressIPv6 = cliArguments.listenAddressIPv6 ?? getConfigValue('listenAddress.ipv6', DEFAULT_LISTEN_ADDRESS_IPV6); -/** @type {string} */ -const listenAddressIPv4 = cliArguments.listenAddressIPv4 ?? getConfigValue('listenAddress.ipv4', DEFAULT_LISTEN_ADDRESS_IPV4); -/** @type {boolean} */ -const enableCorsProxy = cliArguments.corsProxy ?? getConfigValue('enableCorsProxy', DEFAULT_CORS_PROXY, 'boolean'); -const enableWhitelist = cliArguments.whitelist ?? getConfigValue('whitelistMode', DEFAULT_WHITELIST, 'boolean'); -/** @type {string} */ -globalThis.DATA_ROOT = cliArguments.dataRoot ?? getConfigValue('dataRoot', './data'); -/** @type {boolean} */ -const disableCsrf = cliArguments.disableCsrf ?? getConfigValue('disableCsrfProtection', DEFAULT_CSRF_DISABLED, 'boolean'); -const basicAuthMode = cliArguments.basicAuthMode ?? getConfigValue('basicAuthMode', DEFAULT_BASIC_AUTH, 'boolean'); -const perUserBasicAuth = getConfigValue('perUserBasicAuth', DEFAULT_PER_USER_BASIC_AUTH, 'boolean'); -/** @type {boolean} */ -const enableAccounts = getConfigValue('enableUserAccounts', DEFAULT_ACCOUNTS, 'boolean'); +const enableAccounts = getConfigValue('enableUserAccounts', false, 'boolean'); -const uploadsPath = path.join(globalThis.DATA_ROOT, UPLOADS_DIRECTORY); - - -/** @type {boolean | string} */ -let enableIPv6 = stringToBool(cliArguments.enableIPv6) ?? stringToBool(getConfigValue('protocol.ipv6', DEFAULT_ENABLE_IPV6)) ?? DEFAULT_ENABLE_IPV6; -/** @type {boolean | string} */ -let enableIPv4 = stringToBool(cliArguments.enableIPv4) ?? stringToBool(getConfigValue('protocol.ipv4', DEFAULT_ENABLE_IPV4)) ?? DEFAULT_ENABLE_IPV4; - -/** @type {string} */ -const autorunHostname = cliArguments.autorunHostname ?? getConfigValue('autorunHostname', DEFAULT_AUTORUN_HOSTNAME); -/** @type {number} */ -const autorunPortOverride = cliArguments.autorunPortOverride ?? getConfigValue('autorunPortOverride', DEFAULT_AUTORUN_PORT, 'number'); - -/** @type {boolean} */ -const dnsPreferIPv6 = cliArguments.dnsPreferIPv6 ?? getConfigValue('dnsPreferIPv6', DEFAULT_PREFER_IPV6, 'boolean'); - -/** @type {boolean} */ -const avoidLocalhost = cliArguments.avoidLocalhost ?? getConfigValue('avoidLocalhost', DEFAULT_AVOID_LOCALHOST, 'boolean'); - -const proxyEnabled = cliArguments.requestProxyEnabled ?? getConfigValue('requestProxy.enabled', DEFAULT_PROXY_ENABLED, 'boolean'); -const proxyUrl = cliArguments.requestProxyUrl ?? getConfigValue('requestProxy.url', DEFAULT_PROXY_URL); -const proxyBypass = cliArguments.requestProxyBypass ?? getConfigValue('requestProxy.bypass', DEFAULT_PROXY_BYPASS); - -if (dnsPreferIPv6) { +if (cliArgs.dnsPreferIPv6) { // Set default DNS resolution order to IPv6 first dns.setDefaultResultOrder('ipv6first'); console.log('Preferring IPv6 for DNS resolution'); @@ -315,23 +118,6 @@ if (dnsPreferIPv6) { console.log('Preferring IPv4 for DNS resolution'); } - -const ipOptions = [true, 'auto', false]; - -if (!ipOptions.includes(enableIPv6)) { - console.warn(color.red('`protocol: ipv6` option invalid'), '\n use:', ipOptions, '\n setting to:', DEFAULT_ENABLE_IPV6); - enableIPv6 = DEFAULT_ENABLE_IPV6; -} -if (!ipOptions.includes(enableIPv4)) { - console.warn(color.red('`protocol: ipv4` option invalid'), '\n use:', ipOptions, '\n setting to:', DEFAULT_ENABLE_IPV4); - enableIPv4 = DEFAULT_ENABLE_IPV4; -} - -if (enableIPv6 === false && enableIPv4 === false) { - console.error('error: You can\'t disable all internet protocols: at least IPv6 or IPv4 must be enabled.'); - process.exit(1); -} - // CORS Settings // const CORS = cors({ origin: 'null', @@ -340,59 +126,23 @@ const CORS = cors({ app.use(CORS); -if (listen && basicAuthMode) { +if (cliArgs.listen && cliArgs.basicAuthMode) { app.use(basicAuthMiddleware); } -if (enableWhitelist) { +if (cliArgs.whitelistMode) { app.use(whitelistMiddleware()); } -if (listen) { +if (cliArgs.listen) { app.use(accessLoggerMiddleware()); } -if (enableCorsProxy) { +if (cliArgs.enableCorsProxy) { app.use(bodyParser.json({ limit: '200mb', })); - console.log('Enabling CORS proxy'); - - app.use('/proxy/:url(*)', async (req, res) => { - const url = req.params.url; // get the url from the request path - - // Disallow circular requests - const serverUrl = req.protocol + '://' + req.get('host'); - if (url.startsWith(serverUrl)) { - return res.status(400).send('Circular requests are not allowed'); - } - - try { - const headers = JSON.parse(JSON.stringify(req.headers)); - const headersToRemove = [ - 'x-csrf-token', 'host', 'referer', 'origin', 'cookie', - 'x-forwarded-for', 'x-forwarded-protocol', 'x-forwarded-proto', - 'x-forwarded-host', 'x-real-ip', 'sec-fetch-mode', - 'sec-fetch-site', 'sec-fetch-dest', - ]; - - headersToRemove.forEach(header => delete headers[header]); - - const bodyMethods = ['POST', 'PUT', 'PATCH']; - - const response = await fetch(url, { - method: req.method, - headers: headers, - body: bodyMethods.includes(req.method) ? JSON.stringify(req.body) : undefined, - }); - - // Copy over relevant response params to the proxy response - forwardFetchResponse(response, res); - - } catch (error) { - res.status(500).send('Error occurred while trying to proxy to: ' + url + ' ' + error); - } - }); + app.use('/proxy/:url(*)', corsProxyMiddleware); } else { app.use('/proxy/:url(*)', async (_, res) => { const message = 'CORS proxy is disabled. Enable it in config.yaml or use the --corsProxy flag.'; @@ -401,74 +151,6 @@ if (enableCorsProxy) { }); } -function getSessionCookieAge() { - // Defaults to "no expiration" if not set - const configValue = getConfigValue('sessionTimeout', -1, 'number'); - - // Convert to milliseconds - if (configValue > 0) { - return configValue * 1000; - } - - // "No expiration" is just 400 days as per RFC 6265 - if (configValue < 0) { - return 400 * 24 * 60 * 60 * 1000; - } - - // 0 means session cookie is deleted when the browser session ends - // (depends on the implementation of the browser) - return undefined; -} - -/** - * Checks the network interfaces to determine the presence of IPv6 and IPv4 addresses. - * - * @returns {Promise<[boolean, boolean, boolean, boolean]>} A promise that resolves to an array containing: - * - [0]: `hasIPv6` (boolean) - Whether the computer has any IPv6 address, including (`::1`). - * - [1]: `hasIPv4` (boolean) - Whether the computer has any IPv4 address, including (`127.0.0.1`). - * - [2]: `hasIPv6Local` (boolean) - Whether the computer has local IPv6 address (`::1`). - * - [3]: `hasIPv4Local` (boolean) - Whether the computer has local IPv4 address (`127.0.0.1`). - */ -async function getHasIP() { - let hasIPv6 = false; - let hasIPv6Local = false; - - let hasIPv4 = false; - let hasIPv4Local = false; - - const interfaces = os.networkInterfaces(); - - for (const iface of Object.values(interfaces)) { - if (iface === undefined) { - continue; - } - - for (const info of iface) { - if (info.family === 'IPv6') { - hasIPv6 = true; - if (info.address === '::1') { - hasIPv6Local = true; - } - } - - if (info.family === 'IPv4') { - hasIPv4 = true; - if (info.address === '127.0.0.1') { - hasIPv4Local = true; - } - } - if (hasIPv6 && hasIPv4 && hasIPv6Local && hasIPv4Local) break; - } - if (hasIPv6 && hasIPv4 && hasIPv6Local && hasIPv4Local) break; - } - return [ - hasIPv6, - hasIPv4, - hasIPv6Local, - hasIPv4Local, - ]; -} - app.use(cookieSession({ name: getCookieSessionName(), sameSite: 'strict', @@ -480,7 +162,7 @@ app.use(cookieSession({ app.use(setUserDataMiddleware); // CSRF Protection // -if (!disableCsrf) { +if (!cliArgs.disableCsrf) { const csrfSyncProtection = csrfSync({ getTokenFromState: (req) => { if (!req.session) { @@ -542,7 +224,7 @@ app.get('/login', async (request, response) => { } try { - const autoLogin = await tryAutoLogin(request, basicAuthMode); + const autoLogin = await tryAutoLogin(request, cliArgs.basicAuthMode); if (autoLogin) { return response.redirect('/'); @@ -573,6 +255,7 @@ app.get('/api/ping', (request, response) => { }); // File uploads +const uploadsPath = path.join(globalThis.DATA_ROOT, UPLOADS_DIRECTORY); app.use(multer({ dest: uploadsPath, limits: { fieldSize: 10 * 1024 * 1024 } }).single('avatar')); app.use(multerMonkeyPatch); @@ -588,170 +271,14 @@ app.get('/version', async function (_, response) { response.send(data); }); -function cleanUploads() { - try { - if (fs.existsSync(uploadsPath)) { - const uploads = fs.readdirSync(uploadsPath); - - if (!uploads.length) { - return; - } - - console.debug(`Cleaning uploads folder (${uploads.length} files)`); - uploads.forEach(file => { - const pathToFile = path.join(uploadsPath, file); - fs.unlinkSync(pathToFile); - }); - } - } catch (err) { - console.error(err); - } -} - -/** - * Redirect a deprecated API endpoint URL to its replacement. Because fetch, form submissions, and $.ajax follow - * redirects, this is transparent to client-side code. - * @param {string} src The URL to redirect from. - * @param {string} destination The URL to redirect to. - */ -function redirect(src, destination) { - app.use(src, (req, res) => { - console.warn(`API endpoint ${src} is deprecated; use ${destination} instead`); - // HTTP 301 causes the request to become a GET. 308 preserves the request method. - res.redirect(308, destination); - }); -} - -// Redirect deprecated character API endpoints -redirect('/createcharacter', '/api/characters/create'); -redirect('/renamecharacter', '/api/characters/rename'); -redirect('/editcharacter', '/api/characters/edit'); -redirect('/editcharacterattribute', '/api/characters/edit-attribute'); -redirect('/v2/editcharacterattribute', '/api/characters/merge-attributes'); -redirect('/deletecharacter', '/api/characters/delete'); -redirect('/getcharacters', '/api/characters/all'); -redirect('/getonecharacter', '/api/characters/get'); -redirect('/getallchatsofcharacter', '/api/characters/chats'); -redirect('/importcharacter', '/api/characters/import'); -redirect('/dupecharacter', '/api/characters/duplicate'); -redirect('/exportcharacter', '/api/characters/export'); - -// Redirect deprecated chat API endpoints -redirect('/savechat', '/api/chats/save'); -redirect('/getchat', '/api/chats/get'); -redirect('/renamechat', '/api/chats/rename'); -redirect('/delchat', '/api/chats/delete'); -redirect('/exportchat', '/api/chats/export'); -redirect('/importgroupchat', '/api/chats/group/import'); -redirect('/importchat', '/api/chats/import'); -redirect('/getgroupchat', '/api/chats/group/get'); -redirect('/deletegroupchat', '/api/chats/group/delete'); -redirect('/savegroupchat', '/api/chats/group/save'); - -// Redirect deprecated group API endpoints -redirect('/getgroups', '/api/groups/all'); -redirect('/creategroup', '/api/groups/create'); -redirect('/editgroup', '/api/groups/edit'); -redirect('/deletegroup', '/api/groups/delete'); - -// Redirect deprecated worldinfo API endpoints -redirect('/getworldinfo', '/api/worldinfo/get'); -redirect('/deleteworldinfo', '/api/worldinfo/delete'); -redirect('/importworldinfo', '/api/worldinfo/import'); -redirect('/editworldinfo', '/api/worldinfo/edit'); - -// Redirect deprecated stats API endpoints -redirect('/getstats', '/api/stats/get'); -redirect('/recreatestats', '/api/stats/recreate'); -redirect('/updatestats', '/api/stats/update'); - -// Redirect deprecated backgrounds API endpoints -redirect('/getbackgrounds', '/api/backgrounds/all'); -redirect('/delbackground', '/api/backgrounds/delete'); -redirect('/renamebackground', '/api/backgrounds/rename'); -redirect('/downloadbackground', '/api/backgrounds/upload'); // yes, the downloadbackground endpoint actually uploads one - -// Redirect deprecated theme API endpoints -redirect('/savetheme', '/api/themes/save'); - -// Redirect deprecated avatar API endpoints -redirect('/getuseravatars', '/api/avatars/get'); -redirect('/deleteuseravatar', '/api/avatars/delete'); -redirect('/uploaduseravatar', '/api/avatars/upload'); - -// Redirect deprecated quick reply endpoints -redirect('/deletequickreply', '/api/quick-replies/delete'); -redirect('/savequickreply', '/api/quick-replies/save'); - -// Redirect deprecated image endpoints -redirect('/uploadimage', '/api/images/upload'); -redirect('/listimgfiles/:folder', '/api/images/list/:folder'); -redirect('/api/content/import', '/api/content/importURL'); - -// Redirect deprecated moving UI endpoints -redirect('/savemovingui', '/api/moving-ui/save'); - -// Redirect Serp endpoints -redirect('/api/serpapi/search', '/api/search/serpapi'); -redirect('/api/serpapi/visit', '/api/search/visit'); -redirect('/api/serpapi/transcript', '/api/search/transcript'); - -app.use('/api/moving-ui', movingUIRouter); -app.use('/api/images', imagesRouter); -app.use('/api/quick-replies', quickRepliesRouter); -app.use('/api/avatars', avatarsRouter); -app.use('/api/themes', themesRouter); -app.use('/api/openai', openAiRouter); -app.use('/api/google', googleRouter); -app.use('/api/anthropic', anthropicRouter); -app.use('/api/tokenizers', tokenizersRouter); -app.use('/api/presets', presetsRouter); -app.use('/api/secrets', secretsRouter); -app.use('/thumbnail', thumbnailRouter); -app.use('/api/novelai', novelAiRouter); -app.use('/api/extensions', extensionsRouter); -app.use('/api/assets', assetsRouter); -app.use('/api/files', filesRouter); -app.use('/api/characters', charactersRouter); -app.use('/api/chats', chatsRouter); -app.use('/api/groups', groupsRouter); -app.use('/api/worldinfo', worldInfoRouter); -app.use('/api/stats', statsRouter); -app.use('/api/backgrounds', backgroundsRouter); -app.use('/api/sprites', spritesRouter); -app.use('/api/content', contentManagerRouter); -app.use('/api/settings', settingsRouter); -app.use('/api/sd', stableDiffusionRouter); -app.use('/api/horde', hordeRouter); -app.use('/api/vector', vectorsRouter); -app.use('/api/translate', translateRouter); -app.use('/api/extra/classify', classifyRouter); -app.use('/api/extra/caption', captionRouter); -app.use('/api/search', searchRouter); -app.use('/api/backends/text-completions', textCompletionsRouter); -app.use('/api/openrouter', openRouterRouter); -app.use('/api/backends/kobold', koboldRouter); -app.use('/api/backends/chat-completions', chatCompletionsRouter); -app.use('/api/backends/scale-alt', scaleAltRouter); -app.use('/api/speech', speechRouter); -app.use('/api/azure', azureRouter); - -const tavernUrlV6 = new URL( - (cliArguments.ssl ? 'https://' : 'http://') + - (listen ? (ipRegex.v6({ exact: true }).test(listenAddressIPv6) ? listenAddressIPv6 : '[::]') : '[::1]') + - (':' + server_port), -); - -const tavernUrl = new URL( - (cliArguments.ssl ? 'https://' : 'http://') + - (listen ? (ipRegex.v4({ exact: true }).test(listenAddressIPv4) ? listenAddressIPv4 : '0.0.0.0') : '127.0.0.1') + - (':' + server_port), -); +redirectDeprecatedEndpoints(app); +setupPrivateEndpoints(app); /** * Tasks that need to be run before the server starts listening. + * @returns {Promise} */ -const preSetupTasks = async function () { +async function preSetupTasks() { const version = await getVersion(); // Print formatted header @@ -797,56 +324,25 @@ const preSetupTasks = async function () { }); // Add request proxy. - initRequestProxy({ enabled: proxyEnabled, url: proxyUrl, bypass: proxyBypass }); + initRequestProxy({ enabled: cliArgs.requestProxyEnabled, url: cliArgs.requestProxyUrl, bypass: cliArgs.requestProxyBypass }); // Wait for frontend libs to compile await webpackMiddleware.runWebpackCompiler(); -}; - -/** - * Gets the hostname to use for autorun in the browser. - * @param {boolean} useIPv6 If use IPv6 - * @param {boolean} useIPv4 If use IPv4 - * @returns Promise The hostname to use for autorun - */ -async function getAutorunHostname(useIPv6, useIPv4) { - if (autorunHostname === 'auto') { - let localhostResolve = await canResolve('localhost', useIPv6, useIPv4); - - if (useIPv6 && useIPv4) { - return (avoidLocalhost || !localhostResolve) ? '[::1]' : 'localhost'; - } - - if (useIPv6) { - return '[::1]'; - } - - if (useIPv4) { - return '127.0.0.1'; - } - } - - return autorunHostname; } /** * Tasks that need to be run after the server starts listening. - * @param {boolean} v6Failed If the server failed to start on IPv6 - * @param {boolean} v4Failed If the server failed to start on IPv4 - * @param {boolean} useIPv6 If the server is using IPv6 - * @param {boolean} useIPv4 If the server is using IPv4 + * @param {import('./src/server-startup.js').ServerStartupResult} result The result of the server startup + * @returns {Promise} */ -const postSetupTasks = async function (v6Failed, v4Failed, useIPv6, useIPv4) { - const autorunUrl = new URL( - (cliArguments.ssl ? 'https://' : 'http://') + - (await getAutorunHostname(useIPv6, useIPv4)) + - (':') + - ((autorunPortOverride >= 0) ? autorunPortOverride : server_port), - ); - +async function postSetupTasks({ v6Failed, v4Failed, useIPv6, useIPv4 }) { + const autorunHostname = await cliArgs.getAutorunHostname(useIPv6, useIPv4); + const autorunUrl = cliArgs.getAutorunUrl(autorunHostname); console.log('Launching...'); - if (autorun) open(autorunUrl.toString()); + if (cliArgs.autorun) { + open(autorunUrl.toString()); + } setWindowTitle('SillyTavern WebServer'); @@ -854,13 +350,13 @@ const postSetupTasks = async function (v6Failed, v4Failed, useIPv6, useIPv4) { if (useIPv6 && !v6Failed) { logListen += color.green( - ' IPv6: ' + tavernUrlV6.host, + ' IPv6: ' + cliArgs.getIPv6ListenUrl().host, ); } if (useIPv4 && !v4Failed) { logListen += color.green( - ' IPv4: ' + tavernUrl.host, + ' IPv4: ' + cliArgs.getIPv4ListenUrl().host, ); } @@ -868,7 +364,7 @@ const postSetupTasks = async function (v6Failed, v4Failed, useIPv6, useIPv4) { const plainGoToLog = removeColorFormatting(goToLog); console.log(logListen); - if (listen) { + if (cliArgs.listen) { console.log(); console.log('To limit connections to internal localhost only ([::1] or 127.0.0.1), change the setting in config.yaml to "listen: false".'); console.log('Check the "access.log" file in the data directory to inspect incoming connections:', color.green(getAccessLogPath())); @@ -877,25 +373,8 @@ const postSetupTasks = async function (v6Failed, v4Failed, useIPv6, useIPv4) { console.log(goToLog); console.log('\n' + getSeparator(plainGoToLog.length) + '\n'); - - if (basicAuthMode) { - if (perUserBasicAuth && !enableAccounts) { - console.error(color.red( - 'Per-user basic authentication is enabled, but user accounts are disabled. This configuration may be insecure.', - )); - } else if (!perUserBasicAuth) { - const basicAuthUserName = getConfigValue('basicAuthUser.username', ''); - const basicAuthUserPassword = getConfigValue('basicAuthUser.password', ''); - if (!basicAuthUserName || !basicAuthUserPassword) { - console.warn(color.yellow( - 'Basic Authentication is enabled, but username or password is not set or empty!', - )); - } - } - } - setupLogLevel(); -}; +} /** * Loads server plugins from a directory. @@ -912,26 +391,13 @@ async function initializePlugins() { } } -/** - * Set the title of the terminal window - * @param {string} title Desired title for the window - */ -function setWindowTitle(title) { - if (process.platform === 'win32') { - process.title = title; - } - else { - process.stdout.write(`\x1b]2;${title}\x1b\x5c`); - } -} - /** * Prints an error message and exits the process if necessary * @param {string} message The error message to print * @returns {void} */ function logSecurityAlert(message) { - if (basicAuthMode || enableWhitelist) return; // safe! + if (cliArgs.basicAuthMode || cliArgs.whitelistMode) return; // safe! console.error(color.red(message)); if (getConfigValue('securityOverride', false, 'boolean')) { console.warn(color.red('Security has been overridden. If it\'s not a trusted network, change the settings.')); @@ -940,177 +406,9 @@ function logSecurityAlert(message) { process.exit(1); } -/** - * Handles the case where the server failed to start on one or both protocols. - * @param {boolean} v6Failed If the server failed to start on IPv6 - * @param {boolean} v4Failed If the server failed to start on IPv4 - * @param {boolean} useIPv6 If use IPv6 - * @param {boolean} useIPv4 If use IPv4 - */ -function handleServerListenFail(v6Failed, v4Failed, useIPv6, useIPv4) { - if (v6Failed && !useIPv4) { - console.error(color.red('fatal error: Failed to start server on IPv6 and IPv4 disabled')); - process.exit(1); - } - - if (v4Failed && !useIPv6) { - console.error(color.red('fatal error: Failed to start server on IPv4 and IPv6 disabled')); - process.exit(1); - } - - if (v6Failed && v4Failed) { - console.error(color.red('fatal error: Failed to start server on both IPv6 and IPv4')); - process.exit(1); - } -} - -/** - * Creates an HTTPS server. - * @param {URL} url The URL to listen on - * @param {number} ipVersion the ip version to use - * @returns {Promise} A promise that resolves when the server is listening - * @throws {Error} If the server fails to start - */ -function createHttpsServer(url, ipVersion) { - return new Promise((resolve, reject) => { - const server = https.createServer( - { - cert: fs.readFileSync(cliArguments.certPath), - key: fs.readFileSync(cliArguments.keyPath), - }, app); - server.on('error', reject); - server.on('listening', resolve); - - let host = url.hostname; - if (ipVersion === 6) host = urlHostnameToIPv6(url.hostname); - server.listen({ - host: host, - port: Number(url.port || 443), - // see https://nodejs.org/api/net.html#serverlisten for why ipv6Only is used - ipv6Only: true, - }); - }); -} - -/** - * Creates an HTTP server. - * @param {URL} url The URL to listen on - * @param {number} ipVersion the ip version to use - * @returns {Promise} A promise that resolves when the server is listening - * @throws {Error} If the server fails to start - */ -function createHttpServer(url, ipVersion) { - return new Promise((resolve, reject) => { - const server = http.createServer(app); - server.on('error', reject); - server.on('listening', resolve); - - let host = url.hostname; - if (ipVersion === 6) host = urlHostnameToIPv6(url.hostname); - server.listen({ - host: host, - port: Number(url.port || 80), - // see https://nodejs.org/api/net.html#serverlisten for why ipv6Only is used - ipv6Only: true, - }); - }); -} - -/** - * Starts the server using http or https depending on config - * @param {boolean} useIPv6 If use IPv6 - * @param {boolean} useIPv4 If use IPv4 - */ -async function startHTTPorHTTPS(useIPv6, useIPv4) { - let v6Failed = false; - let v4Failed = false; - - const createFunc = cliArguments.ssl ? createHttpsServer : createHttpServer; - - if (useIPv6) { - try { - await createFunc(tavernUrlV6, 6); - } catch (error) { - console.error('non-fatal error: failed to start server on IPv6'); - console.error(error); - - v6Failed = true; - } - } - - if (useIPv4) { - try { - await createFunc(tavernUrl, 4); - } catch (error) { - console.error('non-fatal error: failed to start server on IPv4'); - console.error(error); - - v4Failed = true; - } - } - - return [v6Failed, v4Failed]; -} - -async function startServer() { - let useIPv6 = (enableIPv6 === true); - let useIPv4 = (enableIPv4 === true); - - let hasIPv6 = false, - hasIPv4 = false, - hasIPv6Local = false, - hasIPv4Local = false, - hasIPv6Any = false, - hasIPv4Any = false; - - if (enableIPv6 === 'auto' || enableIPv4 === 'auto') { - [hasIPv6Any, hasIPv4Any, hasIPv6Local, hasIPv4Local] = await getHasIP(); - - hasIPv6 = listen ? hasIPv6Any : hasIPv6Local; - if (enableIPv6 === 'auto') { - useIPv6 = hasIPv6; - } - if (hasIPv6) { - if (useIPv6) { - console.log(color.green('IPv6 support detected')); - } else { - console.log('IPv6 support detected (but disabled)'); - } - } - - hasIPv4 = listen ? hasIPv4Any : hasIPv4Local; - if (enableIPv4 === 'auto') { - useIPv4 = hasIPv4; - } - if (hasIPv4) { - if (useIPv4) { - console.log(color.green('IPv4 support detected')); - } else { - console.log('IPv4 support detected (but disabled)'); - } - } - - if (enableIPv6 === 'auto' && enableIPv4 === 'auto') { - if (!hasIPv6 && !hasIPv4) { - console.error('Both IPv6 and IPv4 are not detected'); - process.exit(1); - } - } - } - - if (!useIPv6 && !useIPv4) { - console.error('Both IPv6 and IPv4 are disabled,\nP.S. you should never see this error, at least at one point it was checked for before this, with the rest of the config options'); - process.exit(1); - } - - const [v6Failed, v4Failed] = await startHTTPorHTTPS(useIPv6, useIPv4); - handleServerListenFail(v6Failed, v4Failed, useIPv6, useIPv4); - postSetupTasks(v6Failed, v4Failed, useIPv6, useIPv4); -} - async function verifySecuritySettings() { // Skip all security checks as listen is set to false - if (!listen) { + if (!cliArgs.listen) { return; } @@ -1133,6 +431,23 @@ async function verifySecuritySettings() { logSecurityAlert('If you are not using basic authentication or whitelisting, you should set a password for all admin users.'); } } + + if (cliArgs.basicAuthMode) { + const perUserBasicAuth = getConfigValue('perUserBasicAuth', false, 'boolean'); + if (perUserBasicAuth && !enableAccounts) { + console.error(color.red( + 'Per-user basic authentication is enabled, but user accounts are disabled. This configuration may be insecure.', + )); + } else if (!perUserBasicAuth) { + const basicAuthUserName = getConfigValue('basicAuthUser.username', ''); + const basicAuthUserPassword = getConfigValue('basicAuthUser.password', ''); + if (!basicAuthUserName || !basicAuthUserPassword) { + console.warn(color.yellow( + 'Basic Authentication is enabled, but username or password is not set or empty!', + )); + } + } + } } /** @@ -1153,4 +468,5 @@ initUserStorage(globalThis.DATA_ROOT) .then(verifySecuritySettings) .then(preSetupTasks) .then(apply404Middleware) - .finally(startServer); + .then(new ServerStartup(app, cliArgs).start) + .then(postSetupTasks); diff --git a/src/command-line.js b/src/command-line.js new file mode 100644 index 000000000..09e2c287f --- /dev/null +++ b/src/command-line.js @@ -0,0 +1,265 @@ +import yargs from 'yargs/yargs'; +import { hideBin } from 'yargs/helpers'; +import ipRegex from 'ip-regex'; +import { canResolve, color, getConfigValue, stringToBool } from './util.js'; + +/** + * @typedef {object} CommandLineArguments + * @property {string} dataRoot + * @property {number} port + * @property {boolean} listen + * @property {string} listenAddressIPv6 + * @property {string} listenAddressIPv4 + * @property {boolean|string} enableIPv4 + * @property {boolean|string} enableIPv6 + * @property {boolean} dnsPreferIPv6 + * @property {boolean} autorun + * @property {string} autorunHostname + * @property {number} autorunPortOverride + * @property {boolean} enableCorsProxy + * @property {boolean} disableCsrf + * @property {boolean} ssl + * @property {string} certPath + * @property {string} keyPath + * @property {boolean} whitelistMode + * @property {boolean} avoidLocalhost + * @property {boolean} basicAuthMode + * @property {boolean} requestProxyEnabled + * @property {string} requestProxyUrl + * @property {string[]} requestProxyBypass + * @property {function(): URL} getIPv4ListenUrl + * @property {function(): URL} getIPv6ListenUrl + * @property {function(boolean, boolean): Promise} getAutorunHostname + * @property {function(string): URL} getAutorunUrl + */ + +/** + * Gets the hostname to use for autorun in the browser. + * @param {boolean} useIPv6 If use IPv6 + * @param {boolean} useIPv4 If use IPv4 + * @returns {Promise} The hostname to use for autorun + */ + +export class CommandLineParser { + constructor() { + /** @type {CommandLineArguments} */ + this.default = Object.freeze({ + dataRoot: './data', + port: 8000, + listen: false, + listenAddressIPv6: '[::]', + listenAddressIPv4: '0.0.0.0', + enableIPv4: true, + enableIPv6: false, + dnsPreferIPv6: false, + autorun: false, + autorunHostname: 'auto', + autorunPortOverride: -1, + enableCorsProxy: false, + disableCsrf: false, + ssl: false, + certPath: 'certs/cert.pem', + keyPath: 'certs/privkey.pem', + whitelistMode: true, + avoidLocalhost: false, + basicAuthMode: false, + requestProxyEnabled: false, + requestProxyUrl: '', + requestProxyBypass: [], + getIPv4ListenUrl: function () { + throw new Error('getIPv4ListenUrl is not implemented'); + }, + getIPv6ListenUrl: function () { + throw new Error('getIPv6ListenUrl is not implemented'); + }, + getAutorunHostname: async function () { + throw new Error('getAutorunHostname is not implemented'); + }, + getAutorunUrl: function () { + throw new Error('getAutorunUrl is not implemented'); + }, + }); + + this.booleanAutoOptions = [true, false, 'auto']; + } + + /** + * Parses command line arguments. + * @param {string[]} args Process startup arguments. + * @returns {CommandLineArguments} Parsed command line arguments. + */ + parse(args) { + const cliArguments = yargs(hideBin(args)) + .usage('Usage: [options]') + .option('enableIPv6', { + type: 'string', + default: null, + describe: 'Enables IPv6 protocol.', + }).option('enableIPv4', { + type: 'string', + default: null, + describe: 'Enables IPv4 protocol.', + }).option('port', { + type: 'number', + default: null, + describe: 'Sets the port under which SillyTavern will run.\nIf not provided falls back to yaml config \'port\'.', + }).option('dnsPreferIPv6', { + type: 'boolean', + default: null, + describe: 'Prefers IPv6 for DNS\nyou should probably have the enabled if you\'re on an IPv6 only network\nIf not provided falls back to yaml config \'dnsPreferIPv6\'.', + }).option('autorun', { + type: 'boolean', + default: null, + describe: 'Automatically launch SillyTavern in the browser.\nAutorun is automatically disabled if --ssl is set to true.\nIf not provided falls back to yaml config \'autorun\'.', + }).option('autorunHostname', { + type: 'string', + default: null, + describe: 'Sets the autorun hostname, probably best left on \'auto\'.\nUse values like \'localhost\', \'st.example.com\'', + }).option('autorunPortOverride', { + type: 'string', + default: null, + describe: 'Overrides the port for autorun with open your browser with this port and ignore what port the server is running on. -1 is use server port', + }).option('listen', { + type: 'boolean', + default: null, + describe: 'SillyTavern is listening on all network interfaces (Wi-Fi, LAN, localhost). If false, will limit it only to internal localhost (127.0.0.1).\nIf not provided falls back to yaml config \'listen\'.', + }).option('listenAddressIPv6', { + type: 'string', + default: null, + describe: 'Set SillyTavern to listen to a specific IPv6 address. If not set, it will fallback to listen to all.', + }).option('listenAddressIPv4', { + type: 'string', + default: null, + describe: 'Set SillyTavern to listen to a specific IPv4 address. If not set, it will fallback to listen to all.', + }).option('corsProxy', { + type: 'boolean', + default: null, + describe: 'Enables CORS proxy\nIf not provided falls back to yaml config \'enableCorsProxy\'', + }).option('disableCsrf', { + type: 'boolean', + default: null, + describe: 'Disables CSRF protection', + }).option('ssl', { + type: 'boolean', + default: false, + describe: 'Enables SSL', + }).option('certPath', { + type: 'string', + default: 'certs/cert.pem', + describe: 'Path to your certificate file.', + }).option('keyPath', { + type: 'string', + default: 'certs/privkey.pem', + describe: 'Path to your private key file.', + }).option('whitelist', { + type: 'boolean', + default: null, + describe: 'Enables whitelist mode', + }).option('dataRoot', { + type: 'string', + default: null, + describe: 'Root directory for data storage', + }).option('avoidLocalhost', { + type: 'boolean', + default: null, + describe: 'Avoids using \'localhost\' for autorun in auto mode.\nuse if you don\'t have \'localhost\' in your hosts file', + }).option('basicAuthMode', { + type: 'boolean', + default: null, + describe: 'Enables basic authentication', + }).option('requestProxyEnabled', { + type: 'boolean', + default: null, + describe: 'Enables a use of proxy for outgoing requests', + }).option('requestProxyUrl', { + type: 'string', + default: null, + describe: 'Request proxy URL (HTTP or SOCKS protocols)', + }).option('requestProxyBypass', { + type: 'array', + describe: 'Request proxy bypass list (space separated list of hosts)', + }).parseSync(); + + /** @type {CommandLineArguments} */ + const result = { + dataRoot: cliArguments.dataRoot ?? getConfigValue('dataRoot', this.default.dataRoot), + port: cliArguments.port ?? getConfigValue('port', this.default.port, 'number'), + listen: cliArguments.listen ?? getConfigValue('listen', this.default.listen, 'boolean'), + listenAddressIPv6: cliArguments.listenAddressIPv6 ?? getConfigValue('listenAddress.ipv6', this.default.listenAddressIPv6), + listenAddressIPv4: cliArguments.listenAddressIPv4 ?? getConfigValue('listenAddress.ipv4', this.default.listenAddressIPv4), + enableIPv4: stringToBool(cliArguments.enableIPv4) ?? stringToBool(getConfigValue('protocol.ipv4', this.default.enableIPv4)) ?? this.default.enableIPv4, + enableIPv6: stringToBool(cliArguments.enableIPv6) ?? stringToBool(getConfigValue('protocol.ipv6', this.default.enableIPv6)) ?? this.default.enableIPv6, + dnsPreferIPv6: cliArguments.dnsPreferIPv6 ?? getConfigValue('dnsPreferIPv6', this.default.dnsPreferIPv6, 'boolean'), + autorun: cliArguments.autorun ?? getConfigValue('autorun', this.default.autorun, 'boolean'), + autorunHostname: cliArguments.autorunHostname ?? getConfigValue('autorunHostname', this.default.autorunHostname), + autorunPortOverride: cliArguments.autorunPortOverride ?? getConfigValue('autorunPortOverride', this.default.autorunPortOverride, 'number'), + enableCorsProxy: cliArguments.corsProxy ?? getConfigValue('enableCorsProxy', this.default.enableCorsProxy, 'boolean'), + disableCsrf: cliArguments.disableCsrf ?? getConfigValue('disableCsrfProtection', this.default.disableCsrf, 'boolean'), + ssl: cliArguments.ssl ?? getConfigValue('ssl.enabled', this.default.ssl, 'boolean'), + certPath: cliArguments.certPath ?? getConfigValue('ssl.certPath', this.default.certPath), + keyPath: cliArguments.keyPath ?? getConfigValue('ssl.keyPath', this.default.keyPath), + whitelistMode: cliArguments.whitelist ?? getConfigValue('whitelistMode', this.default.whitelistMode, 'boolean'), + avoidLocalhost: cliArguments.avoidLocalhost ?? getConfigValue('avoidLocalhost', this.default.avoidLocalhost, 'boolean'), + basicAuthMode: cliArguments.basicAuthMode ?? getConfigValue('basicAuthMode', this.default.basicAuthMode, 'boolean'), + requestProxyEnabled: cliArguments.requestProxyEnabled ?? getConfigValue('requestProxy.enabled', this.default.requestProxyEnabled, 'boolean'), + requestProxyUrl: cliArguments.requestProxyUrl ?? getConfigValue('requestProxy.url', this.default.requestProxyUrl), + requestProxyBypass: cliArguments.requestProxyBypass ?? getConfigValue('requestProxy.bypass', this.default.requestProxyBypass), + getIPv4ListenUrl: function () { + const isValid = ipRegex.v4({ exact: true }).test(this.listenAddressIPv4); + return new URL( + (this.ssl ? 'https://' : 'http://') + + (this.listen ? (isValid ? this.listenAddressIPv4 : '0.0.0.0') : '127.0.0.1') + + (':' + this.port), + ); + }, + getIPv6ListenUrl: function () { + const isValid = ipRegex.v6({ exact: true }).test(this.listenAddressIPv6); + return new URL( + (this.ssl ? 'https://' : 'http://') + + (this.listen ? (isValid ? this.listenAddressIPv6 : '[::]') : '[::1]') + + (':' + this.port), + ); + }, + getAutorunHostname: async function (useIPv6, useIPv4) { + if (this.autorunHostname === 'auto') { + let localhostResolve = await canResolve('localhost', useIPv6, useIPv4); + + if (useIPv6 && useIPv4) { + return (this.avoidLocalhost || !localhostResolve) ? '[::1]' : 'localhost'; + } + + if (useIPv6) { + return '[::1]'; + } + + if (useIPv4) { + return '127.0.0.1'; + } + } + + return this.autorunHostname; + }, + getAutorunUrl: function (hostname) { + const autorunPort = (this.autorunPortOverride >= 0) ? this.autorunPortOverride : this.port; + return new URL( + (this.ssl ? 'https://' : 'http://') + + (hostname) + + (':') + + (autorunPort), + ); + }, + }; + + if (!this.booleanAutoOptions.includes(result.enableIPv6)) { + console.warn(color.red('`protocol: ipv6` option invalid'), '\n use:', this.booleanAutoOptions, '\n setting to:', this.default.enableIPv6); + result.enableIPv6 = this.default.enableIPv6; + } + + if (!this.booleanAutoOptions.includes(result.enableIPv4)) { + console.warn(color.red('`protocol: ipv4` option invalid'), '\n use:', this.booleanAutoOptions, '\n setting to:', this.default.enableIPv4); + result.enableIPv4 = this.default.enableIPv4; + } + + return result; + } +} diff --git a/src/middleware/corsProxy.js b/src/middleware/corsProxy.js new file mode 100644 index 000000000..3b9f1fa9b --- /dev/null +++ b/src/middleware/corsProxy.js @@ -0,0 +1,42 @@ +import fetch from 'node-fetch'; +import { forwardFetchResponse } from '../util.js'; + +/** + * Middleware to proxy requests to a different domain + * @param {import('express').Request} req Express request object + * @param {import('express').Response} res Express response object + */ +export default async function corsProxyMiddleware(req, res) { + const url = req.params.url; // get the url from the request path + + // Disallow circular requests + const serverUrl = req.protocol + '://' + req.get('host'); + if (url.startsWith(serverUrl)) { + return res.status(400).send('Circular requests are not allowed'); + } + + try { + const headers = JSON.parse(JSON.stringify(req.headers)); + const headersToRemove = [ + 'x-csrf-token', 'host', 'referer', 'origin', 'cookie', + 'x-forwarded-for', 'x-forwarded-protocol', 'x-forwarded-proto', + 'x-forwarded-host', 'x-real-ip', 'sec-fetch-mode', + 'sec-fetch-site', 'sec-fetch-dest', + ]; + + headersToRemove.forEach(header => delete headers[header]); + + const bodyMethods = ['POST', 'PUT', 'PATCH']; + + const response = await fetch(url, { + method: req.method, + headers: headers, + body: bodyMethods.includes(req.method) ? JSON.stringify(req.body) : undefined, + }); + + // Copy over relevant response params to the proxy response + forwardFetchResponse(response, res); + } catch (error) { + res.status(500).send('Error occurred while trying to proxy to: ' + url + ' ' + error); + } +} diff --git a/src/server-startup.js b/src/server-startup.js new file mode 100644 index 000000000..656eef5f8 --- /dev/null +++ b/src/server-startup.js @@ -0,0 +1,359 @@ +import https from 'node:https'; +import http from 'node:http'; +import fs from 'node:fs'; +import { color, urlHostnameToIPv6, getHasIP } from './util.js'; + +// Express routers +import { router as movingUIRouter } from './endpoints/moving-ui.js'; +import { router as imagesRouter } from './endpoints/images.js'; +import { router as quickRepliesRouter } from './endpoints/quick-replies.js'; +import { router as avatarsRouter } from './endpoints/avatars.js'; +import { router as themesRouter } from './endpoints/themes.js'; +import { router as openAiRouter } from './endpoints/openai.js'; +import { router as googleRouter } from './endpoints/google.js'; +import { router as anthropicRouter } from './endpoints/anthropic.js'; +import { router as tokenizersRouter } from './endpoints/tokenizers.js'; +import { router as presetsRouter } from './endpoints/presets.js'; +import { router as secretsRouter } from './endpoints/secrets.js'; +import { router as thumbnailRouter } from './endpoints/thumbnails.js'; +import { router as novelAiRouter } from './endpoints/novelai.js'; +import { router as extensionsRouter } from './endpoints/extensions.js'; +import { router as assetsRouter } from './endpoints/assets.js'; +import { router as filesRouter } from './endpoints/files.js'; +import { router as charactersRouter } from './endpoints/characters.js'; +import { router as chatsRouter } from './endpoints/chats.js'; +import { router as groupsRouter } from './endpoints/groups.js'; +import { router as worldInfoRouter } from './endpoints/worldinfo.js'; +import { router as statsRouter } from './endpoints/stats.js'; +import { router as contentManagerRouter } from './endpoints/content-manager.js'; +import { router as settingsRouter } from './endpoints/settings.js'; +import { router as backgroundsRouter } from './endpoints/backgrounds.js'; +import { router as spritesRouter } from './endpoints/sprites.js'; +import { router as stableDiffusionRouter } from './endpoints/stable-diffusion.js'; +import { router as hordeRouter } from './endpoints/horde.js'; +import { router as vectorsRouter } from './endpoints/vectors.js'; +import { router as translateRouter } from './endpoints/translate.js'; +import { router as classifyRouter } from './endpoints/classify.js'; +import { router as captionRouter } from './endpoints/caption.js'; +import { router as searchRouter } from './endpoints/search.js'; +import { router as openRouterRouter } from './endpoints/openrouter.js'; +import { router as chatCompletionsRouter } from './endpoints/backends/chat-completions.js'; +import { router as koboldRouter } from './endpoints/backends/kobold.js'; +import { router as textCompletionsRouter } from './endpoints/backends/text-completions.js'; +import { router as scaleAltRouter } from './endpoints/backends/scale-alt.js'; +import { router as speechRouter } from './endpoints/speech.js'; +import { router as azureRouter } from './endpoints/azure.js'; + +/** + * @typedef {object} ServerStartupResult + * @property {boolean} v6Failed If the server failed to start on IPv6 + * @property {boolean} v4Failed If the server failed to start on IPv4 + * @property {boolean} useIPv6 If use IPv6 + * @property {boolean} useIPv4 If use IPv4 + */ + +/** + * Redirect deprecated API endpoints to their replacements. + * @param {import('express').Express} app The Express app to use + */ +export function redirectDeprecatedEndpoints(app) { + /** + * Redirect a deprecated API endpoint URL to its replacement. Because fetch, form submissions, and $.ajax follow + * redirects, this is transparent to client-side code. + * @param {string} src The URL to redirect from. + * @param {string} destination The URL to redirect to. + */ + function redirect(src, destination) { + app.use(src, (req, res) => { + console.warn(`API endpoint ${src} is deprecated; use ${destination} instead`); + // HTTP 301 causes the request to become a GET. 308 preserves the request method. + res.redirect(308, destination); + }); + } + + redirect('/createcharacter', '/api/characters/create'); + redirect('/renamecharacter', '/api/characters/rename'); + redirect('/editcharacter', '/api/characters/edit'); + redirect('/editcharacterattribute', '/api/characters/edit-attribute'); + redirect('/v2/editcharacterattribute', '/api/characters/merge-attributes'); + redirect('/deletecharacter', '/api/characters/delete'); + redirect('/getcharacters', '/api/characters/all'); + redirect('/getonecharacter', '/api/characters/get'); + redirect('/getallchatsofcharacter', '/api/characters/chats'); + redirect('/importcharacter', '/api/characters/import'); + redirect('/dupecharacter', '/api/characters/duplicate'); + redirect('/exportcharacter', '/api/characters/export'); + redirect('/savechat', '/api/chats/save'); + redirect('/getchat', '/api/chats/get'); + redirect('/renamechat', '/api/chats/rename'); + redirect('/delchat', '/api/chats/delete'); + redirect('/exportchat', '/api/chats/export'); + redirect('/importgroupchat', '/api/chats/group/import'); + redirect('/importchat', '/api/chats/import'); + redirect('/getgroupchat', '/api/chats/group/get'); + redirect('/deletegroupchat', '/api/chats/group/delete'); + redirect('/savegroupchat', '/api/chats/group/save'); + redirect('/getgroups', '/api/groups/all'); + redirect('/creategroup', '/api/groups/create'); + redirect('/editgroup', '/api/groups/edit'); + redirect('/deletegroup', '/api/groups/delete'); + redirect('/getworldinfo', '/api/worldinfo/get'); + redirect('/deleteworldinfo', '/api/worldinfo/delete'); + redirect('/importworldinfo', '/api/worldinfo/import'); + redirect('/editworldinfo', '/api/worldinfo/edit'); + redirect('/getstats', '/api/stats/get'); + redirect('/recreatestats', '/api/stats/recreate'); + redirect('/updatestats', '/api/stats/update'); + redirect('/getbackgrounds', '/api/backgrounds/all'); + redirect('/delbackground', '/api/backgrounds/delete'); + redirect('/renamebackground', '/api/backgrounds/rename'); + redirect('/downloadbackground', '/api/backgrounds/upload'); // yes, the downloadbackground endpoint actually uploads one + redirect('/savetheme', '/api/themes/save'); + redirect('/getuseravatars', '/api/avatars/get'); + redirect('/deleteuseravatar', '/api/avatars/delete'); + redirect('/uploaduseravatar', '/api/avatars/upload'); + redirect('/deletequickreply', '/api/quick-replies/delete'); + redirect('/savequickreply', '/api/quick-replies/save'); + redirect('/uploadimage', '/api/images/upload'); + redirect('/listimgfiles/:folder', '/api/images/list/:folder'); + redirect('/api/content/import', '/api/content/importURL'); + redirect('/savemovingui', '/api/moving-ui/save'); + redirect('/api/serpapi/search', '/api/search/serpapi'); + redirect('/api/serpapi/visit', '/api/search/visit'); + redirect('/api/serpapi/transcript', '/api/search/transcript'); +} + +/** + * Setup the routers for the endpoints. + * @param {import('express').Express} app The Express app to use + */ +export function setupPrivateEndpoints(app) { + app.use('/api/moving-ui', movingUIRouter); + app.use('/api/images', imagesRouter); + app.use('/api/quick-replies', quickRepliesRouter); + app.use('/api/avatars', avatarsRouter); + app.use('/api/themes', themesRouter); + app.use('/api/openai', openAiRouter); + app.use('/api/google', googleRouter); + app.use('/api/anthropic', anthropicRouter); + app.use('/api/tokenizers', tokenizersRouter); + app.use('/api/presets', presetsRouter); + app.use('/api/secrets', secretsRouter); + app.use('/thumbnail', thumbnailRouter); + app.use('/api/novelai', novelAiRouter); + app.use('/api/extensions', extensionsRouter); + app.use('/api/assets', assetsRouter); + app.use('/api/files', filesRouter); + app.use('/api/characters', charactersRouter); + app.use('/api/chats', chatsRouter); + app.use('/api/groups', groupsRouter); + app.use('/api/worldinfo', worldInfoRouter); + app.use('/api/stats', statsRouter); + app.use('/api/backgrounds', backgroundsRouter); + app.use('/api/sprites', spritesRouter); + app.use('/api/content', contentManagerRouter); + app.use('/api/settings', settingsRouter); + app.use('/api/sd', stableDiffusionRouter); + app.use('/api/horde', hordeRouter); + app.use('/api/vector', vectorsRouter); + app.use('/api/translate', translateRouter); + app.use('/api/extra/classify', classifyRouter); + app.use('/api/extra/caption', captionRouter); + app.use('/api/search', searchRouter); + app.use('/api/backends/text-completions', textCompletionsRouter); + app.use('/api/openrouter', openRouterRouter); + app.use('/api/backends/kobold', koboldRouter); + app.use('/api/backends/chat-completions', chatCompletionsRouter); + app.use('/api/backends/scale-alt', scaleAltRouter); + app.use('/api/speech', speechRouter); + app.use('/api/azure', azureRouter); +} + +/** + * Utilities for starting the express server. + */ +export class ServerStartup { + /** + * Creates a new ServerStartup instance. + * @param {import('express').Express} app The Express app to use + * @param {import('./command-line.js').CommandLineArguments} cliArgs The command-line arguments + */ + constructor(app, cliArgs) { + this.app = app; + this.cliArgs = cliArgs; + } + + /** + * Creates an HTTPS server. + * @param {URL} url The URL to listen on + * @param {number} ipVersion the ip version to use + * @returns {Promise} A promise that resolves when the server is listening + * @throws {Error} If the server fails to start + */ + #createHttpsServer(url, ipVersion) { + return new Promise((resolve, reject) => { + const sslOptions = { + cert: fs.readFileSync(this.cliArgs.certPath), + key: fs.readFileSync(this.cliArgs.keyPath), + }; + const server = https.createServer(sslOptions, this.app); + server.on('error', reject); + server.on('listening', resolve); + + let host = url.hostname; + if (ipVersion === 6) host = urlHostnameToIPv6(url.hostname); + server.listen({ + host: host, + port: Number(url.port || 443), + // see https://nodejs.org/api/net.html#serverlisten for why ipv6Only is used + ipv6Only: true, + }); + }); + } + + /** + * Creates an HTTP server. + * @param {URL} url The URL to listen on + * @param {number} ipVersion the ip version to use + * @returns {Promise} A promise that resolves when the server is listening + * @throws {Error} If the server fails to start + */ + #createHttpServer(url, ipVersion) { + return new Promise((resolve, reject) => { + const server = http.createServer(this.app); + server.on('error', reject); + server.on('listening', resolve); + + let host = url.hostname; + if (ipVersion === 6) host = urlHostnameToIPv6(url.hostname); + server.listen({ + host: host, + port: Number(url.port || 80), + // see https://nodejs.org/api/net.html#serverlisten for why ipv6Only is used + ipv6Only: true, + }); + }); + } + + /** + * Starts the server using http or https depending on config + * @param {boolean} useIPv6 If use IPv6 + * @param {boolean} useIPv4 If use IPv4 + * @returns {Promise<[boolean, boolean]>} A promise that resolves with an array of booleans indicating if the server failed to start on IPv6 and IPv4, respectively + */ + async #startHTTPorHTTPS(useIPv6, useIPv4) { + let v6Failed = false; + let v4Failed = false; + + const createFunc = this.cliArgs.ssl ? this.#createHttpsServer.bind(this) : this.#createHttpServer.bind(this); + + if (useIPv6) { + try { + await createFunc(this.cliArgs.getIPv6ListenUrl(), 6); + } catch (error) { + console.error('non-fatal error: failed to start server on IPv6'); + console.error(error); + + v6Failed = true; + } + } + + if (useIPv4) { + try { + await createFunc(this.cliArgs.getIPv4ListenUrl(), 4); + } catch (error) { + console.error('non-fatal error: failed to start server on IPv4'); + console.error(error); + + v4Failed = true; + } + } + + return [v6Failed, v4Failed]; + } + + /** + * Handles the case where the server failed to start on one or both protocols. + * @param {boolean} v6Failed If the server failed to start on IPv6 + * @param {boolean} v4Failed If the server failed to start on IPv4 + * @param {boolean} useIPv6 If use IPv6 + * @param {boolean} useIPv4 If use IPv4 + * @returns {void} + */ + #handleServerListenFail(v6Failed, v4Failed, useIPv6, useIPv4) { + if (v6Failed && !useIPv4) { + console.error(color.red('fatal error: Failed to start server on IPv6 and IPv4 disabled')); + process.exit(1); + } + + if (v4Failed && !useIPv6) { + console.error(color.red('fatal error: Failed to start server on IPv4 and IPv6 disabled')); + process.exit(1); + } + + if (v6Failed && v4Failed) { + console.error(color.red('fatal error: Failed to start server on both IPv6 and IPv4')); + process.exit(1); + } + } + + /** + * Performs the server startup. + * @returns {Promise} A promise that resolves with an object containing the results of the server startup + */ + async start() { + let useIPv6 = (this.cliArgs.enableIPv6 === true); + let useIPv4 = (this.cliArgs.enableIPv4 === true); + + let hasIPv6 = false, + hasIPv4 = false, + hasIPv6Local = false, + hasIPv4Local = false, + hasIPv6Any = false, + hasIPv4Any = false; + + if (this.cliArgs.enableIPv6 === 'auto' || this.cliArgs.enableIPv4 === 'auto') { + [hasIPv6Any, hasIPv4Any, hasIPv6Local, hasIPv4Local] = await getHasIP(); + + hasIPv6 = this.cliArgs.listen ? hasIPv6Any : hasIPv6Local; + if (this.cliArgs.enableIPv6 === 'auto') { + useIPv6 = hasIPv6; + } + if (hasIPv6) { + if (useIPv6) { + console.log(color.green('IPv6 support detected')); + } else { + console.log('IPv6 support detected (but disabled)'); + } + } + + hasIPv4 = this.cliArgs.listen ? hasIPv4Any : hasIPv4Local; + if (this.cliArgs.enableIPv4 === 'auto') { + useIPv4 = hasIPv4; + } + if (hasIPv4) { + if (useIPv4) { + console.log(color.green('IPv4 support detected')); + } else { + console.log('IPv4 support detected (but disabled)'); + } + } + + if (this.cliArgs.enableIPv6 === 'auto' && this.cliArgs.enableIPv4 === 'auto') { + if (!hasIPv6 && !hasIPv4) { + console.error('Both IPv6 and IPv4 are not detected'); + process.exit(1); + } + } + } + + if (!useIPv6 && !useIPv4) { + console.error('Both IPv6 and IPv4 are disabled'); + process.exit(1); + } + + const [v6Failed, v4Failed] = await this.#startHTTPorHTTPS(useIPv6, useIPv4); + this.#handleServerListenFail(v6Failed, v4Failed, useIPv6, useIPv4); + return { v6Failed, v4Failed, useIPv6, useIPv4 }; + } +} diff --git a/src/users.js b/src/users.js index b02c6e935..f6e383b38 100644 --- a/src/users.js +++ b/src/users.js @@ -14,7 +14,7 @@ import archiver from 'archiver'; import _ from 'lodash'; import { sync as writeFileAtomicSync } from 'write-file-atomic'; -import { USER_DIRECTORY_TEMPLATE, DEFAULT_USER, PUBLIC_DIRECTORIES, SETTINGS_FILE } from './constants.js'; +import { USER_DIRECTORY_TEMPLATE, DEFAULT_USER, PUBLIC_DIRECTORIES, SETTINGS_FILE, UPLOADS_DIRECTORY } from './constants.js'; import { getConfigValue, color, delay, generateTimestamp } from './util.js'; import { readSecret, writeSecret } from './endpoints/secrets.js'; import { getContentOfType } from './endpoints/content-manager.js'; @@ -120,6 +120,27 @@ export async function ensurePublicDirectoriesExist() { return directoriesList; } +export function cleanUploads() { + try { + const uploadsPath = path.join(globalThis.DATA_ROOT, UPLOADS_DIRECTORY); + if (fs.existsSync(uploadsPath)) { + const uploads = fs.readdirSync(uploadsPath); + + if (!uploads.length) { + return; + } + + console.debug(`Cleaning uploads folder (${uploads.length} files)`); + uploads.forEach(file => { + const pathToFile = path.join(uploadsPath, file); + fs.unlinkSync(pathToFile); + }); + } + } catch (err) { + console.error(err); + } +} + /** * Gets a list of all user directories. * @returns {Promise} - The list of user directories @@ -478,6 +499,25 @@ export function getCookieSessionName() { return `session-${suffix}`; } +export function getSessionCookieAge() { + // Defaults to "no expiration" if not set + const configValue = getConfigValue('sessionTimeout', -1, 'number'); + + // Convert to milliseconds + if (configValue > 0) { + return configValue * 1000; + } + + // "No expiration" is just 400 days as per RFC 6265 + if (configValue < 0) { + return 400 * 24 * 60 * 60 * 1000; + } + + // 0 means session cookie is deleted when the browser session ends + // (depends on the implementation of the browser) + return undefined; +} + /** * Hashes a password using scrypt with the provided salt. * @param {string} password Password to hash diff --git a/src/util.js b/src/util.js index e98eb4978..aa2075d61 100644 --- a/src/util.js +++ b/src/util.js @@ -6,6 +6,7 @@ import { Readable } from 'node:stream'; import { createRequire } from 'node:module'; import { Buffer } from 'node:buffer'; import { promises as dnsPromise } from 'node:dns'; +import os from 'os'; import yaml from 'yaml'; import { sync as commandExistsSync } from 'command-exists'; @@ -770,6 +771,56 @@ export async function canResolve(name, useIPv6 = true, useIPv4 = true) { } } +/** + * Checks the network interfaces to determine the presence of IPv6 and IPv4 addresses. + * + * @returns {Promise<[boolean, boolean, boolean, boolean]>} A promise that resolves to an array containing: + * - [0]: `hasIPv6` (boolean) - Whether the computer has any IPv6 address, including (`::1`). + * - [1]: `hasIPv4` (boolean) - Whether the computer has any IPv4 address, including (`127.0.0.1`). + * - [2]: `hasIPv6Local` (boolean) - Whether the computer has local IPv6 address (`::1`). + * - [3]: `hasIPv4Local` (boolean) - Whether the computer has local IPv4 address (`127.0.0.1`). + */ +export async function getHasIP() { + let hasIPv6 = false; + let hasIPv6Local = false; + + let hasIPv4 = false; + let hasIPv4Local = false; + + const interfaces = os.networkInterfaces(); + + for (const iface of Object.values(interfaces)) { + if (iface === undefined) { + continue; + } + + for (const info of iface) { + if (info.family === 'IPv6') { + hasIPv6 = true; + if (info.address === '::1') { + hasIPv6Local = true; + } + } + + if (info.family === 'IPv4') { + hasIPv4 = true; + if (info.address === '127.0.0.1') { + hasIPv4Local = true; + } + } + if (hasIPv6 && hasIPv4 && hasIPv6Local && hasIPv4Local) break; + } + if (hasIPv6 && hasIPv4 && hasIPv6Local && hasIPv4Local) break; + } + return [ + hasIPv6, + hasIPv4, + hasIPv6Local, + hasIPv4Local, + ]; +} + + /** * Converts various JavaScript primitives to boolean values. * Handles special case for "true"/"false" strings (case-insensitive) @@ -1005,3 +1056,16 @@ export function safeReadFileSync(filePath, options = { encoding: 'utf-8' }) { if (fs.existsSync(filePath)) return fs.readFileSync(filePath, options); return null; } + +/** + * Set the title of the terminal window + * @param {string} title Desired title for the window + */ +export function setWindowTitle(title) { + if (process.platform === 'win32') { + process.title = title; + } + else { + process.stdout.write(`\x1b]2;${title}\x1b\x5c`); + } +}