diff --git a/server.js b/server.js index 58f7b72bc..8354e2d21 100644 --- a/server.js +++ b/server.js @@ -51,12 +51,12 @@ util.inspect.defaultOptions.maxStringLength = null; // local library imports const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware'); const characterCardParser = require('./src/character-card-parser.js'); -const contentManager = require('./src/content-manager'); +const contentManager = require('./src/endpoints/content-manager'); const statsHelpers = require('./statsHelpers.js'); -const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/secrets'); +const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets'); const { delay, getVersion, deepMerge, getConfigValue, color, uuidv4 } = require('./src/util'); -const { invalidateThumbnail, ensureThumbnailCache } = require('./src/thumbnails'); -const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/tokenizers'); +const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails'); +const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers'); const { convertClaudePrompt } = require('./src/chat-completion'); // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. @@ -2002,7 +2002,7 @@ app.post('/importcharacter', urlencodedParser, async function (request, response let uploadPath = path.join(UPLOADS_PATH, filedata.filename); var format = request.body.file_type; const defaultAvatarPath = './public/img/ai4.png'; - const { importRisuSprites } = require('./src/sprites'); + const { importRisuSprites } = require('./src/endpoints/sprites'); //console.log(format); if (filedata) { if (format == 'json') { @@ -3576,55 +3576,55 @@ async function fetchJSON(url, args = {}) { // ** END ** // OpenAI API -require('./src/openai').registerEndpoints(app, jsonParser, urlencodedParser); +require('./src/endpoints/openai').registerEndpoints(app, jsonParser, urlencodedParser); // Tokenizers -require('./src/tokenizers').registerEndpoints(app, jsonParser); +require('./src/endpoints/tokenizers').registerEndpoints(app, jsonParser); // Preset management -require('./src/presets').registerEndpoints(app, jsonParser); +require('./src/endpoints/presets').registerEndpoints(app, jsonParser); // Secrets managemenet -require('./src/secrets').registerEndpoints(app, jsonParser); +require('./src/endpoints/secrets').registerEndpoints(app, jsonParser); // Thumbnail generation -require('./src/thumbnails').registerEndpoints(app, jsonParser); +require('./src/endpoints/thumbnails').registerEndpoints(app, jsonParser); // NovelAI generation -require('./src/novelai').registerEndpoints(app, jsonParser); +require('./src/endpoints/novelai').registerEndpoints(app, jsonParser); // Third-party extensions -require('./src/extensions').registerEndpoints(app, jsonParser); +require('./src/endpoints/extensions').registerEndpoints(app, jsonParser); // Asset management -require('./src/assets').registerEndpoints(app, jsonParser); +require('./src/endpoints/assets').registerEndpoints(app, jsonParser); // Character sprite management -require('./src/sprites').registerEndpoints(app, jsonParser, urlencodedParser); +require('./src/endpoints/sprites').registerEndpoints(app, jsonParser, urlencodedParser); // Custom content management -require('./src/content-manager').registerEndpoints(app, jsonParser); +require('./src/endpoints/content-manager').registerEndpoints(app, jsonParser); // Stable Diffusion generation -require('./src/stable-diffusion').registerEndpoints(app, jsonParser); +require('./src/endpoints/stable-diffusion').registerEndpoints(app, jsonParser); // LLM and SD Horde generation -require('./src/horde').registerEndpoints(app, jsonParser); +require('./src/endpoints/horde').registerEndpoints(app, jsonParser); // Vector storage DB -require('./src/vectors').registerEndpoints(app, jsonParser); +require('./src/endpoints/vectors').registerEndpoints(app, jsonParser); // Chat translation -require('./src/translate').registerEndpoints(app, jsonParser); +require('./src/endpoints/translate').registerEndpoints(app, jsonParser); // Emotion classification -require('./src/classify').registerEndpoints(app, jsonParser); +require('./src/endpoints/classify').registerEndpoints(app, jsonParser); // Image captioning -require('./src/caption').registerEndpoints(app, jsonParser); +require('./src/endpoints/caption').registerEndpoints(app, jsonParser); // Web search extension -require('./src/serpapi').registerEndpoints(app, jsonParser); +require('./src/endpoints/serpapi').registerEndpoints(app, jsonParser); const tavernUrl = new URL( (cliArguments.ssl ? 'https://' : 'http://') + diff --git a/src/assets.js b/src/endpoints/assets.js similarity index 99% rename from src/assets.js rename to src/endpoints/assets.js index 02c41bc6a..5264e3576 100644 --- a/src/assets.js +++ b/src/endpoints/assets.js @@ -4,7 +4,7 @@ const sanitize = require('sanitize-filename'); const fetch = require('node-fetch').default; const { finished } = require('stream/promises'); const writeFileSyncAtomic = require('write-file-atomic').sync; -const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('./constants'); +const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants'); const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d']; diff --git a/src/caption.js b/src/endpoints/caption.js similarity index 93% rename from src/caption.js rename to src/endpoints/caption.js index 4ffcfec8e..81a8d029d 100644 --- a/src/caption.js +++ b/src/endpoints/caption.js @@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) { try { const { image } = req.body; - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const rawImage = await module.default.getRawImage(image); if (!rawImage) { diff --git a/src/classify.js b/src/endpoints/classify.js similarity index 92% rename from src/classify.js rename to src/endpoints/classify.js index 8773deb28..dc0b8fb90 100644 --- a/src/classify.js +++ b/src/endpoints/classify.js @@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) { app.post('/api/extra/classify/labels', jsonParser, async (req, res) => { try { - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = Object.keys(pipe.model.config.label2id); return res.json({ labels: result }); @@ -27,7 +27,7 @@ function registerEndpoints(app, jsonParser) { if (Object.hasOwn(cacheObject, text)) { return cacheObject[text]; } else { - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = await pipe(text, { topk: 5 }); result.sort((a, b) => b.score - a.score); diff --git a/src/content-manager.js b/src/endpoints/content-manager.js similarity index 99% rename from src/content-manager.js rename to src/endpoints/content-manager.js index 955e9fc63..cc4796980 100644 --- a/src/content-manager.js +++ b/src/endpoints/content-manager.js @@ -2,11 +2,11 @@ const fs = require('fs'); const path = require('path'); const fetch = require('node-fetch').default; const sanitize = require('sanitize-filename'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const contentDirectory = path.join(process.cwd(), 'default/content'); const contentLogPath = path.join(contentDirectory, 'content.log'); const contentIndexPath = path.join(contentDirectory, 'index.json'); -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); const presetFolders = [DIRECTORIES.koboldAI_Settings, DIRECTORIES.openAI_Settings, DIRECTORIES.novelAI_Settings, DIRECTORIES.textGen_Settings]; /** diff --git a/src/extensions.js b/src/endpoints/extensions.js similarity index 99% rename from src/extensions.js rename to src/endpoints/extensions.js index 0ee083cce..4e3e54d39 100644 --- a/src/extensions.js +++ b/src/endpoints/extensions.js @@ -2,7 +2,7 @@ const path = require('path'); const fs = require('fs'); const { default: simpleGit } = require('simple-git'); const sanitize = require('sanitize-filename'); -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); /** * This function extracts the extension information from the manifest file. diff --git a/src/horde.js b/src/endpoints/horde.js similarity index 99% rename from src/horde.js rename to src/endpoints/horde.js index e649971a2..c662be317 100644 --- a/src/horde.js +++ b/src/endpoints/horde.js @@ -1,6 +1,6 @@ const fetch = require('node-fetch').default; -const AIHorde = require('./ai_horde'); -const { getVersion, delay } = require('./util'); +const AIHorde = require('../ai_horde'); +const { getVersion, delay } = require('../util'); const { readSecret, SECRET_KEYS } = require('./secrets'); const ANONYMOUS_KEY = '0000000000'; diff --git a/src/novelai.js b/src/endpoints/novelai.js similarity index 99% rename from src/novelai.js rename to src/endpoints/novelai.js index 161a636a1..70a3305ce 100644 --- a/src/novelai.js +++ b/src/endpoints/novelai.js @@ -2,7 +2,7 @@ const fetch = require('node-fetch').default; const util = require('util'); const { Readable } = require('stream'); const { readSecret, SECRET_KEYS } = require('./secrets'); -const { readAllChunks, extractFileFromZipBuffer } = require('./util'); +const { readAllChunks, extractFileFromZipBuffer } = require('../util'); const API_NOVELAI = 'https://api.novelai.net'; diff --git a/src/openai.js b/src/endpoints/openai.js similarity index 100% rename from src/openai.js rename to src/endpoints/openai.js diff --git a/src/presets.js b/src/endpoints/presets.js similarity index 98% rename from src/presets.js rename to src/endpoints/presets.js index 24eecf917..3b2ceba51 100644 --- a/src/presets.js +++ b/src/endpoints/presets.js @@ -2,7 +2,7 @@ const fs = require('fs'); const path = require('path'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); const { getDefaultPresetFile, getDefaultPresets } = require('./content-manager'); /** diff --git a/src/secrets.js b/src/endpoints/secrets.js similarity index 99% rename from src/secrets.js rename to src/endpoints/secrets.js index 63436b0c0..c612836cc 100644 --- a/src/secrets.js +++ b/src/endpoints/secrets.js @@ -1,6 +1,6 @@ const fs = require('fs'); const path = require('path'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const writeFileAtomicSync = require('write-file-atomic').sync; const SECRETS_FILE = path.join(process.cwd(), './secrets.json'); diff --git a/src/serpapi.js b/src/endpoints/serpapi.js similarity index 100% rename from src/serpapi.js rename to src/endpoints/serpapi.js diff --git a/src/sprites.js b/src/endpoints/sprites.js similarity index 98% rename from src/sprites.js rename to src/endpoints/sprites.js index f708b4692..67b282e9d 100644 --- a/src/sprites.js +++ b/src/endpoints/sprites.js @@ -4,8 +4,8 @@ const path = require('path'); const mime = require('mime-types'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES, UPLOADS_PATH } = require('./constants'); -const { getImageBuffers } = require('./util'); +const { DIRECTORIES, UPLOADS_PATH } = require('../constants'); +const { getImageBuffers } = require('../util'); /** * Gets the path to the sprites folder for the provided character name diff --git a/src/stable-diffusion.js b/src/endpoints/stable-diffusion.js similarity index 99% rename from src/stable-diffusion.js rename to src/endpoints/stable-diffusion.js index f13bc78a0..f6c94be98 100644 --- a/src/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -1,8 +1,8 @@ const fetch = require('node-fetch').default; const sanitize = require('sanitize-filename'); -const { getBasicAuthHeader, delay } = require('./util'); +const { getBasicAuthHeader, delay } = require('../util.js'); const fs = require('fs'); -const { DIRECTORIES } = require('./constants.js'); +const { DIRECTORIES } = require('../constants.js'); const writeFileAtomicSync = require('write-file-atomic').sync; /** @@ -341,7 +341,7 @@ function registerEndpoints(app, jsonParser) { try { const task = 'text-generation'; - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(task); const result = await pipe(prompt, { num_beams: 1, max_new_tokens: 256, do_sample: true }); diff --git a/src/thumbnails.js b/src/endpoints/thumbnails.js similarity index 98% rename from src/thumbnails.js rename to src/endpoints/thumbnails.js index 4120ae3db..d150f0cfb 100644 --- a/src/thumbnails.js +++ b/src/endpoints/thumbnails.js @@ -3,8 +3,8 @@ const path = require('path'); const sanitize = require('sanitize-filename'); const jimp = require('jimp'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('./constants'); -const { getConfigValue } = require('./util'); +const { DIRECTORIES } = require('../constants'); +const { getConfigValue } = require('../util'); /** * Gets a path to thumbnail folder based on the type. diff --git a/src/tokenizers.js b/src/endpoints/tokenizers.js similarity index 99% rename from src/tokenizers.js rename to src/endpoints/tokenizers.js index 644a1d22a..d73af29b1 100644 --- a/src/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -3,7 +3,7 @@ const path = require('path'); const { SentencePieceProcessor } = require('@agnai/sentencepiece-js'); const tiktoken = require('@dqbd/tiktoken'); const { Tokenizer } = require('@agnai/web-tokenizers'); -const { convertClaudePrompt } = require('./chat-completion'); +const { convertClaudePrompt } = require('../chat-completion'); const { readSecret, SECRET_KEYS } = require('./secrets'); /** diff --git a/src/translate.js b/src/endpoints/translate.js similarity index 99% rename from src/translate.js rename to src/endpoints/translate.js index 2528b9b4b..a7155c5e4 100644 --- a/src/translate.js +++ b/src/endpoints/translate.js @@ -1,7 +1,7 @@ const fetch = require('node-fetch').default; const https = require('https'); const { readSecret, SECRET_KEYS } = require('./secrets'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate'; const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate'; diff --git a/src/vectors.js b/src/endpoints/vectors.js similarity index 97% rename from src/vectors.js rename to src/endpoints/vectors.js index a21e246e8..a6b8e5d56 100644 --- a/src/vectors.js +++ b/src/endpoints/vectors.js @@ -11,11 +11,11 @@ const sanitize = require('sanitize-filename'); async function getVector(source, text) { switch (source) { case 'openai': - return require('./openai-vectors').getOpenAIVector(text); + return require('../openai-vectors').getOpenAIVector(text); case 'transformers': - return require('./embedding').getTransformersVector(text); + return require('../embedding').getTransformersVector(text); case 'palm': - return require('./palm-vectors').getPaLMVector(text); + return require('../palm-vectors').getPaLMVector(text); } throw new Error(`Unknown vector source ${source}`); diff --git a/src/openai-vectors.js b/src/openai-vectors.js index e325d6ecb..ecb245065 100644 --- a/src/openai-vectors.js +++ b/src/openai-vectors.js @@ -1,5 +1,5 @@ const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('./secrets'); +const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); /** * Gets the vector for the given text from OpenAI ada model diff --git a/src/palm-vectors.js b/src/palm-vectors.js index af885068f..788b474cd 100644 --- a/src/palm-vectors.js +++ b/src/palm-vectors.js @@ -1,5 +1,5 @@ const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('./secrets'); +const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); /** * Gets the vector for the given text from PaLM gecko model