From 8283d4dc08e7b3f2108fb3efe9a35241e2c05bec Mon Sep 17 00:00:00 2001 From: LenAnderson Date: Mon, 4 Dec 2023 13:28:58 +0000 Subject: [PATCH 01/10] fix: ctx button id after reordering --- public/scripts/extensions/quick-reply/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/scripts/extensions/quick-reply/index.js b/public/scripts/extensions/quick-reply/index.js index ad5bb74e0..be5eb0414 100644 --- a/public/scripts/extensions/quick-reply/index.js +++ b/public/scripts/extensions/quick-reply/index.js @@ -692,9 +692,11 @@ function saveQROrder() { //update html-level order data to match new sort let i = 1; $('#quickReplyContainer').children().each(function () { + const oldOrder = $(this).attr('data-order'); $(this).attr('data-order', i); $(this).find('input').attr('id', `quickReply${i}Label`); $(this).find('textarea').attr('id', `quickReply${i}Mes`); + $(this).find(`#quickReply${oldOrder}CtxButton`).attr('id', `quickReply${i}CtxButton`); i++; }); From 76bfad8cdccf42e54c33523201fb9f2ee5ef8e49 Mon Sep 17 00:00:00 2001 From: LenAnderson Date: Mon, 4 Dec 2023 13:33:28 +0000 Subject: [PATCH 02/10] fix: listeners calling wrong index after reorder --- public/scripts/extensions/quick-reply/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/scripts/extensions/quick-reply/index.js b/public/scripts/extensions/quick-reply/index.js index be5eb0414..50f3aba72 100644 --- a/public/scripts/extensions/quick-reply/index.js +++ b/public/scripts/extensions/quick-reply/index.js @@ -636,9 +636,9 @@ function generateQuickReplyElements() { $('#quickReplyContainer').empty().append(quickReplyHtml); for (let i = 1; i <= extension_settings.quickReply.numberOfSlots; i++) { - $(`#quickReply${i}Mes`).on('input', function () { onQuickReplyInput(i); }); - $(`#quickReply${i}Label`).on('input', function () { onQuickReplyLabelInput(i); }); - $(`#quickReply${i}CtxButton`).on('click', function () { onQuickReplyCtxButtonClick(i); }); + $(`#quickReply${i}Mes`).on('input', function () { onQuickReplyInput(this.closest('[data-order]').getAttribute('data-order')); }); + $(`#quickReply${i}Label`).on('input', function () { onQuickReplyLabelInput(this.closest('[data-order]').getAttribute('data-order')); }); + $(`#quickReply${i}CtxButton`).on('click', function () { onQuickReplyCtxButtonClick(this.closest('[data-order]').getAttribute('data-order')); }); $(`#quickReplyContainer > [data-order="${i}"]`).attr('data-contextMenu', JSON.stringify(extension_settings.quickReply.quickReplySlots[i - 1]?.contextMenu ?? [])); } From 24546621e354560596d6693d63a3176c4418ce66 Mon Sep 17 00:00:00 2001 From: valadaptive Date: Sun, 3 Dec 2023 08:52:43 -0500 Subject: [PATCH 03/10] Move endpoints into their own folder --- server.js | 44 ++++++++++++------------- src/{ => endpoints}/assets.js | 2 +- src/{ => endpoints}/caption.js | 2 +- src/{ => endpoints}/classify.js | 4 +-- src/{ => endpoints}/content-manager.js | 4 +-- src/{ => endpoints}/extensions.js | 2 +- src/{ => endpoints}/horde.js | 4 +-- src/{ => endpoints}/novelai.js | 2 +- src/{ => endpoints}/openai.js | 0 src/{ => endpoints}/presets.js | 2 +- src/{ => endpoints}/secrets.js | 2 +- src/{ => endpoints}/serpapi.js | 0 src/{ => endpoints}/sprites.js | 4 +-- src/{ => endpoints}/stable-diffusion.js | 6 ++-- src/{ => endpoints}/thumbnails.js | 4 +-- src/{ => endpoints}/tokenizers.js | 2 +- src/{ => endpoints}/translate.js | 2 +- src/{ => endpoints}/vectors.js | 6 ++-- src/openai-vectors.js | 2 +- src/palm-vectors.js | 2 +- 20 files changed, 48 insertions(+), 48 deletions(-) rename src/{ => endpoints}/assets.js (99%) rename src/{ => endpoints}/caption.js (93%) rename src/{ => endpoints}/classify.js (92%) rename src/{ => endpoints}/content-manager.js (99%) rename src/{ => endpoints}/extensions.js (99%) rename src/{ => endpoints}/horde.js (99%) rename src/{ => endpoints}/novelai.js (99%) rename src/{ => endpoints}/openai.js (100%) rename src/{ => endpoints}/presets.js (98%) rename src/{ => endpoints}/secrets.js (99%) rename src/{ => endpoints}/serpapi.js (100%) rename src/{ => endpoints}/sprites.js (98%) rename src/{ => endpoints}/stable-diffusion.js (99%) rename src/{ => endpoints}/thumbnails.js (98%) rename src/{ => endpoints}/tokenizers.js (99%) rename src/{ => endpoints}/translate.js (99%) rename src/{ => endpoints}/vectors.js (97%) diff --git a/server.js b/server.js index 58f7b72bc..8354e2d21 100644 --- a/server.js +++ b/server.js @@ -51,12 +51,12 @@ util.inspect.defaultOptions.maxStringLength = null; // local library imports const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware'); const characterCardParser = require('./src/character-card-parser.js'); -const contentManager = require('./src/content-manager'); +const contentManager = require('./src/endpoints/content-manager'); const statsHelpers = require('./statsHelpers.js'); -const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/secrets'); +const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets'); const { delay, getVersion, deepMerge, getConfigValue, color, uuidv4 } = require('./src/util'); -const { invalidateThumbnail, ensureThumbnailCache } = require('./src/thumbnails'); -const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/tokenizers'); +const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails'); +const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers'); const { convertClaudePrompt } = require('./src/chat-completion'); // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. @@ -2002,7 +2002,7 @@ app.post('/importcharacter', urlencodedParser, async function (request, response let uploadPath = path.join(UPLOADS_PATH, filedata.filename); var format = request.body.file_type; const defaultAvatarPath = './public/img/ai4.png'; - const { importRisuSprites } = require('./src/sprites'); + const { importRisuSprites } = require('./src/endpoints/sprites'); //console.log(format); if (filedata) { if (format == 'json') { @@ -3576,55 +3576,55 @@ async function fetchJSON(url, args = {}) { // ** END ** // OpenAI API -require('./src/openai').registerEndpoints(app, jsonParser, urlencodedParser); +require('./src/endpoints/openai').registerEndpoints(app, jsonParser, urlencodedParser); // Tokenizers -require('./src/tokenizers').registerEndpoints(app, jsonParser); +require('./src/endpoints/tokenizers').registerEndpoints(app, jsonParser); // Preset management -require('./src/presets').registerEndpoints(app, jsonParser); +require('./src/endpoints/presets').registerEndpoints(app, jsonParser); // Secrets managemenet -require('./src/secrets').registerEndpoints(app, jsonParser); +require('./src/endpoints/secrets').registerEndpoints(app, jsonParser); // Thumbnail generation -require('./src/thumbnails').registerEndpoints(app, jsonParser); +require('./src/endpoints/thumbnails').registerEndpoints(app, jsonParser); // NovelAI generation -require('./src/novelai').registerEndpoints(app, jsonParser); +require('./src/endpoints/novelai').registerEndpoints(app, jsonParser); // Third-party extensions -require('./src/extensions').registerEndpoints(app, jsonParser); +require('./src/endpoints/extensions').registerEndpoints(app, jsonParser); // Asset management -require('./src/assets').registerEndpoints(app, jsonParser); +require('./src/endpoints/assets').registerEndpoints(app, jsonParser); // Character sprite management -require('./src/sprites').registerEndpoints(app, jsonParser, urlencodedParser); +require('./src/endpoints/sprites').registerEndpoints(app, jsonParser, urlencodedParser); // Custom content management -require('./src/content-manager').registerEndpoints(app, jsonParser); +require('./src/endpoints/content-manager').registerEndpoints(app, jsonParser); // Stable Diffusion generation -require('./src/stable-diffusion').registerEndpoints(app, jsonParser); +require('./src/endpoints/stable-diffusion').registerEndpoints(app, jsonParser); // LLM and SD Horde generation -require('./src/horde').registerEndpoints(app, jsonParser); +require('./src/endpoints/horde').registerEndpoints(app, jsonParser); // Vector storage DB -require('./src/vectors').registerEndpoints(app, jsonParser); +require('./src/endpoints/vectors').registerEndpoints(app, jsonParser); // Chat translation -require('./src/translate').registerEndpoints(app, jsonParser); +require('./src/endpoints/translate').registerEndpoints(app, jsonParser); // Emotion classification -require('./src/classify').registerEndpoints(app, jsonParser); +require('./src/endpoints/classify').registerEndpoints(app, jsonParser); // Image captioning -require('./src/caption').registerEndpoints(app, jsonParser); +require('./src/endpoints/caption').registerEndpoints(app, jsonParser); // Web search extension -require('./src/serpapi').registerEndpoints(app, jsonParser); +require('./src/endpoints/serpapi').registerEndpoints(app, jsonParser); const tavernUrl = new URL( (cliArguments.ssl ? 'https://' : 'http://') + diff --git a/src/assets.js b/src/endpoints/assets.js similarity index 99% rename from src/assets.js rename to src/endpoints/assets.js index 02c41bc6a..5264e3576 100644 --- a/src/assets.js +++ b/src/endpoints/assets.js @@ -4,7 +4,7 @@ const sanitize = require('sanitize-filename'); const fetch = require('node-fetch').default; const { finished } = require('stream/promises'); const writeFileSyncAtomic = require('write-file-atomic').sync; -const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('./constants'); +const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants'); const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d']; diff --git a/src/caption.js b/src/endpoints/caption.js similarity index 93% rename from src/caption.js rename to src/endpoints/caption.js index 4ffcfec8e..81a8d029d 100644 --- a/src/caption.js +++ b/src/endpoints/caption.js @@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) { try { const { image } = req.body; - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const rawImage = await module.default.getRawImage(image); if (!rawImage) { diff --git a/src/classify.js b/src/endpoints/classify.js similarity index 92% rename from src/classify.js rename to src/endpoints/classify.js index 8773deb28..dc0b8fb90 100644 --- a/src/classify.js +++ b/src/endpoints/classify.js @@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) { app.post('/api/extra/classify/labels', jsonParser, async (req, res) => { try { - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = Object.keys(pipe.model.config.label2id); return res.json({ labels: result }); @@ -27,7 +27,7 @@ function registerEndpoints(app, jsonParser) { if (Object.hasOwn(cacheObject, text)) { return cacheObject[text]; } else { - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = await pipe(text, { topk: 5 }); result.sort((a, b) => b.score - a.score); diff --git a/src/content-manager.js b/src/endpoints/content-manager.js similarity index 99% rename from src/content-manager.js rename to src/endpoints/content-manager.js index 955e9fc63..cc4796980 100644 --- a/src/content-manager.js +++ b/src/endpoints/content-manager.js @@ -2,11 +2,11 @@ const fs = require('fs'); const path = require('path'); const fetch = require('node-fetch').default; const sanitize = require('sanitize-filename'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const contentDirectory = path.join(process.cwd(), 'default/content'); const contentLogPath = path.join(contentDirectory, 'content.log'); const contentIndexPath = path.join(contentDirectory, 'index.json'); -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); const presetFolders = [DIRECTORIES.koboldAI_Settings, DIRECTORIES.openAI_Settings, DIRECTORIES.novelAI_Settings, DIRECTORIES.textGen_Settings]; /** diff --git a/src/extensions.js b/src/endpoints/extensions.js similarity index 99% rename from src/extensions.js rename to src/endpoints/extensions.js index 0ee083cce..4e3e54d39 100644 --- a/src/extensions.js +++ b/src/endpoints/extensions.js @@ -2,7 +2,7 @@ const path = require('path'); const fs = require('fs'); const { default: simpleGit } = require('simple-git'); const sanitize = require('sanitize-filename'); -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); /** * This function extracts the extension information from the manifest file. diff --git a/src/horde.js b/src/endpoints/horde.js similarity index 99% rename from src/horde.js rename to src/endpoints/horde.js index e649971a2..c662be317 100644 --- a/src/horde.js +++ b/src/endpoints/horde.js @@ -1,6 +1,6 @@ const fetch = require('node-fetch').default; -const AIHorde = require('./ai_horde'); -const { getVersion, delay } = require('./util'); +const AIHorde = require('../ai_horde'); +const { getVersion, delay } = require('../util'); const { readSecret, SECRET_KEYS } = require('./secrets'); const ANONYMOUS_KEY = '0000000000'; diff --git a/src/novelai.js b/src/endpoints/novelai.js similarity index 99% rename from src/novelai.js rename to src/endpoints/novelai.js index 161a636a1..70a3305ce 100644 --- a/src/novelai.js +++ b/src/endpoints/novelai.js @@ -2,7 +2,7 @@ const fetch = require('node-fetch').default; const util = require('util'); const { Readable } = require('stream'); const { readSecret, SECRET_KEYS } = require('./secrets'); -const { readAllChunks, extractFileFromZipBuffer } = require('./util'); +const { readAllChunks, extractFileFromZipBuffer } = require('../util'); const API_NOVELAI = 'https://api.novelai.net'; diff --git a/src/openai.js b/src/endpoints/openai.js similarity index 100% rename from src/openai.js rename to src/endpoints/openai.js diff --git a/src/presets.js b/src/endpoints/presets.js similarity index 98% rename from src/presets.js rename to src/endpoints/presets.js index 24eecf917..3b2ceba51 100644 --- a/src/presets.js +++ b/src/endpoints/presets.js @@ -2,7 +2,7 @@ const fs = require('fs'); const path = require('path'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); const { getDefaultPresetFile, getDefaultPresets } = require('./content-manager'); /** diff --git a/src/secrets.js b/src/endpoints/secrets.js similarity index 99% rename from src/secrets.js rename to src/endpoints/secrets.js index 63436b0c0..c612836cc 100644 --- a/src/secrets.js +++ b/src/endpoints/secrets.js @@ -1,6 +1,6 @@ const fs = require('fs'); const path = require('path'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const writeFileAtomicSync = require('write-file-atomic').sync; const SECRETS_FILE = path.join(process.cwd(), './secrets.json'); diff --git a/src/serpapi.js b/src/endpoints/serpapi.js similarity index 100% rename from src/serpapi.js rename to src/endpoints/serpapi.js diff --git a/src/sprites.js b/src/endpoints/sprites.js similarity index 98% rename from src/sprites.js rename to src/endpoints/sprites.js index f708b4692..67b282e9d 100644 --- a/src/sprites.js +++ b/src/endpoints/sprites.js @@ -4,8 +4,8 @@ const path = require('path'); const mime = require('mime-types'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES, UPLOADS_PATH } = require('./constants'); -const { getImageBuffers } = require('./util'); +const { DIRECTORIES, UPLOADS_PATH } = require('../constants'); +const { getImageBuffers } = require('../util'); /** * Gets the path to the sprites folder for the provided character name diff --git a/src/stable-diffusion.js b/src/endpoints/stable-diffusion.js similarity index 99% rename from src/stable-diffusion.js rename to src/endpoints/stable-diffusion.js index f13bc78a0..f6c94be98 100644 --- a/src/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -1,8 +1,8 @@ const fetch = require('node-fetch').default; const sanitize = require('sanitize-filename'); -const { getBasicAuthHeader, delay } = require('./util'); +const { getBasicAuthHeader, delay } = require('../util.js'); const fs = require('fs'); -const { DIRECTORIES } = require('./constants.js'); +const { DIRECTORIES } = require('../constants.js'); const writeFileAtomicSync = require('write-file-atomic').sync; /** @@ -341,7 +341,7 @@ function registerEndpoints(app, jsonParser) { try { const task = 'text-generation'; - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(task); const result = await pipe(prompt, { num_beams: 1, max_new_tokens: 256, do_sample: true }); diff --git a/src/thumbnails.js b/src/endpoints/thumbnails.js similarity index 98% rename from src/thumbnails.js rename to src/endpoints/thumbnails.js index 4120ae3db..d150f0cfb 100644 --- a/src/thumbnails.js +++ b/src/endpoints/thumbnails.js @@ -3,8 +3,8 @@ const path = require('path'); const sanitize = require('sanitize-filename'); const jimp = require('jimp'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('./constants'); -const { getConfigValue } = require('./util'); +const { DIRECTORIES } = require('../constants'); +const { getConfigValue } = require('../util'); /** * Gets a path to thumbnail folder based on the type. diff --git a/src/tokenizers.js b/src/endpoints/tokenizers.js similarity index 99% rename from src/tokenizers.js rename to src/endpoints/tokenizers.js index 644a1d22a..d73af29b1 100644 --- a/src/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -3,7 +3,7 @@ const path = require('path'); const { SentencePieceProcessor } = require('@agnai/sentencepiece-js'); const tiktoken = require('@dqbd/tiktoken'); const { Tokenizer } = require('@agnai/web-tokenizers'); -const { convertClaudePrompt } = require('./chat-completion'); +const { convertClaudePrompt } = require('../chat-completion'); const { readSecret, SECRET_KEYS } = require('./secrets'); /** diff --git a/src/translate.js b/src/endpoints/translate.js similarity index 99% rename from src/translate.js rename to src/endpoints/translate.js index 2528b9b4b..a7155c5e4 100644 --- a/src/translate.js +++ b/src/endpoints/translate.js @@ -1,7 +1,7 @@ const fetch = require('node-fetch').default; const https = require('https'); const { readSecret, SECRET_KEYS } = require('./secrets'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate'; const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate'; diff --git a/src/vectors.js b/src/endpoints/vectors.js similarity index 97% rename from src/vectors.js rename to src/endpoints/vectors.js index a21e246e8..a6b8e5d56 100644 --- a/src/vectors.js +++ b/src/endpoints/vectors.js @@ -11,11 +11,11 @@ const sanitize = require('sanitize-filename'); async function getVector(source, text) { switch (source) { case 'openai': - return require('./openai-vectors').getOpenAIVector(text); + return require('../openai-vectors').getOpenAIVector(text); case 'transformers': - return require('./embedding').getTransformersVector(text); + return require('../embedding').getTransformersVector(text); case 'palm': - return require('./palm-vectors').getPaLMVector(text); + return require('../palm-vectors').getPaLMVector(text); } throw new Error(`Unknown vector source ${source}`); diff --git a/src/openai-vectors.js b/src/openai-vectors.js index e325d6ecb..ecb245065 100644 --- a/src/openai-vectors.js +++ b/src/openai-vectors.js @@ -1,5 +1,5 @@ const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('./secrets'); +const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); /** * Gets the vector for the given text from OpenAI ada model diff --git a/src/palm-vectors.js b/src/palm-vectors.js index af885068f..788b474cd 100644 --- a/src/palm-vectors.js +++ b/src/palm-vectors.js @@ -1,5 +1,5 @@ const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('./secrets'); +const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); /** * Gets the vector for the given text from PaLM gecko model From db4da0b8aa15251de3fbdcdcedb811eca09c0510 Mon Sep 17 00:00:00 2001 From: valadaptive Date: Sun, 3 Dec 2023 08:52:43 -0500 Subject: [PATCH 04/10] Move endpoints into their own folder --- server.js | 44 ++++++++++++------------- src/{ => endpoints}/assets.js | 2 +- src/{ => endpoints}/caption.js | 2 +- src/{ => endpoints}/classify.js | 4 +-- src/{ => endpoints}/content-manager.js | 4 +-- src/{ => endpoints}/extensions.js | 2 +- src/{ => endpoints}/horde.js | 4 +-- src/{ => endpoints}/novelai.js | 2 +- src/{ => endpoints}/openai.js | 0 src/{ => endpoints}/presets.js | 2 +- src/{ => endpoints}/secrets.js | 2 +- src/{ => endpoints}/serpapi.js | 0 src/{ => endpoints}/sprites.js | 4 +-- src/{ => endpoints}/stable-diffusion.js | 6 ++-- src/{ => endpoints}/thumbnails.js | 4 +-- src/{ => endpoints}/tokenizers.js | 2 +- src/{ => endpoints}/translate.js | 2 +- src/{ => endpoints}/vectors.js | 6 ++-- src/openai-vectors.js | 2 +- src/palm-vectors.js | 2 +- 20 files changed, 48 insertions(+), 48 deletions(-) rename src/{ => endpoints}/assets.js (99%) rename src/{ => endpoints}/caption.js (93%) rename src/{ => endpoints}/classify.js (92%) rename src/{ => endpoints}/content-manager.js (99%) rename src/{ => endpoints}/extensions.js (99%) rename src/{ => endpoints}/horde.js (99%) rename src/{ => endpoints}/novelai.js (99%) rename src/{ => endpoints}/openai.js (100%) rename src/{ => endpoints}/presets.js (98%) rename src/{ => endpoints}/secrets.js (99%) rename src/{ => endpoints}/serpapi.js (100%) rename src/{ => endpoints}/sprites.js (98%) rename src/{ => endpoints}/stable-diffusion.js (99%) rename src/{ => endpoints}/thumbnails.js (98%) rename src/{ => endpoints}/tokenizers.js (99%) rename src/{ => endpoints}/translate.js (99%) rename src/{ => endpoints}/vectors.js (97%) diff --git a/server.js b/server.js index 58f7b72bc..8354e2d21 100644 --- a/server.js +++ b/server.js @@ -51,12 +51,12 @@ util.inspect.defaultOptions.maxStringLength = null; // local library imports const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware'); const characterCardParser = require('./src/character-card-parser.js'); -const contentManager = require('./src/content-manager'); +const contentManager = require('./src/endpoints/content-manager'); const statsHelpers = require('./statsHelpers.js'); -const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/secrets'); +const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets'); const { delay, getVersion, deepMerge, getConfigValue, color, uuidv4 } = require('./src/util'); -const { invalidateThumbnail, ensureThumbnailCache } = require('./src/thumbnails'); -const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/tokenizers'); +const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails'); +const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers'); const { convertClaudePrompt } = require('./src/chat-completion'); // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. @@ -2002,7 +2002,7 @@ app.post('/importcharacter', urlencodedParser, async function (request, response let uploadPath = path.join(UPLOADS_PATH, filedata.filename); var format = request.body.file_type; const defaultAvatarPath = './public/img/ai4.png'; - const { importRisuSprites } = require('./src/sprites'); + const { importRisuSprites } = require('./src/endpoints/sprites'); //console.log(format); if (filedata) { if (format == 'json') { @@ -3576,55 +3576,55 @@ async function fetchJSON(url, args = {}) { // ** END ** // OpenAI API -require('./src/openai').registerEndpoints(app, jsonParser, urlencodedParser); +require('./src/endpoints/openai').registerEndpoints(app, jsonParser, urlencodedParser); // Tokenizers -require('./src/tokenizers').registerEndpoints(app, jsonParser); +require('./src/endpoints/tokenizers').registerEndpoints(app, jsonParser); // Preset management -require('./src/presets').registerEndpoints(app, jsonParser); +require('./src/endpoints/presets').registerEndpoints(app, jsonParser); // Secrets managemenet -require('./src/secrets').registerEndpoints(app, jsonParser); +require('./src/endpoints/secrets').registerEndpoints(app, jsonParser); // Thumbnail generation -require('./src/thumbnails').registerEndpoints(app, jsonParser); +require('./src/endpoints/thumbnails').registerEndpoints(app, jsonParser); // NovelAI generation -require('./src/novelai').registerEndpoints(app, jsonParser); +require('./src/endpoints/novelai').registerEndpoints(app, jsonParser); // Third-party extensions -require('./src/extensions').registerEndpoints(app, jsonParser); +require('./src/endpoints/extensions').registerEndpoints(app, jsonParser); // Asset management -require('./src/assets').registerEndpoints(app, jsonParser); +require('./src/endpoints/assets').registerEndpoints(app, jsonParser); // Character sprite management -require('./src/sprites').registerEndpoints(app, jsonParser, urlencodedParser); +require('./src/endpoints/sprites').registerEndpoints(app, jsonParser, urlencodedParser); // Custom content management -require('./src/content-manager').registerEndpoints(app, jsonParser); +require('./src/endpoints/content-manager').registerEndpoints(app, jsonParser); // Stable Diffusion generation -require('./src/stable-diffusion').registerEndpoints(app, jsonParser); +require('./src/endpoints/stable-diffusion').registerEndpoints(app, jsonParser); // LLM and SD Horde generation -require('./src/horde').registerEndpoints(app, jsonParser); +require('./src/endpoints/horde').registerEndpoints(app, jsonParser); // Vector storage DB -require('./src/vectors').registerEndpoints(app, jsonParser); +require('./src/endpoints/vectors').registerEndpoints(app, jsonParser); // Chat translation -require('./src/translate').registerEndpoints(app, jsonParser); +require('./src/endpoints/translate').registerEndpoints(app, jsonParser); // Emotion classification -require('./src/classify').registerEndpoints(app, jsonParser); +require('./src/endpoints/classify').registerEndpoints(app, jsonParser); // Image captioning -require('./src/caption').registerEndpoints(app, jsonParser); +require('./src/endpoints/caption').registerEndpoints(app, jsonParser); // Web search extension -require('./src/serpapi').registerEndpoints(app, jsonParser); +require('./src/endpoints/serpapi').registerEndpoints(app, jsonParser); const tavernUrl = new URL( (cliArguments.ssl ? 'https://' : 'http://') + diff --git a/src/assets.js b/src/endpoints/assets.js similarity index 99% rename from src/assets.js rename to src/endpoints/assets.js index 02c41bc6a..5264e3576 100644 --- a/src/assets.js +++ b/src/endpoints/assets.js @@ -4,7 +4,7 @@ const sanitize = require('sanitize-filename'); const fetch = require('node-fetch').default; const { finished } = require('stream/promises'); const writeFileSyncAtomic = require('write-file-atomic').sync; -const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('./constants'); +const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants'); const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d']; diff --git a/src/caption.js b/src/endpoints/caption.js similarity index 93% rename from src/caption.js rename to src/endpoints/caption.js index 4ffcfec8e..81a8d029d 100644 --- a/src/caption.js +++ b/src/endpoints/caption.js @@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) { try { const { image } = req.body; - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const rawImage = await module.default.getRawImage(image); if (!rawImage) { diff --git a/src/classify.js b/src/endpoints/classify.js similarity index 92% rename from src/classify.js rename to src/endpoints/classify.js index 8773deb28..dc0b8fb90 100644 --- a/src/classify.js +++ b/src/endpoints/classify.js @@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) { app.post('/api/extra/classify/labels', jsonParser, async (req, res) => { try { - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = Object.keys(pipe.model.config.label2id); return res.json({ labels: result }); @@ -27,7 +27,7 @@ function registerEndpoints(app, jsonParser) { if (Object.hasOwn(cacheObject, text)) { return cacheObject[text]; } else { - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = await pipe(text, { topk: 5 }); result.sort((a, b) => b.score - a.score); diff --git a/src/content-manager.js b/src/endpoints/content-manager.js similarity index 99% rename from src/content-manager.js rename to src/endpoints/content-manager.js index 955e9fc63..cc4796980 100644 --- a/src/content-manager.js +++ b/src/endpoints/content-manager.js @@ -2,11 +2,11 @@ const fs = require('fs'); const path = require('path'); const fetch = require('node-fetch').default; const sanitize = require('sanitize-filename'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const contentDirectory = path.join(process.cwd(), 'default/content'); const contentLogPath = path.join(contentDirectory, 'content.log'); const contentIndexPath = path.join(contentDirectory, 'index.json'); -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); const presetFolders = [DIRECTORIES.koboldAI_Settings, DIRECTORIES.openAI_Settings, DIRECTORIES.novelAI_Settings, DIRECTORIES.textGen_Settings]; /** diff --git a/src/extensions.js b/src/endpoints/extensions.js similarity index 99% rename from src/extensions.js rename to src/endpoints/extensions.js index 0ee083cce..4e3e54d39 100644 --- a/src/extensions.js +++ b/src/endpoints/extensions.js @@ -2,7 +2,7 @@ const path = require('path'); const fs = require('fs'); const { default: simpleGit } = require('simple-git'); const sanitize = require('sanitize-filename'); -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); /** * This function extracts the extension information from the manifest file. diff --git a/src/horde.js b/src/endpoints/horde.js similarity index 99% rename from src/horde.js rename to src/endpoints/horde.js index e649971a2..c662be317 100644 --- a/src/horde.js +++ b/src/endpoints/horde.js @@ -1,6 +1,6 @@ const fetch = require('node-fetch').default; -const AIHorde = require('./ai_horde'); -const { getVersion, delay } = require('./util'); +const AIHorde = require('../ai_horde'); +const { getVersion, delay } = require('../util'); const { readSecret, SECRET_KEYS } = require('./secrets'); const ANONYMOUS_KEY = '0000000000'; diff --git a/src/novelai.js b/src/endpoints/novelai.js similarity index 99% rename from src/novelai.js rename to src/endpoints/novelai.js index 161a636a1..70a3305ce 100644 --- a/src/novelai.js +++ b/src/endpoints/novelai.js @@ -2,7 +2,7 @@ const fetch = require('node-fetch').default; const util = require('util'); const { Readable } = require('stream'); const { readSecret, SECRET_KEYS } = require('./secrets'); -const { readAllChunks, extractFileFromZipBuffer } = require('./util'); +const { readAllChunks, extractFileFromZipBuffer } = require('../util'); const API_NOVELAI = 'https://api.novelai.net'; diff --git a/src/openai.js b/src/endpoints/openai.js similarity index 100% rename from src/openai.js rename to src/endpoints/openai.js diff --git a/src/presets.js b/src/endpoints/presets.js similarity index 98% rename from src/presets.js rename to src/endpoints/presets.js index 24eecf917..3b2ceba51 100644 --- a/src/presets.js +++ b/src/endpoints/presets.js @@ -2,7 +2,7 @@ const fs = require('fs'); const path = require('path'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); const { getDefaultPresetFile, getDefaultPresets } = require('./content-manager'); /** diff --git a/src/secrets.js b/src/endpoints/secrets.js similarity index 99% rename from src/secrets.js rename to src/endpoints/secrets.js index 63436b0c0..c612836cc 100644 --- a/src/secrets.js +++ b/src/endpoints/secrets.js @@ -1,6 +1,6 @@ const fs = require('fs'); const path = require('path'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const writeFileAtomicSync = require('write-file-atomic').sync; const SECRETS_FILE = path.join(process.cwd(), './secrets.json'); diff --git a/src/serpapi.js b/src/endpoints/serpapi.js similarity index 100% rename from src/serpapi.js rename to src/endpoints/serpapi.js diff --git a/src/sprites.js b/src/endpoints/sprites.js similarity index 98% rename from src/sprites.js rename to src/endpoints/sprites.js index f708b4692..67b282e9d 100644 --- a/src/sprites.js +++ b/src/endpoints/sprites.js @@ -4,8 +4,8 @@ const path = require('path'); const mime = require('mime-types'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES, UPLOADS_PATH } = require('./constants'); -const { getImageBuffers } = require('./util'); +const { DIRECTORIES, UPLOADS_PATH } = require('../constants'); +const { getImageBuffers } = require('../util'); /** * Gets the path to the sprites folder for the provided character name diff --git a/src/stable-diffusion.js b/src/endpoints/stable-diffusion.js similarity index 99% rename from src/stable-diffusion.js rename to src/endpoints/stable-diffusion.js index f13bc78a0..f6c94be98 100644 --- a/src/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -1,8 +1,8 @@ const fetch = require('node-fetch').default; const sanitize = require('sanitize-filename'); -const { getBasicAuthHeader, delay } = require('./util'); +const { getBasicAuthHeader, delay } = require('../util.js'); const fs = require('fs'); -const { DIRECTORIES } = require('./constants.js'); +const { DIRECTORIES } = require('../constants.js'); const writeFileAtomicSync = require('write-file-atomic').sync; /** @@ -341,7 +341,7 @@ function registerEndpoints(app, jsonParser) { try { const task = 'text-generation'; - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(task); const result = await pipe(prompt, { num_beams: 1, max_new_tokens: 256, do_sample: true }); diff --git a/src/thumbnails.js b/src/endpoints/thumbnails.js similarity index 98% rename from src/thumbnails.js rename to src/endpoints/thumbnails.js index 4120ae3db..d150f0cfb 100644 --- a/src/thumbnails.js +++ b/src/endpoints/thumbnails.js @@ -3,8 +3,8 @@ const path = require('path'); const sanitize = require('sanitize-filename'); const jimp = require('jimp'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('./constants'); -const { getConfigValue } = require('./util'); +const { DIRECTORIES } = require('../constants'); +const { getConfigValue } = require('../util'); /** * Gets a path to thumbnail folder based on the type. diff --git a/src/tokenizers.js b/src/endpoints/tokenizers.js similarity index 99% rename from src/tokenizers.js rename to src/endpoints/tokenizers.js index 644a1d22a..d73af29b1 100644 --- a/src/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -3,7 +3,7 @@ const path = require('path'); const { SentencePieceProcessor } = require('@agnai/sentencepiece-js'); const tiktoken = require('@dqbd/tiktoken'); const { Tokenizer } = require('@agnai/web-tokenizers'); -const { convertClaudePrompt } = require('./chat-completion'); +const { convertClaudePrompt } = require('../chat-completion'); const { readSecret, SECRET_KEYS } = require('./secrets'); /** diff --git a/src/translate.js b/src/endpoints/translate.js similarity index 99% rename from src/translate.js rename to src/endpoints/translate.js index 2528b9b4b..a7155c5e4 100644 --- a/src/translate.js +++ b/src/endpoints/translate.js @@ -1,7 +1,7 @@ const fetch = require('node-fetch').default; const https = require('https'); const { readSecret, SECRET_KEYS } = require('./secrets'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate'; const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate'; diff --git a/src/vectors.js b/src/endpoints/vectors.js similarity index 97% rename from src/vectors.js rename to src/endpoints/vectors.js index a21e246e8..a6b8e5d56 100644 --- a/src/vectors.js +++ b/src/endpoints/vectors.js @@ -11,11 +11,11 @@ const sanitize = require('sanitize-filename'); async function getVector(source, text) { switch (source) { case 'openai': - return require('./openai-vectors').getOpenAIVector(text); + return require('../openai-vectors').getOpenAIVector(text); case 'transformers': - return require('./embedding').getTransformersVector(text); + return require('../embedding').getTransformersVector(text); case 'palm': - return require('./palm-vectors').getPaLMVector(text); + return require('../palm-vectors').getPaLMVector(text); } throw new Error(`Unknown vector source ${source}`); diff --git a/src/openai-vectors.js b/src/openai-vectors.js index e325d6ecb..ecb245065 100644 --- a/src/openai-vectors.js +++ b/src/openai-vectors.js @@ -1,5 +1,5 @@ const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('./secrets'); +const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); /** * Gets the vector for the given text from OpenAI ada model diff --git a/src/palm-vectors.js b/src/palm-vectors.js index af885068f..788b474cd 100644 --- a/src/palm-vectors.js +++ b/src/palm-vectors.js @@ -1,5 +1,5 @@ const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('./secrets'); +const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); /** * Gets the vector for the given text from PaLM gecko model From 2180610363726a5a7f4da4432cead7fd7b4a8e31 Mon Sep 17 00:00:00 2001 From: valadaptive Date: Sun, 3 Dec 2023 09:19:25 -0500 Subject: [PATCH 05/10] Rename /viewsecrets to /api/secrets/view --- public/scripts/secrets.js | 2 +- src/endpoints/secrets.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/public/scripts/secrets.js b/public/scripts/secrets.js index 5fef67e2f..84279641d 100644 --- a/public/scripts/secrets.js +++ b/public/scripts/secrets.js @@ -49,7 +49,7 @@ function updateSecretDisplay() { } async function viewSecrets() { - const response = await fetch('/viewsecrets', { + const response = await fetch('/api/secrets/view', { method: 'POST', headers: getRequestHeaders(), }); diff --git a/src/endpoints/secrets.js b/src/endpoints/secrets.js index c612836cc..a0282c5e1 100644 --- a/src/endpoints/secrets.js +++ b/src/endpoints/secrets.js @@ -169,7 +169,7 @@ function registerEndpoints(app, jsonParser) { } }); - app.post('/viewsecrets', jsonParser, async (_, response) => { + app.post('/api/secrets/view', jsonParser, async (_, response) => { const allowKeysExposure = getConfigValue('allowKeysExposure', false); if (!allowKeysExposure) { From baae99831de68e2163ea9926eaec2dcd7dc1c5fb Mon Sep 17 00:00:00 2001 From: valadaptive Date: Sun, 3 Dec 2023 08:52:43 -0500 Subject: [PATCH 06/10] Move endpoints into their own folder --- server.js | 44 ++++++++++++------------- src/{ => endpoints}/assets.js | 2 +- src/{ => endpoints}/caption.js | 2 +- src/{ => endpoints}/classify.js | 4 +-- src/{ => endpoints}/content-manager.js | 4 +-- src/{ => endpoints}/extensions.js | 2 +- src/{ => endpoints}/horde.js | 4 +-- src/{ => endpoints}/novelai.js | 2 +- src/{ => endpoints}/openai.js | 0 src/{ => endpoints}/presets.js | 2 +- src/{ => endpoints}/secrets.js | 2 +- src/{ => endpoints}/serpapi.js | 0 src/{ => endpoints}/sprites.js | 4 +-- src/{ => endpoints}/stable-diffusion.js | 6 ++-- src/{ => endpoints}/thumbnails.js | 4 +-- src/{ => endpoints}/tokenizers.js | 2 +- src/{ => endpoints}/translate.js | 2 +- src/{ => endpoints}/vectors.js | 6 ++-- src/openai-vectors.js | 2 +- src/palm-vectors.js | 2 +- 20 files changed, 48 insertions(+), 48 deletions(-) rename src/{ => endpoints}/assets.js (99%) rename src/{ => endpoints}/caption.js (93%) rename src/{ => endpoints}/classify.js (92%) rename src/{ => endpoints}/content-manager.js (99%) rename src/{ => endpoints}/extensions.js (99%) rename src/{ => endpoints}/horde.js (99%) rename src/{ => endpoints}/novelai.js (99%) rename src/{ => endpoints}/openai.js (100%) rename src/{ => endpoints}/presets.js (98%) rename src/{ => endpoints}/secrets.js (99%) rename src/{ => endpoints}/serpapi.js (100%) rename src/{ => endpoints}/sprites.js (98%) rename src/{ => endpoints}/stable-diffusion.js (99%) rename src/{ => endpoints}/thumbnails.js (98%) rename src/{ => endpoints}/tokenizers.js (99%) rename src/{ => endpoints}/translate.js (99%) rename src/{ => endpoints}/vectors.js (97%) diff --git a/server.js b/server.js index 58f7b72bc..8354e2d21 100644 --- a/server.js +++ b/server.js @@ -51,12 +51,12 @@ util.inspect.defaultOptions.maxStringLength = null; // local library imports const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware'); const characterCardParser = require('./src/character-card-parser.js'); -const contentManager = require('./src/content-manager'); +const contentManager = require('./src/endpoints/content-manager'); const statsHelpers = require('./statsHelpers.js'); -const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/secrets'); +const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets'); const { delay, getVersion, deepMerge, getConfigValue, color, uuidv4 } = require('./src/util'); -const { invalidateThumbnail, ensureThumbnailCache } = require('./src/thumbnails'); -const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/tokenizers'); +const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails'); +const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers'); const { convertClaudePrompt } = require('./src/chat-completion'); // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. @@ -2002,7 +2002,7 @@ app.post('/importcharacter', urlencodedParser, async function (request, response let uploadPath = path.join(UPLOADS_PATH, filedata.filename); var format = request.body.file_type; const defaultAvatarPath = './public/img/ai4.png'; - const { importRisuSprites } = require('./src/sprites'); + const { importRisuSprites } = require('./src/endpoints/sprites'); //console.log(format); if (filedata) { if (format == 'json') { @@ -3576,55 +3576,55 @@ async function fetchJSON(url, args = {}) { // ** END ** // OpenAI API -require('./src/openai').registerEndpoints(app, jsonParser, urlencodedParser); +require('./src/endpoints/openai').registerEndpoints(app, jsonParser, urlencodedParser); // Tokenizers -require('./src/tokenizers').registerEndpoints(app, jsonParser); +require('./src/endpoints/tokenizers').registerEndpoints(app, jsonParser); // Preset management -require('./src/presets').registerEndpoints(app, jsonParser); +require('./src/endpoints/presets').registerEndpoints(app, jsonParser); // Secrets managemenet -require('./src/secrets').registerEndpoints(app, jsonParser); +require('./src/endpoints/secrets').registerEndpoints(app, jsonParser); // Thumbnail generation -require('./src/thumbnails').registerEndpoints(app, jsonParser); +require('./src/endpoints/thumbnails').registerEndpoints(app, jsonParser); // NovelAI generation -require('./src/novelai').registerEndpoints(app, jsonParser); +require('./src/endpoints/novelai').registerEndpoints(app, jsonParser); // Third-party extensions -require('./src/extensions').registerEndpoints(app, jsonParser); +require('./src/endpoints/extensions').registerEndpoints(app, jsonParser); // Asset management -require('./src/assets').registerEndpoints(app, jsonParser); +require('./src/endpoints/assets').registerEndpoints(app, jsonParser); // Character sprite management -require('./src/sprites').registerEndpoints(app, jsonParser, urlencodedParser); +require('./src/endpoints/sprites').registerEndpoints(app, jsonParser, urlencodedParser); // Custom content management -require('./src/content-manager').registerEndpoints(app, jsonParser); +require('./src/endpoints/content-manager').registerEndpoints(app, jsonParser); // Stable Diffusion generation -require('./src/stable-diffusion').registerEndpoints(app, jsonParser); +require('./src/endpoints/stable-diffusion').registerEndpoints(app, jsonParser); // LLM and SD Horde generation -require('./src/horde').registerEndpoints(app, jsonParser); +require('./src/endpoints/horde').registerEndpoints(app, jsonParser); // Vector storage DB -require('./src/vectors').registerEndpoints(app, jsonParser); +require('./src/endpoints/vectors').registerEndpoints(app, jsonParser); // Chat translation -require('./src/translate').registerEndpoints(app, jsonParser); +require('./src/endpoints/translate').registerEndpoints(app, jsonParser); // Emotion classification -require('./src/classify').registerEndpoints(app, jsonParser); +require('./src/endpoints/classify').registerEndpoints(app, jsonParser); // Image captioning -require('./src/caption').registerEndpoints(app, jsonParser); +require('./src/endpoints/caption').registerEndpoints(app, jsonParser); // Web search extension -require('./src/serpapi').registerEndpoints(app, jsonParser); +require('./src/endpoints/serpapi').registerEndpoints(app, jsonParser); const tavernUrl = new URL( (cliArguments.ssl ? 'https://' : 'http://') + diff --git a/src/assets.js b/src/endpoints/assets.js similarity index 99% rename from src/assets.js rename to src/endpoints/assets.js index 02c41bc6a..5264e3576 100644 --- a/src/assets.js +++ b/src/endpoints/assets.js @@ -4,7 +4,7 @@ const sanitize = require('sanitize-filename'); const fetch = require('node-fetch').default; const { finished } = require('stream/promises'); const writeFileSyncAtomic = require('write-file-atomic').sync; -const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('./constants'); +const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants'); const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d']; diff --git a/src/caption.js b/src/endpoints/caption.js similarity index 93% rename from src/caption.js rename to src/endpoints/caption.js index 4ffcfec8e..81a8d029d 100644 --- a/src/caption.js +++ b/src/endpoints/caption.js @@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) { try { const { image } = req.body; - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const rawImage = await module.default.getRawImage(image); if (!rawImage) { diff --git a/src/classify.js b/src/endpoints/classify.js similarity index 92% rename from src/classify.js rename to src/endpoints/classify.js index 8773deb28..dc0b8fb90 100644 --- a/src/classify.js +++ b/src/endpoints/classify.js @@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) { app.post('/api/extra/classify/labels', jsonParser, async (req, res) => { try { - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = Object.keys(pipe.model.config.label2id); return res.json({ labels: result }); @@ -27,7 +27,7 @@ function registerEndpoints(app, jsonParser) { if (Object.hasOwn(cacheObject, text)) { return cacheObject[text]; } else { - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = await pipe(text, { topk: 5 }); result.sort((a, b) => b.score - a.score); diff --git a/src/content-manager.js b/src/endpoints/content-manager.js similarity index 99% rename from src/content-manager.js rename to src/endpoints/content-manager.js index 955e9fc63..cc4796980 100644 --- a/src/content-manager.js +++ b/src/endpoints/content-manager.js @@ -2,11 +2,11 @@ const fs = require('fs'); const path = require('path'); const fetch = require('node-fetch').default; const sanitize = require('sanitize-filename'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const contentDirectory = path.join(process.cwd(), 'default/content'); const contentLogPath = path.join(contentDirectory, 'content.log'); const contentIndexPath = path.join(contentDirectory, 'index.json'); -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); const presetFolders = [DIRECTORIES.koboldAI_Settings, DIRECTORIES.openAI_Settings, DIRECTORIES.novelAI_Settings, DIRECTORIES.textGen_Settings]; /** diff --git a/src/extensions.js b/src/endpoints/extensions.js similarity index 99% rename from src/extensions.js rename to src/endpoints/extensions.js index 0ee083cce..4e3e54d39 100644 --- a/src/extensions.js +++ b/src/endpoints/extensions.js @@ -2,7 +2,7 @@ const path = require('path'); const fs = require('fs'); const { default: simpleGit } = require('simple-git'); const sanitize = require('sanitize-filename'); -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); /** * This function extracts the extension information from the manifest file. diff --git a/src/horde.js b/src/endpoints/horde.js similarity index 99% rename from src/horde.js rename to src/endpoints/horde.js index e649971a2..c662be317 100644 --- a/src/horde.js +++ b/src/endpoints/horde.js @@ -1,6 +1,6 @@ const fetch = require('node-fetch').default; -const AIHorde = require('./ai_horde'); -const { getVersion, delay } = require('./util'); +const AIHorde = require('../ai_horde'); +const { getVersion, delay } = require('../util'); const { readSecret, SECRET_KEYS } = require('./secrets'); const ANONYMOUS_KEY = '0000000000'; diff --git a/src/novelai.js b/src/endpoints/novelai.js similarity index 99% rename from src/novelai.js rename to src/endpoints/novelai.js index 161a636a1..70a3305ce 100644 --- a/src/novelai.js +++ b/src/endpoints/novelai.js @@ -2,7 +2,7 @@ const fetch = require('node-fetch').default; const util = require('util'); const { Readable } = require('stream'); const { readSecret, SECRET_KEYS } = require('./secrets'); -const { readAllChunks, extractFileFromZipBuffer } = require('./util'); +const { readAllChunks, extractFileFromZipBuffer } = require('../util'); const API_NOVELAI = 'https://api.novelai.net'; diff --git a/src/openai.js b/src/endpoints/openai.js similarity index 100% rename from src/openai.js rename to src/endpoints/openai.js diff --git a/src/presets.js b/src/endpoints/presets.js similarity index 98% rename from src/presets.js rename to src/endpoints/presets.js index 24eecf917..3b2ceba51 100644 --- a/src/presets.js +++ b/src/endpoints/presets.js @@ -2,7 +2,7 @@ const fs = require('fs'); const path = require('path'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); const { getDefaultPresetFile, getDefaultPresets } = require('./content-manager'); /** diff --git a/src/secrets.js b/src/endpoints/secrets.js similarity index 99% rename from src/secrets.js rename to src/endpoints/secrets.js index 63436b0c0..c612836cc 100644 --- a/src/secrets.js +++ b/src/endpoints/secrets.js @@ -1,6 +1,6 @@ const fs = require('fs'); const path = require('path'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const writeFileAtomicSync = require('write-file-atomic').sync; const SECRETS_FILE = path.join(process.cwd(), './secrets.json'); diff --git a/src/serpapi.js b/src/endpoints/serpapi.js similarity index 100% rename from src/serpapi.js rename to src/endpoints/serpapi.js diff --git a/src/sprites.js b/src/endpoints/sprites.js similarity index 98% rename from src/sprites.js rename to src/endpoints/sprites.js index f708b4692..67b282e9d 100644 --- a/src/sprites.js +++ b/src/endpoints/sprites.js @@ -4,8 +4,8 @@ const path = require('path'); const mime = require('mime-types'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES, UPLOADS_PATH } = require('./constants'); -const { getImageBuffers } = require('./util'); +const { DIRECTORIES, UPLOADS_PATH } = require('../constants'); +const { getImageBuffers } = require('../util'); /** * Gets the path to the sprites folder for the provided character name diff --git a/src/stable-diffusion.js b/src/endpoints/stable-diffusion.js similarity index 99% rename from src/stable-diffusion.js rename to src/endpoints/stable-diffusion.js index f13bc78a0..f6c94be98 100644 --- a/src/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -1,8 +1,8 @@ const fetch = require('node-fetch').default; const sanitize = require('sanitize-filename'); -const { getBasicAuthHeader, delay } = require('./util'); +const { getBasicAuthHeader, delay } = require('../util.js'); const fs = require('fs'); -const { DIRECTORIES } = require('./constants.js'); +const { DIRECTORIES } = require('../constants.js'); const writeFileAtomicSync = require('write-file-atomic').sync; /** @@ -341,7 +341,7 @@ function registerEndpoints(app, jsonParser) { try { const task = 'text-generation'; - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(task); const result = await pipe(prompt, { num_beams: 1, max_new_tokens: 256, do_sample: true }); diff --git a/src/thumbnails.js b/src/endpoints/thumbnails.js similarity index 98% rename from src/thumbnails.js rename to src/endpoints/thumbnails.js index 4120ae3db..d150f0cfb 100644 --- a/src/thumbnails.js +++ b/src/endpoints/thumbnails.js @@ -3,8 +3,8 @@ const path = require('path'); const sanitize = require('sanitize-filename'); const jimp = require('jimp'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('./constants'); -const { getConfigValue } = require('./util'); +const { DIRECTORIES } = require('../constants'); +const { getConfigValue } = require('../util'); /** * Gets a path to thumbnail folder based on the type. diff --git a/src/tokenizers.js b/src/endpoints/tokenizers.js similarity index 99% rename from src/tokenizers.js rename to src/endpoints/tokenizers.js index 644a1d22a..d73af29b1 100644 --- a/src/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -3,7 +3,7 @@ const path = require('path'); const { SentencePieceProcessor } = require('@agnai/sentencepiece-js'); const tiktoken = require('@dqbd/tiktoken'); const { Tokenizer } = require('@agnai/web-tokenizers'); -const { convertClaudePrompt } = require('./chat-completion'); +const { convertClaudePrompt } = require('../chat-completion'); const { readSecret, SECRET_KEYS } = require('./secrets'); /** diff --git a/src/translate.js b/src/endpoints/translate.js similarity index 99% rename from src/translate.js rename to src/endpoints/translate.js index 2528b9b4b..a7155c5e4 100644 --- a/src/translate.js +++ b/src/endpoints/translate.js @@ -1,7 +1,7 @@ const fetch = require('node-fetch').default; const https = require('https'); const { readSecret, SECRET_KEYS } = require('./secrets'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate'; const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate'; diff --git a/src/vectors.js b/src/endpoints/vectors.js similarity index 97% rename from src/vectors.js rename to src/endpoints/vectors.js index a21e246e8..a6b8e5d56 100644 --- a/src/vectors.js +++ b/src/endpoints/vectors.js @@ -11,11 +11,11 @@ const sanitize = require('sanitize-filename'); async function getVector(source, text) { switch (source) { case 'openai': - return require('./openai-vectors').getOpenAIVector(text); + return require('../openai-vectors').getOpenAIVector(text); case 'transformers': - return require('./embedding').getTransformersVector(text); + return require('../embedding').getTransformersVector(text); case 'palm': - return require('./palm-vectors').getPaLMVector(text); + return require('../palm-vectors').getPaLMVector(text); } throw new Error(`Unknown vector source ${source}`); diff --git a/src/openai-vectors.js b/src/openai-vectors.js index e325d6ecb..ecb245065 100644 --- a/src/openai-vectors.js +++ b/src/openai-vectors.js @@ -1,5 +1,5 @@ const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('./secrets'); +const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); /** * Gets the vector for the given text from OpenAI ada model diff --git a/src/palm-vectors.js b/src/palm-vectors.js index af885068f..788b474cd 100644 --- a/src/palm-vectors.js +++ b/src/palm-vectors.js @@ -1,5 +1,5 @@ const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('./secrets'); +const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); /** * Gets the vector for the given text from PaLM gecko model From c073a89e2cdfcddc01e427683b6e8fc631010f99 Mon Sep 17 00:00:00 2001 From: valadaptive Date: Sun, 3 Dec 2023 08:52:43 -0500 Subject: [PATCH 07/10] Move endpoints into their own folder --- server.js | 44 ++++++++++++------------- src/{ => endpoints}/assets.js | 2 +- src/{ => endpoints}/caption.js | 2 +- src/{ => endpoints}/classify.js | 4 +-- src/{ => endpoints}/content-manager.js | 4 +-- src/{ => endpoints}/extensions.js | 2 +- src/{ => endpoints}/horde.js | 4 +-- src/{ => endpoints}/novelai.js | 2 +- src/{ => endpoints}/openai.js | 0 src/{ => endpoints}/presets.js | 2 +- src/{ => endpoints}/secrets.js | 2 +- src/{ => endpoints}/serpapi.js | 0 src/{ => endpoints}/sprites.js | 4 +-- src/{ => endpoints}/stable-diffusion.js | 6 ++-- src/{ => endpoints}/thumbnails.js | 4 +-- src/{ => endpoints}/tokenizers.js | 2 +- src/{ => endpoints}/translate.js | 2 +- src/{ => endpoints}/vectors.js | 6 ++-- src/openai-vectors.js | 2 +- src/palm-vectors.js | 2 +- 20 files changed, 48 insertions(+), 48 deletions(-) rename src/{ => endpoints}/assets.js (99%) rename src/{ => endpoints}/caption.js (93%) rename src/{ => endpoints}/classify.js (92%) rename src/{ => endpoints}/content-manager.js (99%) rename src/{ => endpoints}/extensions.js (99%) rename src/{ => endpoints}/horde.js (99%) rename src/{ => endpoints}/novelai.js (99%) rename src/{ => endpoints}/openai.js (100%) rename src/{ => endpoints}/presets.js (98%) rename src/{ => endpoints}/secrets.js (99%) rename src/{ => endpoints}/serpapi.js (100%) rename src/{ => endpoints}/sprites.js (98%) rename src/{ => endpoints}/stable-diffusion.js (99%) rename src/{ => endpoints}/thumbnails.js (98%) rename src/{ => endpoints}/tokenizers.js (99%) rename src/{ => endpoints}/translate.js (99%) rename src/{ => endpoints}/vectors.js (97%) diff --git a/server.js b/server.js index 58f7b72bc..8354e2d21 100644 --- a/server.js +++ b/server.js @@ -51,12 +51,12 @@ util.inspect.defaultOptions.maxStringLength = null; // local library imports const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware'); const characterCardParser = require('./src/character-card-parser.js'); -const contentManager = require('./src/content-manager'); +const contentManager = require('./src/endpoints/content-manager'); const statsHelpers = require('./statsHelpers.js'); -const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/secrets'); +const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets'); const { delay, getVersion, deepMerge, getConfigValue, color, uuidv4 } = require('./src/util'); -const { invalidateThumbnail, ensureThumbnailCache } = require('./src/thumbnails'); -const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/tokenizers'); +const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails'); +const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers'); const { convertClaudePrompt } = require('./src/chat-completion'); // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. @@ -2002,7 +2002,7 @@ app.post('/importcharacter', urlencodedParser, async function (request, response let uploadPath = path.join(UPLOADS_PATH, filedata.filename); var format = request.body.file_type; const defaultAvatarPath = './public/img/ai4.png'; - const { importRisuSprites } = require('./src/sprites'); + const { importRisuSprites } = require('./src/endpoints/sprites'); //console.log(format); if (filedata) { if (format == 'json') { @@ -3576,55 +3576,55 @@ async function fetchJSON(url, args = {}) { // ** END ** // OpenAI API -require('./src/openai').registerEndpoints(app, jsonParser, urlencodedParser); +require('./src/endpoints/openai').registerEndpoints(app, jsonParser, urlencodedParser); // Tokenizers -require('./src/tokenizers').registerEndpoints(app, jsonParser); +require('./src/endpoints/tokenizers').registerEndpoints(app, jsonParser); // Preset management -require('./src/presets').registerEndpoints(app, jsonParser); +require('./src/endpoints/presets').registerEndpoints(app, jsonParser); // Secrets managemenet -require('./src/secrets').registerEndpoints(app, jsonParser); +require('./src/endpoints/secrets').registerEndpoints(app, jsonParser); // Thumbnail generation -require('./src/thumbnails').registerEndpoints(app, jsonParser); +require('./src/endpoints/thumbnails').registerEndpoints(app, jsonParser); // NovelAI generation -require('./src/novelai').registerEndpoints(app, jsonParser); +require('./src/endpoints/novelai').registerEndpoints(app, jsonParser); // Third-party extensions -require('./src/extensions').registerEndpoints(app, jsonParser); +require('./src/endpoints/extensions').registerEndpoints(app, jsonParser); // Asset management -require('./src/assets').registerEndpoints(app, jsonParser); +require('./src/endpoints/assets').registerEndpoints(app, jsonParser); // Character sprite management -require('./src/sprites').registerEndpoints(app, jsonParser, urlencodedParser); +require('./src/endpoints/sprites').registerEndpoints(app, jsonParser, urlencodedParser); // Custom content management -require('./src/content-manager').registerEndpoints(app, jsonParser); +require('./src/endpoints/content-manager').registerEndpoints(app, jsonParser); // Stable Diffusion generation -require('./src/stable-diffusion').registerEndpoints(app, jsonParser); +require('./src/endpoints/stable-diffusion').registerEndpoints(app, jsonParser); // LLM and SD Horde generation -require('./src/horde').registerEndpoints(app, jsonParser); +require('./src/endpoints/horde').registerEndpoints(app, jsonParser); // Vector storage DB -require('./src/vectors').registerEndpoints(app, jsonParser); +require('./src/endpoints/vectors').registerEndpoints(app, jsonParser); // Chat translation -require('./src/translate').registerEndpoints(app, jsonParser); +require('./src/endpoints/translate').registerEndpoints(app, jsonParser); // Emotion classification -require('./src/classify').registerEndpoints(app, jsonParser); +require('./src/endpoints/classify').registerEndpoints(app, jsonParser); // Image captioning -require('./src/caption').registerEndpoints(app, jsonParser); +require('./src/endpoints/caption').registerEndpoints(app, jsonParser); // Web search extension -require('./src/serpapi').registerEndpoints(app, jsonParser); +require('./src/endpoints/serpapi').registerEndpoints(app, jsonParser); const tavernUrl = new URL( (cliArguments.ssl ? 'https://' : 'http://') + diff --git a/src/assets.js b/src/endpoints/assets.js similarity index 99% rename from src/assets.js rename to src/endpoints/assets.js index 02c41bc6a..5264e3576 100644 --- a/src/assets.js +++ b/src/endpoints/assets.js @@ -4,7 +4,7 @@ const sanitize = require('sanitize-filename'); const fetch = require('node-fetch').default; const { finished } = require('stream/promises'); const writeFileSyncAtomic = require('write-file-atomic').sync; -const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('./constants'); +const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants'); const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d']; diff --git a/src/caption.js b/src/endpoints/caption.js similarity index 93% rename from src/caption.js rename to src/endpoints/caption.js index 4ffcfec8e..81a8d029d 100644 --- a/src/caption.js +++ b/src/endpoints/caption.js @@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) { try { const { image } = req.body; - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const rawImage = await module.default.getRawImage(image); if (!rawImage) { diff --git a/src/classify.js b/src/endpoints/classify.js similarity index 92% rename from src/classify.js rename to src/endpoints/classify.js index 8773deb28..dc0b8fb90 100644 --- a/src/classify.js +++ b/src/endpoints/classify.js @@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) { app.post('/api/extra/classify/labels', jsonParser, async (req, res) => { try { - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = Object.keys(pipe.model.config.label2id); return res.json({ labels: result }); @@ -27,7 +27,7 @@ function registerEndpoints(app, jsonParser) { if (Object.hasOwn(cacheObject, text)) { return cacheObject[text]; } else { - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = await pipe(text, { topk: 5 }); result.sort((a, b) => b.score - a.score); diff --git a/src/content-manager.js b/src/endpoints/content-manager.js similarity index 99% rename from src/content-manager.js rename to src/endpoints/content-manager.js index 955e9fc63..cc4796980 100644 --- a/src/content-manager.js +++ b/src/endpoints/content-manager.js @@ -2,11 +2,11 @@ const fs = require('fs'); const path = require('path'); const fetch = require('node-fetch').default; const sanitize = require('sanitize-filename'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const contentDirectory = path.join(process.cwd(), 'default/content'); const contentLogPath = path.join(contentDirectory, 'content.log'); const contentIndexPath = path.join(contentDirectory, 'index.json'); -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); const presetFolders = [DIRECTORIES.koboldAI_Settings, DIRECTORIES.openAI_Settings, DIRECTORIES.novelAI_Settings, DIRECTORIES.textGen_Settings]; /** diff --git a/src/extensions.js b/src/endpoints/extensions.js similarity index 99% rename from src/extensions.js rename to src/endpoints/extensions.js index 0ee083cce..4e3e54d39 100644 --- a/src/extensions.js +++ b/src/endpoints/extensions.js @@ -2,7 +2,7 @@ const path = require('path'); const fs = require('fs'); const { default: simpleGit } = require('simple-git'); const sanitize = require('sanitize-filename'); -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); /** * This function extracts the extension information from the manifest file. diff --git a/src/horde.js b/src/endpoints/horde.js similarity index 99% rename from src/horde.js rename to src/endpoints/horde.js index e649971a2..c662be317 100644 --- a/src/horde.js +++ b/src/endpoints/horde.js @@ -1,6 +1,6 @@ const fetch = require('node-fetch').default; -const AIHorde = require('./ai_horde'); -const { getVersion, delay } = require('./util'); +const AIHorde = require('../ai_horde'); +const { getVersion, delay } = require('../util'); const { readSecret, SECRET_KEYS } = require('./secrets'); const ANONYMOUS_KEY = '0000000000'; diff --git a/src/novelai.js b/src/endpoints/novelai.js similarity index 99% rename from src/novelai.js rename to src/endpoints/novelai.js index 161a636a1..70a3305ce 100644 --- a/src/novelai.js +++ b/src/endpoints/novelai.js @@ -2,7 +2,7 @@ const fetch = require('node-fetch').default; const util = require('util'); const { Readable } = require('stream'); const { readSecret, SECRET_KEYS } = require('./secrets'); -const { readAllChunks, extractFileFromZipBuffer } = require('./util'); +const { readAllChunks, extractFileFromZipBuffer } = require('../util'); const API_NOVELAI = 'https://api.novelai.net'; diff --git a/src/openai.js b/src/endpoints/openai.js similarity index 100% rename from src/openai.js rename to src/endpoints/openai.js diff --git a/src/presets.js b/src/endpoints/presets.js similarity index 98% rename from src/presets.js rename to src/endpoints/presets.js index 24eecf917..3b2ceba51 100644 --- a/src/presets.js +++ b/src/endpoints/presets.js @@ -2,7 +2,7 @@ const fs = require('fs'); const path = require('path'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('./constants'); +const { DIRECTORIES } = require('../constants'); const { getDefaultPresetFile, getDefaultPresets } = require('./content-manager'); /** diff --git a/src/secrets.js b/src/endpoints/secrets.js similarity index 99% rename from src/secrets.js rename to src/endpoints/secrets.js index 63436b0c0..c612836cc 100644 --- a/src/secrets.js +++ b/src/endpoints/secrets.js @@ -1,6 +1,6 @@ const fs = require('fs'); const path = require('path'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const writeFileAtomicSync = require('write-file-atomic').sync; const SECRETS_FILE = path.join(process.cwd(), './secrets.json'); diff --git a/src/serpapi.js b/src/endpoints/serpapi.js similarity index 100% rename from src/serpapi.js rename to src/endpoints/serpapi.js diff --git a/src/sprites.js b/src/endpoints/sprites.js similarity index 98% rename from src/sprites.js rename to src/endpoints/sprites.js index f708b4692..67b282e9d 100644 --- a/src/sprites.js +++ b/src/endpoints/sprites.js @@ -4,8 +4,8 @@ const path = require('path'); const mime = require('mime-types'); const sanitize = require('sanitize-filename'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES, UPLOADS_PATH } = require('./constants'); -const { getImageBuffers } = require('./util'); +const { DIRECTORIES, UPLOADS_PATH } = require('../constants'); +const { getImageBuffers } = require('../util'); /** * Gets the path to the sprites folder for the provided character name diff --git a/src/stable-diffusion.js b/src/endpoints/stable-diffusion.js similarity index 99% rename from src/stable-diffusion.js rename to src/endpoints/stable-diffusion.js index f13bc78a0..f6c94be98 100644 --- a/src/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -1,8 +1,8 @@ const fetch = require('node-fetch').default; const sanitize = require('sanitize-filename'); -const { getBasicAuthHeader, delay } = require('./util'); +const { getBasicAuthHeader, delay } = require('../util.js'); const fs = require('fs'); -const { DIRECTORIES } = require('./constants.js'); +const { DIRECTORIES } = require('../constants.js'); const writeFileAtomicSync = require('write-file-atomic').sync; /** @@ -341,7 +341,7 @@ function registerEndpoints(app, jsonParser) { try { const task = 'text-generation'; - const module = await import('./transformers.mjs'); + const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(task); const result = await pipe(prompt, { num_beams: 1, max_new_tokens: 256, do_sample: true }); diff --git a/src/thumbnails.js b/src/endpoints/thumbnails.js similarity index 98% rename from src/thumbnails.js rename to src/endpoints/thumbnails.js index 4120ae3db..d150f0cfb 100644 --- a/src/thumbnails.js +++ b/src/endpoints/thumbnails.js @@ -3,8 +3,8 @@ const path = require('path'); const sanitize = require('sanitize-filename'); const jimp = require('jimp'); const writeFileAtomicSync = require('write-file-atomic').sync; -const { DIRECTORIES } = require('./constants'); -const { getConfigValue } = require('./util'); +const { DIRECTORIES } = require('../constants'); +const { getConfigValue } = require('../util'); /** * Gets a path to thumbnail folder based on the type. diff --git a/src/tokenizers.js b/src/endpoints/tokenizers.js similarity index 99% rename from src/tokenizers.js rename to src/endpoints/tokenizers.js index 644a1d22a..d73af29b1 100644 --- a/src/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -3,7 +3,7 @@ const path = require('path'); const { SentencePieceProcessor } = require('@agnai/sentencepiece-js'); const tiktoken = require('@dqbd/tiktoken'); const { Tokenizer } = require('@agnai/web-tokenizers'); -const { convertClaudePrompt } = require('./chat-completion'); +const { convertClaudePrompt } = require('../chat-completion'); const { readSecret, SECRET_KEYS } = require('./secrets'); /** diff --git a/src/translate.js b/src/endpoints/translate.js similarity index 99% rename from src/translate.js rename to src/endpoints/translate.js index 2528b9b4b..a7155c5e4 100644 --- a/src/translate.js +++ b/src/endpoints/translate.js @@ -1,7 +1,7 @@ const fetch = require('node-fetch').default; const https = require('https'); const { readSecret, SECRET_KEYS } = require('./secrets'); -const { getConfigValue } = require('./util'); +const { getConfigValue } = require('../util'); const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate'; const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate'; diff --git a/src/vectors.js b/src/endpoints/vectors.js similarity index 97% rename from src/vectors.js rename to src/endpoints/vectors.js index a21e246e8..a6b8e5d56 100644 --- a/src/vectors.js +++ b/src/endpoints/vectors.js @@ -11,11 +11,11 @@ const sanitize = require('sanitize-filename'); async function getVector(source, text) { switch (source) { case 'openai': - return require('./openai-vectors').getOpenAIVector(text); + return require('../openai-vectors').getOpenAIVector(text); case 'transformers': - return require('./embedding').getTransformersVector(text); + return require('../embedding').getTransformersVector(text); case 'palm': - return require('./palm-vectors').getPaLMVector(text); + return require('../palm-vectors').getPaLMVector(text); } throw new Error(`Unknown vector source ${source}`); diff --git a/src/openai-vectors.js b/src/openai-vectors.js index e325d6ecb..ecb245065 100644 --- a/src/openai-vectors.js +++ b/src/openai-vectors.js @@ -1,5 +1,5 @@ const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('./secrets'); +const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); /** * Gets the vector for the given text from OpenAI ada model diff --git a/src/palm-vectors.js b/src/palm-vectors.js index af885068f..788b474cd 100644 --- a/src/palm-vectors.js +++ b/src/palm-vectors.js @@ -1,5 +1,5 @@ const fetch = require('node-fetch').default; -const { SECRET_KEYS, readSecret } = require('./secrets'); +const { SECRET_KEYS, readSecret } = require('./endpoints/secrets'); /** * Gets the vector for the given text from PaLM gecko model From 57bc95133e0a43cfc1b0c115b65c569a537b67f1 Mon Sep 17 00:00:00 2001 From: valadaptive Date: Sun, 3 Dec 2023 09:42:54 -0500 Subject: [PATCH 08/10] Rename tokenizer routes They're all under tokenizers/ now, and there are "count", "encode", and "decode" endpoints. This forms a clearer hierarchy. --- public/scripts/tokenizers.js | 42 ++++++++++++++++++------------------ src/endpoints/tokenizers.js | 32 +++++++++++++-------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index ce52b6704..ecab34705 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -161,17 +161,17 @@ function callTokenizer(type, str, padding) { case tokenizers.NONE: return guesstimate(str) + padding; case tokenizers.GPT2: - return countTokensRemote('/api/tokenize/gpt2', str, padding); + return countTokensRemote('/api/tokenizers/gpt2/encode', str, padding); case tokenizers.LLAMA: - return countTokensRemote('/api/tokenize/llama', str, padding); + return countTokensRemote('/api/tokenizers/llama/encode', str, padding); case tokenizers.NERD: - return countTokensRemote('/api/tokenize/nerdstash', str, padding); + return countTokensRemote('/api/tokenizers/nerdstash/encode', str, padding); case tokenizers.NERD2: - return countTokensRemote('/api/tokenize/nerdstash_v2', str, padding); + return countTokensRemote('/api/tokenizers/nerdstash_v2/encode', str, padding); case tokenizers.MISTRAL: - return countTokensRemote('/api/tokenize/mistral', str, padding); + return countTokensRemote('/api/tokenizers/mistral/encode', str, padding); case tokenizers.YI: - return countTokensRemote('/api/tokenize/yi', str, padding); + return countTokensRemote('/api/tokenizers/yi/encode', str, padding); case tokenizers.API: return countTokensRemote('/tokenize_via_api', str, padding); default: @@ -349,7 +349,7 @@ export function countTokensOpenAI(messages, full = false) { jQuery.ajax({ async: false, type: 'POST', // - url: shouldTokenizeAI21 ? '/api/tokenize/ai21' : `/api/tokenize/openai?model=${model}`, + url: shouldTokenizeAI21 ? '/api/tokenizers/ai21/count' : `/api/tokenizers/openai/count?model=${model}`, data: JSON.stringify([message]), dataType: 'json', contentType: 'application/json', @@ -509,20 +509,20 @@ function decodeTextTokensRemote(endpoint, ids, model = '') { export function getTextTokens(tokenizerType, str) { switch (tokenizerType) { case tokenizers.GPT2: - return getTextTokensRemote('/api/tokenize/gpt2', str); + return getTextTokensRemote('/api/tokenizers/gpt2/encode', str); case tokenizers.LLAMA: - return getTextTokensRemote('/api/tokenize/llama', str); + return getTextTokensRemote('/api/tokenizers/llama/encode', str); case tokenizers.NERD: - return getTextTokensRemote('/api/tokenize/nerdstash', str); + return getTextTokensRemote('/api/tokenizers/nerdstash/encode', str); case tokenizers.NERD2: - return getTextTokensRemote('/api/tokenize/nerdstash_v2', str); + return getTextTokensRemote('/api/tokenizers/nerdstash_v2/encode', str); case tokenizers.MISTRAL: - return getTextTokensRemote('/api/tokenize/mistral', str); + return getTextTokensRemote('/api/tokenizers/mistral/encode', str); case tokenizers.YI: - return getTextTokensRemote('/api/tokenize/yi', str); + return getTextTokensRemote('/api/tokenizers/yi/encode', str); case tokenizers.OPENAI: { const model = getTokenizerModel(); - return getTextTokensRemote('/api/tokenize/openai-encode', str, model); + return getTextTokensRemote('/api/tokenizers/openai/encode', str, model); } case tokenizers.API: return getTextTokensRemote('/tokenize_via_api', str); @@ -540,20 +540,20 @@ export function getTextTokens(tokenizerType, str) { export function decodeTextTokens(tokenizerType, ids) { switch (tokenizerType) { case tokenizers.GPT2: - return decodeTextTokensRemote('/api/decode/gpt2', ids); + return decodeTextTokensRemote('/api/tokenizers/gpt2/decode', ids); case tokenizers.LLAMA: - return decodeTextTokensRemote('/api/decode/llama', ids); + return decodeTextTokensRemote('/api/tokenizers/llama/decode', ids); case tokenizers.NERD: - return decodeTextTokensRemote('/api/decode/nerdstash', ids); + return decodeTextTokensRemote('/api/tokenizers/nerdstash/decode', ids); case tokenizers.NERD2: - return decodeTextTokensRemote('/api/decode/nerdstash_v2', ids); + return decodeTextTokensRemote('/api/tokenizers/nerdstash_v2/decode', ids); case tokenizers.MISTRAL: - return decodeTextTokensRemote('/api/decode/mistral', ids); + return decodeTextTokensRemote('/api/tokenizers/mistral/decode', ids); case tokenizers.YI: - return decodeTextTokensRemote('/api/decode/yi', ids); + return decodeTextTokensRemote('/api/tokenizers/yi/decode', ids); case tokenizers.OPENAI: { const model = getTokenizerModel(); - return decodeTextTokensRemote('/api/decode/openai', ids, model); + return decodeTextTokensRemote('/api/tokenizers/openai/decode', ids, model); } default: console.warn('Calling decodeTextTokens with unsupported tokenizer type', tokenizerType); diff --git a/src/endpoints/tokenizers.js b/src/endpoints/tokenizers.js index d73af29b1..b0e6b10db 100644 --- a/src/endpoints/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -365,7 +365,7 @@ async function loadTokenizers() { * @param {any} jsonParser JSON parser middleware */ function registerEndpoints(app, jsonParser) { - app.post('/api/tokenize/ai21', jsonParser, async function (req, res) { + app.post('/api/tokenizers/ai21/count', jsonParser, async function (req, res) { if (!req.body) return res.sendStatus(400); const options = { method: 'POST', @@ -387,20 +387,20 @@ function registerEndpoints(app, jsonParser) { } }); - app.post('/api/tokenize/llama', jsonParser, createSentencepieceEncodingHandler(spp_llama)); - app.post('/api/tokenize/nerdstash', jsonParser, createSentencepieceEncodingHandler(spp_nerd)); - app.post('/api/tokenize/nerdstash_v2', jsonParser, createSentencepieceEncodingHandler(spp_nerd_v2)); - app.post('/api/tokenize/mistral', jsonParser, createSentencepieceEncodingHandler(spp_mistral)); - app.post('/api/tokenize/yi', jsonParser, createSentencepieceEncodingHandler(spp_yi)); - app.post('/api/tokenize/gpt2', jsonParser, createTiktokenEncodingHandler('gpt2')); - app.post('/api/decode/llama', jsonParser, createSentencepieceDecodingHandler(spp_llama)); - app.post('/api/decode/nerdstash', jsonParser, createSentencepieceDecodingHandler(spp_nerd)); - app.post('/api/decode/nerdstash_v2', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2)); - app.post('/api/decode/mistral', jsonParser, createSentencepieceDecodingHandler(spp_mistral)); - app.post('/api/decode/yi', jsonParser, createSentencepieceDecodingHandler(spp_yi)); - app.post('/api/decode/gpt2', jsonParser, createTiktokenDecodingHandler('gpt2')); + app.post('/api/tokenizers/llama/encode', jsonParser, createSentencepieceEncodingHandler(spp_llama)); + app.post('/api/tokenizers/nerdstash/encode', jsonParser, createSentencepieceEncodingHandler(spp_nerd)); + app.post('/api/tokenizers/nerdstash_v2/encode', jsonParser, createSentencepieceEncodingHandler(spp_nerd_v2)); + app.post('/api/tokenizers/mistral/encode', jsonParser, createSentencepieceEncodingHandler(spp_mistral)); + app.post('/api/tokenizers/yi/encode', jsonParser, createSentencepieceEncodingHandler(spp_yi)); + app.post('/api/tokenizers/gpt2/encode', jsonParser, createTiktokenEncodingHandler('gpt2')); + app.post('/api/tokenizers/llama/decode', jsonParser, createSentencepieceDecodingHandler(spp_llama)); + app.post('/api/tokenizers/nerdstash/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd)); + app.post('/api/tokenizers/nerdstash_v2/decode', jsonParser, createSentencepieceDecodingHandler(spp_nerd_v2)); + app.post('/api/tokenizers/mistral/decode', jsonParser, createSentencepieceDecodingHandler(spp_mistral)); + app.post('/api/tokenizers/yi/decode', jsonParser, createSentencepieceDecodingHandler(spp_yi)); + app.post('/api/tokenizers/gpt2/decode', jsonParser, createTiktokenDecodingHandler('gpt2')); - app.post('/api/tokenize/openai-encode', jsonParser, async function (req, res) { + app.post('/api/tokenizers/openai/encode', jsonParser, async function (req, res) { try { const queryModel = String(req.query.model || ''); @@ -435,7 +435,7 @@ function registerEndpoints(app, jsonParser) { } }); - app.post('/api/decode/openai', jsonParser, async function (req, res) { + app.post('/api/tokenizers/openai/decode', jsonParser, async function (req, res) { try { const queryModel = String(req.query.model || ''); @@ -469,7 +469,7 @@ function registerEndpoints(app, jsonParser) { } }); - app.post('/api/tokenize/openai', jsonParser, async function (req, res) { + app.post('/api/tokenizers/openai/count', jsonParser, async function (req, res) { try { if (!req.body) return res.sendStatus(400); From 9d4990d25dbee76878b8be60e3e509a75502e21b Mon Sep 17 00:00:00 2001 From: valadaptive Date: Sun, 3 Dec 2023 09:25:09 -0500 Subject: [PATCH 09/10] Rename sd-next/upscalers to sd/next/upscalers --- public/scripts/extensions/stable-diffusion/index.js | 2 +- src/endpoints/stable-diffusion.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index 7e193db8a..9a57bcd8e 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -967,7 +967,7 @@ async function getAutoRemoteUpscalers() { async function getVladRemoteUpscalers() { try { - const result = await fetch('/api/sd-next/upscalers', { + const result = await fetch('/api/sd/sd-next/upscalers', { method: 'POST', headers: getRequestHeaders(), body: JSON.stringify(getSdRequestBody()), diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index f6c94be98..2375bc44b 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -291,7 +291,7 @@ function registerEndpoints(app, jsonParser) { } }); - app.post('/api/sd-next/upscalers', jsonParser, async (request, response) => { + app.post('/api/sd/sd-next/upscalers', jsonParser, async (request, response) => { try { const url = new URL(request.body.url); url.pathname = '/sdapi/v1/upscalers'; From 8bf18891b37db966572dde1a696d07210b15fd16 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 4 Dec 2023 18:36:05 +0200 Subject: [PATCH 10/10] Add data model field for WI entry group --- public/scripts/world-info.js | 5 +++++ server.js | 1 + 2 files changed, 6 insertions(+) diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index f156a8cd3..5c9d52775 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -1374,6 +1374,7 @@ const newEntryTemplate = { probability: 100, useProbability: true, depth: DEFAULT_DEPTH, + group: '', }; function createWorldInfoEntry(name, data, fromSlashCommand = false) { @@ -1970,6 +1971,7 @@ function convertAgnaiMemoryBook(inputObj) { displayIndex: index, probability: null, useProbability: false, + group: '', }; }); @@ -1996,6 +1998,7 @@ function convertRisuLorebook(inputObj) { displayIndex: index, probability: entry.activationPercent ?? null, useProbability: entry.activationPercent ?? false, + group: '', }; }); @@ -2027,6 +2030,7 @@ function convertNovelLorebook(inputObj) { displayIndex: index, probability: null, useProbability: false, + group: '', }; }); @@ -2060,6 +2064,7 @@ function convertCharacterBook(characterBook) { useProbability: entry.extensions?.useProbability ?? false, depth: entry.extensions?.depth ?? DEFAULT_DEPTH, selectiveLogic: entry.extensions?.selectiveLogic ?? 0, + group: entry.extensions?.group ?? '', }; }); diff --git a/server.js b/server.js index 8354e2d21..61ab6857d 100644 --- a/server.js +++ b/server.js @@ -1876,6 +1876,7 @@ function convertWorldInfoToCharacterBook(name, entries) { useProbability: entry.useProbability ?? false, depth: entry.depth ?? 4, selectiveLogic: entry.selectiveLogic ?? 0, + group: entry.group ?? '', }, };