rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1446 from valadaptive/move-endpoints 

Move endpoints into their own folder
This commit is contained in:
Cohee 2023-12-04 18:46:59 +02:00 committed by GitHub
parent b697a3c93f 24546621e3
commit 11a60e5eec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 48 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
server.js
View File

@ -51,12 +51,12 @@ util.inspect.defaultOptions.maxStringLength = null;
// local library imports // local library imports
const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware'); const basicAuthMiddleware = require('./src/middleware/basicAuthMiddleware');
const characterCardParser = require('./src/character-card-parser.js'); const characterCardParser = require('./src/character-card-parser.js');
const contentManager = require('./src/content-manager'); const contentManager = require('./src/endpoints/content-manager');
const statsHelpers = require('./statsHelpers.js'); const statsHelpers = require('./statsHelpers.js');
const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/secrets'); const { readSecret, migrateSecrets, SECRET_KEYS } = require('./src/endpoints/secrets');
const { delay, getVersion, deepMerge, getConfigValue, color, uuidv4 } = require('./src/util'); const { delay, getVersion, deepMerge, getConfigValue, color, uuidv4 } = require('./src/util');
const { invalidateThumbnail, ensureThumbnailCache } = require('./src/thumbnails'); const { invalidateThumbnail, ensureThumbnailCache } = require('./src/endpoints/thumbnails');
const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/tokenizers'); const { getTokenizerModel, getTiktokenTokenizer, loadTokenizers, TEXT_COMPLETION_MODELS, getSentencepiceTokenizer, sentencepieceTokenizers } = require('./src/endpoints/tokenizers');
const { convertClaudePrompt } = require('./src/chat-completion'); const { convertClaudePrompt } = require('./src/chat-completion');
// Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0. // Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0.
@ -2002,7 +2002,7 @@ app.post('/importcharacter', urlencodedParser, async function (request, response
let uploadPath = path.join(UPLOADS_PATH, filedata.filename); let uploadPath = path.join(UPLOADS_PATH, filedata.filename);
var format = request.body.file_type; var format = request.body.file_type;
const defaultAvatarPath = './public/img/ai4.png'; const defaultAvatarPath = './public/img/ai4.png';
const { importRisuSprites } = require('./src/sprites'); const { importRisuSprites } = require('./src/endpoints/sprites');
//console.log(format); //console.log(format);
if (filedata) { if (filedata) {
if (format == 'json') { if (format == 'json') {
@ -3576,55 +3576,55 @@ async function fetchJSON(url, args = {}) {
// ** END ** // ** END **
// OpenAI API // OpenAI API
require('./src/openai').registerEndpoints(app, jsonParser, urlencodedParser); require('./src/endpoints/openai').registerEndpoints(app, jsonParser, urlencodedParser);
// Tokenizers // Tokenizers
require('./src/tokenizers').registerEndpoints(app, jsonParser); require('./src/endpoints/tokenizers').registerEndpoints(app, jsonParser);
// Preset management // Preset management
require('./src/presets').registerEndpoints(app, jsonParser); require('./src/endpoints/presets').registerEndpoints(app, jsonParser);
// Secrets managemenet // Secrets managemenet
require('./src/secrets').registerEndpoints(app, jsonParser); require('./src/endpoints/secrets').registerEndpoints(app, jsonParser);
// Thumbnail generation // Thumbnail generation
require('./src/thumbnails').registerEndpoints(app, jsonParser); require('./src/endpoints/thumbnails').registerEndpoints(app, jsonParser);
// NovelAI generation // NovelAI generation
require('./src/novelai').registerEndpoints(app, jsonParser); require('./src/endpoints/novelai').registerEndpoints(app, jsonParser);
// Third-party extensions // Third-party extensions
require('./src/extensions').registerEndpoints(app, jsonParser); require('./src/endpoints/extensions').registerEndpoints(app, jsonParser);
// Asset management // Asset management
require('./src/assets').registerEndpoints(app, jsonParser); require('./src/endpoints/assets').registerEndpoints(app, jsonParser);
// Character sprite management // Character sprite management
require('./src/sprites').registerEndpoints(app, jsonParser, urlencodedParser); require('./src/endpoints/sprites').registerEndpoints(app, jsonParser, urlencodedParser);
// Custom content management // Custom content management
require('./src/content-manager').registerEndpoints(app, jsonParser); require('./src/endpoints/content-manager').registerEndpoints(app, jsonParser);
// Stable Diffusion generation // Stable Diffusion generation
require('./src/stable-diffusion').registerEndpoints(app, jsonParser); require('./src/endpoints/stable-diffusion').registerEndpoints(app, jsonParser);
// LLM and SD Horde generation // LLM and SD Horde generation
require('./src/horde').registerEndpoints(app, jsonParser); require('./src/endpoints/horde').registerEndpoints(app, jsonParser);
// Vector storage DB // Vector storage DB
require('./src/vectors').registerEndpoints(app, jsonParser); require('./src/endpoints/vectors').registerEndpoints(app, jsonParser);
// Chat translation // Chat translation
require('./src/translate').registerEndpoints(app, jsonParser); require('./src/endpoints/translate').registerEndpoints(app, jsonParser);
// Emotion classification // Emotion classification
require('./src/classify').registerEndpoints(app, jsonParser); require('./src/endpoints/classify').registerEndpoints(app, jsonParser);
// Image captioning // Image captioning
require('./src/caption').registerEndpoints(app, jsonParser); require('./src/endpoints/caption').registerEndpoints(app, jsonParser);
// Web search extension // Web search extension
require('./src/serpapi').registerEndpoints(app, jsonParser); require('./src/endpoints/serpapi').registerEndpoints(app, jsonParser);
const tavernUrl = new URL( const tavernUrl = new URL(
(cliArguments.ssl ? 'https://' : 'http://') + (cliArguments.ssl ? 'https://' : 'http://') +

2
src/assets.js → src/endpoints/assets.js
View File

@ -4,7 +4,7 @@ const sanitize = require('sanitize-filename');
const fetch = require('node-fetch').default; const fetch = require('node-fetch').default;
const { finished } = require('stream/promises'); const { finished } = require('stream/promises');
const writeFileSyncAtomic = require('write-file-atomic').sync; const writeFileSyncAtomic = require('write-file-atomic').sync;
const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('./constants'); const { DIRECTORIES, UNSAFE_EXTENSIONS } = require('../constants');
const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d']; const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d'];

2
src/caption.js → src/endpoints/caption.js
View File

@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) {
try { try {
const { image } = req.body; const { image } = req.body;
const module = await import('./transformers.mjs'); const module = await import('../transformers.mjs');
const rawImage = await module.default.getRawImage(image); const rawImage = await module.default.getRawImage(image);
if (!rawImage) { if (!rawImage) {

4
src/classify.js → src/endpoints/classify.js
View File

@ -9,7 +9,7 @@ function registerEndpoints(app, jsonParser) {
app.post('/api/extra/classify/labels', jsonParser, async (req, res) => { app.post('/api/extra/classify/labels', jsonParser, async (req, res) => {
try { try {
const module = await import('./transformers.mjs'); const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(TASK); const pipe = await module.default.getPipeline(TASK);
const result = Object.keys(pipe.model.config.label2id); const result = Object.keys(pipe.model.config.label2id);
return res.json({ labels: result }); return res.json({ labels: result });
@ -27,7 +27,7 @@ function registerEndpoints(app, jsonParser) {
if (Object.hasOwn(cacheObject, text)) { if (Object.hasOwn(cacheObject, text)) {
return cacheObject[text]; return cacheObject[text];
} else { } else {
const module = await import('./transformers.mjs'); const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(TASK); const pipe = await module.default.getPipeline(TASK);
const result = await pipe(text, { topk: 5 }); const result = await pipe(text, { topk: 5 });
result.sort((a, b) => b.score - a.score); result.sort((a, b) => b.score - a.score);

4
src/content-manager.js → src/endpoints/content-manager.js
View File

@ -2,11 +2,11 @@ const fs = require('fs');
const path = require('path'); const path = require('path');
const fetch = require('node-fetch').default; const fetch = require('node-fetch').default;
const sanitize = require('sanitize-filename'); const sanitize = require('sanitize-filename');
const { getConfigValue } = require('./util'); const { getConfigValue } = require('../util');
const contentDirectory = path.join(process.cwd(), 'default/content'); const contentDirectory = path.join(process.cwd(), 'default/content');
const contentLogPath = path.join(contentDirectory, 'content.log'); const contentLogPath = path.join(contentDirectory, 'content.log');
const contentIndexPath = path.join(contentDirectory, 'index.json'); const contentIndexPath = path.join(contentDirectory, 'index.json');
const { DIRECTORIES } = require('./constants'); const { DIRECTORIES } = require('../constants');
const presetFolders = [DIRECTORIES.koboldAI_Settings, DIRECTORIES.openAI_Settings, DIRECTORIES.novelAI_Settings, DIRECTORIES.textGen_Settings]; const presetFolders = [DIRECTORIES.koboldAI_Settings, DIRECTORIES.openAI_Settings, DIRECTORIES.novelAI_Settings, DIRECTORIES.textGen_Settings];
/** /**

2
src/extensions.js → src/endpoints/extensions.js
View File

@ -2,7 +2,7 @@ const path = require('path');
const fs = require('fs'); const fs = require('fs');
const { default: simpleGit } = require('simple-git'); const { default: simpleGit } = require('simple-git');
const sanitize = require('sanitize-filename'); const sanitize = require('sanitize-filename');
const { DIRECTORIES } = require('./constants'); const { DIRECTORIES } = require('../constants');
/** /**
* This function extracts the extension information from the manifest file. * This function extracts the extension information from the manifest file.

4
src/horde.js → src/endpoints/horde.js
View File

@ -1,6 +1,6 @@
const fetch = require('node-fetch').default; const fetch = require('node-fetch').default;
const AIHorde = require('./ai_horde'); const AIHorde = require('../ai_horde');
const { getVersion, delay } = require('./util'); const { getVersion, delay } = require('../util');
const { readSecret, SECRET_KEYS } = require('./secrets'); const { readSecret, SECRET_KEYS } = require('./secrets');
const ANONYMOUS_KEY = '0000000000'; const ANONYMOUS_KEY = '0000000000';

2
src/novelai.js → src/endpoints/novelai.js
View File

@ -2,7 +2,7 @@ const fetch = require('node-fetch').default;
const util = require('util'); const util = require('util');
const { Readable } = require('stream'); const { Readable } = require('stream');
const { readSecret, SECRET_KEYS } = require('./secrets'); const { readSecret, SECRET_KEYS } = require('./secrets');
const { readAllChunks, extractFileFromZipBuffer } = require('./util'); const { readAllChunks, extractFileFromZipBuffer } = require('../util');
const API_NOVELAI = 'https://api.novelai.net'; const API_NOVELAI = 'https://api.novelai.net';

0
src/openai.js → src/endpoints/openai.js
View File

2
src/presets.js → src/endpoints/presets.js
View File

@ -2,7 +2,7 @@ const fs = require('fs');
const path = require('path'); const path = require('path');
const sanitize = require('sanitize-filename'); const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync; const writeFileAtomicSync = require('write-file-atomic').sync;
const { DIRECTORIES } = require('./constants'); const { DIRECTORIES } = require('../constants');
const { getDefaultPresetFile, getDefaultPresets } = require('./content-manager'); const { getDefaultPresetFile, getDefaultPresets } = require('./content-manager');
/** /**

2
src/secrets.js → src/endpoints/secrets.js
View File

@ -1,6 +1,6 @@
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
const { getConfigValue } = require('./util'); const { getConfigValue } = require('../util');
const writeFileAtomicSync = require('write-file-atomic').sync; const writeFileAtomicSync = require('write-file-atomic').sync;
const SECRETS_FILE = path.join(process.cwd(), './secrets.json'); const SECRETS_FILE = path.join(process.cwd(), './secrets.json');

0
src/serpapi.js → src/endpoints/serpapi.js
View File

4
src/sprites.js → src/endpoints/sprites.js
View File

@ -4,8 +4,8 @@ const path = require('path');
const mime = require('mime-types'); const mime = require('mime-types');
const sanitize = require('sanitize-filename'); const sanitize = require('sanitize-filename');
const writeFileAtomicSync = require('write-file-atomic').sync; const writeFileAtomicSync = require('write-file-atomic').sync;
const { DIRECTORIES, UPLOADS_PATH } = require('./constants'); const { DIRECTORIES, UPLOADS_PATH } = require('../constants');
const { getImageBuffers } = require('./util'); const { getImageBuffers } = require('../util');
/** /**
* Gets the path to the sprites folder for the provided character name * Gets the path to the sprites folder for the provided character name

6
src/stable-diffusion.js → src/endpoints/stable-diffusion.js
View File

@ -1,8 +1,8 @@
const fetch = require('node-fetch').default; const fetch = require('node-fetch').default;
const sanitize = require('sanitize-filename'); const sanitize = require('sanitize-filename');
const { getBasicAuthHeader, delay } = require('./util'); const { getBasicAuthHeader, delay } = require('../util.js');
const fs = require('fs'); const fs = require('fs');
const { DIRECTORIES } = require('./constants.js'); const { DIRECTORIES } = require('../constants.js');
const writeFileAtomicSync = require('write-file-atomic').sync; const writeFileAtomicSync = require('write-file-atomic').sync;
/** /**
@ -341,7 +341,7 @@ function registerEndpoints(app, jsonParser) {
try { try {
const task = 'text-generation'; const task = 'text-generation';
const module = await import('./transformers.mjs'); const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(task); const pipe = await module.default.getPipeline(task);
const result = await pipe(prompt, { num_beams: 1, max_new_tokens: 256, do_sample: true }); const result = await pipe(prompt, { num_beams: 1, max_new_tokens: 256, do_sample: true });

4
src/thumbnails.js → src/endpoints/thumbnails.js
View File

@ -3,8 +3,8 @@ const path = require('path');
const sanitize = require('sanitize-filename'); const sanitize = require('sanitize-filename');
const jimp = require('jimp'); const jimp = require('jimp');
const writeFileAtomicSync = require('write-file-atomic').sync; const writeFileAtomicSync = require('write-file-atomic').sync;
const { DIRECTORIES } = require('./constants'); const { DIRECTORIES } = require('../constants');
const { getConfigValue } = require('./util'); const { getConfigValue } = require('../util');
/** /**
* Gets a path to thumbnail folder based on the type. * Gets a path to thumbnail folder based on the type.

2
src/tokenizers.js → src/endpoints/tokenizers.js
View File

@ -3,7 +3,7 @@ const path = require('path');
const { SentencePieceProcessor } = require('@agnai/sentencepiece-js'); const { SentencePieceProcessor } = require('@agnai/sentencepiece-js');
const tiktoken = require('@dqbd/tiktoken'); const tiktoken = require('@dqbd/tiktoken');
const { Tokenizer } = require('@agnai/web-tokenizers'); const { Tokenizer } = require('@agnai/web-tokenizers');
const { convertClaudePrompt } = require('./chat-completion'); const { convertClaudePrompt } = require('../chat-completion');
const { readSecret, SECRET_KEYS } = require('./secrets'); const { readSecret, SECRET_KEYS } = require('./secrets');
/** /**

2
src/translate.js → src/endpoints/translate.js
View File

@ -1,7 +1,7 @@
const fetch = require('node-fetch').default; const fetch = require('node-fetch').default;
const https = require('https'); const https = require('https');
const { readSecret, SECRET_KEYS } = require('./secrets'); const { readSecret, SECRET_KEYS } = require('./secrets');
const { getConfigValue } = require('./util'); const { getConfigValue } = require('../util');
const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate'; const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate';
const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate'; const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate';

6
src/vectors.js → src/endpoints/vectors.js
View File

@ -11,11 +11,11 @@ const sanitize = require('sanitize-filename');
async function getVector(source, text) { async function getVector(source, text) {
switch (source) { switch (source) {
case 'openai': case 'openai':
return require('./openai-vectors').getOpenAIVector(text); return require('../openai-vectors').getOpenAIVector(text);
case 'transformers': case 'transformers':
return require('./embedding').getTransformersVector(text); return require('../embedding').getTransformersVector(text);
case 'palm': case 'palm':
return require('./palm-vectors').getPaLMVector(text); return require('../palm-vectors').getPaLMVector(text);
} }
throw new Error(`Unknown vector source ${source}`); throw new Error(`Unknown vector source ${source}`);

2
src/openai-vectors.js
View File

@ -1,5 +1,5 @@
const fetch = require('node-fetch').default; const fetch = require('node-fetch').default;
const { SECRET_KEYS, readSecret } = require('./secrets'); const { SECRET_KEYS, readSecret } = require('./endpoints/secrets');
/** /**
* Gets the vector for the given text from OpenAI ada model * Gets the vector for the given text from OpenAI ada model

2
src/palm-vectors.js
View File

@ -1,5 +1,5 @@
const fetch = require('node-fetch').default; const fetch = require('node-fetch').default;
const { SECRET_KEYS, readSecret } = require('./secrets'); const { SECRET_KEYS, readSecret } = require('./endpoints/secrets');
/** /**
* Gets the vector for the given text from PaLM gecko model * Gets the vector for the given text from PaLM gecko model