rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-02-18 21:20:39 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'staging' into woo-yeah

Browse Source
This commit is contained in:
Cohee 2025-01-24 23:50:15 +02:00
commit 9a2968d1eb
13 changed files with 140 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
default/config.yaml
View File

@ -133,24 +133,26 @@ whitelistImportDomains:
## headers:
## User-Agent: "Googlebot/2.1 (+http://www.google.com/bot.html)"
requestOverrides: []
# -- EXTENSIONS CONFIGURATION --
# Enable UI extensions
enableExtensions: true
# Automatically update extensions when a release version changes
enableExtensionsAutoUpdate: true
# EXTENSIONS CONFIGURATION
extensions:
# Enable UI extensions
enabled: true
# Automatically update extensions when a release version changes
autoUpdate: true
models:
# Enables automatic model download from HuggingFace
autoDownload: true
# Additional models for extensions. Expects model IDs from HuggingFace model hub in ONNX format
classification: Cohee/distilbert-base-uncased-go-emotions-onnx
captioning: Xenova/vit-gpt2-image-captioning
embedding: Cohee/jina-embeddings-v2-base-en
speechToText: Xenova/whisper-small
textToSpeech: Xenova/speecht5_tts
# Additional model tokenizers can be downloaded on demand.
# Disabling will fallback to another locally available tokenizer.
enableDownloadableTokenizers: true
# Extension settings
extras:
# Disables automatic model download from HuggingFace
disableAutoDownload: false
# Extra models for plugins. Expects model IDs from HuggingFace model hub in ONNX format
classificationModel: Cohee/distilbert-base-uncased-go-emotions-onnx
captioningModel: Xenova/vit-gpt2-image-captioning
embeddingModel: Cohee/jina-embeddings-v2-base-en
speechToTextModel: Xenova/whisper-small
textToSpeechModel: Xenova/speecht5_tts
# -- OPENAI CONFIGURATION --
# A placeholder message to use in strict prompt post-processing mode when the prompt doesn't start with a user message
promptPlaceholder: "[Start a new chat]"

2
jsconfig.json
View File

@ -15,7 +15,7 @@
"**/node_modules/**",
"**/dist/**",
"**/.git/**",
"public/lib/**",
"public/**",
"backups/**",
"data/**",
"cache/**",

40
post-install.js
View File

@ -64,6 +64,46 @@ const keyMigrationMap = [
newKey: 'backups.chat.throttleInterval',
migrate: (value) => value,
},
{
oldKey: 'enableExtensions',
newKey: 'extensions.enabled',
migrate: (value) => value,
},
{
oldKey: 'enableExtensionsAutoUpdate',
newKey: 'extensions.autoUpdate',
migrate: (value) => value,
},
{
oldKey: 'extras.disableAutoDownload',
newKey: 'extensions.models.autoDownload',
migrate: (value) => !value,
},
{
oldKey: 'extras.classificationModel',
newKey: 'extensions.models.classification',
migrate: (value) => value,
},
{
oldKey: 'extras.captioningModel',
newKey: 'extensions.models.captioning',
migrate: (value) => value,
},
{
oldKey: 'extras.embeddingModel',
newKey: 'extensions.models.embedding',
migrate: (value) => value,
},
{
oldKey: 'extras.speechToTextModel',
newKey: 'extensions.models.speechToText',
migrate: (value) => value,
},
{
oldKey: 'extras.textToSpeechModel',
newKey: 'extensions.models.textToSpeech',
migrate: (value) => value,
},
];
/**

16
public/locales/zh-cn.json
View File

@ -1191,9 +1191,9 @@
"welcome_message_part_8": "您可随时通过",
"welcome_message_part_9": "图标来更改此设置。",
"Persona Name:": "用户角色名称：",
"Temporarily disable automatic replies from this character": "暂时禁用此角色的自动回复",
"Enable automatic replies from this character": "启用此角色的自动回复",
"Trigger a message from this character": "从此角色触发消息",
"Temporarily disable automatic replies from this character": "临时禁言此角色",
"Enable automatic replies from this character": "解除禁言此角色",
"Trigger a message from this character": "强制触发该角色发言",
"Move up": "向上移动",
"Move down": "向下移动",
"View character card": "查看角色卡片",
@ -1838,7 +1838,7 @@
"Enter the Git URL of the extension to install": "输入扩展程序的 Git URL 以安装",
"Disclaimer:": "免责声明：",
"Please be aware that using external extensions can have unintended side effects and may pose security risks. Always make sure you trust the source before importing an extension. We are not responsible for any damage caused by third-party extensions.": "使用外部的扩展程序可能存在意料外的副作用和安全隐患。在导入扩展程序前，请一定确认其来源可信。我们不为第三方扩展程序造成的任何损失负责。",
"Prompt Itemization": "将提示词分条",
"Prompt Itemization": "提示词拆分",
"Show Raw Prompt": "显示原始提示词",
"Copy Prompt": "复制提示词",
"Show Prompt Differences": "显示提示词差异",
@ -2045,8 +2045,8 @@
"Post a GitHub issue": "在 GitHub 发布问题",
"Contact the developers": "联系开发者",
"If you're connected to an API, try asking me something!": "若您已经配置好API，尝试发送些什么吧！",
"Title/Memo": "标题/备忘录",
"Strategy": "Strategy",
"Position": "位置",
"Trigger %": "触发率 ％"
"Title/Memo": "标题（备忘）",
"Strategy": "触发策略",
"Position": "插入位置",
"Trigger %": "触发概率%"
}

8
public/scripts/st-context.js
View File

@ -12,6 +12,7 @@ import {
extension_prompts,
Generate,
generateQuietPrompt,
getCharacters,
getCurrentChatId,
getRequestHeaders,
getThumbnailUrl,
@ -55,7 +56,7 @@ import { MacrosParser } from './macros.js';
import { oai_settings } from './openai.js';
import { callGenericPopup, Popup, POPUP_RESULT, POPUP_TYPE } from './popup.js';
import { power_user, registerDebugFunction } from './power-user.js';
import { isMobile, shouldSendOnEnter } from './RossAscends-mods.js';
import { humanizedDateTime, isMobile, shouldSendOnEnter } from './RossAscends-mods.js';
import { ScraperManager } from './scrapers.js';
import { executeSlashCommands, executeSlashCommandsWithOptions, registerSlashCommand } from './slash-commands.js';
import { SlashCommand } from './slash-commands/SlashCommand.js';
@ -65,7 +66,7 @@ import { tag_map, tags } from './tags.js';
import { textgenerationwebui_settings } from './textgen-settings.js';
import { tokenizers, getTextTokens, getTokenCount, getTokenCountAsync, getTokenizerModel } from './tokenizers.js';
import { ToolManager } from './tool-calling.js';
import { timestampToMoment } from './utils.js';
import { timestampToMoment, uuidv4 } from './utils.js';
export function getContext() {
return {
@ -167,6 +168,9 @@ export function getContext() {
chatCompletionSettings: oai_settings,
textCompletionSettings: textgenerationwebui_settings,
powerUserSettings: power_user,
getCharacters,
uuidv4,
humanizedDateTime,
};
}

3
src/endpoints/avatars.js
View File

@ -9,6 +9,7 @@ import { sync as writeFileAtomicSync } from 'write-file-atomic';
import { jsonParser, urlencodedParser } from '../express-common.js';
import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js';
import { getImages, tryParse } from '../util.js';
import { getFileNameValidationFunction } from '../middleware/validateFileName.js';
export const router = express.Router();
@ -17,7 +18,7 @@ router.post('/get', jsonParser, function (request, response) {
response.send(JSON.stringify(images));
});
router.post('/delete', jsonParser, function (request, response) {
router.post('/delete', jsonParser, getFileNameValidationFunction('avatar'), function (request, response) {
if (!request.body) return response.sendStatus(400);
if (request.body.avatar !== sanitize(request.body.avatar)) {

3
src/endpoints/backgrounds.js
View File

@ -7,6 +7,7 @@ import sanitize from 'sanitize-filename';
import { jsonParser, urlencodedParser } from '../express-common.js';
import { invalidateThumbnail } from './thumbnails.js';
import { getImages } from '../util.js';
import { getFileNameValidationFunction } from '../middleware/validateFileName.js';
export const router = express.Router();
@ -15,7 +16,7 @@ router.post('/all', jsonParser, function (request, response) {
response.send(JSON.stringify(images));
});
router.post('/delete', jsonParser, function (request, response) {
router.post('/delete', jsonParser, getFileNameValidationFunction('bg'), function (request, response) {
if (!request.body) return response.sendStatus(400);
if (request.body.bg !== sanitize(request.body.bg)) {

19
src/endpoints/characters.js
View File

@ -14,6 +14,7 @@ import jimp from 'jimp';
import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js';
import { jsonParser, urlencodedParser } from '../express-common.js';
import { default as validateAvatarUrlMiddleware, getFileNameValidationFunction } from '../middleware/validateFileName.js';
import { deepMerge, humanizedISO8601DateTime, tryParse, extractFileFromZipBuffer, MemoryLimitedMap, getConfigValue } from '../util.js';
import { TavernCardValidator } from '../validator/TavernCardValidator.js';
import { parse, write } from '../character-card-parser.js';
@ -756,7 +757,7 @@ router.post('/create', urlencodedParser, async function (request, response) {
}
});
router.post('/rename', jsonParser, async function (request, response) {
router.post('/rename', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
if (!request.body.avatar_url || !request.body.new_name) {
return response.sendStatus(400);
}
@ -803,7 +804,7 @@ router.post('/rename', jsonParser, async function (request, response) {
}
});
router.post('/edit', urlencodedParser, async function (request, response) {
router.post('/edit', urlencodedParser, validateAvatarUrlMiddleware, async function (request, response) {
if (!request.body) {
console.error('Error: no response body detected');
response.status(400).send('Error: no response body detected');
@ -852,7 +853,7 @@ router.post('/edit', urlencodedParser, async function (request, response) {
* @param {Object} response - The HTTP response object.
* @returns {void}
*/
router.post('/edit-attribute', jsonParser, async function (request, response) {
router.post('/edit-attribute', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
console.log(request.body);
if (!request.body) {
console.error('Error: no response body detected');
@ -898,7 +899,7 @@ router.post('/edit-attribute', jsonParser, async function (request, response) {
*
* @returns {void}
* */
router.post('/merge-attributes', jsonParser, async function (request, response) {
router.post('/merge-attributes', jsonParser, getFileNameValidationFunction('avatar'), async function (request, response) {
try {
const update = request.body;
const avatarPath = path.join(request.user.directories.characters, update.avatar);
@ -929,7 +930,7 @@ router.post('/merge-attributes', jsonParser, async function (request, response)
}
});
router.post('/delete', jsonParser, async function (request, response) {
router.post('/delete', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
if (!request.body || !request.body.avatar_url) {
return response.sendStatus(400);
}
@ -992,7 +993,7 @@ router.post('/all', jsonParser, async function (request, response) {
}
});
router.post('/get', jsonParser, async function (request, response) {
router.post('/get', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
try {
if (!request.body) return response.sendStatus(400);
const item = request.body.avatar_url;
@ -1011,7 +1012,7 @@ router.post('/get', jsonParser, async function (request, response) {
}
});
router.post('/chats', jsonParser, async function (request, response) {
router.post('/chats', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
if (!request.body) return response.sendStatus(400);
const characterDirectory = (request.body.avatar_url).replace('.png', '');
@ -1160,7 +1161,7 @@ router.post('/import', urlencodedParser, async function (request, response) {
}
});
router.post('/duplicate', jsonParser, async function (request, response) {
router.post('/duplicate', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
try {
if (!request.body.avatar_url) {
console.log('avatar URL not found in request body');
@ -1207,7 +1208,7 @@ router.post('/duplicate', jsonParser, async function (request, response) {
}
});
router.post('/export', jsonParser, async function (request, response) {
router.post('/export', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
try {
if (!request.body.format || !request.body.avatar_url) {
return response.sendStatus(400);

15
src/endpoints/chats.js
View File

@ -9,6 +9,7 @@ import { sync as writeFileAtomicSync } from 'write-file-atomic';
import _ from 'lodash';
import { jsonParser, urlencodedParser } from '../express-common.js';
import validateAvatarUrlMiddleware from '../middleware/validateFileName.js';
import {
getConfigValue,
humanizedISO8601DateTime,
@ -294,7 +295,7 @@ function importRisuChat(userName, characterName, jsonData) {
export const router = express.Router();
router.post('/save', jsonParser, function (request, response) {
router.post('/save', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
try {
const directoryName = String(request.body.avatar_url).replace('.png', '');
const chatData = request.body.chat;
@ -310,7 +311,7 @@ router.post('/save', jsonParser, function (request, response) {
}
});
router.post('/get', jsonParser, function (request, response) {
router.post('/get', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
try {
const dirName = String(request.body.avatar_url).replace('.png', '');
const directoryPath = path.join(request.user.directories.chats, dirName);
@ -347,7 +348,7 @@ router.post('/get', jsonParser, function (request, response) {
});
router.post('/rename', jsonParser, async function (request, response) {
router.post('/rename', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
if (!request.body || !request.body.original_file || !request.body.renamed_file) {
return response.sendStatus(400);
}
@ -372,7 +373,7 @@ router.post('/rename', jsonParser, async function (request, response) {
return response.send({ ok: true, sanitizedFileName });
});
router.post('/delete', jsonParser, function (request, response) {
router.post('/delete', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
const dirName = String(request.body.avatar_url).replace('.png', '');
const fileName = String(request.body.chatfile);
const filePath = path.join(request.user.directories.chats, dirName, sanitize(fileName));
@ -388,7 +389,7 @@ router.post('/delete', jsonParser, function (request, response) {
return response.send('ok');
});
router.post('/export', jsonParser, async function (request, response) {
router.post('/export', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
if (!request.body.file || (!request.body.avatar_url && request.body.is_group === false)) {
return response.sendStatus(400);
}
@ -478,7 +479,7 @@ router.post('/group/import', urlencodedParser, function (request, response) {
}
});
router.post('/import', urlencodedParser, function (request, response) {
router.post('/import', urlencodedParser, validateAvatarUrlMiddleware, function (request, response) {
if (!request.body) return response.sendStatus(400);
const format = request.body.file_type;
@ -626,7 +627,7 @@ router.post('/group/save', jsonParser, (request, response) => {
return response.send({ ok: true });
});
router.post('/search', jsonParser, function (request, response) {
router.post('/search', jsonParser, validateAvatarUrlMiddleware, function (request, response) {
try {
const { query, avatar_url, group_id } = request.body;
let chatFiles = [];

9
src/endpoints/settings.js
View File

@ -9,9 +9,10 @@ import { SETTINGS_FILE } from '../constants.js';
import { getConfigValue, generateTimestamp, removeOldBackups } from '../util.js';
import { jsonParser } from '../express-common.js';
import { getAllUserHandles, getUserDirectories } from '../users.js';
import { getFileNameValidationFunction } from '../middleware/validateFileName.js';
const ENABLE_EXTENSIONS = getConfigValue('enableExtensions', true);
const ENABLE_EXTENSIONS_AUTO_UPDATE = getConfigValue('enableExtensionsAutoUpdate', true);
const ENABLE_EXTENSIONS = !!getConfigValue('extensions.enabled', true);
const ENABLE_EXTENSIONS_AUTO_UPDATE = !!getConfigValue('extensions.autoUpdate', true);
const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false);
// 10 minutes
@ -296,7 +297,7 @@ router.post('/get-snapshots', jsonParser, async (request, response) => {
}
});
router.post('/load-snapshot', jsonParser, async (request, response) => {
router.post('/load-snapshot', jsonParser, getFileNameValidationFunction('name'), async (request, response) => {
try {
const userFilesPattern = getFilePrefix(request.user.profile.handle);
@ -330,7 +331,7 @@ router.post('/make-snapshot', jsonParser, async (request, response) => {
}
});
router.post('/restore-snapshot', jsonParser, async (request, response) => {
router.post('/restore-snapshot', jsonParser, getFileNameValidationFunction('name'), async (request, response) => {
try {
const userFilesPattern = getFilePrefix(request.user.profile.handle);

2
src/endpoints/vectors.js
View File

@ -164,7 +164,7 @@ function getSourceSettings(source, request) {
};
case 'transformers':
return {
model: getConfigValue('extras.embeddingModel', ''),
model: getConfigValue('extensions.models.embedding', ''),
};
case 'palm':
return {

34
src/middleware/validateFileName.js Normal file
View File

@ -0,0 +1,34 @@
import path from 'node:path';
/**
* Gets a middleware function that validates the field in the request body.
* @param {string} fieldName Field name
* @returns {import('express').RequestHandler} Middleware function
*/
export function getFileNameValidationFunction(fieldName) {
/**
* Validates the field in the request body.
* @param {import('express').Request} req Request object
* @param {import('express').Response} res Response object
* @param {import('express').NextFunction} next Next middleware
*/
return function validateAvatarUrlMiddleware(req, res, next) {
if (req.body && fieldName in req.body && typeof req.body[fieldName] === 'string') {
const forbiddenRegExp = path.sep === '/' ? /[/\x00]/ : /[/\x00\\]/;
if (forbiddenRegExp.test(req.body[fieldName])) {
console.error('An error occurred while validating the request body', {
handle: req.user.profile.handle,
path: req.originalUrl,
field: fieldName,
value: req.body[fieldName],
});
return res.sendStatus(400);
}
}
next();
};
}
const avatarUrlValidationFunction = getFileNameValidationFunction('avatar_url');
export default avatarUrlValidationFunction;

12
src/transformers.js
View File

@ -19,31 +19,31 @@ const tasks = {
'text-classification': {
defaultModel: 'Cohee/distilbert-base-uncased-go-emotions-onnx',
pipeline: null,
configField: 'extras.classificationModel',
configField: 'extensions.models.classification',
quantized: true,
},
'image-to-text': {
defaultModel: 'Xenova/vit-gpt2-image-captioning',
pipeline: null,
configField: 'extras.captioningModel',
configField: 'extensions.models.captioning',
quantized: true,
},
'feature-extraction': {
defaultModel: 'Xenova/all-mpnet-base-v2',
pipeline: null,
configField: 'extras.embeddingModel',
configField: 'extensions.models.embedding',
quantized: true,
},
'automatic-speech-recognition': {
defaultModel: 'Xenova/whisper-small',
pipeline: null,
configField: 'extras.speechToTextModel',
configField: 'extensions.models.speechToText',
quantized: true,
},
'text-to-speech': {
defaultModel: 'Xenova/speecht5_tts',
pipeline: null,
configField: 'extras.textToSpeechModel',
configField: 'extensions.models.textToSpeech',
quantized: false,
},
};
@ -132,7 +132,7 @@ export async function getPipeline(task, forceModel = '') {
const cacheDir = path.join(globalThis.DATA_ROOT, '_cache');
const model = forceModel || getModelForTask(task);
const localOnly = getConfigValue('extras.disableAutoDownload', false);
const localOnly = !getConfigValue('extensions.models.autoDownload', true);
console.log('Initializing transformers.js pipeline for task', task, 'with model', model);
const instance = await pipeline(task, model, { cache_dir: cacheDir, quantized: tasks[task].quantized ?? true, local_files_only: localOnly });
tasks[task].pipeline = instance;