rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Initial vLLM support

Browse Source
This commit is contained in:
sasha0552
2024-05-02 22:40:40 +00:00
committed by GitHub
parent 1a219e32fe
commit 2bd239fe81
15 changed files with 231 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
src/constants.js
View File

@ -200,6 +200,7 @@ const UPLOADS_PATH = './uploads';
const TEXTGEN_TYPES = {
OOBA: 'ooba',
MANCER: 'mancer',
VLLM: 'vllm',
APHRODITE: 'aphrodite',
TABBY: 'tabby',
KOBOLDCPP: 'koboldcpp',
@ -298,6 +299,49 @@ const OPENROUTER_KEYS = [
'stop',
];
// https://github.com/vllm-project/vllm/blob/0f8a91401c89ac0a8018def3756829611b57727f/vllm/entrypoints/openai/protocol.py#L220
const VLLM_KEYS = [
'model',
'prompt',
'best_of',
'echo',
'frequency_penalty',
'logit_bias',
'logprobs',
'max_tokens',
'n',
'presence_penalty',
'seed',
'stop',
'stream',
'suffix',
'temperature',
'top_p',
'user',
'use_beam_search',
'top_k',
'min_p',
'repetition_penalty',
'length_penalty',
'early_stopping',
'stop_token_ids',
'ignore_eos',
'min_tokens',
'skip_special_tokens',
'spaces_between_special_tokens',
'truncate_prompt_tokens',
'include_stop_str_in_output',
'response_format',
'guided_json',
'guided_regex',
'guided_choice',
'guided_grammar',
'guided_decoding_backend',
'guided_whitespace_pattern',
];
module.exports = {
DEFAULT_USER,
DEFAULT_AVATAR,
@ -318,4 +362,5 @@ module.exports = {
DREAMGEN_KEYS,
OPENROUTER_HEADERS,
OPENROUTER_KEYS,
VLLM_KEYS,
};