Merge pull request #2177 from sasha0552/vllm-support

vLLM support
This commit is contained in:
Cohee
2024-05-03 20:03:05 +03:00
committed by GitHub
15 changed files with 230 additions and 86 deletions

View File

@ -22,7 +22,7 @@ import {
parseTabbyLogprobs,
} from './scripts/textgen-settings.js';
const { MANCER, TOGETHERAI, OOBA, APHRODITE, OLLAMA, INFERMATICAI, DREAMGEN, OPENROUTER } = textgen_types;
const { MANCER, TOGETHERAI, OOBA, VLLM, APHRODITE, OLLAMA, INFERMATICAI, DREAMGEN, OPENROUTER } = textgen_types;
import {
world_info,
@ -218,7 +218,7 @@ import {
import { getBackgrounds, initBackgrounds, loadBackgroundSettings, background_settings } from './scripts/backgrounds.js';
import { hideLoader, showLoader } from './scripts/loader.js';
import { BulkEditOverlay, CharacterContextMenu } from './scripts/BulkEditOverlay.js';
import { loadMancerModels, loadOllamaModels, loadTogetherAIModels, loadInfermaticAIModels, loadOpenRouterModels, loadAphroditeModels, loadDreamGenModels } from './scripts/textgen-models.js';
import { loadMancerModels, loadOllamaModels, loadTogetherAIModels, loadInfermaticAIModels, loadOpenRouterModels, loadVllmModels, loadAphroditeModels, loadDreamGenModels } from './scripts/textgen-models.js';
import { appendFileContent, hasPendingFileAttachment, populateFileAttachment, decodeStyleTags, encodeStyleTags, isExternalMediaAllowed, getCurrentEntityId } from './scripts/chats.js';
import { initPresetManager } from './scripts/preset-manager.js';
import { evaluateMacros } from './scripts/macros.js';
@ -1071,6 +1071,9 @@ async function getStatusTextgen() {
} else if (textgen_settings.type === OPENROUTER) {
loadOpenRouterModels(data?.data);
online_status = textgen_settings.openrouter_model;
} else if (textgen_settings.type === VLLM) {
loadVllmModels(data?.data);
online_status = textgen_settings.vllm_model;
} else if (textgen_settings.type === APHRODITE) {
loadAphroditeModels(data?.data);
online_status = textgen_settings.aphrodite_model;
@ -4832,6 +4835,7 @@ function parseAndSaveLogprobs(data, continueFrom) {
case textgen_types.LLAMACPP: {
logprobs = data?.completion_probabilities?.map(x => parseTextgenLogprobs(x.content, [x])) || null;
} break;
case textgen_types.VLLM:
case textgen_types.APHRODITE:
case textgen_types.MANCER:
case textgen_types.TABBY: {
@ -4888,7 +4892,7 @@ function extractMultiSwipes(data, type) {
return swipes;
}
if (main_api === 'openai' || (main_api === 'textgenerationwebui' && [MANCER, APHRODITE].includes(textgen_settings.type))) {
if (main_api === 'openai' || (main_api === 'textgenerationwebui' && [MANCER, VLLM, APHRODITE].includes(textgen_settings.type))) {
if (!Array.isArray(data.choices)) {
return swipes;
}
@ -7922,6 +7926,11 @@ const CONNECT_API_MAP = {
button: '#api_button_textgenerationwebui',
type: textgen_types.MANCER,
},
'vllm': {
selected: 'textgenerationwebui',
button: '#api_button_textgenerationwebui',
type: textgen_types.VLLM,
},
'aphrodite': {
selected: 'textgenerationwebui',
button: '#api_button_textgenerationwebui',
@ -8923,6 +8932,7 @@ jQuery(async function () {
$('#api_button_textgenerationwebui').on('click', async function (e) {
const keys = [
{ id: 'api_key_mancer', secret: SECRET_KEYS.MANCER },
{ id: 'api_key_vllm', secret: SECRET_KEYS.VLLM },
{ id: 'api_key_aphrodite', secret: SECRET_KEYS.APHRODITE },
{ id: 'api_key_tabby', secret: SECRET_KEYS.TABBY },
{ id: 'api_key_togetherai', secret: SECRET_KEYS.TOGETHERAI },