rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

#371 Add llama.cpp inference server support

Browse Source
This commit is contained in:
Cohee
2023-12-18 22:38:28 +02:00
parent 6e8104873e
commit edd737e8bd
9 changed files with 136 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
public/scripts/tokenizers.js
View File

@ -6,7 +6,7 @@ import { getStringHash } from './utils.js';
import { kai_flags } from './kai-settings.js';
import { textgen_types, textgenerationwebui_settings as textgen_settings } from './textgen-settings.js';
const { OOBA, TABBY, KOBOLDCPP, MANCER, TOGETHERAI } = textgen_types;
const { OOBA, TABBY, KOBOLDCPP, APHRODITE, LLAMACPP } = textgen_types;
export const CHARACTERS_PER_TOKEN_RATIO = 3.35;
const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown';
@ -190,7 +190,7 @@ export function getTokenizerBestMatch(forApi) {
// - Tokenizer haven't reported an error previously
const hasTokenizerError = sessionStorage.getItem(TOKENIZER_WARNING_KEY);
const isConnected = online_status !== 'no_connection';
const isTokenizerSupported = textgen_settings.type === OOBA || textgen_settings.type === TABBY || textgen_settings.type === KOBOLDCPP;
const isTokenizerSupported = [OOBA, TABBY, KOBOLDCPP, LLAMACPP].includes(textgen_settings.type);
if (!hasTokenizerError && isConnected) {
if (forApi === 'kobold' && kai_flags.can_use_tokenization) {
@ -538,10 +538,7 @@ function getTextgenAPITokenizationParams(str) {
text: str,
api_type: textgen_settings.type,
url: api_server_textgenerationwebui,
legacy_api:
textgen_settings.legacy_api &&
textgen_settings.type !== MANCER &&
textgen_settings.type !== TOGETHERAI,
legacy_api: textgen_settings.legacy_api && (textgen_settings.type === OOBA || textgen_settings.type === APHRODITE),
};
}