Aphrodite tokenizer
This commit is contained in:
parent
4b845dd442
commit
10d78f9a25
|
@ -35,7 +35,7 @@ export const SENTENCEPIECE_TOKENIZERS = [
|
||||||
//tokenizers.NERD2,
|
//tokenizers.NERD2,
|
||||||
];
|
];
|
||||||
|
|
||||||
export const TEXTGEN_TOKENIZERS = [OOBA, TABBY, KOBOLDCPP, LLAMACPP];
|
export const TEXTGEN_TOKENIZERS = [OOBA, TABBY, KOBOLDCPP, LLAMACPP, APHRODITE];
|
||||||
|
|
||||||
const TOKENIZER_URLS = {
|
const TOKENIZER_URLS = {
|
||||||
[tokenizers.GPT2]: {
|
[tokenizers.GPT2]: {
|
||||||
|
|
|
@ -628,6 +628,10 @@ router.post('/remote/textgenerationwebui/encode', jsonParser, async function (re
|
||||||
url += '/tokenize';
|
url += '/tokenize';
|
||||||
args.body = JSON.stringify({ 'content': text });
|
args.body = JSON.stringify({ 'content': text });
|
||||||
break;
|
break;
|
||||||
|
case TEXTGEN_TYPES.APHRODITE:
|
||||||
|
url += '/v1/tokenize';
|
||||||
|
args.body = JSON.stringify({ 'prompt': text });
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
url += '/v1/internal/encode';
|
url += '/v1/internal/encode';
|
||||||
args.body = JSON.stringify({ 'text': text });
|
args.body = JSON.stringify({ 'text': text });
|
||||||
|
|
Loading…
Reference in New Issue