rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Revert MANUAL_SELECTION

Browse Source 
This partially reverts commit c89804677a (keeps the turbo-0301 removal).
This commit is contained in:
Succubyss
2024-12-31 11:13:45 -06:00
parent 3a29756109
commit 97dc041777
1 changed files with 1 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
public/scripts/tokenizers.js
View File

@@ -33,7 +33,6 @@ export const tokenizers = {
NEMO: 17,
DEEPSEEK: 18,
BEST_MATCH: 99,
MANUAL_SELECTION: 411,
};
// A list of local tokenizers that support encoding and decoding token ids.
@@ -1073,14 +1072,9 @@ function decodeTextTokensFromServer(endpoint, ids, resolve) {
* Encodes a string to tokens using the server API.
* @param {number} tokenizerType Tokenizer type.
* @param {string} str String to tokenize.
* @param {string} overrideModel Tokenizer for {tokenizers.MANUAL_SELECTION}.
* @returns {number[]} Array of token ids.
*/
export function getTextTokens(tokenizerType, str, overrideModel = undefined) {
if (overrideModel && tokenizerType !== tokenizers.MANUAL_SELECTION) {
console.warn('overrideModel must be undefined unless using tokenizers.MANUAL_SELECTION', tokenizerType);
return [];
}
export function getTextTokens(tokenizerType, str) {
switch (tokenizerType) {
case tokenizers.API_CURRENT:
return getTextTokens(currentRemoteTokenizerAPI(), str);
@@ -1101,9 +1095,6 @@ export function getTextTokens(tokenizerType, str, overrideModel = undefined) {
console.warn('This tokenizer type does not support encoding', tokenizerType);
return [];
}
if (tokenizerType === tokenizers.MANUAL_SELECTION) {
endpointUrl += `?model=${overrideModel}`;
}
if (tokenizerType === tokenizers.OPENAI) {
endpointUrl += `?model=${getTokenizerModel()}`;
}