rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add text trimming commands

Browse Source
This commit is contained in:
Cohee
2023-11-26 13:55:22 +02:00
parent 7b3f2a8986
commit e6c96553d0
3 changed files with 113 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
public/scripts/tokenizers.js
View File

@@ -467,7 +467,11 @@ function getTextTokensRemote(endpoint, str, model = '') {
* @param {string} endpoint API endpoint.
* @param {number[]} ids Array of token ids
*/
function decodeTextTokensRemote(endpoint, ids) {
function decodeTextTokensRemote(endpoint, ids, model = '') {
if (model) {
endpoint += `?model=${model}`;
}
let text = '';
jQuery.ajax({
async: false,
@@ -533,6 +537,9 @@ export function decodeTextTokens(tokenizerType, ids) {
return decodeTextTokensRemote('/api/decode/mistral', ids);
case tokenizers.YI:
return decodeTextTokensRemote('/api/decode/yi', ids);
case tokenizers.OPENAI:
const model = getTokenizerModel();
return decodeTextTokensRemote('/api/decode/openai', ids, model);
default:
console.warn("Calling decodeTextTokens with unsupported tokenizer type", tokenizerType);
return '';