Add text trimming commands

This commit is contained in:
Cohee
2023-11-26 13:55:22 +02:00
parent 7b3f2a8986
commit e6c96553d0
3 changed files with 113 additions and 3 deletions

View File

@ -467,7 +467,11 @@ function getTextTokensRemote(endpoint, str, model = '') {
* @param {string} endpoint API endpoint.
* @param {number[]} ids Array of token ids
*/
function decodeTextTokensRemote(endpoint, ids) {
function decodeTextTokensRemote(endpoint, ids, model = '') {
if (model) {
endpoint += `?model=${model}`;
}
let text = '';
jQuery.ajax({
async: false,
@ -533,6 +537,9 @@ export function decodeTextTokens(tokenizerType, ids) {
return decodeTextTokensRemote('/api/decode/mistral', ids);
case tokenizers.YI:
return decodeTextTokensRemote('/api/decode/yi', ids);
case tokenizers.OPENAI:
const model = getTokenizerModel();
return decodeTextTokensRemote('/api/decode/openai', ids, model);
default:
console.warn("Calling decodeTextTokens with unsupported tokenizer type", tokenizerType);
return '';