rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Implement Token Probabilities UI using logprobs

Browse Source
This commit is contained in:
khanon
2024-01-22 23:00:31 -06:00
parent 9b42be2334
commit 60044c18a4
16 changed files with 921 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
public/scripts/tokenizers.js
View File

@ -669,9 +669,11 @@ function getTextTokensFromKoboldAPI(str) {
* Calls the underlying tokenizer model to decode token ids to text.
* @param {string} endpoint API endpoint.
* @param {number[]} ids Array of token ids
* @returns {({ text: string, chunks?: string[] })} Decoded token text as a single string and individual chunks (if available).
*/
function decodeTextTokensFromServer(endpoint, ids) {
let text = '';
let chunks = [];
jQuery.ajax({
async: false,
type: 'POST',
@ -681,9 +683,10 @@ function decodeTextTokensFromServer(endpoint, ids) {
contentType: 'application/json',
success: function (data) {
text = data.text;
chunks = data.chunks;
},
});
return text;
return { text, chunks };
}
/**
@ -725,6 +728,7 @@ export function getTextTokens(tokenizerType, str) {
* Decodes token ids to text using the server API.
* @param {number} tokenizerType Tokenizer type.
* @param {number[]} ids Array of token ids
* @returns {({ text: string, chunks?: string[] })} Decoded token text as a single string and individual chunks (if available).
*/
export function decodeTextTokens(tokenizerType, ids) {
// Currently, neither remote API can decode, but this may change in the future. Put this guard here to be safe
@ -734,12 +738,12 @@ export function decodeTextTokens(tokenizerType, ids) {
const tokenizerEndpoints = TOKENIZER_URLS[tokenizerType];
if (!tokenizerEndpoints) {
console.warn('Unknown tokenizer type', tokenizerType);
return [];
return { text: '', chunks: [] };
}
let endpointUrl = tokenizerEndpoints.decode;
if (!endpointUrl) {
console.warn('This tokenizer type does not support decoding', tokenizerType);
return [];
return { text: '', chunks: [] };
}
if (tokenizerType === tokenizers.OPENAI) {
endpointUrl += `?model=${getTokenizerModel()}`;