rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

tokenizer changes and fixes. + a toggle

Browse Source
This commit is contained in:
based
2023-12-14 16:31:08 +10:00
parent e26159c00d
commit 3e82a7d439
5 changed files with 24 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/endpoints/tokenizers.js
View File

@ -4,7 +4,7 @@ const express = require('express');
const { SentencePieceProcessor } = require('@agnai/sentencepiece-js');
const tiktoken = require('@dqbd/tiktoken');
const { Tokenizer } = require('@agnai/web-tokenizers');
const { convertClaudePrompt } = require('../chat-completion');
const { convertClaudePrompt, convertGooglePrompt } = require('../chat-completion');
const { readSecret, SECRET_KEYS } = require('./secrets');
const { TEXTGEN_TYPES } = require('../constants');
const { jsonParser } = require('../express-common');
@ -395,13 +395,12 @@ router.post('/google/count', jsonParser, async function (req, res) {
accept: 'application/json',
'content-type': 'application/json',
},
body: JSON.stringify({ prompt: { text: req.body[0].content } }),
body: JSON.stringify({ contents: convertGooglePrompt(req.body) }),
};
try {
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${req.query.model}:countTextTokens?key=${readSecret(SECRET_KEYS.MAKERSUITE)}`, options);
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${req.query.model}:countTokens?key=${readSecret(SECRET_KEYS.MAKERSUITE)}`, options);
const data = await response.json();
console.log(data)
return res.send({ 'token_count': data?.tokenCount || 0 });
return res.send({ 'token_count': data?.totalTokens || 0 });
} catch (err) {
console.error(err);
return res.send({ 'token_count': 0 });