Compare commits
9 Commits
23b15d1f1d
...
26ddcfc458
Author | SHA1 | Date |
---|---|---|
箱庭XTer | 26ddcfc458 | |
Cohee | be7eb8b2b5 | |
Cohee | 3b6372431a | |
sirius422 | 389ee7917f | |
Cohee | 212e61d2a1 | |
Cohee | 1b60e4a013 | |
Aisu Wata | 93cd93ada3 | |
XTer | 6c44f5b3fd | |
XTer | 42083b371b |
|
@ -0,0 +1,276 @@
|
|||
|
||||
import { saveTtsProviderSettings } from './index.js';
|
||||
|
||||
export { GSVITtsProvider };
|
||||
|
||||
class GSVITtsProvider {
|
||||
//########//
|
||||
// Config //
|
||||
//########//
|
||||
|
||||
settings;
|
||||
ready = false;
|
||||
separator = '. ';
|
||||
|
||||
characterList = {};
|
||||
voices = [];
|
||||
/**
|
||||
* Perform any text processing before passing to TTS engine.
|
||||
* @param {string} text Input text
|
||||
* @returns {string} Processed text
|
||||
*/
|
||||
processText(text) {
|
||||
text = text.replace('<br>', '\n'); // Replace <br> with newline
|
||||
return text;
|
||||
}
|
||||
|
||||
languageLabels = {
|
||||
'多语种混合': '多语种混合',
|
||||
'中文': '中文',
|
||||
'英文': '英文',
|
||||
'日文': '日文',
|
||||
'中英混合': '中英混合',
|
||||
'日英混合': '日英混合',
|
||||
};
|
||||
defaultSettings = {
|
||||
provider_endpoint: 'http://127.0.0.1:5000',
|
||||
|
||||
language: '多语种混合',
|
||||
|
||||
cha_name: '',
|
||||
character_emotion: 'default',
|
||||
|
||||
speed: 1,
|
||||
|
||||
top_k: 6,
|
||||
top_p: 0.85,
|
||||
temperature: 0.75,
|
||||
batch_size: 10,
|
||||
|
||||
stream: false,
|
||||
stream_chunk_size: 100,
|
||||
};
|
||||
|
||||
|
||||
// 新增获取角色和情绪的方法
|
||||
async fetchCharacterList() {
|
||||
const response = await fetch(this.settings.provider_endpoint + '/character_list');
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${await response.text()}`);
|
||||
}
|
||||
const characterList = await response.json();
|
||||
this.characterList = characterList;
|
||||
this.voices = Object.keys(characterList);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
get settingsHtml() {
|
||||
let html = `
|
||||
<label for="gsvi_api_language">Text Language</label>
|
||||
<select id="gsvi_api_language">`;
|
||||
|
||||
for (let language in this.languageLabels) {
|
||||
if (this.languageLabels[language] == this.settings?.language) {
|
||||
html += `<option value="${this.languageLabels[language]}" selected="selected">${language}</option>`;
|
||||
continue;
|
||||
}
|
||||
|
||||
html += `<option value="${this.languageLabels[language]}">${language}</option>`;
|
||||
}
|
||||
|
||||
html += `
|
||||
</select>
|
||||
<label>GSVI Settings:</label><br/>
|
||||
<label for="gsvi_tts_endpoint">Provider Endpoint:</label>
|
||||
<input id="gsvi_tts_endpoint" type="text" class="text_pole" maxlength="250" value="${this.defaultSettings.provider_endpoint}"/>
|
||||
|
||||
|
||||
|
||||
<div>Goto <a target="_blank" href="https://www.yuque.com/xter/zibxlp/knu8p82lb5ipufqy">API Document</a>.</div>
|
||||
|
||||
<label for="gsvi_speed">Speed: <span id="gsvi_tts_speed_output">${this.defaultSettings.speed}</span></label>
|
||||
<input id="gsvi_speed" type="range" value="${this.defaultSettings.speed}" min="0.5" max="2" step="0.01" />
|
||||
|
||||
<label for="gsvi_top_k">Top K: <span id="gsvi_top_k_output">${this.defaultSettings.top_k}</span></label>
|
||||
<input id="gsvi_top_k" type="range" value="${this.defaultSettings.top_k}" min="0" max="100" step="1" />
|
||||
|
||||
<label for="gsvi_top_p">Top P: <span id="gsvi_top_p_output">${this.defaultSettings.top_p}</span></label>
|
||||
<input id="gsvi_top_p" type="range" value="${this.defaultSettings.top_p}" min="0" max="1" step="0.01" />
|
||||
|
||||
<label for="gsvi_temperature">Temperature: <span id="gsvi_tts_temperature_output">${this.defaultSettings.temperature}</span></label>
|
||||
<input id="gsvi_temperature" type="range" value="${this.defaultSettings.temperature}" min="0.01" max="1" step="0.01" />
|
||||
|
||||
<label for="gsvi_batch_size">Batch Size: <span id="gsvi_batch_size_output">${this.defaultSettings.batch_size}</span></label>
|
||||
<input id="gsvi_batch_size" type="range" value="${this.defaultSettings.batch_size}" min="1" max="35" step="1" />
|
||||
|
||||
<label for="gsvi_tts_streaming" class="checkbox_label">
|
||||
<input id="gsvi_tts_streaming" type="checkbox" ${this.defaultSettings.stream ? 'checked' : ''}/>
|
||||
<span>Streaming</span>
|
||||
</label>
|
||||
|
||||
<label for="gsvi_stream_chunk_size">Stream Chunk Size: <span id="gsvi_stream_chunk_size_output">${this.defaultSettings.stream_chunk_size}</span></label>
|
||||
<input id="gsvi_stream_chunk_size" type="range" value="${this.defaultSettings.stream_chunk_size}" min="100" max="400" step="1" />
|
||||
<title>About GSVI (GPT-Sovits Inference)</title>
|
||||
<p>
|
||||
GSVI (GPT-Sovits Inference) is an inference enhancement project based on
|
||||
<a href="https://github.com/RVC-Boss/GPT-SoVITS" target="_blank">GPT-Sovits</a>, allowing you to run an API interface locally, offering emotion-rich speech-to-text and convenient model management features.
|
||||
</p>
|
||||
<p>
|
||||
For more information, visit the
|
||||
<a href="https://github.com/X-T-E-R/GPT-SoVITS-Inference" target="_blank">GSVI project page</a>.
|
||||
</p>
|
||||
`;
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
onSettingsChange() {
|
||||
// Update provider settings based on input fields
|
||||
this.settings.provider_endpoint = $('#gsvi_tts_endpoint').val();
|
||||
this.settings.language = $('#gsvi_api_language').val();
|
||||
|
||||
|
||||
// Update the rest of TTS settings based on input fields
|
||||
this.settings.speed = parseFloat($('#gsvi_speed').val());
|
||||
this.settings.temperature = parseFloat($('#gsvi_temperature').val());
|
||||
this.settings.top_k = parseInt($('#gsvi_top_k').val(), 10);
|
||||
this.settings.top_p = parseFloat($('#gsvi_top_p').val());
|
||||
this.settings.batch_size = parseInt($('#gsvi_batch_size').val(), 10);
|
||||
this.settings.stream = $('#gsvi_tts_streaming').is(':checked');
|
||||
this.settings.stream_chunk_size = parseInt($('#gsvi_stream_chunk_size').val(), 10);
|
||||
|
||||
// Update UI to reflect changes
|
||||
|
||||
$('#gsvi_tts_speed_output').text(this.settings.speed);
|
||||
$('#gsvi_tts_temperature_output').text(this.settings.temperature);
|
||||
$('#gsvi_top_k_output').text(this.settings.top_k);
|
||||
$('#gsvi_top_p_output').text(this.settings.top_p);
|
||||
$('#gsvi_stream_chunk_size_output').text(this.settings.stream_chunk_size);
|
||||
$('#gsvi_batch_size_output').text(this.settings.batch_size);
|
||||
|
||||
|
||||
|
||||
|
||||
// Persist settings changes
|
||||
saveTtsProviderSettings();
|
||||
|
||||
}
|
||||
|
||||
async loadSettings(settings) {
|
||||
// Populate Provider UI given input settings
|
||||
if (Object.keys(settings).length === 0) {
|
||||
console.info('Using default TTS Provider settings');
|
||||
}
|
||||
|
||||
// Only accept keys defined in defaultSettings
|
||||
this.settings = { ...this.defaultSettings, ...settings };
|
||||
|
||||
// Fetch character and emotion list
|
||||
// Set initial values from the settings
|
||||
$('#gsvi_tts_endpoint').val(this.settings.provider_endpoint);
|
||||
$('#gsvi_api_language').val(this.settings.language);
|
||||
|
||||
$('#gsvi_speed').val(this.settings.speed);
|
||||
$('#gsvi_temperature').val(this.settings.temperature);
|
||||
$('#gsvi_top_k').val(this.settings.top_k);
|
||||
$('#gsvi_top_p').val(this.settings.top_p);
|
||||
$('#gsvi_batch_size').val(this.settings.batch_size);
|
||||
$('#gsvi_tts_streaming').prop('checked', this.settings.stream);
|
||||
$('#gsvi_stream_chunk_size').val(this.settings.stream_chunk_size);
|
||||
|
||||
// Update UI to reflect initial settings
|
||||
$('#gsvi_tts_speed_output').text(this.settings.speed);
|
||||
$('#gsvi_tts_temperature_output').text(this.settings.temperature);
|
||||
$('#gsvi_top_k_output').text(this.settings.top_k);
|
||||
$('#gsvi_top_p_output').text(this.settings.top_p);
|
||||
$('#gsvi_stream_chunk_size_output').text(this.settings.stream_chunk_size);
|
||||
|
||||
// Register event listeners to update settings on user interaction
|
||||
// (Similar to before, ensure event listeners for character and emotion selection are included)
|
||||
// Register input/change event listeners to update settings on user interaction
|
||||
$('#gsvi_tts_endpoint').on('input', () => { this.onSettingsChange(); });
|
||||
$('#gsvi_api_language').on('change', () => { this.onSettingsChange(); });
|
||||
|
||||
$('#gsvi_speed').on('input', () => { this.onSettingsChange(); });
|
||||
$('#gsvi_temperature').on('input', () => { this.onSettingsChange(); });
|
||||
$('#gsvi_top_k').on('input', () => { this.onSettingsChange(); });
|
||||
$('#gsvi_top_p').on('input', () => { this.onSettingsChange(); });
|
||||
$('#gsvi_batch_size').on('input', () => { this.onSettingsChange(); });
|
||||
$('#gsvi_tts_streaming').on('change', () => { this.onSettingsChange(); });
|
||||
$('#gsvi_stream_chunk_size').on('input', () => { this.onSettingsChange(); });
|
||||
|
||||
await this.checkReady();
|
||||
console.debug('GSVI: Settings loaded');
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Perform a simple readiness check by trying to fetch voiceIds
|
||||
async checkReady() {
|
||||
await Promise.allSettled([this.fetchCharacterList()]);
|
||||
}
|
||||
|
||||
async onRefreshClick() {
|
||||
return;
|
||||
}
|
||||
|
||||
//#################//
|
||||
// TTS Interfaces //
|
||||
//#################//
|
||||
|
||||
async getVoice(voiceName) {
|
||||
if (this.voices.length == 0) {
|
||||
this.fetchCharacterList();
|
||||
}
|
||||
if (!this.voices.includes(voiceName)) {
|
||||
throw `TTS Voice name ${voiceName} not found`;
|
||||
}
|
||||
return { name: voiceName, voice_id: voiceName, preview_url: false, lang: 'zh-CN' };
|
||||
}
|
||||
|
||||
async generateTts(text, voiceId) {
|
||||
const response = await this.fetchTtsGeneration(text, voiceId);
|
||||
return response;
|
||||
}
|
||||
|
||||
//###########//
|
||||
// API CALLS //
|
||||
//###########//
|
||||
async fetchTtsVoiceObjects() {
|
||||
if (this.voices.length == 0) {
|
||||
await this.fetchCharacterList();
|
||||
}
|
||||
console.log(this.voices);
|
||||
const voices = this.voices.map(x => ({ name: x, voice_id: x, preview_url: false, lang: 'zh-CN' }));
|
||||
return voices;
|
||||
}
|
||||
|
||||
|
||||
async fetchTtsGeneration(inputText, voiceId) {
|
||||
console.info(`Generating new TTS for voice_id ${voiceId}`);
|
||||
|
||||
|
||||
const params = new URLSearchParams();
|
||||
params.append('text', inputText);
|
||||
params.append('cha_name', voiceId);
|
||||
params.append('text_language', this.settings.language);
|
||||
params.append('batch_size', this.settings.batch_size.toString());
|
||||
params.append('speed', this.settings.speed.toString());
|
||||
params.append('top_k', this.settings.top_k.toString());
|
||||
params.append('top_p', this.settings.top_p.toString());
|
||||
params.append('temperature', this.settings.temperature.toString());
|
||||
params.append('stream', this.settings.stream.toString());
|
||||
|
||||
|
||||
return `${this.settings.provider_endpoint}/tts?${params.toString()}`;
|
||||
|
||||
}
|
||||
|
||||
// Interface not used by GSVI TTS
|
||||
async fetchTtsFromHistory(history_item_id) {
|
||||
return Promise.resolve(history_item_id);
|
||||
}
|
||||
|
||||
}
|
|
@ -11,6 +11,7 @@ import { power_user } from '../../power-user.js';
|
|||
import { registerSlashCommand } from '../../slash-commands.js';
|
||||
import { OpenAITtsProvider } from './openai.js';
|
||||
import { XTTSTtsProvider } from './xtts.js';
|
||||
import { GSVITtsProvider } from './gsvi.js';
|
||||
import { AllTalkTtsProvider } from './alltalk.js';
|
||||
import { SpeechT5TtsProvider } from './speecht5.js';
|
||||
export { talkingAnimation };
|
||||
|
@ -72,6 +73,7 @@ const ttsProviders = {
|
|||
ElevenLabs: ElevenLabsTtsProvider,
|
||||
Silero: SileroTtsProvider,
|
||||
XTTSv2: XTTSTtsProvider,
|
||||
GSVI: GSVITtsProvider,
|
||||
System: SystemTtsProvider,
|
||||
Coqui: CoquiTtsProvider,
|
||||
Edge: EdgeTtsProvider,
|
||||
|
|
|
@ -3250,7 +3250,8 @@ async function onExportPresetClick() {
|
|||
delete preset.proxy_password;
|
||||
|
||||
const presetJsonString = JSON.stringify(preset, null, 4);
|
||||
download(presetJsonString, oai_settings.preset_settings_openai, 'application/json');
|
||||
const presetFileName = `${oai_settings.preset_settings_openai}.json`;
|
||||
download(presetJsonString, presetFileName, 'application/json');
|
||||
}
|
||||
|
||||
async function onLogitBiasPresetImportFileChange(e) {
|
||||
|
@ -3298,7 +3299,8 @@ function onLogitBiasPresetExportClick() {
|
|||
}
|
||||
|
||||
const presetJsonString = JSON.stringify(oai_settings.bias_presets[oai_settings.bias_preset_selected], null, 4);
|
||||
download(presetJsonString, oai_settings.bias_preset_selected, 'application/json');
|
||||
const presetFileName = `${oai_settings.bias_preset_selected}.json`;
|
||||
download(presetJsonString, presetFileName, 'application/json');
|
||||
}
|
||||
|
||||
async function onDeletePresetClick() {
|
||||
|
|
|
@ -1000,6 +1000,7 @@ body .panelControlBar {
|
|||
padding-left: 10px;
|
||||
width: 100%;
|
||||
overflow-x: hidden;
|
||||
overflow-y: clip;
|
||||
}
|
||||
|
||||
.mes_text {
|
||||
|
|
61
server.js
61
server.js
|
@ -45,7 +45,6 @@ const {
|
|||
forwardFetchResponse,
|
||||
} = require('./src/util');
|
||||
const { ensureThumbnailCache } = require('./src/endpoints/thumbnails');
|
||||
const { loadTokenizers } = require('./src/endpoints/tokenizers');
|
||||
|
||||
// Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0.
|
||||
// https://github.com/nodejs/node/issues/47822#issuecomment-1564708870
|
||||
|
@ -543,22 +542,12 @@ const setupTasks = async function () {
|
|||
}
|
||||
console.log();
|
||||
|
||||
// TODO: do endpoint init functions depend on certain directories existing or not existing? They should be callable
|
||||
// in any order for encapsulation reasons, but right now it's unknown if that would break anything.
|
||||
await userModule.initUserStorage(dataRoot);
|
||||
|
||||
if (listen && !basicAuthMode && enableAccounts) {
|
||||
await userModule.checkAccountsProtection();
|
||||
}
|
||||
|
||||
await settingsEndpoint.init();
|
||||
const directories = await userModule.ensurePublicDirectoriesExist();
|
||||
await userModule.migrateUserData();
|
||||
const directories = await userModule.getUserDirectoriesList();
|
||||
await contentManager.checkForNewContent(directories);
|
||||
await ensureThumbnailCache();
|
||||
cleanUploads();
|
||||
|
||||
await loadTokenizers();
|
||||
await settingsEndpoint.init();
|
||||
await statsEndpoint.init();
|
||||
|
||||
const cleanupPlugins = await loadPlugins();
|
||||
|
@ -581,7 +570,6 @@ const setupTasks = async function () {
|
|||
exitProcess();
|
||||
});
|
||||
|
||||
|
||||
console.log('Launching...');
|
||||
|
||||
if (autorun) open(autorunUrl.toString());
|
||||
|
@ -601,6 +589,9 @@ const setupTasks = async function () {
|
|||
}
|
||||
}
|
||||
|
||||
if (listen && !basicAuthMode && enableAccounts) {
|
||||
await userModule.checkAccountsProtection();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
|
@ -642,21 +633,27 @@ function setWindowTitle(title) {
|
|||
}
|
||||
}
|
||||
|
||||
if (cliArguments.ssl) {
|
||||
https.createServer(
|
||||
{
|
||||
cert: fs.readFileSync(cliArguments.certPath),
|
||||
key: fs.readFileSync(cliArguments.keyPath),
|
||||
}, app)
|
||||
.listen(
|
||||
Number(tavernUrl.port) || 443,
|
||||
tavernUrl.hostname,
|
||||
setupTasks,
|
||||
);
|
||||
} else {
|
||||
http.createServer(app).listen(
|
||||
Number(tavernUrl.port) || 80,
|
||||
tavernUrl.hostname,
|
||||
setupTasks,
|
||||
);
|
||||
}
|
||||
// User storage module needs to be initialized before starting the server
|
||||
userModule.initUserStorage(dataRoot)
|
||||
.then(userModule.ensurePublicDirectoriesExist)
|
||||
.then(userModule.migrateUserData)
|
||||
.finally(() => {
|
||||
if (cliArguments.ssl) {
|
||||
https.createServer(
|
||||
{
|
||||
cert: fs.readFileSync(cliArguments.certPath),
|
||||
key: fs.readFileSync(cliArguments.keyPath),
|
||||
}, app)
|
||||
.listen(
|
||||
Number(tavernUrl.port) || 443,
|
||||
tavernUrl.hostname,
|
||||
setupTasks,
|
||||
);
|
||||
} else {
|
||||
http.createServer(app).listen(
|
||||
Number(tavernUrl.port) || 80,
|
||||
tavernUrl.hostname,
|
||||
setupTasks,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
|
|
@ -10,6 +10,10 @@ const { TEXTGEN_TYPES } = require('../constants');
|
|||
const { jsonParser } = require('../express-common');
|
||||
const { setAdditionalHeaders } = require('../additional-headers');
|
||||
|
||||
/**
|
||||
* @typedef { (req: import('express').Request, res: import('express').Response) => Promise<any> } TokenizationHandler
|
||||
*/
|
||||
|
||||
/**
|
||||
* @type {{[key: string]: import("@dqbd/tiktoken").Tiktoken}} Tokenizers cache
|
||||
*/
|
||||
|
@ -48,16 +52,30 @@ const TEXT_COMPLETION_MODELS = [
|
|||
|
||||
const CHARS_PER_TOKEN = 3.35;
|
||||
|
||||
/**
|
||||
* Sentencepiece tokenizer for tokenizing text.
|
||||
*/
|
||||
class SentencePieceTokenizer {
|
||||
/**
|
||||
* @type {import('@agnai/sentencepiece-js').SentencePieceProcessor} Sentencepiece tokenizer instance
|
||||
*/
|
||||
#instance;
|
||||
/**
|
||||
* @type {string} Path to the tokenizer model
|
||||
*/
|
||||
#model;
|
||||
|
||||
/**
|
||||
* Creates a new Sentencepiece tokenizer.
|
||||
* @param {string} model Path to the tokenizer model
|
||||
*/
|
||||
constructor(model) {
|
||||
this.#model = model;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the Sentencepiece tokenizer instance.
|
||||
* @returns {Promise<import('@agnai/sentencepiece-js').SentencePieceProcessor|null>} Sentencepiece tokenizer instance
|
||||
*/
|
||||
async get() {
|
||||
if (this.#instance) {
|
||||
|
@ -76,18 +94,61 @@ class SentencePieceTokenizer {
|
|||
}
|
||||
}
|
||||
|
||||
const spp_llama = new SentencePieceTokenizer('src/sentencepiece/llama.model');
|
||||
const spp_nerd = new SentencePieceTokenizer('src/sentencepiece/nerdstash.model');
|
||||
const spp_nerd_v2 = new SentencePieceTokenizer('src/sentencepiece/nerdstash_v2.model');
|
||||
const spp_mistral = new SentencePieceTokenizer('src/sentencepiece/mistral.model');
|
||||
const spp_yi = new SentencePieceTokenizer('src/sentencepiece/yi.model');
|
||||
let claude_tokenizer;
|
||||
/**
|
||||
* Web tokenizer for tokenizing text.
|
||||
*/
|
||||
class WebTokenizer {
|
||||
/**
|
||||
* @type {Tokenizer} Web tokenizer instance
|
||||
*/
|
||||
#instance;
|
||||
/**
|
||||
* @type {string} Path to the tokenizer model
|
||||
*/
|
||||
#model;
|
||||
|
||||
/**
|
||||
* Creates a new Web tokenizer.
|
||||
* @param {string} model Path to the tokenizer model
|
||||
*/
|
||||
constructor(model) {
|
||||
this.#model = model;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the Web tokenizer instance.
|
||||
* @returns {Promise<Tokenizer|null>} Web tokenizer instance
|
||||
*/
|
||||
async get() {
|
||||
if (this.#instance) {
|
||||
return this.#instance;
|
||||
}
|
||||
|
||||
try {
|
||||
const arrayBuffer = fs.readFileSync(this.#model).buffer;
|
||||
this.#instance = await Tokenizer.fromJSON(arrayBuffer);
|
||||
console.log('Instantiated the tokenizer for', path.parse(this.#model).name);
|
||||
return this.#instance;
|
||||
} catch (error) {
|
||||
console.error('Web tokenizer failed to load: ' + this.#model, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const spp_llama = new SentencePieceTokenizer('src/tokenizers/llama.model');
|
||||
const spp_nerd = new SentencePieceTokenizer('src/tokenizers/nerdstash.model');
|
||||
const spp_nerd_v2 = new SentencePieceTokenizer('src/tokenizers/nerdstash_v2.model');
|
||||
const spp_mistral = new SentencePieceTokenizer('src/tokenizers/mistral.model');
|
||||
const spp_yi = new SentencePieceTokenizer('src/tokenizers/yi.model');
|
||||
const claude_tokenizer = new WebTokenizer('src/tokenizers/claude.json');
|
||||
|
||||
const sentencepieceTokenizers = [
|
||||
'llama',
|
||||
'nerdstash',
|
||||
'nerdstash_v2',
|
||||
'mistral',
|
||||
'yi',
|
||||
];
|
||||
|
||||
/**
|
||||
|
@ -112,6 +173,10 @@ function getSentencepiceTokenizer(model) {
|
|||
return spp_nerd_v2;
|
||||
}
|
||||
|
||||
if (model.includes('yi')) {
|
||||
return spp_yi;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
@ -168,13 +233,23 @@ async function getTiktokenChunks(tokenizer, ids) {
|
|||
return chunks;
|
||||
}
|
||||
|
||||
async function getWebTokenizersChunks(tokenizer, ids) {
|
||||
/**
|
||||
* Gets the token chunks for the given token IDs using the Web tokenizer.
|
||||
* @param {Tokenizer} tokenizer Web tokenizer instance
|
||||
* @param {number[]} ids Token IDs
|
||||
* @returns {string[]} Token chunks
|
||||
*/
|
||||
function getWebTokenizersChunks(tokenizer, ids) {
|
||||
const chunks = [];
|
||||
|
||||
for (let i = 0; i < ids.length; i++) {
|
||||
const id = ids[i];
|
||||
const chunkText = await tokenizer.decode(new Uint32Array([id]));
|
||||
for (let i = 0, lastProcessed = 0; i < ids.length; i++) {
|
||||
const chunkIds = ids.slice(lastProcessed, i + 1);
|
||||
const chunkText = tokenizer.decode(new Int32Array(chunkIds));
|
||||
if (chunkText === '<27>') {
|
||||
continue;
|
||||
}
|
||||
chunks.push(chunkText);
|
||||
lastProcessed = i + 1;
|
||||
}
|
||||
|
||||
return chunks;
|
||||
|
@ -237,17 +312,12 @@ function getTiktokenTokenizer(model) {
|
|||
return tokenizer;
|
||||
}
|
||||
|
||||
async function loadClaudeTokenizer(modelPath) {
|
||||
try {
|
||||
const arrayBuffer = fs.readFileSync(modelPath).buffer;
|
||||
const instance = await Tokenizer.fromJSON(arrayBuffer);
|
||||
return instance;
|
||||
} catch (error) {
|
||||
console.error('Claude tokenizer failed to load: ' + modelPath, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts the tokens for the given messages using the Claude tokenizer.
|
||||
* @param {Tokenizer} tokenizer Web tokenizer
|
||||
* @param {object[]} messages Array of messages
|
||||
* @returns {number} Number of tokens
|
||||
*/
|
||||
function countClaudeTokens(tokenizer, messages) {
|
||||
// Should be fine if we use the old conversion method instead of the messages API one i think?
|
||||
const convertedPrompt = convertClaudePrompt(messages, false, '', false, false, '', false);
|
||||
|
@ -264,9 +334,14 @@ function countClaudeTokens(tokenizer, messages) {
|
|||
/**
|
||||
* Creates an API handler for encoding Sentencepiece tokens.
|
||||
* @param {SentencePieceTokenizer} tokenizer Sentencepiece tokenizer
|
||||
* @returns {any} Handler function
|
||||
* @returns {TokenizationHandler} Handler function
|
||||
*/
|
||||
function createSentencepieceEncodingHandler(tokenizer) {
|
||||
/**
|
||||
* Request handler for encoding Sentencepiece tokens.
|
||||
* @param {import('express').Request} request
|
||||
* @param {import('express').Response} response
|
||||
*/
|
||||
return async function (request, response) {
|
||||
try {
|
||||
if (!request.body) {
|
||||
|
@ -276,7 +351,7 @@ function createSentencepieceEncodingHandler(tokenizer) {
|
|||
const text = request.body.text || '';
|
||||
const instance = await tokenizer?.get();
|
||||
const { ids, count } = await countSentencepieceTokens(tokenizer, text);
|
||||
const chunks = await instance?.encodePieces(text);
|
||||
const chunks = instance?.encodePieces(text);
|
||||
return response.send({ ids, count, chunks });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
@ -288,9 +363,14 @@ function createSentencepieceEncodingHandler(tokenizer) {
|
|||
/**
|
||||
* Creates an API handler for decoding Sentencepiece tokens.
|
||||
* @param {SentencePieceTokenizer} tokenizer Sentencepiece tokenizer
|
||||
* @returns {any} Handler function
|
||||
* @returns {TokenizationHandler} Handler function
|
||||
*/
|
||||
function createSentencepieceDecodingHandler(tokenizer) {
|
||||
/**
|
||||
* Request handler for decoding Sentencepiece tokens.
|
||||
* @param {import('express').Request} request
|
||||
* @param {import('express').Response} response
|
||||
*/
|
||||
return async function (request, response) {
|
||||
try {
|
||||
if (!request.body) {
|
||||
|
@ -299,6 +379,7 @@ function createSentencepieceDecodingHandler(tokenizer) {
|
|||
|
||||
const ids = request.body.ids || [];
|
||||
const instance = await tokenizer?.get();
|
||||
if (!instance) throw new Error('Failed to load the Sentencepiece tokenizer');
|
||||
const ops = ids.map(id => instance.decodeIds([id]));
|
||||
const chunks = await Promise.all(ops);
|
||||
const text = chunks.join('');
|
||||
|
@ -313,9 +394,14 @@ function createSentencepieceDecodingHandler(tokenizer) {
|
|||
/**
|
||||
* Creates an API handler for encoding Tiktoken tokens.
|
||||
* @param {string} modelId Tiktoken model ID
|
||||
* @returns {any} Handler function
|
||||
* @returns {TokenizationHandler} Handler function
|
||||
*/
|
||||
function createTiktokenEncodingHandler(modelId) {
|
||||
/**
|
||||
* Request handler for encoding Tiktoken tokens.
|
||||
* @param {import('express').Request} request
|
||||
* @param {import('express').Response} response
|
||||
*/
|
||||
return async function (request, response) {
|
||||
try {
|
||||
if (!request.body) {
|
||||
|
@ -337,9 +423,14 @@ function createTiktokenEncodingHandler(modelId) {
|
|||
/**
|
||||
* Creates an API handler for decoding Tiktoken tokens.
|
||||
* @param {string} modelId Tiktoken model ID
|
||||
* @returns {any} Handler function
|
||||
* @returns {TokenizationHandler} Handler function
|
||||
*/
|
||||
function createTiktokenDecodingHandler(modelId) {
|
||||
/**
|
||||
* Request handler for decoding Tiktoken tokens.
|
||||
* @param {import('express').Request} request
|
||||
* @param {import('express').Response} response
|
||||
*/
|
||||
return async function (request, response) {
|
||||
try {
|
||||
if (!request.body) {
|
||||
|
@ -358,14 +449,6 @@ function createTiktokenDecodingHandler(modelId) {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the model tokenizers.
|
||||
* @returns {Promise<void>} Promise that resolves when the tokenizers are loaded
|
||||
*/
|
||||
async function loadTokenizers() {
|
||||
claude_tokenizer = await loadClaudeTokenizer('src/claude.json');
|
||||
}
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.post('/ai21/count', jsonParser, async function (req, res) {
|
||||
|
@ -446,8 +529,10 @@ router.post('/openai/encode', jsonParser, async function (req, res) {
|
|||
|
||||
if (queryModel.includes('claude')) {
|
||||
const text = req.body.text || '';
|
||||
const tokens = Object.values(claude_tokenizer.encode(text));
|
||||
const chunks = await getWebTokenizersChunks(claude_tokenizer, tokens);
|
||||
const instance = await claude_tokenizer.get();
|
||||
if (!instance) throw new Error('Failed to load the Claude tokenizer');
|
||||
const tokens = Object.values(instance.encode(text));
|
||||
const chunks = getWebTokenizersChunks(instance, tokens);
|
||||
return res.send({ ids: tokens, count: tokens.length, chunks });
|
||||
}
|
||||
|
||||
|
@ -481,7 +566,9 @@ router.post('/openai/decode', jsonParser, async function (req, res) {
|
|||
|
||||
if (queryModel.includes('claude')) {
|
||||
const ids = req.body.ids || [];
|
||||
const chunkText = await claude_tokenizer.decode(new Uint32Array(ids));
|
||||
const instance = await claude_tokenizer.get();
|
||||
if (!instance) throw new Error('Failed to load the Claude tokenizer');
|
||||
const chunkText = instance.decode(new Int32Array(ids));
|
||||
return res.send({ text: chunkText });
|
||||
}
|
||||
|
||||
|
@ -503,7 +590,9 @@ router.post('/openai/count', jsonParser, async function (req, res) {
|
|||
const model = getTokenizerModel(queryModel);
|
||||
|
||||
if (model === 'claude') {
|
||||
num_tokens = countClaudeTokens(claude_tokenizer, req.body);
|
||||
const instance = await claude_tokenizer.get();
|
||||
if (!instance) throw new Error('Failed to load the Claude tokenizer');
|
||||
num_tokens = countClaudeTokens(instance, req.body);
|
||||
return res.send({ 'token_count': num_tokens });
|
||||
}
|
||||
|
||||
|
@ -665,7 +754,6 @@ module.exports = {
|
|||
getTokenizerModel,
|
||||
getTiktokenTokenizer,
|
||||
countClaudeTokens,
|
||||
loadTokenizers,
|
||||
getSentencepiceTokenizer,
|
||||
sentencepieceTokenizers,
|
||||
router,
|
||||
|
|
11
src/users.js
11
src/users.js
|
@ -112,6 +112,16 @@ async function ensurePublicDirectoriesExist() {
|
|||
return directoriesList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a list of all user directories.
|
||||
* @returns {Promise<import('./users').UserDirectoryList[]>} - The list of user directories
|
||||
*/
|
||||
async function getUserDirectoriesList() {
|
||||
const userHandles = await getAllUserHandles();
|
||||
const directoriesList = userHandles.map(handle => getUserDirectories(handle));
|
||||
return directoriesList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform migration from the old user data format to the new one.
|
||||
*/
|
||||
|
@ -707,6 +717,7 @@ module.exports = {
|
|||
toAvatarKey,
|
||||
initUserStorage,
|
||||
ensurePublicDirectoriesExist,
|
||||
getUserDirectoriesList,
|
||||
getAllUserHandles,
|
||||
getUserDirectories,
|
||||
setUserDataMiddleware,
|
||||
|
|
Loading…
Reference in New Issue