From 0e5100180b38c47049cb51ec690e9de87f2ece0d Mon Sep 17 00:00:00 2001
From: Succubyss <87207237+Succubyss@users.noreply.github.com>
Date: Fri, 13 Dec 2024 16:29:34 -0600
Subject: [PATCH] expose tokenizers & getTextTokens in getContext

---
 public/scripts/st-context.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/public/scripts/st-context.js b/public/scripts/st-context.js
index 4f0abdd8d..9cbf18210 100644
--- a/public/scripts/st-context.js
+++ b/public/scripts/st-context.js
@@ -63,7 +63,7 @@ import { ARGUMENT_TYPE, SlashCommandArgument, SlashCommandNamedArgument } from '
 import { SlashCommandParser } from './slash-commands/SlashCommandParser.js';
 import { tag_map, tags } from './tags.js';
 import { textgenerationwebui_settings } from './textgen-settings.js';
-import { getTokenCount, getTokenCountAsync, getTokenizerModel } from './tokenizers.js';
+import { tokenizers, getTextTokens, getTokenCount, getTokenCountAsync, getTokenizerModel } from './tokenizers.js';
 import { ToolManager } from './tool-calling.js';
 import { timestampToMoment } from './utils.js';
 
@@ -95,6 +95,8 @@ export function getContext() {
         sendStreamingRequest,
         sendGenerationRequest,
         stopGeneration,
+        tokenizers,
+        getTextTokens,
         /** @deprecated Use getTokenCountAsync instead */
         getTokenCount,
         getTokenCountAsync,