add in process files for OAI tokenization merge

This commit is contained in:
RossAscends
2023-05-15 01:45:36 +09:00
parent d765e9bad9
commit 133caa58d2
4 changed files with 54 additions and 10 deletions

View File

@ -648,7 +648,7 @@ class TokenHandler {
}
count(messages, full, type) {
console.log(messages);
//console.log(messages);
const token_count = this.countTokenFn(messages, full);
this.counts[type] += token_count;