rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Fix fairseq newline handling issues

Browse Source
This commit is contained in:
Gnome Ann
2022-02-12 13:23:59 -05:00
parent c1af8f72c3
commit f682c1229a
2 changed files with 46 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
utils.py
View File

@ -1,6 +1,8 @@
from threading import Timer
import re
vars = None
#==================================================================#
# Decorator to prevent a function's actions from being run until
# at least x seconds have passed without the function being called
@ -111,8 +113,15 @@ def cleanfilename(filename):
filename = "".join(c for c in filename if c not in filteredcharacters).rstrip()
return filename
#==================================================================#
# Newline substitution for fairseq models
#==================================================================#
def encodenewlines(txt):
if(vars.newlinemode == "s"):
return txt.replace('\n', "</s>")
return txt
def decodenewlines(txt):
if(vars.newlinemode == "s"):
return txt.replace("</s>", '\n')
return txt