From 91b6289897c66f0a662d1b2b39306a536b6c713e Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Tue, 21 Dec 2021 19:35:57 -0500 Subject: [PATCH 01/18] Upload Lua pattern matching libraries --- extern/lualibs/lpeg.lua | 1 + extern/lualibs/lulpeg.lua | 138 ++++++++++++++++++++++++++++++++++++++ extern/lualibs/pegex.lua | 6 ++ 3 files changed, 145 insertions(+) create mode 100644 extern/lualibs/lpeg.lua create mode 100644 extern/lualibs/lulpeg.lua create mode 100644 extern/lualibs/pegex.lua diff --git a/extern/lualibs/lpeg.lua b/extern/lualibs/lpeg.lua new file mode 100644 index 00000000..cfcc7b45 --- /dev/null +++ b/extern/lualibs/lpeg.lua @@ -0,0 +1 @@ +return require("lulpeg") diff --git a/extern/lualibs/lulpeg.lua b/extern/lualibs/lulpeg.lua new file mode 100644 index 00000000..9712c4a3 --- /dev/null +++ b/extern/lualibs/lulpeg.lua @@ -0,0 +1,138 @@ +-- LuLPEG @a10d246290254ab5c206ed9df1394db9275166ab | /lulpeg.lua | https://github.com/pygy/LuLPeg | License: Public-domain-equivalent except for /src/re.lua, which is MIT-licensed | Minified using https://www.npmjs.com/package/luamin/v/1.0.4 and then the following regex replacement was used on the output: /(\([^()]+?\))(\([^()]+?or[^()]+?\)\([^()]+?\))/ was replaced with "\1;\2" (without the quotes) +-- Copyright (C) 2013 Lua.org, PUC-Rio. +-- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +-- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +local _ENV,a,b,c,d=_ENV or _G,{},{},true,require;local function require(...)local e=...if a[e]then return a[e]elseif b[e]then a[e]=b[e](e)return a[e]else return d(e)end end;do local _ENV=_ENV;b['util']=function(...)local getmetatable,setmetatable,load,loadstring,next,pairs,pcall,print,rawget,rawset,select,tostring,type,unpack=getmetatable,setmetatable,load,loadstring,next,pairs,pcall,print,rawget,rawset,select,tostring,type,unpack;local f,g,h=require"math",require"string",require"table"local i,j,k,l,m=f.max,g.match,g.gsub,h.concat,h.insert;local n=require"compat"local function o()end;local p,q,r;if pcall and not n.lua52 and not c then local function s(t,u)error("illegal global read: "..tostring(u),2)end;local function v(t,u,w)error("illegal global write: "..tostring(u)..": "..tostring(w),2)end;local x=setmetatable({},{__index=s,__newindex=v})p=function()pcall(setfenv,3,x)end;function q(y)rawget(x,y)end;function r(y,w)rawset(x,y,w)end else p=o end;local _ENV=p()local z={nop=o,noglobals=p,getglobal=q,setglobal=r}z.unpack=h.unpack or unpack;z.pack=h.pack or function(...)return{n=select('#',...),...}end;if n.lua51 then local A=load;function z.load(B,C,D,x)local E;if type(B)=='string'then E=loadstring(B)else E=A(B,C)end;if x then setfenv(E,x)end;return E end else z.load=load end;if n.luajit and n.jit then function z.max(F)local G=0;for u=1,#F do G=i(G,F[u])end;return G end elseif n.luajit then local H=z.unpack;function z.max(F)local I=#F;if I<=30 or I>10240 then local G=0;for u=1,#F do local J=F[u]if J>G then G=J end end;return G else return i(H(F))end end else local H=z.unpack;local K=1000;function z.max(L)local I=#L;if I==0 then return-1 end;local M=1;local N=K;local G=L[1]repeat if N>I then N=I end;local O=i(H(L,M,N))if O>G then G=O end;M=M+K;N=N+K until M>=I;return G end end;local function P(h,D)local Q=getmetatable(h)or{}if Q.__mode then error("The mode has already been set on table "..tostring(h)..".")end;Q.__mode=D;return setmetatable(h,Q)end;z.setmode=P;function z.weakboth(h)return P(h,"kv")end;function z.weakkey(h)return P(h,"k")end;function z.weakval(h)return P(h,"v")end;function z.strip_mt(h)return setmetatable(h,nil)end;local R;do local S,T=0,{}function R(w)if not T[w]then S=S+1;T[w]=S end;return T[w]end end;z.getuniqueid=R;do local U=0;function z.gensym()U=U+1;return"___SYM_"..U end end;function z.passprint(...)print(...)return...end;local V,W,X,Y,Z;local _=2;local function a0(w,a1)a1=a1 or 0;Z={}local a2={}V(w,a2,a1,a1)local a3=l(a2,"")return a3 end;z.val_to_str=a0;function V(w,a2,a1,a4)a4=a4 or 1;if"string"==type(w)then w=k(w,"\n","\n"..(" "):rep(a1*_+a4))if j(k(w,"[^'\"]",""),'^"+$')then a2[#a2+1]=l{"'","",w,"'"}else a2[#a2+1]=l{'"',k(w,'"','\\"'),'"'}end elseif"cdata"==type(w)then Y(w,a2,a1)elseif"table"==type(w)then if Z[w]then a2[#a2+1]=Z[w]else Z[w]=tostring(w)X(w,a2,a1)end else a2[#a2+1]=tostring(w)end end;function W(y,a2,a1)if"string"==type(y)and j(y,"^[_%a][_%a%d]*$")then a2[#a2+1]=k(y,"\n",(" "):rep(a1*_+1).."\n")else a2[#a2+1]="[ "V(y,a2,a1)a2[#a2+1]=" ]"end end;function Y(w,a2,a1)a2[#a2+1]=(" "):rep(a1*_)a2[#a2+1]="["print(#a2)for u=0,#w do if u%16==0 and u~=0 then a2[#a2+1]="\n"a2[#a2+1]=(" "):rep(a1*_+2)end;a2[#a2+1]=w[u]and 1 or 0;a2[#a2+1]=u~=#w and", "or""end;print(#a2,a2[1],a2[2])a2[#a2+1]="]"end;function X(a5,a2,a1)a2[#a2+1]=Z[a5]a2[#a2+1]="{\n"for y,w in pairs(a5)do local a4=1;a2[#a2+1]=(" "):rep((a1+1)*_)W(y,a2,a1+1)if a2[#a2]==" ]"and a2[#a2-2]=="[ "then a4=8+#a2[#a2-1]end;a2[#a2+1]=" = "V(w,a2,a1+1,a4)a2[#a2+1]="\n"end;a2[#a2+1]=(" "):rep(a1*_)a2[#a2+1]="}"end;function z.expose(w)print(a0(w))return w end;function z.map(F,a6,...)if type(F)=="function"then F,a6=a6,F end;local a3={}for u=1,#F do a3[u]=a6(F[u],...)end;return a3 end;function z.selfmap(F,a6,...)if type(F)=="function"then F,a6=a6,F end;for u=1,#F do F[u]=a6(F[u],...)end;return F end;local function a7(a5,a6,...)if type(a5)=="function"then a5,a6=a6,a5 end;local a3={}for y,w in next,a5 do a3[y]=a6(w,...)end;return a3 end;z.map_all=a7;local function a8(F,a6,a2)local a9=1;if not a2 then a2=F[1]a9=2 end;for u=a9,#F do a2=a6(a2,F[u])end;return a2 end;z.fold=a8;local function aa(F,a6,a2)local ab=0;if not a2 then a2=F[#F]ab=1 end;for u=#F-ab,1,-1 do a2=a6(F[u],a2)end;return a2 end;z.foldr=aa;local function ac(F,ad,ae,a2)local a9=1;if not a2 then a2=ad(F[1])a9=2 end;for u=a9,#F do a2=ae(a2,ad(F[u]))end;return a2 end;z.map_fold=ac;local function af(F,ad,ae,a2)local ab=0;if not a2 then a2=ad(F[#a2])ab=1 end;for u=#F-ab,1,-1 do a2=ae(ad(F[u],a2))end;return a2 end;z.map_foldr=ac;function z.zip(ag,ah)local a3,I={},i(#ag,#ah)for u=1,I do a3[u]={ag[u],ah[u]}end;return a3 end;function z.zip_all(ai,aj)local a3={}for y,w in pairs(ai)do a3[y]={w,aj[y]}end;for y,w in pairs(aj)do if a3[y]==nil then a3[y]={ai[y],w}end end;return a3 end;function z.filter(F,a6)local a3={}for u=1,#F do if a6(F[u])then m(a3,F[u])end end end;local function ak(...)return...end;z.id=ak;local function al(am,an)return am and an end;local function ao(am,an)return am or an end;function z.copy(a5)return a7(a5,ak)end;function z.all(F,ad)if ad then return ac(F,ad,al)else return a8(F,al)end end;function z.any(F,ad)if ad then return ac(F,ad,ao)else return a8(F,ao)end end;function z.get(ap)return function(a5)return a5[ap]end end;function z.lt(aq)return function(ar)return ar0 then return u>I and I+1 or u else return I+u<0 and 1 or I+u+1 end end;local function aV()return{kind={},bounds={},openclose={},aux={}}end;local function aW(aX,aM,aY,aZ,...)if aX then print("@!!! Match !!!@",aM)end;aM=aT(aM)assert(type(aY)=="string","string expected for the match subject")aZ=aU(aZ,#aY)if aX then print(("-"):rep(30))print(aM.pkind)aG.pprint(aM)end;local a_=aL(aM,{})local b0=aV()local b1={grammars={},args={n=select('#',...),...},tags={}}local success,b2,aQ=a_(aY,aZ,b0,1,b1)if aX then print("!!! Done Matching !!! success: ",success,"final position",b2,"final cap index",aQ,"#caps",#b0.openclose)end;if success then aP(b0.kind,aQ)aP(b0.aux,aQ)if aX then print("trimmed cap index = ",#b0+1)aG.cprint(b0,aY,1)end;local b3,t,b4=aS(b0,aY,1,1)if aX then print("#values",b4)aE(b3)end;if b4==0 then return b2 else return H(b3,1,b4)end else if aX then print("Failed")end;return nil end end;function aG.match(...)return aW(false,...)end;function aG.dmatch(...)return aW(true,...)end;for t,w in pairs{"C","Cf","Cg","Cs","Ct","Clb","div_string","div_table","div_number","div_function"}do aK[w]=load(([=[ + local compile, expose, type, LL = ... + return function (pt, ccache) + local matcher, this_aux = compile(pt.pattern, ccache), pt.aux + return function (sbj, si, caps, ci, state) + local ref_ci = ci + local kind, bounds, openclose, aux + = caps.kind, caps.bounds, caps.openclose, caps.aux + kind [ci] = "XXXX" + bounds [ci] = si + openclose [ci] = 0 + caps.aux [ci] = (this_aux or false) + local success + success, si, ci + = matcher(sbj, si, caps, ci + 1, state) + if success then + if ci == ref_ci + 1 then + caps.openclose[ref_ci] = si + else + kind [ci] = "XXXX" + bounds [ci] = si + openclose [ci] = ref_ci - ci + aux [ci] = this_aux or false + ci = ci + 1 + end + else + ci = ci - 1 + end + return success, si, ci + end + end]=]):gsub("XXXX",w),w.." compiler")(aL,aE,type,aG)end;aK["Carg"]=function(aM,aN)local b5=aM.aux;return function(aY,aZ,b0,aQ,b6)if b6.args.n#aY,aZ,aQ end;aK["eos"]=function(aM)return bm end;local function bn(aY,aZ,b0,aQ,b6)local bo,t=az(aY,aZ),aZ+1;if bo then return true,aZ+1,aQ else return false,aZ,aQ end end;aK["one"]=function(aM)return bn end;aK["any"]=function(aM)local S=aM.aux;if S==1 then return bn else S=aM.aux-1;return function(aY,aZ,b0,aQ,b6)local b5=aZ+S;if b5<=#aY then return true,b5+1,aQ else return false,aZ,aQ end end end end;do local function bp(at)for y,w in pairs(at.aux)do if not aI(w)then error(("rule 'A' is not a pattern"):gsub("A",tostring(y)))end end end;aK["grammar"]=function(aM,aN)bp(aM)local bq=a7(aM.aux,aL,aN)local br=bq[1]return function(aY,aZ,b0,aQ,b6)m(b6.grammars,bq)local success,b7,aQ=br(aY,aZ,b0,aQ,b6)aB(b6.grammars)return success,b7,aQ end end end;local bs={kind={},bounds={},openclose={},aux={}}aK["behind"]=function(aM,aN)local a_,S=aL(aM.pattern,aN),aM.aux;return function(aY,aZ,b0,aQ,b6)if aZ<=S then return false,aZ,aQ end;local success=a_(aY,aZ-S,bs,aQ,b6)bs.aux={}return success,aZ,aQ end end;aK["range"]=function(aM)local bt=aM.aux;return function(aY,aZ,b0,aQ,b6)local bo,b7=az(aY,aZ),aZ+1;for u=1,#bt do local bu=bt[u]if bo and bu[bo]then return true,b7,aQ end end;return false,aZ,aQ end end;aK["set"]=function(aM)local g=aM.aux;return function(aY,aZ,b0,aQ,b6)local bo,b7=az(aY,aZ),aZ+1;if g[bo]then return true,b7,aQ else return false,aZ,aQ end end end;aK["range"]=aK.set;aK["ref"]=function(aM,aN)local bv=aM.aux;local aq;return function(aY,aZ,b0,aQ,b6)if not aq then if#b6.grammars==0 then error(("rule 'XXXX' used outside a grammar"):gsub("XXXX",tostring(bv)))elseif not b6.grammars[#b6.grammars][bv]then error(("rule 'XXXX' undefined in given grammar"):gsub("XXXX",tostring(bv)))end;aq=b6.grammars[#b6.grammars][bv]end;local success,b7,bw=aq(aY,aZ,b0,aQ,b6)return success,b7,bw end end;local bx=[=[ + success, si, ci = XXXX(sbj, si, caps, ci, state) + if success then + return true, si, ci + else + end]=]local function by(bd,aM,aN)if aM[2].pkind==bd then return aL(aM[1],aN),by(bd,aM[2],aN)else return aL(aM[1],aN),aL(aM[2],aN)end end;aK["choice"]=function(aM,aN)local bz={by("choice",aM,aN)}local bA,bB={},{}for u=1,#bz do local f="ch"..u;bA[#bA+1]=f;bB[#bA]=bx:gsub("XXXX",f)end;bA[#bA+1]="clear_captures"bz[#bA]=aP;local bC=l{"local ",l(bA,", "),[=[ = ... + return function (sbj, si, caps, ci, state) + local aux, success = caps.aux, false + ]=],l(bB,"\n"),[=[-- + return false, si, ci + end]=]}return load(bC,"Choice")(H(bz))end;local bD=[=[ + success, si, ci = XXXX(sbj, si, caps, ci, state) + if not success then + return false, ref_si, ref_ci + end]=]aK["sequence"]=function(aM,aN)local bE={by("sequence",aM,aN)}local bA,bB={},{}for u=1,#bE do local f="seq"..u;bA[#bA+1]=f;bB[#bA]=bD:gsub("XXXX",f)end;bA[#bA+1]="clear_captures"bE[#bA]=aP;local bC=l{"local ",l(bA,", "),[=[ = ... + return function (sbj, si, caps, ci, state) + local ref_si, ref_ci, success = si, ci + ]=],l(bB,"\n"),[=[ + return true, si, ci + end]=]}return load(bC,"Sequence")(H(bE))end;aK["at most"]=function(aM,aN)local a_,b5=aL(aM.pattern,aN),aM.aux;b5=-b5;return function(aY,aZ,b0,aQ,b6)local success=true;for u=1,b5 do success,aZ,aQ=a_(aY,aZ,b0,aQ,b6)if not success then break end end;return true,aZ,aQ end end;aK["at least"]=function(aM,aN)local a_,b5=aL(aM.pattern,aN),aM.aux;if b5==0 then return function(aY,aZ,b0,aQ,b6)local bF,bG;while true do local success;bF,bG=aZ,aQ;success,aZ,aQ=a_(aY,aZ,b0,aQ,b6)if not success then aZ,aQ=bF,bG;break end end;return true,aZ,aQ end elseif b5==1 then return function(aY,aZ,b0,aQ,b6)local bF,bG;local success=true;success,aZ,aQ=a_(aY,aZ,b0,aQ,b6)if not success then return false,aZ,aQ end;while true do local success;bF,bG=aZ,aQ;success,aZ,aQ=a_(aY,aZ,b0,aQ,b6)if not success then aZ,aQ=bF,bG;break end end;return true,aZ,aQ end else return function(aY,aZ,b0,aQ,b6)local bF,bG;local success=true;for t=1,b5 do success,aZ,aQ=a_(aY,aZ,b0,aQ,b6)if not success then return false,aZ,aQ end end;while true do local success;bF,bG=aZ,aQ;success,aZ,aQ=a_(aY,aZ,b0,aQ,b6)if not success then aZ,aQ=bF,bG;break end end;return true,aZ,aQ end end end;aK["unm"]=function(aM,aN)if aM.pkind=="any"and aM.aux==1 then return bm end;local a_=aL(aM.pattern,aN)return function(aY,aZ,b0,aQ,b6)local success,t,t=a_(aY,aZ,b0,aQ,b6)return not success,aZ,aQ end end;aK["lookahead"]=function(aM,aN)local a_=aL(aM.pattern,aN)return function(aY,aZ,b0,aQ,b6)local success,t,t=a_(aY,aZ,b0,aQ,b6)return success,aZ,aQ end end end end end;do local _ENV=_ENV;b['datastructures']=function(...)local getmetatable,pairs,setmetatable,type=getmetatable,pairs,setmetatable,type;local f,h,ay=require"math",require"table",require"util"local n=require"compat"local bH;if n.luajit then bH=require"ffi"end;local _ENV=ay.noglobals()local bI,load,bJ=ay.extend,ay.load,ay.max;local i,l,m,bK=f.max,h.concat,h.insert,h.sort;local bL={}local bM,bN,bO;local bP={}local function bQ(bR)local av=setmetatable(load(l{"return{ [0]=false",(", false"):rep(bR)," }"})(),bP)return av end;if n.jit then local bS,bT={v={}}function bP.__index(g,u)if u==nil or u>g.upper then return nil end;return g.v[u]end;function bP.__len(g)return g.upper end;function bP.__newindex(g,u,w)g.v[u]=w end;bT=bH.metatype('struct { int upper; bool v[?]; }',bP)function bM(h)if type(h)=="number"then local a3=bT(h+1)a3.upper=h;return a3 end;local bR=bJ(h)bS.upper=bR;if bR>255 then error"bool_set overflow"end;local av=bT(bR+1)av.upper=bR;for u=1,#h do av[h[u]]=true end;return av end;function bN(g)return type(g)=="cdata"and bH.istype(g,bT)end;bO=bN else function bM(h)if type(h)=="number"then return bQ(h)end;local av=bQ(bJ(h))for u=1,#h do av[h[u]]=true end;return av end;function bN(g)return false end;function bO(g)return getmetatable(g)==bP end end;local function bU(bV,bW)bW=bV<=bW and bW or-1;local av=bM(bW)for u=bV,bW do av[u]=true end;return av end;local bX,bY={},{}local function bZ(g,b_)if type(g)=="number"then b_[g]=true;return b_ else return g end end;local function c0(am,an)bX[am]=nil;bY[an]=nil end;local function c1(am,an)local bR=i(type(am)=="number"and am or#am,type(an)=="number"and an or#an)local c2,c3=bZ(am,bX),bZ(an,bY)local a3=bM(bR)for u=0,bR do a3[u]=c2[u]or c3[u]or false end;c0(am,an)return a3 end;local function c4(am,an)local a3={}for u=0,255 do a3[u]=am[u]and not an[u]end;return a3 end;local function c5(g)local c6={}for u=0,255 do c6[#c6+1]=g[u]==true and u or nil end;return l(c6,", ")end;bL.binary={set={new=bM,union=c1,difference=c4,tostring=c5},Range=bU,isboolset=bN,isbyteset=bO,isset=bO}local c7={}local function c8(h)local av=setmetatable({},c7)for u=1,#h do av[h[u]]=true end;return av end;local function c9(am,a3)for y in pairs(am)do a3[y]=true end;return a3 end;local function ca(am,an)am,an=type(am)=="number"and c8{am}or am,type(an)=="number"and c8{an}or an;local a3=c8{}c9(am,a3)c9(an,a3)return a3 end;local function cb(am,an)local c6={}am,an=type(am)=="number"and c8{am}or am,type(an)=="number"and c8{an}or an;for cc in pairs(am)do if am[cc]and not an[cc]then c6[#c6+1]=cc end end;return c8(c6)end;local function cd(g)local c6={}for cc in pairs(g)do m(c6,cc)end;bK(c6)return l(c6,",")end;local function ce(g)return getmetatable(g)==c7 end;local function cf(br,cg)local c6={}for u=br,cg do c6[#c6+1]=u end;return c8(c6)end;bL.other={set={new=c8,union=ca,tostring=cd,difference=cb},Range=cf,isboolset=bN,isbyteset=bO,isset=ce,isrange=function(am)return false end}return function(aF,aG)local ch=(aF.options or{}).charset or"binary"if type(ch)=="string"then ch=ch=="binary"and"binary"or"other"else ch=ch.binary and"binary"or"other"end;return bI(aF,bL[ch])end end end;do local _ENV=_ENV;b['re']=function(...)return function(aF,aG)local tonumber,type,print,error=tonumber,type,print,error;local setmetatable=setmetatable;local f=aG;local ci=f;local Q=getmetatable(ci.P(0))local cj=_VERSION;if cj=="Lua 5.2"then _ENV=nil end;local ck=f.P(1)local cl={nl=f.P"\n"}local cm;local cn;local co;local function cp()ci.locale(cl)cl.a=cl.alpha;cl.c=cl.cntrl;cl.d=cl.digit;cl.g=cl.graph;cl.l=cl.lower;cl.p=cl.punct;cl.s=cl.space;cl.u=cl.upper;cl.w=cl.alnum;cl.x=cl.xdigit;cl.A=ck-cl.a;cl.C=ck-cl.c;cl.D=ck-cl.d;cl.G=ck-cl.g;cl.L=ck-cl.l;cl.P=ck-cl.p;cl.S=ck-cl.s;cl.U=ck-cl.u;cl.W=ck-cl.w;cl.X=ck-cl.x;cm={}cn={}co={}local Q={__mode="v"}setmetatable(cm,Q)setmetatable(cn,Q)setmetatable(co,Q)end;cp()local function cq(ak,cr)local bj=cr and cr[ak]if not bj then error("undefined name: "..ak)end;return bj end;local function cs(g,u)local ct=#g=1 do if b5%2>=1 then cw=cw*cv end;cv=cv*cv;b5=b5/2 end;return cw end;local function cx(g,u,bj)if type(bj)~="string"then return nil end;local cy=#bj+u;if g:sub(u,cy-1)==bj then return cy else return nil end end;local bh=(cl.space+"--"*(ck-cl.nl)^0)^0;local bv=f.R("AZ","az","__")*f.R("AZ","az","__","09")^0;local cz=bh*"<-"local cA=f.P"/"+")"+"}"+":}"+"~}"+"|}"+bv*cz+-1;bv=f.C(bv)local cB=bv*f.Carg(1)local cC=f.C(f.R"09"^1)*bh/tonumber;local cD="'"*f.C((ck-"'")^0)*"'"+'"'*f.C((ck-'"')^0)*'"'local cE="%"*cB/function(bj,cF)local cG=cF and cF[bj]or cl[bj]if not cG then error("name '"..bj.."' undefined")end;return cG end;local cH=f.Cs(ck*f.P"-"/""*(ck-"]"))/ci.R;local cI=cE+cH+f.C(ck)local cJ="["*f.C(f.P"^"^-1)*f.Cf(cI*(cI-"]")^0,Q.__add)/function(bj,cv)return bj=="^"and ck-cv or cv end*"]"local function cK(h,y,cL)if h[y]then error("'"..y.."' already defined as a rule")else h[y]=cL end;return h end;local function cM(b5,bu)return cK({b5},b5,bu)end;local function cN(b5,an)if not an then error("rule '"..b5 .."' used outside a grammar")else return ci.V(b5)end end;local cL=f.P{"Exp",Exp=bh*(f.V"Grammar"+f.Cf(f.V"Seq"*("/"*bh*f.V"Seq")^0,Q.__add)),Seq=f.Cf(f.Cc(f.P"")*f.V"Prefix"^0,Q.__mul)*(f.L(cA)+cs),Prefix="&"*bh*f.V"Prefix"/Q.__len+"!"*bh*f.V"Prefix"/Q.__unm+f.V"Suffix",Suffix=f.Cf(f.V"Primary"*bh*((f.P"+"*f.Cc(1,Q.__pow)+f.P"*"*f.Cc(0,Q.__pow)+f.P"?"*f.Cc(-1,Q.__pow)+"^"*(f.Cg(cC*f.Cc(cu))+f.Cg(f.C(f.S"+-"*f.R"09"^1)*f.Cc(Q.__pow)))+"->"*bh*(f.Cg((cD+cC)*f.Cc(Q.__div))+f.P"{}"*f.Cc(nil,f.Ct)+f.Cg(cB/cq*f.Cc(Q.__div)))+"=>"*bh*f.Cg(cB/cq*f.Cc(f.Cmt)))*bh)^0,function(am,an,as)return as(am,an)end),Primary="("*f.V"Exp"*")"+cD/ci.P+cJ+cE+"{:"*(bv*":"+f.Cc(nil))*f.V"Exp"*":}"/function(b5,cv)return ci.Cg(cv,b5)end+"="*bv/function(b5)return ci.Cmt(ci.Cb(b5),cx)end+f.P"{}"/ci.Cp+"{~"*f.V"Exp"*"~}"/ci.Cs+"{|"*f.V"Exp"*"|}"/ci.Ct+"{"*f.V"Exp"*"}"/ci.C+f.P"."*f.Cc(ck)+(bv*-cz+"<"*bv*">")*f.Cb("G")/cN,Definition=bv*cz*f.V"Exp",Grammar=f.Cg(f.Cc(true),"G")*f.Cf(f.V"Definition"/cM*f.Cg(f.V"Definition")^0,cK)/ci.P}local cO=bh*f.Cg(f.Cc(false),"G")*cL/ci.P*(-ck+cs)local function aL(cv,cr)if ci.type(cv)=="pattern"then return cv end;local cP=cO:match(cv,1,cr)if not cP then error("incorrect pattern",3)end;return cP end;local function cQ(g,cv,u)local cP=cm[cv]if not cP then cP=aL(cv)cm[cv]=cP end;return cP:match(g,u or 1)end;local function cR(g,cv,u)local cP=cn[cv]if not cP then cP=aL(cv)/0;cP=ci.P{ci.Cp()*cP*ci.Cp()+1*ci.V(1)}cn[cv]=cP end;local u,cy=cP:match(g,u or 1)if u then return u,cy-1 else return u end end;local function cS(g,cv,cT)local at=co[cv]or{}co[cv]=at;local cP=at[cT]if not cP then cP=aL(cv)cP=ci.Cs((cP/cT+1)^0)at[cT]=cP end;return cP:match(g)end;local cU={compile=aL,match=cQ,find=cR,gsub=cS,updatelocale=cp}return cU end end end;do local _ENV=_ENV;b['charsets']=function(...)local g,h,ay=require"string",require"table",require"util"local _ENV=ay.noglobals()local cV=ay.copy;local cW,aA,az,l,m=g.char,g.sub,g.byte,h.concat,h.insert;local function cX(cY)if cY<128 then return 0,cY elseif cY<192 then error("Byte values between 0x80 to 0xBF cannot start a multibyte sequence")elseif cY<224 then return 1,cY-192 elseif cY<240 then return 2,cY-224 elseif cY<248 then return 3,cY-240 elseif cY<252 then return 4,cY-248 elseif cY<254 then return 5,cY-252 else error("Byte values between 0xFE and OxFF cannot start a multibyte sequence")end end;local function cZ(c_,br,cg)br=br or 1;cg=cg or#c_;local ab,bo=0;for u=br,cg do local an=az(c_,u)if ab==0 then bo=u;success,ab=pcall(cX,an)if not success then return false,bo-1 end else if not(127#c_ then return end;local bj=az(c_,u)local ab,ar=cX(bj)for u=u+1,u+ab do bj=az(c_,u)ar=ar*64+bj-128 end;return u+ab,u,ar end;local function d1(c_,u)u=u and u+1 or 1;if u>#c_ then return end;local ab=cX(az(c_,u))return u+ab,u,aA(c_,u,u+ab)end;local function d2(c_)local d3={}for t,t,bj in d0,c_ do m(d3,bj)end;return d3 end;local function d4(c_)local d3={}for t,t,bj in d1,c_ do m(d3,bj)end;return d3 end;local function d5(c_,u)if u>#c_ then return end;local bj=az(c_,u)local ab,ar=cX(bj)for u=u+1,u+ab do bj=az(c_,u)ar=ar*64+bj-128 end;return ar,u+ab+1 end;local function d6(d7)if not d7 then return end;return function(c_)local a3={}local d8,u=true;while d8 do d8,u=d7(c_,u)a3[#a3]=d8 end;return a3 end end;local function d9(bo)if not bo then return end;return function(F)local a3={}for u=1,#F do m(a3,bo(F[u]))end;return l(a3)end end;local function da(c_,u)local cY,db,dc,dd,de,df=az(c_,u)if cY<128 then return cY,u+1 elseif cY<192 then error("Byte values between 0x80 to 0xBF cannot start a multibyte sequence")elseif cY<224 then return(cY-192)*64+az(c_,u+1),u+2 elseif cY<240 then de,df=az(c_,u+1,u+2)return(cY-224)*4096+de%64*64+df%64,u+3 elseif cY<248 then dd,de,df=az(c_,u+1,u+2,1+3)return(cY-240)*262144+dd%64*4096+de%64*64+df%64,u+4 elseif cY<252 then dc,dd,de,df=az(c_,u+1,u+2,1+3,u+4)return(cY-248)*16777216+dc%64*262144+dd%64*4096+de%64*64+df%64,u+5 elseif cY<254 then db,dc,dd,de,df=az(c_,u+1,u+2,1+3,u+4,u+5)return(cY-252)*1073741824+db%64*16777216+dc%64*262144+dd%64*4096+de%64*64+df%64,u+6 else error("Byte values between 0xFE and OxFF cannot start a multibyte sequence")end end;local function dg(c_,u)if u>#c_ then return end;local ab=cX(az(c_,u))return aA(c_,u,u+ab),u+ab+1 end;local function dh(bj)if bj<128 then return cW(bj)elseif bj<2048 then return cW(192+bj/64,128+bj%64)elseif bj<55296 or 57343=#c_ then return end;return u,u,aA(c_,u,u)end;local function dk(c_,u)u=u and u+1 or 1;if u>#c_ then return end;return u,u,az(c_,u)end;local function dl(c_)local d3={}for u=1,#c_ do m(d3,az(c_,u))end;return d3 end;local function dm(c_)local d3={}for u=1,#c_ do m(d3,aA(c_,u,u))end;return d3 end;local function dn(c_,u)return az(c_,u),u+1 end;local function dp(c_,u)return aA(c_,u,u),u+1 end;local dq={binary={name="binary",binary=true,validate=di,split_char=dm,split_int=dl,next_char=dk,next_int=dj,get_char=dp,get_int=dn,tochar=cW},["UTF-8"]={name="UTF-8",validate=cZ,split_char=d4,split_int=d2,next_char=d1,next_int=d0,get_char=dg,get_int=d5}}return function(aF)local ch=aF.options.charset or"binary"if dq[ch]then aF.charset=cV(dq[ch])aF.binary_split_int=dl else error("NYI: custom charsets")end end end end;do local _ENV=_ENV;b['evaluator']=function(...)local select,tonumber,tostring,type=select,tonumber,tostring,type;local g,h,ay=require"string",require"table",require"util"local aA,l=g.sub,h.concat;local H=ay.unpack;local _ENV=ay.noglobals()return function(aF,aG)local dr={}local function ds(b0,aY,dt,aQ,b4)local bf,bd=b0.openclose,b0.kind;while bd[aQ]and bf[aQ]>=0 do aQ,b4=dr[bd[aQ]](b0,aY,dt,aQ,b4)end;return aQ,b4 end;function dr.C(b0,aY,dt,aQ,b4)if b0.openclose[aQ]>0 then dt[b4]=aA(aY,b0.bounds[aQ],b0.openclose[aQ]-1)return aQ+1,b4+1 end;dt[b4]=false;local du,dv=ds(b0,aY,dt,aQ+1,b4+1)dt[b4]=aA(aY,b0.bounds[aQ],b0.bounds[du]-1)return du+1,dv end;local function dw(b0,dx,aQ)local bg,bf,bd=b0.aux,b0.openclose,b0.kind;repeat aQ=aQ-1;local dy,dz=bg[aQ],bf[aQ]if dz<0 then aQ=aQ+dz end;if dz~=0 and bd[aQ]=="Clb"and dx==dy then return aQ end until aQ==1;dx=type(dx)=="string"and"'"..dx.."'"or tostring(dx)error("back reference "..dx.." not found")end;function dr.Cb(b0,aY,dt,aQ,b4)local dA=dw(b0,b0.aux[aQ],aQ)dA,b4=dr.Cg(b0,aY,dt,dA,b4)return aQ+1,b4 end;function dr.Cc(b0,aY,dt,aQ,b4)local dB=b0.aux[aQ]for u=1,dB.n do b4,dt[b4]=b4+1,dB[u]end;return aQ+1,b4 end;dr["Cf"]=function()error("NYI: Cf")end;function dr.Cf(b0,aY,dt,aQ,b4)if b0.openclose[aQ]>0 then error"No First Value"end;local a6,dC,dD=b0.aux[aQ],{}aQ=aQ+1;aQ,dD=dr[b0.kind[aQ]](b0,aY,dC,aQ,1)if dD==1 then error"No first value"end;local dE=dC[1]while b0.kind[aQ]and b0.openclose[aQ]>=0 do aQ,dD=dr[b0.kind[aQ]](b0,aY,dC,aQ,1)dE=a6(dE,H(dC,1,dD-1))end;dt[b4]=dE;return aQ+1,b4+1 end;function dr.Cg(b0,aY,dt,aQ,b4)if b0.openclose[aQ]>0 then dt[b4]=aA(aY,b0.bounds[aQ],b0.openclose[aQ]-1)return aQ+1,b4+1 end;local du,dv=ds(b0,aY,dt,aQ+1,b4)if dv==b4 then dt[dv]=aA(aY,b0.bounds[aQ],b0.bounds[du]-1)dv=dv+1 end;return du+1,dv end;function dr.Clb(b0,aY,dt,aQ,b4)local dz=b0.openclose;if dz[aQ]>0 then return aQ+1,b4 end;local dF=0;repeat if dz[aQ]==0 then dF=dF+1 elseif dz[aQ]<0 then dF=dF-1 end;aQ=aQ+1 until dF==0;return aQ,b4 end;function dr.Cp(b0,aY,dt,aQ,b4)dt[b4]=b0.bounds[aQ]return aQ+1,b4+1 end;function dr.Ct(b0,aY,dt,aQ,b4)local bg,bf,bd=b0.aux,b0.openclose,b0.kind;local dG={}dt[b4]=dG;if bf[aQ]>0 then return aQ+1,b4+1 end;local dH,dI=1,{}aQ=aQ+1;while bd[aQ]and bf[aQ]>=0 do if bd[aQ]=="Clb"then local dx,dJ=bg[aQ],1;aQ,dJ=dr.Cg(b0,aY,dI,aQ,1)if dJ~=1 then dG[dx]=dI[1]end else aQ,dH=dr[bd[aQ]](b0,aY,dG,aQ,dH)end end;return aQ+1,b4+1 end;local dK=1/0;function dr.value(b0,aY,dt,aQ,b4)local ar;if b0.aux[aQ]~=dK or b0.openclose[aQ]~=dK then ar=b0.aux[aQ]end;dt[b4]=ar;return aQ+1,b4+1 end;function dr.Cs(b0,aY,dt,aQ,b4)if b0.openclose[aQ]>0 then dt[b4]=aA(aY,b0.bounds[aQ],b0.openclose[aQ]-1)else local be,bd,bf=b0.bounds,b0.kind,b0.openclose;local br,dL,dM,dN,dO=be[aQ],{},{},1,1;local dP;aQ=aQ+1;while bf[aQ]>=0 do dP=be[aQ]dL[dN]=aA(aY,br,dP-1)dN=dN+1;aQ,dO=dr[bd[aQ]](b0,aY,dM,aQ,1)if dO>1 then dL[dN]=dM[1]dN=dN+1;br=bf[aQ-1]>0 and bf[aQ-1]or be[aQ-1]else br=dP end end;dL[dN]=aA(aY,br,be[aQ]-1)dt[b4]=l(dL)end;return aQ+1,b4+1 end;local function dQ(a2,dR,...)local b5=select('#',...)for u=1,b5 do dR,a2[dR]=dR+1,select(u,...)end;return dR end;function dr.div_function(b0,aY,dt,aQ,b4)local a6=b0.aux[aQ]local dS,dT;if b0.openclose[aQ]>0 then dS,dT={aA(aY,b0.bounds[aQ],b0.openclose[aQ]-1)},2 else dS={}aQ,dT=ds(b0,aY,dS,aQ+1,1)end;aQ=aQ+1;b4=dQ(dt,b4,a6(H(dS,1,dT-1)))return aQ,b4 end;function dr.div_number(b0,aY,dt,aQ,b4)local dU=b0.aux[aQ]local dV,dW;if b0.openclose[aQ]>0 then dV,dW={aA(aY,b0.bounds[aQ],b0.openclose[aQ]-1)},2 else dV={}aQ,dW=ds(b0,aY,dV,aQ+1,1)end;aQ=aQ+1;if dU>=dW then error("no capture '"..dU.."' in /number capture.")end;dt[b4]=dV[dU]return aQ,b4+1 end;local function dX(b0,aQ)local dY=b0.openclose;local dZ={open=b0.bounds[aQ]}if dY[aQ]>0 then dZ.close=dY[aQ]return aQ+1,dZ,0 end;local d_=aQ;local dF=1;aQ=aQ+1;repeat local dz=dY[aQ]if dF==1 and dz>=0 then dZ[#dZ+1]=aQ end;if dz==0 then dF=dF+1 elseif dz<0 then dF=dF-1 end;aQ=aQ+1 until dF==0;dZ.close=b0.bounds[aQ-1]return aQ,dZ,#dZ end;function dr.div_string(b0,aY,dt,aQ,b4)local b5,dZ;local e0;local e0,e1={},{}local e2=b0.aux[aQ]aQ,dZ,b5=dX(b0,aQ)dt[b4]=e2:gsub("%%([%d%%])",function(e3)if e3=="%"then return"%"end;e3=tonumber(e3)if not e0[e3]then if e3>b5 then error("no capture at index "..e3 .." in /string capture.")end;if e3==0 then e0[e3]=aA(aY,dZ.open,dZ.close-1)else local t,b4=dr[b0.kind[dZ[e3]]](b0,aY,e1,dZ[e3],1)if b4==1 then error("no values in capture at index"..e3 .." in /string capture.")end;e0[e3]=e1[1]end end;return e0[e3]end)return aQ,b4+1 end;function dr.div_table(b0,aY,dt,aQ,b4)local dU=b0.aux[aQ]local e4;if b0.openclose[aQ]>0 then e4=aA(aY,b0.bounds[aQ],b0.openclose[aQ]-1)else local e5,t={}aQ,t=ds(b0,aY,e5,aQ+1,1)e4=e5[1]end;aQ=aQ+1;if dU[e4]then dt[b4]=dU[e4]return aQ,b4+1 else return aQ,b4 end end;function aG.evaluate(b0,aY,aQ)local dt={}local t,b4=ds(b0,aY,dt,aQ,1)return dt,1,b4-1 end end end end;do local _ENV=_ENV;b['printers']=function(...)return function(aF,aG)local ipairs,pairs,print,tostring,type=ipairs,pairs,print,tostring,type;local g,h,ay=require"string",require"table",require"util"local e6=aF.set.tostring;local _ENV=ay.noglobals()local cW,aA,l=g.char,g.sub,h.concat;local aE,load,aC=ay.expose,ay.load,ay.map;local e7={["\f"]="\\f",["\n"]="\\n",["\r"]="\\r",["\t"]="\\t",["\v"]="\\v",["\127"]="\\ESC"}local function by(bd,c6)if c6[2].pkind==bd then return c6[1],by(bd,c6[2])else return c6[1],c6[2]end end;for u=0,8 do e7[cW(u)]="\\"..u end;for u=14,31 do e7[cW(u)]="\\"..u end;local function e8(ax)return ax:gsub("%c",e7)end;local function e9(av)return cW(load("return "..e6(av))())end;local ea={}local function eb(aM,ab,ec)return ea[aM.pkind](aM,ab,ec)end;function aG.pprint(ed)local aM=aG.P(ed)print"\nPrint pattern"eb(aM,"","")print"--- /pprint\n"return ed end;for y,w in pairs{string=[[ "P( \""..escape(pt.as_is).."\" )" ]],char=[[ "P( \""..escape(to_char(pt.aux)).."\" )"]],["true"]=[[ "P( true )" ]],["false"]=[[ "P( false )" ]],eos=[[ "~EOS~" ]],one=[[ "P( one )" ]],any=[[ "P( "..pt.aux.." )" ]],set=[[ "S( "..'"'..escape(set_repr(pt.aux))..'"'.." )" ]],["function"]=[[ "P( "..pt.aux.." )" ]],ref=[[ + "V( ", + (type(pt.aux) == "string" and "\""..pt.aux.."\"") + or tostring(pt.aux) + , " )" + ]],range=[[ + "R( ", + escape(t_concat(map( + pt.as_is, + function(e) return '"'..e..'"' end) + , ", ")) + ," )" + ]]}do ea[y]=load(([==[ + local k, map, t_concat, to_char, escape, set_repr = ... + return function (pt, offset, prefix) + print(t_concat{offset,prefix,XXXX}) + end + ]==]):gsub("XXXX",w),y.." printer")(y,aC,l,cW,e8,e9)end;for y,w in pairs{["behind"]=[[ LL_pprint(pt.pattern, offset, "B ") ]],["at least"]=[[ LL_pprint(pt.pattern, offset, pt.aux.." ^ ") ]],["at most"]=[[ LL_pprint(pt.pattern, offset, pt.aux.." ^ ") ]],unm=[[LL_pprint(pt.pattern, offset, "- ")]],lookahead=[[LL_pprint(pt.pattern, offset, "# ")]],choice=[[ + print(offset..prefix.."+") + local ch, i = {}, 1 + while pt.pkind == "choice" do + ch[i], pt, i = pt[1], pt[2], i + 1 + end + ch[i] = pt + map(ch, LL_pprint, offset.." :", "") + ]],sequence=[=[ + print(offset..prefix.."*") + local acc, p2 = {} + offset = offset .. " |" + while true do + if pt.pkind ~= "sequence" then -- last element + if pt.pkind == "char" then + acc[#acc + 1] = pt.aux + print(offset..'P( "'..s.char(u.unpack(acc))..'" )') + else + if #acc ~= 0 then + print(offset..'P( "'..s.char(u.unpack(acc))..'" )') + end + LL_pprint(pt, offset, "") + end + break + elseif pt[1].pkind == "char" then + acc[#acc + 1] = pt[1].aux + elseif #acc ~= 0 then + print(offset..'P( "'..s.char(u.unpack(acc))..'" )') + acc = {} + LL_pprint(pt[1], offset, "") + else + LL_pprint(pt[1], offset, "") + end + pt = pt[2] + end + ]=],grammar=[[ + print(offset..prefix.."Grammar") + for k, pt in pairs(pt.aux) do + local prefix = ( type(k)~="string" + and tostring(k) + or "\""..k.."\"" ) + LL_pprint(pt, offset.." ", prefix .. " = ") + end + ]]}do ea[y]=load(([[ + local map, LL_pprint, pkind, s, u, flatten = ... + return function (pt, offset, prefix) + XXXX + end + ]]):gsub("XXXX",w),y.." printer")(aC,eb,type,g,ay,by)end;for t,ee in pairs{"C","Cs","Ct"}do ea[ee]=function(aM,ab,ec)print(ab..ec..ee)eb(aM.pattern,ab.." ","")end end;for t,ee in pairs{"Cg","Clb","Cf","Cmt","div_number","/zero","div_function","div_table"}do ea[ee]=function(aM,ab,ec)print(ab..ec..ee.." "..tostring(aM.aux or""))eb(aM.pattern,ab.." ","")end end;ea["div_string"]=function(aM,ab,ec)print(ab..ec..'/string "'..tostring(aM.aux or"")..'"')eb(aM.pattern,ab.." ","")end;for t,ee in pairs{"Carg","Cp"}do ea[ee]=function(aM,ab,ec)print(ab..ec..ee.."( "..tostring(aM.aux).." )")end end;ea["Cb"]=function(aM,ab,ec)print(ab..ec.."Cb( \""..aM.aux.."\" )")end;ea["Cc"]=function(aM,ab,ec)print(ab..ec.."Cc("..l(aC(aM.aux,tostring),", ").." )")end;local ef={}local eg=" "local function eh(b5)b5=tostring(b5)b5=b5 .."."..(" "):rep(4-#b5)return b5 end;local function ei(b0,aQ,a1,aY,b5)local bf,bd=b0.openclose,b0.kind;a1=a1 or 0;while bd[aQ]and bf[aQ]>=0 do if b0.openclose[aQ]>0 then print(l({eh(b5),eg:rep(a1),b0.kind[aQ],": start = ",tostring(b0.bounds[aQ])," finish = ",tostring(b0.openclose[aQ]),b0.aux[aQ]and" aux = "or"",b0.aux[aQ]and(type(b0.aux[aQ])=="string"and'"'..tostring(b0.aux[aQ])..'"'or tostring(b0.aux[aQ]))or""," \t",aA(aY,b0.bounds[aQ],b0.openclose[aQ]-1)}))if type(b0.aux[aQ])=="table"then aE(b0.aux[aQ])end else local bd=b0.kind[aQ]local br=b0.bounds[aQ]print(l({eh(b5),eg:rep(a1),bd,": start = ",br,b0.aux[aQ]and" aux = "or"",b0.aux[aQ]and(type(b0.aux[aQ])=="string"and'"'..tostring(b0.aux[aQ])..'"'or tostring(b0.aux[aQ]))or""}))aQ,b5=ei(b0,aQ+1,a1+1,aY,b5+1)print(l({eh(b5),eg:rep(a1),"/",bd," finish = ",tostring(b0.bounds[aQ])," \t",aA(aY,br,(b0.bounds[aQ]or 1)-1)}))end;b5=b5+1;aQ=aQ+1 end;return aQ,b5 end;function aG.cprint(b0,aQ,aY)aQ=aQ or 1;print"\nCapture Printer:\n================"ei(b0,aQ,0,aY,1)print"================\n/Cprinter\n"end;return{pprint=aG.pprint,cprint=aG.cprint}end end end;do local _ENV=_ENV;b['analyzer']=function(...)local ay=require"util"local o,ej=ay.nop,ay.weakkey;local ek,el,em=ej{},ej{},ej{}return{hasV=o,hasCmt=o,length=o,hasCapture=o}end end;do local _ENV=_ENV;b['locale']=function(...)local bI=require"util".extend;local _ENV=require"util".noglobals()return function(aF,aG)local en,bh=aG.R,aG.S;local eo={}eo["cntrl"]=en"\0\31"+"\127"eo["digit"]=en"09"eo["lower"]=en"az"eo["print"]=en" ~"eo["space"]=bh" \f\n\r\t\v"eo["upper"]=en"AZ"eo["alpha"]=eo["lower"]+eo["upper"]eo["alnum"]=eo["alpha"]+eo["digit"]eo["graph"]=eo["print"]-eo["space"]eo["punct"]=eo["graph"]-eo["alnum"]eo["xdigit"]=eo["digit"]+en"af"+en"AF"function aG.locale(h)return bI(h or{},eo)end end end end;do local _ENV=_ENV;b['match']=function(...)end end;do local _ENV=_ENV;b['factorizer']=function(...)local ipairs,pairs,print,setmetatable=ipairs,pairs,print,setmetatable;local ay=require"util"local ak,o,ep,ej=ay.id,ay.nop,ay.setify,ay.weakkey;local _ENV=ay.noglobals()local function eq(am,an,er)local ak,es=er.id,er.brk;if am==ak then return true,an elseif an==ak then return true,am elseif am==es then return true,es else return false end end;local et=ep{"unm","lookahead","C","Cf","Cg","Cs","Ct","/zero"}local eu=ep{"behind","at least","at most","Clb","Cmt","div_string","div_number","div_table","div_function"}local ev=ep{"char","set","range"}local ew;ew=setmetatable({},{__mode="k",__index=function(self,aM)local bd,a3=aM.pkind,false;if bd=="Cmt"or bd=="ref"then a3=true elseif et[bd]or eu[bd]then a3=ew[aM.pattern]elseif bd=="choice"or bd=="sequence"then a3=ew[aM[1]]or ew[aM[2]]end;ew[aM]=a3;return a3 end})return function(aF,aG)if aF.options.factorize==false then return{choice=o,sequence=o,lookahead=o,unm=o}end;local ex,aT=aF.constructors,aG.P;local ey,ez=ex.constant.truept,ex.constant.falsept;local eA=aF.set.union;local eB=ep{"char","set"}local eC={["/zero"]="__div",["div_number"]="__div",["div_string"]="__div",["div_table"]="__div",["div_function"]="__div",["at least"]="__exp",["at most"]="__exp",["Clb"]="Cg"}local function eD(am,an)do local eE,a3=eq(am,an,{id=ez,brk=ey})if eE then return a3 end end;local eF,eG=am.pkind,an.pkind;if am==an and not ew[am]then return am elseif eF=="choice"then local a2,u={},1;while am.pkind=="choice"do a2[u],am,u=am[1],am[2],u+1 end;a2[u]=am;for J=u,1,-1 do an=a2[J]+an end;return an elseif eB[eF]and eB[eG]then return ex.aux("set",eA(am.aux,an.aux))elseif eB[eF]and eG=="any"and an.aux==1 or eB[eG]and eF=="any"and am.aux==1 then return eF=="any"and am or an elseif eF==eG then if(et[eF]or eu[eF])and am.aux==an.aux then return aG[eC[eF]or eF](am.pattern+an.pattern,am.aux)elseif eF==eG and eF=="sequence"then if am[1]==an[1]and not ew[am[1]]then return am[1]*(am[2]+an[2])end end end;return false end;local function eH(aM)return aM end;local function bE(am,an)do local eE,a3=eq(am,an,{id=ey,brk=ez})if eE then return a3 end end;local eF,eG=am.pkind,an.pkind;if eF=="sequence"then local a2,u={},1;while am.pkind=="sequence"do a2[u],am,u=am[1],am[2],u+1 end;a2[u]=am;for J=u,1,-1 do an=a2[J]*an end;return an elseif(eF=="one"or eF=="any")and(eG=="one"or eG=="any")then return aT(am.aux+an.aux)end;return false end;local function eI(aM)if aM==ey then return ez elseif aM==ez then return ey elseif aM.pkind=="unm"then return#aM.pattern elseif aM.pkind=="lookahead"then return-aM.pattern end end;return{choice=eD,lookahead=eH,sequence=bE,unm=eI}end end end;do local _ENV=_ENV;b['API']=function(...)local assert,error,ipairs,pairs,pcall,print,require,select,tonumber,tostring,type=assert,error,ipairs,pairs,pcall,print,require,select,tonumber,tostring,type;local h,ay=require"table",require"util"local _ENV=ay.noglobals()local l=h.concat;local eJ,cV,a8,load,ac,af,ep,aD,H=ay.checkstring,ay.copy,ay.fold,ay.load,ay.map_fold,ay.map_foldr,ay.setify,ay.pack,ay.unpack;local function eK(T,aJ)error("Character at position "..T+1 .." is not a valid "..aJ.." one.",2)end;return function(aF,aG)local ch=aF.charset;local ex,aI=aF.constructors,aG.ispattern;local ey,ez,eL=ex.constant.truept,ex.constant.falsept,ex.constant.Cppt;local eM,eN=ch.split_int,ch.validate;local cH,eO,eA,e6=aF.Range,aF.set.new,aF.set.union,aF.set.tostring;local eP,eQ,eR,eS;local function eT(bj)return ex.aux("char",bj)end;local function aT(...)local w,b5=...,select('#',...)if b5==0 then error"bad argument #1 to 'P' (value expected)"end;local aO=type(w)if aI(w)then return w elseif aO=="function"then return aG.Cmt("",w)elseif aO=="string"then local success,T=eN(w)if not success then eK(T,ch.name)end;if w==""then return ey end;return af(eM(w),eT,aF.sequence)elseif aO=="table"then local at=cV(w)if at[1]==nil then error("grammar has no initial rule")end;if not aI(at[1])then at[1]=aG.V(at[1])end;return ex.none("grammar",at)elseif aO=="boolean"then return w and ey or ez elseif aO=="number"then if w==0 then return ey elseif w>0 then return ex.aux("any",w)else return-ex.aux("any",-w)end else error("bad argument #1 to 'P' (lpeg-pattern expected, got "..aO..")")end end;aG.P=aT;local function eU(av)if av==""then return ez else local success;av=eJ(av,"S")return ex.aux("set",eO(eM(av)),av)end end;aG.S=eU;local function eV(...)if select('#',...)==0 then return aT(false)else local eW=cH(1,0)for t,bu in ipairs{...}do bu=eJ(bu,"R")assert(#bu==2,"bad argument #1 to 'R' (range must have two characters)")eW=eA(eW,cH(H(eM(bu))))end;return ex.aux("set",eW)end end;aG.R=eV;local function eX(bv)assert(bv~=nil)return ex.aux("ref",bv)end;aG.V=eX;do local eY=ep{"set","range","one","char"}local eZ=ep{"true","false","lookahead","unm"}local e_=ep{"Carg","Cb","C","Cf","Cg","Cs","Ct","/zero","Clb","Cmt","Cc","Cp","div_string","div_number","div_table","div_function","at least","at most","behind"}local function f0(aM,bq,f1)local aO=aM.pkind;if e_[aO]then return false elseif eY[aO]then return 1 elseif eZ[aO]then return 0 elseif aO=="string"then return#aM.as_is elseif aO=="any"then return aM.aux elseif aO=="choice"then local f2,f3=f0(aM[1],bq,f1),f0(aM[2],bq,f1)return f2==f3 and f2 elseif aO=="sequence"then local f2,f3=f0(aM[1],bq,f1),f0(aM[2],bq,f1)return f2 and f3 and f2+f3 elseif aO=="grammar"then if aM.aux[1].pkind=="ref"then return f0(aM.aux[aM.aux[1].aux],aM.aux,{})else return f0(aM.aux[1],aM.aux,{})end elseif aO=="ref"then if f1[aM]then return false end;f1[aM]=true;return f0(bq[aM.aux],bq,f1)else print(aO,"is not handled by fixedlen()")end end;function aG.B(aM)aM=aT(aM)local I=f0(aM)assert(I,"A 'behind' pattern takes a fixed length pattern as argument.")if I>=260 then error("Subpattern too long in 'behind' pattern constructor.")end;return ex.both("behind",aM,I)end end;local function f4(am,an)return tostring(am)..tostring(an)end;local function eD(am,an)local bv=tostring(am)..tostring(an)local f5=aF.ptcache.choice[bv]if not f5 then f5=eP(am,an)or ex.binary("choice",am,an)aF.ptcache.choice[bv]=f5 end;return f5 end;function aG.__add(am,an)return eD(aT(am),aT(an))end;local function bE(am,an)local bv=tostring(am)..tostring(an)local f6=aF.ptcache.sequence[bv]if not f6 then f6=eR(am,an)or ex.binary("sequence",am,an)aF.ptcache.sequence[bv]=f6 end;return f6 end;aF.sequence=bE;function aG.__mul(am,an)return bE(aT(am),aT(an))end;local function f7(aM)if aM==ey or aM==ez or aM.pkind=="unm"or aM.pkind=="lookahead"then return aM end;return ex.subpt("lookahead",aM)end;aG.__len=f7;aG.L=f7;local function f8(aM)return eS(aM)or ex.subpt("unm",aM)end;aG.__unm=f8;local function f9(am,an)am,an=aT(am),aT(an)return f8(an)*am end;aG.__sub=f9;local function fa(aM,b5)local success;success,b5=pcall(tonumber,b5)assert(success and type(b5)=="number","Invalid type encountered at right side of '^'.")return ex.both(b5<0 and"at most"or"at least",aM,b5)end;aG.__pow=fa;for t,ee in pairs{"C","Cs","Ct"}do aG[ee]=function(aM)aM=aT(aM)return ex.subpt(ee,aM)end end;aG["Cb"]=function(bg)return ex.aux("Cb",bg)end;aG["Carg"]=function(bg)assert(type(bg)=="number","Number expected as parameter to Carg capture.")assert(0"*k({[0]="\\>"})local af=c"{"*f(d"09"^1)*c"}"/function(L)return tonumber(L),tonumber(L)end;local ag=c"{"*f(d"09"^1)*c","*f(d"09"^1)*c"}"/function(ah,ai)return tonumber(ah),tonumber(ai)end;local aj=c"{"*f(d"09"^1)*c",}"/function(L)return tonumber(L),nil end;local ak=af+ag+aj;local al=j("groups")*m()/function(z,am)local an=#z+1;z[an]={am}z.open[#z.open+1]=an end;local ao=j("groups")*m()/function(z,am)local an=z.open[#z.open]z.open[#z.open]=nil;z[an][2]=am;return an end;local ap=c"("*al;local aq=c")"*ao;local ar=c"(?:"local as=c")"local at=c{"pattern",pattern=n(i(1),"groups")*o((c"^"*n(k(1),"anchorstart")+c(0))*g"subpat"*(c"$"*-c(1)*n(k(1),"anchorend")+-c(1)))/function(v)v[0]="pattern"return v end,subpat=g"branch"*(c"|"*g"branch")^0/function(...)return{[0]="alt",...}end,branch=g"concat",concat=k(nil)*g"piece"^0/function(au,...)return{[0]="concat",...}end,piece=g"atom_multi",atom_multi=g"atom_plus"+g"atom_star"+g"atom_query"+g"atom_count"+g"atom",atom_plus=g"atom"*c"+"/function(av)return{[0]="+",av}end,atom_star=g"atom"*c"*"/function(av)return{[0]="*",av}end,atom_query=g"atom"*c"?"/function(av)return{[0]="?",av}end,atom_count=g"atom"*ak/function(av,ah,ai)return{[0]="{}",min=ah,max=ai,av}end,anongroup=ar*g"subpat"*as,group=ap*g"subpat"*aq/function(aw,C)return{[0]="group",aw,C}end,atom=O+ad+ae+a9+a7+g"anongroup"+g"group"+a8+aa+a0}local function ax(ay,v,az)local aA=az;local aB=#v;if aA==nil then aA=v[aB]aB=aB-1 end;for aC=aB,1,-1 do aA=ay(v[aC],aA)end;return aA end;local function aD(ay,v)local y={}for aC=1,#v do y[aC]=ay(v[aC])end;return y end;local function aE(a5,a6)return a5+a6 end;local function aF(aG)local v=aG[0]if v=="char"then assert(#aG==1)return c(aG[1])elseif v=="range"then assert(#aG==2)return d(aG[1]..aG[2])elseif v=="set"then return e(aG[1])else error("Got charset bit: "..tostring(v).."/"..tostring(v and v[0]))end end;local function aH(aI,A,aJ)local v=aI[0]if v=="pattern"then assert(#aI==1)local aK=aH(aI[1],A,aJ)if aI.anchorend then aK=aK*-c(1)end;aK=s*aK*t;if not aI.anchorstart then aK=c{aK+1*g(1)}end;return aK elseif v=="group"then assert(#aI==2)aJ.numGroups=aJ.numGroups+1;local C=tostring(aI[2])local aL=n(m()/q,"e"..C)*A;local aK=aH(aI[1],aL,aJ)aK=n(m(),"s"..C)*aK;return aK elseif v=="alt"then if#aI==1 then return aH(aI[1],A,aJ)else local aM=aD(function(r)return aH(r,A,aJ)end,aI)return ax(aE,aM)end elseif v=="concat"then return ax(function(aN,aO)return aH(aN,aO,aJ)end,aI,A)elseif v=="char"then assert(#aI==1)return c(aI[1])*A elseif v=="charset"then local aP=ax(aE,aD(aF,aI))if aI.negate then aP=1-aP end;return aP*A elseif v=="*"then return c{"A",A=aH(aI[1],g"A",aJ)+A}elseif v=="+"then return aH(aI[1],c{"A",A=aH(aI[1],g"A",aJ)+A},aJ)elseif v=="."then assert(#aI==0)return(c(1)-c"\n")*A elseif v=="?"then assert(#aI==1)return aH(aI[1],A,aJ)+A elseif v=="{}"then assert(#aI==1)local aw=aI[1]local ah=aI.min;local ai=aI.max;local aQ={[0]="concat"}for aC=1,ah do aQ[#aQ+1]=aw end;if ai==nil then aQ[#aQ+1]={[0]="*",aw}else local aR={[0]="?",aw}for aC=ah+1,ai do aQ[#aQ+1]=aR end end;return aH(aQ,A,aJ)elseif v=="\\<"then assert(#aI==0)return-h(ab)*#ab*A elseif v=="\\>"then assert(#aI==0)return h(ab)*-#ab*A elseif v=="backref"then local C=aI[1]return p(c(0)*j("s"..C)*j("e"..C),function(aS,am,w,aT)local aU=aS:sub(w,aT)local aV=aS:sub(am,am+aT-w)if aU==aV then return am+aT-w+1 else return false end end)else error("Not implemented op: "..tostring(v).."/"..tostring(aI))end end;function a.parse(aW)return at:match(aW,1,{open={}})end;function a.compile(aW)local aI=a.parse(aW)if aI==nil then error("Failed to parse regular expression: {"..aW.."}",2)end;local aJ={numGroups=0}local aX=aH(aI,c(0),aJ)return setmetatable({_pat=o(aX),numgroups=aJ.numGroups},u)end;b.setmaxstack(1000)return a From 380b54167a0c634a5b808fea8f20a91cb23f9a9f Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Tue, 21 Dec 2021 22:12:24 -0500 Subject: [PATCH 02/18] Make transformers warpers dynamically update their parameters So that if you change, e.g., `top_p`, from a Lua generation modifier or from the settings menu during generation, the rest of the generation will use the new setting value instead of retaining the settings it had when generation began. --- aiserver.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/aiserver.py b/aiserver.py index 2fb91b1e..e31eedd8 100644 --- a/aiserver.py +++ b/aiserver.py @@ -7,6 +7,7 @@ # External packages import eventlet +from transformers.generation_logits_process import RepetitionPenaltyLogitsProcessor eventlet.monkey_patch() import os os.environ['EVENTLET_THREADPOOL_SIZE'] = '1' @@ -617,6 +618,18 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly", "TPUMeshTransforme # Patch transformers to use our custom logit warpers from transformers import LogitsProcessorList, LogitsWarper, LogitsProcessor, TopKLogitsWarper, TopPLogitsWarper, TemperatureLogitsWarper + + def dynamic_processor_wrap(cls, field_name, var_name): + old_call = cls.__call__ + def new_call(self, *args, **kwargs): + setattr(self, field_name, getattr(vars, var_name)) + return old_call(self, *args, **kwargs) + cls.__call__ = new_call + dynamic_processor_wrap(RepetitionPenaltyLogitsProcessor, "penalty", "rep_pen") + dynamic_processor_wrap(TopKLogitsWarper, "top_k", "top_k") + dynamic_processor_wrap(TopPLogitsWarper, "top_p", "top_p") + dynamic_processor_wrap(TemperatureLogitsWarper, "temperature", "temp") + class TailFreeLogitsWarper(LogitsWarper): def __init__(self, tfs: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1): @@ -628,6 +641,8 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly", "TPUMeshTransforme self.min_tokens_to_keep = min_tokens_to_keep def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: + self.tfs = vars.tfs + if self.filter_value >= 1.0: return scores sorted_logits, sorted_indices = torch.sort(scores, descending=True) From 8c63f7c03432ec356b4dc8a57f85963531029002 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Tue, 21 Dec 2021 23:43:56 -0500 Subject: [PATCH 03/18] Change `|nil` in Lua type annotations to `?` --- bridge.lua | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/bridge.lua b/bridge.lua index dcdd51f7..8e5955f3 100644 --- a/bridge.lua +++ b/bridge.lua @@ -3,7 +3,7 @@ ---@param _python? table ---@param _bridged? table ----@return KoboldLib, KoboldCoreLib|nil +---@return KoboldLib, KoboldCoreLib? return function(_python, _bridged) --========================================================================== @@ -14,7 +14,7 @@ return function(_python, _bridged) ---@generic K, V ---@param t table ---@param k? K - ---@return K|nil, V|nil + ---@return K?, V? function next(t, k) local meta = getmetatable(t) return ((meta ~= nil and type(rawget(t, "_name")) == "string" and string.match(rawget(t, "_name"), "^Kobold") and type(meta._kobold_next) == "function") and meta._kobold_next or old_next)(t, k) @@ -214,7 +214,7 @@ return function(_python, _bridged) koboldbridge.generated = {} koboldbridge.generated_cols = 0 koboldbridge.outputs = {} - koboldbridge.feedback = nil ---@type string|nil + koboldbridge.feedback = nil ---@type string? ---@return nil local function maybe_require_regeneration() @@ -373,7 +373,7 @@ return function(_python, _bridged) local KoboldWorldInfoFolder_mt = setmetatable({}, metawrapper) ---@param u integer - ---@return KoboldWorldInfoEntry|nil + ---@return KoboldWorldInfoEntry? function KoboldWorldInfoFolder:finduid(u) if not check_validity(self) or type(u) ~= "number" then return @@ -440,7 +440,7 @@ return function(_python, _bridged) KoboldWorldInfoFolder_mt.__pairs = KoboldWorldInfoEntry_mt.__pairs ---@param t KoboldWorldInfoFolder|KoboldWorldInfo - ---@return KoboldWorldInfoEntry|nil + ---@return KoboldWorldInfoEntry? function KoboldWorldInfoFolder_mt.__index(t, k) if not check_validity(t) then return @@ -495,7 +495,7 @@ return function(_python, _bridged) local KoboldWorldInfoFolderSelector_mt = setmetatable({}, metawrapper) ---@param u integer - ---@return KoboldWorldInfoFolder|nil + ---@return KoboldWorldInfoFolder? function KoboldWorldInfoFolderSelector:finduid(u) if not check_validity(self) or type(u) ~= "number" then return @@ -528,7 +528,7 @@ return function(_python, _bridged) KoboldWorldInfoFolderSelector_mt.__pairs = KoboldWorldInfoEntry_mt.__pairs ---@param t KoboldWorldInfoFolderSelector - ---@return KoboldWorldInfoFolder|nil + ---@return KoboldWorldInfoFolder? function KoboldWorldInfoFolderSelector_mt.__index(t, k) if not check_validity(t) or type(k) ~= "number" or math.tointeger(k) == nil or k < 1 or k > #t then return @@ -1259,19 +1259,19 @@ return function(_python, _bridged) koboldbridge.userscripts = {} ---@type table koboldbridge.num_userscripts = 0 - koboldbridge.inmod = nil ---@type function|nil - koboldbridge.genmod = nil ---@type function|nil - koboldbridge.outmod = nil ---@type function|nil + koboldbridge.inmod = nil ---@type function? + koboldbridge.genmod = nil ---@type function? + koboldbridge.outmod = nil ---@type function? ---@class KoboldUserScript - ---@field inmod function|nil - ---@field genmod function|nil - ---@field outmod function|nil + ---@field inmod function? + ---@field genmod function? + ---@field outmod function? ---@class KoboldCoreScript - ---@field inmod function|nil - ---@field genmod function|nil - ---@field outmod function|nil + ---@field inmod function? + ---@field genmod function? + ---@field outmod function? ---------------------------------------------------------------------------- @@ -1280,9 +1280,9 @@ return function(_python, _bridged) ---@field filename string ---@field modulename string ---@field description string - ---@field inmod function|nil - ---@field genmod function|nil - ---@field outmod function|nil + ---@field inmod function? + ---@field genmod function? + ---@field outmod function? local KoboldUserScriptModule = setmetatable({ _name = "KoboldUserScriptModule", }, metawrapper) @@ -1346,7 +1346,7 @@ return function(_python, _bridged) ---@param t KoboldUserScriptList ---@param k integer - ---@return KoboldUserScriptModule|nil + ---@return KoboldUserScriptModule? function KoboldUserScriptList_mt.__index(t, k) if type(k) == "number" and math.tointeger(k) ~= nil then return koboldbridge.userscripts[k] @@ -1428,7 +1428,7 @@ return function(_python, _bridged) ---@param modname string ---@param env table ---@param search_path? string - ---@return any, string|nil + ---@return any, string? local function requirex(modname, env, search_path) if search_path == nil then search_path = bridged.lib_path @@ -1470,7 +1470,7 @@ return function(_python, _bridged) end local function _safe_require(_g) ---@param modname string - ---@return any, string|nil + ---@return any, string? return function(modname) return requirex(modname, _g) end From d538782b1e78bcb7a43720c9f1982544e73c838f Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Wed, 22 Dec 2021 02:59:31 -0500 Subject: [PATCH 04/18] Add individual configuration files for userscripts --- aiserver.py | 4 +- bridge.lua | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 139 insertions(+), 6 deletions(-) diff --git a/aiserver.py b/aiserver.py index e31eedd8..f971cab3 100644 --- a/aiserver.py +++ b/aiserver.py @@ -7,7 +7,6 @@ # External packages import eventlet -from transformers.generation_logits_process import RepetitionPenaltyLogitsProcessor eventlet.monkey_patch() import os os.environ['EVENTLET_THREADPOOL_SIZE'] = '1' @@ -617,7 +616,7 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly", "TPUMeshTransforme # Patch transformers to use our custom logit warpers - from transformers import LogitsProcessorList, LogitsWarper, LogitsProcessor, TopKLogitsWarper, TopPLogitsWarper, TemperatureLogitsWarper + from transformers import LogitsProcessorList, LogitsWarper, LogitsProcessor, TopKLogitsWarper, TopPLogitsWarper, TemperatureLogitsWarper, RepetitionPenaltyLogitsProcessor def dynamic_processor_wrap(cls, field_name, var_name): old_call = cls.__call__ @@ -1461,6 +1460,7 @@ vars.lua_state = lupa.LuaRuntime(unpack_returned_tuples=True) bridged = { "corescript_path": os.path.join(os.path.dirname(os.path.realpath(__file__)), "cores"), "userscript_path": os.path.join(os.path.dirname(os.path.realpath(__file__)), "userscripts"), + "config_path": os.path.join(os.path.dirname(os.path.realpath(__file__)), "userscripts"), "lib_path": os.path.join(os.path.dirname(os.path.realpath(__file__)), "extern", "lualibs"), "load_callback": load_callback, "print": lua_print, diff --git a/bridge.lua b/bridge.lua index 8e5955f3..2542aedc 100644 --- a/bridge.lua +++ b/bridge.lua @@ -153,6 +153,8 @@ return function(_python, _bridged) ---@field generated_cols integer ---@field outputs table ---@field num_outputs integer + ---@field feedback string + ---@field is_config_file_open boolean local kobold = setmetatable({}, metawrapper) local KoboldLib_mt = setmetatable({}, metawrapper) local KoboldLib_getters = setmetatable({}, metawrapper) @@ -216,6 +218,11 @@ return function(_python, _bridged) koboldbridge.outputs = {} koboldbridge.feedback = nil ---@type string? + function koboldbridge:clear_userscript_metadata() + self.logging_name = nil + self.filename = nil + end + ---@return nil local function maybe_require_regeneration() if koboldbridge.userstate == "genmod" or koboldbridge.userstate == "outmod" then @@ -224,6 +231,84 @@ return function(_python, _bridged) end + --========================================================================== + -- Userscript API: Configuration + --========================================================================== + + local config_files = {} ---@type table + local config_file_filename_map = {} ---@type table + + ---@return file*? + local function open_and_handle_errors(...) + local file, err_msg = io.open(...) + if err_msg ~= nil then + koboldbridge.obliterate_multiverse() + error(err_msg) + return + end + return file + end + + ---@param file file* + local function new_close_pre(file) + if file == nil then + file = io.output() + end + local filename = config_file_filename_map[file] + if filename ~= nil then + config_file_filename_map[file] = nil + config_files[filename] = nil + end + end + + ---@param f fun(file?: file*) + local function _new_close(f) + ---@param file file*? + return function(file) + new_close_pre(file) + return f(file) + end + end + debug.getmetatable(io.stdout).__index.close = _new_close(io.stdout.close) + + ---@param filename string + ---@return boolean + local function is_config_file_open(filename) + return config_files[filename] ~= nil + end + + ---@param filename string + ---@param clear? boolean + ---@return file* + local function get_config_file(filename, clear) + if not is_config_file_open(filename) then + local config_filepath = join_folder_and_filename(bridged.config_path, filename .. ".conf") + open_and_handle_errors(config_filepath, "a"):close() + config_files[filename] = open_and_handle_errors(config_filepath, clear and "w+b" or "r+b") + config_file_filename_map[config_files[filename]] = filename + end + return config_files[filename] + end + + ---@param clear? boolean + ---@return file* + function kobold.get_config_file(clear) + return get_config_file(koboldbridge.filename, clear) + end + + ---@param t KoboldLib + ---@return boolean + function KoboldLib_getters.is_config_file_open(t) + return is_config_file_open(koboldbridge.filename) + end + + ---@param t KoboldLib + ---@param v boolean + function KoboldLib_setters.is_config_file_open(t, v) + error("`KoboldLib.is_config_file_open` is a read-only attribute") + end + + --========================================================================== -- Userscript API: World Info --========================================================================== @@ -1257,7 +1342,8 @@ return function(_python, _bridged) -- Core script API --========================================================================== - koboldbridge.userscripts = {} ---@type table + koboldbridge.userscripts = {} ---@type table + koboldbridge.userscriptmodule_filename_map = {} ---@type table koboldbridge.num_userscripts = 0 koboldbridge.inmod = nil ---@type function? koboldbridge.genmod = nil ---@type function? @@ -1267,6 +1353,7 @@ return function(_python, _bridged) ---@field inmod function? ---@field genmod function? ---@field outmod function? + ---@field config file* ---@class KoboldCoreScript ---@field inmod function? @@ -1280,6 +1367,7 @@ return function(_python, _bridged) ---@field filename string ---@field modulename string ---@field description string + ---@field is_config_file_open boolean ---@field inmod function? ---@field genmod function? ---@field outmod function? @@ -1297,6 +1385,12 @@ return function(_python, _bridged) outmod = false, } + ---@param clear? boolean + ---@return file* + function KoboldUserScriptModule:get_config_file(clear) + return get_config_file(koboldbridge.userscriptmodule_filename_map[self], clear) + end + ---@generic K ---@param t KoboldUserScriptModule ---@param k K @@ -1316,6 +1410,8 @@ return function(_python, _bridged) function KoboldUserScriptModule_mt.__index(t, k) if type(k) == "string" and KoboldUserScriptModule_fields[k] ~= nil then return rawget(t, "_" .. k) + elseif k == "is_config_file_open" then + return is_config_file_open(koboldbridge.userscriptmodule_filename_map[t]) end return rawget(t, k) end @@ -1405,6 +1501,7 @@ return function(_python, _bridged) local envs = {} koboldbridge.logging_name = nil + koboldbridge.filename = nil local old_load = load local function _safe_load(_g) @@ -1610,6 +1707,8 @@ return function(_python, _bridged) output = io.output, read = io.read, write = io.write, + close = _new_close(io.close), + lines = io.lines, flush = io.flush, type = io.type, }, @@ -1654,7 +1753,11 @@ return function(_python, _bridged) end function koboldbridge.obliterate_multiverse() + for k, v in pairs(config_files) do + pcall(v.close, v) + end envs = {} + koboldbridge.userscripts = {} koboldbridge.num_userscripts = 0 koboldbridge.inmod = nil koboldbridge.genmod = nil @@ -1669,22 +1772,49 @@ return function(_python, _bridged) ---@return nil function koboldbridge.load_userscripts(filenames, modulenames, descriptions) set_require_path(bridged.userscript_path) + config_files = {} + config_file_filename_map = {} koboldbridge.userscripts = {} + koboldbridge.userscriptmodule_filename_map = {} koboldbridge.num_userscripts = 0 for i, filename in _python.enumerate(filenames) do bridged.load_callback(filename, modulenames[i]) koboldbridge.logging_name = modulenames[i] + koboldbridge.filename = filename ---@type KoboldUserScript local _userscript = old_loadfile(join_folder_and_filename(bridged.userscript_path, filename), "t", koboldbridge.get_universe(filename))() koboldbridge.logging_name = nil + koboldbridge.filename = nil local userscript = deepcopy(KoboldUserScriptModule) - rawset(userscript, "_inmod", function() koboldbridge.logging_name = modulenames[i]; if _userscript.inmod ~= nil then _userscript.inmod() end end) - rawset(userscript, "_genmod", function() koboldbridge.logging_name = modulenames[i]; if _userscript.genmod ~= nil then _userscript.genmod() end end) - rawset(userscript, "_outmod", function() koboldbridge.logging_name = modulenames[i]; if _userscript.outmod ~= nil then _userscript.outmod() end end) + rawset(userscript, "_inmod", function() + koboldbridge.logging_name = modulenames[i] + koboldbridge.filename = filename + if _userscript.inmod ~= nil then + _userscript.inmod() + end + koboldbridge:clear_userscript_metadata() + end) + rawset(userscript, "_genmod", function() + koboldbridge.logging_name = modulenames[i] + koboldbridge.filename = filename + if _userscript.genmod ~= nil then + _userscript.genmod() + end + koboldbridge:clear_userscript_metadata() + end) + rawset(userscript, "_outmod", function() + koboldbridge.logging_name = modulenames[i] + koboldbridge.filename = filename + if _userscript.outmod ~= nil then + _userscript.outmod() + end + koboldbridge:clear_userscript_metadata() + end) rawset(userscript, "_filename", filename) rawset(userscript, "_modulename", modulenames[i]) rawset(userscript, "_description", descriptions[i]) koboldbridge.userscripts[i+1] = userscript + koboldbridge.userscriptmodule_filename_map[userscript] = filename koboldbridge.num_userscripts = i + 1 end end @@ -1700,6 +1830,7 @@ return function(_python, _bridged) function koboldbridge.execute_inmod() local r + koboldbridge:clear_userscript_metadata() koboldbridge.restart_sequence = nil koboldbridge.userstate = "inmod" koboldbridge.regeneration_required = false @@ -1722,6 +1853,7 @@ return function(_python, _bridged) ---@return any, boolean function koboldbridge.execute_genmod() local r + koboldbridge:clear_userscript_metadata() koboldbridge.generating = true koboldbridge.userstate = "genmod" if koboldbridge.genmod ~= nil then @@ -1758,6 +1890,7 @@ return function(_python, _bridged) function koboldbridge.execute_outmod() local r + koboldbridge:clear_userscript_metadata() koboldbridge.generating = false koboldbridge.userstate = "outmod" koboldbridge.num_outputs = kobold.settings.numseqs From 988a68e2379df85a4dca9115c409addd21e4d059 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Wed, 22 Dec 2021 11:47:47 -0500 Subject: [PATCH 05/18] Use local variables as much as possible in bridge.lua --- bridge.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bridge.lua b/bridge.lua index 2542aedc..020af586 100644 --- a/bridge.lua +++ b/bridge.lua @@ -42,7 +42,7 @@ return function(_python, _bridged) ---@param path string ---@return nil - function set_require_path(path) + local function set_require_path(path) local config = {} local i = 1 for substring in string.gmatch(package.config, "[^\n]+") do @@ -56,7 +56,7 @@ return function(_python, _bridged) ---@param path string ---@param filename string ---@return string - function join_folder_and_filename(path, filename) + local function join_folder_and_filename(path, filename) return path .. string.match(package.config, "[^\n]+") .. filename end @@ -67,7 +67,7 @@ return function(_python, _bridged) local bridged = {} for k in _python.iter(_bridged) do - v = _bridged[k] + local v = _bridged[k] bridged[k] = type(v) == "userdata" and _python.as_attrgetter(v) or v end set_require_path(bridged.lib_path) @@ -766,7 +766,7 @@ return function(_python, _bridged) local actions = koboldbridge.userstate == "genmod" and bridged.vars._actions or bridged.vars.actions local nxt, iterator = _python.iter(actions) local run_once = false - local f = function() + local function f() if not bridged.vars.gamestarted then return end @@ -794,7 +794,7 @@ return function(_python, _bridged) local actions = koboldbridge.userstate == "genmod" and bridged.vars._actions or bridged.vars.actions local nxt, iterator = _python.iter(_python.builtins.reversed(actions)) local last_run = false - local f = function() + local function f() if not bridged.vars.gamestarted or last_run then return end From bc26b7c7aff6ad487222f79513221abe26cea954 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Wed, 22 Dec 2021 12:05:59 -0500 Subject: [PATCH 06/18] Correct some type annotations in bridge.lua --- bridge.lua | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/bridge.lua b/bridge.lua index 020af586..7b9a986f 100644 --- a/bridge.lua +++ b/bridge.lua @@ -249,7 +249,7 @@ return function(_python, _bridged) return file end - ---@param file file* + ---@param file? file* local function new_close_pre(file) if file == nil then file = io.output() @@ -263,7 +263,7 @@ return function(_python, _bridged) ---@param f fun(file?: file*) local function _new_close(f) - ---@param file file*? + ---@param file? file* return function(file) new_close_pre(file) return f(file) @@ -1350,15 +1350,14 @@ return function(_python, _bridged) koboldbridge.outmod = nil ---@type function? ---@class KoboldUserScript - ---@field inmod function? - ---@field genmod function? - ---@field outmod function? - ---@field config file* + ---@field inmod? function + ---@field genmod? function + ---@field outmod? function ---@class KoboldCoreScript - ---@field inmod function? - ---@field genmod function? - ---@field outmod function? + ---@field inmod? function + ---@field genmod? function + ---@field outmod? function ---------------------------------------------------------------------------- @@ -1368,9 +1367,9 @@ return function(_python, _bridged) ---@field modulename string ---@field description string ---@field is_config_file_open boolean - ---@field inmod function? - ---@field genmod function? - ---@field outmod function? + ---@field inmod? function + ---@field genmod? function + ---@field outmod? function local KoboldUserScriptModule = setmetatable({ _name = "KoboldUserScriptModule", }, metawrapper) From 175416c1ba203769da00627b7f6785b996449aa8 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Wed, 22 Dec 2021 12:06:22 -0500 Subject: [PATCH 07/18] Use `table.pack` in bridge.lua metawrapper newindex metamethod --- bridge.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridge.lua b/bridge.lua index 7b9a986f..9075268f 100644 --- a/bridge.lua +++ b/bridge.lua @@ -120,12 +120,12 @@ return function(_python, _bridged) _needs_unwrap = true wrapped = true end - local r = {wrapped_func(...)} + local r = table.pack(wrapped_func(...)) if _needs_unwrap then metatables:restore() wrapped = false end - return table.unpack(r) + return table.unpack(r, 1, r.n) end) else return rawset(t, k, wrapped_func) From 452b46146b9478ede4208e18a73900fab060223a Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Wed, 22 Dec 2021 12:19:36 -0500 Subject: [PATCH 08/18] Messages should be hidden on loading a save, softprompt or script --- static/application.js | 5 +++++ templates/index.html | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/static/application.js b/static/application.js index 3296348e..ad747d48 100644 --- a/static/application.js +++ b/static/application.js @@ -2205,6 +2205,7 @@ $(document).ready(function(){ }); load_accept.on("click", function(ev) { + hideMessage(); newly_loaded = true; socket.send({'cmd': 'loadrequest', 'data': ''}); hideLoadPopup(); @@ -2215,6 +2216,7 @@ $(document).ready(function(){ }); sp_accept.on("click", function(ev) { + hideMessage(); socket.send({'cmd': 'sprequest', 'data': ''}); hideSPPopup(); }); @@ -2225,6 +2227,7 @@ $(document).ready(function(){ }); us_accept.on("click", function(ev) { + hideMessage(); socket.send({'cmd': 'usloaded', 'data': usloaded.find(".uslistitem").map(function() { return $(this).attr("name"); }).toArray()}); socket.send({'cmd': 'usload', 'data': ''}); hideUSPopup(); @@ -2235,6 +2238,7 @@ $(document).ready(function(){ }); ns_accept.on("click", function(ev) { + hideMessage(); socket.send({'cmd': 'newgame', 'data': ''}); hideNewStoryPopup(); }); @@ -2267,6 +2271,7 @@ $(document).ready(function(){ }); rs_accept.on("click", function(ev) { + hideMessage(); socket.send({'cmd': 'rndgame', 'data': topic.val()}); hideRandomStoryPopup(); }); diff --git a/templates/index.html b/templates/index.html index 8451bbae..5997720f 100644 --- a/templates/index.html +++ b/templates/index.html @@ -10,7 +10,7 @@ - + From fc04ff3a08956adde55e2668c555d8c2bc6da747 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Wed, 22 Dec 2021 13:12:35 -0500 Subject: [PATCH 09/18] World info folders can now be collapsed by clicking on the folder icon --- aiserver.py | 6 ++++++ static/application.js | 40 +++++++++++++++++++++++++++++++++++----- static/custom.css | 24 +++++++++++++++++++++++- templates/index.html | 4 ++-- 4 files changed, 66 insertions(+), 8 deletions(-) diff --git a/aiserver.py b/aiserver.py index f971cab3..bc9f67d4 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1710,6 +1710,12 @@ def get_message(msg): elif(msg['cmd'] == 'wiexpandfolder'): assert 0 <= int(msg['data']) < len(vars.worldinfo) emit('from_server', {'cmd': 'wiexpandfolder', 'data': msg['data']}, broadcast=True) + elif(msg['cmd'] == 'wifoldercollapsecontent'): + vars.wifolders_d[msg['data']]['collapsed'] = True + emit('from_server', {'cmd': 'wifoldercollapsecontent', 'data': msg['data']}, broadcast=True) + elif(msg['cmd'] == 'wifolderexpandcontent'): + vars.wifolders_d[msg['data']]['collapsed'] = False + emit('from_server', {'cmd': 'wifolderexpandcontent', 'data': msg['data']}, broadcast=True) elif(msg['cmd'] == 'wiupdate'): num = int(msg['num']) fields = ("key", "keysecondary", "content", "comment") diff --git a/static/application.js b/static/application.js index ad747d48..5746f05e 100644 --- a/static/application.js +++ b/static/application.js @@ -419,7 +419,7 @@ function addWiLine(ob) { function addWiFolder(uid, ob) { if(uid !== null) { var uninitialized = $("#wilistfoldercontainer"+null); - var html = "
\ + var html = "
\
\
\ \ @@ -428,7 +428,7 @@ function addWiFolder(uid, ob) {
\
\
\ - \ + \
\
\
\ @@ -461,7 +461,7 @@ function addWiFolder(uid, ob) { var onfocusout = function () { socket.send({'cmd': 'wifolderupdate', 'uid': uid, 'data': { name: $("#wifoldername"+uid).val(), - collapsed: false, + collapsed: !$("#btn_wifolderexpand"+uid).hasClass("folder-expanded"), }}); }; $("#wifoldergutter"+uid).on("click", function () { @@ -488,9 +488,22 @@ function addWiFolder(uid, ob) { $(this).parent().parent().find(".wisortable-body").removeClass("hidden"); $(this).parent().css("max-height", "").find(".wifoldername").find(".form-control").css("max-height", ""); }); + $("#btn_wifolderexpand"+uid).on("click", function () { + if($(this).hasClass("folder-expanded")) { + socket.send({'cmd': 'wifoldercollapsecontent', 'data': uid}); + } else { + socket.send({'cmd': 'wifolderexpandcontent', 'data': uid}); + } + }) adjustWiFolderNameHeight($("#wifoldername"+uid)[0]); + if(ob.collapsed) { + setTimeout(function() { + var container = $("#wilistfoldercontainer"+uid); + hide([container.find(".wifoldergutter-container"), container.find(".wisortable-body")]); + }, 2); + } } else { - wi_menu.append("
\ + wi_menu.append("
\
\
\ \ @@ -499,7 +512,7 @@ function addWiFolder(uid, ob) {
\
\
\ - \ + \
\
\
\ @@ -564,6 +577,18 @@ function hideWiFolderDeleteConfirm(num) { hide([$("#btn_wifolderdel"+num), $("#btn_wifoldercan"+num)]); } +function collapseWiFolderContent(uid) { + hide([$("#wifoldergutter"+uid), $(".wisortable-body[folder-uid="+uid+"]")]); + $("#btn_wifolderexpand"+uid).removeClass("folder-expanded"); + $("#wilistfoldercontainer"+uid).removeClass("folder-expanded"); +} + +function expandWiFolderContent(uid) { + show([$("#wifoldergutter"+uid), $(".wisortable-body[folder-uid="+uid+"]")]); + $("#btn_wifolderexpand"+uid).addClass("folder-expanded"); + $("#wilistfoldercontainer"+uid).addClass("folder-expanded"); +} + function enableWiSelective(num) { hide([$("#wikey"+num)]); $("#wikeyprimary"+num).val($("#wikey"+num).val()); @@ -1922,6 +1947,10 @@ $(document).ready(function(){ expandWiLine(msg.data); } else if(msg.cmd == "wiexpandfolder") { expandWiFolderLine(msg.data); + } else if(msg.cmd == "wifoldercollapsecontent") { + collapseWiFolderContent(msg.data); + } else if(msg.cmd == "wifolderexpandcontent") { + expandWiFolderContent(msg.data); } else if(msg.cmd == "wiselon") { enableWiSelective(msg.data); } else if(msg.cmd == "wiseloff") { @@ -2062,6 +2091,7 @@ $(document).ready(function(){ // Make the userscripts menu sortable var us_sortable_settings = { + placeholder: "ussortable-placeholder", delay: 2, cursor: "move", tolerance: "pointer", diff --git a/static/custom.css b/static/custom.css index f4933258..5bc6139b 100644 --- a/static/custom.css +++ b/static/custom.css @@ -603,7 +603,12 @@ chunk.editing, chunk.editing * { background-color: #3bf723; } -.wisortable-container { +.ussortable-placeholder { + height: 4px; + background-color: #3bf723; +} + +.wisortable-container.folder-expanded { padding-bottom: 50px; } @@ -702,6 +707,23 @@ chunk.editing, chunk.editing * { position: relative; } +.folder-expand { + opacity: 20%; +} + +.folder-expand:hover { + opacity: 44%; + cursor: pointer; +} + +.folder-expand.folder-expanded { + opacity: 80% !important; +} + +.folder-expand.folder-expanded:hover { + opacity: 100% !important; +} + .selective-key-icon { position: absolute !important; top: 5px !important; diff --git a/templates/index.html b/templates/index.html index 5997720f..45fcb78e 100644 --- a/templates/index.html +++ b/templates/index.html @@ -10,12 +10,12 @@ - + - + From 20a7b6a260734c51bee2c704ee2008a130c9993f Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Wed, 22 Dec 2021 13:58:01 -0500 Subject: [PATCH 10/18] Correct "LuLPEG" to "LuLPeg" --- extern/lualibs/lulpeg.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/lualibs/lulpeg.lua b/extern/lualibs/lulpeg.lua index 9712c4a3..82308904 100644 --- a/extern/lualibs/lulpeg.lua +++ b/extern/lualibs/lulpeg.lua @@ -1,4 +1,4 @@ --- LuLPEG @a10d246290254ab5c206ed9df1394db9275166ab | /lulpeg.lua | https://github.com/pygy/LuLPeg | License: Public-domain-equivalent except for /src/re.lua, which is MIT-licensed | Minified using https://www.npmjs.com/package/luamin/v/1.0.4 and then the following regex replacement was used on the output: /(\([^()]+?\))(\([^()]+?or[^()]+?\)\([^()]+?\))/ was replaced with "\1;\2" (without the quotes) +-- LuLPeg @a10d246290254ab5c206ed9df1394db9275166ab | /lulpeg.lua | https://github.com/pygy/LuLPeg | License: Public-domain-equivalent except for /src/re.lua, which is MIT-licensed | Minified using https://www.npmjs.com/package/luamin/v/1.0.4 and then the following regex replacement was used on the output: /(\([^()]+?\))(\([^()]+?or[^()]+?\)\([^()]+?\))/ was replaced with "\1;\2" (without the quotes) -- Copyright (C) 2013 Lua.org, PUC-Rio. -- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. From 1e1b45d47a227bc4fd79ecad79df5ade71bc8159 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Wed, 22 Dec 2021 14:24:31 -0500 Subject: [PATCH 11/18] Add support for multiple library paths in bridge.lua --- aiserver.py | 2 +- bridge.lua | 28 +++++++++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/aiserver.py b/aiserver.py index bc9f67d4..742b80f3 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1461,7 +1461,7 @@ bridged = { "corescript_path": os.path.join(os.path.dirname(os.path.realpath(__file__)), "cores"), "userscript_path": os.path.join(os.path.dirname(os.path.realpath(__file__)), "userscripts"), "config_path": os.path.join(os.path.dirname(os.path.realpath(__file__)), "userscripts"), - "lib_path": os.path.join(os.path.dirname(os.path.realpath(__file__)), "extern", "lualibs"), + "lib_paths": vars.lua_state.table(os.path.join(os.path.dirname(os.path.realpath(__file__)), "lualibs"), os.path.join(os.path.dirname(os.path.realpath(__file__)), "extern", "lualibs")), "load_callback": load_callback, "print": lua_print, "warn": lua_warn, diff --git a/bridge.lua b/bridge.lua index 9075268f..2ca2eed0 100644 --- a/bridge.lua +++ b/bridge.lua @@ -40,16 +40,23 @@ return function(_python, _bridged) return original end - ---@param path string + ---@param paths string|table ---@return nil - local function set_require_path(path) + local function set_require_path(paths) + if type(paths) == "string" then + paths = {paths} + end local config = {} local i = 1 for substring in string.gmatch(package.config, "[^\n]+") do config[i] = substring i = i + 1 end - package.path = path .. config[1] .. config[3] .. ".lua" .. config[2] .. path .. config[1] .. config[3] .. config[1] .. "init.lua" + local _paths = {} + for i, path in ipairs(paths) do + _paths[i] = path .. config[1] .. config[3] .. ".lua" .. config[2] .. path .. config[1] .. config[3] .. config[1] .. "init.lua" + end + package.path = table.concat(_paths, config[2]) package.cpath = "" end @@ -70,7 +77,7 @@ return function(_python, _bridged) local v = _bridged[k] bridged[k] = type(v) == "userdata" and _python.as_attrgetter(v) or v end - set_require_path(bridged.lib_path) + set_require_path(bridged.lib_paths) --========================================================================== @@ -1523,11 +1530,11 @@ return function(_python, _bridged) local old_package_searchers = package.searchers ---@param modname string ---@param env table - ---@param search_path? string + ---@param search_paths? string|table ---@return any, string? - local function requirex(modname, env, search_path) - if search_path == nil then - search_path = bridged.lib_path + local function requirex(modname, env, search_paths) + if search_paths == nil then + search_paths = bridged.lib_paths end if modname == "bridge" then return function() return env.kobold, env.koboldcore end @@ -1545,7 +1552,7 @@ return function(_python, _bridged) local loader, path local errors = {} local n_errors = 0 - set_require_path(search_path) + set_require_path(search_paths) for k, v in ipairs(old_package_searchers) do loader, path = v(modname) if allowsearch and type(loader) == "function" then @@ -1555,7 +1562,7 @@ return function(_python, _bridged) errors[n_errors] = "\n\t" .. loader end end - set_require_path(bridged.lib_path) + set_require_path(bridged.lib_paths) if not allowsearch or type(loader) ~= "function" then error("module '" .. modname .. "' not found:" .. table.concat(errors)) return @@ -1770,7 +1777,6 @@ return function(_python, _bridged) ---@return nil function koboldbridge.load_userscripts(filenames, modulenames, descriptions) - set_require_path(bridged.userscript_path) config_files = {} config_file_filename_map = {} koboldbridge.userscripts = {} From c549ea04a99b58862d9fddbadf344d168f9cbcc4 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Wed, 22 Dec 2021 17:29:07 -0500 Subject: [PATCH 12/18] Always use all logit warpers Now that the logit warper parameters can be changed mid-generation by generation modifiers, the logit warpers have to be always on. --- aiserver.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/aiserver.py b/aiserver.py index 742b80f3..d22ba7c1 100644 --- a/aiserver.py +++ b/aiserver.py @@ -717,14 +717,10 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly", "TPUMeshTransforme beams: int = 1, ) -> LogitsProcessorList: warper_list = LogitsProcessorList() - if(top_k is not None and top_k > 0): - warper_list.append(TopKLogitsWarper(top_k=top_k, min_tokens_to_keep=1 + (beams > 1))) - if(top_p is not None and top_p < 1.0): - warper_list.append(TopPLogitsWarper(top_p=top_p, min_tokens_to_keep=1 + (beams > 1))) - if(tfs is not None and tfs < 1.0): - warper_list.append(TailFreeLogitsWarper(tfs=tfs, min_tokens_to_keep=1 + (beams > 1))) - if(temp is not None and temp != 1.0): - warper_list.append(TemperatureLogitsWarper(temperature=temp)) + warper_list.append(TopKLogitsWarper(top_k=top_k, min_tokens_to_keep=1 + (beams > 1))) + warper_list.append(TopPLogitsWarper(top_p=top_p, min_tokens_to_keep=1 + (beams > 1))) + warper_list.append(TailFreeLogitsWarper(tfs=tfs, min_tokens_to_keep=1 + (beams > 1))) + warper_list.append(TemperatureLogitsWarper(temperature=temp)) return warper_list def new_sample(self, *args, **kwargs): From 7e06c25011b1e18255ff29b0df27bb272b821ed6 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Wed, 22 Dec 2021 23:33:27 -0500 Subject: [PATCH 13/18] Display icons for active userscripts and softprompts Also fixes the userscript menu so that the active userscripts preserve the previously selected order as was originally intended. --- aiserver.py | 54 ++++++++++++++++++---- static/application.js | 59 ++++++++++++++++++++++++ static/custom.css | 104 ++++++++++++++++++++++++++++++++++-------- templates/index.html | 23 ++++++++-- 4 files changed, 208 insertions(+), 32 deletions(-) diff --git a/aiserver.py b/aiserver.py index d22ba7c1..1a5db449 100644 --- a/aiserver.py +++ b/aiserver.py @@ -112,9 +112,11 @@ class vars: lua_kobold = None # `kobold` from` bridge.lua lua_koboldcore = None # `koboldcore` from bridge.lua lua_logname = ... # Name of previous userscript that logged to terminal + lua_running = False # Whether or not Lua is running (i.e. wasn't stopped due to an error) lua_edited = set() # Set of chunk numbers that were edited from a Lua generation modifier lua_deleted = set() # Set of chunk numbers that were deleted from a Lua generation modifier userscripts = [] # List of userscripts to load + last_userscripts = [] # List of previous userscript filenames from the previous time userscripts were send via usstatitems corescript = "default.lua" # Filename of corescript to load # badwords = [] # Array of str/chr values that should be removed from output badwordsids = [[13460], [6880], [50256], [42496], [4613], [17414], [22039], [16410], [27], [29], [38430], [37922], [15913], [24618], [28725], [58], [47175], [36937], [26700], [12878], [16471], [37981], [5218], [29795], [13412], [45160], [3693], [49778], [4211], [20598], [36475], [33409], [44167], [32406], [29847], [29342], [42669], [685], [25787], [7359], [3784], [5320], [33994], [33490], [34516], [43734], [17635], [24293], [9959], [23785], [21737], [28401], [18161], [26358], [32509], [1279], [38155], [18189], [26894], [6927], [14610], [23834], [11037], [14631], [26933], [46904], [22330], [25915], [47934], [38214], [1875], [14692], [41832], [13163], [25970], [29565], [44926], [19841], [37250], [49029], [9609], [44438], [16791], [17816], [30109], [41888], [47527], [42924], [23984], [49074], [33717], [31161], [49082], [30138], [31175], [12240], [14804], [7131], [26076], [33250], [3556], [38381], [36338], [32756], [46581], [17912], [49146]] # Tokenized array of badwords used to prevent AI artifacting @@ -139,6 +141,7 @@ class vars: importjs = {} # Temporary storage for import data loadselect = "" # Temporary storage for story filename to load spselect = "" # Temporary storage for soft prompt filename to load + spmeta = None # Metadata of current soft prompt, or None if not using a soft prompt sp = None # Current soft prompt tensor (as a NumPy array) sp_length = 0 # Length of current soft prompt in tokens, or 0 if not using a soft prompt svowname = "" # Filename that was flagged for overwrite confirm @@ -1039,10 +1042,13 @@ def load_lua_scripts(): vars.lua_koboldbridge.obliterate_multiverse() tpool.execute(vars.lua_koboldbridge.load_corescript, vars.corescript) tpool.execute(vars.lua_koboldbridge.load_userscripts, filenames, modulenames, descriptions) + vars.lua_running = True except lupa.LuaError as e: vars.lua_koboldbridge.obliterate_multiverse() + vars.lua_running = False if(vars.serverstarted): emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error, please check console.'}, broadcast=True) + sendUSStatItems() print("{0}{1}{2}".format(colors.RED, "***LUA ERROR***: ", colors.END), end="", file=sys.stderr) print("{0}{1}{2}".format(colors.RED, str(e).replace("\033", ""), colors.END), file=sys.stderr) print("{0}{1}{2}".format(colors.YELLOW, "Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.", colors.END), file=sys.stderr) @@ -1414,7 +1420,9 @@ def execute_inmod(): tpool.execute(vars.lua_koboldbridge.execute_inmod) except lupa.LuaError as e: vars.lua_koboldbridge.obliterate_multiverse() + vars.lua_running = False emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error, please check console.'}, broadcast=True) + sendUSStatItems() print("{0}{1}{2}".format(colors.RED, "***LUA ERROR***: ", colors.END), end="", file=sys.stderr) print("{0}{1}{2}".format(colors.RED, str(e).replace("\033", ""), colors.END), file=sys.stderr) print("{0}{1}{2}".format(colors.YELLOW, "Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.", colors.END), file=sys.stderr) @@ -1430,7 +1438,9 @@ def execute_outmod(): tpool.execute(vars.lua_koboldbridge.execute_outmod) except lupa.LuaError as e: vars.lua_koboldbridge.obliterate_multiverse() + vars.lua_running = False emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error, please check console.'}, broadcast=True) + sendUSStatItems() print("{0}{1}{2}".format(colors.RED, "***LUA ERROR***: ", colors.END), end="", file=sys.stderr) print("{0}{1}{2}".format(colors.RED, str(e).replace("\033", ""), colors.END), file=sys.stderr) print("{0}{1}{2}".format(colors.YELLOW, "Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.", colors.END), file=sys.stderr) @@ -1515,7 +1525,10 @@ def do_connect(): emit('from_server', {'cmd': 'runs_remotely'}) if(vars.allowsp): emit('from_server', {'cmd': 'allowsp', 'data': vars.allowsp}) - + + sendUSStatItems() + emit('from_server', {'cmd': 'spstatitems', 'data': {vars.spselect: vars.spmeta} if len(vars.spselect) else {}}, broadcast=True) + if(not vars.gamestarted): setStartState() sendsettings() @@ -1751,7 +1764,8 @@ def get_message(msg): elif(msg['cmd'] == 'splistrequest'): getsplist() elif(msg['cmd'] == 'uslistrequest'): - getuslist() + unloaded, loaded = getuslist() + emit('from_server', {'cmd': 'buildus', 'data': {"unloaded": unloaded, "loaded": loaded}}) elif(msg['cmd'] == 'usloaded'): vars.userscripts = [] for userscript in msg['data']: @@ -1763,8 +1777,8 @@ def get_message(msg): settingschanged() elif(msg['cmd'] == 'usload'): load_lua_scripts() - elif(msg['cmd'] == 'usload'): - getuslist() + unloaded, loaded = getuslist() + sendUSStatItems() elif(msg['cmd'] == 'loadselect'): vars.loadselect = msg["data"] elif(msg['cmd'] == 'spselect'): @@ -1773,6 +1787,7 @@ def get_message(msg): loadRequest(fileops.storypath(vars.loadselect)) elif(msg['cmd'] == 'sprequest'): spRequest(vars.spselect) + emit('from_server', {'cmd': 'spstatitems', 'data': {vars.spselect: vars.spmeta} if len(vars.spselect) else {}}, broadcast=True) elif(msg['cmd'] == 'deletestory'): deletesave(msg['data']) elif(msg['cmd'] == 'renamestory'): @@ -1810,7 +1825,17 @@ def get_message(msg): refresh_settings() elif(not vars.remote and msg['cmd'] == 'importwi'): wiimportrequest() - + +#==================================================================# +# Send userscripts list to client +#==================================================================# +def sendUSStatItems(): + _, loaded = getuslist() + loaded = loaded if vars.lua_running else [] + last_userscripts = [e["filename"] for e in loaded] + emit('from_server', {'cmd': 'usstatitems', 'data': loaded, 'flash': last_userscripts != vars.last_userscripts}, broadcast=True) + vars.last_userscripts = last_userscripts + #==================================================================# # Send start message and tell Javascript to set UI state #==================================================================# @@ -2443,7 +2468,9 @@ def generate(txt, minimum, maximum, found_entries=None): except Exception as e: if(issubclass(type(e), lupa.LuaError)): vars.lua_koboldbridge.obliterate_multiverse() + vars.lua_running = False emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error, please check console.'}, broadcast=True) + sendUSStatItems() print("{0}{1}{2}".format(colors.RED, "***LUA ERROR***: ", colors.END), end="", file=sys.stderr) print("{0}{1}{2}".format(colors.RED, str(e).replace("\033", ""), colors.END), file=sys.stderr) print("{0}{1}{2}".format(colors.YELLOW, "Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.", colors.END), file=sys.stderr) @@ -2670,7 +2697,9 @@ def tpumtjgenerate(txt, minimum, maximum, found_entries=None): except Exception as e: if(issubclass(type(e), lupa.LuaError)): vars.lua_koboldbridge.obliterate_multiverse() + vars.lua_running = False emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error, please check console.'}, broadcast=True) + sendUSStatItems() print("{0}{1}{2}".format(colors.RED, "***LUA ERROR***: ", colors.END), end="", file=sys.stderr) print("{0}{1}{2}".format(colors.RED, str(e).replace("\033", ""), colors.END), file=sys.stderr) print("{0}{1}{2}".format(colors.YELLOW, "Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.", colors.END), file=sys.stderr) @@ -3548,7 +3577,7 @@ def getsplist(): emit('from_server', {'cmd': 'buildsp', 'data': fileops.getspfiles(vars.modeldim)}) #==================================================================# -# Show list of userscripts +# Get list of userscripts #==================================================================# def getuslist(): files = {i: v for i, v in enumerate(fileops.getusfiles())} @@ -3556,11 +3585,14 @@ def getuslist(): unloaded = [] userscripts = set(vars.userscripts) for i in range(len(files)): - if files[i]["filename"] in userscripts: - loaded.append(files[i]) - else: + if files[i]["filename"] not in userscripts: unloaded.append(files[i]) - emit('from_server', {'cmd': 'buildus', 'data': {"unloaded": unloaded, "loaded": loaded}}) + files = {files[k]["filename"]: files[k] for k in files} + userscripts = set(files.keys()) + for filename in vars.userscripts: + if filename in userscripts: + loaded.append(files[filename]) + return unloaded, loaded #==================================================================# # Load a saved story via file browser @@ -3693,6 +3725,8 @@ def spRequest(filename): z, version, shape, fortran_order, dtype = fileops.checksp(filename, vars.modeldim) assert isinstance(z, zipfile.ZipFile) + with z.open('meta.json') as f: + vars.spmeta = json.load(f) z.close() with np.load(fileops.sppath(filename), allow_pickle=False) as f: diff --git a/static/application.js b/static/application.js index 5746f05e..410f2809 100644 --- a/static/application.js +++ b/static/application.js @@ -1053,6 +1053,59 @@ function hideRandomStoryPopup() { rspopup.addClass("hidden"); } +function statFlash(ref) { + ref.addClass("status-flash"); + setTimeout(function () { + ref.addClass("colorfade"); + ref.removeClass("status-flash"); + setTimeout(function () { + ref.removeClass("colorfade"); + }, 1000); + }, 50); +} + +function updateUSStatItems(items, flash) { + var stat_us = $("#stat-us"); + var stat_usactive = $("#stat-usactive"); + if(flash || stat_usactive.find("li").length != items.length) { + statFlash(stat_us.closest(".statusicon").add("#usiconlabel")); + } + stat_usactive.html(""); + if(items.length == 0) { + stat_us.html("No userscripts active"); + $("#usiconlabel").html(""); + stat_us.closest(".statusicon").removeClass("active"); + return; + } + stat_us.html("Active userscripts:"); + stat_us.closest(".statusicon").addClass("active"); + var i; + for(i = 0; i < items.length; i++) { + stat_usactive.append($("
  • "+items[i].modulename+" <"+items[i].filename+">
    • ")); + } + $("#usiconlabel").html(items.length); +} + +function updateSPStatItems(items) { + var stat_sp = $("#stat-sp"); + var stat_spactive = $("#stat-spactive"); + var key = null; + var old_val = stat_spactive.html(); + Object.keys(items).forEach(function(k) {key = k;}); + if(key === null) { + stat_sp.html("No soft prompt active"); + stat_sp.closest(".statusicon").removeClass("active"); + stat_spactive.html(""); + } else { + stat_sp.html("Active soft prompt:"); + stat_sp.closest(".statusicon").addClass("active"); + stat_spactive.html((items[key].name || key)+" <"+key+">"); + } + if(stat_spactive.html() !== old_val) { + statFlash(stat_sp.closest(".statusicon")); + } +} + function setStartState() { enableSendBtn(); enableButtons([button_actmem, button_actwi]); @@ -1905,6 +1958,10 @@ $(document).ready(function(){ } else if(msg.cmd == "allowtoggle") { // Allow toggle change states to propagate allowtoggle = msg.data; + } else if(msg.cmd == "usstatitems") { + updateUSStatItems(msg.data, msg.flash); + } else if(msg.cmd == "spstatitems") { + updateSPStatItems(msg.data); } else if(msg.cmd == "popupshow") { // Show/Hide Popup popupShow(msg.data); @@ -2060,6 +2117,8 @@ $(document).ready(function(){ connect_status.html("Lost connection..."); connect_status.removeClass("color_green"); connect_status.addClass("color_orange"); + updateUSStatItems([], false); + updateSPStatItems({}); }); // Register editing events diff --git a/static/custom.css b/static/custom.css index 5bc6139b..d17c0c1c 100644 --- a/static/custom.css +++ b/static/custom.css @@ -29,11 +29,12 @@ chunk.editing, chunk.editing * { #topmenu { background-color: #337ab7; padding: 10px; + display: flex; } #menuitems { - display: grid; - grid-template-columns: 80% 20%; + display: flex; + width: 100%; } #navbar { @@ -66,10 +67,7 @@ chunk.editing, chunk.editing * { #connectstatusdiv { display: flex; -} - -#connectstatusdiv span { - align-self: center; + text-align: right; } #gamescreen { @@ -437,10 +435,10 @@ chunk.editing, chunk.editing * { } .colorfade, .colorfade * { - -moz-transition:color 1s ease-in; - -o-transition:color 1s ease-in; - -webkit-transition:color 1s ease-in; - transition:color 1s ease-in; + -moz-transition: color 1s ease-in, text-shadow 1s ease-in; + -o-transition: color 1s ease-in, text-shadow 1s ease-in; + -webkit-transition: color 1s ease-in, text-shadow 1s ease-in; + transition: color 1s ease-in, text-shadow 1s ease-in; } .color_orange { @@ -484,6 +482,11 @@ chunk.editing, chunk.editing * { color: #3bf723 !important; } +.status-flash, .status-flash { + color: #fce94f !important; + text-shadow: 0 0 50px #fce94f, 0 0 50px #fce94f, 0 0 10px #fce94f, 0 0 10px #fce94f, 0 0 10px #fce94f, 0 0 10px #fce94f, 0 0 10px #fce94f; +} + .flex { display: flex; align-items: center; @@ -656,11 +659,30 @@ chunk.editing, chunk.editing * { text-decoration: none; } -.helpicon:hover { - cursor: pointer; +.statusicon { + display: inline-block; + font-weight: bold; + text-align: center; + padding-left: 8px; + padding-right: 8px; + font-size: 30px !important; + font-weight: bold; + text-align: center; + font-size: 1.4ex; + line-height: 1.8ex; + text-decoration: none; + color: #68a2d4; } -.helpicon:hover .helptext { +.statusicon.active { + color: #3bf723; +} + +.helpicon:hover, .statusicon:hover { + cursor: pointer; +} + +.helpicon:hover .helptext, .statusicon:hover .statustext, .statusicon.statustoggled .statustext { display: inline-block; width: 250px; background-color: #1f2931; @@ -672,13 +694,59 @@ chunk.editing, chunk.editing * { padding: 15px; margin-left:10px; border: 1px solid #337ab7; - - position: absolute; - z-index: 1; } -.helptext { +.statusicon:hover .statustext.statustext-wide, .statusicon.statustoggled .statustext.statustext-wide { + width: 350px; +} + +.statusiconlabel { + pointer-events: none; + color: #337ab7; + text-align: center; + font-weight: bold; + font-size: 13px; +} + +#usiconlabel { + transform: translate(-3px, 10px); + -moz-transform: translate(-3px, 10px); + -webkit-transform: translate(-3px, 10px); + -ms-transform: translate(-3px, 10px); + -o-transform: translate(-3px, 10px); +} + +.status-container { + z-index: 1; + text-shadow: none !important; +} + +.helptext, .statustext { display: none; + font-family: sans-serif; + position: absolute; + z-index: 1; + text-shadow: none !important; +} + +.statustext { + transform: translate(-105%, 30px); + -moz-transform: translate(-105%, 30px); + -webkit-transform: translate(-105%, 30px); + -ms-transform: translate(-105%, 30px); + -o-transform: translate(-105%, 30px); +} + +.statusheader { + padding-bottom: 10px; +} + +#stat-usactive { + text-align: left; + height: 270px; + overflow-y: scroll; + position: relative; + padding-left: 20px; } .justifyleft { @@ -915,7 +983,7 @@ chunk.editing, chunk.editing * { } .navcontainer { - + width: 100%; } .nowrap { diff --git a/templates/index.html b/templates/index.html index 45fcb78e..12939306 100644 --- a/templates/index.html +++ b/templates/index.html @@ -10,12 +10,12 @@ - + - + @@ -80,8 +80,23 @@
    -
    - Waiting for connection... +
    + Waiting for connection... +
    + +
    +
    +
    From 2a4d7448be55dc32a39987a796290ea929644457 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Wed, 22 Dec 2021 23:46:25 -0500 Subject: [PATCH 14/18] Make `dynamic_processor_wrap` execute warper conditionally The top-k warper doesn't work properly with an argument of 0, so there is now the ability to not execute the warper if a condition is not met. --- aiserver.py | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/aiserver.py b/aiserver.py index 1a5db449..6f9bb30d 100644 --- a/aiserver.py +++ b/aiserver.py @@ -621,16 +621,19 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly", "TPUMeshTransforme # Patch transformers to use our custom logit warpers from transformers import LogitsProcessorList, LogitsWarper, LogitsProcessor, TopKLogitsWarper, TopPLogitsWarper, TemperatureLogitsWarper, RepetitionPenaltyLogitsProcessor - def dynamic_processor_wrap(cls, field_name, var_name): + def dynamic_processor_wrap(cls, field_name, var_name, cond=None): old_call = cls.__call__ def new_call(self, *args, **kwargs): setattr(self, field_name, getattr(vars, var_name)) - return old_call(self, *args, **kwargs) + assert len(args) == 2 + if(cond is None or cond(getattr(vars, var_name))): + return old_call(self, *args, **kwargs) + return args[1] cls.__call__ = new_call - dynamic_processor_wrap(RepetitionPenaltyLogitsProcessor, "penalty", "rep_pen") - dynamic_processor_wrap(TopKLogitsWarper, "top_k", "top_k") - dynamic_processor_wrap(TopPLogitsWarper, "top_p", "top_p") - dynamic_processor_wrap(TemperatureLogitsWarper, "temperature", "temp") + dynamic_processor_wrap(RepetitionPenaltyLogitsProcessor, "penalty", "rep_pen", cond=lambda x: x != 1.0) + dynamic_processor_wrap(TopKLogitsWarper, "top_k", "top_k", cond=lambda x: x > 0) + dynamic_processor_wrap(TopPLogitsWarper, "top_p", "top_p", cond=lambda x: x < 1.0) + dynamic_processor_wrap(TemperatureLogitsWarper, "temperature", "temp", cond=lambda x: x != 1.0) class TailFreeLogitsWarper(LogitsWarper): @@ -712,27 +715,17 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly", "TPUMeshTransforme new_get_logits_processor.old_get_logits_processor = transformers.generation_utils.GenerationMixin._get_logits_processor transformers.generation_utils.GenerationMixin._get_logits_processor = new_get_logits_processor - def new_get_logits_warper( - top_k: int = None, - top_p: float = None, - tfs: float = None, - temp: float = None, - beams: int = 1, - ) -> LogitsProcessorList: + def new_get_logits_warper(beams: int = 1,) -> LogitsProcessorList: warper_list = LogitsProcessorList() - warper_list.append(TopKLogitsWarper(top_k=top_k, min_tokens_to_keep=1 + (beams > 1))) - warper_list.append(TopPLogitsWarper(top_p=top_p, min_tokens_to_keep=1 + (beams > 1))) - warper_list.append(TailFreeLogitsWarper(tfs=tfs, min_tokens_to_keep=1 + (beams > 1))) - warper_list.append(TemperatureLogitsWarper(temperature=temp)) + warper_list.append(TopKLogitsWarper(top_k=1, min_tokens_to_keep=1 + (beams > 1))) + warper_list.append(TopPLogitsWarper(top_p=0.5, min_tokens_to_keep=1 + (beams > 1))) + warper_list.append(TailFreeLogitsWarper(tfs=0.5, min_tokens_to_keep=1 + (beams > 1))) + warper_list.append(TemperatureLogitsWarper(temperature=0.5)) return warper_list def new_sample(self, *args, **kwargs): assert kwargs.pop("logits_warper", None) is not None kwargs["logits_warper"] = new_get_logits_warper( - top_k=vars.top_k, - top_p=vars.top_p, - tfs=vars.tfs, - temp=vars.temp, beams=1, ) return new_sample.old_sample(self, *args, **kwargs) From 00f611b207d0f497e6f6086b464dfc55d7f87ec1 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Thu, 23 Dec 2021 13:02:11 -0500 Subject: [PATCH 15/18] Save softprompt filename in settings --- aiserver.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/aiserver.py b/aiserver.py index 771c8054..dd03de24 100644 --- a/aiserver.py +++ b/aiserver.py @@ -116,6 +116,7 @@ class vars: lua_running = False # Whether or not Lua is running (i.e. wasn't stopped due to an error) lua_edited = set() # Set of chunk numbers that were edited from a Lua generation modifier lua_deleted = set() # Set of chunk numbers that were deleted from a Lua generation modifier + spfilename = "" # Filename of soft prompt to load, or an empty string if not using a soft prompt userscripts = [] # List of userscripts to load last_userscripts = [] # List of previous userscript filenames from the previous time userscripts were send via usstatitems corescript = "default.lua" # Filename of corescript to load @@ -1539,7 +1540,7 @@ def do_connect(): emit('from_server', {'cmd': 'allowsp', 'data': vars.allowsp}) sendUSStatItems() - emit('from_server', {'cmd': 'spstatitems', 'data': {vars.spselect: vars.spmeta} if len(vars.spselect) else {}}, broadcast=True) + emit('from_server', {'cmd': 'spstatitems', 'data': {vars.spfilename: vars.spmeta} if vars.allowsp and len(vars.spfilename) else {}}, broadcast=True) if(not vars.gamestarted): setStartState() @@ -1799,7 +1800,7 @@ def get_message(msg): loadRequest(fileops.storypath(vars.loadselect)) elif(msg['cmd'] == 'sprequest'): spRequest(vars.spselect) - emit('from_server', {'cmd': 'spstatitems', 'data': {vars.spselect: vars.spmeta} if len(vars.spselect) else {}}, broadcast=True) + emit('from_server', {'cmd': 'spstatitems', 'data': {vars.spfilename: vars.spmeta} if vars.allowsp and len(vars.spfilename) else {}}, broadcast=True) elif(msg['cmd'] == 'deletestory'): deletesave(msg['data']) elif(msg['cmd'] == 'renamestory'): @@ -1905,6 +1906,7 @@ def savesettings(): js["userscripts"] = vars.userscripts js["corescript"] = vars.corescript + js["softprompt"] = vars.spfilename # Write it if not os.path.exists('settings'): @@ -1974,6 +1976,11 @@ def loadsettings(): else: vars.corescript = "default.lua" + if(vars.allowsp and "softprompt" in js and type(js["softprompt"]) is str and all(q not in js["softprompt"] for q in ("..", ":")) and all(js["softprompt"][0] not in q for q in ("/", "\\"))): + spRequest(vars.spfilename) + else: + vars.spfilename = "" + file.close() #==================================================================# @@ -3726,6 +3733,9 @@ def loadRequest(loadpath, filename=None): # Load a soft prompt from a file #==================================================================# def spRequest(filename): + vars.spfilename = "" + savesettings() + if(len(filename) == 0): vars.sp = None vars.sp_length = 0 @@ -3769,6 +3779,9 @@ def spRequest(filename): else: vars.sp = torch.from_numpy(tensor) + vars.spfilename = filename + savesettings() + #==================================================================# # Import an AIDungon game exported with Mimi's tool #==================================================================# From 4a852d7f95bf221331905b8fc6ef33046d4f750a Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Thu, 23 Dec 2021 15:43:32 -0500 Subject: [PATCH 16/18] Fix `restorePrompt()` in application.js When the prompt is deleted by the user, the topmost remaining chunk of the story that has at most one non-whitespace character is now made the new prompt chunk. Also fixed issues in Chromium-based browsers (desktop and Android) where selecting all text in the story, typing some new text to replace the entire story and then defocusing causes the editor to break. --- static/application.js | 55 +++++++++++++++++++++++++++++++++++++++---- templates/index.html | 2 +- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/static/application.js b/static/application.js index 410f2809..4e62accc 100644 --- a/static/application.js +++ b/static/application.js @@ -1476,14 +1476,59 @@ function syncAllModifiedChunks(including_selected_chunks=false) { } function restorePrompt() { - if(game_text[0].firstChild && game_text[0].firstChild.nodeType === 3) { - saved_prompt = formatChunkInnerText(game_text[0].firstChild); + if($("#n0").length && formatChunkInnerText($("#n0")[0]).length === 0) { + $("#n0").remove(); + } + var shadow_text = $("" + game_text.html() + ""); + var detected = false; + var ref = null; + try { + if(shadow_text.length && shadow_text[0].firstChild && (shadow_text[0].firstChild.nodeType === 3 || shadow_text[0].firstChild.tagName === "BR")) { + detected = true; + ref = shadow_text; + } else if(game_text.length && game_text[0].firstChild && game_text[0].firstChild.nodeType === 3 || game_text[0].firstChild.tagName === "BR") { + detected = true; + ref = game_text; + } + } catch (e) { + detected = false; + } + if(detected) { unbindGametext(); - game_text[0].innerText = ""; + var text = []; + while(true) { + if(ref.length && ref[0].firstChild && ref[0].firstChild.nodeType === 3) { + text.push(ref[0].firstChild.textContent.replace(/\u00a0/g, " ")); + } else if(ref.length && ref[0].firstChild && ref[0].firstChild.tagName === "BR") { + text.push("\n"); + } else { + break; + } + ref[0].removeChild(ref[0].firstChild); + } + text = text.join("").trim(); + if(text.length) { + saved_prompt = text; + } + game_text[0].innerHTML = ""; bindGametext(); } - if($("#n0").length) { - $("#n0").remove(); + game_text.children().each(function() { + if(this.tagName !== "CHUNK") { + this.parentNode.removeChild(this); + } + }); + if(!detected) { + game_text.children().each(function() { + if(this.innerText.trim().length) { + saved_prompt = this.innerText.trim(); + socket.send({'cmd': 'inlinedelete', 'data': this.getAttribute("n")}); + this.parentNode.removeChild(this); + return false; + } + socket.send({'cmd': 'inlinedelete', 'data': this.getAttribute("n")}); + this.parentNode.removeChild(this); + }); } var prompt_chunk = document.createElement("chunk"); prompt_chunk.setAttribute("n", "0"); diff --git a/templates/index.html b/templates/index.html index 12939306..fda8fd34 100644 --- a/templates/index.html +++ b/templates/index.html @@ -10,7 +10,7 @@ - + From c305076cf32b98843f49c62c98f6fd17b9c9af86 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Thu, 23 Dec 2021 16:14:09 -0500 Subject: [PATCH 17/18] Fix the behaviour of the `softprompt` setting --- aiserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aiserver.py b/aiserver.py index dd03de24..e6e5c80b 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1977,7 +1977,7 @@ def loadsettings(): vars.corescript = "default.lua" if(vars.allowsp and "softprompt" in js and type(js["softprompt"]) is str and all(q not in js["softprompt"] for q in ("..", ":")) and all(js["softprompt"][0] not in q for q in ("/", "\\"))): - spRequest(vars.spfilename) + spRequest(js["softprompt"]) else: vars.spfilename = "" @@ -3734,7 +3734,7 @@ def loadRequest(loadpath, filename=None): #==================================================================# def spRequest(filename): vars.spfilename = "" - savesettings() + settingschanged() if(len(filename) == 0): vars.sp = None @@ -3780,7 +3780,7 @@ def spRequest(filename): vars.sp = torch.from_numpy(tensor) vars.spfilename = filename - savesettings() + settingschanged() #==================================================================# # Import an AIDungon game exported with Mimi's tool From b2def30d9d3df44f23b74d17dffdae0482d9d048 Mon Sep 17 00:00:00 2001 From: Gnome Ann <> Date: Thu, 23 Dec 2021 16:35:52 -0500 Subject: [PATCH 18/18] Update Lua modeltype and model API --- aiserver.py | 23 ++++++++++++----------- bridge.lua | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/aiserver.py b/aiserver.py index e6e5c80b..e4280aec 100644 --- a/aiserver.py +++ b/aiserver.py @@ -82,6 +82,7 @@ class vars: submission = "" # Same as above, but after applying input formatting lastctx = "" # The last context submitted to the generator model = "" # Model ID string chosen at startup + model_orig = "" # Original model string before being changed by auto model type detection model_type = "" # Model Type (Automatically taken from the model config) noai = False # Runs the script without starting up the transformers pipeline aibusy = False # Stops submissions while the AI is working @@ -184,7 +185,7 @@ def getModelSelection(): while(vars.model == ''): modelsel = input("Model #> ") if(modelsel.isnumeric() and int(modelsel) > 0 and int(modelsel) <= len(modellist)): - vars.model = modellist[int(modelsel)-1][1] + vars.model = vars.model_orig = modellist[int(modelsel)-1][1] else: print("{0}Please enter a valid selection.{1}".format(colors.RED, colors.END)) @@ -365,7 +366,7 @@ parser.add_argument("--override_rename", action='store_true', help="Renaming sto parser.add_argument("--configname", help="Force a fixed configuration name to aid with config management.") args = parser.parse_args() -vars.model = args.model; +vars.model = vars.model_orig = args.model; if args.remote: vars.remote = True; @@ -1386,23 +1387,23 @@ def lua_get_modeltype(): return "readonly" if(vars.model in ("Colab", "OAI", "InferKit")): return "api" - if(vars.model in ("GPT2Custom", "NeoCustom")): + if(vars.model not in ("TPUMeshTransformerGPTJ",) and (vars.model in ("GPT2Custom", "NeoCustom") or vars.model_type in ("gpt2", "gpt_neo", "gptj"))): hidden_size = get_hidden_size_from_model(model) - if(vars.model in ("gpt2",) or (vars.model == "GPT2Custom" and hidden_size == 768)): + if(vars.model in ("gpt2",) or (vars.model_type == "gpt2" and hidden_size == 768)): return "gpt2" - if(vars.model in ("gpt2-medium",) or (vars.model == "GPT2Custom" and hidden_size == 1024)): + if(vars.model in ("gpt2-medium",) or (vars.model_type == "gpt2" and hidden_size == 1024)): return "gpt2-medium" - if(vars.model in ("gpt2-large",) or (vars.model == "GPT2Custom" and hidden_size == 1280)): + if(vars.model in ("gpt2-large",) or (vars.model_type == "gpt2" and hidden_size == 1280)): return "gpt2-large" - if(vars.model in ("gpt2-xl",) or (vars.model == "GPT2Custom" and hidden_size == 1600)): + if(vars.model in ("gpt2-xl",) or (vars.model_type == "gpt2" and hidden_size == 1600)): return "gpt2-xl" - if(vars.model == "NeoCustom" and hidden_size == 768): + if(vars.model_type == "gpt_neo" and hidden_size == 768): return "gpt-neo-125M" - if(vars.model in ("EleutherAI/gpt-neo-1.3B",) or (vars.model == "NeoCustom" and hidden_size == 2048)): + if(vars.model in ("EleutherAI/gpt-neo-1.3B",) or (vars.model_type == "gpt_neo" and hidden_size == 2048)): return "gpt-neo-1.3B" - if(vars.model in ("EleutherAI/gpt-neo-2.7B",) or (vars.model == "NeoCustom" and hidden_size == 2560)): + if(vars.model in ("EleutherAI/gpt-neo-2.7B",) or (vars.model_type == "gpt_neo" and hidden_size == 2560)): return "gpt-neo-2.7B" - if(vars.model in ("EleutherAI/gpt-j-6B",) or (vars.model == "NeoCustom" and hidden_size == 4096) or (vars.model == "TPUMeshTransformerGPTJ" and tpu_mtj_backend.params["d_model"] == 4096)): + if(vars.model in ("EleutherAI/gpt-j-6B",) or (vars.model == "TPUMeshTransformerGPTJ" and tpu_mtj_backend.params["d_model"] == 4096) or (vars.model_type in ("gpt_neo", "gptj") and hidden_size == 4096)): return "gpt-j-6B" return "unknown" diff --git a/bridge.lua b/bridge.lua index 2ca2eed0..39acc222 100644 --- a/bridge.lua +++ b/bridge.lua @@ -1038,7 +1038,7 @@ return function(_python, _bridged) ---@param t KoboldLib ---@return string function KoboldLib_getters.model(t) - return bridged.vars.model + return bridged.vars.model_orig end ---@param t KoboldLib