mirror of
https://github.com/KoboldAI/KoboldAI-Client.git
synced 2025-02-17 04:00:44 +01:00
Merge branch 'united' into patch
This commit is contained in:
Gnome Ann
commit
03b16ed920
1
.gitattributes
vendored
1
.gitattributes
vendored
@ -1 +1,2 @@
|
||||
*.min.lua linguist-vendored
|
||||
*documentation.html linguist-vendored
|
||||
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@ -13,8 +13,11 @@ miniconda3/*
|
||||
__pycache__
|
||||
*.log
|
||||
cache/*
|
||||
userscripts/*.*
|
||||
userscripts/*
|
||||
!userscripts/examples
|
||||
!userscripts/kaipreset_*.lua
|
||||
!userscripts/Readme.*
|
||||
!userscripts/api_documentation.*
|
||||
softprompts/*
|
||||
|
||||
# Ignore PyCharm project files.
|
||||
|
||||
230
aiserver.py
230
aiserver.py
@ -22,7 +22,7 @@ import packaging
|
||||
import contextlib
|
||||
import traceback
|
||||
import threading
|
||||
from typing import Any, Callable, TypeVar, Union, Dict, Set, List
|
||||
from typing import Any, Callable, TypeVar, Tuple, Union, Dict, Set, List
|
||||
|
||||
import requests
|
||||
import html
|
||||
@ -69,7 +69,7 @@ modellist = [
|
||||
["C1 6B (Chatbot)", "hakurei/c1-6B", "12GB"],
|
||||
["Picard 2.7B (Novel)", "KoboldAI/GPT-Neo-2.7B-Picard", "6GB"],
|
||||
["Horni 2.7B (NSFW)", "KoboldAI/GPT-Neo-2.7B-Horni", "6GB"],
|
||||
["Horni-LN 2.7B (Novel/NSFW)", "KoboldAI/GPT-Neo-2.7B-Horni-LN", "6GB"],
|
||||
["Horni-LN 2.7B (Novel)", "KoboldAI/GPT-Neo-2.7B-Horni-LN", "6GB"],
|
||||
["Shinen 2.7B (NSFW)", "KoboldAI/GPT-Neo-2.7B-Shinen", "6GB"],
|
||||
["GPT-J 6B", "EleutherAI/gpt-j-6B", "12GB"],
|
||||
["GPT-Neo 2.7B", "EleutherAI/gpt-neo-2.7B", "6GB"],
|
||||
@ -158,6 +158,7 @@ class vars:
|
||||
spmeta = None # Metadata of current soft prompt, or None if not using a soft prompt
|
||||
sp = None # Current soft prompt tensor (as a NumPy array)
|
||||
sp_length = 0 # Length of current soft prompt in tokens, or 0 if not using a soft prompt
|
||||
has_genmod = False # Whether or not at least one loaded Lua userscript has a generation modifier
|
||||
svowname = "" # Filename that was flagged for overwrite confirm
|
||||
saveow = False # Whether or not overwrite confirm has been displayed
|
||||
genseqs = [] # Temporary storage for generated sequences
|
||||
@ -185,6 +186,7 @@ class vars:
|
||||
remote = False
|
||||
nopromptgen = False
|
||||
rngpersist = False
|
||||
nogenmod = False
|
||||
|
||||
#==================================================================#
|
||||
# Function to get model selection at startup
|
||||
@ -387,6 +389,7 @@ parser.add_argument("--breakmodel_gpulayers", type=str, help="If using a model t
|
||||
parser.add_argument("--override_delete", action='store_true', help="Deleting stories from inside the browser is disabled if you are using --remote and enabled otherwise. Using this option will instead allow deleting stories if using --remote and prevent deleting stories otherwise.")
|
||||
parser.add_argument("--override_rename", action='store_true', help="Renaming stories from inside the browser is disabled if you are using --remote and enabled otherwise. Using this option will instead allow renaming stories if using --remote and prevent renaming stories otherwise.")
|
||||
parser.add_argument("--configname", help="Force a fixed configuration name to aid with config management.")
|
||||
parser.add_argument("--colab", action='store_true', help="Optimize for Google Colab.")
|
||||
|
||||
args: argparse.Namespace = None
|
||||
if(os.environ.get("KOBOLDAI_ARGS") is not None):
|
||||
@ -394,8 +397,14 @@ if(os.environ.get("KOBOLDAI_ARGS") is not None):
|
||||
args = parser.parse_args(shlex.split(os.environ["KOBOLDAI_ARGS"]))
|
||||
else:
|
||||
args = parser.parse_args()
|
||||
|
||||
vars.model = args.model;
|
||||
|
||||
if args.colab:
|
||||
args.remote = True;
|
||||
args.override_rename = True;
|
||||
args.override_delete = True;
|
||||
|
||||
if args.remote:
|
||||
vars.remote = True;
|
||||
|
||||
@ -453,7 +462,7 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly", "TPUMeshTransforme
|
||||
vars.model_type = "gpt_neo"
|
||||
print("{0}Looking for GPU support...{1}".format(colors.PURPLE, colors.END), end="")
|
||||
vars.hascuda = torch.cuda.is_available()
|
||||
vars.bmsupported = vars.model_type in ("gpt_neo", "gptj")
|
||||
vars.bmsupported = vars.model_type in ("gpt_neo", "gptj") and not args.colab
|
||||
if(args.breakmodel is not None and args.breakmodel):
|
||||
print("WARNING: --breakmodel is no longer supported. Breakmodel mode is now automatically enabled when --layers is used (see --help for details).", file=sys.stderr)
|
||||
if(args.breakmodel_layers is not None):
|
||||
@ -930,7 +939,6 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly", "TPUMeshTransforme
|
||||
except ValueError as e:
|
||||
model = GPTNeoForCausalLM.from_pretrained(vars.model.replace('/', '_'), cache_dir="cache/", **lowmem)
|
||||
else:
|
||||
print("Model does not exist locally, attempting to download from Huggingface...")
|
||||
try:
|
||||
tokenizer = AutoTokenizer.from_pretrained(vars.model, cache_dir="cache/")
|
||||
except ValueError as e:
|
||||
@ -940,11 +948,13 @@ if(not vars.model in ["InferKit", "Colab", "OAI", "ReadOnly", "TPUMeshTransforme
|
||||
model = AutoModelForCausalLM.from_pretrained(vars.model, cache_dir="cache/", **lowmem)
|
||||
except ValueError as e:
|
||||
model = GPTNeoForCausalLM.from_pretrained(vars.model, cache_dir="cache/", **lowmem)
|
||||
model = model.half()
|
||||
import shutil
|
||||
shutil.rmtree("cache/")
|
||||
model.save_pretrained(vars.model.replace('/', '_'))
|
||||
tokenizer.save_pretrained(vars.model.replace('/', '_'))
|
||||
|
||||
if not args.colab:
|
||||
model = model.half()
|
||||
import shutil
|
||||
shutil.rmtree("cache/")
|
||||
model.save_pretrained(vars.model.replace('/', '_'))
|
||||
tokenizer.save_pretrained(vars.model.replace('/', '_'))
|
||||
|
||||
if(vars.hascuda):
|
||||
if(vars.usegpu):
|
||||
@ -991,7 +1001,7 @@ else:
|
||||
-1,
|
||||
tpu_mtj_backend.params["d_model"],
|
||||
)
|
||||
vars.sp = tensor
|
||||
vars.sp = tpu_mtj_backend.shard_xmap(tensor)
|
||||
soft_tokens = np.arange(
|
||||
tpu_mtj_backend.params["n_vocab"] + tpu_mtj_backend.params["n_vocab_padding"],
|
||||
tpu_mtj_backend.params["n_vocab"] + tpu_mtj_backend.params["n_vocab_padding"] + vars.sp_length,
|
||||
@ -999,6 +1009,49 @@ else:
|
||||
)
|
||||
return soft_tokens
|
||||
|
||||
def tpumtjgenerate_warper_callback(scores) -> "np.array":
|
||||
scores_shape = scores.shape
|
||||
scores_list = scores.tolist()
|
||||
vars.lua_koboldbridge.logits = vars.lua_state.table()
|
||||
for r, row in enumerate(scores_list):
|
||||
vars.lua_koboldbridge.logits[r+1] = vars.lua_state.table(*row)
|
||||
vars.lua_koboldbridge.vocab_size = scores_shape[-1]
|
||||
|
||||
execute_genmod()
|
||||
|
||||
scores = np.array(
|
||||
tuple(tuple(row.values()) for row in vars.lua_koboldbridge.logits.values()),
|
||||
dtype=scores.dtype,
|
||||
)
|
||||
assert scores.shape == scores_shape
|
||||
|
||||
return scores
|
||||
|
||||
def tpumtjgenerate_stopping_callback(generated, n_generated, excluded_world_info) -> Tuple[List[set], bool, bool]:
|
||||
vars.generated_tkns += 1
|
||||
|
||||
assert len(excluded_world_info) == len(generated)
|
||||
regeneration_required = vars.lua_koboldbridge.regeneration_required
|
||||
halt = not vars.lua_koboldbridge.generating or vars.generated_tkns >= vars.genamt
|
||||
vars.lua_koboldbridge.regeneration_required = False
|
||||
|
||||
global past
|
||||
|
||||
for i in range(vars.numseqs):
|
||||
vars.lua_koboldbridge.generated[i+1][vars.generated_tkns] = int(generated[i, tpu_mtj_backend.params["seq"] + n_generated - 1].item())
|
||||
|
||||
if(not vars.dynamicscan or halt):
|
||||
return excluded_world_info, regeneration_required, halt
|
||||
|
||||
for i, t in enumerate(generated):
|
||||
decoded = tokenizer.decode(past[i]) + tokenizer.decode(t[tpu_mtj_backend.params["seq"] : tpu_mtj_backend.params["seq"] + n_generated])
|
||||
_, found = checkworldinfo(decoded, force_use_txt=True)
|
||||
found -= excluded_world_info[i]
|
||||
if(len(found) != 0):
|
||||
regeneration_required = True
|
||||
break
|
||||
return excluded_world_info, regeneration_required, halt
|
||||
|
||||
# If we're running Colab or OAI, we still need a tokenizer.
|
||||
if(vars.model == "Colab"):
|
||||
from transformers import GPT2TokenizerFast
|
||||
@ -1011,21 +1064,12 @@ else:
|
||||
print("{0}Initializing Mesh Transformer JAX, please wait...{1}".format(colors.PURPLE, colors.END))
|
||||
assert vars.model == "TPUMeshTransformerGPTJ" and vars.custmodpth and os.path.isdir(vars.custmodpth)
|
||||
import tpu_mtj_backend
|
||||
tpu_mtj_backend.warper_callback = tpumtjgenerate_warper_callback
|
||||
tpu_mtj_backend.stopping_callback = tpumtjgenerate_stopping_callback
|
||||
tpu_mtj_backend.load_model(vars.custmodpth)
|
||||
vars.allowsp = True
|
||||
vars.modeldim = int(tpu_mtj_backend.params["d_model"])
|
||||
tokenizer = tpu_mtj_backend.tokenizer
|
||||
soft_tokens = tpumtjgetsofttokens()
|
||||
threading.Thread( # Compile backend code in background
|
||||
target=tpu_mtj_backend.infer,
|
||||
args=(np.uint32((23403, 727, 20185)),),
|
||||
kwargs={
|
||||
"soft_embeddings": vars.sp,
|
||||
"soft_tokens": soft_tokens,
|
||||
"gen_len": 1,
|
||||
"numseqs": vars.numseqs,
|
||||
},
|
||||
).start()
|
||||
|
||||
# Set up Flask routes
|
||||
@app.route('/')
|
||||
@ -1133,13 +1177,18 @@ def load_lua_scripts():
|
||||
modulenames.append(lst[i]["modulename"])
|
||||
descriptions.append(lst[i]["description"])
|
||||
|
||||
vars.has_genmod = False
|
||||
|
||||
try:
|
||||
vars.lua_koboldbridge.obliterate_multiverse()
|
||||
tpool.execute(vars.lua_koboldbridge.load_corescript, vars.corescript)
|
||||
tpool.execute(vars.lua_koboldbridge.load_userscripts, filenames, modulenames, descriptions)
|
||||
vars.has_genmod = tpool.execute(vars.lua_koboldbridge.load_userscripts, filenames, modulenames, descriptions)
|
||||
vars.lua_running = True
|
||||
except lupa.LuaError as e:
|
||||
vars.lua_koboldbridge.obliterate_multiverse()
|
||||
try:
|
||||
vars.lua_koboldbridge.obliterate_multiverse()
|
||||
except:
|
||||
pass
|
||||
vars.lua_running = False
|
||||
if(vars.serverstarted):
|
||||
emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error; please check console.'}, broadcast=True)
|
||||
@ -1996,6 +2045,10 @@ def get_message(msg):
|
||||
vars.rngpersist = msg['data']
|
||||
settingschanged()
|
||||
refresh_settings()
|
||||
elif(msg['cmd'] == 'setnogenmod'):
|
||||
vars.nogenmod = msg['data']
|
||||
settingschanged()
|
||||
refresh_settings()
|
||||
elif(not vars.remote and msg['cmd'] == 'importwi'):
|
||||
wiimportrequest()
|
||||
|
||||
@ -2066,6 +2119,8 @@ def savesettings():
|
||||
js["dynamicscan"] = vars.dynamicscan
|
||||
js["nopromptgen"] = vars.nopromptgen
|
||||
js["rngpersist"] = vars.rngpersist
|
||||
js["nogenmod"] = vars.nogenmod
|
||||
|
||||
js["antemplate"] = vars.setauthornotetemplate
|
||||
|
||||
js["userscripts"] = vars.userscripts
|
||||
@ -2131,6 +2186,8 @@ def loadsettings():
|
||||
vars.nopromptgen = js["nopromptgen"]
|
||||
if("rngpersist" in js):
|
||||
vars.rngpersist = js["rngpersist"]
|
||||
if("nogenmod" in js):
|
||||
vars.nogenmod = js["nogenmod"]
|
||||
|
||||
if("antemplate" in js):
|
||||
vars.setauthornotetemplate = js["antemplate"]
|
||||
@ -2896,32 +2953,90 @@ def sendtocolab(txt, min, max):
|
||||
# Send text to TPU mesh transformer backend
|
||||
#==================================================================#
|
||||
def tpumtjgenerate(txt, minimum, maximum, found_entries=None):
|
||||
vars.generated_tkns = 0
|
||||
|
||||
if(found_entries is None):
|
||||
found_entries = set()
|
||||
found_entries = tuple(found_entries.copy() for _ in range(vars.numseqs))
|
||||
|
||||
print("{0}Min:{1}, Max:{2}, Txt:{3}{4}".format(colors.YELLOW, minimum, maximum, tokenizer.decode(txt), colors.END))
|
||||
|
||||
vars._actions = vars.actions
|
||||
vars._prompt = vars.prompt
|
||||
if(vars.dynamicscan):
|
||||
vars._actions = vars._actions.copy()
|
||||
|
||||
# Submit input text to generator
|
||||
try:
|
||||
if(vars.dynamicscan):
|
||||
raise ValueError("Dynamic world info scanning is not supported by the TPU backend yet")
|
||||
|
||||
soft_tokens = tpumtjgetsofttokens()
|
||||
|
||||
genout = tpool.execute(
|
||||
tpu_mtj_backend.infer,
|
||||
np.uint32(txt),
|
||||
gen_len = maximum-minimum+1,
|
||||
temp=vars.temp,
|
||||
top_p=vars.top_p,
|
||||
top_k=vars.top_k,
|
||||
tfs=vars.tfs,
|
||||
numseqs=vars.numseqs,
|
||||
repetition_penalty=vars.rep_pen,
|
||||
soft_embeddings=vars.sp,
|
||||
soft_tokens=soft_tokens,
|
||||
)
|
||||
global past
|
||||
|
||||
if(vars.dynamicscan or (not vars.nogenmod and vars.has_genmod)):
|
||||
|
||||
context = np.tile(np.uint32(txt), (vars.numseqs, 1))
|
||||
past = np.empty((vars.numseqs, 0), dtype=np.uint32)
|
||||
|
||||
while(True):
|
||||
genout, n_generated, regeneration_required, halt = tpool.execute(
|
||||
tpu_mtj_backend.infer_dynamic,
|
||||
context,
|
||||
gen_len = maximum-minimum+1,
|
||||
temp=vars.temp,
|
||||
top_p=vars.top_p,
|
||||
top_k=vars.top_k,
|
||||
tfs=vars.tfs,
|
||||
numseqs=vars.numseqs,
|
||||
repetition_penalty=vars.rep_pen,
|
||||
soft_embeddings=vars.sp,
|
||||
soft_tokens=soft_tokens,
|
||||
excluded_world_info=found_entries,
|
||||
)
|
||||
|
||||
past = np.pad(past, ((0, 0), (0, n_generated)))
|
||||
for r in range(vars.numseqs):
|
||||
for c in range(vars.lua_koboldbridge.generated_cols):
|
||||
assert vars.lua_koboldbridge.generated[r+1][c+1] is not None
|
||||
past[r, c] = vars.lua_koboldbridge.generated[r+1][c+1]
|
||||
|
||||
if(halt or not regeneration_required):
|
||||
break
|
||||
print("(regeneration triggered)")
|
||||
|
||||
encoded = []
|
||||
for i in range(vars.numseqs):
|
||||
txt = tokenizer.decode(past[i])
|
||||
winfo, mem, anotetxt, _found_entries = calcsubmitbudgetheader(txt, force_use_txt=True)
|
||||
found_entries[i].update(_found_entries)
|
||||
txt, _, _ = calcsubmitbudget(len(vars._actions), winfo, mem, anotetxt, vars._actions, submission=txt)
|
||||
encoded.append(np.array(txt, dtype=np.uint32))
|
||||
max_length = len(max(encoded, key=len))
|
||||
encoded = np.stack(tuple(np.pad(e, (max_length - len(e), 0), constant_values=tpu_mtj_backend.pad_token_id) for e in encoded))
|
||||
context = np.concatenate(
|
||||
(
|
||||
encoded,
|
||||
past,
|
||||
),
|
||||
axis=-1,
|
||||
)
|
||||
|
||||
else:
|
||||
genout = tpool.execute(
|
||||
tpu_mtj_backend.infer_static,
|
||||
np.uint32(txt),
|
||||
gen_len = maximum-minimum+1,
|
||||
temp=vars.temp,
|
||||
top_p=vars.top_p,
|
||||
top_k=vars.top_k,
|
||||
tfs=vars.tfs,
|
||||
numseqs=vars.numseqs,
|
||||
repetition_penalty=vars.rep_pen,
|
||||
soft_embeddings=vars.sp,
|
||||
soft_tokens=soft_tokens,
|
||||
)
|
||||
past = genout
|
||||
for i in range(vars.numseqs):
|
||||
vars.lua_koboldbridge.generated[i+1] = vars.lua_state.table(*genout[i].tolist())
|
||||
|
||||
except Exception as e:
|
||||
if(issubclass(type(e), lupa.LuaError)):
|
||||
@ -2937,10 +3052,10 @@ def tpumtjgenerate(txt, minimum, maximum, found_entries=None):
|
||||
print("{0}{1}{2}".format(colors.RED, traceback.format_exc().replace("\033", ""), colors.END), file=sys.stderr)
|
||||
set_aibusy(0)
|
||||
return
|
||||
|
||||
|
||||
for i in range(vars.numseqs):
|
||||
vars.lua_koboldbridge.generated[i+1] = vars.lua_state.table(*genout[i].tolist())
|
||||
vars.lua_koboldbridge.outputs[i+1] = tokenizer.decode(genout[i])
|
||||
vars.lua_koboldbridge.outputs[i+1] = tokenizer.decode(past[i])
|
||||
genout = past
|
||||
|
||||
execute_outmod()
|
||||
if(vars.lua_koboldbridge.regeneration_required):
|
||||
@ -3103,6 +3218,7 @@ def refresh_settings():
|
||||
emit('from_server', {'cmd': 'updatedynamicscan', 'data': vars.dynamicscan}, broadcast=True)
|
||||
emit('from_server', {'cmd': 'updatenopromptgen', 'data': vars.nopromptgen}, broadcast=True)
|
||||
emit('from_server', {'cmd': 'updaterngpersist', 'data': vars.rngpersist}, broadcast=True)
|
||||
emit('from_server', {'cmd': 'updatenogenmod', 'data': vars.nogenmod}, broadcast=True)
|
||||
|
||||
emit('from_server', {'cmd': 'updatefrmttriminc', 'data': vars.formatoptns["frmttriminc"]}, broadcast=True)
|
||||
emit('from_server', {'cmd': 'updatefrmtrmblln', 'data': vars.formatoptns["frmtrmblln"]}, broadcast=True)
|
||||
@ -4014,7 +4130,7 @@ def spRequest(filename):
|
||||
-1,
|
||||
tpu_mtj_backend.params["d_model"],
|
||||
)
|
||||
vars.sp = np.float32(tensor)
|
||||
vars.sp = tpu_mtj_backend.shard_xmap(np.float32(tensor))
|
||||
else:
|
||||
vars.sp = torch.from_numpy(tensor)
|
||||
|
||||
@ -4359,6 +4475,34 @@ def randomGameRequest(topic, memory=""):
|
||||
loadmodelsettings()
|
||||
loadsettings()
|
||||
|
||||
# Precompile TPU backend if required
|
||||
if(vars.model in ("TPUMeshTransformerGPTJ",)):
|
||||
soft_tokens = tpumtjgetsofttokens()
|
||||
if(vars.dynamicscan or (not vars.nogenmod and vars.has_genmod)):
|
||||
threading.Thread(
|
||||
target=tpu_mtj_backend.infer_dynamic,
|
||||
args=(np.tile(np.uint32((23403, 727, 20185)), (vars.numseqs, 1)),),
|
||||
kwargs={
|
||||
"soft_embeddings": vars.sp,
|
||||
"soft_tokens": soft_tokens,
|
||||
"gen_len": 1,
|
||||
"use_callback": False,
|
||||
"numseqs": vars.numseqs,
|
||||
"excluded_world_info": list(set() for _ in range(vars.numseqs)),
|
||||
},
|
||||
).start()
|
||||
else:
|
||||
threading.Thread(
|
||||
target=tpu_mtj_backend.infer_static,
|
||||
args=(np.uint32((23403, 727, 20185)),),
|
||||
kwargs={
|
||||
"soft_embeddings": vars.sp,
|
||||
"soft_tokens": soft_tokens,
|
||||
"gen_len": 1,
|
||||
"numseqs": vars.numseqs,
|
||||
},
|
||||
).start()
|
||||
|
||||
#==================================================================#
|
||||
# Final startup commands to launch Flask app
|
||||
#==================================================================#
|
||||
|
||||
13
bridge.lua
13
bridge.lua
@ -1851,13 +1851,14 @@ return function(_python, _bridged)
|
||||
-- API for aiserver.py
|
||||
--==========================================================================
|
||||
|
||||
---@return nil
|
||||
---@return boolean
|
||||
function koboldbridge.load_userscripts(filenames, modulenames, descriptions)
|
||||
config_files = {}
|
||||
config_file_filename_map = {}
|
||||
koboldbridge.userscripts = {}
|
||||
koboldbridge.userscriptmodule_filename_map = {}
|
||||
koboldbridge.num_userscripts = 0
|
||||
local has_genmod = false
|
||||
for i, filename in _python.enumerate(filenames) do
|
||||
bridged.load_callback(filename, modulenames[i])
|
||||
koboldbridge.logging_name = modulenames[i]
|
||||
@ -1865,12 +1866,15 @@ return function(_python, _bridged)
|
||||
local f, err = old_loadfile(join_folder_and_filename(bridged.userscript_path, filename), "t", koboldbridge.get_universe(filename))
|
||||
if err ~= nil then
|
||||
error(err)
|
||||
return
|
||||
return false
|
||||
end
|
||||
---@type KoboldUserScript
|
||||
local _userscript = f()
|
||||
koboldbridge.logging_name = nil
|
||||
koboldbridge.filename = nil
|
||||
if _userscript.genmod ~= nil then
|
||||
has_genmod = true
|
||||
end
|
||||
local userscript = deepcopy(KoboldUserScriptModule)
|
||||
rawset(userscript, "_inmod", function()
|
||||
koboldbridge.logging_name = modulenames[i]
|
||||
@ -1903,6 +1907,7 @@ return function(_python, _bridged)
|
||||
koboldbridge.userscriptmodule_filename_map[userscript] = filename
|
||||
koboldbridge.num_userscripts = i + 1
|
||||
end
|
||||
return has_genmod
|
||||
end
|
||||
|
||||
---@return nil
|
||||
@ -1949,7 +1954,9 @@ return function(_python, _bridged)
|
||||
koboldbridge.userstate = "genmod"
|
||||
if koboldbridge.genmod ~= nil then
|
||||
local _generated = deepcopy(koboldbridge.generated)
|
||||
r = koboldbridge.genmod()
|
||||
if not bridged.vars.nogenmod then
|
||||
r = koboldbridge.genmod()
|
||||
end
|
||||
setmetatable(koboldbridge.logits, nil)
|
||||
for kr, vr in old_next, koboldbridge.logits, nil do
|
||||
setmetatable(vr, nil)
|
||||
|
||||
@ -106,9 +106,9 @@ if [ "$init" != "skip" ]; then
|
||||
|
||||
cd /content/KoboldAI-Client
|
||||
|
||||
cp -rn stories/* /content/drive/MyDrive/KoboldAI/stories/
|
||||
cp -rn userscripts/* /content/drive/MyDrive/KoboldAI/userscripts/
|
||||
cp -rn softprompts/* /content/drive/MyDrive/KoboldAI/softprompts/
|
||||
cp -rn stories/*.* /content/drive/MyDrive/KoboldAI/stories/
|
||||
cp -rn userscripts/*.* /content/drive/MyDrive/KoboldAI/userscripts/
|
||||
cp -rn softprompts/*.* /content/drive/MyDrive/KoboldAI/softprompts/
|
||||
rm stories
|
||||
rm -rf stories/
|
||||
rm userscripts
|
||||
|
||||
@ -4,6 +4,7 @@ TITLE CMD for KoboldAI Runtime
|
||||
SET /P M=<loader.settings
|
||||
IF %M%==1 GOTO drivemap
|
||||
IF %M%==2 GOTO subfolder
|
||||
IF %M%==3 GOTO drivemap_B
|
||||
|
||||
:subfolder
|
||||
SET TEMP=%~DP0MINICONDA3
|
||||
@ -16,4 +17,11 @@ subst K: miniconda3 >nul
|
||||
SET TEMP=K:\
|
||||
SET TMP=K:\
|
||||
call K:\python\condabin\activate
|
||||
cmd /k
|
||||
|
||||
:drivemap_B
|
||||
subst B: miniconda3 >nul
|
||||
SET TEMP=B:\
|
||||
SET TMP=B:\
|
||||
call B:\python\condabin\activate
|
||||
cmd /k
|
||||
@ -7,6 +7,7 @@ services:
|
||||
network_mode: "host"
|
||||
volumes:
|
||||
- /tmp/.X11-unix:/tmp/.X11-unix
|
||||
- /etc/protocols:/etc/protocols:ro
|
||||
- ../:/content/
|
||||
- $HOME/.Xauthority:/home/micromamba/.Xauthority:rw
|
||||
devices:
|
||||
|
||||
@ -7,6 +7,7 @@ services:
|
||||
network_mode: "host"
|
||||
volumes:
|
||||
- /tmp/.X11-unix:/tmp/.X11-unix
|
||||
- /etc/protocols:/etc/protocols:ro
|
||||
- ../:/content/
|
||||
- $HOME/.Xauthority:/root/.Xauthority:rw
|
||||
devices:
|
||||
|
||||
@ -162,6 +162,17 @@ gensettingstf = [{
|
||||
"step": 1,
|
||||
"default": 0,
|
||||
"tooltip": "When enabled, the Memory text box in the Random Story dialog will be prefilled by default with your current story's memory instead of being empty."
|
||||
},
|
||||
{
|
||||
"uitype": "toggle",
|
||||
"unit": "bool",
|
||||
"label": "No Genmod",
|
||||
"id": "setnogenmod",
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"step": 1,
|
||||
"default": 0,
|
||||
"tooltip": "Disables userscript generation modifiers."
|
||||
}]
|
||||
|
||||
gensettingsik =[{
|
||||
|
||||
@ -27,7 +27,7 @@ IF %D%==1 rmdir /s /q miniconda3
|
||||
|
||||
:Mode
|
||||
echo Which installation mode would you like?
|
||||
echo 1. Temporary Drive Letter (Mounts the folder as drive K:, more stable and portable)
|
||||
echo 1. Temporary Drive Letter (Mounts the folder as drive B:, more stable and portable)
|
||||
echo 2. Subfolder (Traditional method, can't run in folder paths that contain spaces)
|
||||
echo.
|
||||
SET /P M=Type the number of the desired option and then press ENTER:
|
||||
@ -38,20 +38,20 @@ GOTO MODE
|
||||
|
||||
|
||||
:drivemap
|
||||
echo 1 > loader.settings
|
||||
subst K: /D >nul
|
||||
echo 3 > loader.settings
|
||||
subst B: /D >nul
|
||||
mkdir miniconda3
|
||||
subst K: miniconda3
|
||||
SET TEMP=K:\
|
||||
SET TMP=K:\
|
||||
copy umamba.exe K:\umamba.exe
|
||||
K:
|
||||
umamba.exe create -r K:\python\ -n base
|
||||
IF %B%==1 umamba.exe install --no-shortcuts -r K:\python\ -n base -f "%~dp0\environments\huggingface.yml" -y --always-copy
|
||||
IF %B%==2 umamba.exe install --no-shortcuts -r K:\python\ -n base -f "%~dp0\environments\finetuneanon.yml" -y --always-copy
|
||||
umamba.exe -r K:\ clean -a -y
|
||||
rd K:\Python\pkgs /S /Q
|
||||
subst K: /d
|
||||
subst B: miniconda3
|
||||
SET TEMP=B:\
|
||||
SET TMP=B:\
|
||||
copy umamba.exe B:\umamba.exe
|
||||
B:
|
||||
umamba.exe create -r B:\python\ -n base
|
||||
IF %B%==1 umamba.exe install --no-shortcuts -r B:\python\ -n base -f "%~dp0\environments\huggingface.yml" -y --always-copy
|
||||
IF %B%==2 umamba.exe install --no-shortcuts -r B:\python\ -n base -f "%~dp0\environments\finetuneanon.yml" -y --always-copy
|
||||
umamba.exe -r B:\ clean -a -y
|
||||
rd B:\Python\pkgs /S /Q
|
||||
subst B: /d
|
||||
pause
|
||||
exit
|
||||
|
||||
|
||||
@ -4,6 +4,7 @@ TITLE Jupyter for KoboldAI Runtime
|
||||
SET /P M=<loader.settings
|
||||
IF %M%==1 GOTO drivemap
|
||||
IF %M%==2 GOTO subfolder
|
||||
IF %M%==3 GOTO drivemap_B
|
||||
|
||||
:subfolder
|
||||
umamba.exe install --no-shortcuts -r miniconda3 -n base -c conda-forge jupyter
|
||||
@ -17,4 +18,12 @@ umamba.exe install --no-shortcuts -r K:\python\ -n base -c conda-forge jupyter
|
||||
call K:\python\condabin\activate
|
||||
jupyter notebook
|
||||
subst K: /D
|
||||
cmd /k
|
||||
|
||||
:drivemap_B
|
||||
subst B: miniconda3 >nul
|
||||
umamba.exe install --no-shortcuts -r B:\python\ -n base -c conda-forge jupyter
|
||||
call B:\python\condabin\activate
|
||||
jupyter notebook
|
||||
subst B: /D
|
||||
cmd /k
|
||||
11
play.bat
11
play.bat
@ -4,6 +4,7 @@ TITLE KoboldAI - Server
|
||||
SET /P M=<loader.settings
|
||||
IF %M%==1 GOTO drivemap
|
||||
IF %M%==2 GOTO subfolder
|
||||
IF %M%==3 GOTO drivemap_B
|
||||
|
||||
:subfolder
|
||||
ECHO Runtime launching in subfolder mode
|
||||
@ -22,3 +23,13 @@ call K:\python\condabin\activate
|
||||
python aiserver.py %*
|
||||
subst K: /D
|
||||
cmd /k
|
||||
|
||||
:drivemap_B
|
||||
ECHO Runtime launching in B: drive mode
|
||||
subst B: miniconda3 >nul
|
||||
SET TEMP=B:\
|
||||
SET TMP=B:\
|
||||
call B:\python\condabin\activate
|
||||
python aiserver.py %*
|
||||
subst B: /D
|
||||
cmd /k
|
||||
@ -2200,6 +2200,9 @@ $(document).ready(function(){
|
||||
if(!$("#setrngpersist").prop("checked")) {
|
||||
$("#rngmemory").val("");
|
||||
}
|
||||
} else if(msg.cmd == "updatenogenmod") {
|
||||
// Update toggle state
|
||||
$("#setnogenmod").prop('checked', msg.data).change();
|
||||
} else if(msg.cmd == "runs_remotely") {
|
||||
remote = true;
|
||||
hide([button_savetofile, button_import, button_importwi]);
|
||||
|
||||
@ -192,6 +192,7 @@ body.connected #formatmenu, #formatmenu.always-available {
|
||||
overflow:auto;
|
||||
background-color: #404040;
|
||||
color: #ffffff;
|
||||
resize: vertical;
|
||||
}
|
||||
|
||||
#btnmode {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import multiprocessing
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar
|
||||
import progressbar
|
||||
import time
|
||||
import os
|
||||
@ -20,6 +20,13 @@ from mesh_transformer.transformer_shard import CausalTransformer, CausalTransfor
|
||||
params: Dict[str, Any] = {}
|
||||
|
||||
|
||||
def warper_callback(logits) -> np.array:
|
||||
raise NotImplementedError("`tpu_mtj_backend.warper_callback()` needs to be defined")
|
||||
|
||||
def stopping_callback(generated, n_generated, excluded_world_info) -> Tuple[List[set], bool, bool]:
|
||||
raise NotImplementedError("`tpu_mtj_backend.stopping_callback()` needs to be defined")
|
||||
|
||||
|
||||
def show_spinner():
|
||||
bar = progressbar.ProgressBar(max_value=progressbar.UnknownLength, widgets=[progressbar.Timer(), ' ', progressbar.BouncingBar(left='[', right=']', marker='█')])
|
||||
i = 0
|
||||
@ -28,7 +35,155 @@ def show_spinner():
|
||||
time.sleep(0.1)
|
||||
i += 1
|
||||
|
||||
def apply_repetition_penalty(logits, tokens, repetition_penalty):
|
||||
|
||||
__F = TypeVar("__F", bound=Callable)
|
||||
__T = TypeVar("__T")
|
||||
|
||||
def __move_xmap(f: __F, out_axis: str) -> __F:
|
||||
return maps.xmap(
|
||||
f,
|
||||
in_axes=(["shard", ...], ["batch", ...]),
|
||||
out_axes=[out_axis, ...],
|
||||
axis_resources={'shard': 'mp', 'batch': 'dp'},
|
||||
)
|
||||
|
||||
def __shard_xmap(batch_dim=1):
|
||||
xmap = __move_xmap(lambda s, b: s, "shard")
|
||||
def inner(x: __T) -> __T:
|
||||
return xmap(x, np.empty(batch_dim))
|
||||
return inner
|
||||
|
||||
def __batch_xmap(shard_dim=1):
|
||||
xmap = __move_xmap(lambda s, b: b, "batch")
|
||||
def inner(x: __T) -> __T:
|
||||
return xmap(np.empty(shard_dim), x)
|
||||
return inner
|
||||
|
||||
|
||||
def apply_repetition_penalty_dynamic(logits, tokens, repetition_penalty):
|
||||
'''
|
||||
This gets called by generate_loop_fn to apply repetition penalty
|
||||
to the 1D array logits using the provided 1D array of tokens to penalize
|
||||
'''
|
||||
# Make a new array with the same length as the tokens array but with
|
||||
# each element replaced by the value at the corresponding index in the
|
||||
# logits array; e.g.
|
||||
# if logits is [77, 5, 3, 98] and tokens is [0, 1, 2, 3, 2, 3, 1],
|
||||
# then penalty_logits will be [77, 5, 3, 98, 3, 98, 5]
|
||||
penalty_logits = np.take(logits, tokens)
|
||||
# Divide positive values by repetition_penalty and multiply negative
|
||||
# values by repetition_penalty (the academic publication that described
|
||||
# this technique actually just only divided, but that would cause tokens
|
||||
# with negative logits to become more likely, which is obviously wrong)
|
||||
penalty_logits = np.where(
|
||||
penalty_logits > 0,
|
||||
penalty_logits/repetition_penalty,
|
||||
penalty_logits*repetition_penalty,
|
||||
)
|
||||
# Finally, put those penalized logit values back into their original
|
||||
# positions in the logits array
|
||||
logits[tokens] = penalty_logits
|
||||
return logits
|
||||
|
||||
def kobold_sample_dynamic(key, logits, top_p=0.9, temp=0.5, top_k=0, tfs=1.0):
|
||||
'''
|
||||
This gets called by generate_loop_fn to apply a series of 4 filters
|
||||
to the logits (top-k, then top-p, then TFS, then temperature) before
|
||||
picking one token using the modified logits
|
||||
'''
|
||||
# Top-k (keep only the k tokens with the highest logits and remove
|
||||
# the rest, by setting their logits to negative infinity)
|
||||
def top_k_filter(logits):
|
||||
# After sorting the logits array in descending order,
|
||||
# sorted_indices_to_remove is a 1D array that is True for tokens
|
||||
# in the sorted logits array we want to remove and False for ones
|
||||
# we want to keep, in this case the first top_k elements will be
|
||||
# False and the rest will be True
|
||||
sorted_indices_to_remove = np.arange(len(logits)) >= top_k
|
||||
# Unsort the logits array back to its original configuration and
|
||||
# remove tokens we need to remove
|
||||
_, indices_to_remove = jax.lax.sort_key_val(
|
||||
np.argsort(-logits),
|
||||
sorted_indices_to_remove,
|
||||
)
|
||||
return np.where(indices_to_remove, -np.inf, logits)
|
||||
if top_k > 0:
|
||||
logits = top_k_filter(logits)
|
||||
# Top-p (after sorting the remaining tokens again in descending order of
|
||||
# logit, remove the ones that have cumulative softmax probability
|
||||
# greater than p)
|
||||
def top_p_filter(logits):
|
||||
# Sort the logits array in descending order, replace every element
|
||||
# with e (Euler's number) to the power of that element, and divide
|
||||
# each element of the new array by the sum of the elements in the
|
||||
# new array
|
||||
sorted_logits = -np.sort(-logits)
|
||||
probabilities = np.array(jax.nn.softmax(sorted_logits), copy=True)
|
||||
# Calculate cumulative_probabilities as the prefix-sum array of
|
||||
# probabilities
|
||||
cumulative_probabilities = np.cumsum(probabilities, axis=-1)
|
||||
# We want to remove tokens with cumulative probability higher
|
||||
# than top_p
|
||||
sorted_indices_to_remove = cumulative_probabilities > top_p
|
||||
# Don't ever remove the token with the highest logit, even if
|
||||
# the probability is higher than top_p
|
||||
sorted_indices_to_remove[0] = False
|
||||
# Unsort and remove
|
||||
_, indices_to_remove = jax.lax.sort_key_val(
|
||||
np.argsort(-logits),
|
||||
sorted_indices_to_remove,
|
||||
)
|
||||
return np.where(indices_to_remove, -np.inf, logits)
|
||||
if top_p < 1.0:
|
||||
logits = top_p_filter(logits)
|
||||
# Tail free sampling (basically top-p a second time on remaining tokens
|
||||
# except it's the "cumulative normalized absolute second finite
|
||||
# differences of the softmax probabilities" instead of just the
|
||||
# cumulative softmax probabilities)
|
||||
def tail_free_filter(logits):
|
||||
# Sort in descending order
|
||||
sorted_logits = -np.sort(-logits)
|
||||
# Softmax again
|
||||
probabilities = np.array(jax.nn.softmax(sorted_logits), copy=True)
|
||||
# Calculate the second finite differences of that array (i.e.
|
||||
# calculate the difference array and then calculate the difference
|
||||
# array of the difference array)
|
||||
d2 = np.diff(np.diff(probabilities))
|
||||
# Get the absolute values of all those second finite differences
|
||||
d2 = np.abs(d2)
|
||||
# Normalize (all elements in the array are divided by the sum of the
|
||||
# array's elements)
|
||||
d2 = d2 / d2.sum(axis=-1, keepdims=True)
|
||||
# Get the prefix-sum array
|
||||
cumulative_d2 = np.cumsum(d2, axis=-1)
|
||||
# We will remove the tokens with a cumulative normalized absolute
|
||||
# second finite difference larger than the TFS value
|
||||
sorted_indices_to_remove = cumulative_d2 > tfs
|
||||
# Don't remove the token with the highest logit
|
||||
sorted_indices_to_remove[0] = False
|
||||
# Since the d2 array has two fewer elements than the logits array,
|
||||
# we'll add two extra Trues to the end
|
||||
sorted_indices_to_remove = np.pad(
|
||||
sorted_indices_to_remove,
|
||||
(0, 2),
|
||||
constant_values=True,
|
||||
)
|
||||
# Unsort and remove
|
||||
_, indices_to_remove = jax.lax.sort_key_val(
|
||||
np.argsort(-logits),
|
||||
sorted_indices_to_remove,
|
||||
)
|
||||
return np.where(indices_to_remove, -np.inf, logits)
|
||||
if tfs < 1.0:
|
||||
logits = tail_free_filter(logits)
|
||||
# Temperature (just divide the logits by the temperature)
|
||||
logits /= temp
|
||||
# Finally, pick one token using the softmax thingy again (it gives
|
||||
# an array whose elements sum to 1 so it can be used nicely as a
|
||||
# probability distribution)
|
||||
return jax.random.categorical(key, logits, -1).astype(np.uint32)
|
||||
|
||||
def apply_repetition_penalty_static(logits, tokens, repetition_penalty):
|
||||
'''
|
||||
This gets called by generate_loop_fn to apply repetition penalty
|
||||
to the 1D array logits using the provided 1D array of tokens to penalize
|
||||
@ -52,7 +207,7 @@ def apply_repetition_penalty(logits, tokens, repetition_penalty):
|
||||
# positions in the logits array
|
||||
return logits.at[tokens].set(penalty_logits)
|
||||
|
||||
def kobold_sample(key, logits, top_p=0.9, temp=0.5, top_k=0, tfs=1.0):
|
||||
def kobold_sample_static(key, logits, top_p=0.9, temp=0.5, top_k=0, tfs=1.0):
|
||||
'''
|
||||
This gets called by generate_loop_fn to apply a series of 4 filters
|
||||
to the logits (top-k, then top-p, then TFS, then temperature) before
|
||||
@ -147,18 +302,66 @@ def kobold_sample(key, logits, top_p=0.9, temp=0.5, top_k=0, tfs=1.0):
|
||||
# Finally, pick one token using the softmax thingy again (it gives
|
||||
# an array whose elements sum to 1 so it can be used nicely as a
|
||||
# probability distribution)
|
||||
return jax.random.categorical(key, logits, -1).astype(jnp.uint32)[jnp.newaxis]
|
||||
return jax.random.categorical(key, logits, -1).astype(jnp.uint32)
|
||||
|
||||
pad_token_id = 50256
|
||||
|
||||
def sample_func(data, key, numseqs_aux, badwords, repetition_penalty, sampler_options):
|
||||
numseqs = numseqs_aux.shape[0]
|
||||
gi = data[0][1]
|
||||
def sample_loop_fn(carry):
|
||||
generated, generated_index, logits, _ = carry[0][0]
|
||||
sample_key = carry[1]
|
||||
# Get the pseudo-random number generator key that will
|
||||
# be used by kobold_sample_dynamic to randomly pick a token
|
||||
sample_key, new_key = jax.random.split(sample_key, num=2)
|
||||
# Apply repetition penalty to all tokens that are
|
||||
# currently inside the "generated" array
|
||||
logits = apply_repetition_penalty_dynamic(
|
||||
logits,
|
||||
generated,
|
||||
repetition_penalty
|
||||
)
|
||||
# Remove any tokens in the badwords list by setting
|
||||
# their logits to negative infinity which effectively
|
||||
# makes their probabilities of being chosen zero
|
||||
logits[badwords] = -np.inf
|
||||
# Use the sampler (kobold_sample_dynamic) to pick one token
|
||||
# based on the logits array as a 0D uint32 array
|
||||
# (higher logit means higher probability of being
|
||||
# picked, non-linearly)
|
||||
next_token = kobold_sample_dynamic(
|
||||
sample_key,
|
||||
logits,
|
||||
**sampler_options,
|
||||
)
|
||||
# Remember what token was picked
|
||||
generated[generated_index] = next_token
|
||||
generated_index += 1
|
||||
# Re-pack the current sample_loop_fn's state so we can
|
||||
# get back the same variables the next time
|
||||
carry[0][0] = [generated, generated_index, logits, next_token]
|
||||
carry[0].append(carry[0].pop(0))
|
||||
return carry[0], new_key
|
||||
# return jax.lax.while_loop(
|
||||
# lambda carry: carry[0][0][1] == gi,
|
||||
# sample_loop_fn,
|
||||
# (data, key),
|
||||
# )
|
||||
carry = (data, key)
|
||||
while carry[0][0][1] == gi:
|
||||
carry = sample_loop_fn(carry)
|
||||
return carry
|
||||
|
||||
class PenalizingCausalTransformer(CausalTransformer):
|
||||
def __init__(self, config):
|
||||
# Initialize
|
||||
super().__init__(config)
|
||||
def generate(state, key, ctx, ctx_length, gen_length, numseqs_aux, sampler_options, soft_embeddings=None):
|
||||
def generate_static(state, key, ctx, ctx_length, gen_length, numseqs_aux, sampler_options, soft_embeddings=None):
|
||||
numseqs = numseqs_aux.shape[0]
|
||||
# These are the tokens that we don't want the AI to ever write
|
||||
self.badwords = jnp.array([6880, 50256, 42496, 4613, 17414, 22039, 16410, 27, 29, 38430, 37922, 15913, 24618, 28725, 58, 47175, 36937, 26700, 12878, 16471, 37981, 5218, 29795, 13412, 45160, 3693, 49778, 4211, 20598, 36475, 33409, 44167, 32406, 29847, 29342, 42669, 685, 25787, 7359, 3784, 5320, 33994, 33490, 34516, 43734, 17635, 24293, 9959, 23785, 21737, 28401, 18161, 26358, 32509, 1279, 38155, 18189, 26894, 6927, 14610, 23834, 11037, 14631, 26933, 46904, 22330, 25915, 47934, 38214, 1875, 14692, 41832, 13163, 25970, 29565, 44926, 19841, 37250, 49029, 9609, 44438, 16791, 17816, 30109, 41888, 47527, 42924, 23984, 49074, 33717, 31161, 49082, 30138, 31175, 12240, 14804, 7131, 26076, 33250, 3556, 38381, 36338, 32756, 46581, 17912, 49146])
|
||||
@hk.transform
|
||||
def generate_sample(context, ctx_length):
|
||||
# Give the initial context to the transformer
|
||||
transformer = CausalTransformerShard(config)
|
||||
@ -184,7 +387,7 @@ class PenalizingCausalTransformer(CausalTransformer):
|
||||
generated, generated_index, sequence_index, next_token, decode_state = carry[0][0]
|
||||
sample_key = carry[1]
|
||||
# Get the pseudo-random number generator key that will
|
||||
# be used by kobold_sample to randomly pick a token
|
||||
# be used by kobold_sample_static to randomly pick a token
|
||||
sample_key, new_key = jax.random.split(sample_key)
|
||||
# Give the context to the model and get the logits it
|
||||
# spits out
|
||||
@ -201,7 +404,7 @@ class PenalizingCausalTransformer(CausalTransformer):
|
||||
# Apply repetition penalty to all tokens that are
|
||||
# currently inside the "generated" array
|
||||
if repetition_penalty is not None:
|
||||
logits = apply_repetition_penalty(
|
||||
logits = apply_repetition_penalty_static(
|
||||
logits,
|
||||
generated,
|
||||
repetition_penalty
|
||||
@ -210,33 +413,31 @@ class PenalizingCausalTransformer(CausalTransformer):
|
||||
# their logits to negative infinity which effectively
|
||||
# makes their probabilities of being chosen zero
|
||||
logits = logits.at[self.badwords].set(-jnp.inf)
|
||||
# Use the sampler (kobold_sample) to pick one token
|
||||
# based on the logits array as a 1D array with 1 element
|
||||
# Use the sampler (kobold_sample_static) to pick one token
|
||||
# based on the logits array as a 0D uint32 array
|
||||
# (higher logit means higher probability of being
|
||||
# picked, non-linearly)
|
||||
next_token = kobold_sample(
|
||||
next_token = kobold_sample_static(
|
||||
sample_key,
|
||||
logits,
|
||||
**sampler_options,
|
||||
)
|
||||
# Remember what token was picked
|
||||
generated = generated.at[generated_index].set(next_token[0])
|
||||
generated = generated.at[generated_index].set(next_token)
|
||||
generated_index += 1
|
||||
# Re-pack the current generate_loop_fn's state so we can
|
||||
# get back the same variables the next time
|
||||
carry[0][0] = (generated, generated_index, sequence_index, next_token, new_state)
|
||||
carry[0][0] = (generated, generated_index, sequence_index, next_token[jnp.newaxis], new_state)
|
||||
carry[0].append(carry[0].pop(0))
|
||||
return carry[0], new_key
|
||||
final_state = jax.lax.while_loop(
|
||||
return jax.lax.while_loop(
|
||||
lambda carry: carry[0][0][1] - config["seq"] < gen_length,
|
||||
generate_loop_fn,
|
||||
(initial_states, sample_key),
|
||||
)
|
||||
return final_state
|
||||
generate_fn = hk.transform(generate_sample).apply
|
||||
return generate_fn(state["params"], key, ctx, ctx_length)
|
||||
self.generate_xmap = jax.experimental.maps.xmap(
|
||||
fun=generate,
|
||||
return generate_sample.apply(state["params"], key, ctx, ctx_length)
|
||||
self.generate_static_xmap = jax.experimental.maps.xmap(
|
||||
fun=generate_static,
|
||||
in_axes=(
|
||||
["shard", ...],
|
||||
["batch", ...],
|
||||
@ -250,12 +451,136 @@ class PenalizingCausalTransformer(CausalTransformer):
|
||||
out_axes=["shard", "batch", ...],
|
||||
axis_resources={'shard': 'mp', 'batch': 'dp'},
|
||||
)
|
||||
def generate(self, ctx, ctx_length, gen_length, numseqs, sampler_options, return_logits=False, soft_embeddings=None):
|
||||
def generate_initial(state, key, ctx, ctx_length, numseqs_aux, soft_embeddings=None):
|
||||
numseqs = numseqs_aux.shape[0]
|
||||
@hk.transform
|
||||
def generate_initial_inner(context, ctx_length):
|
||||
# Give the initial context to the transformer
|
||||
transformer = CausalTransformerShard(config)
|
||||
def generate_initial_scan_fn(sequence_index, c):
|
||||
_, initial_state = transformer.generate_initial(c, ctx_length, soft_embeddings=soft_embeddings)
|
||||
generated_index = config["seq"]
|
||||
# Add that information to generate_loop_fn's starting state
|
||||
initial_state = (jnp.empty(config["n_vocab"], dtype=jnp.float32), generated_index, sequence_index) + initial_state
|
||||
return sequence_index+1, initial_state
|
||||
_, initial_states = jax.lax.scan(generate_initial_scan_fn, 0, context, numseqs)
|
||||
sample_key = initial_states[-1][0]
|
||||
initial_states = list(list(jax.tree_map(lambda x: x[i], initial_states[:-1])) for i in range(numseqs))
|
||||
return initial_states, sample_key
|
||||
return generate_initial_inner.apply(state["params"], key, ctx, ctx_length)
|
||||
self.generate_initial_xmap = jax.experimental.maps.xmap(
|
||||
fun=generate_initial,
|
||||
in_axes=(
|
||||
["shard", ...],
|
||||
["batch", ...],
|
||||
["batch", ...],
|
||||
["batch", ...],
|
||||
["batch", ...],
|
||||
["shard", ...],
|
||||
),
|
||||
out_axes=["shard", "batch", ...],
|
||||
axis_resources={'shard': 'mp', 'batch': 'dp'},
|
||||
)
|
||||
def generate_once(data, state, numseqs_aux, soft_embeddings=None):
|
||||
numseqs = numseqs_aux.shape[0]
|
||||
@hk.without_apply_rng
|
||||
@hk.transform
|
||||
def generate_once_inner():
|
||||
gi = data[0][1]
|
||||
# Give the initial context to the transformer
|
||||
transformer = CausalTransformerShard(config)
|
||||
# This is the main generation loop
|
||||
def generate_loop_fn(carry):
|
||||
# Unpack current generate_loop_fn state
|
||||
_, generated_index, sequence_index, next_token, decode_state = carry[0][0]
|
||||
# Give the context to the model and get the logits it
|
||||
# spits out
|
||||
# (a 2D array with 1 row and 50400 columns representing
|
||||
# how strongly it thinks each of the 50257 tokens in its
|
||||
# vocabulary should be appended to the context, followed
|
||||
# by 143 apparently useless columns ???)
|
||||
logits, new_state = transformer.generate_once(next_token, decode_state, soft_embeddings=soft_embeddings)
|
||||
# Verify that logits does indeed have that many rows and
|
||||
# columns (if you get an error here, pray for mercy)
|
||||
assert logits.shape == (1, config["n_vocab"])
|
||||
assert logits.dtype == jnp.float32
|
||||
# Flatten it into a 1D array to make it easier to use
|
||||
logits = logits[0]
|
||||
# Re-pack the current generate_loop_fn's state so we can
|
||||
# get back the same variables the next time
|
||||
generated_index += 1
|
||||
carry[0][0] = [logits, generated_index, sequence_index, next_token, new_state]
|
||||
carry[0].append(carry[0].pop(0))
|
||||
return carry[0],
|
||||
return jax.lax.while_loop(
|
||||
lambda carry: carry[0][0][1] == gi,
|
||||
generate_loop_fn,
|
||||
(data,),
|
||||
)
|
||||
return generate_once_inner.apply(state["params"])
|
||||
self.generate_once_xmap = jax.experimental.maps.xmap(
|
||||
fun=generate_once,
|
||||
in_axes=(
|
||||
["shard", "batch", ...],
|
||||
["shard", ...],
|
||||
["batch", ...],
|
||||
["shard", ...],
|
||||
),
|
||||
out_axes=["shard", "batch", ...],
|
||||
axis_resources={'shard': 'mp', 'batch': 'dp'},
|
||||
)
|
||||
def generate_dynamic(self, ctx, ctx_length, gen_length, numseqs, sampler_options, return_logits=False, soft_embeddings=None, excluded_world_info=None, use_callback=True):
|
||||
assert excluded_world_info is not None
|
||||
assert not return_logits
|
||||
assert gen_length.ndim == 1
|
||||
assert soft_embeddings is not None
|
||||
key = hk.PRNGSequence(random.randint(0, 2 ** 60))
|
||||
batch_size = ctx.shape[0]
|
||||
self.batch_size = batch_size
|
||||
_numseqs_aux = jnp.empty((batch_size, numseqs), dtype=np.uint32)
|
||||
numseqs_aux = batch_xmap(_numseqs_aux)
|
||||
sample_data = [
|
||||
[
|
||||
np.pad(ctx[0][i], (0, params["seq"]), constant_values=pad_token_id),
|
||||
params["seq"],
|
||||
None,
|
||||
np.empty((), dtype=np.uint32),
|
||||
]
|
||||
for i in range(numseqs)
|
||||
]
|
||||
repetition_penalty = sampler_options.pop("repetition_penalty", 1.0)
|
||||
n_generated = 0
|
||||
regeneration_required = False
|
||||
halt = False
|
||||
generate_data, sample_key = self.generate_initial_xmap(self.state, jnp.array(key.take(batch_size)), ctx, ctx_length, numseqs_aux, soft_embeddings)
|
||||
sample_key = np.asarray(sample_key[0, 0])
|
||||
while True:
|
||||
generate_data, = self.generate_once_xmap(generate_data, self.state, numseqs_aux, soft_embeddings)
|
||||
for i in range(numseqs):
|
||||
sample_data[i][2] = np.array(generate_data[i][0][0, 0], copy=True)
|
||||
if use_callback:
|
||||
logits = np.float32(tuple(d[2] for d in sample_data))
|
||||
logits = warper_callback(logits)
|
||||
for i in range(numseqs):
|
||||
sample_data[i][2] = logits[i]
|
||||
sample_data, sample_key = sample_func(sample_data, sample_key, _numseqs_aux, badwords, repetition_penalty, sampler_options)
|
||||
n_generated += 1
|
||||
for i in range(numseqs):
|
||||
generate_data[i][3] = np.tile(sample_data[i][0][sample_data[i][1]-1][np.newaxis, np.newaxis], (params["cores_per_replica"], 1, 1))
|
||||
if use_callback:
|
||||
generated = np.uint32(tuple(d[0] for d in sample_data))
|
||||
excluded_world_info, regeneration_required, halt = stopping_callback(generated, n_generated, excluded_world_info)
|
||||
if regeneration_required or halt:
|
||||
break
|
||||
else:
|
||||
break
|
||||
return sample_data, n_generated, regeneration_required, halt
|
||||
def generate_static(self, ctx, ctx_length, gen_length, numseqs, sampler_options, return_logits=False, soft_embeddings=None):
|
||||
assert not return_logits
|
||||
key = hk.PRNGSequence(random.randint(0, 2 ** 60))
|
||||
batch_size = ctx.shape[0]
|
||||
self.batch_size = batch_size
|
||||
return self.generate_xmap(
|
||||
return self.generate_static_xmap(
|
||||
self.state,
|
||||
jnp.array(key.take(batch_size)),
|
||||
ctx,
|
||||
@ -267,7 +592,7 @@ class PenalizingCausalTransformer(CausalTransformer):
|
||||
)
|
||||
|
||||
|
||||
def infer(
|
||||
def infer_dynamic(
|
||||
context: np.array,
|
||||
top_p=0.9,
|
||||
temp=0.5,
|
||||
@ -278,7 +603,53 @@ def infer(
|
||||
gen_len=80,
|
||||
soft_embeddings: Optional[np.array] = None,
|
||||
soft_tokens: Optional[np.array] = None,
|
||||
) -> List[str]:
|
||||
excluded_world_info = None,
|
||||
use_callback=True,
|
||||
) -> Tuple[List[np.array], int, bool, bool]:
|
||||
assert excluded_world_info is not None
|
||||
maps.thread_resources.env = thread_resources_env
|
||||
total_batch = 1
|
||||
tokens = context
|
||||
if(soft_tokens is not None):
|
||||
tokens = np.uint32(np.concatenate((np.tile(soft_tokens, (tokens.shape[0], 1)), tokens), axis=-1))
|
||||
provided_ctx = tokens.shape[-1]
|
||||
pad_amount = seq - provided_ctx
|
||||
padded_tokens = np.pad(tokens, ((0, 0), (pad_amount, 0)), constant_values=pad_token_id)
|
||||
batched_tokens = np.array([padded_tokens] * total_batch)
|
||||
samples = []
|
||||
generator_params = {
|
||||
"temp": float(temp),
|
||||
"top_p": float(top_p),
|
||||
"tfs": float(tfs),
|
||||
"repetition_penalty": float(repetition_penalty),
|
||||
"top_k": int(top_k),
|
||||
}
|
||||
output = network.generate_dynamic(
|
||||
batched_tokens,
|
||||
np.ones(total_batch, dtype=np.uint32) * provided_ctx,
|
||||
np.ones(total_batch, dtype=np.uint32) * gen_len,
|
||||
numseqs,
|
||||
generator_params,
|
||||
soft_embeddings=soft_embeddings,
|
||||
excluded_world_info=excluded_world_info,
|
||||
use_callback=use_callback,
|
||||
)
|
||||
for out in output[0]:
|
||||
samples.append(out[0][params["seq"] : params["seq"] + gen_len])
|
||||
return (samples,) + output[1:]
|
||||
|
||||
def infer_static(
|
||||
context: np.array,
|
||||
top_p=0.9,
|
||||
temp=0.5,
|
||||
top_k=0,
|
||||
tfs=1.0,
|
||||
repetition_penalty=1.0,
|
||||
numseqs=1,
|
||||
gen_len=80,
|
||||
soft_embeddings: Optional[np.array] = None,
|
||||
soft_tokens: Optional[np.array] = None,
|
||||
) -> List[np.array]:
|
||||
maps.thread_resources.env = thread_resources_env
|
||||
total_batch = 1
|
||||
tokens = context
|
||||
@ -296,7 +667,7 @@ def infer(
|
||||
"repetition_penalty": repetition_penalty * np.ones(total_batch),
|
||||
"top_k": np.full(total_batch, top_k, dtype=np.uint32)
|
||||
}
|
||||
output = network.generate(
|
||||
output = network.generate_static(
|
||||
batched_tokens,
|
||||
np.ones(total_batch, dtype=np.uint32) * provided_ctx,
|
||||
np.ones(total_batch, dtype=np.uint32) * gen_len,
|
||||
@ -354,6 +725,14 @@ def load_model(path: str, driver_version="tpu_driver0.1_dev20210607", **kwargs)
|
||||
maps.thread_resources.env = thread_resources_env
|
||||
tokenizer = transformers.GPT2TokenizerFast.from_pretrained('gpt2')
|
||||
|
||||
global shard_xmap, batch_xmap
|
||||
shard_xmap = __shard_xmap()
|
||||
batch_xmap = __batch_xmap(shard_dim=cores_per_replica)
|
||||
|
||||
global badwords
|
||||
# These are the tokens that we don't want the AI to ever write
|
||||
badwords = jnp.array([6880, 50256, 42496, 4613, 17414, 22039, 16410, 27, 29, 38430, 37922, 15913, 24618, 28725, 58, 47175, 36937, 26700, 12878, 16471, 37981, 5218, 29795, 13412, 45160, 3693, 49778, 4211, 20598, 36475, 33409, 44167, 32406, 29847, 29342, 42669, 685, 25787, 7359, 3784, 5320, 33994, 33490, 34516, 43734, 17635, 24293, 9959, 23785, 21737, 28401, 18161, 26358, 32509, 1279, 38155, 18189, 26894, 6927, 14610, 23834, 11037, 14631, 26933, 46904, 22330, 25915, 47934, 38214, 1875, 14692, 41832, 13163, 25970, 29565, 44926, 19841, 37250, 49029, 9609, 44438, 16791, 17816, 30109, 41888, 47527, 42924, 23984, 49074, 33717, 31161, 49082, 30138, 31175, 12240, 14804, 7131, 26076, 33250, 3556, 38381, 36338, 32756, 46581, 17912, 49146])
|
||||
|
||||
if not path.endswith("/"):
|
||||
path += "/"
|
||||
|
||||
|
||||
@ -5,6 +5,7 @@ TITLE KoboldAI - Updater
|
||||
SET /P M=<loader.settings
|
||||
IF %M%==1 GOTO drivemap
|
||||
IF %M%==2 GOTO subfolder
|
||||
IF %M%==3 GOTO drivemap_B
|
||||
|
||||
:subfolder
|
||||
SET TEMP=%~DP0MINICONDA3
|
||||
@ -19,6 +20,13 @@ SET TMP=K:\
|
||||
call K:\python\condabin\activate
|
||||
GOTO GIT
|
||||
|
||||
:drivemap_B
|
||||
subst B: miniconda3 >nul
|
||||
SET TEMP=B:\
|
||||
SET TMP=B:\
|
||||
call B:\python\condabin\activate
|
||||
GOTO GIT
|
||||
|
||||
:GIT
|
||||
ECHO 1. KoboldAI Main (The Official stable version of KoboldAI)
|
||||
ECHO 2. KoboldAI United (Development Version, new features but may break at any time)
|
||||
|
||||
2
userscripts/Readme.txt
Normal file
2
userscripts/Readme.txt
Normal file
@ -0,0 +1,2 @@
|
||||
This folder contains userscripts for KoboldAI, any script that begins with kaipreset_ is treated as a official part of KoboldAI and can be overwritten by updates.
|
||||
If you would like to make modifications to any of the lua scripts it is recommended you that you either rename the script, or create a new copy with a new name.
|
||||
720
userscripts/api_documentation.html
vendored
Normal file
720
userscripts/api_documentation.html
vendored
Normal file
@ -0,0 +1,720 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>API</title>
|
||||
<link rel="stylesheet" href="https://stackedit.io/style.css" />
|
||||
</head>
|
||||
|
||||
<body class="stackedit">
|
||||
<div class="stackedit__left">
|
||||
<div class="stackedit__toc">
|
||||
|
||||
<ul>
|
||||
<li><a href="#kobold">kobold</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#kobold.decode">kobold.decode()</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#kobold.encode">kobold.encode()</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#kobold.get_config_file">kobold.get_config_file()</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#kobold.halt_generation">kobold.halt_generation()</a></li>
|
||||
<li><a href="#kobold.restart_generation">kobold.restart_generation()</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#kobold.authorsnote">kobold.authorsnote</a></li>
|
||||
<li><a href="#kobold.authorsnotetemplate">kobold.authorsnotetemplate</a></li>
|
||||
<li><a href="#kobold.custmodpth">kobold.custmodpth</a></li>
|
||||
<li><a href="#kobold.feedback">kobold.feedback</a></li>
|
||||
<li><a href="#kobold.generated">kobold.generated</a></li>
|
||||
<li><a href="#kobold.generated_cols">kobold.generated_cols</a></li>
|
||||
<li><a href="#kobold.generated_rows">kobold.generated_rows</a></li>
|
||||
<li><a href="#kobold.is_config_file_open">kobold.is_config_file_open</a></li>
|
||||
<li><a href="#kobold.logits">kobold.logits</a></li>
|
||||
<li><a href="#kobold.logits_cols">kobold.logits_cols</a></li>
|
||||
<li><a href="#kobold.logits_rows">kobold.logits_rows</a></li>
|
||||
<li><a href="#kobold.memory">kobold.memory</a></li>
|
||||
<li><a href="#kobold.modelbackend">kobold.modelbackend</a></li>
|
||||
<li><a href="#kobold.modeltype">kobold.modeltype</a></li>
|
||||
<li><a href="#kobold.num_outputs">kobold.num_outputs</a></li>
|
||||
<li><a href="#kobold.outputs">kobold.outputs</a></li>
|
||||
<li><a href="#kobold.settings">kobold.settings</a></li>
|
||||
<li><a href="#kobold.story">kobold.story</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#kobold.storyforward_iter">kobold.story:forward_iter()</a></li>
|
||||
<li><a href="#kobold.storyreverse_iter">kobold.story:reverse_iter()</a></li>
|
||||
<li><a href="#koboldstorychunk">KoboldStoryChunk</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#koboldstorychunk.content">KoboldStoryChunk.content</a></li>
|
||||
<li><a href="#koboldstorychunk.num">KoboldStoryChunk.num</a></li>
|
||||
<li><a href="#kobold.submission">kobold.submission</a></li>
|
||||
<li><a href="#kobold.worldinfo">kobold.worldinfo</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#kobold.worldinfocompute_context">kobold.worldinfo:compute_context()</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#kobold.worldinfofinduid">kobold.worldinfo:finduid()</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#kobold.worldinfois_valid">kobold.worldinfo:is_valid()</a></li>
|
||||
<li><a href="#kobold.worldinfo.folders">kobold.worldinfo.folders</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#koboldworldinfoentry">KoboldWorldInfoEntry</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#koboldworldinfoentrycompute_context">KoboldWorldInfoEntry:compute_context()</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#koboldworldinfoentryis_valid">KoboldWorldInfoEntry:is_valid()</a></li>
|
||||
<li><a href="#koboldworldinfoentry.comment">KoboldWorldInfoEntry.comment</a></li>
|
||||
<li><a href="#koboldworldinfoentry.constant">KoboldWorldInfoEntry.constant</a></li>
|
||||
<li><a href="#koboldworldinfoentry.content">KoboldWorldInfoEntry.content</a></li>
|
||||
<li><a href="#koboldworldinfoentry.folder">KoboldWorldInfoEntry.folder</a></li>
|
||||
<li><a href="#koboldworldinfoentry.key">KoboldWorldInfoEntry.key</a></li>
|
||||
<li><a href="#koboldworldinfoentry.keysecondary">KoboldWorldInfoEntry.keysecondary</a></li>
|
||||
<li><a href="#koboldworldinfoentry.num">KoboldWorldInfoEntry.num</a></li>
|
||||
<li><a href="#koboldworldinfoentry.selective">KoboldWorldInfoEntry.selective</a></li>
|
||||
<li><a href="#koboldworldinfoentry.uid">KoboldWorldInfoEntry.uid</a></li>
|
||||
<li><a href="#koboldworldinfofolderselectorfinduid">KoboldWorldInfoFolderSelector:finduid()</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#koboldworldinfofolderselectoris_valid">KoboldWorldInfoFolderSelector:is_valid()</a></li>
|
||||
<li><a href="#koboldworldinfofolder">KoboldWorldInfoFolder</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#koboldworldinfofoldercompute_context">KoboldWorldInfoFolder:compute_context()</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#koboldworldinfofolderfinduid">KoboldWorldInfoFolder:finduid()</a>
|
||||
<ul>
|
||||
<li></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#koboldworldinfofolderis_valid">KoboldWorldInfoFolder:is_valid()</a></li>
|
||||
<li><a href="#koboldworldinfofolder.name">KoboldWorldInfoFolder.name</a></li>
|
||||
<li><a href="#koboldworldinfofolder.uid">KoboldWorldInfoFolder.uid</a></li>
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="stackedit__right">
|
||||
<div class="stackedit__html">
|
||||
<h1 id="kobold">kobold</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">global kobold<span class="token punctuation">:</span> KoboldLib
|
||||
</code></pre>
|
||||
<h3 id="methods">Methods:</h3>
|
||||
<ul>
|
||||
<li><code>kobold.decode()</code></li>
|
||||
<li><code>kobold.encode()</code></li>
|
||||
<li><code>kobold.get_config_file()</code></li>
|
||||
<li><code>kobold.halt_generation()</code></li>
|
||||
<li><code>kobold.restart_generation()</code></li>
|
||||
</ul>
|
||||
<h3 id="fields">Fields:</h3>
|
||||
<ul>
|
||||
<li><code>kobold.authorsnote</code></li>
|
||||
<li><code>kobold.authorsnotetemplate</code></li>
|
||||
<li><code>kobold.custmodpth</code></li>
|
||||
<li><code>kobold.feedback</code></li>
|
||||
<li><code>kobold.generated</code></li>
|
||||
<li><code>kobold.generated_cols</code></li>
|
||||
<li><code>kobold.generated_rows</code></li>
|
||||
<li><code>kobold.is_config_file_open</code></li>
|
||||
<li><code>kobold.logits</code></li>
|
||||
<li><code>kobold.logits_cols</code></li>
|
||||
<li><code>kobold.logits_rows</code></li>
|
||||
<li><code>kobold.memory</code></li>
|
||||
<li><code>kobold.modelbackend</code></li>
|
||||
<li><code>kobold.modeltype</code></li>
|
||||
<li><code>kobold.num_outputs</code></li>
|
||||
<li><code>kobold.outputs</code></li>
|
||||
<li><code>kobold.settings</code></li>
|
||||
<li><code>kobold.story</code></li>
|
||||
<li><code>kobold.submission</code></li>
|
||||
<li><code>kobold.worldinfo</code></li>
|
||||
</ul>
|
||||
<h1 id="kobold.decode">kobold.decode()</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldLib<span class="token punctuation">.</span><span class="token function">decode</span><span class="token punctuation">(</span>tok<span class="token punctuation">:</span> integer<span class="token operator">|</span>table<span class="token operator"><</span>integer<span class="token punctuation">,</span> integer<span class="token operator">></span><span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> string
|
||||
</code></pre>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<p>Decodes the given token or list of tokens using the current tokenizer. If <code>kobold.backend</code> is <code>'readonly'</code> or <code>'api'</code>, the tokenizer used is the GPT-2 tokenizer, otherwise the model’s tokenizer is used. This function is the inverse of <code>kobold.encode()</code>.</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token function">print</span><span class="token punctuation">(</span>kobold<span class="token punctuation">.</span><span class="token function">decode</span><span class="token punctuation">(</span><span class="token punctuation">{</span><span class="token number">15496</span><span class="token punctuation">,</span> <span class="token number">2159</span><span class="token punctuation">}</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token comment">-- 'Hello World'</span>
|
||||
</code></pre>
|
||||
<h3 id="parameters">Parameters:</h3>
|
||||
<ul>
|
||||
<li>tok (<code>integer|table<integer, integer></code>): Array of token IDs to decode, or the token ID of a single token.</li>
|
||||
</ul>
|
||||
<h3 id="returns">Returns:</h3>
|
||||
<ul>
|
||||
<li><code>string</code>: Decoded tokens.</li>
|
||||
</ul>
|
||||
<h1 id="kobold.encode">kobold.encode()</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldLib<span class="token punctuation">.</span><span class="token function">encode</span><span class="token punctuation">(</span>str<span class="token punctuation">:</span> string<span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> table<span class="token operator"><</span>integer<span class="token punctuation">,</span> integer<span class="token operator">></span>
|
||||
</code></pre>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<p>Encodes the given string using the current tokenizer into an array of tokens. If <code>kobold.backend</code> is <code>'readonly'</code> or <code>'api'</code>, the tokenizer used is the GPT-2 tokenizer, otherwise the model’s tokenizer is used. This function is the inverse of <code>kobold.decode()</code>.</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">local</span> tokens <span class="token operator">=</span> kobold<span class="token punctuation">.</span><span class="token function">encode</span><span class="token punctuation">(</span><span class="token string">"Hello World"</span><span class="token punctuation">)</span>
|
||||
<span class="token function">print</span><span class="token punctuation">(</span><span class="token operator">#</span>tokens<span class="token punctuation">)</span> <span class="token comment">-- 2</span>
|
||||
<span class="token function">print</span><span class="token punctuation">(</span>tokens<span class="token punctuation">[</span><span class="token number">1</span><span class="token punctuation">]</span><span class="token punctuation">)</span> <span class="token comment">-- 15496</span>
|
||||
<span class="token function">print</span><span class="token punctuation">(</span>tokens<span class="token punctuation">[</span><span class="token number">2</span><span class="token punctuation">]</span><span class="token punctuation">)</span> <span class="token comment">-- 2159</span>
|
||||
</code></pre>
|
||||
<h3 id="parameters-1">Parameters:</h3>
|
||||
<ul>
|
||||
<li>tok (<code>integer|table<integer, integer></code>): Array of token IDs to decode, or the token ID of a single token.</li>
|
||||
</ul>
|
||||
<h3 id="returns-1">Returns:</h3>
|
||||
<ul>
|
||||
<li><code>string</code>: Decoded tokens.</li>
|
||||
</ul>
|
||||
<h1 id="kobold.get_config_file">kobold.get_config_file()</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldLib<span class="token punctuation">.</span><span class="token function">get_config_file</span><span class="token punctuation">(</span>clear?<span class="token punctuation">:</span> boolean<span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> file<span class="token operator">*</span>
|
||||
</code></pre>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<p>Returns a file handle representing your script’s configuration file, which is usually the file in the userscripts folder with the same filename as your script but with “.conf” appended at the end. This function throws an error on failure.</p>
|
||||
<p>If the configuration file does not exist when this function is called, the configuration file will first be created as a new empty file.</p>
|
||||
<p>If KoboldAI does not possess an open file handle to the configuration file, this function opens the file in <code>w+b</code> mode if the <code>clear</code> parameter is a truthy value, otherwise the file is opened in <code>r+b</code> mode. These are mostly the same – the file is opened in binary read-write mode and then seeked to the start of the file – except the former mode deletes the contents of the file prior to opening it and the latter mode does not.</p>
|
||||
<p>If KoboldAI does possess an open file handle to the configuration file, that open file handle is returned without seeking or deleting the contents of the file. You can check if KoboldAI possesses an open file handle to the configuration file by using <code>kobold.is_config_file_open</code>.</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">local</span> example_config <span class="token operator">=</span> <span class="token string">"return 'Hello World'"</span>
|
||||
<span class="token keyword">local</span> cfg
|
||||
<span class="token keyword">do</span>
|
||||
<span class="token comment">-- If config file is empty, write example config</span>
|
||||
<span class="token keyword">local</span> f <span class="token operator"><</span>close<span class="token operator">></span> <span class="token operator">=</span> kobold<span class="token punctuation">.</span><span class="token function">get_config_file</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
|
||||
f<span class="token punctuation">:</span><span class="token function">seek</span><span class="token punctuation">(</span><span class="token string">"set"</span><span class="token punctuation">)</span>
|
||||
<span class="token keyword">if</span> f<span class="token punctuation">:</span><span class="token function">read</span><span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span> <span class="token operator">==</span> <span class="token keyword">nil</span> <span class="token keyword">then</span> f<span class="token punctuation">:</span><span class="token function">write</span><span class="token punctuation">(</span>example_config<span class="token punctuation">)</span> <span class="token keyword">end</span>
|
||||
f<span class="token punctuation">:</span><span class="token function">seek</span><span class="token punctuation">(</span><span class="token string">"set"</span><span class="token punctuation">)</span>
|
||||
|
||||
<span class="token comment">-- Read config</span>
|
||||
<span class="token keyword">local</span> err
|
||||
cfg<span class="token punctuation">,</span> err <span class="token operator">=</span> <span class="token function">load</span><span class="token punctuation">(</span>f<span class="token punctuation">:</span><span class="token function">read</span><span class="token punctuation">(</span><span class="token string">"a"</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
|
||||
<span class="token keyword">if</span> err <span class="token operator">~=</span> <span class="token keyword">nil</span> <span class="token keyword">then</span> <span class="token function">error</span><span class="token punctuation">(</span>err<span class="token punctuation">)</span> <span class="token keyword">end</span>
|
||||
cfg <span class="token operator">=</span> <span class="token function">cfg</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
|
||||
<span class="token keyword">end</span>
|
||||
</code></pre>
|
||||
<h3 id="parameters-2">Parameters:</h3>
|
||||
<ul>
|
||||
<li>clear? (<code>bool</code>): If KoboldAI does not possess an open file handle to the configuration object, this determines whether the file will be opened in <code>w+b</code> or <code>r+b</code> mode. This parameter defaults to <code>false</code>.</li>
|
||||
</ul>
|
||||
<h3 id="returns-2">Returns:</h3>
|
||||
<ul>
|
||||
<li><code>file*</code>: File handle for the configuration file.</li>
|
||||
</ul>
|
||||
<h1 id="kobold.halt_generation">kobold.halt_generation()</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldLib<span class="token punctuation">.</span><span class="token function">halt_generation</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> <span class="token keyword">nil</span>
|
||||
</code></pre>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<p>If called from an input modifier, prevents the user’s input from being sent to the model and skips directly to the output modifier.</p>
|
||||
<p>If called from a generation modifier, stops generation after the current token is generated and then skips to the output modifier. In other words, if, when you call this function, <code>kobold.generated</code> has n columns, it will have exactly n+1 columns when the output modifier is called.</p>
|
||||
<p>If called from an output modifier, has no effect.</p>
|
||||
<h1 id="kobold.restart_generation">kobold.restart_generation()</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldLib<span class="token punctuation">.</span><span class="token function">restart_generation</span><span class="token punctuation">(</span>sequence?<span class="token punctuation">:</span> integer<span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> <span class="token keyword">nil</span>
|
||||
</code></pre>
|
||||
<p><em><strong>Callable from:</strong></em> output modifier</p>
|
||||
<p>After the current output is sent to the GUI, starts another generation using the empty string as the submission.</p>
|
||||
<p>Whatever ends up being the output selected by the user or by the <code>sequence</code> parameter will be saved in <code>kobold.feedback</code> when the new generation begins.</p>
|
||||
<h3 id="parameters-3">Parameters:</h3>
|
||||
<ul>
|
||||
<li>sequence? (<code>integer</code>): If you have multiple Gens Per Action, this can be used to choose which sequence to use as the output, where 1 is the first, 2 is the second and so on. If you set this to 0, the user will be prompted to choose the sequence instead. Defaults to 0.</li>
|
||||
</ul>
|
||||
<h1 id="kobold.authorsnote">kobold.authorsnote</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>authorsnote<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere (triggers regeneration when written to from generation modifier)</p>
|
||||
<p>The author’s note as set from the “Memory” button in the GUI.</p>
|
||||
<p>Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.</p>
|
||||
<h1 id="kobold.authorsnotetemplate">kobold.authorsnotetemplate</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>authorsnotetemplate<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere (triggers regeneration when written to from generation modifier)</p>
|
||||
<p>The author’s note template as set from the “Memory” button in the GUI.</p>
|
||||
<p>Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.</p>
|
||||
<h1 id="kobold.custmodpth">kobold.custmodpth</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>custmodpth<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<p>Path to a directory that the user chose either via the file dialog that appears when KoboldAI asks you to choose the path to your custom model or via the <code>--path</code> command-line flag.</p>
|
||||
<p>If the user loaded a built-in model from the menu, this is instead the model ID of the model on Hugging Face’s model hub, such as “KoboldAI/GPT-Neo-2.7B-Picard” or “hakurei/lit-6B”.</p>
|
||||
<h1 id="kobold.feedback">kobold.feedback</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>feedback<span class="token punctuation">:</span> string?
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<p>If this is a repeat generation caused by <code>kobold.restart_generation()</code>, this will be a string containing the previous output. If not, this will be <code>nil</code>.</p>
|
||||
<h1 id="kobold.generated">kobold.generated</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>generated<span class="token punctuation">:</span> table<span class="token operator"><</span>integer<span class="token punctuation">,</span> table<span class="token operator"><</span>integer<span class="token punctuation">,</span> integer<span class="token operator">>></span>?
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> generation modifier and output modifier, but only if <code>kobold.modelbackend</code> is not <code>'api'</code> or <code>'readonly'</code><br>
|
||||
<em><strong>Writable from:</strong></em> generation modifier</p>
|
||||
<p>Two-dimensional array of tokens generated thus far. Each row represents one sequence, each column one token.</p>
|
||||
<h1 id="kobold.generated_cols">kobold.generated_cols</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>generated_cols<span class="token punctuation">:</span> integer
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<p>Number of columns in <code>kobold.generated</code>. In other words, the number of tokens generated thus far, which is equal to the number of times that the generation modifier has been called, not including the current time if this is being read from a generation modifier.</p>
|
||||
<p>If <code>kobold.modelbackend</code> is <code>'api'</code> or <code>'readonly'</code>, this returns 0 instead.</p>
|
||||
<h1 id="kobold.generated_rows">kobold.generated_rows</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>generated_rows<span class="token punctuation">:</span> integer
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<p>Number of rows in <code>kobold.generated</code>, equal to <code>kobold.settings.numseqs</code>.</p>
|
||||
<h1 id="kobold.is_config_file_open">kobold.is_config_file_open</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>is_config_file_open<span class="token punctuation">:</span> boolean
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<p>Whether or not KoboldAI possesses an open file handle to your script’s configuration file. See <code>kobold.get_config_file()</code> for more details.</p>
|
||||
<h1 id="kobold.logits">kobold.logits</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>logits<span class="token punctuation">:</span> table<span class="token operator"><</span>integer<span class="token punctuation">,</span> table<span class="token operator"><</span>integer<span class="token punctuation">,</span> number<span class="token operator">>></span>?
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> generation modifier, but only if <code>kobold.modelbackend</code> is not <code>'api'</code> or <code>'readonly'</code><br>
|
||||
<em><strong>Writable from:</strong></em> generation modifier</p>
|
||||
<p>Two-dimensional array of <a href="https://datascience.stackexchange.com/questions/31041/what-does-logits-in-machine-learning-mean">logits</a> prior to being filtered by top-p sampling, etc. Each row represents one sequence, each column one of the tokens in the model’s vocabulary. The ith column represents the logit score of token i-1, so if you want to access the logit score of token 18435 (" Hello" with a leading space), you need to access column 18436. You may alter this two-dimensional array to encourage or deter certain tokens from appearing in the output in a stochastic manner.</p>
|
||||
<p>Don’t modify this table unnecessarily unless you know what you are doing! The bias example scripts show how to use this feature properly.</p>
|
||||
<h1 id="kobold.logits_cols">kobold.logits_cols</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>logits_cols<span class="token punctuation">:</span> integer
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<p>Number of columns in <code>kobold.logits</code>, equal to the vocabulary size of the current model. Most models based on GPT-2 (e.g. GPT-Neo and GPT-J) have a vocabulary size of 50257. GPT-J models in particular have a vocabulary size of 50400 instead, although GPT-J models aren’t trained to use the rightmost 143 tokens of the logits array.</p>
|
||||
<p>If <code>kobold.modelbackend</code> is <code>'api'</code> or <code>'readonly'</code>, this returns 0 instead.</p>
|
||||
<h1 id="kobold.logits_rows">kobold.logits_rows</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>logits_rows<span class="token punctuation">:</span> integer
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<p>Number of rows in <code>kobold.generated</code>, equal to <code>kobold.settings.numseqs</code>. a local KoboldAI install.</p>
|
||||
<h1 id="kobold.memory">kobold.memory</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>memory<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere (triggers regeneration when written to from generation modifier)</p>
|
||||
<p>The memory as set from the “Memory” button in the GUI.</p>
|
||||
<p>Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.</p>
|
||||
<h1 id="kobold.modelbackend">kobold.modelbackend</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>modelbackend<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<p>One of the following values:</p>
|
||||
<ul>
|
||||
<li><code>'api'</code>: InferKit, OpenAI or legacy Colab mode</li>
|
||||
<li><code>'readonly'</code>: Read-only no AI mode</li>
|
||||
<li><code>'transformers'</code>: Models running on your own computer, and Colab GPU backend (currently used for 2.7B models on Colab)</li>
|
||||
<li><code>'mtj'</code>: Colab TPU backend (currently used for 6B models on Colab)</li>
|
||||
</ul>
|
||||
<h1 id="kobold.modeltype">kobold.modeltype</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>modeltype<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<p>One of the following values:</p>
|
||||
<ul>
|
||||
<li><code>'api'</code>: InferKit, OpenAI or legacy Colab mode</li>
|
||||
<li><code>'readonly'</code>: Read-only no AI mode</li>
|
||||
<li><code>'unknown'</code></li>
|
||||
<li><code>'gpt2'</code>: GPT-2-Small</li>
|
||||
<li><code>'gpt2-medium'</code></li>
|
||||
<li><code>'gpt2-large'</code></li>
|
||||
<li><code>'gpt2-xl'</code></li>
|
||||
<li><code>'gpt-neo-125M'</code></li>
|
||||
<li><code>'gpt-neo-1.3B'</code></li>
|
||||
<li><code>'gpt-neo-2.7B'</code></li>
|
||||
<li><code>'gpt-j-6B'</code></li>
|
||||
</ul>
|
||||
<h1 id="kobold.num_outputs">kobold.num_outputs</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>num_outputs<span class="token punctuation">:</span> integer
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<p>Number of rows in <code>kobold.outputs</code>. This is equal to <code>kobold.settings.numseqs</code> unless you’re using a non-Colab third-party API such as OpenAI or InferKit, in which case this is 1. If you decide to write to <code>kobold.settings.numseqs</code> from an output modifier, this value remains unchanged.</p>
|
||||
<h1 id="kobold.outputs">kobold.outputs</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>outputs<span class="token punctuation">:</span> table<span class="token operator"><</span>integer<span class="token punctuation">,</span> string<span class="token operator">></span>
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> output modifier<br>
|
||||
<em><strong>Writable from:</strong></em> output modifier</p>
|
||||
<p>Model output before applying output formatting. One row per “Gens Per Action”, unless you’re using OpenAI or InferKit, in which case this always has exactly one row.</p>
|
||||
<h1 id="kobold.settings">kobold.settings</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>settings<span class="token punctuation">:</span> KoboldSettings
|
||||
</code></pre>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere (does not affect other scripts when written to since each script has its own copy of this object)</p>
|
||||
<p>Contains most of the settings. They have the same names as in gensettings.py, so the top-p value is <code>kobold.settings.settopp</code>.</p>
|
||||
<p>All the settings can be read from anywhere and written from anywhere, except <code>kobold.settings.numseqs</code> which can only be written to from an input modifier or output modifier.</p>
|
||||
<p>Modifying certain fields from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess. Currently, only the following fields and their aliases cause this to occur:</p>
|
||||
<ul>
|
||||
<li><code>kobold.settings.settknmax</code> (Max Tokens)</li>
|
||||
<li><code>kobold.settings.anotedepth</code> (Author’s Note Depth)</li>
|
||||
<li><code>kobold.settings.setwidepth</code> (World Info Depth)</li>
|
||||
<li><code>kobold.settings.setuseprompt</code> (Always Use Prompt)</li>
|
||||
</ul>
|
||||
<h1 id="kobold.story">kobold.story</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>story<span class="token punctuation">:</span> KoboldStory
|
||||
</code></pre>
|
||||
<p>Contains the chunks of the current story. Don’t use <code>pairs</code> or <code>ipairs</code> to iterate over the story chunks, use <code>kobold.story:forward_iter()</code> or <code>kobold.story:reverse_iter()</code>, which guarantee amortized worst-case iteration time complexity linear to the number of chunks in the story regardless of what the highest chunk number is.</p>
|
||||
<p>You can index this object to get a story chunk (as a <code>KoboldStoryChunk</code> object) by its number, which is an integer. The prompt chunk, if it exists, is guaranteed to be chunk 0. Aside from that, the chunk numbers are not guaranteed to be contiguous or ordered in any way.</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">local</span> prompt_chunk <span class="token operator">=</span> kobold<span class="token punctuation">.</span>story<span class="token punctuation">[</span><span class="token number">0</span><span class="token punctuation">]</span> <span class="token comment">-- KoboldStoryChunk object referring to the prompt chunk</span>
|
||||
</code></pre>
|
||||
<h3 id="methods-1">Methods:</h3>
|
||||
<ul>
|
||||
<li><code>kobold.story:forward_iter()</code></li>
|
||||
<li><code>kobold.story:reverse_iter()</code></li>
|
||||
</ul>
|
||||
<h1 id="kobold.storyforward_iter">kobold.story:forward_iter()</h1>
|
||||
<p><em><strong>Callable from</strong></em>: anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldStory<span class="token punctuation">:</span><span class="token function">forward_iter</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> <span class="token function">fun</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">:</span> KoboldStoryChunk<span class="token punctuation">,</span> table<span class="token punctuation">,</span> <span class="token keyword">nil</span>
|
||||
</code></pre>
|
||||
<p>Returns a stateful iterator that efficiently iterates through story chunks from top to bottom.</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">for</span> chunk <span class="token keyword">in</span> kobold<span class="token punctuation">.</span>story<span class="token punctuation">:</span><span class="token function">forward_iter</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token keyword">do</span>
|
||||
<span class="token function">print</span><span class="token punctuation">(</span>chunk<span class="token punctuation">.</span>num<span class="token punctuation">,</span> chunk<span class="token punctuation">.</span>content<span class="token punctuation">)</span>
|
||||
<span class="token keyword">end</span>
|
||||
</code></pre>
|
||||
<h1 id="kobold.storyreverse_iter">kobold.story:reverse_iter()</h1>
|
||||
<p><em><strong>Callable from</strong></em>: anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldStory<span class="token punctuation">:</span><span class="token function">reverse_iter</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> <span class="token function">fun</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">:</span> KoboldStoryChunk<span class="token punctuation">,</span> table<span class="token punctuation">,</span> <span class="token keyword">nil</span>
|
||||
</code></pre>
|
||||
<p>Returns a stateful iterator that efficiently iterates through story chunks from bottom to top.</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">for</span> chunk <span class="token keyword">in</span> kobold<span class="token punctuation">.</span>story<span class="token punctuation">:</span><span class="token function">reverse_iter</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token keyword">do</span>
|
||||
<span class="token function">print</span><span class="token punctuation">(</span>chunk<span class="token punctuation">.</span>num<span class="token punctuation">,</span> chunk<span class="token punctuation">.</span>content<span class="token punctuation">)</span>
|
||||
<span class="token keyword">end</span>
|
||||
</code></pre>
|
||||
<h1 id="koboldstorychunk">KoboldStoryChunk</h1>
|
||||
<p>Represents a story chunk.</p>
|
||||
<h3 id="fields-1">Fields:</h3>
|
||||
<ul>
|
||||
<li><code>KoboldStoryChunk.content</code></li>
|
||||
<li><code>KoboldStoryChunk.num</code></li>
|
||||
</ul>
|
||||
<h1 id="koboldstorychunk.content">KoboldStoryChunk.content</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere (triggers regeneration when written to from generation modifier)</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldStoryChunk<span class="token punctuation">.</span>content<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p>The text inside of the story chunk.</p>
|
||||
<p>Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.</p>
|
||||
<h1 id="koboldstorychunk.num">KoboldStoryChunk.num</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldStoryChunk<span class="token punctuation">.</span>num<span class="token punctuation">:</span> integer
|
||||
</code></pre>
|
||||
<p>The number of the story chunk. Chunk 0 is guaranteed to be the prompt chunk if it exists; no guarantees can be made about the numbers of other chunks.</p>
|
||||
<h1 id="kobold.submission">kobold.submission</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> input modifier</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field kobold<span class="token punctuation">.</span>submission<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p>The user-submitted text after being formatted by input formatting. If this is a repeated generation incurred by <code>kobold.restart_generation()</code>, then this is the empty string.</p>
|
||||
<h1 id="kobold.worldinfo">kobold.worldinfo</h1>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldLib<span class="token punctuation">.</span>worldinfo<span class="token punctuation">:</span> KoboldWorldInfo
|
||||
</code></pre>
|
||||
<p>Represents the world info entries.</p>
|
||||
<p>Indexing this object at index i returns the ith world info entry from the top in amortized constant worst-case time as a <code>KoboldWorldInfoEntry</code>. This includes world info entries that are inside folders.</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">local</span> entry <span class="token operator">=</span> kobold<span class="token punctuation">.</span>worldinfo<span class="token punctuation">[</span><span class="token number">5</span><span class="token punctuation">]</span> <span class="token comment">-- Retrieves fifth entry from top as a KoboldWorldInfoEntry</span>
|
||||
</code></pre>
|
||||
<p>You can use <code>ipairs</code> or a numeric loop to iterate from top to bottom:</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">for</span> index<span class="token punctuation">,</span> entry <span class="token keyword">in</span> <span class="token function">ipairs</span><span class="token punctuation">(</span>kobold<span class="token punctuation">.</span>worldinfo<span class="token punctuation">)</span><span class="token punctuation">:</span>
|
||||
<span class="token function">print</span><span class="token punctuation">(</span>index<span class="token punctuation">,</span> entry<span class="token punctuation">.</span>content<span class="token punctuation">)</span>
|
||||
<span class="token keyword">end</span>
|
||||
</code></pre>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">for</span> index <span class="token operator">=</span> <span class="token number">1</span><span class="token punctuation">,</span> <span class="token operator">#</span>kobold<span class="token punctuation">.</span>worldinfo <span class="token keyword">do</span><span class="token punctuation">:</span>
|
||||
<span class="token function">print</span><span class="token punctuation">(</span>index<span class="token punctuation">,</span> kobold<span class="token punctuation">.</span>worldinfo<span class="token punctuation">[</span>index<span class="token punctuation">]</span><span class="token punctuation">.</span>content<span class="token punctuation">)</span>
|
||||
<span class="token keyword">end</span>
|
||||
</code></pre>
|
||||
<h3 id="methods-2">Methods:</h3>
|
||||
<ul>
|
||||
<li><code>kobold.story:compute_context()</code></li>
|
||||
<li><code>kobold.story:finduid()</code></li>
|
||||
<li><code>kobold.story:is_valid()</code></li>
|
||||
</ul>
|
||||
<h3 id="fields-2">Fields:</h3>
|
||||
<ul>
|
||||
<li><code>kobold.story.folders</code></li>
|
||||
</ul>
|
||||
<h1 id="kobold.worldinfocompute_context">kobold.worldinfo:compute_context()</h1>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldWorldInfo<span class="token punctuation">:</span><span class="token function">compute_context</span><span class="token punctuation">(</span>submission<span class="token punctuation">:</span> string<span class="token punctuation">,</span> entries?<span class="token punctuation">:</span> KoboldWorldInfoEntry<span class="token operator">|</span>table<span class="token operator"><</span>any<span class="token punctuation">,</span> KoboldWorldInfoEntry<span class="token operator">></span><span class="token punctuation">,</span> kwargs?<span class="token punctuation">:</span> table<span class="token operator"><</span>string<span class="token punctuation">,</span> any<span class="token operator">></span><span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> string
|
||||
</code></pre>
|
||||
<p>Computes the context that would be sent to the generator with the user’s current settings if <code>submission</code> were the user’s input after being formatted by input formatting. The context would include memory at the top, followed by active world info entries, followed by some story chunks with the author’s note somewhere, followed by <code>submission</code>.</p>
|
||||
<h3 id="parameters-4">Parameters</h3>
|
||||
<ul>
|
||||
<li>submission (<code>string</code>): String to use as simulated user’s input after being formatted by input formatting.</li>
|
||||
<li>entries? (<code>KoboldWorldInfoEntry|table<any, KoboldWorldInfoEntry></code>): A <code>KoboldWorldInfoEntry</code> or table thereof that indicates an allowed subset of world info entries to include in the context. Defaults to all world info entries.</li>
|
||||
<li>kwargs? (<code>table<string, any></code>): Table of optional keyword arguments from the following list. Defaults to <code>{}</code>.
|
||||
<ul>
|
||||
<li>scan_story? (<code>boolean</code>): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to <code>false</code>, only the <code>submission</code> is scanned for world info keys. Defaults to <code>true</code>.</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<h3 id="returns-3">Returns</h3>
|
||||
<p><code>string</code>: Computed context.</p>
|
||||
<h1 id="kobold.worldinfofinduid">kobold.worldinfo:finduid()</h1>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldWorldInfo<span class="token punctuation">:</span><span class="token function">finduid</span><span class="token punctuation">(</span>u<span class="token punctuation">:</span> integer<span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> KoboldWorldInfoEntry?
|
||||
</code></pre>
|
||||
<p>Returns the world info entry with the given UID in amortized constant worst-case time, or <code>nil</code> if not found.</p>
|
||||
<h3 id="parameters-5">Parameters</h3>
|
||||
<ul>
|
||||
<li>u (<code>integer</code>): UID.</li>
|
||||
</ul>
|
||||
<h3 id="returns-4">Returns</h3>
|
||||
<ul>
|
||||
<li><code>KoboldWorldInfoEntry?</code>: The world info entry with requested UID, or <code>nil</code> if no such entry exists.</li>
|
||||
</ul>
|
||||
<h1 id="kobold.worldinfois_valid">kobold.worldinfo:is_valid()</h1>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldWorldInfo<span class="token punctuation">:</span><span class="token function">is_valid</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> boolean
|
||||
</code></pre>
|
||||
<p>This always returns true.</p>
|
||||
<h1 id="kobold.worldinfo.folders">kobold.worldinfo.folders</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfo<span class="token punctuation">.</span>folders<span class="token punctuation">:</span> KoboldWorldInfoFolderSelector
|
||||
</code></pre>
|
||||
<p>Can be indexed in amortized constant worst-case time and iterated over and has a <code>finduid</code> method just like <code>kobold.worldinfo</code>, but gets folders (as <code>KoboldWorldInfoFolder</code> objects) instead.</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">local</span> folder <span class="token operator">=</span> kobold<span class="token punctuation">.</span>worldinfo<span class="token punctuation">.</span>folders<span class="token punctuation">[</span><span class="token number">5</span><span class="token punctuation">]</span> <span class="token comment">-- Retrieves fifth folder from top as a KoboldWorldInfoFolder</span>
|
||||
</code></pre>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">for</span> index<span class="token punctuation">,</span> folder <span class="token keyword">in</span> <span class="token function">ipairs</span><span class="token punctuation">(</span>kobold<span class="token punctuation">.</span>worldinfo<span class="token punctuation">.</span>folders<span class="token punctuation">)</span><span class="token punctuation">:</span>
|
||||
<span class="token function">print</span><span class="token punctuation">(</span>index<span class="token punctuation">,</span> folder<span class="token punctuation">.</span>name<span class="token punctuation">)</span>
|
||||
<span class="token keyword">end</span>
|
||||
</code></pre>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">for</span> index <span class="token operator">=</span> <span class="token number">1</span><span class="token punctuation">,</span> <span class="token operator">#</span>kobold<span class="token punctuation">.</span>worldinfo<span class="token punctuation">.</span>folders <span class="token keyword">do</span><span class="token punctuation">:</span>
|
||||
<span class="token function">print</span><span class="token punctuation">(</span>index<span class="token punctuation">,</span> kobold<span class="token punctuation">.</span>worldinfo<span class="token punctuation">.</span>folders<span class="token punctuation">[</span>index<span class="token punctuation">]</span><span class="token punctuation">.</span>name<span class="token punctuation">)</span>
|
||||
<span class="token keyword">end</span>
|
||||
</code></pre>
|
||||
<h3 id="methods-3">Methods</h3>
|
||||
<ul>
|
||||
<li><code>kobold.story.folders:finduid()</code></li>
|
||||
<li><code>kobold.story.folders:is_valid()</code></li>
|
||||
</ul>
|
||||
<h1 id="koboldworldinfoentry">KoboldWorldInfoEntry</h1>
|
||||
<p>Represents a world info entry.</p>
|
||||
<h3 id="methods-4">Methods:</h3>
|
||||
<ul>
|
||||
<li><code>KoboldWorldInfoEntry:compute_context()</code></li>
|
||||
<li><code>KoboldWorldInfoEntry:is_valid()</code></li>
|
||||
</ul>
|
||||
<h3 id="fields-3">Fields:</h3>
|
||||
<ul>
|
||||
<li><code>KoboldWorldInfoEntry.comment</code></li>
|
||||
<li><code>KoboldWorldInfoEntry.constant</code></li>
|
||||
<li><code>KoboldWorldInfoEntry.content</code></li>
|
||||
<li><code>KoboldWorldInfoEntry.folder</code></li>
|
||||
<li><code>KoboldWorldInfoEntry.key</code></li>
|
||||
<li><code>KoboldWorldInfoEntry.keysecondary</code></li>
|
||||
<li><code>KoboldWorldInfoEntry.num</code></li>
|
||||
<li><code>KoboldWorldInfoEntry.selective</code></li>
|
||||
<li><code>KoboldWorldInfoEntry.uid</code></li>
|
||||
</ul>
|
||||
<h1 id="koboldworldinfoentrycompute_context">KoboldWorldInfoEntry:compute_context()</h1>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldWorldInfoEntry<span class="token punctuation">:</span><span class="token function">compute_context</span><span class="token punctuation">(</span>submission<span class="token punctuation">:</span> string<span class="token punctuation">,</span> kwargs?<span class="token punctuation">:</span> table<span class="token operator"><</span>string<span class="token punctuation">,</span> any<span class="token operator">></span><span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> string
|
||||
</code></pre>
|
||||
<p>The same as calling <code>kobold.worldinfo:compute_context()</code> with this world info entry as the argument.</p>
|
||||
<h3 id="parameters-6">Parameters</h3>
|
||||
<ul>
|
||||
<li>submission (<code>string</code>): String to use as simulated user’s input after being formatted by input formatting.</li>
|
||||
<li>kwargs? (<code>table<string, any></code>): Table of optional keyword arguments from the following list. Defaults to <code>{}</code>.
|
||||
<ul>
|
||||
<li>scan_story? (<code>boolean</code>): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to <code>false</code>, only the <code>submission</code> is scanned for world info keys. Defaults to <code>true</code>.</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<h3 id="returns-5">Returns</h3>
|
||||
<p><code>string</code>: Computed context.</p>
|
||||
<h1 id="koboldworldinfoentryis_valid">KoboldWorldInfoEntry:is_valid()</h1>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldWorldInfoEntry<span class="token punctuation">:</span><span class="token function">is_valid</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> boolean
|
||||
</code></pre>
|
||||
<p>Returns true if this world info entry still exists (i.e. wasn’t deleted), otherwise returns false.</p>
|
||||
<h1 id="koboldworldinfoentry.comment">KoboldWorldInfoEntry.comment</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfoEntry<span class="token punctuation">.</span>comment<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p>The world info entry’s comment that appears in its topmost text box.</p>
|
||||
<h1 id="koboldworldinfoentry.constant">KoboldWorldInfoEntry.constant</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere (triggers regeneration when written to from generation modifier)</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfoEntry<span class="token punctuation">.</span>constant<span class="token punctuation">:</span> boolean
|
||||
</code></pre>
|
||||
<p>Whether or not this world info entry is constant. Constant world info entries are always included in the context regardless of whether or not its keys match the story chunks in the context.</p>
|
||||
<p>Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.</p>
|
||||
<h1 id="koboldworldinfoentry.content">KoboldWorldInfoEntry.content</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere (triggers regeneration when written to from generation modifier)</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfoEntry<span class="token punctuation">.</span>content<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p>The text in the “What To Remember” text box that gets included in the context when the world info entry is active.</p>
|
||||
<p>Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.</p>
|
||||
<h1 id="koboldworldinfoentry.folder">KoboldWorldInfoEntry.folder</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfoEntryfolder<span class="token punctuation">:</span> integer?
|
||||
</code></pre>
|
||||
<p>UID of the folder the world info entry is in, or <code>nil</code> if it’s outside of a folder.</p>
|
||||
<h1 id="koboldworldinfoentry.key">KoboldWorldInfoEntry.key</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere (triggers regeneration when written to from generation modifier)</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfoEntry<span class="token punctuation">.</span>key<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p>For non-selective world info entries, this is the world info entry’s comma-separated list of keys. For selective world info entries, this is the comma-separated list of primary keys.</p>
|
||||
<p>Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.</p>
|
||||
<h1 id="koboldworldinfoentry.keysecondary">KoboldWorldInfoEntry.keysecondary</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere (triggers regeneration when written to from generation modifier)</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfoEntry<span class="token punctuation">.</span>keysecondary<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p>For non-selective world info entries, the value of this field is undefined and writing to it has no effect. For selective world info entries, this is the comma-separated list of secondary keys.</p>
|
||||
<p>Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.</p>
|
||||
<h1 id="koboldworldinfoentry.num">KoboldWorldInfoEntry.num</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfoEntry<span class="token punctuation">.</span>num<span class="token punctuation">:</span> integer
|
||||
</code></pre>
|
||||
<p>This is 0 if the entry is the first one from the top, 1 if second from the top, and so on.</p>
|
||||
<h1 id="koboldworldinfoentry.selective">KoboldWorldInfoEntry.selective</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere (triggers regeneration when written to from generation modifier)</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfoEntry<span class="token punctuation">.</span>selective<span class="token punctuation">:</span> boolean
|
||||
</code></pre>
|
||||
<p>Whether or not the world info entry is selective. Selective entries have both primary and secondary keys.</p>
|
||||
<p>Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.</p>
|
||||
<h1 id="koboldworldinfoentry.uid">KoboldWorldInfoEntry.uid</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfoEntry<span class="token punctuation">.</span>uid<span class="token punctuation">:</span> integer
|
||||
</code></pre>
|
||||
<p>UID of the world info entry.</p>
|
||||
<h1 id="koboldworldinfofolderselectorfinduid">KoboldWorldInfoFolderSelector:finduid()</h1>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldWorldInfoFolderSelector<span class="token punctuation">:</span><span class="token function">finduid</span><span class="token punctuation">(</span>u<span class="token punctuation">:</span> integer<span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> KoboldWorldInfoFolder?
|
||||
</code></pre>
|
||||
<p>Returns the world info folder with the given UID in amortized constant worst-case time, or <code>nil</code> if not found.</p>
|
||||
<h3 id="parameters-7">Parameters</h3>
|
||||
<ul>
|
||||
<li>u (<code>integer</code>): UID.</li>
|
||||
</ul>
|
||||
<h3 id="returns-6">Returns</h3>
|
||||
<ul>
|
||||
<li><code>KoboldWorldInfoFolder?</code>: The world info folder with requested UID, or <code>nil</code> if no such folder exists.</li>
|
||||
</ul>
|
||||
<h1 id="koboldworldinfofolderselectoris_valid">KoboldWorldInfoFolderSelector:is_valid()</h1>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldWorldInfoFolderSelector<span class="token punctuation">:</span><span class="token function">is_valid</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> boolean
|
||||
</code></pre>
|
||||
<p>This always returns true.</p>
|
||||
<h1 id="koboldworldinfofolder">KoboldWorldInfoFolder</h1>
|
||||
<p>Represents a world info folder.</p>
|
||||
<h3 id="methods-5">Methods:</h3>
|
||||
<ul>
|
||||
<li><code>KoboldWorldInfoFolder:compute_context()</code></li>
|
||||
<li><code>KoboldWorldInfoFolder:finduid()</code></li>
|
||||
<li><code>KoboldWorldInfoFolder:is_valid()</code></li>
|
||||
</ul>
|
||||
<h3 id="fields-4">Fields:</h3>
|
||||
<ul>
|
||||
<li><code>KoboldWorldInfoFolder.name</code></li>
|
||||
<li><code>KoboldWorldInfoFolder.uid</code></li>
|
||||
</ul>
|
||||
<h1 id="koboldworldinfofoldercompute_context">KoboldWorldInfoFolder:compute_context()</h1>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldWorldInfoFolder<span class="token punctuation">:</span><span class="token function">compute_context</span><span class="token punctuation">(</span>submission<span class="token punctuation">:</span> string<span class="token punctuation">,</span> entries?<span class="token punctuation">:</span> KoboldWorldInfoEntry<span class="token operator">|</span>table<span class="token operator"><</span>any<span class="token punctuation">,</span> KoboldWorldInfoEntry<span class="token operator">></span><span class="token punctuation">,</span> kwargs?<span class="token punctuation">:</span> table<span class="token operator"><</span>string<span class="token punctuation">,</span> any<span class="token operator">></span><span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> string
|
||||
</code></pre>
|
||||
<p>Computes the context that would be sent to the generator with the user’s current settings if <code>submission</code> were the user’s input after being formatted by input formatting. The context would include memory at the top, followed by active world info entries, followed by some story chunks with the author’s note somewhere, followed by <code>submission</code>.</p>
|
||||
<p>Unlike <code>kobold.worldinfo:compute_context()</code>, this function doesn’t include world info keys outside of the folder.</p>
|
||||
<h3 id="parameters-8">Parameters</h3>
|
||||
<ul>
|
||||
<li>submission (<code>string</code>): String to use as simulated user’s input after being formatted by input formatting.</li>
|
||||
<li>entries? (<code>KoboldWorldInfoEntry|table<any, KoboldWorldInfoEntry></code>): A <code>KoboldWorldInfoEntry</code> or table thereof that indicates an allowed subset of world info entries to include in the context. Entries that are not inside of the folder are still not included. Defaults to all world info entries in the folder.</li>
|
||||
<li>kwargs? (<code>table<string, any></code>): Table of optional keyword arguments from the following list. Defaults to <code>{}</code>.
|
||||
<ul>
|
||||
<li>scan_story? (<code>boolean</code>): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to <code>false</code>, only the <code>submission</code> is scanned for world info keys. Defaults to <code>true</code>.</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<h3 id="returns-7">Returns</h3>
|
||||
<p><code>string</code>: Computed context.</p>
|
||||
<h1 id="koboldworldinfofolderfinduid">KoboldWorldInfoFolder:finduid()</h1>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldWorldInfoFolder<span class="token punctuation">:</span><span class="token function">finduid</span><span class="token punctuation">(</span>u<span class="token punctuation">:</span> integer<span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> KoboldWorldInfoEntry?
|
||||
</code></pre>
|
||||
<p>Returns the world info entry inside of the folder with the given UID in amortized constant worst-case time, or <code>nil</code> if not found.</p>
|
||||
<h3 id="parameters-9">Parameters</h3>
|
||||
<ul>
|
||||
<li>u (<code>integer</code>): UID.</li>
|
||||
</ul>
|
||||
<h3 id="returns-8">Returns</h3>
|
||||
<ul>
|
||||
<li><code>KoboldWorldInfoEntry?</code>: The world info entry with requested UID, or <code>nil</code> if no such entry exists or if it’s outside of the folder.</li>
|
||||
</ul>
|
||||
<h1 id="koboldworldinfofolderis_valid">KoboldWorldInfoFolder:is_valid()</h1>
|
||||
<p><em><strong>Callable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua"><span class="token keyword">function</span> KoboldWorldInfoFolder<span class="token punctuation">:</span><span class="token function">is_valid</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">-</span><span class="token operator">></span> boolean
|
||||
</code></pre>
|
||||
<p>Returns whether or not the folder still exists (i.e. wasn’t deleted).</p>
|
||||
<h1 id="koboldworldinfofolder.name">KoboldWorldInfoFolder.name</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> anywhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfoFolder<span class="token punctuation">.</span>name<span class="token punctuation">:</span> string
|
||||
</code></pre>
|
||||
<p>Name of the world info folder as defined in its one text box.</p>
|
||||
<h1 id="koboldworldinfofolder.uid">KoboldWorldInfoFolder.uid</h1>
|
||||
<p><em><strong>Readable from:</strong></em> anywhere<br>
|
||||
<em><strong>Writable from:</strong></em> nowhere</p>
|
||||
<pre class=" language-lua"><code class="prism language-lua">field KoboldWorldInfoFolder<span class="token punctuation">.</span>uid<span class="token punctuation">:</span> integer
|
||||
</code></pre>
|
||||
<p>UID of the world info folder.</p>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
859
userscripts/api_documentation.md
Normal file
859
userscripts/api_documentation.md
Normal file
@ -0,0 +1,859 @@
|
||||
# kobold
|
||||
|
||||
```lua
|
||||
global kobold: KoboldLib
|
||||
```
|
||||
|
||||
### Methods:
|
||||
* `kobold.decode()`
|
||||
* `kobold.encode()`
|
||||
* `kobold.get_config_file()`
|
||||
* `kobold.halt_generation()`
|
||||
* `kobold.restart_generation()`
|
||||
|
||||
### Fields:
|
||||
* `kobold.authorsnote`
|
||||
* `kobold.authorsnotetemplate`
|
||||
* `kobold.custmodpth`
|
||||
* `kobold.feedback`
|
||||
* `kobold.generated`
|
||||
* `kobold.generated_cols`
|
||||
* `kobold.generated_rows`
|
||||
* `kobold.is_config_file_open`
|
||||
* `kobold.logits`
|
||||
* `kobold.logits_cols`
|
||||
* `kobold.logits_rows`
|
||||
* `kobold.memory`
|
||||
* `kobold.modelbackend`
|
||||
* `kobold.modeltype`
|
||||
* `kobold.num_outputs`
|
||||
* `kobold.outputs`
|
||||
* `kobold.settings`
|
||||
* `kobold.story`
|
||||
* `kobold.submission`
|
||||
* `kobold.worldinfo`
|
||||
|
||||
# kobold.decode()
|
||||
|
||||
```lua
|
||||
function KoboldLib.decode(tok: integer|table<integer, integer>) -> string
|
||||
```
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
Decodes the given token or list of tokens using the current tokenizer. If `kobold.backend` is `'readonly'` or `'api'`, the tokenizer used is the GPT-2 tokenizer, otherwise the model's tokenizer is used. This function is the inverse of `kobold.encode()`.
|
||||
|
||||
```lua
|
||||
print(kobold.decode({15496, 2159})) -- 'Hello World'
|
||||
```
|
||||
|
||||
### Parameters:
|
||||
* tok (`integer|table<integer, integer>`): Array of token IDs to decode, or the token ID of a single token.
|
||||
|
||||
### Returns:
|
||||
* `string`: Decoded tokens.
|
||||
|
||||
# kobold.encode()
|
||||
|
||||
```lua
|
||||
function KoboldLib.encode(str: string) -> table<integer, integer>
|
||||
```
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
Encodes the given string using the current tokenizer into an array of tokens. If `kobold.backend` is `'readonly'` or `'api'`, the tokenizer used is the GPT-2 tokenizer, otherwise the model's tokenizer is used. This function is the inverse of `kobold.decode()`.
|
||||
|
||||
```lua
|
||||
local tokens = kobold.encode("Hello World")
|
||||
print(#tokens) -- 2
|
||||
print(tokens[1]) -- 15496
|
||||
print(tokens[2]) -- 2159
|
||||
```
|
||||
|
||||
### Parameters:
|
||||
* tok (`integer|table<integer, integer>`): Array of token IDs to decode, or the token ID of a single token.
|
||||
|
||||
### Returns:
|
||||
* `string`: Decoded tokens.
|
||||
|
||||
# kobold.get_config_file()
|
||||
|
||||
```lua
|
||||
function KoboldLib.get_config_file(clear?: boolean) -> file*
|
||||
```
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
Returns a file handle representing your script's configuration file, which is usually the file in the userscripts folder with the same filename as your script but with ".conf" appended at the end. This function throws an error on failure.
|
||||
|
||||
If the configuration file does not exist when this function is called, the configuration file will first be created as a new empty file.
|
||||
|
||||
If KoboldAI does not possess an open file handle to the configuration file, this function opens the file in `w+b` mode if the `clear` parameter is a truthy value, otherwise the file is opened in `r+b` mode. These are mostly the same -- the file is opened in binary read-write mode and then seeked to the start of the file -- except the former mode deletes the contents of the file prior to opening it and the latter mode does not.
|
||||
|
||||
If KoboldAI does possess an open file handle to the configuration file, that open file handle is returned without seeking or deleting the contents of the file. You can check if KoboldAI possesses an open file handle to the configuration file by using `kobold.is_config_file_open`.
|
||||
|
||||
```lua
|
||||
local example_config = "return 'Hello World'"
|
||||
local cfg
|
||||
do
|
||||
-- If config file is empty, write example config
|
||||
local f <close> = kobold.get_config_file()
|
||||
f:seek("set")
|
||||
if f:read(1) == nil then f:write(example_config) end
|
||||
f:seek("set")
|
||||
|
||||
-- Read config
|
||||
local err
|
||||
cfg, err = load(f:read("a"))
|
||||
if err ~= nil then error(err) end
|
||||
cfg = cfg()
|
||||
end
|
||||
```
|
||||
|
||||
### Parameters:
|
||||
* clear? (`bool`): If KoboldAI does not possess an open file handle to the configuration object, this determines whether the file will be opened in `w+b` or `r+b` mode. This parameter defaults to `false`.
|
||||
|
||||
### Returns:
|
||||
* `file*`: File handle for the configuration file.
|
||||
|
||||
# kobold.halt_generation()
|
||||
|
||||
```lua
|
||||
function KoboldLib.halt_generation() -> nil
|
||||
```
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
If called from an input modifier, prevents the user's input from being sent to the model and skips directly to the output modifier.
|
||||
|
||||
If called from a generation modifier, stops generation after the current token is generated and then skips to the output modifier. In other words, if, when you call this function, `kobold.generated` has n columns, it will have exactly n+1 columns when the output modifier is called.
|
||||
|
||||
If called from an output modifier, has no effect.
|
||||
|
||||
# kobold.restart_generation()
|
||||
|
||||
```lua
|
||||
function KoboldLib.restart_generation(sequence?: integer) -> nil
|
||||
```
|
||||
|
||||
***Callable from:*** output modifier
|
||||
|
||||
After the current output is sent to the GUI, starts another generation using the empty string as the submission.
|
||||
|
||||
Whatever ends up being the output selected by the user or by the `sequence` parameter will be saved in `kobold.feedback` when the new generation begins.
|
||||
|
||||
### Parameters:
|
||||
* sequence? (`integer`): If you have multiple Gens Per Action, this can be used to choose which sequence to use as the output, where 1 is the first, 2 is the second and so on. If you set this to 0, the user will be prompted to choose the sequence instead. Defaults to 0.
|
||||
|
||||
# kobold.authorsnote
|
||||
|
||||
```lua
|
||||
field KoboldLib.authorsnote: string
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere (triggers regeneration when written to from generation modifier)
|
||||
|
||||
The author's note as set from the "Memory" button in the GUI.
|
||||
|
||||
Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.
|
||||
|
||||
# kobold.authorsnotetemplate
|
||||
|
||||
```lua
|
||||
field KoboldLib.authorsnotetemplate: string
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere (triggers regeneration when written to from generation modifier)
|
||||
|
||||
The author's note template as set from the "Memory" button in the GUI.
|
||||
|
||||
Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.
|
||||
|
||||
# kobold.custmodpth
|
||||
|
||||
```lua
|
||||
field KoboldLib.custmodpth: string
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
Path to a directory that the user chose either via the file dialog that appears when KoboldAI asks you to choose the path to your custom model or via the `--path` command-line flag.
|
||||
|
||||
If the user loaded a built-in model from the menu, this is instead the model ID of the model on Hugging Face's model hub, such as "KoboldAI/GPT-Neo-2.7B-Picard" or "hakurei/lit-6B".
|
||||
|
||||
# kobold.feedback
|
||||
|
||||
```lua
|
||||
field KoboldLib.feedback: string?
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
If this is a repeat generation caused by `kobold.restart_generation()`, this will be a string containing the previous output. If not, this will be `nil`.
|
||||
|
||||
# kobold.generated
|
||||
|
||||
```lua
|
||||
field KoboldLib.generated: table<integer, table<integer, integer>>?
|
||||
```
|
||||
|
||||
***Readable from:*** generation modifier and output modifier, but only if `kobold.modelbackend` is not `'api'` or `'readonly'`
|
||||
***Writable from:*** generation modifier
|
||||
|
||||
Two-dimensional array of tokens generated thus far. Each row represents one sequence, each column one token.
|
||||
|
||||
# kobold.generated_cols
|
||||
|
||||
```lua
|
||||
field KoboldLib.generated_cols: integer
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
Number of columns in `kobold.generated`. In other words, the number of tokens generated thus far, which is equal to the number of times that the generation modifier has been called, not including the current time if this is being read from a generation modifier.
|
||||
|
||||
If `kobold.modelbackend` is `'api'` or `'readonly'`, this returns 0 instead.
|
||||
|
||||
# kobold.generated_rows
|
||||
|
||||
```lua
|
||||
field KoboldLib.generated_rows: integer
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
Number of rows in `kobold.generated`, equal to `kobold.settings.numseqs`.
|
||||
|
||||
# kobold.is_config_file_open
|
||||
|
||||
```lua
|
||||
field KoboldLib.is_config_file_open: boolean
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
Whether or not KoboldAI possesses an open file handle to your script's configuration file. See `kobold.get_config_file()` for more details.
|
||||
|
||||
|
||||
# kobold.logits
|
||||
|
||||
```lua
|
||||
field KoboldLib.logits: table<integer, table<integer, number>>?
|
||||
```
|
||||
|
||||
***Readable from:*** generation modifier, but only if `kobold.modelbackend` is not `'api'` or `'readonly'`
|
||||
***Writable from:*** generation modifier
|
||||
|
||||
Two-dimensional array of [logits](https://datascience.stackexchange.com/questions/31041/what-does-logits-in-machine-learning-mean) prior to being filtered by top-p sampling, etc. Each row represents one sequence, each column one of the tokens in the model's vocabulary. The ith column represents the logit score of token i-1, so if you want to access the logit score of token 18435 (" Hello" with a leading space), you need to access column 18436. You may alter this two-dimensional array to encourage or deter certain tokens from appearing in the output in a stochastic manner.
|
||||
|
||||
Don't modify this table unnecessarily unless you know what you are doing! The bias example scripts show how to use this feature properly.
|
||||
|
||||
# kobold.logits_cols
|
||||
|
||||
```lua
|
||||
field KoboldLib.logits_cols: integer
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
Number of columns in `kobold.logits`, equal to the vocabulary size of the current model. Most models based on GPT-2 (e.g. GPT-Neo and GPT-J) have a vocabulary size of 50257. GPT-J models in particular have a vocabulary size of 50400 instead, although GPT-J models aren't trained to use the rightmost 143 tokens of the logits array.
|
||||
|
||||
If `kobold.modelbackend` is `'api'` or `'readonly'`, this returns 0 instead.
|
||||
|
||||
# kobold.logits_rows
|
||||
|
||||
```lua
|
||||
field KoboldLib.logits_rows: integer
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
Number of rows in `kobold.generated`, equal to `kobold.settings.numseqs`. a local KoboldAI install.
|
||||
|
||||
# kobold.memory
|
||||
|
||||
```lua
|
||||
field KoboldLib.memory: string
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere (triggers regeneration when written to from generation modifier)
|
||||
|
||||
The memory as set from the "Memory" button in the GUI.
|
||||
|
||||
Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.
|
||||
|
||||
# kobold.modelbackend
|
||||
|
||||
```lua
|
||||
field KoboldLib.modelbackend: string
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
One of the following values:
|
||||
|
||||
* `'api'`: InferKit, OpenAI or legacy Colab mode
|
||||
* `'readonly'`: Read-only no AI mode
|
||||
* `'transformers'`: Models running on your own computer, and Colab GPU backend (currently used for 2.7B models on Colab)
|
||||
* `'mtj'`: Colab TPU backend (currently used for 6B models on Colab)
|
||||
|
||||
# kobold.modeltype
|
||||
|
||||
```lua
|
||||
field KoboldLib.modeltype: string
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
One of the following values:
|
||||
|
||||
* `'api'`: InferKit, OpenAI or legacy Colab mode
|
||||
* `'readonly'`: Read-only no AI mode
|
||||
* `'unknown'`
|
||||
* `'gpt2'`: GPT-2-Small
|
||||
* `'gpt2-medium'`
|
||||
* `'gpt2-large'`
|
||||
* `'gpt2-xl'`
|
||||
* `'gpt-neo-125M'`
|
||||
* `'gpt-neo-1.3B'`
|
||||
* `'gpt-neo-2.7B'`
|
||||
* `'gpt-j-6B'`
|
||||
|
||||
# kobold.num_outputs
|
||||
|
||||
```lua
|
||||
field KoboldLib.num_outputs: integer
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
Number of rows in `kobold.outputs`. This is equal to `kobold.settings.numseqs` unless you're using a non-Colab third-party API such as OpenAI or InferKit, in which case this is 1. If you decide to write to `kobold.settings.numseqs` from an output modifier, this value remains unchanged.
|
||||
|
||||
# kobold.outputs
|
||||
|
||||
```lua
|
||||
field KoboldLib.outputs: table<integer, string>
|
||||
```
|
||||
|
||||
***Readable from:*** output modifier
|
||||
***Writable from:*** output modifier
|
||||
|
||||
Model output before applying output formatting. One row per "Gens Per Action", unless you're using OpenAI or InferKit, in which case this always has exactly one row.
|
||||
|
||||
# kobold.settings
|
||||
|
||||
```lua
|
||||
field KoboldLib.settings: KoboldSettings
|
||||
```
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere (does not affect other scripts when written to since each script has its own copy of this object)
|
||||
|
||||
Contains most of the settings. They have the same names as in gensettings.py, so the top-p value is `kobold.settings.settopp`.
|
||||
|
||||
All the settings can be read from anywhere and written from anywhere, except `kobold.settings.numseqs` which can only be written to from an input modifier or output modifier.
|
||||
|
||||
Modifying certain fields from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess. Currently, only the following fields and their aliases cause this to occur:
|
||||
* `kobold.settings.settknmax` (Max Tokens)
|
||||
* `kobold.settings.anotedepth` (Author's Note Depth)
|
||||
* `kobold.settings.setwidepth` (World Info Depth)
|
||||
* `kobold.settings.setuseprompt` (Always Use Prompt)
|
||||
|
||||
# kobold.story
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
```lua
|
||||
field KoboldLib.story: KoboldStory
|
||||
```
|
||||
|
||||
Contains the chunks of the current story. Don't use `pairs` or `ipairs` to iterate over the story chunks, use `kobold.story:forward_iter()` or `kobold.story:reverse_iter()`, which guarantee amortized worst-case iteration time complexity linear to the number of chunks in the story regardless of what the highest chunk number is.
|
||||
|
||||
You can index this object to get a story chunk (as a `KoboldStoryChunk` object) by its number, which is an integer. The prompt chunk, if it exists, is guaranteed to be chunk 0. Aside from that, the chunk numbers are not guaranteed to be contiguous or ordered in any way.
|
||||
|
||||
```lua
|
||||
local prompt_chunk = kobold.story[0] -- KoboldStoryChunk object referring to the prompt chunk
|
||||
```
|
||||
|
||||
### Methods:
|
||||
|
||||
* `kobold.story:forward_iter()`
|
||||
* `kobold.story:reverse_iter()`
|
||||
|
||||
# kobold.story:forward_iter()
|
||||
|
||||
***Callable from***: anywhere
|
||||
|
||||
```lua
|
||||
function KoboldStory:forward_iter() -> fun(): KoboldStoryChunk, table, nil
|
||||
```
|
||||
|
||||
Returns a stateful iterator that efficiently iterates through story chunks from top to bottom.
|
||||
|
||||
```lua
|
||||
for chunk in kobold.story:forward_iter() do
|
||||
print(chunk.num, chunk.content)
|
||||
end
|
||||
```
|
||||
|
||||
# kobold.story:reverse_iter()
|
||||
|
||||
***Callable from***: anywhere
|
||||
|
||||
```lua
|
||||
function KoboldStory:reverse_iter() -> fun(): KoboldStoryChunk, table, nil
|
||||
```
|
||||
|
||||
Returns a stateful iterator that efficiently iterates through story chunks from bottom to top.
|
||||
|
||||
```lua
|
||||
for chunk in kobold.story:reverse_iter() do
|
||||
print(chunk.num, chunk.content)
|
||||
end
|
||||
```
|
||||
|
||||
# KoboldStoryChunk
|
||||
|
||||
Represents a story chunk.
|
||||
|
||||
### Fields:
|
||||
|
||||
* `KoboldStoryChunk.content`
|
||||
* `KoboldStoryChunk.num`
|
||||
|
||||
# KoboldStoryChunk.content
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere (triggers regeneration when written to from generation modifier)
|
||||
|
||||
```lua
|
||||
field KoboldStoryChunk.content: string
|
||||
```
|
||||
|
||||
The text inside of the story chunk.
|
||||
|
||||
Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.
|
||||
|
||||
# KoboldStoryChunk.num
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
```lua
|
||||
field KoboldStoryChunk.num: integer
|
||||
```
|
||||
|
||||
The number of the story chunk. Chunk 0 is guaranteed to be the prompt chunk if it exists; no guarantees can be made about the numbers of other chunks.
|
||||
|
||||
# kobold.submission
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** input modifier
|
||||
|
||||
```lua
|
||||
field kobold.submission: string
|
||||
```
|
||||
|
||||
The user-submitted text after being formatted by input formatting. If this is a repeated generation incurred by `kobold.restart_generation()`, then this is the empty string.
|
||||
|
||||
# kobold.worldinfo
|
||||
|
||||
```lua
|
||||
field KoboldLib.worldinfo: KoboldWorldInfo
|
||||
```
|
||||
|
||||
Represents the world info entries.
|
||||
|
||||
Indexing this object at index i returns the ith world info entry from the top in amortized constant worst-case time as a `KoboldWorldInfoEntry`. This includes world info entries that are inside folders.
|
||||
|
||||
```lua
|
||||
local entry = kobold.worldinfo[5] -- Retrieves fifth entry from top as a KoboldWorldInfoEntry
|
||||
```
|
||||
|
||||
You can use `ipairs` or a numeric loop to iterate from top to bottom:
|
||||
|
||||
```lua
|
||||
for index, entry in ipairs(kobold.worldinfo):
|
||||
print(index, entry.content)
|
||||
end
|
||||
```
|
||||
|
||||
```lua
|
||||
for index = 1, #kobold.worldinfo do:
|
||||
print(index, kobold.worldinfo[index].content)
|
||||
end
|
||||
```
|
||||
|
||||
### Methods:
|
||||
* `kobold.story:compute_context()`
|
||||
* `kobold.story:finduid()`
|
||||
* `kobold.story:is_valid()`
|
||||
|
||||
### Fields:
|
||||
* `kobold.story.folders`
|
||||
|
||||
# kobold.worldinfo:compute_context()
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
```lua
|
||||
function KoboldWorldInfo:compute_context(submission: string, entries?: KoboldWorldInfoEntry|table<any, KoboldWorldInfoEntry>, kwargs?: table<string, any>) -> string
|
||||
```
|
||||
|
||||
Computes the context that would be sent to the generator with the user's current settings if `submission` were the user's input after being formatted by input formatting. The context would include memory at the top, followed by active world info entries, followed by some story chunks with the author's note somewhere, followed by `submission`.
|
||||
|
||||
### Parameters
|
||||
|
||||
* submission (`string`): String to use as simulated user's input after being formatted by input formatting.
|
||||
* entries? (`KoboldWorldInfoEntry|table<any, KoboldWorldInfoEntry>`): A `KoboldWorldInfoEntry` or table thereof that indicates an allowed subset of world info entries to include in the context. Defaults to all world info entries.
|
||||
* kwargs? (`table<string, any>`): Table of optional keyword arguments from the following list. Defaults to `{}`.
|
||||
* scan_story? (`boolean`): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to `false`, only the `submission` is scanned for world info keys. Defaults to `true`.
|
||||
|
||||
### Returns
|
||||
|
||||
`string`: Computed context.
|
||||
|
||||
# kobold.worldinfo:finduid()
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
```lua
|
||||
function KoboldWorldInfo:finduid(u: integer) -> KoboldWorldInfoEntry?
|
||||
```
|
||||
|
||||
Returns the world info entry with the given UID in amortized constant worst-case time, or `nil` if not found.
|
||||
|
||||
### Parameters
|
||||
|
||||
* u (`integer`): UID.
|
||||
|
||||
### Returns
|
||||
|
||||
* `KoboldWorldInfoEntry?`: The world info entry with requested UID, or `nil` if no such entry exists.
|
||||
|
||||
# kobold.worldinfo:is_valid()
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
```lua
|
||||
function KoboldWorldInfo:is_valid() -> boolean
|
||||
```
|
||||
|
||||
This always returns true.
|
||||
|
||||
# kobold.worldinfo.folders
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfo.folders: KoboldWorldInfoFolderSelector
|
||||
```
|
||||
|
||||
Can be indexed in amortized constant worst-case time and iterated over and has a `finduid` method just like `kobold.worldinfo`, but gets folders (as `KoboldWorldInfoFolder` objects) instead.
|
||||
|
||||
```lua
|
||||
local folder = kobold.worldinfo.folders[5] -- Retrieves fifth folder from top as a KoboldWorldInfoFolder
|
||||
```
|
||||
|
||||
```lua
|
||||
for index, folder in ipairs(kobold.worldinfo.folders):
|
||||
print(index, folder.name)
|
||||
end
|
||||
```
|
||||
|
||||
```lua
|
||||
for index = 1, #kobold.worldinfo.folders do:
|
||||
print(index, kobold.worldinfo.folders[index].name)
|
||||
end
|
||||
```
|
||||
|
||||
### Methods
|
||||
* `kobold.story.folders:finduid()`
|
||||
* `kobold.story.folders:is_valid()`
|
||||
|
||||
# KoboldWorldInfoEntry
|
||||
|
||||
Represents a world info entry.
|
||||
|
||||
### Methods:
|
||||
* `KoboldWorldInfoEntry:compute_context()`
|
||||
* `KoboldWorldInfoEntry:is_valid()`
|
||||
|
||||
### Fields:
|
||||
* `KoboldWorldInfoEntry.comment`
|
||||
* `KoboldWorldInfoEntry.constant`
|
||||
* `KoboldWorldInfoEntry.content`
|
||||
* `KoboldWorldInfoEntry.folder`
|
||||
* `KoboldWorldInfoEntry.key`
|
||||
* `KoboldWorldInfoEntry.keysecondary`
|
||||
* `KoboldWorldInfoEntry.num`
|
||||
* `KoboldWorldInfoEntry.selective`
|
||||
* `KoboldWorldInfoEntry.uid`
|
||||
|
||||
# KoboldWorldInfoEntry:compute_context()
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
```lua
|
||||
function KoboldWorldInfoEntry:compute_context(submission: string, kwargs?: table<string, any>) -> string
|
||||
```
|
||||
|
||||
The same as calling `kobold.worldinfo:compute_context()` with this world info entry as the argument.
|
||||
|
||||
### Parameters
|
||||
|
||||
* submission (`string`): String to use as simulated user's input after being formatted by input formatting.
|
||||
* kwargs? (`table<string, any>`): Table of optional keyword arguments from the following list. Defaults to `{}`.
|
||||
* scan_story? (`boolean`): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to `false`, only the `submission` is scanned for world info keys. Defaults to `true`.
|
||||
|
||||
### Returns
|
||||
|
||||
`string`: Computed context.
|
||||
|
||||
# KoboldWorldInfoEntry:is_valid()
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
```lua
|
||||
function KoboldWorldInfoEntry:is_valid() -> boolean
|
||||
```
|
||||
|
||||
Returns true if this world info entry still exists (i.e. wasn't deleted), otherwise returns false.
|
||||
|
||||
# KoboldWorldInfoEntry.comment
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfoEntry.comment: string
|
||||
```
|
||||
|
||||
The world info entry's comment that appears in its topmost text box.
|
||||
|
||||
# KoboldWorldInfoEntry.constant
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere (triggers regeneration when written to from generation modifier)
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfoEntry.constant: boolean
|
||||
```
|
||||
|
||||
Whether or not this world info entry is constant. Constant world info entries are always included in the context regardless of whether or not its keys match the story chunks in the context.
|
||||
|
||||
Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.
|
||||
|
||||
# KoboldWorldInfoEntry.content
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere (triggers regeneration when written to from generation modifier)
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfoEntry.content: string
|
||||
```
|
||||
|
||||
The text in the "What To Remember" text box that gets included in the context when the world info entry is active.
|
||||
|
||||
Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.
|
||||
|
||||
# KoboldWorldInfoEntry.folder
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfoEntryfolder: integer?
|
||||
```
|
||||
|
||||
UID of the folder the world info entry is in, or `nil` if it's outside of a folder.
|
||||
|
||||
# KoboldWorldInfoEntry.key
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere (triggers regeneration when written to from generation modifier)
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfoEntry.key: string
|
||||
```
|
||||
|
||||
For non-selective world info entries, this is the world info entry's comma-separated list of keys. For selective world info entries, this is the comma-separated list of primary keys.
|
||||
|
||||
Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.
|
||||
|
||||
# KoboldWorldInfoEntry.keysecondary
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere (triggers regeneration when written to from generation modifier)
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfoEntry.keysecondary: string
|
||||
```
|
||||
|
||||
For non-selective world info entries, the value of this field is undefined and writing to it has no effect. For selective world info entries, this is the comma-separated list of secondary keys.
|
||||
|
||||
Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.
|
||||
|
||||
# KoboldWorldInfoEntry.num
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfoEntry.num: integer
|
||||
```
|
||||
|
||||
This is 0 if the entry is the first one from the top, 1 if second from the top, and so on.
|
||||
|
||||
# KoboldWorldInfoEntry.selective
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere (triggers regeneration when written to from generation modifier)
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfoEntry.selective: boolean
|
||||
```
|
||||
|
||||
Whether or not the world info entry is selective. Selective entries have both primary and secondary keys.
|
||||
|
||||
Modifying this field from inside of a generation modifier triggers a regeneration, which means that the context is recomputed after modification and generation begins again with the new context and previously generated tokens. This incurs a small performance penalty and should not be performed in excess.
|
||||
|
||||
# KoboldWorldInfoEntry.uid
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfoEntry.uid: integer
|
||||
```
|
||||
|
||||
UID of the world info entry.
|
||||
|
||||
# KoboldWorldInfoFolderSelector:finduid()
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
```lua
|
||||
function KoboldWorldInfoFolderSelector:finduid(u: integer) -> KoboldWorldInfoFolder?
|
||||
```
|
||||
|
||||
Returns the world info folder with the given UID in amortized constant worst-case time, or `nil` if not found.
|
||||
|
||||
### Parameters
|
||||
|
||||
* u (`integer`): UID.
|
||||
|
||||
### Returns
|
||||
|
||||
* `KoboldWorldInfoFolder?`: The world info folder with requested UID, or `nil` if no such folder exists.
|
||||
|
||||
# KoboldWorldInfoFolderSelector:is_valid()
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
```lua
|
||||
function KoboldWorldInfoFolderSelector:is_valid() -> boolean
|
||||
```
|
||||
|
||||
This always returns true.
|
||||
|
||||
# KoboldWorldInfoFolder
|
||||
|
||||
Represents a world info folder.
|
||||
|
||||
### Methods:
|
||||
* `KoboldWorldInfoFolder:compute_context()`
|
||||
* `KoboldWorldInfoFolder:finduid()`
|
||||
* `KoboldWorldInfoFolder:is_valid()`
|
||||
|
||||
### Fields:
|
||||
* `KoboldWorldInfoFolder.name`
|
||||
* `KoboldWorldInfoFolder.uid`
|
||||
|
||||
# KoboldWorldInfoFolder:compute_context()
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
```lua
|
||||
function KoboldWorldInfoFolder:compute_context(submission: string, entries?: KoboldWorldInfoEntry|table<any, KoboldWorldInfoEntry>, kwargs?: table<string, any>) -> string
|
||||
```
|
||||
|
||||
Computes the context that would be sent to the generator with the user's current settings if `submission` were the user's input after being formatted by input formatting. The context would include memory at the top, followed by active world info entries, followed by some story chunks with the author's note somewhere, followed by `submission`.
|
||||
|
||||
Unlike `kobold.worldinfo:compute_context()`, this function doesn't include world info keys outside of the folder.
|
||||
|
||||
### Parameters
|
||||
|
||||
* submission (`string`): String to use as simulated user's input after being formatted by input formatting.
|
||||
* entries? (`KoboldWorldInfoEntry|table<any, KoboldWorldInfoEntry>`): A `KoboldWorldInfoEntry` or table thereof that indicates an allowed subset of world info entries to include in the context. Entries that are not inside of the folder are still not included. Defaults to all world info entries in the folder.
|
||||
* kwargs? (`table<string, any>`): Table of optional keyword arguments from the following list. Defaults to `{}`.
|
||||
* scan_story? (`boolean`): Whether or not to scan the past few actions of the story for world info keys in addition to the submission like how world info normally behaves. If this is set to `false`, only the `submission` is scanned for world info keys. Defaults to `true`.
|
||||
|
||||
### Returns
|
||||
|
||||
`string`: Computed context.
|
||||
|
||||
# KoboldWorldInfoFolder:finduid()
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
```lua
|
||||
function KoboldWorldInfoFolder:finduid(u: integer) -> KoboldWorldInfoEntry?
|
||||
```
|
||||
|
||||
Returns the world info entry inside of the folder with the given UID in amortized constant worst-case time, or `nil` if not found.
|
||||
|
||||
### Parameters
|
||||
|
||||
* u (`integer`): UID.
|
||||
|
||||
### Returns
|
||||
|
||||
* `KoboldWorldInfoEntry?`: The world info entry with requested UID, or `nil` if no such entry exists or if it's outside of the folder.
|
||||
|
||||
# KoboldWorldInfoFolder:is_valid()
|
||||
|
||||
***Callable from:*** anywhere
|
||||
|
||||
```lua
|
||||
function KoboldWorldInfoFolder:is_valid() -> boolean
|
||||
```
|
||||
|
||||
Returns whether or not the folder still exists (i.e. wasn't deleted).
|
||||
|
||||
# KoboldWorldInfoFolder.name
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** anywhere
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfoFolder.name: string
|
||||
```
|
||||
|
||||
Name of the world info folder as defined in its one text box.
|
||||
|
||||
# KoboldWorldInfoFolder.uid
|
||||
|
||||
***Readable from:*** anywhere
|
||||
***Writable from:*** nowhere
|
||||
|
||||
```lua
|
||||
field KoboldWorldInfoFolder.uid: integer
|
||||
```
|
||||
|
||||
UID of the world info folder.
|
||||
2
userscripts/examples/Readme.txt
Normal file
2
userscripts/examples/Readme.txt
Normal file
@ -0,0 +1,2 @@
|
||||
This folder contains example code for KoboldAI, to use these scripts move them to the userscripts folder.
|
||||
Anything in this folder is considered official, make sure to copy it to the userscripts folder first if you do not want changes overwritten or commited as examples.
|
||||
@ -1,16 +1,16 @@
|
||||
:: This file is part of KoboldAI.
|
||||
::
|
||||
:: KoboldAI is free software: you can redistribute it and/or modify
|
||||
:: it under the terms of the GNU Affero General Public License as published by
|
||||
:: the Free Software Foundation, either version 3 of the License, or
|
||||
:: (at your option) any later version.
|
||||
::
|
||||
:: This program is distributed in the hope that it will be useful,
|
||||
:: but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
:: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
:: GNU Affero General Public License for more details.
|
||||
::
|
||||
:: You should have received a copy of the GNU Affero General Public License
|
||||
:: along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
@powershell ./build.ps1 %1
|
||||
:: This file is part of KoboldAI.
|
||||
::
|
||||
:: KoboldAI is free software: you can redistribute it and/or modify
|
||||
:: it under the terms of the GNU Affero General Public License as published by
|
||||
:: the Free Software Foundation, either version 3 of the License, or
|
||||
:: (at your option) any later version.
|
||||
::
|
||||
:: This program is distributed in the hope that it will be useful,
|
||||
:: but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
:: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
:: GNU Affero General Public License for more details.
|
||||
::
|
||||
:: You should have received a copy of the GNU Affero General Public License
|
||||
:: along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
@powershell ./build.ps1 %1
|
||||
@ -1,21 +1,21 @@
|
||||
# This file is part of KoboldAI.
|
||||
#
|
||||
# KoboldAI is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
$path = "out.lua"
|
||||
if ($args[0].length -gt 0) {
|
||||
$path = $args[0]
|
||||
}
|
||||
haxe --lua $path -L littleBigInt --main Main
|
||||
if (-not $?) { exit 1 }
|
||||
(Get-Content $path).replace('_G.require("rex_pcre")', '({flags = function() return {CASELESS = 1, DOTALL = 1, MULTILINE = 1, UCP = 1, UTF8 = 1} end, gsub = function() return "" end, new = function() return {} end})').replace("return _hx_exports", "return _hx_exports.Main").replace(" _hx_bit_raw = _G.require('bit32')", " _hx_bit_raw = {arshift = function(x, n) local y = x >> n; if (x < 0) then y = y | ~(-1 >> n) end return y end, band = function(x, y) return x & y end, bor = function(x, y) return x | y end, bnot = function(x) return ~x end, bxor = function(x, y) return x ~ y end, lshift = function(x, n) return x << n end, rshift = function(x, n) return x >> n end}").replace('__lua_lib_luautf8_Utf8 = _G.require("lua-utf8")', "__lua_lib_luautf8_Utf8 = {byte = _G.string.byte, find = _G.string.find, gmatch = _G.string.gmatch, gsub = _G.string.gsub, lower = _G.string.lower, match = _G.string.match, reverse = _G.string.reverse, sub = _G.string.sub, upper = _G.string.upper}; for k, v in pairs(_G.utf8) do __lua_lib_luautf8_Utf8[k] = v end;").replace("_G.xpcall(Main.main, _hx_error)", 'local err; if not xpcall(Main.main, function(obj) err = ""; local _print = _G.print; _G.print = function(...) local args = table.pack(...) for i = 1, args.n do args[i] = tostring(args[i]) end err = err .. table.concat(args, "\t") .. "\n" end _hx_error(obj); _G.print = _print end) then _G.error(err) return end;') | Set-Content $path
|
||||
# This file is part of KoboldAI.
|
||||
#
|
||||
# KoboldAI is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
$path = "out.lua"
|
||||
if ($args[0].length -gt 0) {
|
||||
$path = $args[0]
|
||||
}
|
||||
haxe --lua $path -L littleBigInt --main Main
|
||||
if (-not $?) { exit 1 }
|
||||
(Get-Content $path).replace('_G.require("rex_pcre")', '({flags = function() return {CASELESS = 1, DOTALL = 1, MULTILINE = 1, UCP = 1, UTF8 = 1} end, gsub = function() return "" end, new = function() return {} end})').replace("return _hx_exports", "return _hx_exports.Main").replace(" _hx_bit_raw = _G.require('bit32')", " _hx_bit_raw = {arshift = function(x, n) local y = x >> n; if (x < 0) then y = y | ~(-1 >> n) end return y end, band = function(x, y) return x & y end, bor = function(x, y) return x | y end, bnot = function(x) return ~x end, bxor = function(x, y) return x ~ y end, lshift = function(x, n) return x << n end, rshift = function(x, n) return x >> n end}").replace('__lua_lib_luautf8_Utf8 = _G.require("lua-utf8")', "__lua_lib_luautf8_Utf8 = {byte = _G.string.byte, find = _G.string.find, gmatch = _G.string.gmatch, gsub = _G.string.gsub, lower = _G.string.lower, match = _G.string.match, reverse = _G.string.reverse, sub = _G.string.sub, upper = _G.string.upper}; for k, v in pairs(_G.utf8) do __lua_lib_luautf8_Utf8[k] = v end;").replace("_G.xpcall(Main.main, _hx_error)", 'local err; if not xpcall(Main.main, function(obj) err = ""; local _print = _G.print; _G.print = function(...) local args = table.pack(...) for i = 1, args.n do args[i] = tostring(args[i]) end err = err .. table.concat(args, "\t") .. "\n" end _hx_error(obj); _G.print = _print end) then _G.error(err) return end;') | Set-Content $path
|
||||
76
userscripts/examples/various_examples.lua
Normal file
76
userscripts/examples/various_examples.lua
Normal file
@ -0,0 +1,76 @@
|
||||
-- Example script
|
||||
-- Description goes on
|
||||
--[[subsequent lines including
|
||||
in multiline comments]]
|
||||
|
||||
kobold = require("bridge")() -- This line is optional and is only for EmmyLua type annotations
|
||||
|
||||
-- You can import libraries that are in extern/lualibs/
|
||||
local inspect = require("inspect")
|
||||
local mt19937ar = require("mt19937ar")
|
||||
|
||||
|
||||
---@class KoboldUserScript
|
||||
local userscript = {}
|
||||
|
||||
|
||||
local twister = mt19937ar.new()
|
||||
local seed = math.random(0, 2147483647)
|
||||
|
||||
local token_num = 0
|
||||
local lifetime_token_num = 0
|
||||
|
||||
-- This gets run when user submits a string to the AI (right after the input
|
||||
-- formatting is applied but before the string is actually sent to the AI)
|
||||
function userscript.inmod()
|
||||
warn("\nINPUT MODIFIER")
|
||||
token_num = 0
|
||||
twister:init_genrand(seed)
|
||||
print("Submitted text: " .. kobold.submission) -- You can also write to kobold.submission to alter the user's input
|
||||
print("top-p sampling value: " .. kobold.settings.settopp)
|
||||
end
|
||||
|
||||
-- This gets run every time the AI generates a token (before the token is
|
||||
-- actually sampled, so this is where you can make certain tokens more likely
|
||||
-- to appear than others)
|
||||
function userscript.genmod()
|
||||
warn("\nGENERATION MODIFIER")
|
||||
|
||||
print("Tokens generated in the current generation: " .. token_num)
|
||||
print("Tokens generated since this script started up: " .. lifetime_token_num)
|
||||
|
||||
local r = twister:genrand_real3()
|
||||
print("Setting top-p sampling value to " .. r)
|
||||
kobold.settings.settopp = r
|
||||
|
||||
local generated = {}
|
||||
for sequence_number, tokens in ipairs(kobold.generated) do
|
||||
generated[sequence_number] = kobold.decode(tokens)
|
||||
end
|
||||
print("Current generated strings: " .. inspect(generated))
|
||||
|
||||
if token_num == math.floor(kobold.settings.genamt/2) then
|
||||
print("\n\n\n\n\n\nMaking all subsequent tokens more likely to be exclamation marks...")
|
||||
end
|
||||
if token_num >= math.floor(kobold.settings.genamt/2) then
|
||||
for i = 1, kobold.settings.numseqs do
|
||||
kobold.logits[i][1] = 13.37
|
||||
end
|
||||
end
|
||||
|
||||
token_num = token_num + 1
|
||||
lifetime_token_num = lifetime_token_num + 1
|
||||
end
|
||||
|
||||
-- This gets run right before the output formatting is applied after generation
|
||||
-- is finished
|
||||
function userscript.outmod()
|
||||
warn("\nOUTPUT MODIFIER")
|
||||
for chunk in kobold.story:reverse_iter() do
|
||||
print(chunk.num, chunk.content)
|
||||
end
|
||||
print("Wrapping first output in brackets")
|
||||
kobold.outputs[1] = "[" .. kobold.outputs[1] .. "]"
|
||||
end
|
||||
|
||||
return userscript
|
||||
Loading…
x
Reference in New Issue
Block a user