Merge branch 'united' of https://github.com/henk717/KoboldAI into model-structure-and-maybe-rwkv

colab/TPU.ipynb

@@ -66,7 +66,7 @@
         "#@title <b><-- Select your model below and then click this to start KoboldAI</b>\n",
         "#@markdown You can find a description of the models below along with instructions on how to start KoboldAI.\n",
         "\n",
-        "Model = \"Nerys 13B V2\" #@param [\"Nerys 13B V2\", \"Nerybus 13B\", \"Erebus 13B\", \"Janeway 13B\", \"Shinen 13B\", \"Skein 20B\", \"Erebus 20B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Shinen 6B\", \"Pygmalion 6B\", \"Lit V2 6B\", \"Lit 6B\", \"NeoX 20B\", \"OPT 13B\", \"Fairseq Dense 13B\", \"GPT-J-6B\"] {allow-input: true}\n",
+        "Model = \"Nerys 13B V2\" #@param [\"Nerys 13B V2\", \"Nerybus 13B\", \"Erebus 13B\", \"Janeway 13B\", \"Shinen 13B\", \"Skein 20B\", \"Erebus 20B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Shinen 6B\", \"Pygmalion 6B\", \"Pygmalion 6B Dev\", \"Lit V2 6B\", \"Lit 6B\", \"NeoX 20B\", \"OPT 13B\", \"Fairseq Dense 13B\", \"GPT-J-6B\"] {allow-input: true}\n",
         "Version = \"Official\" #@param [\"Official\", \"United\"] {allow-input: true}\n",
         "Provider = \"Cloudflare\" #@param [\"Localtunnel\", \"Cloudflare\"]\n",
         "use_google_drive = True #@param {type:\"boolean\"}\n",
@@ -89,6 +89,8 @@
         "  if not os.path.exists(\"/content/drive/MyDrive/\"):\n",
         "    os.mkdir(\"/content/drive/MyDrive/\")\n",
         "\n",
+        "Revision = \"\"\n",
+        "\n",
         "if Model == \"Janeway 13B\":\n",
         "  Model = \"KoboldAI/fairseq-dense-13B-Janeway\"\n",
         "  path = \"\"\n",
@@ -177,7 +179,7 @@
         "else:\n",
         "  tunnel = \"\"\n",
         "\n",
-        "!wget https://koboldai.org/ckds -O - | bash /dev/stdin $path$download -m $Model -g $Version $tunnel"
+        "!wget https://koboldai.org/ckds -O - | bash /dev/stdin $path$download -m $Model -g $Version $tunnel $Revision"
       ]
     },
     {

commandline.bat

@@ -1,6 +1,10 @@
 @echo off
 cd /D %~dp0
+
+:Isolation
 SET CONDA_SHLVL=
+SET PYTHONNOUSERSITE=1
+SET PYTHONPATH=
 
 TITLE CMD for KoboldAI Runtime
 SET /P M=<loader.settings

environments/huggingface.yml

@@ -5,12 +5,13 @@ channels:
   - defaults
 dependencies:
   - colorama
-  - flask-socketio
+  - flask-socketio=5.3.2
-  - flask-session
+  - flask-session=0.4.0
+  - python-socketio=5.7.2
   - pytorch=1.11.*
   - python=3.8.*
   - cudatoolkit=11.1
-  - eventlet
+  - eventlet=0.33.3
   - dnspython=2.2.1
   - markdown
   - bleach=4.1.0
@@ -30,7 +31,7 @@ dependencies:
     - flask-cors
     - lupa==1.10
     - transformers==4.25.1
-    - huggingface_hub>=0.10.1
+    - huggingface_hub==0.12.1
     - safetensors
     - accelerate
     - git+https://github.com/VE-FORBRYDERNE/mkultra

environments/rocm.yml

@@ -4,10 +4,11 @@ channels:
   - defaults
 dependencies:
   - colorama
-  - flask-socketio
+  - flask-socketio=5.3.2
-  - flask-session
+  - flask-session=0.4.0
+  - python-socketio=5.7.2
   - python=3.8.*
-  - eventlet
+  - eventlet=0.33.3
   - dnspython=2.2.1
   - markdown
   - bleach=4.1.0
@@ -29,7 +30,7 @@ dependencies:
     - flask-cors
     - lupa==1.10
     - transformers==4.25.1
-    - huggingface_hub>=0.10.1
+    - huggingface_hub==0.12.1
     - safetensors
     - accelerate
     - git+https://github.com/VE-FORBRYDERNE/mkultra

gensettings.py

@@ -825,7 +825,7 @@ gensettingstf = [
 	"max": 1,
 	"step": 1,
 	"default": 0,
-    "tooltip": "If enabled a specfic seed will be used for the random generator on text generation",
+    "tooltip": "If enabled, a specific seed will be used for the random generator on text generation",
     "menu_path": "Settings",
     "sub_path": "Other",
     "classname": "system",

install_git_transformers.bat

@@ -0,0 +1,41 @@
+@echo off
+cd /D %~dp0
+SET CONDA_SHLVL=
+
+TITLE KoboldAI - Git Transformers Installer
+ECHO This script will replace the Transformers version with the latest Git Transformers which may contain breaking changes.
+ECHO If you wish to return to the approved version of transformers you can run the install_requirements.bat script or KoboldAI Updater.
+pause
+
+SET /P M=<loader.settings
+IF %M%==1 GOTO drivemap
+IF %M%==2 GOTO subfolder
+IF %M%==3 GOTO drivemap_B
+
+:subfolder
+ECHO Runtime launching in subfolder mode
+SET TEMP=%~DP0MINICONDA3
+SET TMP=%~DP0MINICONDA3
+call miniconda3\condabin\activate
+pip install git+https://github.com/huggingface/transformers
+cmd /k
+
+:drivemap
+ECHO Runtime launching in K: drive mode
+subst /D K: >nul
+subst K: miniconda3 >nul
+SET TEMP=K:\
+SET TMP=K:\
+call K:\python\condabin\activate
+pip install git+https://github.com/huggingface/transformers
+cmd /k
+
+:drivemap_B
+ECHO Runtime launching in B: drive mode
+subst /D B: >nul
+subst B: miniconda3 >nul
+SET TEMP=B:\
+SET TMP=B:\
+call B:\python\condabin\activate
+pip install git+https://github.com/huggingface/transformers
+cmd /k

koboldai_settings.py

@@ -968,9 +968,9 @@ class story_settings(settings):
             else:
                 logger.warning(f"Story mismatch in v2 migration. Existing file had story id {v2j['story_id']} but we have {self.story_id}")
 
+        self.gamesaved = True
         with open(self.save_paths.story, "w", encoding="utf-8") as file:
             file.write(self.to_json())
-        self.gamesaved = True
     
     def update_story_path_structure(self, path: str) -> None:
         # Upon loading a file, makes directories that are required for certain
@@ -1202,12 +1202,12 @@ class system_settings(settings):
     local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 
                             'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 
                             'comregex_ui', 'sp', '_horde_pid', 'inference_config', 'image_pipeline', 
-                            'summarizer', 'summary_tokenizer', 'tts_model']
+                            'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states']
     no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 
                          'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy', 
                          'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', 
                          'summary_tokenizer', 'use_colab_tpu', 'noai', 'disable_set_aibusy', 'cloudflare_link', 'tts_model',
-                         'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu']
+                         'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu', 'rng_states']
     settings_name = "system"
     def __init__(self, socketio, koboldai_var):
         self._socketio = socketio
@@ -1263,6 +1263,7 @@ class system_settings(settings):
         self.disable_output_formatting = False
         self.full_determinism = False  # Whether or not full determinism is enabled
         self.seed_specified = False  # Whether or not the current RNG seed was specified by the user (in their settings file)
+        self.rng_states = {} # creates an empty dictionary to store the random number generator (RNG) states for a given seed, which is used to restore the RNG state later on
         self.seed        = None   # The current RNG seed (as an int), or None if unknown
         self.alt_gen = False # Use the calc_ai_text method for generating text to go to the AI
         self.theme_list = [".".join(f.split(".")[:-1]) for f in os.listdir("./themes") if os.path.isfile(os.path.join("./themes", f))]
@@ -1504,7 +1505,7 @@ class KoboldStoryRegister(object):
                                 else:
                                     self.actions[i]["Probabilities"][token_num][token_option]["Used"] = False
             if "Options" in self.actions[i]:
-                for j in range(len(self.actions[i]["Options"])):
+                for j in reversed(range(len(self.actions[i]["Options"]))):
                     if self.actions[i]["Options"][j]["text"] == text:
                         del self.actions[i]["Options"][j]
             if old_text != "":
@@ -2259,6 +2260,8 @@ class KoboldWorldInfo(object):
             self._socketio.emit("world_info_folder", {x: self.world_info_folder[x] for x in self.world_info_folder}, broadcast=True, room="UI_2")
         
     def delete_folder(self, folder):
+        if folder == "root":
+            raise Exception("removing the root folder is not supported")
         keys = [key for key in self.world_info]
         for key in keys:
             if self.world_info[key]['folder'] == folder:
@@ -2272,7 +2275,7 @@ class KoboldWorldInfo(object):
         
     def add_item_to_folder(self, uid, folder, before=None):
         if uid in self.world_info:
-            #fiirst we need to remove the item from whatever folder it's in
+            #first we need to remove the item from whatever folder it's in
             for temp in self.world_info_folder:
                 if uid in self.world_info_folder[temp]:
                     self.world_info_folder[temp].remove(uid)
@@ -2357,7 +2360,8 @@ class KoboldWorldInfo(object):
             raise
         if folder not in self.world_info_folder:
             self.world_info_folder[folder] = []
-        self.world_info_folder[folder].append(uid)
+        if uid not in self.world_info_folder[folder]:
+            self.world_info_folder[folder].append(uid)
         self.story_settings.gamesaved = False
         if sync:
             self.sync_world_info_to_old_format()
@@ -2442,8 +2446,11 @@ class KoboldWorldInfo(object):
             self._socketio.emit("world_info_entry", self.world_info[uid], broadcast=True, room="UI_2")
         
     def delete(self, uid):
+        if self.world_info[uid]['folder'] == "root":
+            raise Exception("removing the root folder is not supported")
+        
         del self.world_info[uid]
-
+        
         try:
             os.remove(os.path.join(self._koboldai_vars.save_paths.wi_images, str(uid)))
         except FileNotFoundError:
@@ -2463,6 +2470,8 @@ class KoboldWorldInfo(object):
         ignore = self._koboldai_vars.calc_ai_text()
     
     def rename_folder(self, old_folder, folder):
+        if old_folder == "root":
+            raise Exception("renaming the root folder is not supported")
         self.story_settings.gamesaved = False
         if folder in self.world_info_folder:
             i=0
@@ -2536,14 +2545,10 @@ class KoboldWorldInfo(object):
                     file.write(base64.b64decode(image_b64))
         
         data["entries"] = {k: self.upgrade_entry(v) for k,v in data["entries"].items()}
-
-        if folder is None:
-            self.world_info_folder = data['folders']
         
         #Add the item
         start_time = time.time()
         for uid, item in data['entries'].items():
-            
             self.add_item(item['title'] if 'title' in item else item['key'][0], 
                           item['key'] if 'key' in item else [], 
                           item['keysecondary'] if 'keysecondary' in item else [], 
@@ -2555,10 +2560,9 @@ class KoboldWorldInfo(object):
                           use_wpp=item['use_wpp'] if 'use_wpp' in item else False, 
                           wpp=item['wpp'] if 'wpp' in item else {'name': "", 'type': "", 'format': "W++", 'attributes': {}},
                           object_type=item.get("object_type"),
+                          v1_uid=item.get("v1_uid"),
                           recalc=False, sync=False)
-        if folder is None:
+
-            #self.world_info = {int(x): data['entries'][x] for x in data['entries']}
-            self.world_info_folder = data['folders']
         logger.debug("Load World Info took {}s".format(time.time()-start_time))
         try:
             start_time = time.time()
@@ -2577,50 +2581,52 @@ class KoboldWorldInfo(object):
         for folder in self.world_info_folder:
             folder_entries[folder] = i
             i-=1
-    
+        
-    
+        #self.wifolders_l = []     # List of World Info folder UIDs
+        self.story_settings.wifolders_l = [folder_entries[x] for x in folder_entries if x != "root"]
+        
         #self.worldinfo_i = []     # List of World Info key/value objects sans uninitialized entries
         self.story_settings.worldinfo_i = [{
-                                            "comment": self.world_info[x]['comment'],
-                                            "constant": self.world_info[x]['constant'],
-                                            "content": self.world_info[x]['content'],
-                                            "folder": folder_entries[self.world_info[x]['folder']],
-                                            "init": True,
                                             "key": ",".join(self.world_info[x]['key']),
                                             "keysecondary": ",".join(self.world_info[x]['keysecondary']),
+                                            "content": self.world_info[x]['content'],
+                                            "comment": self.world_info[x]['comment'],
+                                            "folder": folder_entries[self.world_info[x]['folder']] if self.world_info[x]['folder'] != "root" else None,
                                             "num": x,
+                                            "init": True,
                                             "selective": len(self.world_info[x]['keysecondary'])>0,
+                                            "constant": self.world_info[x]['constant'],
                                             "uid": self.world_info[x]['uid'] if 'v1_uid' not in self.world_info[x] or self.world_info[x]['v1_uid'] is None else self.world_info[x]['v1_uid']
                                         } for x in self.world_info]
-                                        
+        
         #self.worldinfo   = []     # List of World Info key/value objects
         self.story_settings.worldinfo = [x for x in self.story_settings.worldinfo_i]
         #We have to have an uninitialized blank entry for every folder or the old method craps out
         for folder in folder_entries:
             self.story_settings.worldinfo.append({
-                                            "comment": "",
-                                            "constant": False,
-                                            "content": "",
-                                            "folder": folder_entries[folder],
-                                            "init": False,
                                             "key": "",
                                             "keysecondary": "",
+                                            "content": "",
+                                            "comment": "",
+                                            "folder": folder_entries[folder] if folder != "root" else None,
                                             "num": (0 if len(self.world_info) == 0 else max(self.world_info))+(folder_entries[folder]*-1),
+                                            "init": False,
                                             "selective": False,
+                                            "constant": False,
                                             "uid": folder_entries[folder]
                                         })
         
+        mapping = {uid: index for index, uid in enumerate(self.story_settings.wifolders_l)}
+        self.story_settings.worldinfo.sort(key=lambda x: mapping[x["folder"]] if x["folder"] is not None else float("inf"))
+        
         #self.wifolders_d = {}     # Dictionary of World Info folder UID-info pairs
-        self.story_settings.wifolders_d = {folder_entries[x]: {'collapsed': False, 'name': x} for x in folder_entries}
+        self.story_settings.wifolders_d = {str(folder_entries[x]): {'name': x, 'collapsed': False} for x in folder_entries if x != "root"}
         
         #self.worldinfo_u = {}     # Dictionary of World Info UID - key/value pairs
-        self.story_settings.worldinfo_u = {x['uid']: x for x in self.story_settings.worldinfo}
+        self.story_settings.worldinfo_u = {str(y["uid"]): y for x in folder_entries for y in self.story_settings.worldinfo if y["folder"] == (folder_entries[x] if x != "root" else None)}
-        
-        #self.wifolders_l = []     # List of World Info folder UIDs
-        self.story_settings.wifolders_l = [folder_entries[x] for x in folder_entries]
         
         #self.wifolders_u = {}     # Dictionary of pairs of folder UID - list of WI UID
-        self.story_settings.wifolders_u = {folder_entries[x]: [y for y in self.story_settings.worldinfo if y['folder'] == x] for x in folder_entries}
+        self.story_settings.wifolders_u = {str(folder_entries[x]): [y for y in self.story_settings.worldinfo if y['folder'] == folder_entries[x]] for x in folder_entries if x != "root"}
         
     def reset_used_in_game(self):
         for key in self.world_info:

modeling/inference_models/hf_torch.py

@@ -110,19 +110,43 @@ class HFTorchInferenceModel(HFInferenceModel):
                 pre = torch.Tensor(scores)
         return scores
 
-    def _post_load(model_self) -> None:
+    def get_model_type(self) -> str:
-        # Patch stopping_criteria
+        if not self.model_config:
+            return "Read Only"
 
+        if not isinstance(self.model_config, dict):
+            return str(self.model_config.model_type)
+
+        model_type = self.model_config.get("model_type")
+
+        if model_type:
+            return model_type
+
+        if utils.koboldai_vars.mode.endswith("gpt2"):
+            return "gpt2"
+        else:
+            return "Unknown"
+
+    def _post_load(m_self) -> None:
+        if not utils.koboldai_vars.model_type:
+            utils.koboldai_vars.model_type = m_self.get_model_type()
+
+        # Model specific overrides if a model has bad defaults
+        if utils.koboldai_vars.model_type == "llama":
+            m_self.tokenizer.decode_with_prefix_space = True
+            m_self.tokenizer.add_bos_token = False
+
+        # Patch stopping_criteria
         class PTHStopper(StoppingCriteria):
             def __call__(
                 hf_self,
                 input_ids: torch.LongTensor,
                 scores: torch.FloatTensor,
             ) -> None:
-                model_self._post_token_gen(input_ids)
+                m_self._post_token_gen(input_ids)
 
-                for stopper in model_self.stopper_hooks:
+                for stopper in m_self.stopper_hooks:
-                    do_stop = stopper(model_self, input_ids)
+                    do_stop = stopper(m_self, input_ids)
                     if do_stop:
                         return True
                 return False
@@ -235,18 +259,18 @@ class HFTorchInferenceModel(HFInferenceModel):
                 # Handle direct phrases
                 if phrase.startswith("{") and phrase.endswith("}"):
                     no_brackets = phrase[1:-1]
-                    return [model_self.tokenizer.encode(no_brackets)]
+                    return [m_self.tokenizer.encode(no_brackets)]
 
                 # Handle untamperable phrases
                 if not self._allow_leftwards_tampering(phrase):
-                    return [model_self.tokenizer.encode(phrase)]
+                    return [m_self.tokenizer.encode(phrase)]
 
                 # Handle slight alterations to original phrase
                 phrase = phrase.strip(" ")
                 ret = []
 
                 for alt_phrase in [phrase, f" {phrase}"]:
-                    ret.append(model_self.tokenizer.encode(alt_phrase))
+                    ret.append(m_self.tokenizer.encode(alt_phrase))
 
                 return ret
 
@@ -428,8 +452,8 @@ class HFTorchInferenceModel(HFInferenceModel):
                 *args,
                 **kwargs,
             ):
-                scores = model_self._apply_warpers(scores=scores, input_ids=input_ids)
+                scores = m_self._apply_warpers(scores=scores, input_ids=input_ids)
-                visualize_probabilities(model_self, scores)
+                visualize_probabilities(m_self, scores)
                 return scores
 
         def new_get_logits_warper(

play.bat

@@ -1,6 +1,10 @@
 @echo off
 cd /D %~dp0
+
+:Isolation
 SET CONDA_SHLVL=
+SET PYTHONNOUSERSITE=1
+SET PYTHONPATH=
 
 rmdir /S /Q flask_session
 

requirements.txt

@@ -1,13 +1,14 @@
 transformers==4.25.1
-huggingface_hub>=0.10.1
+huggingface_hub==0.12.1
-Flask
+Flask==2.2.3
-Flask-SocketIO
+Flask-SocketIO==5.3.2
+python-socketio==5.7.2
 requests
 torch >= 1.9, < 1.13
 flask-cloudflared==0.0.10
 flask-ngrok
 flask-cors
-eventlet
+eventlet==0.33.3
 dnspython==2.2.1
 lupa==1.10
 markdown
@@ -15,7 +16,7 @@ bleach==4.1.0
 sentencepiece
 protobuf
 accelerate
-flask_session
+flask-session==0.4.0
 marshmallow>=3.13
 apispec-webframeworks
 loguru

requirements_mtj.txt

@@ -6,20 +6,21 @@ dm-haiku == 0.0.5
 jax == 0.2.21
 jaxlib >= 0.1.69, <= 0.3.7
 transformers == 4.25.1
-huggingface_hub >= 0.10.1
+huggingface_hub==0.12.1
 progressbar2
 git+https://github.com/VE-FORBRYDERNE/mesh-transformer-jax@ck
-flask
+Flask==2.2.3
-Flask-SocketIO
+Flask-SocketIO==5.3.2
+python-socketio==5.7.2
 flask-cloudflared==0.0.10
 flask-ngrok
 flask-cors
-eventlet
+eventlet==0.33.3
 dnspython==2.2.1
 lupa==1.10
 markdown
 bleach==4.1.0
-flask-session
+flask-session==0.4.0
 marshmallow>=3.13
 apispec-webframeworks
 loguru

update-koboldai.bat

@@ -1,6 +1,10 @@
 @echo off
 cd /d %~dp0
+
+:Isolation
 SET CONDA_SHLVL=
+SET PYTHONNOUSERSITE=1
+SET PYTHONPATH=
 
 TITLE KoboldAI - Updater
 SET /P M=<loader.settings