Merge branch 'united' of https://github.com/henk717/KoboldAI into model-structure-and-maybe-rwkv

2025-06-05 21:59:24 +02:00 · 2023-03-17 16:20:13 -05:00
parent 8d0bc404a5 90a7eb6153
commit 692dbfeb37
12 changed files with 153 additions and 64 deletions
--- a/colab/TPU.ipynb
+++ b/colab/TPU.ipynb
@@ -66,7 +66,7 @@
        "#@title <b><-- Select your model below and then click this to start KoboldAI</b>\n",
        "#@markdown You can find a description of the models below along with instructions on how to start KoboldAI.\n",
        "\n",
-        "Model = \"Nerys 13B V2\" #@param [\"Nerys 13B V2\", \"Nerybus 13B\", \"Erebus 13B\", \"Janeway 13B\", \"Shinen 13B\", \"Skein 20B\", \"Erebus 20B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Shinen 6B\", \"Pygmalion 6B\", \"Lit V2 6B\", \"Lit 6B\", \"NeoX 20B\", \"OPT 13B\", \"Fairseq Dense 13B\", \"GPT-J-6B\"] {allow-input: true}\n",
+        "Model = \"Nerys 13B V2\" #@param [\"Nerys 13B V2\", \"Nerybus 13B\", \"Erebus 13B\", \"Janeway 13B\", \"Shinen 13B\", \"Skein 20B\", \"Erebus 20B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Shinen 6B\", \"Pygmalion 6B\", \"Pygmalion 6B Dev\", \"Lit V2 6B\", \"Lit 6B\", \"NeoX 20B\", \"OPT 13B\", \"Fairseq Dense 13B\", \"GPT-J-6B\"] {allow-input: true}\n",
        "Version = \"Official\" #@param [\"Official\", \"United\"] {allow-input: true}\n",
        "Provider = \"Cloudflare\" #@param [\"Localtunnel\", \"Cloudflare\"]\n",
        "use_google_drive = True #@param {type:\"boolean\"}\n",
@@ -89,6 +89,8 @@
        "  if not os.path.exists(\"/content/drive/MyDrive/\"):\n",
        "    os.mkdir(\"/content/drive/MyDrive/\")\n",
        "\n",
+        "Revision = \"\"\n",
+        "\n",
        "if Model == \"Janeway 13B\":\n",
        "  Model = \"KoboldAI/fairseq-dense-13B-Janeway\"\n",
        "  path = \"\"\n",
@@ -177,7 +179,7 @@
        "else:\n",
        "  tunnel = \"\"\n",
        "\n",
-        "!wget https://koboldai.org/ckds -O - | bash /dev/stdin $path$download -m $Model -g $Version $tunnel"
+        "!wget https://koboldai.org/ckds -O - | bash /dev/stdin $path$download -m $Model -g $Version $tunnel $Revision"
      ]
    },
    {
--- a/commandline.bat
+++ b/commandline.bat
@@ -1,6 +1,10 @@
@echo off
 cd /D %~dp0
+
+:Isolation
 SET CONDA_SHLVL=
+SET PYTHONNOUSERSITE=1
+SET PYTHONPATH=

 TITLE CMD for KoboldAI Runtime
 SET /P M=<loader.settings
--- a/environments/huggingface.yml
+++ b/environments/huggingface.yml
@@ -5,12 +5,13 @@ channels:
  - defaults
 dependencies:
  - colorama
-  - flask-socketio
-  - flask-session
+  - flask-socketio=5.3.2
+  - flask-session=0.4.0
+  - python-socketio=5.7.2
  - pytorch=1.11.*
  - python=3.8.*
  - cudatoolkit=11.1
-  - eventlet
+  - eventlet=0.33.3
  - dnspython=2.2.1
  - markdown
  - bleach=4.1.0
@@ -30,7 +31,7 @@ dependencies:
    - flask-cors
    - lupa==1.10
    - transformers==4.25.1
-    - huggingface_hub>=0.10.1
+    - huggingface_hub==0.12.1
    - safetensors
    - accelerate
    - git+https://github.com/VE-FORBRYDERNE/mkultra
--- a/environments/rocm.yml
+++ b/environments/rocm.yml
@@ -4,10 +4,11 @@ channels:
  - defaults
 dependencies:
  - colorama
-  - flask-socketio
-  - flask-session
+  - flask-socketio=5.3.2
+  - flask-session=0.4.0
+  - python-socketio=5.7.2
  - python=3.8.*
-  - eventlet
+  - eventlet=0.33.3
  - dnspython=2.2.1
  - markdown
  - bleach=4.1.0
@@ -29,7 +30,7 @@ dependencies:
    - flask-cors
    - lupa==1.10
    - transformers==4.25.1
-    - huggingface_hub>=0.10.1
+    - huggingface_hub==0.12.1
    - safetensors
    - accelerate
    - git+https://github.com/VE-FORBRYDERNE/mkultra
--- a/gensettings.py
+++ b/gensettings.py
@@ -825,7 +825,7 @@ gensettingstf = [
 	"max": 1,
 	"step": 1,
 	"default": 0,
-    "tooltip": "If enabled a specfic seed will be used for the random generator on text generation",
+    "tooltip": "If enabled, a specific seed will be used for the random generator on text generation",
    "menu_path": "Settings",
    "sub_path": "Other",
    "classname": "system",
--- a/install_git_transformers.bat
+++ b/install_git_transformers.bat
@@ -0,0 +1,41 @@
+@echo off
+cd /D %~dp0
+SET CONDA_SHLVL=
+
+TITLE KoboldAI - Git Transformers Installer
+ECHO This script will replace the Transformers version with the latest Git Transformers which may contain breaking changes.
+ECHO If you wish to return to the approved version of transformers you can run the install_requirements.bat script or KoboldAI Updater.
+pause
+
+SET /P M=<loader.settings
+IF %M%==1 GOTO drivemap
+IF %M%==2 GOTO subfolder
+IF %M%==3 GOTO drivemap_B
+
+:subfolder
+ECHO Runtime launching in subfolder mode
+SET TEMP=%~DP0MINICONDA3
+SET TMP=%~DP0MINICONDA3
+call miniconda3\condabin\activate
+pip install git+https://github.com/huggingface/transformers
+cmd /k
+
+:drivemap
+ECHO Runtime launching in K: drive mode
+subst /D K: >nul
+subst K: miniconda3 >nul
+SET TEMP=K:\
+SET TMP=K:\
+call K:\python\condabin\activate
+pip install git+https://github.com/huggingface/transformers
+cmd /k
+
+:drivemap_B
+ECHO Runtime launching in B: drive mode
+subst /D B: >nul
+subst B: miniconda3 >nul
+SET TEMP=B:\
+SET TMP=B:\
+call B:\python\condabin\activate
+pip install git+https://github.com/huggingface/transformers
+cmd /k
--- a/koboldai_settings.py
+++ b/koboldai_settings.py
@@ -968,9 +968,9 @@ class story_settings(settings):
            else:
                logger.warning(f"Story mismatch in v2 migration. Existing file had story id {v2j['story_id']} but we have {self.story_id}")

+        self.gamesaved = True
        with open(self.save_paths.story, "w", encoding="utf-8") as file:
            file.write(self.to_json())
-        self.gamesaved = True
    
    def update_story_path_structure(self, path: str) -> None:
        # Upon loading a file, makes directories that are required for certain
@@ -1202,12 +1202,12 @@ class system_settings(settings):
    local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 
                            'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 
                            'comregex_ui', 'sp', '_horde_pid', 'inference_config', 'image_pipeline', 
-                            'summarizer', 'summary_tokenizer', 'tts_model']
+                            'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states']
    no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 
                         'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy', 
                         'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', 
                         'summary_tokenizer', 'use_colab_tpu', 'noai', 'disable_set_aibusy', 'cloudflare_link', 'tts_model',
-                         'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu']
+                         'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu', 'rng_states']
    settings_name = "system"
    def __init__(self, socketio, koboldai_var):
        self._socketio = socketio
@@ -1263,6 +1263,7 @@ class system_settings(settings):
        self.disable_output_formatting = False
        self.full_determinism = False  # Whether or not full determinism is enabled
        self.seed_specified = False  # Whether or not the current RNG seed was specified by the user (in their settings file)
+        self.rng_states = {} # creates an empty dictionary to store the random number generator (RNG) states for a given seed, which is used to restore the RNG state later on
        self.seed        = None   # The current RNG seed (as an int), or None if unknown
        self.alt_gen = False # Use the calc_ai_text method for generating text to go to the AI
        self.theme_list = [".".join(f.split(".")[:-1]) for f in os.listdir("./themes") if os.path.isfile(os.path.join("./themes", f))]
@@ -1504,7 +1505,7 @@ class KoboldStoryRegister(object):
                                else:
                                    self.actions[i]["Probabilities"][token_num][token_option]["Used"] = False
            if "Options" in self.actions[i]:
-                for j in range(len(self.actions[i]["Options"])):
+                for j in reversed(range(len(self.actions[i]["Options"]))):
                    if self.actions[i]["Options"][j]["text"] == text:
                        del self.actions[i]["Options"][j]
            if old_text != "":
@@ -2259,6 +2260,8 @@ class KoboldWorldInfo(object):
            self._socketio.emit("world_info_folder", {x: self.world_info_folder[x] for x in self.world_info_folder}, broadcast=True, room="UI_2")
        
    def delete_folder(self, folder):
+        if folder == "root":
+            raise Exception("removing the root folder is not supported")
        keys = [key for key in self.world_info]
        for key in keys:
            if self.world_info[key]['folder'] == folder:
@@ -2272,7 +2275,7 @@ class KoboldWorldInfo(object):
        
    def add_item_to_folder(self, uid, folder, before=None):
        if uid in self.world_info:
-            #fiirst we need to remove the item from whatever folder it's in
+            #first we need to remove the item from whatever folder it's in
            for temp in self.world_info_folder:
                if uid in self.world_info_folder[temp]:
                    self.world_info_folder[temp].remove(uid)
@@ -2357,6 +2360,7 @@ class KoboldWorldInfo(object):
            raise
        if folder not in self.world_info_folder:
            self.world_info_folder[folder] = []
+        if uid not in self.world_info_folder[folder]:
            self.world_info_folder[folder].append(uid)
        self.story_settings.gamesaved = False
        if sync:
@@ -2442,6 +2446,9 @@ class KoboldWorldInfo(object):
            self._socketio.emit("world_info_entry", self.world_info[uid], broadcast=True, room="UI_2")
        
    def delete(self, uid):
+        if self.world_info[uid]['folder'] == "root":
+            raise Exception("removing the root folder is not supported")
+        
        del self.world_info[uid]
        
        try:
@@ -2463,6 +2470,8 @@ class KoboldWorldInfo(object):
        ignore = self._koboldai_vars.calc_ai_text()
    
    def rename_folder(self, old_folder, folder):
+        if old_folder == "root":
+            raise Exception("renaming the root folder is not supported")
        self.story_settings.gamesaved = False
        if folder in self.world_info_folder:
            i=0
@@ -2537,13 +2546,9 @@ class KoboldWorldInfo(object):
        
        data["entries"] = {k: self.upgrade_entry(v) for k,v in data["entries"].items()}
        
-        if folder is None:
-            self.world_info_folder = data['folders']
-        
        #Add the item
        start_time = time.time()
        for uid, item in data['entries'].items():
-            
            self.add_item(item['title'] if 'title' in item else item['key'][0], 
                          item['key'] if 'key' in item else [], 
                          item['keysecondary'] if 'keysecondary' in item else [], 
@@ -2555,10 +2560,9 @@ class KoboldWorldInfo(object):
                          use_wpp=item['use_wpp'] if 'use_wpp' in item else False, 
                          wpp=item['wpp'] if 'wpp' in item else {'name': "", 'type': "", 'format': "W++", 'attributes': {}},
                          object_type=item.get("object_type"),
+                          v1_uid=item.get("v1_uid"),
                          recalc=False, sync=False)
-        if folder is None:
-            #self.world_info = {int(x): data['entries'][x] for x in data['entries']}
-            self.world_info_folder = data['folders']
+
        logger.debug("Load World Info took {}s".format(time.time()-start_time))
        try:
            start_time = time.time()
@@ -2578,18 +2582,20 @@ class KoboldWorldInfo(object):
            folder_entries[folder] = i
            i-=1
        
+        #self.wifolders_l = []     # List of World Info folder UIDs
+        self.story_settings.wifolders_l = [folder_entries[x] for x in folder_entries if x != "root"]
        
        #self.worldinfo_i = []     # List of World Info key/value objects sans uninitialized entries
        self.story_settings.worldinfo_i = [{
-                                            "comment": self.world_info[x]['comment'],
-                                            "constant": self.world_info[x]['constant'],
-                                            "content": self.world_info[x]['content'],
-                                            "folder": folder_entries[self.world_info[x]['folder']],
-                                            "init": True,
                                            "key": ",".join(self.world_info[x]['key']),
                                            "keysecondary": ",".join(self.world_info[x]['keysecondary']),
+                                            "content": self.world_info[x]['content'],
+                                            "comment": self.world_info[x]['comment'],
+                                            "folder": folder_entries[self.world_info[x]['folder']] if self.world_info[x]['folder'] != "root" else None,
                                            "num": x,
+                                            "init": True,
                                            "selective": len(self.world_info[x]['keysecondary'])>0,
+                                            "constant": self.world_info[x]['constant'],
                                            "uid": self.world_info[x]['uid'] if 'v1_uid' not in self.world_info[x] or self.world_info[x]['v1_uid'] is None else self.world_info[x]['v1_uid']
                                        } for x in self.world_info]
        
@@ -2598,29 +2604,29 @@ class KoboldWorldInfo(object):
        #We have to have an uninitialized blank entry for every folder or the old method craps out
        for folder in folder_entries:
            self.story_settings.worldinfo.append({
-                                            "comment": "",
-                                            "constant": False,
-                                            "content": "",
-                                            "folder": folder_entries[folder],
-                                            "init": False,
                                            "key": "",
                                            "keysecondary": "",
+                                            "content": "",
+                                            "comment": "",
+                                            "folder": folder_entries[folder] if folder != "root" else None,
                                            "num": (0 if len(self.world_info) == 0 else max(self.world_info))+(folder_entries[folder]*-1),
+                                            "init": False,
                                            "selective": False,
+                                            "constant": False,
                                            "uid": folder_entries[folder]
                                        })
        
+        mapping = {uid: index for index, uid in enumerate(self.story_settings.wifolders_l)}
+        self.story_settings.worldinfo.sort(key=lambda x: mapping[x["folder"]] if x["folder"] is not None else float("inf"))
+        
        #self.wifolders_d = {}     # Dictionary of World Info folder UID-info pairs
-        self.story_settings.wifolders_d = {folder_entries[x]: {'collapsed': False, 'name': x} for x in folder_entries}
+        self.story_settings.wifolders_d = {str(folder_entries[x]): {'name': x, 'collapsed': False} for x in folder_entries if x != "root"}
        
        #self.worldinfo_u = {}     # Dictionary of World Info UID - key/value pairs
-        self.story_settings.worldinfo_u = {x['uid']: x for x in self.story_settings.worldinfo}
-        
-        #self.wifolders_l = []     # List of World Info folder UIDs
-        self.story_settings.wifolders_l = [folder_entries[x] for x in folder_entries]
+        self.story_settings.worldinfo_u = {str(y["uid"]): y for x in folder_entries for y in self.story_settings.worldinfo if y["folder"] == (folder_entries[x] if x != "root" else None)}
        
        #self.wifolders_u = {}     # Dictionary of pairs of folder UID - list of WI UID
-        self.story_settings.wifolders_u = {folder_entries[x]: [y for y in self.story_settings.worldinfo if y['folder'] == x] for x in folder_entries}
+        self.story_settings.wifolders_u = {str(folder_entries[x]): [y for y in self.story_settings.worldinfo if y['folder'] == folder_entries[x]] for x in folder_entries if x != "root"}
        
    def reset_used_in_game(self):
        for key in self.world_info:
--- a/modeling/inference_models/hf_torch.py
+++ b/modeling/inference_models/hf_torch.py
@@ -110,19 +110,43 @@ class HFTorchInferenceModel(HFInferenceModel):
                pre = torch.Tensor(scores)
        return scores

-    def _post_load(model_self) -> None:
-        # Patch stopping_criteria
+    def get_model_type(self) -> str:
+        if not self.model_config:
+            return "Read Only"

+        if not isinstance(self.model_config, dict):
+            return str(self.model_config.model_type)
+
+        model_type = self.model_config.get("model_type")
+
+        if model_type:
+            return model_type
+
+        if utils.koboldai_vars.mode.endswith("gpt2"):
+            return "gpt2"
+        else:
+            return "Unknown"
+
+    def _post_load(m_self) -> None:
+        if not utils.koboldai_vars.model_type:
+            utils.koboldai_vars.model_type = m_self.get_model_type()
+
+        # Model specific overrides if a model has bad defaults
+        if utils.koboldai_vars.model_type == "llama":
+            m_self.tokenizer.decode_with_prefix_space = True
+            m_self.tokenizer.add_bos_token = False
+
+        # Patch stopping_criteria
        class PTHStopper(StoppingCriteria):
            def __call__(
                hf_self,
                input_ids: torch.LongTensor,
                scores: torch.FloatTensor,
            ) -> None:
-                model_self._post_token_gen(input_ids)
+                m_self._post_token_gen(input_ids)

-                for stopper in model_self.stopper_hooks:
-                    do_stop = stopper(model_self, input_ids)
+                for stopper in m_self.stopper_hooks:
+                    do_stop = stopper(m_self, input_ids)
                    if do_stop:
                        return True
                return False
@@ -235,18 +259,18 @@ class HFTorchInferenceModel(HFInferenceModel):
                # Handle direct phrases
                if phrase.startswith("{") and phrase.endswith("}"):
                    no_brackets = phrase[1:-1]
-                    return [model_self.tokenizer.encode(no_brackets)]
+                    return [m_self.tokenizer.encode(no_brackets)]

                # Handle untamperable phrases
                if not self._allow_leftwards_tampering(phrase):
-                    return [model_self.tokenizer.encode(phrase)]
+                    return [m_self.tokenizer.encode(phrase)]

                # Handle slight alterations to original phrase
                phrase = phrase.strip(" ")
                ret = []

                for alt_phrase in [phrase, f" {phrase}"]:
-                    ret.append(model_self.tokenizer.encode(alt_phrase))
+                    ret.append(m_self.tokenizer.encode(alt_phrase))

                return ret

@@ -428,8 +452,8 @@ class HFTorchInferenceModel(HFInferenceModel):
                *args,
                **kwargs,
            ):
-                scores = model_self._apply_warpers(scores=scores, input_ids=input_ids)
-                visualize_probabilities(model_self, scores)
+                scores = m_self._apply_warpers(scores=scores, input_ids=input_ids)
+                visualize_probabilities(m_self, scores)
                return scores

        def new_get_logits_warper(
--- a/play.bat
+++ b/play.bat
@@ -1,6 +1,10 @@
@echo off
 cd /D %~dp0
+
+:Isolation
 SET CONDA_SHLVL=
+SET PYTHONNOUSERSITE=1
+SET PYTHONPATH=

 rmdir /S /Q flask_session

--- a/requirements.txt
+++ b/requirements.txt
@@ -1,13 +1,14 @@
 transformers==4.25.1
-huggingface_hub>=0.10.1
-Flask
-Flask-SocketIO
+huggingface_hub==0.12.1
+Flask==2.2.3
+Flask-SocketIO==5.3.2
+python-socketio==5.7.2
 requests
 torch >= 1.9, < 1.13
 flask-cloudflared==0.0.10
 flask-ngrok
 flask-cors
-eventlet
+eventlet==0.33.3
 dnspython==2.2.1
 lupa==1.10
 markdown
@@ -15,7 +16,7 @@ bleach==4.1.0
 sentencepiece
 protobuf
 accelerate
-flask_session
+flask-session==0.4.0
 marshmallow>=3.13
 apispec-webframeworks
 loguru
--- a/requirements_mtj.txt
+++ b/requirements_mtj.txt
@@ -6,20 +6,21 @@ dm-haiku == 0.0.5
 jax == 0.2.21
 jaxlib >= 0.1.69, <= 0.3.7
 transformers == 4.25.1
-huggingface_hub >= 0.10.1
+huggingface_hub==0.12.1
 progressbar2
 git+https://github.com/VE-FORBRYDERNE/mesh-transformer-jax@ck
-flask
-Flask-SocketIO
+Flask==2.2.3
+Flask-SocketIO==5.3.2
+python-socketio==5.7.2
 flask-cloudflared==0.0.10
 flask-ngrok
 flask-cors
-eventlet
+eventlet==0.33.3
 dnspython==2.2.1
 lupa==1.10
 markdown
 bleach==4.1.0
-flask-session
+flask-session==0.4.0
 marshmallow>=3.13
 apispec-webframeworks
 loguru
--- a/update-koboldai.bat
+++ b/update-koboldai.bat
@@ -1,6 +1,10 @@
@echo off
 cd /d %~dp0
+
+:Isolation
 SET CONDA_SHLVL=
+SET PYTHONNOUSERSITE=1
+SET PYTHONPATH=

 TITLE KoboldAI - Updater
 SET /P M=<loader.settings