mirror of
https://github.com/KoboldAI/KoboldAI-Client.git
synced 2025-02-17 20:20:45 +01:00
Merge branch 'united' into settings
This commit is contained in:
Gnome Ann
commit
ea82867e4d
31
aiserver.py
31
aiserver.py
@ -778,6 +778,7 @@ def spRequest(filename):
|
||||
parser = argparse.ArgumentParser(description="KoboldAI Server")
|
||||
parser.add_argument("--remote", action='store_true', help="Optimizes KoboldAI for Remote Play")
|
||||
parser.add_argument("--ngrok", action='store_true', help="Optimizes KoboldAI for Remote Play using Ngrok")
|
||||
parser.add_argument("--localtunnel", action='store_true', help="Optimizes KoboldAI for Remote Play using Localtunnel")
|
||||
parser.add_argument("--host", action='store_true', help="Optimizes KoboldAI for Remote Play without using a proxy service")
|
||||
parser.add_argument("--port", type=int, help="Specify the port on which the application will be joinable")
|
||||
parser.add_argument("--model", help="Specify the Model Type to skip the Menu")
|
||||
@ -824,6 +825,9 @@ if args.remote:
|
||||
if args.ngrok:
|
||||
vars.host = True;
|
||||
|
||||
if args.localtunnel:
|
||||
vars.host = True;
|
||||
|
||||
if args.host:
|
||||
vars.host = True;
|
||||
|
||||
@ -5327,21 +5331,38 @@ if __name__ == "__main__":
|
||||
|
||||
# Start Flask/SocketIO (Blocking, so this must be last method!)
|
||||
|
||||
#socketio.run(app, host='0.0.0.0', port=5000)
|
||||
#socketio.run(app, host='0.0.0.0', port=port)
|
||||
if(vars.host):
|
||||
if(args.ngrok):
|
||||
if(args.localtunnel):
|
||||
import subprocess, shutil
|
||||
localtunnel = subprocess.Popen([shutil.which('lt'), '-p', str(port), 'http'], stdout=subprocess.PIPE)
|
||||
attempts = 0
|
||||
while attempts < 10:
|
||||
try:
|
||||
cloudflare = str(localtunnel.stdout.readline())
|
||||
cloudflare = (re.search("(?P<url>https?:\/\/[^\s]+loca.lt)", cloudflare).group("url"))
|
||||
break
|
||||
except:
|
||||
attempts += 1
|
||||
time.sleep(3)
|
||||
continue
|
||||
if attempts == 10:
|
||||
print("LocalTunnel could not be created, falling back to cloudflare...")
|
||||
from flask_cloudflared import _run_cloudflared
|
||||
cloudflare = _run_cloudflared(port)
|
||||
elif(args.ngrok):
|
||||
from flask_ngrok import _run_ngrok
|
||||
cloudflare = _run_ngrok()
|
||||
elif(args.remote):
|
||||
from flask_cloudflared import _run_cloudflared
|
||||
cloudflare = _run_cloudflared(port)
|
||||
if(args.ngrok or args.remote):
|
||||
if(args.localtunnel or args.ngrok or args.remote):
|
||||
with open('cloudflare.log', 'w') as cloudflarelog:
|
||||
cloudflarelog.write("KoboldAI has finished loading and is available at the following link : " + cloudflare)
|
||||
print(format(colors.GREEN) + "KoboldAI has finished loading and is available at the following link : " + cloudflare + format(colors.END))
|
||||
else:
|
||||
print("{0}Webserver has started, you can now connect to this machine at port 5000{1}"
|
||||
.format(colors.GREEN, colors.END))
|
||||
print("{0}Webserver has started, you can now connect to this machine at port {1}{2}"
|
||||
.format(colors.GREEN, port, colors.END))
|
||||
vars.serverstarted = True
|
||||
socketio.run(app, host='0.0.0.0', port=port)
|
||||
else:
|
||||
|
||||
162
colab/TPU.ipynb
162
colab/TPU.ipynb
@ -7,7 +7,7 @@
|
||||
"colab_type": "text"
|
||||
},
|
||||
"source": [
|
||||
"<a href=\"https://colab.research.google.com/github/KoboldAI/KoboldAI-Client/blob/main/colab/TPU.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||||
"<a href=\"https://colab.research.google.com/github/henk717/KoboldAI/blob/united/colab/TPU.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -18,7 +18,21 @@
|
||||
"\n",
|
||||
"For more information about KoboldAI check our our Github readme : https://github.com/KoboldAI/KoboldAI-Client/blob/main/readme.md\n",
|
||||
"\n",
|
||||
"More (smaller) models are available in the **[GPU edition](https://colab.research.google.com/github/koboldai/KoboldAI-Client/blob/main/colab/GPU.ipynb)**!"
|
||||
"More (smaller) models are available in the **[GPU edition](https://colab.research.google.com/github/koboldai/KoboldAI-Client/blob/main/colab/GPU.ipynb)**!\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"## How to load KoboldAI: Everything you need to know\n",
|
||||
"1. On a phone? First put your browser in desktop mode because of a Google Colab bug. Otherwise nothing will happen when you click the play button. Then tap the play button next to \"<-- Tap This if you play on Mobile\", you will see an audio player. Keep the audio player playing so Colab does not get shut down in the background.\n",
|
||||
"2. Select the desired model, you will find a description of all the available models further down the page.\n",
|
||||
"3. Click the play button next to \"<-- Select your model below and then click this to start KoboldAI\".\n",
|
||||
"4. Got a message saying no accelerator is available? Click cancel, and try again in a few minutes. If you do not manage to get a session when you frequently try again try at a different time of day, colab can be busy or your priority may have been lowered by frequent usage.\n",
|
||||
"5. After everything is done loading you will get a link that you can use to open KoboldAI. In case of Localtunnel you will also be warned that some people are abusing Localtunnel for phishing, once you acknowledge this warning you will be taken to KoboldAI's interface. If you picked Cloudflare and get a 1033 error refresh the error page after waiting one minute.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"Further down the page you can find descriptions of the models, and tips to get the most out of your Google Colab experience.\n",
|
||||
"\n",
|
||||
"Make sure to keep this page open while you are using KoboldAI, and check back regularly to see if you got a Captcha. Failure to complete the captcha's in time can result in termination of your session or a lower priority towards the TPUs."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "zrLGxVCEaqZx"
|
||||
@ -47,11 +61,13 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#@title <b><-- Select your model below and then click this to start KoboldAI</b>\n",
|
||||
"#@markdown You can find a description of the models below along with instructions on how to start KoboldAI.\n",
|
||||
"\n",
|
||||
"#@title <b><-- Click this to start KoboldAI</b>\n",
|
||||
"Model = \"Skein 6B\" #@param [\"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Lit 6B\", \"Shinen 6B\", \"Generic 6B\", \"C1 6B\"]\n",
|
||||
"Version = \"Official\" #@param [\"Official\", \"United\"] {allow-input: true}\n",
|
||||
"Drive = \"Unextracted (Less Space)\" #@param [\"Unextracted (Less Space)\", \"Extracted (Faster Loading)\"]\n",
|
||||
"#@markdown Extracted models take up more space but load faster the next time you use them, not all models use your Google Drive. See the Model list below for descriptions and space requirements. If your extracted model does not load the next time you try to launch KoboldAI delete the folder from your Google Drive and ensure enough space is available.\n",
|
||||
"Model = \"Janeway 13B\" #@param [\"Janeway 13B\", \"Shinen 13B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Shinen 6B\", \"Lit 6B\", \"Convo 6B\", \"C1 6B\", \"NeoX 20B\", \"KoboldAI/fairseq-dense-13B\", \"EleutherAI/gpt-j-6B\"] {allow-input: true}\n",
|
||||
"Version = \"United\" #@param [\"Official\", \"United\"] {allow-input: true}\n",
|
||||
"Provider = \"Localtunnel\" #@param [\"Localtunnel\", \"Cloudflare\"]\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"try:\n",
|
||||
@ -64,78 +80,95 @@
|
||||
"from google.colab import drive\n",
|
||||
"drive.mount('/content/drive/')\n",
|
||||
"\n",
|
||||
"!wget https://henk.tech/ckds -O - | bash /dev/stdin -i drive\n",
|
||||
"\n",
|
||||
"if Model == \"Skein 6B\":\n",
|
||||
" path = \"gpt-j-6b-skein-jax\"\n",
|
||||
"if Model == \"Janeway 13B\":\n",
|
||||
" Model = \"KoboldAI/fairseq-dense-13B-Janeway\"\n",
|
||||
" path = \"\"\n",
|
||||
" download = \"\"\n",
|
||||
"elif Model == \"Shinen 13B\":\n",
|
||||
" Model = \"KoboldAI/fairseq-dense-13B-Shinen\"\n",
|
||||
" path = \"\"\n",
|
||||
" download = \"\"\n",
|
||||
"elif Model == \"NeoX 20B\":\n",
|
||||
" Model = \"TPUMeshTransformerGPTNeoX\"\n",
|
||||
" path = \" -p gpt-neox-20b-jax\"\n",
|
||||
" location = \"colab\"\n",
|
||||
" download = \"-a https://storage.henk.tech/KoboldAI/skein-jax.txt\"\n",
|
||||
" download = \" -a https://storage.henk.tech/KoboldAI/neox-20b.txt\"\n",
|
||||
" extract = \"\"\n",
|
||||
" Drive = \"Unextracted (Less Space)\"\n",
|
||||
" ![[ -f /content/drive/MyDrive/KoboldAI/settings/gpt-neox-20b-jax.settings ]] || echo -e \"{\\n \\\"apikey\\\": \\\"\\\",\\n \\\"andepth\\\": 3,\\n \\\"temp\\\": 0.5,\\n \\\"top_p\\\": 0.9,\\n \\\"top_k\\\": 0,\\n \\\"tfs\\\": 1.0,\\n \\\"rep_pen\\\": 1.03,\\n \\\"genamt\\\": 80,\\n \\\"max_length\\\": 2048,\\n \\\"ikgen\\\": 200,\\n \\\"formatoptns\\\": {\\n \\\"frmttriminc\\\": true,\\n \\\"frmtrmblln\\\": false,\\n \\\"frmtrmspch\\\": false,\\n \\\"frmtadsnsp\\\": false\\n },\\n \\\"numseqs\\\": 1,\\n \\\"widepth\\\": 3,\\n \\\"useprompt\\\": true,\\n \\\"adventure\\\": false\\n}\" > /content/drive/MyDrive/KoboldAI/settings/gpt-neox-20b-jax.settings\n",
|
||||
"elif Model == \"Skein 6B\":\n",
|
||||
" Model = \"TPUMeshTransformerGPTJ\"\n",
|
||||
" path = \" -p gpt-j-6b-skein-jax\"\n",
|
||||
" location = \"colab\"\n",
|
||||
" download = \" -a https://storage.henk.tech/KoboldAI/skein-jax.txt\"\n",
|
||||
" extract = \"\"\n",
|
||||
" Drive = \"Unextracted (Less Space)\"\n",
|
||||
" ![[ -f /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-skein-jax.settings ]] || echo -e \"{\\n \\\"apikey\\\": \\\"\\\",\\n \\\"andepth\\\": 3,\\n \\\"temp\\\": 0.5,\\n \\\"top_p\\\": 0.9,\\n \\\"top_k\\\": 0,\\n \\\"tfs\\\": 1.0,\\n \\\"rep_pen\\\": 1.1,\\n \\\"genamt\\\": 80,\\n \\\"max_length\\\": 2048,\\n \\\"ikgen\\\": 200,\\n \\\"formatoptns\\\": {\\n \\\"frmttriminc\\\": true,\\n \\\"frmtrmblln\\\": false,\\n \\\"frmtrmspch\\\": false,\\n \\\"frmtadsnsp\\\": false\\n },\\n \\\"numseqs\\\": 1,\\n \\\"widepth\\\": 3,\\n \\\"useprompt\\\": true,\\n \\\"adventure\\\": false\\n}\" > /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-skein-jax.settings\n",
|
||||
"if Model == \"Janeway 6B\":\n",
|
||||
" path = \"gpt-j-6b-janeway-jax\"\n",
|
||||
"elif Model == \"Janeway 6B\":\n",
|
||||
" Model = \"TPUMeshTransformerGPTJ\"\n",
|
||||
" path = \" -p gpt-j-6b-janeway-jax\"\n",
|
||||
" location = \"colab\"\n",
|
||||
" download = \"-a https://storage.henk.tech/KoboldAI/janeway-jax.txt\"\n",
|
||||
" download = \" -a https://storage.henk.tech/KoboldAI/janeway-jax.txt\"\n",
|
||||
" extract = \"\"\n",
|
||||
" Drive = \"Unextracted (Less Space)\"\n",
|
||||
" ![[ -f /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-janeway-jax.settings ]] || echo -e \"{\\n \\\"apikey\\\": \\\"\\\",\\n \\\"andepth\\\": 3,\\n \\\"temp\\\": 0.5,\\n \\\"top_p\\\": 0.9,\\n \\\"top_k\\\": 0,\\n \\\"tfs\\\": 1.0,\\n \\\"rep_pen\\\": 1.1,\\n \\\"rep_pen_slope\\\": 0.7,\\n \\\"rep_pen_range\\\": 1024.0,\\n \\\"genamt\\\": 80,\\n \\\"max_length\\\": 2048,\\n \\\"ikgen\\\": 200,\\n \\\"formatoptns\\\": {\\n \\\"frmttriminc\\\": true,\\n \\\"frmtrmblln\\\": false,\\n \\\"frmtrmspch\\\": false,\\n \\\"frmtadsnsp\\\": false,\\n \\\"singleline\\\": false\\n },\\n \\\"numseqs\\\": 1,\\n \\\"widepth\\\": 3,\\n \\\"useprompt\\\": true,\\n \\\"adventure\\\": false,\\n \\\"chatmode\\\": false,\\n \\\"chatname\\\": \\\"You\\\",\\n \\\"dynamicscan\\\": false,\\n \\\"nopromptgen\\\": false,\\n \\\"rngpersist\\\": false,\\n \\\"nogenmod\\\": false,\\n \\\"autosave\\\": false,\\n \\\"welcome\\\": false,\\n \\\"newlinemode\\\": \\\"n\\\",\\n \\\"antemplate\\\": \\\"[Genre: <|>]\\\",\\n \\\"userscripts\\\": [],\\n \\\"corescript\\\": \\\"default.lua\\\",\\n \\\"softprompt\\\": \\\"\\\"\\n}\" > /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-janeway-jax.settings\n",
|
||||
"if Model == \"Adventure 6B\":\n",
|
||||
" path = \"gpt-j-6b-adventure-jax\"\n",
|
||||
"elif Model == \"Adventure 6B\":\n",
|
||||
" Model = \"TPUMeshTransformerGPTJ\"\n",
|
||||
" path = \" -p gpt-j-6b-adventure-jax\"\n",
|
||||
" location = \"colab\"\n",
|
||||
" download = \"-a https://api.wandb.ai/files/ve-forbryderne/adventure/carol-data/models/gpt-j-6b-adventure-jax/aria2.txt\"\n",
|
||||
" download = \" -a https://api.wandb.ai/files/ve-forbryderne/adventure/carol-data/models/gpt-j-6b-adventure-jax/aria2.txt\"\n",
|
||||
" extract = \"\"\n",
|
||||
" Drive = \"Unextracted (Less Space)\"\n",
|
||||
" ![[ -f /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-adventure-jax.settings ]] || echo -e \"{\\n \\\"apikey\\\": \\\"\\\",\\n \\\"andepth\\\": 3,\\n \\\"temp\\\": 0.5,\\n \\\"top_p\\\": 0.9,\\n \\\"top_k\\\": 0,\\n \\\"tfs\\\": 1.0,\\n \\\"rep_pen\\\": 1.1,\\n \\\"genamt\\\": 80,\\n \\\"max_length\\\": 2048,\\n \\\"ikgen\\\": 200,\\n \\\"formatoptns\\\": {\\n \\\"frmttriminc\\\": true,\\n \\\"frmtrmblln\\\": false,\\n \\\"frmtrmspch\\\": false,\\n \\\"frmtadsnsp\\\": false\\n },\\n \\\"numseqs\\\": 1,\\n \\\"widepth\\\": 3,\\n \\\"useprompt\\\": true,\\n \\\"adventure\\\": true\\n}\" > /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-adventure-jax.settings\n",
|
||||
"if Model == \"Lit 6B\":\n",
|
||||
" path = \"gpt-j-6b-lit-jax\"\n",
|
||||
" location = \"drive\"\n",
|
||||
" download = \"-a https://storage.henk.tech/KoboldAI/aria2.php?file=gpt-j-6b-lit-jax.7z\"\n",
|
||||
" extract = \"-z gpt-j-6b-lit-jax.7z\"\n",
|
||||
" ![[ -f /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-lit-jax.settings ]] || echo -e \"{\\n \\\"apikey\\\": \\\"\\\",\\n \\\"andepth\\\": 3,\\n \\\"temp\\\": 0.5,\\n \\\"top_p\\\": 0.9,\\n \\\"top_k\\\": 0,\\n \\\"tfs\\\": 1.0,\\n \\\"rep_pen\\\": 1.1,\\n \\\"genamt\\\": 80,\\n \\\"max_length\\\": 2048,\\n \\\"ikgen\\\": 200,\\n \\\"formatoptns\\\": {\\n \\\"frmttriminc\\\": true,\\n \\\"frmtrmblln\\\": false,\\n \\\"frmtrmspch\\\": false,\\n \\\"frmtadsnsp\\\": false\\n },\\n \\\"numseqs\\\": 1,\\n \\\"widepth\\\": 3,\\n \\\"useprompt\\\": true,\\n \\\"adventure\\\": false\\n}\" > /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-lit-jax.settings\n",
|
||||
"if Model == \"Shinen 6B\":\n",
|
||||
" path = \"gpt-j-6b-shinen-jax\"\n",
|
||||
"elif Model == \"Lit 6B\":\n",
|
||||
" Model = \"hakurei/lit-6B\"\n",
|
||||
" path = \"\"\n",
|
||||
" download = \"\"\n",
|
||||
"elif Model == \"Shinen 6B\":\n",
|
||||
" Model = \"TPUMeshTransformerGPTJ\"\n",
|
||||
" path = \" -p gpt-j-6b-shinen-jax\"\n",
|
||||
" location = \"colab\"\n",
|
||||
" download = \"-a https://storage.henk.tech/KoboldAI/shinen-jax.txt\"\n",
|
||||
" download = \" -a https://storage.henk.tech/KoboldAI/shinen-jax.txt\"\n",
|
||||
" extract = \"\"\n",
|
||||
" Drive = \"Unextracted (Less Space)\"\n",
|
||||
" ![[ -f /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-shinen-jax.settings ]] || echo -e \"{\\n \\\"apikey\\\": \\\"\\\",\\n \\\"andepth\\\": 3,\\n \\\"temp\\\": 0.5,\\n \\\"top_p\\\": 0.9,\\n \\\"top_k\\\": 0,\\n \\\"tfs\\\": 1.0,\\n \\\"rep_pen\\\": 1.1,\\n \\\"rep_pen_slope\\\": 0.7,\\n \\\"rep_pen_range\\\": 1024.0,\\n \\\"genamt\\\": 80,\\n \\\"max_length\\\": 2048,\\n \\\"ikgen\\\": 200,\\n \\\"formatoptns\\\": {\\n \\\"frmttriminc\\\": true,\\n \\\"frmtrmblln\\\": false,\\n \\\"frmtrmspch\\\": false,\\n \\\"frmtadsnsp\\\": false,\\n \\\"singleline\\\": false\\n },\\n \\\"numseqs\\\": 1,\\n \\\"widepth\\\": 3,\\n \\\"useprompt\\\": true,\\n \\\"adventure\\\": false,\\n \\\"chatmode\\\": false,\\n \\\"chatname\\\": \\\"You\\\",\\n \\\"dynamicscan\\\": false,\\n \\\"nopromptgen\\\": false,\\n \\\"rngpersist\\\": false,\\n \\\"nogenmod\\\": false,\\n \\\"autosave\\\": false,\\n \\\"welcome\\\": false,\\n \\\"newlinemode\\\": \\\"n\\\",\\n \\\"antemplate\\\": \\\"[Genre: <|>]\\\",\\n \\\"userscripts\\\": [],\\n \\\"corescript\\\": \\\"default.lua\\\",\\n \\\"softprompt\\\": \\\"\\\"\\n}\" > /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-shinen-jax.settings\n",
|
||||
"if Model == \"Generic 6B\":\n",
|
||||
" path = \"step_383500\"\n",
|
||||
" location = \"drive\"\n",
|
||||
" download = \"-a https://storage.henk.tech/KoboldAI/aria2.php?file=step_383500_slim.tar.zstd\"\n",
|
||||
" extract = \"-t step_383500_slim.tar.zstd\"\n",
|
||||
" ![[ -f /content/drive/MyDrive/KoboldAI/settings/step_383500.settings ]] || echo -e \"{\\n \\\"apikey\\\": \\\"\\\",\\n \\\"andepth\\\": 3,\\n \\\"temp\\\": 0.5,\\n \\\"top_p\\\": 0.9,\\n \\\"top_k\\\": 0,\\n \\\"tfs\\\": 1.0,\\n \\\"rep_pen\\\": 1.1,\\n \\\"genamt\\\": 80,\\n \\\"max_length\\\": 2048,\\n \\\"ikgen\\\": 200,\\n \\\"formatoptns\\\": {\\n \\\"frmttriminc\\\": true,\\n \\\"frmtrmblln\\\": false,\\n \\\"frmtrmspch\\\": false,\\n \\\"frmtadsnsp\\\": false\\n },\\n \\\"numseqs\\\": 1,\\n \\\"widepth\\\": 3,\\n \\\"useprompt\\\": true,\\n \\\"adventure\\\": false\\n}\" > /content/drive/MyDrive/KoboldAI/settings/step_383500.settings\n",
|
||||
"if Model == \"C1 6B\":\n",
|
||||
" path = \"gpt-j-6b-c1-jax\"\n",
|
||||
" location = \"drive\"\n",
|
||||
" download = \"-a https://storage.henk.tech/KoboldAI/aria2.php?file=gpt-j-6b-c1-jax.7z\"\n",
|
||||
" extract = \"-z gpt-j-6b-c1-jax.7z\"\n",
|
||||
" ![[ -f /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-c1-jax.settings ]] || echo -e \"{\\n \\\"apikey\\\": \\\"\\\",\\n \\\"andepth\\\": 3,\\n \\\"temp\\\": 0.5,\\n \\\"top_p\\\": 0.9,\\n \\\"top_k\\\": 0,\\n \\\"tfs\\\": 1.0,\\n \\\"rep_pen\\\": 1.1,\\n \\\"genamt\\\": 80,\\n \\\"max_length\\\": 2048,\\n \\\"ikgen\\\": 200,\\n \\\"formatoptns\\\": {\\n \\\"frmttriminc\\\": true,\\n \\\"frmtrmblln\\\": false,\\n \\\"frmtrmspch\\\": false,\\n \\\"frmtadsnsp\\\": false\\n },\\n \\\"numseqs\\\": 1,\\n \\\"widepth\\\": 3,\\n \\\"useprompt\\\": true,\\n \\\"chatmode\\\": true\\n}\" > /content/drive/MyDrive/KoboldAI/settings/gpt-j-6b-c1-jax.settings\n",
|
||||
"elif Model == \"Convo 6B\":\n",
|
||||
" Model = \"hitomi-team/convo-6B\"\n",
|
||||
" path = \"\"\n",
|
||||
" download = \"\"\n",
|
||||
"elif Model == \"C1 6B\":\n",
|
||||
" Model = \"hakurei/C1-6B\"\n",
|
||||
" path = \"\"\n",
|
||||
" download = \"\"\n",
|
||||
"else:\n",
|
||||
" path = \"\"\n",
|
||||
" download = \"\"\n",
|
||||
"\n",
|
||||
"if Drive == \"Unextracted (Less Space)\":\n",
|
||||
" xloc = \"colab\"\n",
|
||||
"if Drive == \"Extracted (Faster Loading)\":\n",
|
||||
" xloc = \"drive\"\n",
|
||||
"if Provider == \"Localtunnel\":\n",
|
||||
" tunnel = \"--localtunnel yes\"\n",
|
||||
"else:\n",
|
||||
" tunnel = \"\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"!wget https://henk.tech/ckds -O - | bash /dev/stdin $download -l $location $extract -p $path -m TPUMeshTransformerGPTJ -g $Version -x $xloc"
|
||||
"!wget https://henk.tech/ckds -O - | bash /dev/stdin $path$download -m $Model -g $Version $tunnel"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# TPU Edition Model Descriptions\n",
|
||||
"## TPU Edition Model Descriptions\n",
|
||||
"\n",
|
||||
"| Model | Size | Style | Drive Space | Description |\n",
|
||||
"| ------------------------------ | ------ | --------- | ----------- | ------------------------------------------------------------ |\n",
|
||||
"| Skein 6B by VE_FORBRYDERNE | 6B TPU | Hybrid | 0 GB | Skein is our flagship 6B model, it is a hybrid between a Adventure model and a Novel model. Best used with either Adventure mode or the You Bias userscript enabled. Skein has been trained on high quality Novels along with CYOA adventure stories and is not as wackey as the Adventure model. It also has tagging support. |\n",
|
||||
"| Janeway 6B by Mr Seeker | 6B TPU | Novel | 0 GB | Janeway is a model created from Picard's dataset combined with a brand new collection of ebooks. This model is trained on 20% more content than Picard and has been trained on literature from various genres. Although the model is mainly focussed on SFW, romantic scenes might involve a degree of nudity. |\n",
|
||||
"| Adventure 6B by VE_FORBRYDERNE | 6B TPU | Adventure | 0 GB | Adventure is a 6B model designed to mimick the behavior of AI Dungeon. It is exclusively for Adventure Mode and can take you on the epic and wackey adventures that AI Dungeon players love. It also features the many tropes of AI Dungeon as it has been trained on very similar data. It must be used in second person (You). |\n",
|
||||
"| Lit 6B by Haru | 6B TPU | NSFW | 8 GB / 12 GB | Lit is a great NSFW model trained by Haru on both a large set of Literotica stories and high quality novels along with tagging support. Creating a high quality model for your NSFW stories. This model is exclusively a novel model and is best used in third person. |\n",
|
||||
"| Shinen 6B by Mr Seeker | 6B TPU | NSFW | 0 GB | Shinen is an alternative to the Lit model designed to be more explicit. If Lit is to tame for you Shinen might produce better results. While it is a Novel model it is unsuitable for SFW stories due to its heavy NSFW bias. Shinen will not hold back. It is meant to be used in KoboldAI's regular mode. |\n",
|
||||
"| Generic 6B by EleutherAI | 6B TPU | Generic | 10 GB / 12 GB | GPT-J-6B is what all other models are based on, if you need something that has no specific bias towards any particular subject this is the model for you. Best used when the other models are not suitable for what you wish to do. Such as homework assistance, blog writing, coding and more. It needs more hand holding than other models and is more prone to undesirable formatting changes. |\n",
|
||||
"| C1 6B by Haru | 6B TPU | Chatbot | 8 GB / 12 GB | C1 has been trained on various internet chatrooms, it makes the basis for an interesting chatbot model and has been optimized to be used in the Chatmode. |\n",
|
||||
"| Model | Size | Style | Description |\n",
|
||||
"| --- | --- | --- | --- |\n",
|
||||
"| Janeway by Mr Seeker | 13B | Novel | Janeway is a model created from Picard's dataset combined with a brand new collection of ebooks. This model is trained on 20% more content than Picard and has been trained on literature from various genres. Although the model is mainly focussed on SFW, romantic scenes might involve a degree of nudity. |\n",
|
||||
"| Shinen by Mr Seeker | 13B | NSFW | Shinen is an NSFW model designed to be more explicit. Trained on a variety of stories from the website Sexstories it contains many different kinks. |\n",
|
||||
"| Skein by VE\\_FORBRYDERNE | 6B | Adventure | Skein is best used with Adventure mode enabled, it consists of a 4 times larger adventure dataset than the Adventure model making it excellent for text adventure gaming. On top of that it also consists of light novel training further expanding its knowledge and writing capabilities. It can be used with the You filter bias if you wish to write Novels with it, but dedicated Novel models can perform better for this task. |\n",
|
||||
"| Adventure by VE\\_FORBRYDERNE | 6B | Adventure | Adventure is a 6B model designed to mimick the behavior of AI Dungeon. It is exclusively for Adventure Mode and can take you on the epic and wackey adventures that AI Dungeon players love. It also features the many tropes of AI Dungeon as it has been trained on very similar data. It must be used in second person (You). |\n",
|
||||
"| Lit by Haru | 6B | NSFW | Lit is a great NSFW model trained by Haru on both a large set of Literotica stories and high quality novels along with tagging support. Creating a high quality model for your NSFW stories. This model is exclusively a novel model and is best used in third person. |\n",
|
||||
"| Convo | 6B | Chatbot | Convo-6B is a GPT-J 6B model fine-tuned on a collection of high quality open source datasets which amount to 6 million messages. The primary goal of the model is to provide improved performance and generalization when generating multi-turn dialogue for characters that were not present from within the fine tuning data. The prompted performance has especially improved over the predecessor model [C1-6B](https://huggingface.co/hakurei/c1-6B). |\n",
|
||||
"| C1 by Haru | 6B | Chatbot | C1 has been trained on various internet chatrooms, it makes the basis for an interesting chatbot model and has been optimized to be used in the Chatmode. |\n",
|
||||
"| Neo(X) by EleutherAI | 20B | Generic | NeoX is the largest EleutherAI model currently available, being a generic model it is not particularly trained towards anything and can do a variety of writing, Q&A and coding tasks. 20B's performance is closely compared to the 13B models and it is worth trying both especially if you have a task that does not involve english writing. Its behavior will be similar to the GPT-J-6B model since they are trained on the same dataset but with more sensitivity towards repetition penalty and with more knowledge. |\n",
|
||||
"| Fairseq Dense | 13B | Generic | Trained by Facebook Researchers this model stems from the MOE research project within Fairseq. This particular version has been converted by us for use in KoboldAI. It is known to be on par with the larger 20B model from EleutherAI and considered as better for pop culture and language tasks. Because the model has never seen a new line (enter) it may perform worse on formatting and paragraphing. |\n",
|
||||
"| GPT-J-6B by EleutherAI | 6B | Generic | This model serves as the basis for most other 6B models (Some being based on Fairseq Dense instead). Being trained on the Pile and not biased towards anything in particular it is suitable for a variety of tasks such as writing, Q&A and coding tasks. You will likely get better result with larger generic models or finetuned models. |\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# [GPU Edition Model Descriptions](https://colab.research.google.com/github/KoboldAI/KoboldAI-Client/blob/main/colab/GPU.ipynb)\n",
|
||||
@ -156,21 +189,16 @@
|
||||
"| NSFW | Indicates that the model is strongly biased towards NSFW content and is not suitable for children, work environments or livestreaming. Most NSFW models are also Novel models in nature. |\n",
|
||||
"| Adventure | These models are excellent for people willing to play KoboldAI like a Text Adventure game and are meant to be used with Adventure mode enabled. Even if you wish to use it as a Novel style model you should always have Adventure mode on and set it to story. These models typically have a strong bias towards the use of the word You and without Adventure mode enabled break the story flow and write actions on your behalf. |\n",
|
||||
"| Chatbot | These models are specifically trained for chatting and are best used with the Chatmode enabled. Typically trained on either public chatrooms or private chats. |\n",
|
||||
"| Hybrid | Hybrid models are a blend between different styles, for example they are trained on both Novel stories and Adventure stories. These models are great variety models that you can use for multiple different playstyles and modes, but depending on your usage you may need to enable Adventure Mode or the You bias (in userscripts). |\n",
|
||||
"| Generic | Generic models are not trained towards anything specific, typically used as a basis for other tasks and models. They can do everything the other models can do, but require much more handholding to work properly. Generic models are an ideal basis for tasks that we have no specific model for, or for experiencing a softprompt in its raw form. |\n",
|
||||
"\n",
|
||||
"## How to start KoboldAI in 7 simple steps\n",
|
||||
"Using KoboldAI on Google Colab is easy! Simply follow these steps to get started:\n",
|
||||
"1. Mobile phone? Tap the play button below next to \"<--- Tap this if you play on mobile\" to reveal an audio player, play the silent audio to keep the tab alive so Google will not shut you down when your using KoboldAI. If no audio player is revealed your phone browser does not support Google Colab in the mobile view, go to your browser menu and enable Desktop mode before you continue.\n",
|
||||
"2. Select the model that most describes what you would like to do, by default we have the most recommended model for people willing to try out KoboldAI selected.\n",
|
||||
"3. Click the play button next to \"<--- Click this to start KoboldAI\".\n",
|
||||
"4. Allow Google Drive access, this typically happens trough a popup but sometimes Google Drive access may be requested trough the older method by asking you to click on a link and copy a code. This is normal behavior for Colab and only you will get access to your files, nothing is shared with us.\n",
|
||||
"5. Now the automatic installation and Download process starts, for most models in the TPU edition expect the loading to take between 15 and 30 minutes on average depending on the current Colab download speeds and the model you selected. These downloads happen trough Google's internet connection, you will not be billed by your internet provider and it will not count towards any download limits.\n",
|
||||
"6. After waiting a Trycloudflare link appears, click the link to enjoy KoboldAI. If you get a 1033 error Cloudflare is not done loading, in that case keep refreshing until it goes away. (If it keeps happening after 2 minutes Cloudflare has an issue, in that case you can use Runtime -> Restart and Run All to get a new link).\n",
|
||||
"7. As you play KoboldAI, keep this Colab tab open in the background and check occationally for Captcha's so they do not shut your instance down. If you do get shut down you can always download a copy of your gamesave in the Save menu inside KoboldAI. Stories are never lost as long as you keep KoboldAI open in your browser.\n",
|
||||
"\n",
|
||||
"Get a error message saying you do not have access to a GPU/TPU instance? Do not continue and try again later, KoboldAI will not run correctly without them.\n",
|
||||
"\n"
|
||||
"---\n",
|
||||
"## Tips to get the most out of Google Colab\n",
|
||||
"- Google will occationally show a Captcha, typically after it has been open for 30 minutes but it can be more frequent if you often use Colab. Make sure to do these properly, or you risk getting your instance shut down and getting a lower priority towards the TPU's.\n",
|
||||
"- KoboldAI uses Google Drive to store your files and settings, if you wish to upload a softprompt or userscript this can be done directly on the Google Drive website. You can also use this to download backups of your KoboldAI related files or upload models of your own.\n",
|
||||
"- Don't want to save your stories on Google Drive for privacy reasons? Do not use KoboldAI's save function and instead click Download as .json, this will automatically download the story to your own computer without ever touching Google's harddrives. You can load this back trough the Load from file option.\n",
|
||||
"- Google shut your instance down unexpectedly? You can still make use of the Download as .json button to recover your story as long as you did not close the KoboldAI window. You can then load this back up in your next session.\n",
|
||||
"- Done with KoboldAI? Go to the Runtime menu, click on Manage Sessions and terminate your open sessions that you no longer need. This trick can help you maintain higher priority towards getting a TPU.\n",
|
||||
"- Models stored on Google Drive typically load faster than models we need to download from the internet."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "i0-9ARA3c4Fx"
|
||||
|
||||
@ -67,6 +67,7 @@
|
||||
"!wget https://henk.tech/ckds -O - | bash /dev/stdin -g $Version -i only $Args\n",
|
||||
"\n",
|
||||
"!pip install colabcode\n",
|
||||
"!pip install 'flask>=2.1.0'\n",
|
||||
"from colabcode import ColabCode\n",
|
||||
"ColabCode(authtoken=Authtoken)"
|
||||
]
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
# KoboldAI Easy Colab Deployment Script by Henk717
|
||||
|
||||
# read the options
|
||||
TEMP=`getopt -o m:i:p:c:d:x:a:l:z:g:t:n:b:s: --long model:,init:,path:,configname:,download:,aria2:,dloc:xloc:7z:git:tar:ngrok:branch:savemodel: -- "$@"`
|
||||
TEMP=`getopt -o m:i:p:c:d:x:a:l:z:g:t:n:b:s: --long model:,init:,path:,configname:,download:,aria2:,dloc:,xloc:,7z:,git:,tar:,ngrok:,branch:,savemodel:,localtunnel:,lt: -- "$@"`
|
||||
eval set -- "$TEMP"
|
||||
|
||||
# extract options and their arguments into variables.
|
||||
@ -17,7 +17,9 @@ while true ; do
|
||||
-c|--configname)
|
||||
configname=" --configname $2" ; shift 2 ;;
|
||||
-n|--ngrok)
|
||||
configname=" --ngrok" ; shift 2 ;;
|
||||
ngrok=" --ngrok" ; shift 2 ;;
|
||||
--lt|--localtunnel)
|
||||
localtunnel=" --localtunnel" ; shift 2 ;;
|
||||
-d|--download)
|
||||
download="$2" ; shift 2 ;;
|
||||
-a|--aria2)
|
||||
@ -50,8 +52,8 @@ function launch
|
||||
exit 0
|
||||
else
|
||||
cd /content/KoboldAI-Client
|
||||
echo "Launching KoboldAI with the following options : python3 aiserver.py$model$kmpath$configname$ngrok$savemodel --colab"
|
||||
python3 aiserver.py$model$kmpath$configname$ngrok$savemodel --colab
|
||||
echo "Launching KoboldAI with the following options : python3 aiserver.py$model$kmpath$configname$ngrok$localtunnel$savemodel --colab"
|
||||
python3 aiserver.py$model$kmpath$configname$ngrok$localtunnel$savemodel --colab
|
||||
exit
|
||||
fi
|
||||
}
|
||||
@ -159,8 +161,9 @@ if [ "$init" != "skip" ]; then
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
|
||||
# Make sure Colab has netbase
|
||||
# Make sure Colab has the system dependencies
|
||||
sudo apt install netbase -y
|
||||
npm install -g localtunnel
|
||||
fi
|
||||
|
||||
cd /content
|
||||
|
||||
0
commandline-rocm.sh
Normal file → Executable file
0
commandline-rocm.sh
Normal file → Executable file
0
commandline.sh
Normal file → Executable file
0
commandline.sh
Normal file → Executable file
@ -9,8 +9,7 @@
|
||||
"static_weights": {
|
||||
"model.embed_tokens.weight": {"mtj": {"module": "embedding_shard/~/linear", "param": "w", "transforms": ["no_transpose", "vocab_pad"]}},
|
||||
"model.layer_norm.weight": {"mtj": {"module": "projection_shard/~/replicated_layer_norm", "param": "scale"}},
|
||||
"model.layer_norm.bias": {"mtj": {"module": "projection_shard/~/replicated_layer_norm", "param": "offset"}},
|
||||
"lm_head.weight": {"mtj": {"module": "projection_shard/~/linear", "param": "w", "transforms": ["vocab_pad"]}}
|
||||
"model.layer_norm.bias": {"mtj": {"module": "projection_shard/~/replicated_layer_norm", "param": "offset"}}
|
||||
},
|
||||
"layer_weights": {
|
||||
"model.layers.{layer}.self_attn.q_proj.weight": {"mtj": {"module": "layer_{layer}/~/linear", "param": "w"}},
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
|
||||
bin/micromamba create -f environments/rocm.yml -r runtime -n koboldai-rocm -y
|
||||
bin/micromamba run -r runtime -n koboldai-rocm python aiserver.py
|
||||
bin/micromamba run -r runtime -n koboldai-rocm python aiserver.py $*
|
||||
|
||||
2
play.sh
2
play.sh
@ -1,3 +1,3 @@
|
||||
wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
|
||||
bin/micromamba create -f environments/huggingface.yml -r runtime -n koboldai -y
|
||||
bin/micromamba run -r runtime -n koboldai python aiserver.py
|
||||
bin/micromamba run -r runtime -n koboldai python aiserver.py $*
|
||||
|
||||
@ -89,7 +89,7 @@ class LazyTensor:
|
||||
def __repr__(self):
|
||||
return self.__view(repr)
|
||||
|
||||
def materialize(self, checkpoint: Union[zipfile.ZipFile, zipfile.ZipExtFile], map_location=None) -> torch.Tensor:
|
||||
def materialize(self, checkpoint: Union[zipfile.ZipFile, zipfile.ZipExtFile], map_location=None, no_grad=True) -> torch.Tensor:
|
||||
size = reduce(lambda x, y: x * y, self.shape, 1)
|
||||
dtype = self.dtype
|
||||
nbytes = size if dtype is torch.bool else size * ((torch.finfo if dtype.is_floating_point else torch.iinfo)(dtype).bits >> 3)
|
||||
@ -106,7 +106,7 @@ class LazyTensor:
|
||||
storage = torch.serialization._get_restore_location(map_location)(storage, self.location)
|
||||
tensor = torch.tensor([], dtype=storage.dtype, device=storage.device)
|
||||
tensor.set_(storage, 0, self.shape, self.stride)
|
||||
tensor.requires_grad = self.requires_grad
|
||||
tensor.requires_grad = not no_grad and self.requires_grad
|
||||
tensor._backward_hooks = self.backward_hooks
|
||||
return tensor
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user