SillyTavern/public/scripts/gpt-2-3-tokenizer/vocab.bpe.js

1 line
467 KiB
JavaScript
Raw Normal View History

2023-07-20 19:32:15 +02:00
export default "#version: 0.2\nĠ t\nĠ a\nh e\ni n\nr e\no n\nĠt he\ne r\nĠ s\na t\nĠ w\nĠ o\ne n\nĠ c\ni t\ni s\na n\no r\ne s\nĠ b\ne d\nĠ f\nin g\nĠ p\no u\nĠa n\na l\na r\nĠt o\nĠ m\nĠo f\nĠ in\nĠ d\nĠ h\nĠan d\ni c\na s\nl e\nĠt h\ni on\no m\nl l\nen t\nĠ n\nĠ l\ns t\nĠ re\nv e\nĠ e\nr o\nl y\nĠb e\nĠ g\nĠ T\nc t\nĠ S\ni d\no t\nĠ I\nu t\ne t\nĠ A\nĠ is\nĠ on\ni m\na m\no w\na y\na d\ns e\nĠth at\nĠ C\ni g\nĠf or\na c\nĠ y\nv er\nu r\nĠ u\nl d\nĠs t\nĠ M\n' s\nĠ he\nĠ it\nat ion\nit h\ni r\nc e\nĠy ou\ni l\nĠ B\nĠw h\no l\nĠ P\nĠw ith\nĠ 1\nt er\nc h\nĠa s\nĠw e\nĠ (\nn d\ni ll\nĠ D\ni f\nĠ 2\na g\ner s\nk e\nĠ \"\nĠ H\ne m\nĠc on\nĠ W\nĠ R\nhe r\nĠw as\nĠ r\no d\nĠ F\nu l\nat e\nĠa t\nr i\np p\no re\nĠT he\nĠs e\nu s\nĠp ro\nĠh a\nu m\nĠa re\nĠd e\na in\nan d\nĠo r\nig h\nes t\nis t\na b\nr om\nĠ N\nt h\nĠc om\nĠ G\nu n\no p\n0 0\nĠ L\nĠn ot\nes s\nĠe x\nĠ v\nre s\nĠ E\ne w\nit y\nan t\nĠb y\ne l\no s\nor t\no c\nq u\nĠf rom\nĠha ve\nĠs u\ni ve\nou ld\nĠs h\nĠth is\nn t\nr a\np e\nigh t\nar t\nm ent\nĠa l\nu st\nen d\n- -\nal l\nĠ O\nac k\nĠc h\nĠ le\ni es\nre d\nar d\nâ Ģ\nou t\nĠ J\nĠa b\ne ar\ni v\nal ly\nou r\no st\ng h\np t\nĠp l\nas t\nĠc an\na k\nom e\nu d\nT he\nĠh is\nĠd o\nĠg o\nĠh as\ng e\n' t\nĠ U\nr ou\nĠs a\nĠ j\nĠb ut\nĠw or\nĠa ll\ne ct\nĠ k\nam e\nĠw ill\no k\nĠw he\nĠthe y\nid e\n0 1\nf f\nic h\np l\nt her\nĠt r\n. .\nĠin t\ni e\nu re\nag e\nĠn e\ni al\na p\nin e\nic e\nĠm e\nĠo ut\nan s\non e\non g\nion s\nĠwh o\nĠ K\nĠu p\nĠthe ir\nĠa d\nĠ 3\nĠu s\nat ed\nou s\nĠm ore\nu e\no g\nĠS t\nin d\ni ke\nĠs o\nim e\np er\n. \"\nb er\ni z\na ct\nĠon e\nĠsa id\nĠ -\na re\nĠyou r\nc c\nĠT h\nĠc l\ne p\na ke\nab le\ni p\nĠcon t\nĠwh ich\ni a\nĠ im\nĠab out\nĠwe re\nver y\nu b\nĠh ad\nĠ en\nĠcom p\n, \"\nĠI n\nĠu n\nĠa g\ni re\nac e\na u\nar y\nĠw ould\nas s\nr y\ âĢ\nc l\no ok\ne re\ns o\ V\nig n\ni b\nĠof f\nĠt e\nv en\ Y\ni le\no se\nit e\nor m\nĠ2 01\nĠre s\nĠm an\nĠp er\nĠo ther\nor d\nul t\nĠbe en\nĠl ike\nas e\nan ce\nk s\nay s\now n\nen ce\nĠd is\nct ion\nĠan y\nĠa pp\nĠs p\nin t\nres s\nation s\na il\ 4\nic al\nĠthe m\nĠhe r\nou nt\nĠC h\nĠa r\ if\nĠthe re\nĠp e\nĠy ear\na v\nĠm y\nĠs ome\nĠwhe n\nou gh\nac h\nĠth an\nr u\non d\nic k\nĠo ver\nve l\ qu\ Ċ\nĠs c\nre at\nre e\nĠI t\nou nd\np ort\nĠal so\nĠp art\nf ter\nĠk n\nĠbe c\nĠt ime\nen s\ 5\nop le\nĠwh at\nĠn o\nd u\nm er\nan g\nĠn ew\n-- --\nĠg et\nor y\nit ion\ning s\nĠj ust\nĠint o\ 0\nent s\no ve\nt e\nĠpe ople\nĠp re\nĠit s\nĠre c\nĠt w\ni an\nir st\nar k\nor s\nĠwor k\nad e\no b\nĠs he\nĠo ur\nw n\nin k\nl ic\nĠ1 9\nĠH e\nis h\nnd er\nau se\nĠh im\non s\ [\ ro\nf orm\ni ld\nat es\nver s\nĠon ly\no ll\nĠs pe\nc k\ne ll\nam p\nĠa cc\nĠb l\ni ous\nur n\nf t\no od\nĠh ow\nhe d\ '\nĠa fter\na w\nĠat t\no v\nn e\nĠpl ay\ner v\nic t\nĠc ould\nit t\nĠa m\nĠf irst\nĠ 6\nĠa ct\nĠ $\ne c\nh ing\nu al\nu ll\nĠcom m\no y\no ld\nc es\nat er\nĠf e\nĠbe t\nw e\nif f\nĠtw o\noc k\nĠb ack\n) .\nid ent\nĠu nder\nrou gh\nse l\nx t\nĠm ay\nrou nd\nĠp o\np h\nis s\nĠd es\nĠm ost\nĠd id\nĠad d\nj ect\nĠin c\nf ore\nĠp ol\non t\nĠag ain\ncl ud\nter n\nĠkn ow\nĠne ed\nĠcon s\nĠc o\nĠ .\nĠw ant\nĠse e\nĠ 7\nn ing\ni ew\nĠTh is\nc ed\nĠe ven\nĠin d\nt y\nĠW e\nat h\nĠthe se\nĠp r\nĠu se\nĠbec ause\nĠf l\nn g\nĠn ow\nĠâĢ ĵ\nc om\nis e\nĠm ake\nĠthe n\now er\nĠe very\nĠU n\nĠse c\nos s\nu ch\nĠe m\nĠ =\nĠR e\ni ed\nr it\nĠin v\nle ct\nĠsu pp\nat ing\nĠl ook\nm an\npe ct\nĠ 8\nro w\nĠb u\nĠwhe re\nif ic\nĠyear s\ni ly\nĠd iff\nĠsh ould\nĠre m\nT h\nI n\nĠe v\nd ay\n' re\nri b\nĠre l\ns s\nĠde f\nĠr ight\nĠs y\n) ,\nl es\n00 0\nhe n\nĠth rough\nĠT r\n_ _\nĠw ay\nĠd on\ ,\nĠ1 0\nas ed\nĠas s\nub lic\nĠre g\nĠA nd\ni x\ very\nĠin clud\not her\nĠim p\not h\nĠsu b\nĠâĢ Ķ\nĠbe ing\nar g\nĠW h\n= =\nib le\nĠdo es\nan ge\nr am\ 9\ner t\np s\nit ed\nation a