mirror of
https://github.com/KoboldAI/KoboldAI-Client.git
synced 2025-06-05 21:59:24 +02:00
Model: Successful load implementation
The goal of this series of commits is to have an implementation-agnostic interface for models, thus being less reliant on HF Transformers for model support. A model object will have a method for generation, a list of callbacks to be run on every token generation, a list of samplers that will modify probabilities, etc. Basically anything HF can do should be easily implementable with the new interface :^) Currently I've tested the loading of pre-downloaded models with breakmodel between GPUs and that works, though essentially no testing has been done in the larger scheme of things. Currently this is about the only supported configuration, and generation isn't very functional.
This commit is contained in:
18
utils.py
18
utils.py
@@ -633,4 +633,20 @@ def get_missing_module_names(model: PreTrainedModel, names: List[str]) -> List[s
|
||||
else:
|
||||
recurse(c[1], head=name + ".")
|
||||
recurse(model)
|
||||
return missing_names
|
||||
return missing_names
|
||||
|
||||
class UIProgressBarFile(object):
|
||||
"""Write TQDM progress to the UI."""
|
||||
def write(self, bar):
|
||||
bar = bar.replace("\r", "").replace("\n", "").replace(chr(0), "")
|
||||
if bar != "" and [ord(num) for num in bar] != [27, 91, 65]: #No idea why we're getting the 27, 1, 65 character set, just killing to so we can move on
|
||||
#logger.info(bar)
|
||||
print('\r' + bar, end='')
|
||||
time.sleep(0.01)
|
||||
try:
|
||||
emit('from_server', {'cmd': 'model_load_status', 'data': bar.replace(" ", " ")}, broadcast=True, room="UI_1")
|
||||
except:
|
||||
pass
|
||||
|
||||
def flush(self):
|
||||
pass
|
Reference in New Issue
Block a user