rixty/KoboldAI-Client
1
1
Fork 0
mirror of https://github.com/KoboldAI/KoboldAI-Client.git synced 2025-06-05 21:59:24 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #38 from one-some/ui2-probs

Browse Source 
Fix probabilities
This commit is contained in:
ebolam
2022-08-17 18:27:37 -04:00
committed by GitHub
parent 63cd4560e0 f4fc6b5969
commit 89be79bd0b
1 changed files with 16 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
aiserver.py
View File

@@ -1716,20 +1716,26 @@ def patch_transformers():
assert scores.ndim == 2 assert scores.ndim == 2
assert input_ids.ndim == 2 assert input_ids.ndim == 2
if koboldai_vars.numseqs > 1 or not koboldai_vars.show_probs: if not koboldai_vars.show_probs:
return scores return scores
probs = F.softmax(scores, dim = -1).cpu().numpy()[0] for batch_index, batch in enumerate(scores):
probs = F.softmax(batch, dim = -1).cpu().numpy()
token_prob_info = [] token_prob_info = []
for token_id, score in sorted(enumerate(probs), key=lambda x: x[1], reverse=True)[:8]: for token_id, score in sorted(enumerate(probs), key=lambda x: x[1], reverse=True)[:8]:
token_prob_info.append({ token_prob_info.append({
"tokenId": token_id, "tokenId": token_id,
"decoded": utils.decodenewlines(tokenizer.decode(token_id)), "decoded": utils.decodenewlines(tokenizer.decode(token_id)),
"score": float(score), "score": float(score),
}) })
if len(scores) == 1:
koboldai_vars.actions.set_probabilities(token_prob_info)
else:
koboldai_vars.actions.set_option_probabilities(token_prob_info, batch_index)
#koboldai_vars.token_stream_queue.probability_buffer = token_prob_info
return scores return scores
def new_get_logits_processor(*args, **kwargs) -> LogitsProcessorList: def new_get_logits_processor(*args, **kwargs) -> LogitsProcessorList: