From 6c87dfe10d98a1597c27309211c9471e7a4d8b53 Mon Sep 17 00:00:00 2001 From: sasha0552 Date: Thu, 20 Jun 2024 01:13:32 +0000 Subject: [PATCH 01/11] Add a token counting endpoint for vLLM --- public/scripts/tokenizers.js | 5 +++-- src/endpoints/tokenizers.js | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index 39a19d0ca..5c017ad83 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -7,7 +7,7 @@ import { kai_flags } from './kai-settings.js'; import { textgen_types, textgenerationwebui_settings as textgen_settings, getTextGenServer, getTextGenModel } from './textgen-settings.js'; import { getCurrentDreamGenModelTokenizer, getCurrentOpenRouterModelTokenizer, openRouterModels } from './textgen-models.js'; -const { OOBA, TABBY, KOBOLDCPP, APHRODITE, LLAMACPP, OPENROUTER, DREAMGEN } = textgen_types; +const { OOBA, TABBY, KOBOLDCPP, VLLM, APHRODITE, LLAMACPP, OPENROUTER, DREAMGEN } = textgen_types; export const CHARACTERS_PER_TOKEN_RATIO = 3.35; const TOKENIZER_WARNING_KEY = 'tokenizationWarningShown'; @@ -39,7 +39,7 @@ export const SENTENCEPIECE_TOKENIZERS = [ //tokenizers.NERD2, ]; -export const TEXTGEN_TOKENIZERS = [OOBA, TABBY, KOBOLDCPP, LLAMACPP, APHRODITE]; +export const TEXTGEN_TOKENIZERS = [OOBA, TABBY, KOBOLDCPP, LLAMACPP, VLLM, APHRODITE]; const TOKENIZER_URLS = { [tokenizers.GPT2]: { @@ -765,6 +765,7 @@ function getTextgenAPITokenizationParams(str) { api_type: textgen_settings.type, url: getTextGenServer(), legacy_api: textgen_settings.legacy_api && (textgen_settings.type === OOBA || textgen_settings.type === APHRODITE), + vllm_model: textgen_settings.vllm_model, }; } diff --git a/src/endpoints/tokenizers.js b/src/endpoints/tokenizers.js index 0c8c99034..6b150e32b 100644 --- a/src/endpoints/tokenizers.js +++ b/src/endpoints/tokenizers.js @@ -784,6 +784,7 @@ router.post('/remote/textgenerationwebui/encode', jsonParser, async function (re const text = String(request.body.text) || ''; const baseUrl = String(request.body.url); const legacyApi = Boolean(request.body.legacy_api); + const vllmModel = String(request.body.vllm_model) || ''; try { const args = { @@ -814,7 +815,9 @@ router.post('/remote/textgenerationwebui/encode', jsonParser, async function (re args.body = JSON.stringify({ 'content': text }); break; case TEXTGEN_TYPES.VLLM: - return response.send({ error: true }); + url += '/tokenize'; + args.body = JSON.stringify({ 'model': vllmModel, 'prompt': text }); + break; case TEXTGEN_TYPES.APHRODITE: url += '/v1/tokenize'; args.body = JSON.stringify({ 'prompt': text }); @@ -834,7 +837,7 @@ router.post('/remote/textgenerationwebui/encode', jsonParser, async function (re } const data = await result.json(); - const count = legacyApi ? data?.results[0]?.tokens : (data?.length ?? data?.value ?? data?.tokens?.length); + const count = legacyApi ? data?.results[0]?.tokens : (data?.length ?? data?.count ?? data?.value ?? data?.tokens?.length); const ids = legacyApi ? [] : (data?.tokens ?? data?.ids ?? []); return response.send({ count, ids }); From ff23808d3b910267ff6effad62a2758d1262b5d2 Mon Sep 17 00:00:00 2001 From: Wolfsblvt Date: Sat, 6 Jul 2024 03:23:02 +0200 Subject: [PATCH 02/11] Add WI toggle to include/exclude names in scanning --- default/content/settings.json | 1 + public/index.html | 6 ++++++ public/script.js | 3 ++- public/scripts/world-info.js | 15 +++++++++++++-- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/default/content/settings.json b/default/content/settings.json index c04b88bfb..f558c769f 100644 --- a/default/content/settings.json +++ b/default/content/settings.json @@ -13,6 +13,7 @@ }, "world_info_depth": 2, "world_info_budget": 25, + "world_info_include_names": true, "world_info_recursive": true, "world_info_overflow_alert": false, "world_info_case_sensitive": false, diff --git a/public/index.html b/public/index.html index 6109568bb..5b9f5b455 100644 --- a/public/index.html +++ b/public/index.html @@ -3542,6 +3542,12 @@
+
+ diff --git a/public/lib/diff_match_patch.js b/public/lib/diff_match_patch.js new file mode 100644 index 000000000..2fe320a1b --- /dev/null +++ b/public/lib/diff_match_patch.js @@ -0,0 +1,55 @@ +var diff_match_patch=function(){this.Diff_Timeout=1;this.Diff_EditCost=4;this.Match_Threshold=.5;this.Match_Distance=1E3;this.Patch_DeleteThreshold=.5;this.Patch_Margin=4;this.Match_MaxBits=32},DIFF_DELETE=-1,DIFF_INSERT=1,DIFF_EQUAL=0;diff_match_patch.Diff=function(a,b){this[0]=a;this[1]=b};diff_match_patch.Diff.prototype.length=2;diff_match_patch.Diff.prototype.toString=function(){return this[0]+","+this[1]}; +diff_match_patch.prototype.diff_main=function(a,b,c,d){"undefined"==typeof d&&(d=0>=this.Diff_Timeout?Number.MAX_VALUE:(new Date).getTime()+1E3*this.Diff_Timeout);if(null==a||null==b)throw Error("Null input. (diff_main)");if(a==b)return a?[new diff_match_patch.Diff(DIFF_EQUAL,a)]:[];"undefined"==typeof c&&(c=!0);var e=c,f=this.diff_commonPrefix(a,b);c=a.substring(0,f);a=a.substring(f);b=b.substring(f);f=this.diff_commonSuffix(a,b);var g=a.substring(a.length-f);a=a.substring(0,a.length-f);b=b.substring(0, +b.length-f);a=this.diff_compute_(a,b,e,d);c&&a.unshift(new diff_match_patch.Diff(DIFF_EQUAL,c));g&&a.push(new diff_match_patch.Diff(DIFF_EQUAL,g));this.diff_cleanupMerge(a);return a}; +diff_match_patch.prototype.diff_compute_=function(a,b,c,d){if(!a)return[new diff_match_patch.Diff(DIFF_INSERT,b)];if(!b)return[new diff_match_patch.Diff(DIFF_DELETE,a)];var e=a.length>b.length?a:b,f=a.length>b.length?b:a,g=e.indexOf(f);return-1!=g?(c=[new diff_match_patch.Diff(DIFF_INSERT,e.substring(0,g)),new diff_match_patch.Diff(DIFF_EQUAL,f),new diff_match_patch.Diff(DIFF_INSERT,e.substring(g+f.length))],a.length>b.length&&(c[0][0]=c[2][0]=DIFF_DELETE),c):1==f.length?[new diff_match_patch.Diff(DIFF_DELETE, +a),new diff_match_patch.Diff(DIFF_INSERT,b)]:(e=this.diff_halfMatch_(a,b))?(b=e[1],f=e[3],a=e[4],e=this.diff_main(e[0],e[2],c,d),c=this.diff_main(b,f,c,d),e.concat([new diff_match_patch.Diff(DIFF_EQUAL,a)],c)):c&&100c);t++){for(var v=-t+p;v<=t-x;v+=2){var n=f+v;var r=v==-t||v!=t&&h[n-1]d)x+=2;else if(y>e)p+=2;else if(m&&(n=f+k-v,0<=n&&n= +u)return this.diff_bisectSplit_(a,b,r,y,c)}}for(v=-t+w;v<=t-q;v+=2){n=f+v;u=v==-t||v!=t&&l[n-1]d)q+=2;else if(r>e)w+=2;else if(!m&&(n=f+k-v,0<=n&&n=u)))return this.diff_bisectSplit_(a,b,r,y,c)}}return[new diff_match_patch.Diff(DIFF_DELETE,a),new diff_match_patch.Diff(DIFF_INSERT,b)]}; +diff_match_patch.prototype.diff_bisectSplit_=function(a,b,c,d,e){var f=a.substring(0,c),g=b.substring(0,d);a=a.substring(c);b=b.substring(d);f=this.diff_main(f,g,!1,e);e=this.diff_main(a,b,!1,e);return f.concat(e)}; +diff_match_patch.prototype.diff_linesToChars_=function(a,b){function c(a){for(var b="",c=0,g=-1,h=d.length;gd?a=a.substring(c-d):c=a.length?[h,k,l,m,g]:null}if(0>=this.Diff_Timeout)return null; +var d=a.length>b.length?a:b,e=a.length>b.length?b:a;if(4>d.length||2*e.lengthd[4].length?g:d:d:g;else return null;if(a.length>b.length){d=g[0];e=g[1];var h=g[2];var l=g[3]}else h=g[0],l=g[1],d=g[2],e=g[3];return[d,e,h,l,g[4]]}; +diff_match_patch.prototype.diff_cleanupSemantic=function(a){for(var b=!1,c=[],d=0,e=null,f=0,g=0,h=0,l=0,k=0;f=e){if(d>=b.length/2||d>=c.length/2)a.splice(f,0,new diff_match_patch.Diff(DIFF_EQUAL,c.substring(0,d))),a[f-1][1]=b.substring(0,b.length-d),a[f+1][1]=c.substring(d),f++}else if(e>=b.length/2||e>=c.length/2)a.splice(f,0,new diff_match_patch.Diff(DIFF_EQUAL,b.substring(0,e))),a[f-1][0]=DIFF_INSERT,a[f-1][1]=c.substring(0,c.length-e),a[f+1][0]=DIFF_DELETE, +a[f+1][1]=b.substring(e),f++;f++}f++}}; +diff_match_patch.prototype.diff_cleanupSemanticLossless=function(a){function b(a,b){if(!a||!b)return 6;var c=a.charAt(a.length-1),d=b.charAt(0),e=c.match(diff_match_patch.nonAlphaNumericRegex_),f=d.match(diff_match_patch.nonAlphaNumericRegex_),g=e&&c.match(diff_match_patch.whitespaceRegex_),h=f&&d.match(diff_match_patch.whitespaceRegex_);c=g&&c.match(diff_match_patch.linebreakRegex_);d=h&&d.match(diff_match_patch.linebreakRegex_);var k=c&&a.match(diff_match_patch.blanklineEndRegex_),l=d&&b.match(diff_match_patch.blanklineStartRegex_); +return k||l?5:c||d?4:e&&!g&&h?3:g||h?2:e||f?1:0}for(var c=1;c=k&&(k=m,g=d,h=e,l=f)}a[c-1][1]!=g&&(g?a[c-1][1]=g:(a.splice(c- +1,1),c--),a[c][1]=h,l?a[c+1][1]=l:(a.splice(c+1,1),c--))}c++}};diff_match_patch.nonAlphaNumericRegex_=/[^a-zA-Z0-9]/;diff_match_patch.whitespaceRegex_=/\s/;diff_match_patch.linebreakRegex_=/[\r\n]/;diff_match_patch.blanklineEndRegex_=/\n\r?\n$/;diff_match_patch.blanklineStartRegex_=/^\r?\n\r?\n/; +diff_match_patch.prototype.diff_cleanupEfficiency=function(a){for(var b=!1,c=[],d=0,e=null,f=0,g=!1,h=!1,l=!1,k=!1;fb)break;e=c;f=d}return a.length!=g&&a[g][0]===DIFF_DELETE?f:f+(b-e)}; +diff_match_patch.prototype.diff_prettyHtml=function(a){for(var b=[],c=/&/g,d=//g,f=/\n/g,g=0;g");switch(h){case DIFF_INSERT:b[g]=''+l+"";break;case DIFF_DELETE:b[g]=''+l+"";break;case DIFF_EQUAL:b[g]=""+l+""}}return b.join("")}; +diff_match_patch.prototype.diff_text1=function(a){for(var b=[],c=0;cthis.Match_MaxBits)throw Error("Pattern too long for this browser.");var e=this.match_alphabet_(b),f=this,g=this.Match_Threshold,h=a.indexOf(b,c);-1!=h&&(g=Math.min(d(0,h),g),h=a.lastIndexOf(b,c+b.length),-1!=h&&(g=Math.min(d(0,h),g)));var l=1<=k;q--){var t=e[a.charAt(q-1)];m[q]=0===w?(m[q+1]<<1|1)&t:(m[q+1]<<1|1)&t|(x[q+1]|x[q])<<1|1|x[q+1];if(m[q]&l&&(t=d(w,q-1),t<=g))if(g=t,h=q-1,h>c)k=Math.max(1,2*c-h);else break}if(d(w+1,c)>g)break;x=m}return h}; +diff_match_patch.prototype.match_alphabet_=function(a){for(var b={},c=0;c=2*this.Patch_Margin&&e&&(this.patch_addContext_(a,h),c.push(a),a=new diff_match_patch.patch_obj,e=0,h=d,f=g)}k!==DIFF_INSERT&&(f+=m.length);k!==DIFF_DELETE&&(g+=m.length)}e&&(this.patch_addContext_(a,h),c.push(a));return c}; +diff_match_patch.prototype.patch_deepCopy=function(a){for(var b=[],c=0;cthis.Match_MaxBits){var k=this.match_main(b,h.substring(0,this.Match_MaxBits),g);-1!=k&&(l=this.match_main(b,h.substring(h.length-this.Match_MaxBits),g+h.length-this.Match_MaxBits),-1==l||k>=l)&&(k=-1)}else k=this.match_main(b,h, +g);if(-1==k)e[f]=!1,d-=a[f].length2-a[f].length1;else if(e[f]=!0,d=k-g,g=-1==l?b.substring(k,k+h.length):b.substring(k,l+this.Match_MaxBits),h==g)b=b.substring(0,k)+this.diff_text2(a[f].diffs)+b.substring(k+h.length);else if(g=this.diff_main(h,g,!1),h.length>this.Match_MaxBits&&this.diff_levenshtein(g)/h.length>this.Patch_DeleteThreshold)e[f]=!1;else{this.diff_cleanupSemanticLossless(g);h=0;var m;for(l=0;le[0][1].length){var f=b-e[0][1].length;e[0][1]=c.substring(e[0][1].length)+e[0][1];d.start1-=f;d.start2-=f;d.length1+=f;d.length2+=f}d=a[a.length-1];e=d.diffs; +0==e.length||e[e.length-1][0]!=DIFF_EQUAL?(e.push(new diff_match_patch.Diff(DIFF_EQUAL,c)),d.length1+=b,d.length2+=b):b>e[e.length-1][1].length&&(f=b-e[e.length-1][1].length,e[e.length-1][1]+=c.substring(0,f),d.length1+=f,d.length2+=f);return c}; +diff_match_patch.prototype.patch_splitMax=function(a){for(var b=this.Match_MaxBits,c=0;c2*b?(h.length1+=k.length,e+=k.length,l=!1,h.diffs.push(new diff_match_patch.Diff(g,k)),d.diffs.shift()):(k=k.substring(0,b-h.length1-this.Patch_Margin),h.length1+=k.length,e+=k.length,g===DIFF_EQUAL?(h.length2+=k.length,f+=k.length):l=!1,h.diffs.push(new diff_match_patch.Diff(g,k)),k==d.diffs[0][1]?d.diffs.shift():d.diffs[0][1]=d.diffs[0][1].substring(k.length))}g=this.diff_text2(h.diffs); +g=g.substring(g.length-this.Patch_Margin);k=this.diff_text1(d.diffs).substring(0,this.Patch_Margin);""!==k&&(h.length1+=k.length,h.length2+=k.length,0!==h.diffs.length&&h.diffs[h.diffs.length-1][0]===DIFF_EQUAL?h.diffs[h.diffs.length-1][1]+=k:h.diffs.push(new diff_match_patch.Diff(DIFF_EQUAL,k)));l||a.splice(++c,0,h)}}};diff_match_patch.prototype.patch_toText=function(a){for(var b=[],c=0;c"; + const rawPromptWrapper = document.getElementById('rawPromptWrapper'); + rawPromptWrapper.innerHTML = ds; + $('#rawPromptPopup').slideToggle(); + }); + } else { + popup.dlg.querySelector('#diffPrevPrompt').style.display='none'; + } popup.dlg.querySelector('#copyPromptToClipboard').addEventListener('click', function () { let rawPrompt = itemizedPrompts[PromptArrayItemForRawPromptDisplay].rawPrompt; let rawPromptValues = rawPrompt; diff --git a/public/scripts/templates/itemizationChat.html b/public/scripts/templates/itemizationChat.html index 969a31757..839a96618 100644 --- a/public/scripts/templates/itemizationChat.html +++ b/public/scripts/templates/itemizationChat.html @@ -2,6 +2,7 @@ Prompt Itemization + Tokenizer: {{selectedTokenizer}}
API Used: {{this_main_api}}
diff --git a/public/scripts/templates/itemizationText.html b/public/scripts/templates/itemizationText.html index b3855e027..499297be1 100644 --- a/public/scripts/templates/itemizationText.html +++ b/public/scripts/templates/itemizationText.html @@ -2,6 +2,7 @@ Prompt Itemization + Tokenizer: {{selectedTokenizer}}
API Used: {{this_main_api}}
From 792c077267f7370bcfc6554ebb8a4b2cadb45fe3 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 7 Jul 2024 10:15:53 +0300 Subject: [PATCH 11/11] Fix for chat completion prompts --- .eslintrc.js | 1 + public/script.js | 47 +++++++++++++++++++++++------------------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 02433c082..b60cf4976 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -45,6 +45,7 @@ module.exports = { Readability: 'readonly', isProbablyReaderable: 'readonly', ePub: 'readonly', + diff_match_patch: 'readonly', }, }, ], diff --git a/public/script.js b/public/script.js index 650750826..de5d90db7 100644 --- a/public/script.js +++ b/public/script.js @@ -4907,7 +4907,7 @@ export function findItemizedPromptSet(itemizedPrompts, incomingMesId) { console.log(`wanting to raw display of ArrayItem: ${PromptArrayItemForRawPromptDisplay} which is mesID ${incomingMesId}`); console.log(itemizedPrompts[thisPromptSet]); break; - } else if (itemizedPrompts[i].rawPrompt){ + } else if (itemizedPrompts[i].rawPrompt) { priorPromptArrayItemForRawPromptDisplay = i; } } @@ -4927,35 +4927,39 @@ async function promptItemize(itemizedPrompts, requestedMesId) { } const params = await itemizedParams(itemizedPrompts, thisPromptSet); + const flatten = (rawPrompt) => Array.isArray(rawPrompt) ? rawPrompt.map(x => x.content).join('\n') : rawPrompt; const template = params.this_main_api == 'openai' ? await renderTemplateAsync('itemizationChat', params) : await renderTemplateAsync('itemizationText', params); const popup = new Popup(template, POPUP_TYPE.TEXT); - if (priorPromptArrayItemForRawPromptDisplay){ - popup.dlg.querySelector('#diffPrevPrompt').style.display=''; - popup.dlg.querySelector('#diffPrevPrompt').addEventListener('click', function () { - var dmp = new diff_match_patch(); - var text1 = itemizedPrompts[priorPromptArrayItemForRawPromptDisplay].rawPrompt; - var text2 = itemizedPrompts[PromptArrayItemForRawPromptDisplay].rawPrompt; + + /** @type {HTMLElement} */ + const diffPrevPrompt = popup.dlg.querySelector('#diffPrevPrompt'); + if (priorPromptArrayItemForRawPromptDisplay) { + diffPrevPrompt.style.display = ''; + diffPrevPrompt.addEventListener('click', function () { + const dmp = new diff_match_patch(); + const text1 = flatten(itemizedPrompts[priorPromptArrayItemForRawPromptDisplay].rawPrompt); + const text2 = flatten(itemizedPrompts[PromptArrayItemForRawPromptDisplay].rawPrompt); dmp.Diff_Timeout = 2.0; - - var d = dmp.diff_main(text1, text2); - var ds = dmp.diff_prettyHtml(d); + const d = dmp.diff_main(text1, text2); + let ds = dmp.diff_prettyHtml(d); // make it readable - ds = ds.replaceAll("background:#e6ffe6;", "background:#b9f3b9; color:black;"); - ds = ds.replaceAll("background:#ffe6e6;", "background:#f5b4b4; color:black;"); - ds = ds.replaceAll("¶", ""); - ds = "
" + ds + "
"; + ds = ds.replaceAll('background:#e6ffe6;', 'background:#b9f3b9; color:black;'); + ds = ds.replaceAll('background:#ffe6e6;', 'background:#f5b4b4; color:black;'); + ds = ds.replaceAll('¶', ''); + const container = document.createElement('div'); + container.innerHTML = DOMPurify.sanitize(ds); const rawPromptWrapper = document.getElementById('rawPromptWrapper'); - rawPromptWrapper.innerHTML = ds; - $('#rawPromptPopup').slideToggle(); + rawPromptWrapper.replaceChildren(container); + $('#rawPromptPopup').slideToggle(); }); } else { - popup.dlg.querySelector('#diffPrevPrompt').style.display='none'; + diffPrevPrompt.style.display = 'none'; } popup.dlg.querySelector('#copyPromptToClipboard').addEventListener('click', function () { let rawPrompt = itemizedPrompts[PromptArrayItemForRawPromptDisplay].rawPrompt; @@ -4975,16 +4979,11 @@ async function promptItemize(itemizedPrompts, requestedMesId) { console.log(itemizedPrompts); console.log(itemizedPrompts[PromptArrayItemForRawPromptDisplay].rawPrompt); - let rawPrompt = itemizedPrompts[PromptArrayItemForRawPromptDisplay].rawPrompt; - let rawPromptValues = rawPrompt; - - if (Array.isArray(rawPrompt)) { - rawPromptValues = rawPrompt.map(x => x.content).join('\n'); - } + const rawPrompt = flatten(itemizedPrompts[PromptArrayItemForRawPromptDisplay].rawPrompt); //let DisplayStringifiedPrompt = JSON.stringify(itemizedPrompts[PromptArrayItemForRawPromptDisplay].rawPrompt).replace(/\n+/g, '
'); const rawPromptWrapper = document.getElementById('rawPromptWrapper'); - rawPromptWrapper.innerText = rawPromptValues; + rawPromptWrapper.innerText = rawPrompt; $('#rawPromptPopup').slideToggle(); });