mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-06-05 21:59:27 +02:00
Merge branch 'staging' into parser-v2
This commit is contained in:
LenAnderson
@ -18,7 +18,7 @@
|
||||
"input_suffix": "<|eot_id|>",
|
||||
"system_suffix": "<|eot_id|>",
|
||||
"user_alignment_message": "",
|
||||
"system_same_as_user": false,
|
||||
"system_same_as_user": true,
|
||||
"last_system_sequence": "",
|
||||
"name": "Llama 3 Instruct"
|
||||
}
|
||||
|
||||
9
package-lock.json
generated
9
package-lock.json
generated
@ -27,6 +27,7 @@
|
||||
"form-data": "^4.0.0",
|
||||
"google-translate-api-browser": "^3.0.1",
|
||||
"gpt3-tokenizer": "^1.1.5",
|
||||
"he": "^1.2.0",
|
||||
"helmet": "^7.1.0",
|
||||
"ip-matching": "^2.1.2",
|
||||
"ipaddr.js": "^2.0.1",
|
||||
@ -2800,6 +2801,14 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/he": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
|
||||
"integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
|
||||
"bin": {
|
||||
"he": "bin/he"
|
||||
}
|
||||
},
|
||||
"node_modules/helmet": {
|
||||
"version": "7.1.0",
|
||||
"resolved": "https://registry.npmjs.org/helmet/-/helmet-7.1.0.tgz",
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
"form-data": "^4.0.0",
|
||||
"google-translate-api-browser": "^3.0.1",
|
||||
"gpt3-tokenizer": "^1.1.5",
|
||||
"he": "^1.2.0",
|
||||
"helmet": "^7.1.0",
|
||||
"ip-matching": "^2.1.2",
|
||||
"ipaddr.js": "^2.0.1",
|
||||
|
||||
@ -86,6 +86,10 @@
|
||||
margin: 5px;
|
||||
}
|
||||
|
||||
.marginLeft5 {
|
||||
margin-left: 5px;
|
||||
}
|
||||
|
||||
.overflowYAuto {
|
||||
overflow-y: auto;
|
||||
}
|
||||
@ -257,6 +261,10 @@
|
||||
flex-basis: 48%
|
||||
}
|
||||
|
||||
.flexBasis30p {
|
||||
flex-basis: 30%;
|
||||
}
|
||||
|
||||
.flex-container {
|
||||
display: flex;
|
||||
gap: 5px;
|
||||
|
||||
@ -141,10 +141,10 @@
|
||||
</a>
|
||||
</h4>
|
||||
<div class="flex-container flexNoGap">
|
||||
<select id="settings_preset" data-preset-manager-for="kobold" class="flex1 flexBasis100p text_pole">
|
||||
<select id="settings_preset" data-preset-manager-for="kobold" class="flex1 text_pole">
|
||||
<option value="gui" data-i18n="guikoboldaisettings">GUI KoboldAI Settings</option>
|
||||
</select>
|
||||
<div class="flex-container flexBasis100p justifyCenter">
|
||||
<div class="flex-container marginLeft5 ">
|
||||
<input type="file" hidden data-preset-manager-file="kobold" accept=".json, .settings">
|
||||
<i data-newbie-hidden data-preset-manager-update="kobold" class="menu_button fa-solid fa-save" title="Update current preset" data-i18n="[title]Update current preset"></i>
|
||||
<i data-newbie-hidden data-preset-manager-new="kobold" class="menu_button fa-solid fa-file-circle-plus" title="Save preset as" data-i18n="[title]Save preset as"></i>
|
||||
@ -163,10 +163,10 @@
|
||||
</a>
|
||||
</h4>
|
||||
<div class="flex-container flexNoGap">
|
||||
<select id="settings_preset_novel" class="flex1 flexBasis100p text_pole" data-preset-manager-for="novel">
|
||||
<select id="settings_preset_novel" class="flex1 text_pole" data-preset-manager-for="novel">
|
||||
<option value="gui" data-i18n="default">Default</option>
|
||||
</select>
|
||||
<div class="flex-container flexBasis100p justifyCenter">
|
||||
<div class="flex-container marginLeft5 ">
|
||||
<input type="file" hidden data-preset-manager-file="novel" accept=".json, .settings">
|
||||
<i data-newbie-hidden data-preset-manager-update="novel" class="menu_button fa-solid fa-save" title="Update current preset" data-i18n="[title]Update current preset"></i>
|
||||
<i data-newbie-hidden data-preset-manager-new="novel" class="menu_button fa-solid fa-file-circle-plus" title="Save preset as" data-i18n="[title]Save preset as"></i>
|
||||
@ -184,7 +184,7 @@
|
||||
<select id="settings_preset_openai" class="flex1 text_pole" data-preset-manager-for="openai">
|
||||
<option value="gui" data-i18n="default">Default</option>
|
||||
</select>
|
||||
<div class="flex-container flexBasis100p justifyCenter">
|
||||
<div class="flex-container marginLeft5 ">
|
||||
<input id="openai_preset_import_file" type="file" accept=".json,.settings" hidden />
|
||||
<i id="update_oai_preset" class="menu_button fa-solid fa-save" title="Update current preset" data-i18n="[title]Update current preset"></i>
|
||||
<i id="new_oai_preset" class="menu_button fa-solid fa-file-circle-plus" title="Save preset as" data-i18n="[title]Save preset as"></i>
|
||||
@ -200,7 +200,7 @@
|
||||
<div class="flex-container flexNoGap">
|
||||
<select id="settings_preset_textgenerationwebui" class="flex1 text_pole" data-preset-manager-for="textgenerationwebui">
|
||||
</select>
|
||||
<div class="flex-container flexBasis100p justifyCenter">
|
||||
<div class="flex-container marginLeft5 ">
|
||||
<input type="file" hidden data-preset-manager-file="textgenerationwebui" accept=".json, .settings">
|
||||
<i data-newbie-hidden data-preset-manager-update="textgenerationwebui" class="menu_button fa-solid fa-save" title="Update current preset" data-i18n="[title]Update current preset"></i>
|
||||
<i data-newbie-hidden data-preset-manager-new="textgenerationwebui" class="menu_button fa-solid fa-file-circle-plus" title="Save preset as" data-i18n="[title]Save preset as"></i>
|
||||
@ -321,7 +321,7 @@
|
||||
</div>
|
||||
<div data-newbie-hidden class="range-block">
|
||||
<div class="range-block-title" data-i18n="Rep. Pen. Range.">
|
||||
Repetition Penalty Range
|
||||
Rep Pen Range
|
||||
</div>
|
||||
<div class="range-block-range-and-counter">
|
||||
<div class="range-block-range">
|
||||
@ -373,7 +373,7 @@
|
||||
</div>
|
||||
<div data-newbie-hidden class="range-block">
|
||||
<div class="range-block-title" data-i18n="Tail Free Sampling">
|
||||
Tail Free Sampling
|
||||
TFS
|
||||
</div>
|
||||
<div class="range-block-range-and-counter">
|
||||
<div class="range-block-range">
|
||||
@ -788,7 +788,7 @@
|
||||
<div id="advanced-ai-config-block" class="width100p">
|
||||
<div id="kobold_api-settings">
|
||||
<div class="flex-container gap10h5v justifyCenter">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="temperature">Temperature</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Temperature controls the randomness in token selection" title="Temperature controls the randomness in token selection: - low temperature (<1.0) leads to more predictable text, favoring higher probability tokens. - high temperature (>1.0) increases creativity and diversity in the output by giving lower probability tokens a better chance. Set to 1.0 for the original probabilities."></div>
|
||||
@ -796,7 +796,7 @@
|
||||
<input class="neo-range-slider" type="range" id="temp" name="volume" min="0.0" max="4.0" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0.0" max="4.0" step="0.01" data-for="temp" id="temp_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Top K">Top K</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Top K sets a maximum amount of top tokens that can be chosen from. E.g Top K is 20, this means only the 20 highest ranking tokens will be kept (regardless of their probabilities being diverse or limited). Set to 0 to disable."></div>
|
||||
@ -804,7 +804,7 @@
|
||||
<input class="neo-range-slider" type="range" id="top_k" name="volume" min="0" max="100" step="1">
|
||||
<input class="neo-range-input" type="number" min="0" max="100" step="1" data-for="top_k" id="top_k_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
Top P
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Top P (a.k.a. nucleus sampling) adds up all the top tokens required to add up to the target percentage. E.g If the Top 2 tokens are both 25%, and Top P is 0.50, only the Top 2 tokens are considered. Set to 1.0 to disable."></div>
|
||||
@ -812,7 +812,7 @@
|
||||
<input class="neo-range-slider" type="range" id="top_p" name="volume" min="0" max="1" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="top_p" id="top_p_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Typical P">Typical P</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Typical P Sampling prioritizes tokens based on their deviation from the average entropy of the set. It maintains tokens whose cumulative probability is close to a predefined threshold (e.g., 0.5), emphasizing those with average information content. Set to 1.0 to disable."></div>
|
||||
@ -820,7 +820,7 @@
|
||||
<input class="neo-range-slider" type="range" id="typical_p" name="volume" min="0" max="1" step="0.001">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="typical_p" id="typical_p_counter">
|
||||
</div>
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Min P">Min P</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Min P sets a base minimum probability. This is scaled according to the top token's probability. E.g If Top token is 80% probability, and Min P is 0.1, only tokens higher than 8% would be considered. Set to 0 to disable."></div>
|
||||
@ -828,7 +828,7 @@
|
||||
<input class="neo-range-slider" type="range" id="min_p" name="volume" min="0" max="1" step="0.001">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="min_p" id="min_p_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Top A">Top A</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Top A sets a threshold for token selection based on the square of the highest token probability. E.g if the Top-A value is 0.2 and the top token's probability is 50%, tokens with probabilities below 5% (0.2 * 0.5^2) are excluded. Set to 0 to disable."></div>
|
||||
@ -836,29 +836,29 @@
|
||||
<input class="neo-range-slider" type="range" id="top_a" name="volume" min="0" max="1" step="0.001">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="top_a" id="top_a_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Tail Free Sampling">Tail Free Sampling</span>
|
||||
<span data-i18n="Tail Free Sampling">TFS</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Tail-Free Sampling (TFS) searches for a tail of low-probability tokens in the distribution, by analyzing the rate of change in token probabilities using derivatives. It retains tokens up to a threshold (e.g., 0.3) based on the normalized second derivative. The closer to 0, the more discarded tokens. Set to 1.0 to disable."></div>
|
||||
</small>
|
||||
<input class="neo-range-slider" type="range" id="tfs" name="volume" min="0" max="1" step="0.001">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="tfs" id="tfs_counter">
|
||||
</div>
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="rep.pen">Repetition Penalty</span>
|
||||
</small>
|
||||
<input class="neo-range-slider" type="range" id="rep_pen" name="volume" min="1" max="3" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="1" max="3" step="0.01" data-for="rep_pen" id="rep_pen_counter">
|
||||
</div>
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="rep.pen range">Repetition Penalty Range</span>
|
||||
<span data-i18n="rep.pen range">Rep Pen Range</span>
|
||||
</small>
|
||||
<input class="neo-range-slider" type="range" id="rep_pen_range" name="volume" min="0" max="4096" step="1">
|
||||
<input class="neo-range-input" type="number" min="0" max="4096" step="1" data-for="rep_pen_range" id="rep_pen_range_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Rep. Pen. Slope">Repetition Penalty Slope</span>
|
||||
</small>
|
||||
@ -892,7 +892,7 @@
|
||||
</div>
|
||||
<hr class="wide100p">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter justifyCenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter justifyCenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<label class="checkbox_label alignItemsBaseline" for="use_default_badwordsids">
|
||||
<input id="use_default_badwordsids" type="checkbox" />
|
||||
<span>
|
||||
@ -901,7 +901,7 @@
|
||||
</span>
|
||||
</label>
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter textAlignCenter flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter textAlignCenter flexBasis30p flexGrow flexShrink gap0">
|
||||
<!-- <hr class="wide100p"> -->
|
||||
<small data-i18n="Seed">Seed</small>
|
||||
<!-- Max value is 2**64 - 1 -->
|
||||
@ -1185,7 +1185,7 @@
|
||||
<input type="number" id="n_textgenerationwebui" class="text_pole textAlignCenter" min="1" value="1" step="1" />
|
||||
</div>
|
||||
<div class="flex-container gap10h5v justifyCenter">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="temperature">Temperature</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Temperature controls the randomness in token selection" title="Temperature controls the randomness in token selection: - low temperature (<1.0) leads to more predictable text, favoring higher probability tokens. - high temperature (>1.0) increases creativity and diversity in the output by giving lower probability tokens a better chance. Set to 1.0 for the original probabilities."></div>
|
||||
@ -1193,7 +1193,7 @@
|
||||
<input class="neo-range-slider" type="range" id="temp_textgenerationwebui" name="volume" min="0.0" max="5.0" step="0.01" x-setting-id="temp">
|
||||
<input class="neo-range-input" type="number" min="0.0" max="5.0" step="0.01" data-for="temp_textgenerationwebui" id="temp_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Top K">Top K</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Top K sets a maximum amount of top tokens that can be chosen from" title="Top K sets a maximum amount of top tokens that can be chosen from. E.g Top K is 20, this means only the 20 highest ranking tokens will be kept (regardless of their probabilities being diverse or limited). Set to 0 (or -1, depending on your backend) to disable."></div>
|
||||
@ -1201,7 +1201,7 @@
|
||||
<input class="neo-range-slider" type="range" id="top_k_textgenerationwebui" name="volume" min="-1" max="200" step="1">
|
||||
<input class="neo-range-input" type="number" min="-1" max="200" step="1" data-for="top_k_textgenerationwebui" id="top_k_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Top P">Top P</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Top P (a.k.a. nucleus sampling)" title="Top P (a.k.a. nucleus sampling) adds up all the top tokens required to add up to the target percentage. E.g If the Top 2 tokens are both 25%, and Top P is 0.50, only the Top 2 tokens are considered. Set to 1.0 to disable."></div>
|
||||
@ -1209,7 +1209,7 @@
|
||||
<input class="neo-range-slider" type="range" id="top_p_textgenerationwebui" name="volume" min="0" max="1" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="top_p_textgenerationwebui" id="top_p_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Typical P">Typical P</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Typical P Sampling prioritizes tokens based on their deviation from the average entropy of the set" title="Typical P Sampling prioritizes tokens based on their deviation from the average entropy of the set. It maintains tokens whose cumulative probability is close to a predefined threshold (e.g., 0.5), emphasizing those with average information content. Set to 1.0 to disable."></div>
|
||||
@ -1217,7 +1217,7 @@
|
||||
<input class="neo-range-slider" type="range" id="typical_p_textgenerationwebui" name="volume" min="0" max="1" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="typical_p_textgenerationwebui" id="typical_p_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Min P">Min P</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Min P sets a base minimum probability" title="Min P sets a base minimum probability. This is scaled according to the top token's probability. E.g If Top token is 80% probability, and Min P is 0.1, only tokens higher than 8% would be considered. Set to 0 to disable."></div>
|
||||
@ -1225,7 +1225,7 @@
|
||||
<input class="neo-range-slider" type="range" id="min_p_textgenerationwebui" name="volume" min="0" max="1" step="0.001">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="min_p_textgenerationwebui" id="min_p_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Top A">Top A</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Top A sets a threshold for token selection based on the square of the highest token probability" title="Top A sets a threshold for token selection based on the square of the highest token probability. E.g if the Top-A value is 0.2 and the top token's probability is 50%, tokens with probabilities below 5% (0.2 * 0.5^2) are excluded. Set to 0 to disable."></div>
|
||||
@ -1233,15 +1233,15 @@
|
||||
<input class="neo-range-slider" type="range" id="top_a_textgenerationwebui" name="volume" min="0" max="1" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="top_a_textgenerationwebui" id="top_a_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Tail Free Sampling">Tail Free Sampling</span>
|
||||
<span data-i18n="Tail Free Sampling">TFS</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Tail-Free Sampling (TFS)" title="Tail-Free Sampling (TFS) searches for a tail of low-probability tokens in the distribution, by analyzing the rate of change in token probabilities using derivatives. It retains tokens up to a threshold (e.g., 0.3) based on the normalized second derivative. The closer to 0, the more discarded tokens. Set to 1.0 to disable."></div>
|
||||
</small>
|
||||
<input class="neo-range-slider" type="range" id="tfs_textgenerationwebui" name="volume" min="0" max="1" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="tfs_textgenerationwebui" id="tfs_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden data-tg-type="ooba,mancer" class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden data-tg-type="ooba,mancer" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Epsilon Cutoff">Epsilon Cutoff</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Epsilon cutoff sets a probability floor below which tokens are excluded from being sampled" title="Epsilon cutoff sets a probability floor below which tokens are excluded from being sampled. In units of 1e-4; a reasonable value is 3. Set to 0 to disable."></div>
|
||||
@ -1249,7 +1249,7 @@
|
||||
<input class="neo-range-slider" type="range" id="epsilon_cutoff_textgenerationwebui" name="volume" min="0" max="9" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="9" step="0.01" data-for="epsilon_cutoff_textgenerationwebui" id="epsilon_cutoff_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden data-tg-type="ooba,mancer" class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden data-tg-type="ooba,mancer" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Eta Cutoff">Eta Cutoff</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Eta cutoff is the main parameter of the special Eta Sampling technique. In units of 1e-4; a reasonable value is 3. Set to 0 to disable. See the paper Truncation Sampling as Language Model Desmoothing by Hewitt et al. (2022) for details." title="Eta cutoff is the main parameter of the special Eta Sampling technique. In units of 1e-4; a reasonable value is 3. Set to 0 to disable. See the paper Truncation Sampling as Language Model Desmoothing by Hewitt et al. (2022) for details."></div>
|
||||
@ -1257,42 +1257,42 @@
|
||||
<input class="neo-range-slider" type="range" id="eta_cutoff_textgenerationwebui" name="volume" min="0" max="20" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="20" step="0.01" data-for="eta_cutoff_textgenerationwebui" id="eta_cutoff_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="rep.pen">Repetition Penalty</small>
|
||||
<input class="neo-range-slider" type="range" id="rep_pen_textgenerationwebui" name="volume" min="1" max="3" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="1" max="3" step="0.01" data-for="rep_pen_textgenerationwebui" id="rep_pen_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-forAphro="False" class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<small data-i18n="rep.pen range">Repetition Penalty Range</small>
|
||||
<div data-forAphro="False" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="rep.pen range">Rep Pen Range</small>
|
||||
<input class="neo-range-slider" type="range" id="rep_pen_range_textgenerationwebui" name="volume" min="-1" max="8192" step="1">
|
||||
<input class="neo-range-input" type="number" min="-1" max="8192" step="1" data-for="rep_pen_range_textgenerationwebui" id="rep_pen_range_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-forAphro="False" data-tg-type="ooba" data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-forAphro="False" data-tg-type="ooba" data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="Encoder Rep. Pen.">Encoder Penalty</small>
|
||||
<input class="neo-range-slider" type="range" id="encoder_rep_pen_textgenerationwebui" name="volume" min="0.8" max="1.5" step="0.01" />
|
||||
<input class="neo-range-input" type="number" min="0.8" max="1.5" step="0.01" data-for="encoder_rep_pen_textgenerationwebui" id="encoder_rep_pen_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="Frequency Penalty">Frequency Penalty</small>
|
||||
<input class="neo-range-slider" type="range" id="freq_pen_textgenerationwebui" name="volume" min="-2" max="2" step="0.01" />
|
||||
<input class="neo-range-input" type="number" data-for="freq_pen_textgenerationwebui" min="-2" max="2" step="0.01" id="freq_pen_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="Presence Penalty">Presence Penalty</small>
|
||||
<input class="neo-range-slider" type="range" id="presence_pen_textgenerationwebui" name="volume" min="-2" max="2" step="0.01" />
|
||||
<input class="neo-range-input" type="number" min="-2" max="2" step="0.01" data-for="presence_pen_textgenerationwebui" id="presence_pen_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-forAphro="False" data-tg-type="ooba" data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-forAphro="False" data-tg-type="ooba" data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="No Repeat Ngram Size">No Repeat Ngram Size</small>
|
||||
<input class="neo-range-slider" type="range" id="no_repeat_ngram_size_textgenerationwebui" name="volume" min="0" max="20" step="1">
|
||||
<input class="neo-range-input" type="number" min="0" max="20" step="1" data-for="no_repeat_ngram_size_textgenerationwebui" id="no_repeat_ngram_size_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden data-tg-type="mancer, ooba, dreamgen" class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden data-tg-type="mancer, ooba, dreamgen" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="Min Length">Min Length</small>
|
||||
<input class="neo-range-slider" type="range" id="min_length_textgenerationwebui" name="volume" min="0" max="2000" step="1" />
|
||||
<input class="neo-range-input" type="number" min="0" max="2000" step="1" data-for="min_length_textgenerationwebui" id="min_length_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden data-tg-type="ooba" class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden data-tg-type="ooba" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="Max Tokens Second">Maximum tokens/second</small>
|
||||
<input class="neo-range-slider" type="range" id="max_tokens_second_textgenerationwebui" name="volume" min="0" max="20" step="1" />
|
||||
<input class="neo-range-input" type="number" min="0" max="20" step="1" data-for="max_tokens_second_textgenerationwebui" id="max_tokens_second_counter_textgenerationwebui">
|
||||
@ -3544,7 +3544,7 @@
|
||||
<div id="ui_preset_export_button" class="menu_button menu_button_icon margin0" title="Export a theme file" data-i18n="[title]Export a theme file">
|
||||
<i class="fa-solid fa-file-export"></i>
|
||||
</div>
|
||||
<div id="ui-preset-delete-button" class="menu_button menu_button_icon margin0" title="Delete a theme" data-i18n="[title]Delete a theme" >
|
||||
<div id="ui-preset-delete-button" class="menu_button menu_button_icon margin0" title="Delete a theme" data-i18n="[title]Delete a theme">
|
||||
<i class="fa-solid fa-trash-can"></i>
|
||||
</div>
|
||||
</div>
|
||||
@ -4488,9 +4488,7 @@
|
||||
<div class="flex1 flexGap5" title="Inserted before each part of the joined fields.">
|
||||
<label for="rm_group_generation_mode_join_prefix" class="flexnowrap width100p whitespacenowrap">
|
||||
<span data-i18n="Join Prefix">Join Prefix</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p"
|
||||
data-i18n="[title]When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)"
|
||||
title="When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)">
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)" title="When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)">
|
||||
</div>
|
||||
</label>
|
||||
<textarea id="rm_group_generation_mode_join_prefix" class="text_pole wide100p textarea_compact autoSetHeight" maxlength="2000" placeholder="—" rows="1"></textarea>
|
||||
@ -4498,9 +4496,7 @@
|
||||
<div class="flex1 flexGap5" title="Inserted after each part of the joined fields.">
|
||||
<label for="rm_group_generation_mode_join_suffix" class="flexnowrap width100p whitespacenowrap">
|
||||
<span data-i18n="Join Suffix">Join Suffix</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p"
|
||||
data-i18n="[title]When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)"
|
||||
title="When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)">
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)" title="When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)">
|
||||
</div>
|
||||
</label>
|
||||
<textarea id="rm_group_generation_mode_join_suffix" class="text_pole wide100p textarea_compact autoSetHeight" maxlength="2000" placeholder="—" rows="1"></textarea>
|
||||
@ -5008,7 +5004,7 @@
|
||||
<option value="3" data-role="" data-i18n="After AN">
|
||||
↓AN
|
||||
</option>
|
||||
<option value="4" data-role="0" data-i18n="at Depth System" >
|
||||
<option value="4" data-role="0" data-i18n="at Depth System">
|
||||
@D ⚙️
|
||||
</option>
|
||||
<option value="4" data-role="1" data-i18n="at Depth User">
|
||||
@ -6032,10 +6028,7 @@
|
||||
<div class="fa-fw fa-solid fa-grip drag-grabber"></div>
|
||||
<div class="fa-fw fa-solid fa-circle-xmark dragClose" id="closeZoom"></div>
|
||||
</div>
|
||||
<img class="zoomed_avatar_img" src=""
|
||||
data-izoomify-url=""
|
||||
data-izoomify-magnify="1.8"
|
||||
data-izoomify-duration="300" alt="">
|
||||
<img class="zoomed_avatar_img" src="" data-izoomify-url="" data-izoomify-magnify="1.8" data-izoomify-duration="300" alt="">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -4132,8 +4132,12 @@ async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, qu
|
||||
// regenerate with character speech reenforced
|
||||
// to make sure we leave on swipe type while also adding the name2 appendage
|
||||
await delay(1000);
|
||||
// A message was already deleted on regeneration, so instead treat is as a normal gen
|
||||
if (type === 'regenerate') {
|
||||
type = 'normal';
|
||||
}
|
||||
// The first await is for waiting for the generate to start. The second one is waiting for it to finish
|
||||
const result = await await Generate(type, { automatic_trigger, force_name2: true, quiet_prompt, skipWIAN, force_chid, maxLoops: maxLoops - 1 });
|
||||
const result = await await Generate(type, { automatic_trigger, force_name2: true, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName, maxLoops: maxLoops - 1 });
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -6769,8 +6773,9 @@ function select_rm_info(type, charId, previousCharId = null) {
|
||||
importFlashTimeout = setTimeout(function () {
|
||||
if (type === 'char_import' || type === 'char_create') {
|
||||
// Find the page at which the character is located
|
||||
const avatarFileName = `${charId}.png`;
|
||||
const charData = getEntitiesList({ doFilter: true });
|
||||
const charIndex = charData.findIndex((x) => x?.item?.avatar?.startsWith(charId));
|
||||
const charIndex = charData.findIndex((x) => x?.item?.avatar?.startsWith(avatarFileName));
|
||||
|
||||
if (charIndex === -1) {
|
||||
console.log(`Could not find character ${charId} in the list`);
|
||||
@ -6780,7 +6785,7 @@ function select_rm_info(type, charId, previousCharId = null) {
|
||||
try {
|
||||
const perPage = Number(localStorage.getItem('Characters_PerPage')) || per_page_default;
|
||||
const page = Math.floor(charIndex / perPage) + 1;
|
||||
const selector = `#rm_print_characters_block [title^="${charId}"]`;
|
||||
const selector = `#rm_print_characters_block [title*="${avatarFileName}"]`;
|
||||
$('#rm_print_characters_pagination').pagination('go', page);
|
||||
|
||||
waitUntilCondition(() => document.querySelector(selector) !== null).then(() => {
|
||||
|
||||
@ -1398,7 +1398,8 @@ class PromptManager {
|
||||
`;
|
||||
|
||||
const rangeBlockDiv = promptManagerDiv.querySelector('.range-block');
|
||||
rangeBlockDiv.insertAdjacentHTML('beforeend', footerHtml);
|
||||
const headerDiv = promptManagerDiv.querySelector('.completion_prompt_manager_header');
|
||||
headerDiv.insertAdjacentHTML('afterend', footerHtml);
|
||||
rangeBlockDiv.querySelector('#prompt-manager-reset-character').addEventListener('click', this.handleCharacterReset);
|
||||
|
||||
const footerDiv = rangeBlockDiv.querySelector(`.${this.configuration.prefix}prompt_manager_footer`);
|
||||
@ -1427,7 +1428,12 @@ class PromptManager {
|
||||
|
||||
rangeBlockDiv.insertAdjacentHTML('beforeend', exportPopup);
|
||||
|
||||
let exportPopper = Popper.createPopper(
|
||||
// Destroy previous popper instance if it exists
|
||||
if (this.exportPopper) {
|
||||
this.exportPopper.destroy();
|
||||
}
|
||||
|
||||
this.exportPopper = Popper.createPopper(
|
||||
document.getElementById('prompt-manager-export'),
|
||||
document.getElementById('prompt-manager-export-format-popup'),
|
||||
{ placement: 'bottom' },
|
||||
@ -1440,7 +1446,7 @@ class PromptManager {
|
||||
if (show) popup.removeAttribute('data-show');
|
||||
else popup.setAttribute('data-show', '');
|
||||
|
||||
exportPopper.update();
|
||||
this.exportPopper.update();
|
||||
};
|
||||
|
||||
footerDiv.querySelector('#prompt-manager-import').addEventListener('click', this.handleImport);
|
||||
|
||||
@ -1202,7 +1202,7 @@ export function initRossMods() {
|
||||
|
||||
if (event.ctrlKey && /^[1-9]$/.test(event.key)) {
|
||||
// This will eventually be to trigger quick replies
|
||||
event.preventDefault();
|
||||
// event.preventDefault();
|
||||
console.log('Ctrl +' + event.key + ' pressed!');
|
||||
}
|
||||
}
|
||||
|
||||
@ -32,6 +32,7 @@ import {
|
||||
getStringHash,
|
||||
humanFileSize,
|
||||
saveBase64AsFile,
|
||||
extractTextFromOffice,
|
||||
} from './utils.js';
|
||||
import { extension_settings, renderExtensionTemplateAsync, saveMetadataDebounced } from './extensions.js';
|
||||
import { POPUP_RESULT, POPUP_TYPE, callGenericPopup } from './popup.js';
|
||||
@ -46,6 +47,12 @@ import { ScraperManager } from './scrapers.js';
|
||||
* @property {string} [text] File text
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {function} ConverterFunction
|
||||
* @param {File} file File object
|
||||
* @returns {Promise<string>} Converted file text
|
||||
*/
|
||||
|
||||
const fileSizeLimit = 1024 * 1024 * 10; // 10 MB
|
||||
const ATTACHMENT_SOURCE = {
|
||||
GLOBAL: 'global',
|
||||
@ -53,20 +60,60 @@ const ATTACHMENT_SOURCE = {
|
||||
CHARACTER: 'character',
|
||||
};
|
||||
|
||||
/**
|
||||
* @type {Record<string, ConverterFunction>} File converters
|
||||
*/
|
||||
const converters = {
|
||||
'application/pdf': extractTextFromPDF,
|
||||
'text/html': extractTextFromHTML,
|
||||
'text/markdown': extractTextFromMarkdown,
|
||||
'application/epub+zip': extractTextFromEpub,
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': extractTextFromOffice,
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': extractTextFromOffice,
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.presentation': extractTextFromOffice,
|
||||
'application/vnd.oasis.opendocument.text': extractTextFromOffice,
|
||||
'application/vnd.oasis.opendocument.presentation': extractTextFromOffice,
|
||||
'application/vnd.oasis.opendocument.spreadsheet': extractTextFromOffice,
|
||||
};
|
||||
|
||||
/**
|
||||
* Finds a matching key in the converters object.
|
||||
* @param {string} type MIME type
|
||||
* @returns {string} Matching key
|
||||
*/
|
||||
function findConverterKey(type) {
|
||||
return Object.keys(converters).find((key) => {
|
||||
// Match exact type
|
||||
if (type === key) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Match wildcards
|
||||
if (key.endsWith('*')) {
|
||||
return type.startsWith(key.substring(0, key.length - 1));
|
||||
}
|
||||
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the file type has a converter function.
|
||||
* @param {string} type MIME type
|
||||
* @returns {boolean} True if the file type is convertible, false otherwise.
|
||||
*/
|
||||
function isConvertible(type) {
|
||||
return Object.keys(converters).includes(type);
|
||||
return Boolean(findConverterKey(type));
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the converter function for a file type.
|
||||
* @param {string} type MIME type
|
||||
* @returns {ConverterFunction} Converter function
|
||||
*/
|
||||
function getConverter(type) {
|
||||
const key = findConverterKey(type);
|
||||
return key && converters[key];
|
||||
}
|
||||
|
||||
/**
|
||||
@ -152,7 +199,7 @@ export async function populateFileAttachment(message, inputId = 'file_form_input
|
||||
|
||||
if (isConvertible(file.type)) {
|
||||
try {
|
||||
const converter = converters[file.type];
|
||||
const converter = getConverter(file.type);
|
||||
const fileText = await converter(file);
|
||||
base64Data = window.btoa(unescape(encodeURIComponent(fileText)));
|
||||
} catch (error) {
|
||||
@ -584,19 +631,60 @@ async function openFilePopup(attachment) {
|
||||
callGenericPopup(modalTemplate, POPUP_TYPE.TEXT, '', { wide: true, large: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Edit a file attachment in a notepad-like modal.
|
||||
* @param {FileAttachment} attachment Attachment to edit
|
||||
* @param {string} source Attachment source
|
||||
* @param {function} callback Callback function
|
||||
*/
|
||||
async function editAttachment(attachment, source, callback) {
|
||||
const originalFileText = attachment.text || (await getFileAttachment(attachment.url));
|
||||
const template = $(await renderExtensionTemplateAsync('attachments', 'notepad'));
|
||||
|
||||
let editedFileText = originalFileText;
|
||||
template.find('[name="notepadFileContent"]').val(editedFileText).on('input', function () {
|
||||
editedFileText = String($(this).val());
|
||||
});
|
||||
|
||||
let editedFileName = attachment.name;
|
||||
template.find('[name="notepadFileName"]').val(editedFileName).on('input', function () {
|
||||
editedFileName = String($(this).val());
|
||||
});
|
||||
|
||||
const result = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: true, large: true, okButton: 'Save', cancelButton: 'Cancel' });
|
||||
|
||||
if (result !== POPUP_RESULT.AFFIRMATIVE) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (editedFileText === originalFileText && editedFileName === attachment.name) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nullCallback = () => { };
|
||||
await deleteAttachment(attachment, source, nullCallback, false);
|
||||
const file = new File([editedFileText], editedFileName, { type: 'text/plain' });
|
||||
await uploadFileAttachmentToServer(file, source);
|
||||
|
||||
callback();
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes an attachment from the server and the chat.
|
||||
* @param {FileAttachment} attachment Attachment to delete
|
||||
* @param {string} source Source of the attachment
|
||||
* @param {function} callback Callback function
|
||||
* @param {boolean} [confirm=true] If true, show a confirmation dialog
|
||||
* @returns {Promise<void>} A promise that resolves when the attachment is deleted.
|
||||
*/
|
||||
async function deleteAttachment(attachment, source, callback) {
|
||||
const confirm = await callGenericPopup('Are you sure you want to delete this attachment?', POPUP_TYPE.CONFIRM);
|
||||
async function deleteAttachment(attachment, source, callback, confirm = true) {
|
||||
if (confirm) {
|
||||
const result = await callGenericPopup('Are you sure you want to delete this attachment?', POPUP_TYPE.CONFIRM);
|
||||
|
||||
if (confirm !== POPUP_RESULT.AFFIRMATIVE) {
|
||||
if (result !== POPUP_RESULT.AFFIRMATIVE) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
ensureAttachmentsExist();
|
||||
|
||||
@ -672,6 +760,7 @@ async function openAttachmentManager() {
|
||||
attachmentTemplate.find('.attachmentListItemSize').text(humanFileSize(attachment.size));
|
||||
attachmentTemplate.find('.attachmentListItemCreated').text(new Date(attachment.created).toLocaleString());
|
||||
attachmentTemplate.find('.viewAttachmentButton').on('click', () => openFilePopup(attachment));
|
||||
attachmentTemplate.find('.editAttachmentButton').on('click', () => editAttachment(attachment, source, renderAttachments));
|
||||
attachmentTemplate.find('.deleteAttachmentButton').on('click', () => deleteAttachment(attachment, source, renderAttachments));
|
||||
template.find(sources[source]).append(attachmentTemplate);
|
||||
}
|
||||
@ -842,7 +931,7 @@ async function runScraper(scraperId, target, callback) {
|
||||
* @param {string} target Target for the attachment
|
||||
* @returns
|
||||
*/
|
||||
async function uploadFileAttachmentToServer(file, target) {
|
||||
export async function uploadFileAttachmentToServer(file, target) {
|
||||
const isValid = await validateFile(file);
|
||||
|
||||
if (!isValid) {
|
||||
@ -855,7 +944,7 @@ async function uploadFileAttachmentToServer(file, target) {
|
||||
|
||||
if (isConvertible(file.type)) {
|
||||
try {
|
||||
const converter = converters[file.type];
|
||||
const converter = getConverter(file.type);
|
||||
const fileText = await converter(file);
|
||||
base64Data = window.btoa(unescape(encodeURIComponent(fileText)));
|
||||
} catch (error) {
|
||||
@ -932,6 +1021,44 @@ export function getDataBankAttachments() {
|
||||
return [...globalAttachments, ...chatAttachments, ...characterAttachments];
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all attachments for a specific source.
|
||||
* @param {string} source Attachment source
|
||||
* @returns {FileAttachment[]} List of attachments
|
||||
*/
|
||||
export function getDataBankAttachmentsForSource(source) {
|
||||
ensureAttachmentsExist();
|
||||
|
||||
switch (source) {
|
||||
case ATTACHMENT_SOURCE.GLOBAL:
|
||||
return extension_settings.attachments ?? [];
|
||||
case ATTACHMENT_SOURCE.CHAT:
|
||||
return chat_metadata.attachments ?? [];
|
||||
case ATTACHMENT_SOURCE.CHARACTER:
|
||||
return extension_settings.character_attachments?.[characters[this_chid]?.avatar] ?? [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers a file converter function.
|
||||
* @param {string} mimeType MIME type
|
||||
* @param {ConverterFunction} converter Function to convert file
|
||||
* @returns {void}
|
||||
*/
|
||||
export function registerFileConverter(mimeType, converter) {
|
||||
if (typeof mimeType !== 'string' || typeof converter !== 'function') {
|
||||
console.error('Invalid converter registration');
|
||||
return;
|
||||
}
|
||||
|
||||
if (Object.keys(converters).includes(mimeType)) {
|
||||
console.error('Converter already registered');
|
||||
return;
|
||||
}
|
||||
|
||||
converters[mimeType] = converter;
|
||||
}
|
||||
|
||||
jQuery(function () {
|
||||
$(document).on('click', '.mes_hide', async function () {
|
||||
const messageBlock = $(this).closest('.mes');
|
||||
|
||||
@ -102,6 +102,7 @@
|
||||
<small class="attachmentListItemCreated"></small>
|
||||
<small class="attachmentListItemSize"></small>
|
||||
<div class="viewAttachmentButton right_menu_button fa-solid fa-magnifying-glass" title="View attachment content"></div>
|
||||
<div class="editAttachmentButton right_menu_button fa-solid fa-pencil" title="Edit attachment"></div>
|
||||
<div class="deleteAttachmentButton right_menu_button fa-solid fa-trash" title="Delete attachment"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
10
public/scripts/extensions/attachments/notepad.html
Normal file
10
public/scripts/extensions/attachments/notepad.html
Normal file
@ -0,0 +1,10 @@
|
||||
<div class="flex-container flexFlowColumn height100p">
|
||||
<label for="notepadFileName">
|
||||
File Name
|
||||
</label>
|
||||
<input type="text" class="text_pole" id="notepadFileName" name="notepadFileName" value="" />
|
||||
<labels>
|
||||
File Content
|
||||
</label>
|
||||
<textarea id="notepadFileContent" name="notepadFileContent" class="text_pole textarea_compact monospace flex1" placeholder="Enter your notes here."></textarea>
|
||||
</div>
|
||||
20
public/scripts/extensions/attachments/youtube-scrape.html
Normal file
20
public/scripts/extensions/attachments/youtube-scrape.html
Normal file
@ -0,0 +1,20 @@
|
||||
<div>
|
||||
<strong data-i18n="Enter a video URL to download its transcript.">
|
||||
Enter a video URL or ID to download its transcript.
|
||||
</strong>
|
||||
<div data-i18n="Examples:" class="m-t-1">
|
||||
Examples:
|
||||
</div>
|
||||
<ul class="justifyLeft">
|
||||
<li>https://www.youtube.com/watch?v=jV1vkHv4zq8</li>
|
||||
<li>https://youtu.be/nlLhw1mtCFA</li>
|
||||
<li>TDpxx5UqrVU</li>
|
||||
</ul>
|
||||
<label>
|
||||
Language code (optional 2-letter ISO code):
|
||||
</label>
|
||||
<input type="text" class="text_pole" name="youtubeLanguageCode" placeholder="e.g. en">
|
||||
<label>
|
||||
Video ID:
|
||||
</label>
|
||||
</div>
|
||||
@ -37,6 +37,8 @@ const p = a => `<p>${a}</p>`;
|
||||
|
||||
const MODULE_NAME = 'sd';
|
||||
const UPDATE_INTERVAL = 1000;
|
||||
// This is a 1x1 transparent PNG
|
||||
const PNG_PIXEL = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';
|
||||
|
||||
const sources = {
|
||||
extras: 'extras',
|
||||
@ -2650,6 +2652,8 @@ async function generateComfyImage(prompt, negativePrompt) {
|
||||
const avatarBlob = await response.blob();
|
||||
const avatarBase64 = await getBase64Async(avatarBlob);
|
||||
workflow = workflow.replace('"%user_avatar%"', JSON.stringify(avatarBase64));
|
||||
} else {
|
||||
workflow = workflow.replace('"%user_avatar%"', JSON.stringify(PNG_PIXEL));
|
||||
}
|
||||
}
|
||||
if (/%char_avatar%/gi.test(workflow)) {
|
||||
@ -2658,6 +2662,8 @@ async function generateComfyImage(prompt, negativePrompt) {
|
||||
const avatarBlob = await response.blob();
|
||||
const avatarBase64 = await getBase64Async(avatarBlob);
|
||||
workflow = workflow.replace('"%char_avatar%"', JSON.stringify(avatarBase64));
|
||||
} else {
|
||||
workflow = workflow.replace('"%char_avatar%"', JSON.stringify(PNG_PIXEL));
|
||||
}
|
||||
}
|
||||
console.log(`{
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { callPopup, cancelTtsPlay, eventSource, event_types, name2, saveSettingsDebounced } from '../../../script.js';
|
||||
import { callPopup, cancelTtsPlay, eventSource, event_types, name2, saveSettingsDebounced, substituteParams } from '../../../script.js';
|
||||
import { ModuleWorkerWrapper, doExtrasFetch, extension_settings, getApiUrl, getContext, modules } from '../../extensions.js';
|
||||
import { delay, escapeRegex, getBase64Async, getStringHash, onlyUnique } from '../../utils.js';
|
||||
import { EdgeTtsProvider } from './edge.js';
|
||||
@ -425,6 +425,9 @@ async function processTtsQueue() {
|
||||
currentTtsJob = ttsJobQueue.shift();
|
||||
let text = extension_settings.tts.narrate_translated_only ? (currentTtsJob?.extra?.display_text || currentTtsJob.mes) : currentTtsJob.mes;
|
||||
|
||||
// Substitute macros
|
||||
text = substituteParams(text);
|
||||
|
||||
if (extension_settings.tts.skip_codeblocks) {
|
||||
text = text.replace(/^\s{4}.*$/gm, '').trim();
|
||||
text = text.replace(/```.*?```/gs, '').trim();
|
||||
|
||||
@ -53,6 +53,7 @@ const settings = {
|
||||
|
||||
// For files
|
||||
enabled_files: false,
|
||||
translate_files: false,
|
||||
size_threshold: 10,
|
||||
chunk_size: 5000,
|
||||
chunk_count: 2,
|
||||
@ -437,6 +438,12 @@ async function retrieveFileChunks(queryText, collectionId) {
|
||||
*/
|
||||
async function vectorizeFile(fileText, fileName, collectionId, chunkSize) {
|
||||
try {
|
||||
if (settings.translate_files && typeof window['translate'] === 'function') {
|
||||
console.log(`Vectors: Translating file ${fileName} to English...`);
|
||||
const translatedText = await window['translate'](fileText, 'en');
|
||||
fileText = translatedText;
|
||||
}
|
||||
|
||||
const toast = toastr.info('Vectorization may take some time, please wait...', `Ingesting file ${fileName}`);
|
||||
const chunks = splitRecursive(fileText, chunkSize);
|
||||
console.debug(`Vectors: Split file ${fileName} into ${chunks.length} chunks`, chunks);
|
||||
@ -1121,6 +1128,12 @@ jQuery(async () => {
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
$('#vectors_translate_files').prop('checked', settings.translate_files).on('input', () => {
|
||||
settings.translate_files = !!$('#vectors_translate_files').prop('checked');
|
||||
Object.assign(extension_settings.vectors, settings);
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
const validSecret = !!secret_state[SECRET_KEYS.NOMICAI];
|
||||
const placeholder = validSecret ? '✔️ Key saved' : '❌ Missing key';
|
||||
$('#api_key_nomicai').attr('placeholder', placeholder);
|
||||
|
||||
@ -107,6 +107,13 @@
|
||||
</label>
|
||||
|
||||
<div id="vectors_files_settings" class="marginTopBot5">
|
||||
<label class="checkbox_label" for="vectors_translate_files" title="This can help with retrieval accuracy if using embedding models that are trained on English data. Uses the selected API from Chat Translation extension settings.">
|
||||
<input id="vectors_translate_files" type="checkbox" class="checkbox">
|
||||
<span data-i18n="Translate files into English before processing">
|
||||
Translate files into English before processing
|
||||
</span>
|
||||
<i class="fa-solid fa-flask" title="Experimental feature"></i>
|
||||
</label>
|
||||
<div class="flex justifyCenter" title="These settings apply to files attached directly to messages.">
|
||||
<span>Message attachments</span>
|
||||
</div>
|
||||
|
||||
@ -188,9 +188,7 @@ export async function getGroupChat(groupId, reload = false) {
|
||||
|
||||
if (Array.isArray(data) && data.length) {
|
||||
data[0].is_group = true;
|
||||
for (let key of data) {
|
||||
chat.push(key);
|
||||
}
|
||||
chat.splice(0, chat.length, ...data);
|
||||
await printMessages();
|
||||
} else {
|
||||
sendSystemMessage(system_message_types.GROUP, '', { isSmallSys: true });
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { chat, chat_metadata, main_api, getMaxContextSize, getCurrentChatId } from '../script.js';
|
||||
import { chat, chat_metadata, main_api, getMaxContextSize, getCurrentChatId, substituteParams } from '../script.js';
|
||||
import { timestampToMoment, isDigitsOnly, getStringHash } from './utils.js';
|
||||
import { textgenerationwebui_banned_in_macros } from './textgen-settings.js';
|
||||
import { replaceInstructMacros } from './instruct-mode.js';
|
||||
@ -6,6 +6,12 @@ import { replaceVariableMacros } from './variables.js';
|
||||
|
||||
// Register any macro that you want to leave in the compiled story string
|
||||
Handlebars.registerHelper('trim', () => '{{trim}}');
|
||||
// Catch-all helper for any macro that is not defined for story strings
|
||||
Handlebars.registerHelper('helperMissing', function () {
|
||||
const options = arguments[arguments.length - 1];
|
||||
const macroName = options.name;
|
||||
return substituteParams(`{{${macroName}}}`);
|
||||
});
|
||||
|
||||
/**
|
||||
* Gets a hashed id of the current chat from the metadata.
|
||||
|
||||
@ -77,6 +77,52 @@ export class ScraperManager {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a text file from a string.
|
||||
* @implements {Scraper}
|
||||
*/
|
||||
class Notepad {
|
||||
constructor() {
|
||||
this.id = 'text';
|
||||
this.name = 'Notepad';
|
||||
this.description = 'Create a text file from scratch.';
|
||||
this.iconClass = 'fa-solid fa-note-sticky';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the scraper is available.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async isAvailable() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a text file from a string.
|
||||
* @returns {Promise<File[]>} File attachments scraped from the text
|
||||
*/
|
||||
async scrape() {
|
||||
const template = $(await renderExtensionTemplateAsync('attachments', 'notepad', {}));
|
||||
let fileName = `Untitled - ${new Date().toLocaleString()}`;
|
||||
let text = '';
|
||||
template.find('input[name="notepadFileName"]').val(fileName).on('input', function () {
|
||||
fileName = String($(this).val()).trim();
|
||||
});
|
||||
template.find('textarea[name="notepadFileContent"]').on('input', function () {
|
||||
text = String($(this).val());
|
||||
});
|
||||
|
||||
const result = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: true, large: true, okButton: 'Save', cancelButton: 'Cancel' });
|
||||
|
||||
if (!result || text === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
const file = new File([text], `Notepad - ${fileName}.txt`, { type: 'text/plain' });
|
||||
return [file];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrape data from a webpage.
|
||||
* @implements {Scraper}
|
||||
@ -93,8 +139,8 @@ class WebScraper {
|
||||
* Check if the scraper is available.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
isAvailable() {
|
||||
return Promise.resolve(true);
|
||||
async isAvailable() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -167,8 +213,8 @@ class FileScraper {
|
||||
* Check if the scraper is available.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
isAvailable() {
|
||||
return Promise.resolve(true);
|
||||
async isAvailable() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -179,7 +225,7 @@ class FileScraper {
|
||||
return new Promise(resolve => {
|
||||
const fileInput = document.createElement('input');
|
||||
fileInput.type = 'file';
|
||||
fileInput.accept = '.txt, .md, .pdf, .html, .htm, .epub';
|
||||
fileInput.accept = '*/*';
|
||||
fileInput.multiple = true;
|
||||
fileInput.onchange = () => resolve(Array.from(fileInput.files));
|
||||
fileInput.click();
|
||||
@ -199,6 +245,10 @@ class FandomScraper {
|
||||
this.iconClass = 'fa-solid fa-fire';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the scraper is available.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async isAvailable() {
|
||||
try {
|
||||
const result = await fetch('/api/plugins/fandom/probe', {
|
||||
@ -289,6 +339,78 @@ class FandomScraper {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrape transcript from a YouTube video.
|
||||
* @implements {Scraper}
|
||||
*/
|
||||
class YouTubeScraper {
|
||||
constructor() {
|
||||
this.id = 'youtube';
|
||||
this.name = 'YouTube';
|
||||
this.description = 'Download a transcript from a YouTube video.';
|
||||
this.iconClass = 'fa-solid fa-closed-captioning';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the scraper is available.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async isAvailable() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the ID of a YouTube video from a URL.
|
||||
* @param {string} url URL of the YouTube video
|
||||
* @returns {string} ID of the YouTube video
|
||||
*/
|
||||
parseId(url){
|
||||
const regex = /^.*(?:(?:youtu\.be\/|v\/|vi\/|u\/\w\/|embed\/|shorts\/)|(?:(?:watch)?\?v(?:i)?=|&v(?:i)?=))([^#&?]*).*/;
|
||||
const match = url.match(regex);
|
||||
return (match?.length && match[1] ? match[1] : url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrape transcript from a YouTube video.
|
||||
* @returns {Promise<File[]>} File attachments scraped from the YouTube video
|
||||
*/
|
||||
async scrape() {
|
||||
let lang = '';
|
||||
const template = $(await renderExtensionTemplateAsync('attachments', 'youtube-scrape', {}));
|
||||
const videoUrl = await callGenericPopup(template, POPUP_TYPE.INPUT, '', { wide: false, large: false, okButton: 'Scrape', cancelButton: 'Cancel', rows: 2 });
|
||||
|
||||
template.find('input[name="youtubeLanguageCode"]').on('input', function () {
|
||||
lang = String($(this).val()).trim();
|
||||
});
|
||||
|
||||
if (!videoUrl) {
|
||||
return;
|
||||
}
|
||||
|
||||
const id = this.parseId(String(videoUrl).trim());
|
||||
const toast = toastr.info('Working, please wait...');
|
||||
|
||||
const result = await fetch('/api/serpapi/transcript', {
|
||||
method: 'POST',
|
||||
headers: getRequestHeaders(),
|
||||
body: JSON.stringify({ id, lang }),
|
||||
});
|
||||
|
||||
if (!result.ok) {
|
||||
const error = await result.text();
|
||||
throw new Error(error);
|
||||
}
|
||||
|
||||
const transcript = await result.text();
|
||||
toastr.clear(toast);
|
||||
|
||||
const file = new File([transcript], `YouTube - ${id} - ${Date.now()}.txt`, { type: 'text/plain' });
|
||||
return [file];
|
||||
}
|
||||
}
|
||||
|
||||
ScraperManager.registerDataBankScraper(new FileScraper());
|
||||
ScraperManager.registerDataBankScraper(new Notepad());
|
||||
ScraperManager.registerDataBankScraper(new WebScraper());
|
||||
ScraperManager.registerDataBankScraper(new FandomScraper());
|
||||
ScraperManager.registerDataBankScraper(new YouTubeScraper());
|
||||
|
||||
@ -1185,16 +1185,23 @@ export function uuidv4() {
|
||||
}
|
||||
|
||||
function postProcessText(text, collapse = true) {
|
||||
// Remove carriage returns
|
||||
text = text.replace(/\r/g, '');
|
||||
// Replace tabs with spaces
|
||||
text = text.replace(/\t/g, ' ');
|
||||
// Normalize unicode spaces
|
||||
text = text.replace(/\u00A0/g, ' ');
|
||||
// Collapse multiple newlines into one
|
||||
if (collapse) {
|
||||
text = collapseNewlines(text);
|
||||
// Trim leading and trailing whitespace, and remove empty lines
|
||||
text = text.split('\n').map(l => l.trim()).filter(Boolean).join('\n');
|
||||
} else {
|
||||
// Replace more than 4 newlines with 4 newlines
|
||||
text = text.replace(/\n{4,}/g, '\n\n\n\n');
|
||||
// Trim lines that contain nothing but whitespace
|
||||
text = text.split('\n').map(l => /^\s+$/.test(l) ? '' : l).join('\n');
|
||||
}
|
||||
// Remove carriage returns
|
||||
text = text.replace(/\r/g, '');
|
||||
// Normalize unicode spaces
|
||||
text = text.replace(/\u00A0/g, ' ');
|
||||
// Collapse multiple spaces into one (except for newlines)
|
||||
text = text.replace(/ {2,}/g, ' ');
|
||||
// Remove leading and trailing spaces
|
||||
@ -1348,6 +1355,47 @@ export async function extractTextFromEpub(blob) {
|
||||
return postProcessText(text.join('\n'), false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts text from an Office document using the server plugin.
|
||||
* @param {File} blob File to extract text from
|
||||
* @returns {Promise<string>} A promise that resolves to the extracted text.
|
||||
*/
|
||||
export async function extractTextFromOffice(blob) {
|
||||
async function checkPluginAvailability() {
|
||||
try {
|
||||
const result = await fetch('/api/plugins/office/probe', {
|
||||
method: 'POST',
|
||||
headers: getRequestHeaders(),
|
||||
});
|
||||
|
||||
return result.ok;
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const isPluginAvailable = await checkPluginAvailability();
|
||||
|
||||
if (!isPluginAvailable) {
|
||||
throw new Error('Importing Office documents requires a server plugin. Please refer to the documentation for more information.');
|
||||
}
|
||||
|
||||
const base64 = await getBase64Async(blob);
|
||||
|
||||
const response = await fetch('/api/plugins/office/parse', {
|
||||
method: 'POST',
|
||||
headers: getRequestHeaders(),
|
||||
body: JSON.stringify({ data: base64 }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to parse the Office document');
|
||||
}
|
||||
|
||||
const data = await response.text();
|
||||
return postProcessText(data, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets a value in an object by a path.
|
||||
* @param {object} obj Object to set value in
|
||||
|
||||
@ -899,7 +899,7 @@ router.post('/generate', jsonParser, function (request, response) {
|
||||
}
|
||||
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.PERPLEXITY) {
|
||||
apiUrl = API_PERPLEXITY;
|
||||
apiKey = readSecret(SECRET_KEYS.PERPLEXITY);
|
||||
apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);
|
||||
headers = {};
|
||||
bodyParams = {};
|
||||
request.body.messages = postProcessPrompt(request.body.messages, 'claude', request.body.char_name, request.body.user_name);
|
||||
|
||||
@ -48,6 +48,92 @@ router.post('/search', jsonParser, async (request, response) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Get the transcript of a YouTube video
|
||||
* @copyright https://github.com/Kakulukian/youtube-transcript (MIT License)
|
||||
*/
|
||||
router.post('/transcript', jsonParser, async (request, response) => {
|
||||
try {
|
||||
const he = require('he');
|
||||
const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
|
||||
const id = request.body.id;
|
||||
const lang = request.body.lang;
|
||||
|
||||
if (!id) {
|
||||
console.log('Id is required for /transcript');
|
||||
return response.sendStatus(400);
|
||||
}
|
||||
|
||||
const videoPageResponse = await fetch(`https://www.youtube.com/watch?v=${id}`, {
|
||||
headers: {
|
||||
...(lang && { 'Accept-Language': lang }),
|
||||
'User-Agent': visitHeaders['User-Agent'],
|
||||
},
|
||||
});
|
||||
|
||||
const videoPageBody = await videoPageResponse.text();
|
||||
const splittedHTML = videoPageBody.split('"captions":');
|
||||
|
||||
if (splittedHTML.length <= 1) {
|
||||
if (videoPageBody.includes('class="g-recaptcha"')) {
|
||||
throw new Error('Too many requests');
|
||||
}
|
||||
if (!videoPageBody.includes('"playabilityStatus":')) {
|
||||
throw new Error('Video is not available');
|
||||
}
|
||||
throw new Error('Transcript not available');
|
||||
}
|
||||
|
||||
const captions = (() => {
|
||||
try {
|
||||
return JSON.parse(splittedHTML[1].split(',"videoDetails')[0].replace('\n', ''));
|
||||
} catch (e) {
|
||||
return undefined;
|
||||
}
|
||||
})()?.['playerCaptionsTracklistRenderer'];
|
||||
|
||||
if (!captions) {
|
||||
throw new Error('Transcript disabled');
|
||||
}
|
||||
|
||||
if (!('captionTracks' in captions)) {
|
||||
throw new Error('Transcript not available');
|
||||
}
|
||||
|
||||
if (lang && !captions.captionTracks.some(track => track.languageCode === lang)) {
|
||||
throw new Error('Transcript not available in this language');
|
||||
}
|
||||
|
||||
const transcriptURL = (lang ? captions.captionTracks.find(track => track.languageCode === lang) : captions.captionTracks[0]).baseUrl;
|
||||
const transcriptResponse = await fetch(transcriptURL, {
|
||||
headers: {
|
||||
...(lang && { 'Accept-Language': lang }),
|
||||
'User-Agent': visitHeaders['User-Agent'],
|
||||
},
|
||||
});
|
||||
|
||||
if (!transcriptResponse.ok) {
|
||||
throw new Error('Transcript request failed');
|
||||
}
|
||||
|
||||
const transcriptBody = await transcriptResponse.text();
|
||||
const results = [...transcriptBody.matchAll(RE_XML_TRANSCRIPT)];
|
||||
const transcript = results.map((result) => ({
|
||||
text: result[3],
|
||||
duration: parseFloat(result[2]),
|
||||
offset: parseFloat(result[1]),
|
||||
lang: lang ?? captions.captionTracks[0].languageCode,
|
||||
}));
|
||||
// The text is double-encoded
|
||||
const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
|
||||
|
||||
return response.send(transcriptText);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
return response.sendStatus(500);
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/visit', jsonParser, async (request, response) => {
|
||||
try {
|
||||
const url = request.body.url;
|
||||
|
||||
Reference in New Issue
Block a user