Compare commits
11 Commits
b3bbec83b6
...
c2256c2ac7
| Author | SHA1 | Date |
|---|---|---|
Cohee
|
c2256c2ac7 | |
|
Cohee
|
78ce23750e | |
|
Cohee
|
e6ddbd1418 | |
|
Cohee
|
344146d837 | |
|
Cohee
|
dcb98b2969 | |
|
Cohee
|
15f0e491bf | |
|
Cohee
|
70c4e82b89 | |
|
Cohee
|
1e0b76f4fc | |
|
Cohee
|
3a5886f53a | |
|
Cohee
|
db78346bef | |
RossAscends
|
cee1774554 |
|
|
@ -27,6 +27,7 @@
|
|||
"form-data": "^4.0.0",
|
||||
"google-translate-api-browser": "^3.0.1",
|
||||
"gpt3-tokenizer": "^1.1.5",
|
||||
"he": "^1.2.0",
|
||||
"helmet": "^7.1.0",
|
||||
"ip-matching": "^2.1.2",
|
||||
"ipaddr.js": "^2.0.1",
|
||||
|
|
@ -2800,6 +2801,14 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/he": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
|
||||
"integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
|
||||
"bin": {
|
||||
"he": "bin/he"
|
||||
}
|
||||
},
|
||||
"node_modules/helmet": {
|
||||
"version": "7.1.0",
|
||||
"resolved": "https://registry.npmjs.org/helmet/-/helmet-7.1.0.tgz",
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
"form-data": "^4.0.0",
|
||||
"google-translate-api-browser": "^3.0.1",
|
||||
"gpt3-tokenizer": "^1.1.5",
|
||||
"he": "^1.2.0",
|
||||
"helmet": "^7.1.0",
|
||||
"ip-matching": "^2.1.2",
|
||||
"ipaddr.js": "^2.0.1",
|
||||
|
|
|
|||
|
|
@ -86,6 +86,10 @@
|
|||
margin: 5px;
|
||||
}
|
||||
|
||||
.marginLeft5 {
|
||||
margin-left: 5px;
|
||||
}
|
||||
|
||||
.overflowYAuto {
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
|
@ -257,6 +261,10 @@
|
|||
flex-basis: 48%
|
||||
}
|
||||
|
||||
.flexBasis30p {
|
||||
flex-basis: 30%;
|
||||
}
|
||||
|
||||
.flex-container {
|
||||
display: flex;
|
||||
gap: 5px;
|
||||
|
|
@ -555,4 +563,4 @@ textarea:disabled {
|
|||
height: 30px;
|
||||
text-align: center;
|
||||
padding: 5px;
|
||||
}
|
||||
}
|
||||
|
|
@ -141,10 +141,10 @@
|
|||
</a>
|
||||
</h4>
|
||||
<div class="flex-container flexNoGap">
|
||||
<select id="settings_preset" data-preset-manager-for="kobold" class="flex1 flexBasis100p text_pole">
|
||||
<select id="settings_preset" data-preset-manager-for="kobold" class="flex1 text_pole">
|
||||
<option value="gui" data-i18n="guikoboldaisettings">GUI KoboldAI Settings</option>
|
||||
</select>
|
||||
<div class="flex-container flexBasis100p justifyCenter">
|
||||
<div class="flex-container marginLeft5 ">
|
||||
<input type="file" hidden data-preset-manager-file="kobold" accept=".json, .settings">
|
||||
<i data-newbie-hidden data-preset-manager-update="kobold" class="menu_button fa-solid fa-save" title="Update current preset" data-i18n="[title]Update current preset"></i>
|
||||
<i data-newbie-hidden data-preset-manager-new="kobold" class="menu_button fa-solid fa-file-circle-plus" title="Save preset as" data-i18n="[title]Save preset as"></i>
|
||||
|
|
@ -163,10 +163,10 @@
|
|||
</a>
|
||||
</h4>
|
||||
<div class="flex-container flexNoGap">
|
||||
<select id="settings_preset_novel" class="flex1 flexBasis100p text_pole" data-preset-manager-for="novel">
|
||||
<select id="settings_preset_novel" class="flex1 text_pole" data-preset-manager-for="novel">
|
||||
<option value="gui" data-i18n="default">Default</option>
|
||||
</select>
|
||||
<div class="flex-container flexBasis100p justifyCenter">
|
||||
<div class="flex-container marginLeft5 ">
|
||||
<input type="file" hidden data-preset-manager-file="novel" accept=".json, .settings">
|
||||
<i data-newbie-hidden data-preset-manager-update="novel" class="menu_button fa-solid fa-save" title="Update current preset" data-i18n="[title]Update current preset"></i>
|
||||
<i data-newbie-hidden data-preset-manager-new="novel" class="menu_button fa-solid fa-file-circle-plus" title="Save preset as" data-i18n="[title]Save preset as"></i>
|
||||
|
|
@ -184,7 +184,7 @@
|
|||
<select id="settings_preset_openai" class="flex1 text_pole" data-preset-manager-for="openai">
|
||||
<option value="gui" data-i18n="default">Default</option>
|
||||
</select>
|
||||
<div class="flex-container flexBasis100p justifyCenter">
|
||||
<div class="flex-container marginLeft5 ">
|
||||
<input id="openai_preset_import_file" type="file" accept=".json,.settings" hidden />
|
||||
<i id="update_oai_preset" class="menu_button fa-solid fa-save" title="Update current preset" data-i18n="[title]Update current preset"></i>
|
||||
<i id="new_oai_preset" class="menu_button fa-solid fa-file-circle-plus" title="Save preset as" data-i18n="[title]Save preset as"></i>
|
||||
|
|
@ -200,7 +200,7 @@
|
|||
<div class="flex-container flexNoGap">
|
||||
<select id="settings_preset_textgenerationwebui" class="flex1 text_pole" data-preset-manager-for="textgenerationwebui">
|
||||
</select>
|
||||
<div class="flex-container flexBasis100p justifyCenter">
|
||||
<div class="flex-container marginLeft5 ">
|
||||
<input type="file" hidden data-preset-manager-file="textgenerationwebui" accept=".json, .settings">
|
||||
<i data-newbie-hidden data-preset-manager-update="textgenerationwebui" class="menu_button fa-solid fa-save" title="Update current preset" data-i18n="[title]Update current preset"></i>
|
||||
<i data-newbie-hidden data-preset-manager-new="textgenerationwebui" class="menu_button fa-solid fa-file-circle-plus" title="Save preset as" data-i18n="[title]Save preset as"></i>
|
||||
|
|
@ -321,7 +321,7 @@
|
|||
</div>
|
||||
<div data-newbie-hidden class="range-block">
|
||||
<div class="range-block-title" data-i18n="Rep. Pen. Range.">
|
||||
Repetition Penalty Range
|
||||
Rep Pen Range
|
||||
</div>
|
||||
<div class="range-block-range-and-counter">
|
||||
<div class="range-block-range">
|
||||
|
|
@ -373,7 +373,7 @@
|
|||
</div>
|
||||
<div data-newbie-hidden class="range-block">
|
||||
<div class="range-block-title" data-i18n="Tail Free Sampling">
|
||||
Tail Free Sampling
|
||||
TFS
|
||||
</div>
|
||||
<div class="range-block-range-and-counter">
|
||||
<div class="range-block-range">
|
||||
|
|
@ -788,7 +788,7 @@
|
|||
<div id="advanced-ai-config-block" class="width100p">
|
||||
<div id="kobold_api-settings">
|
||||
<div class="flex-container gap10h5v justifyCenter">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="temperature">Temperature</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Temperature controls the randomness in token selection" title="Temperature controls the randomness in token selection: - low temperature (<1.0) leads to more predictable text, favoring higher probability tokens. - high temperature (>1.0) increases creativity and diversity in the output by giving lower probability tokens a better chance. Set to 1.0 for the original probabilities."></div>
|
||||
|
|
@ -796,7 +796,7 @@
|
|||
<input class="neo-range-slider" type="range" id="temp" name="volume" min="0.0" max="4.0" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0.0" max="4.0" step="0.01" data-for="temp" id="temp_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Top K">Top K</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Top K sets a maximum amount of top tokens that can be chosen from. E.g Top K is 20, this means only the 20 highest ranking tokens will be kept (regardless of their probabilities being diverse or limited). Set to 0 to disable."></div>
|
||||
|
|
@ -804,7 +804,7 @@
|
|||
<input class="neo-range-slider" type="range" id="top_k" name="volume" min="0" max="100" step="1">
|
||||
<input class="neo-range-input" type="number" min="0" max="100" step="1" data-for="top_k" id="top_k_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
Top P
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Top P (a.k.a. nucleus sampling) adds up all the top tokens required to add up to the target percentage. E.g If the Top 2 tokens are both 25%, and Top P is 0.50, only the Top 2 tokens are considered. Set to 1.0 to disable."></div>
|
||||
|
|
@ -812,7 +812,7 @@
|
|||
<input class="neo-range-slider" type="range" id="top_p" name="volume" min="0" max="1" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="top_p" id="top_p_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Typical P">Typical P</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Typical P Sampling prioritizes tokens based on their deviation from the average entropy of the set. It maintains tokens whose cumulative probability is close to a predefined threshold (e.g., 0.5), emphasizing those with average information content. Set to 1.0 to disable."></div>
|
||||
|
|
@ -820,7 +820,7 @@
|
|||
<input class="neo-range-slider" type="range" id="typical_p" name="volume" min="0" max="1" step="0.001">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="typical_p" id="typical_p_counter">
|
||||
</div>
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Min P">Min P</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Min P sets a base minimum probability. This is scaled according to the top token's probability. E.g If Top token is 80% probability, and Min P is 0.1, only tokens higher than 8% would be considered. Set to 0 to disable."></div>
|
||||
|
|
@ -828,7 +828,7 @@
|
|||
<input class="neo-range-slider" type="range" id="min_p" name="volume" min="0" max="1" step="0.001">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="min_p" id="min_p_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Top A">Top A</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Top A sets a threshold for token selection based on the square of the highest token probability. E.g if the Top-A value is 0.2 and the top token's probability is 50%, tokens with probabilities below 5% (0.2 * 0.5^2) are excluded. Set to 0 to disable."></div>
|
||||
|
|
@ -836,29 +836,29 @@
|
|||
<input class="neo-range-slider" type="range" id="top_a" name="volume" min="0" max="1" step="0.001">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="top_a" id="top_a_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Tail Free Sampling">Tail Free Sampling</span>
|
||||
<span data-i18n="Tail Free Sampling">TFS</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" title="Tail-Free Sampling (TFS) searches for a tail of low-probability tokens in the distribution, by analyzing the rate of change in token probabilities using derivatives. It retains tokens up to a threshold (e.g., 0.3) based on the normalized second derivative. The closer to 0, the more discarded tokens. Set to 1.0 to disable."></div>
|
||||
</small>
|
||||
<input class="neo-range-slider" type="range" id="tfs" name="volume" min="0" max="1" step="0.001">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="tfs" id="tfs_counter">
|
||||
</div>
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="rep.pen">Repetition Penalty</span>
|
||||
</small>
|
||||
<input class="neo-range-slider" type="range" id="rep_pen" name="volume" min="1" max="3" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="1" max="3" step="0.01" data-for="rep_pen" id="rep_pen_counter">
|
||||
</div>
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="rep.pen range">Repetition Penalty Range</span>
|
||||
<span data-i18n="rep.pen range">Rep Pen Range</span>
|
||||
</small>
|
||||
<input class="neo-range-slider" type="range" id="rep_pen_range" name="volume" min="0" max="4096" step="1">
|
||||
<input class="neo-range-input" type="number" min="0" max="4096" step="1" data-for="rep_pen_range" id="rep_pen_range_counter">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Rep. Pen. Slope">Repetition Penalty Slope</span>
|
||||
</small>
|
||||
|
|
@ -892,7 +892,7 @@
|
|||
</div>
|
||||
<hr class="wide100p">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter justifyCenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter justifyCenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<label class="checkbox_label alignItemsBaseline" for="use_default_badwordsids">
|
||||
<input id="use_default_badwordsids" type="checkbox" />
|
||||
<span>
|
||||
|
|
@ -901,7 +901,7 @@
|
|||
</span>
|
||||
</label>
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter textAlignCenter flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter textAlignCenter flexBasis30p flexGrow flexShrink gap0">
|
||||
<!-- <hr class="wide100p"> -->
|
||||
<small data-i18n="Seed">Seed</small>
|
||||
<!-- Max value is 2**64 - 1 -->
|
||||
|
|
@ -1185,7 +1185,7 @@
|
|||
<input type="number" id="n_textgenerationwebui" class="text_pole textAlignCenter" min="1" value="1" step="1" />
|
||||
</div>
|
||||
<div class="flex-container gap10h5v justifyCenter">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="temperature">Temperature</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Temperature controls the randomness in token selection" title="Temperature controls the randomness in token selection: - low temperature (<1.0) leads to more predictable text, favoring higher probability tokens. - high temperature (>1.0) increases creativity and diversity in the output by giving lower probability tokens a better chance. Set to 1.0 for the original probabilities."></div>
|
||||
|
|
@ -1193,7 +1193,7 @@
|
|||
<input class="neo-range-slider" type="range" id="temp_textgenerationwebui" name="volume" min="0.0" max="5.0" step="0.01" x-setting-id="temp">
|
||||
<input class="neo-range-input" type="number" min="0.0" max="5.0" step="0.01" data-for="temp_textgenerationwebui" id="temp_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Top K">Top K</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Top K sets a maximum amount of top tokens that can be chosen from" title="Top K sets a maximum amount of top tokens that can be chosen from. E.g Top K is 20, this means only the 20 highest ranking tokens will be kept (regardless of their probabilities being diverse or limited). Set to 0 (or -1, depending on your backend) to disable."></div>
|
||||
|
|
@ -1201,7 +1201,7 @@
|
|||
<input class="neo-range-slider" type="range" id="top_k_textgenerationwebui" name="volume" min="-1" max="200" step="1">
|
||||
<input class="neo-range-input" type="number" min="-1" max="200" step="1" data-for="top_k_textgenerationwebui" id="top_k_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Top P">Top P</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Top P (a.k.a. nucleus sampling)" title="Top P (a.k.a. nucleus sampling) adds up all the top tokens required to add up to the target percentage. E.g If the Top 2 tokens are both 25%, and Top P is 0.50, only the Top 2 tokens are considered. Set to 1.0 to disable."></div>
|
||||
|
|
@ -1209,7 +1209,7 @@
|
|||
<input class="neo-range-slider" type="range" id="top_p_textgenerationwebui" name="volume" min="0" max="1" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="top_p_textgenerationwebui" id="top_p_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Typical P">Typical P</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Typical P Sampling prioritizes tokens based on their deviation from the average entropy of the set" title="Typical P Sampling prioritizes tokens based on their deviation from the average entropy of the set. It maintains tokens whose cumulative probability is close to a predefined threshold (e.g., 0.5), emphasizing those with average information content. Set to 1.0 to disable."></div>
|
||||
|
|
@ -1217,7 +1217,7 @@
|
|||
<input class="neo-range-slider" type="range" id="typical_p_textgenerationwebui" name="volume" min="0" max="1" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="typical_p_textgenerationwebui" id="typical_p_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Min P">Min P</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Min P sets a base minimum probability" title="Min P sets a base minimum probability. This is scaled according to the top token's probability. E.g If Top token is 80% probability, and Min P is 0.1, only tokens higher than 8% would be considered. Set to 0 to disable."></div>
|
||||
|
|
@ -1225,7 +1225,7 @@
|
|||
<input class="neo-range-slider" type="range" id="min_p_textgenerationwebui" name="volume" min="0" max="1" step="0.001">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.001" data-for="min_p_textgenerationwebui" id="min_p_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Top A">Top A</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Top A sets a threshold for token selection based on the square of the highest token probability" title="Top A sets a threshold for token selection based on the square of the highest token probability. E.g if the Top-A value is 0.2 and the top token's probability is 50%, tokens with probabilities below 5% (0.2 * 0.5^2) are excluded. Set to 0 to disable."></div>
|
||||
|
|
@ -1233,15 +1233,15 @@
|
|||
<input class="neo-range-slider" type="range" id="top_a_textgenerationwebui" name="volume" min="0" max="1" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="top_a_textgenerationwebui" id="top_a_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Tail Free Sampling">Tail Free Sampling</span>
|
||||
<span data-i18n="Tail Free Sampling">TFS</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Tail-Free Sampling (TFS)" title="Tail-Free Sampling (TFS) searches for a tail of low-probability tokens in the distribution, by analyzing the rate of change in token probabilities using derivatives. It retains tokens up to a threshold (e.g., 0.3) based on the normalized second derivative. The closer to 0, the more discarded tokens. Set to 1.0 to disable."></div>
|
||||
</small>
|
||||
<input class="neo-range-slider" type="range" id="tfs_textgenerationwebui" name="volume" min="0" max="1" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="1" step="0.01" data-for="tfs_textgenerationwebui" id="tfs_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden data-tg-type="ooba,mancer" class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden data-tg-type="ooba,mancer" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Epsilon Cutoff">Epsilon Cutoff</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Epsilon cutoff sets a probability floor below which tokens are excluded from being sampled" title="Epsilon cutoff sets a probability floor below which tokens are excluded from being sampled. In units of 1e-4; a reasonable value is 3. Set to 0 to disable."></div>
|
||||
|
|
@ -1249,7 +1249,7 @@
|
|||
<input class="neo-range-slider" type="range" id="epsilon_cutoff_textgenerationwebui" name="volume" min="0" max="9" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="9" step="0.01" data-for="epsilon_cutoff_textgenerationwebui" id="epsilon_cutoff_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden data-tg-type="ooba,mancer" class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden data-tg-type="ooba,mancer" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small>
|
||||
<span data-i18n="Eta Cutoff">Eta Cutoff</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]Eta cutoff is the main parameter of the special Eta Sampling technique. In units of 1e-4; a reasonable value is 3. Set to 0 to disable. See the paper Truncation Sampling as Language Model Desmoothing by Hewitt et al. (2022) for details." title="Eta cutoff is the main parameter of the special Eta Sampling technique. In units of 1e-4; a reasonable value is 3. Set to 0 to disable. See the paper Truncation Sampling as Language Model Desmoothing by Hewitt et al. (2022) for details."></div>
|
||||
|
|
@ -1257,42 +1257,42 @@
|
|||
<input class="neo-range-slider" type="range" id="eta_cutoff_textgenerationwebui" name="volume" min="0" max="20" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="0" max="20" step="0.01" data-for="eta_cutoff_textgenerationwebui" id="eta_cutoff_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="rep.pen">Repetition Penalty</small>
|
||||
<input class="neo-range-slider" type="range" id="rep_pen_textgenerationwebui" name="volume" min="1" max="3" step="0.01">
|
||||
<input class="neo-range-input" type="number" min="1" max="3" step="0.01" data-for="rep_pen_textgenerationwebui" id="rep_pen_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-forAphro="False" class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<small data-i18n="rep.pen range">Repetition Penalty Range</small>
|
||||
<div data-forAphro="False" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="rep.pen range">Rep Pen Range</small>
|
||||
<input class="neo-range-slider" type="range" id="rep_pen_range_textgenerationwebui" name="volume" min="-1" max="8192" step="1">
|
||||
<input class="neo-range-input" type="number" min="-1" max="8192" step="1" data-for="rep_pen_range_textgenerationwebui" id="rep_pen_range_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-forAphro="False" data-tg-type="ooba" data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-forAphro="False" data-tg-type="ooba" data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="Encoder Rep. Pen.">Encoder Penalty</small>
|
||||
<input class="neo-range-slider" type="range" id="encoder_rep_pen_textgenerationwebui" name="volume" min="0.8" max="1.5" step="0.01" />
|
||||
<input class="neo-range-input" type="number" min="0.8" max="1.5" step="0.01" data-for="encoder_rep_pen_textgenerationwebui" id="encoder_rep_pen_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="Frequency Penalty">Frequency Penalty</small>
|
||||
<input class="neo-range-slider" type="range" id="freq_pen_textgenerationwebui" name="volume" min="-2" max="2" step="0.01" />
|
||||
<input class="neo-range-input" type="number" data-for="freq_pen_textgenerationwebui" min="-2" max="2" step="0.01" id="freq_pen_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="Presence Penalty">Presence Penalty</small>
|
||||
<input class="neo-range-slider" type="range" id="presence_pen_textgenerationwebui" name="volume" min="-2" max="2" step="0.01" />
|
||||
<input class="neo-range-input" type="number" min="-2" max="2" step="0.01" data-for="presence_pen_textgenerationwebui" id="presence_pen_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-forAphro="False" data-tg-type="ooba" data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-forAphro="False" data-tg-type="ooba" data-newbie-hidden class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="No Repeat Ngram Size">No Repeat Ngram Size</small>
|
||||
<input class="neo-range-slider" type="range" id="no_repeat_ngram_size_textgenerationwebui" name="volume" min="0" max="20" step="1">
|
||||
<input class="neo-range-input" type="number" min="0" max="20" step="1" data-for="no_repeat_ngram_size_textgenerationwebui" id="no_repeat_ngram_size_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden data-tg-type="mancer, ooba, dreamgen" class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden data-tg-type="mancer, ooba, dreamgen" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="Min Length">Min Length</small>
|
||||
<input class="neo-range-slider" type="range" id="min_length_textgenerationwebui" name="volume" min="0" max="2000" step="1" />
|
||||
<input class="neo-range-input" type="number" min="0" max="2000" step="1" data-for="min_length_textgenerationwebui" id="min_length_counter_textgenerationwebui">
|
||||
</div>
|
||||
<div data-newbie-hidden data-tg-type="ooba" class="alignitemscenter flex-container flexFlowColumn flexBasis48p flexGrow flexShrink gap0">
|
||||
<div data-newbie-hidden data-tg-type="ooba" class="alignitemscenter flex-container flexFlowColumn flexBasis30p flexGrow flexShrink gap0">
|
||||
<small data-i18n="Max Tokens Second">Maximum tokens/second</small>
|
||||
<input class="neo-range-slider" type="range" id="max_tokens_second_textgenerationwebui" name="volume" min="0" max="20" step="1" />
|
||||
<input class="neo-range-input" type="number" min="0" max="20" step="1" data-for="max_tokens_second_textgenerationwebui" id="max_tokens_second_counter_textgenerationwebui">
|
||||
|
|
@ -3544,7 +3544,7 @@
|
|||
<div id="ui_preset_export_button" class="menu_button menu_button_icon margin0" title="Export a theme file" data-i18n="[title]Export a theme file">
|
||||
<i class="fa-solid fa-file-export"></i>
|
||||
</div>
|
||||
<div id="ui-preset-delete-button" class="menu_button menu_button_icon margin0" title="Delete a theme" data-i18n="[title]Delete a theme" >
|
||||
<div id="ui-preset-delete-button" class="menu_button menu_button_icon margin0" title="Delete a theme" data-i18n="[title]Delete a theme">
|
||||
<i class="fa-solid fa-trash-can"></i>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -4461,9 +4461,7 @@
|
|||
<div class="flex1 flexGap5" title="Inserted before each part of the joined fields.">
|
||||
<label for="rm_group_generation_mode_join_prefix" class="flexnowrap width100p whitespacenowrap">
|
||||
<span data-i18n="Join Prefix">Join Prefix</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p"
|
||||
data-i18n="[title]When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)"
|
||||
title="When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)">
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)" title="When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)">
|
||||
</div>
|
||||
</label>
|
||||
<textarea id="rm_group_generation_mode_join_prefix" class="text_pole wide100p textarea_compact autoSetHeight" maxlength="2000" placeholder="—" rows="1"></textarea>
|
||||
|
|
@ -4471,9 +4469,7 @@
|
|||
<div class="flex1 flexGap5" title="Inserted after each part of the joined fields.">
|
||||
<label for="rm_group_generation_mode_join_suffix" class="flexnowrap width100p whitespacenowrap">
|
||||
<span data-i18n="Join Suffix">Join Suffix</span>
|
||||
<div class="fa-solid fa-circle-info opacity50p"
|
||||
data-i18n="[title]When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)"
|
||||
title="When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)">
|
||||
<div class="fa-solid fa-circle-info opacity50p" data-i18n="[title]When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)" title="When 'Join character cards' is selected, all respective fields of the characters are being joined together. This means that in the story string for example all character descriptions will be joined to one big text. If you want those fields to be separated, you can define a prefix or suffix here. This value supports normal macros and will also replace {{char}} with the relevant char's name and <FIELDNAME> with the name of the part (e.g.: description, personality, scenario, etc.)">
|
||||
</div>
|
||||
</label>
|
||||
<textarea id="rm_group_generation_mode_join_suffix" class="text_pole wide100p textarea_compact autoSetHeight" maxlength="2000" placeholder="—" rows="1"></textarea>
|
||||
|
|
@ -4981,7 +4977,7 @@
|
|||
<option value="3" data-role="" data-i18n="After AN">
|
||||
↓AN
|
||||
</option>
|
||||
<option value="4" data-role="0" data-i18n="at Depth System" >
|
||||
<option value="4" data-role="0" data-i18n="at Depth System">
|
||||
@D ⚙️
|
||||
</option>
|
||||
<option value="4" data-role="1" data-i18n="at Depth User">
|
||||
|
|
@ -6005,10 +6001,7 @@
|
|||
<div class="fa-fw fa-solid fa-grip drag-grabber"></div>
|
||||
<div class="fa-fw fa-solid fa-circle-xmark dragClose" id="closeZoom"></div>
|
||||
</div>
|
||||
<img class="zoomed_avatar_img" src=""
|
||||
data-izoomify-url=""
|
||||
data-izoomify-magnify="1.8"
|
||||
data-izoomify-duration="300" alt="">
|
||||
<img class="zoomed_avatar_img" src="" data-izoomify-url="" data-izoomify-magnify="1.8" data-izoomify-duration="300" alt="">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -6072,4 +6065,4 @@
|
|||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
</html>
|
||||
|
|
@ -4132,8 +4132,12 @@ async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, qu
|
|||
// regenerate with character speech reenforced
|
||||
// to make sure we leave on swipe type while also adding the name2 appendage
|
||||
await delay(1000);
|
||||
// A message was already deleted on regeneration, so instead treat is as a normal gen
|
||||
if (type === 'regenerate') {
|
||||
type = 'normal';
|
||||
}
|
||||
// The first await is for waiting for the generate to start. The second one is waiting for it to finish
|
||||
const result = await await Generate(type, { automatic_trigger, force_name2: true, quiet_prompt, skipWIAN, force_chid, maxLoops: maxLoops - 1 });
|
||||
const result = await await Generate(type, { automatic_trigger, force_name2: true, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage, quietName, maxLoops: maxLoops - 1 });
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1202,7 +1202,7 @@ export function initRossMods() {
|
|||
|
||||
if (event.ctrlKey && /^[1-9]$/.test(event.key)) {
|
||||
// This will eventually be to trigger quick replies
|
||||
event.preventDefault();
|
||||
// event.preventDefault();
|
||||
console.log('Ctrl +' + event.key + ' pressed!');
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -842,7 +842,7 @@ async function runScraper(scraperId, target, callback) {
|
|||
* @param {string} target Target for the attachment
|
||||
* @returns
|
||||
*/
|
||||
async function uploadFileAttachmentToServer(file, target) {
|
||||
export async function uploadFileAttachmentToServer(file, target) {
|
||||
const isValid = await validateFile(file);
|
||||
|
||||
if (!isValid) {
|
||||
|
|
@ -932,6 +932,24 @@ export function getDataBankAttachments() {
|
|||
return [...globalAttachments, ...chatAttachments, ...characterAttachments];
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all attachments for a specific source.
|
||||
* @param {string} source Attachment source
|
||||
* @returns {FileAttachment[]} List of attachments
|
||||
*/
|
||||
export function getDataBankAttachmentsForSource(source) {
|
||||
ensureAttachmentsExist();
|
||||
|
||||
switch (source) {
|
||||
case ATTACHMENT_SOURCE.GLOBAL:
|
||||
return extension_settings.attachments ?? [];
|
||||
case ATTACHMENT_SOURCE.CHAT:
|
||||
return chat_metadata.attachments ?? [];
|
||||
case ATTACHMENT_SOURCE.CHARACTER:
|
||||
return extension_settings.character_attachments?.[characters[this_chid]?.avatar] ?? [];
|
||||
}
|
||||
}
|
||||
|
||||
jQuery(function () {
|
||||
$(document).on('click', '.mes_hide', async function () {
|
||||
const messageBlock = $(this).closest('.mes');
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
<div>
|
||||
<strong data-i18n="Enter a video URL to download its transcript.">
|
||||
Enter a video URL or ID to download its transcript.
|
||||
</strong>
|
||||
<div data-i18n="Examples:" class="m-t-1">
|
||||
Examples:
|
||||
</div>
|
||||
<ul class="justifyLeft">
|
||||
<li>https://www.youtube.com/watch?v=jV1vkHv4zq8</li>
|
||||
<li>https://youtu.be/nlLhw1mtCFA</li>
|
||||
<li>TDpxx5UqrVU</li>
|
||||
</ul>
|
||||
<label>
|
||||
Language code (optional 2-letter ISO code):
|
||||
</label>
|
||||
<input type="text" class="text_pole" name="youtubeLanguageCode" placeholder="e.g. en">
|
||||
<label>
|
||||
Video ID:
|
||||
</label>
|
||||
</div>
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import { callPopup, cancelTtsPlay, eventSource, event_types, name2, saveSettingsDebounced } from '../../../script.js';
|
||||
import { callPopup, cancelTtsPlay, eventSource, event_types, name2, saveSettingsDebounced, substituteParams } from '../../../script.js';
|
||||
import { ModuleWorkerWrapper, doExtrasFetch, extension_settings, getApiUrl, getContext, modules } from '../../extensions.js';
|
||||
import { delay, escapeRegex, getBase64Async, getStringHash, onlyUnique } from '../../utils.js';
|
||||
import { EdgeTtsProvider } from './edge.js';
|
||||
|
|
@ -425,6 +425,9 @@ async function processTtsQueue() {
|
|||
currentTtsJob = ttsJobQueue.shift();
|
||||
let text = extension_settings.tts.narrate_translated_only ? (currentTtsJob?.extra?.display_text || currentTtsJob.mes) : currentTtsJob.mes;
|
||||
|
||||
// Substitute macros
|
||||
text = substituteParams(text);
|
||||
|
||||
if (extension_settings.tts.skip_codeblocks) {
|
||||
text = text.replace(/^\s{4}.*$/gm, '').trim();
|
||||
text = text.replace(/```.*?```/gs, '').trim();
|
||||
|
|
|
|||
|
|
@ -93,8 +93,8 @@ class WebScraper {
|
|||
* Check if the scraper is available.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
isAvailable() {
|
||||
return Promise.resolve(true);
|
||||
async isAvailable() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -167,8 +167,8 @@ class FileScraper {
|
|||
* Check if the scraper is available.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
isAvailable() {
|
||||
return Promise.resolve(true);
|
||||
async isAvailable() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -199,6 +199,10 @@ class FandomScraper {
|
|||
this.iconClass = 'fa-solid fa-fire';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the scraper is available.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async isAvailable() {
|
||||
try {
|
||||
const result = await fetch('/api/plugins/fandom/probe', {
|
||||
|
|
@ -289,6 +293,77 @@ class FandomScraper {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrape transcript from a YouTube video.
|
||||
* @implements {Scraper}
|
||||
*/
|
||||
class YouTubeScraper {
|
||||
constructor() {
|
||||
this.id = 'youtube';
|
||||
this.name = 'YouTube';
|
||||
this.description = 'Download a transcript from a YouTube video.';
|
||||
this.iconClass = 'fa-solid fa-closed-captioning';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the scraper is available.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async isAvailable() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the ID of a YouTube video from a URL.
|
||||
* @param {string} url URL of the YouTube video
|
||||
* @returns {string} ID of the YouTube video
|
||||
*/
|
||||
parseId(url){
|
||||
const regex = /^.*(?:(?:youtu\.be\/|v\/|vi\/|u\/\w\/|embed\/|shorts\/)|(?:(?:watch)?\?v(?:i)?=|&v(?:i)?=))([^#&?]*).*/;
|
||||
const match = url.match(regex);
|
||||
return (match?.length && match[1] ? match[1] : url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrape transcript from a YouTube video.
|
||||
* @returns {Promise<File[]>} File attachments scraped from the YouTube video
|
||||
*/
|
||||
async scrape() {
|
||||
let lang = '';
|
||||
const template = $(await renderExtensionTemplateAsync('attachments', 'youtube-scrape', {}));
|
||||
const videoUrl = await callGenericPopup(template, POPUP_TYPE.INPUT, '', { wide: false, large: false, okButton: 'Scrape', cancelButton: 'Cancel', rows: 2 });
|
||||
|
||||
template.find('input[name="youtubeLanguageCode"]').on('input', function () {
|
||||
lang = String($(this).val()).trim();
|
||||
});
|
||||
|
||||
if (!videoUrl) {
|
||||
return;
|
||||
}
|
||||
|
||||
const id = this.parseId(String(videoUrl).trim());
|
||||
const toast = toastr.info('Working, please wait...');
|
||||
|
||||
const result = await fetch('/api/serpapi/transcript', {
|
||||
method: 'POST',
|
||||
headers: getRequestHeaders(),
|
||||
body: JSON.stringify({ id, lang }),
|
||||
});
|
||||
|
||||
if (!result.ok) {
|
||||
const error = await result.text();
|
||||
throw new Error(error);
|
||||
}
|
||||
|
||||
const transcript = await result.text();
|
||||
toastr.clear(toast);
|
||||
|
||||
const file = new File([transcript], `YouTube - ${id} - ${Date.now()}.txt`, { type: 'text/plain' });
|
||||
return [file];
|
||||
}
|
||||
}
|
||||
|
||||
ScraperManager.registerDataBankScraper(new FileScraper());
|
||||
ScraperManager.registerDataBankScraper(new WebScraper());
|
||||
ScraperManager.registerDataBankScraper(new FandomScraper());
|
||||
ScraperManager.registerDataBankScraper(new YouTubeScraper());
|
||||
|
|
|
|||
|
|
@ -1185,16 +1185,23 @@ export function uuidv4() {
|
|||
}
|
||||
|
||||
function postProcessText(text, collapse = true) {
|
||||
// Remove carriage returns
|
||||
text = text.replace(/\r/g, '');
|
||||
// Replace tabs with spaces
|
||||
text = text.replace(/\t/g, ' ');
|
||||
// Normalize unicode spaces
|
||||
text = text.replace(/\u00A0/g, ' ');
|
||||
// Collapse multiple newlines into one
|
||||
if (collapse) {
|
||||
text = collapseNewlines(text);
|
||||
// Trim leading and trailing whitespace, and remove empty lines
|
||||
text = text.split('\n').map(l => l.trim()).filter(Boolean).join('\n');
|
||||
} else {
|
||||
// Replace more than 4 newlines with 4 newlines
|
||||
text = text.replace(/\n{4,}/g, '\n\n\n\n');
|
||||
// Trim lines that contain nothing but whitespace
|
||||
text = text.split('\n').map(l => /^\s+$/.test(l) ? '' : l).join('\n');
|
||||
}
|
||||
// Remove carriage returns
|
||||
text = text.replace(/\r/g, '');
|
||||
// Normalize unicode spaces
|
||||
text = text.replace(/\u00A0/g, ' ');
|
||||
// Collapse multiple spaces into one (except for newlines)
|
||||
text = text.replace(/ {2,}/g, ' ');
|
||||
// Remove leading and trailing spaces
|
||||
|
|
|
|||
|
|
@ -899,7 +899,7 @@ router.post('/generate', jsonParser, function (request, response) {
|
|||
}
|
||||
} else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.PERPLEXITY) {
|
||||
apiUrl = API_PERPLEXITY;
|
||||
apiKey = readSecret(SECRET_KEYS.PERPLEXITY);
|
||||
apiKey = readSecret(request.user.directories, SECRET_KEYS.PERPLEXITY);
|
||||
headers = {};
|
||||
bodyParams = {};
|
||||
request.body.messages = postProcessPrompt(request.body.messages, 'claude', request.body.char_name, request.body.user_name);
|
||||
|
|
|
|||
|
|
@ -48,6 +48,92 @@ router.post('/search', jsonParser, async (request, response) => {
|
|||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Get the transcript of a YouTube video
|
||||
* @copyright https://github.com/Kakulukian/youtube-transcript (MIT License)
|
||||
*/
|
||||
router.post('/transcript', jsonParser, async (request, response) => {
|
||||
try {
|
||||
const he = require('he');
|
||||
const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
|
||||
const id = request.body.id;
|
||||
const lang = request.body.lang;
|
||||
|
||||
if (!id) {
|
||||
console.log('Id is required for /transcript');
|
||||
return response.sendStatus(400);
|
||||
}
|
||||
|
||||
const videoPageResponse = await fetch(`https://www.youtube.com/watch?v=${id}`, {
|
||||
headers: {
|
||||
...(lang && { 'Accept-Language': lang }),
|
||||
'User-Agent': visitHeaders['User-Agent'],
|
||||
},
|
||||
});
|
||||
|
||||
const videoPageBody = await videoPageResponse.text();
|
||||
const splittedHTML = videoPageBody.split('"captions":');
|
||||
|
||||
if (splittedHTML.length <= 1) {
|
||||
if (videoPageBody.includes('class="g-recaptcha"')) {
|
||||
throw new Error('Too many requests');
|
||||
}
|
||||
if (!videoPageBody.includes('"playabilityStatus":')) {
|
||||
throw new Error('Video is not available');
|
||||
}
|
||||
throw new Error('Transcript not available');
|
||||
}
|
||||
|
||||
const captions = (() => {
|
||||
try {
|
||||
return JSON.parse(splittedHTML[1].split(',"videoDetails')[0].replace('\n', ''));
|
||||
} catch (e) {
|
||||
return undefined;
|
||||
}
|
||||
})()?.['playerCaptionsTracklistRenderer'];
|
||||
|
||||
if (!captions) {
|
||||
throw new Error('Transcript disabled');
|
||||
}
|
||||
|
||||
if (!('captionTracks' in captions)) {
|
||||
throw new Error('Transcript not available');
|
||||
}
|
||||
|
||||
if (lang && !captions.captionTracks.some(track => track.languageCode === lang)) {
|
||||
throw new Error('Transcript not available in this language');
|
||||
}
|
||||
|
||||
const transcriptURL = (lang ? captions.captionTracks.find(track => track.languageCode === lang) : captions.captionTracks[0]).baseUrl;
|
||||
const transcriptResponse = await fetch(transcriptURL, {
|
||||
headers: {
|
||||
...(lang && { 'Accept-Language': lang }),
|
||||
'User-Agent': visitHeaders['User-Agent'],
|
||||
},
|
||||
});
|
||||
|
||||
if (!transcriptResponse.ok) {
|
||||
throw new Error('Transcript request failed');
|
||||
}
|
||||
|
||||
const transcriptBody = await transcriptResponse.text();
|
||||
const results = [...transcriptBody.matchAll(RE_XML_TRANSCRIPT)];
|
||||
const transcript = results.map((result) => ({
|
||||
text: result[3],
|
||||
duration: parseFloat(result[2]),
|
||||
offset: parseFloat(result[1]),
|
||||
lang: lang ?? captions.captionTracks[0].languageCode,
|
||||
}));
|
||||
// The text is double-encoded
|
||||
const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');
|
||||
|
||||
return response.send(transcriptText);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
return response.sendStatus(500);
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/visit', jsonParser, async (request, response) => {
|
||||
try {
|
||||
const url = request.body.url;
|
||||
|
|
|
|||
Loading…
Reference in New Issue