mirror of
https://github.com/SillyTavern/SillyTavern.git
synced 2025-06-05 21:59:27 +02:00
Merge branch 'staging' into InfermaticAI
This commit is contained in:
NWilson
@ -47,6 +47,20 @@
|
||||
"ban_eos_token": false,
|
||||
"skip_special_tokens": true,
|
||||
"streaming": false,
|
||||
"sampler_priority": [
|
||||
"temperature",
|
||||
"dynamic_temperature",
|
||||
"quadratic_sampling",
|
||||
"top_k",
|
||||
"top_p",
|
||||
"typical_p",
|
||||
"epsilon_cutoff",
|
||||
"eta_cutoff",
|
||||
"tfs",
|
||||
"top_a",
|
||||
"min_p",
|
||||
"mirostat"
|
||||
],
|
||||
"mirostat_mode": 0,
|
||||
"mirostat_tau": 5,
|
||||
"mirostat_eta": 0.1,
|
||||
|
||||
@ -1507,7 +1507,7 @@
|
||||
</h4>
|
||||
<textarea id="grammar_string_textgenerationwebui" rows="4" class="text_pole textarea_compact monospace" data-i18n="[placeholder]Type in the desired custom grammar" placeholder="Type in the desired custom grammar"></textarea>
|
||||
</div>
|
||||
<div data-newbie-hidden data-tg-type="koboldcpp" class="range-block flexFlowColumn">
|
||||
<div data-newbie-hidden data-tg-type="koboldcpp" class="range-block flexFlowColumn wide100p">
|
||||
<hr class="wide100p">
|
||||
<div class="range-block-title">
|
||||
<span data-i18n="Samplers Order">Samplers Order</span>
|
||||
@ -1550,6 +1550,33 @@
|
||||
<span data-i18n="Load default order">Load default order</span>
|
||||
</div>
|
||||
</div>
|
||||
<div data-newbie-hidden data-tg-type="ooba" class="range-block flexFlowColumn wide100p">
|
||||
<hr class="wide100p">
|
||||
<h4 class="range-block-title justifyCenter">
|
||||
<span data-i18n="Sampler Priority">Sampler Priority</span>
|
||||
<div class="margin5 fa-solid fa-circle-info opacity50p" title="Ooba only. Determines the order of samplers."></div>
|
||||
</h4>
|
||||
<div class="toggle-description widthUnset" data-i18n="Ooba only. Determines the order of samplers.">
|
||||
Ooba only. Determines the order of samplers.
|
||||
</div>
|
||||
<div id="sampler_priority_container" class="prompt_order">
|
||||
<div data-name="temperature" draggable="true"><span>Temperature</span><small></small></div>
|
||||
<div data-name="dynamic_temperature" draggable="true"><span>Dynamic Temperature</span><small></small></div>
|
||||
<div data-name="quadratic_sampling" draggable="true"><span>Quadratic / Smooth Sampling</span><small></small></div>
|
||||
<div data-name="top_k" draggable="true"><span>Top K</span><small></small></div>
|
||||
<div data-name="top_p" draggable="true"><span>Top P</span><small></small></div>
|
||||
<div data-name="typical_p" draggable="true"><span>Typical P</span><small></small></div>
|
||||
<div data-name="epsilon_cutoff" draggable="true"><span>Epsilon Cutoff</span><small></small></div>
|
||||
<div data-name="eta_cutoff" draggable="true"><span>Eta Cutoff</span><small></small></div>
|
||||
<div data-name="tfs" draggable="true"><span>Tail Free Sampling</span><small></small></div>
|
||||
<div data-name="top_a" draggable="true"><span>Top A</span><small></small></div>
|
||||
<div data-name="min_p" draggable="true"><span>Min P</span><small></small></div>
|
||||
<div data-name="mirostat" draggable="true"><span>Mirostat</span><small></small></div>
|
||||
</div>
|
||||
<div id="textgenerationwebui_default_order" class="menu_button menu_button_icon">
|
||||
<span data-i18n="Load default order">Load default order</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div><!-- end of textgen settings-->
|
||||
<div id="openai_settings">
|
||||
|
||||
@ -2273,12 +2273,21 @@ export async function generateQuietPrompt(quiet_prompt, quietToLoud, skipWIAN, q
|
||||
return generateFinished;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes slash commands and returns the new text and whether the generation was interrupted.
|
||||
* @param {string} message Text to be sent
|
||||
* @returns {Promise<boolean>} Whether the message sending was interrupted
|
||||
*/
|
||||
async function processCommands(message) {
|
||||
if (!message || !message.trim().startsWith('/')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const previousText = String($('#send_textarea').val());
|
||||
const result = await executeSlashCommands(message);
|
||||
|
||||
if (!result || typeof result !== 'object') {
|
||||
return null;
|
||||
return false;
|
||||
}
|
||||
|
||||
const currentText = String($('#send_textarea').val());
|
||||
@ -2881,7 +2890,7 @@ async function Generate(type, { automatic_trigger, force_name2, quiet_prompt, qu
|
||||
let message_already_generated = isImpersonate ? `${name1}: ` : `${name2}: `;
|
||||
|
||||
if (!(dryRun || type == 'regenerate' || type == 'swipe' || type == 'quiet')) {
|
||||
const interruptedByCommand = await processCommands($('#send_textarea').val());
|
||||
const interruptedByCommand = await processCommands(String($('#send_textarea').val()));
|
||||
|
||||
if (interruptedByCommand) {
|
||||
//$("#send_textarea").val('').trigger('input');
|
||||
@ -7872,9 +7881,9 @@ async function importFromURL(items, files) {
|
||||
}
|
||||
}
|
||||
|
||||
async function doImpersonate() {
|
||||
async function doImpersonate(_, prompt) {
|
||||
$('#send_textarea').val('');
|
||||
$('#option_impersonate').trigger('click', { fromSlashCommand: true });
|
||||
$('#option_impersonate').trigger('click', { fromSlashCommand: true, additionalPrompt: prompt });
|
||||
}
|
||||
|
||||
async function doDeleteChat() {
|
||||
@ -8041,7 +8050,7 @@ jQuery(async function () {
|
||||
|
||||
registerSlashCommand('dupe', DupeChar, [], '– duplicates the currently selected character', true, true);
|
||||
registerSlashCommand('api', connectAPISlash, [], `<span class="monospace">(${Object.keys(CONNECT_API_MAP).join(', ')})</span> – connect to an API`, true, true);
|
||||
registerSlashCommand('impersonate', doImpersonate, ['imp'], '– calls an impersonation response', true, true);
|
||||
registerSlashCommand('impersonate', doImpersonate, ['imp'], '<span class="monospace">[prompt]</span> – calls an impersonation response, with an optional additional prompt', true, true);
|
||||
registerSlashCommand('delchat', doDeleteChat, [], '– deletes the current chat', true, true);
|
||||
registerSlashCommand('getchatname', doGetChatName, [], '– returns the name of the current chat file into the pipe', false, true);
|
||||
registerSlashCommand('closechat', doCloseChat, [], '– closes the current chat', true, true);
|
||||
@ -8695,6 +8704,13 @@ jQuery(async function () {
|
||||
const fromSlashCommand = customData?.fromSlashCommand || false;
|
||||
var id = $(this).attr('id');
|
||||
|
||||
// Check whether a custom prompt was provided via custom data (for example through a slash command)
|
||||
const additionalPrompt = customData?.additionalPrompt?.trim() || undefined;
|
||||
const buildOrFillAdditionalArgs = (args = {}) => ({
|
||||
...args,
|
||||
...(additionalPrompt !== undefined && { quiet_prompt: additionalPrompt, quietToLoud: true }),
|
||||
});
|
||||
|
||||
if (id == 'option_select_chat') {
|
||||
if ((selected_group && !is_group_generating) || (this_chid !== undefined && !is_send_press) || fromSlashCommand) {
|
||||
await displayPastChats();
|
||||
@ -8730,7 +8746,7 @@ jQuery(async function () {
|
||||
}
|
||||
else {
|
||||
is_send_press = true;
|
||||
Generate('regenerate');
|
||||
Generate('regenerate', buildOrFillAdditionalArgs());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -8738,14 +8754,14 @@ jQuery(async function () {
|
||||
else if (id == 'option_impersonate') {
|
||||
if (is_send_press == false || fromSlashCommand) {
|
||||
is_send_press = true;
|
||||
Generate('impersonate');
|
||||
Generate('impersonate', buildOrFillAdditionalArgs());
|
||||
}
|
||||
}
|
||||
|
||||
else if (id == 'option_continue') {
|
||||
if (is_send_press == false || fromSlashCommand) {
|
||||
is_send_press = true;
|
||||
Generate('continue');
|
||||
Generate('continue', buildOrFillAdditionalArgs());
|
||||
}
|
||||
}
|
||||
|
||||
@ -9882,6 +9898,7 @@ jQuery(async function () {
|
||||
<li>Chub characters (direct link or id)<br>Example: <tt>Anonymous/example-character</tt></li>
|
||||
<li>Chub lorebooks (direct link or id)<br>Example: <tt>lorebooks/bartleby/example-lorebook</tt></li>
|
||||
<li>JanitorAI character (direct link or id)<br>Example: <tt>https://janitorai.com/characters/ddd1498a-a370-4136-b138-a8cd9461fdfe_character-aqua-the-useless-goddess</tt></li>
|
||||
<li>Pygmalion.chat character (link)<br>Example: <tt>https://pygmalion.chat/character/a7ca95a1-0c88-4e23-91b3-149db1e78ab9</tt></li>
|
||||
<li>More coming soon...</li>
|
||||
<ul>`;
|
||||
const input = await callPopup(html, 'input', '', { okButton: 'Import', rows: 4 });
|
||||
|
||||
@ -119,7 +119,7 @@ const scale_max = 8191;
|
||||
const claude_max = 9000; // We have a proper tokenizer, so theoretically could be larger (up to 9k)
|
||||
const claude_100k_max = 99000;
|
||||
let ai21_max = 9200; //can easily fit 9k gpt tokens because j2's tokenizer is efficient af
|
||||
const unlocked_max = 100 * 1024;
|
||||
const unlocked_max = max_200k;
|
||||
const oai_max_temp = 2.0;
|
||||
const claude_max_temp = 1.0; //same as j2
|
||||
const j2_max_topk = 10.0;
|
||||
@ -3739,9 +3739,11 @@ async function testApiConnection() {
|
||||
}
|
||||
|
||||
function reconnectOpenAi() {
|
||||
if (main_api == 'openai') {
|
||||
setOnlineStatus('no_connection');
|
||||
resultCheckStatus();
|
||||
$('#api_button_openai').trigger('click');
|
||||
}
|
||||
}
|
||||
|
||||
function onProxyPasswordShowClick() {
|
||||
@ -4202,11 +4204,7 @@ $(document).ready(async function () {
|
||||
oai_settings.chat_completion_source = String($(this).find(':selected').val());
|
||||
toggleChatCompletionForms();
|
||||
saveSettingsDebounced();
|
||||
|
||||
if (main_api == 'openai') {
|
||||
reconnectOpenAi();
|
||||
}
|
||||
|
||||
eventSource.emit(event_types.CHATCOMPLETION_SOURCE_CHANGED, oai_settings.chat_completion_source);
|
||||
});
|
||||
|
||||
|
||||
@ -140,7 +140,7 @@ const getSlashCommandsHelp = parser.getHelpString.bind(parser);
|
||||
|
||||
parser.addCommand('?', helpCommandCallback, ['help'], ' – get help on macros, chat formatting and commands', true, true);
|
||||
parser.addCommand('name', setNameCallback, ['persona'], '<span class="monospace">(name)</span> – sets user name and persona avatar (if set)', true, true);
|
||||
parser.addCommand('sync', syncCallback, [], ' – syncs user name in user-attributed messages in the current chat', true, true);
|
||||
parser.addCommand('sync', syncCallback, [], ' – syncs the user persona in user-attributed messages in the current chat', true, true);
|
||||
parser.addCommand('lock', bindCallback, ['bind'], ' – locks/unlocks a persona (name and avatar) to the current chat', true, true);
|
||||
parser.addCommand('bg', setBackgroundCallback, ['background'], '<span class="monospace">(filename)</span> – sets a background according to filename, partial names allowed', false, true);
|
||||
parser.addCommand('sendas', sendMessageAs, [], ' – sends message as a specific character. Uses character avatar if it exists in the characters list. Example that will send "Hello, guys!" from "Chloe": <tt>/sendas name="Chloe" Hello, guys!</tt>', true, true);
|
||||
@ -150,7 +150,7 @@ parser.addCommand('comment', sendCommentMessage, [], '<span class="monospace">(t
|
||||
parser.addCommand('single', setStoryModeCallback, ['story'], ' – sets the message style to single document mode without names or avatars visible', true, true);
|
||||
parser.addCommand('bubble', setBubbleModeCallback, ['bubbles'], ' – sets the message style to bubble chat mode', true, true);
|
||||
parser.addCommand('flat', setFlatModeCallback, ['default'], ' – sets the message style to flat chat mode', true, true);
|
||||
parser.addCommand('continue', continueChatCallback, ['cont'], ' – continues the last message in the chat', true, true);
|
||||
parser.addCommand('continue', continueChatCallback, ['cont'], '<span class="monospace">[prompt]</span> – continues the last message in the chat, with an optional additional prompt', true, true);
|
||||
parser.addCommand('go', goToCharacterCallback, ['char'], '<span class="monospace">(name)</span> – opens up a chat with the character or group by its name', true, true);
|
||||
parser.addCommand('sysgen', generateSystemMessage, [], '<span class="monospace">(prompt)</span> – generates a system message using a specified prompt', true, true);
|
||||
parser.addCommand('ask', askCharacter, [], '<span class="monospace">(prompt)</span> – asks a specified character card a prompt', true, true);
|
||||
@ -1168,7 +1168,7 @@ async function openChat(id) {
|
||||
await reloadCurrentChat();
|
||||
}
|
||||
|
||||
function continueChatCallback() {
|
||||
function continueChatCallback(_, prompt) {
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
await waitUntilCondition(() => !is_send_press && !is_group_generating, 10000, 100);
|
||||
@ -1179,7 +1179,7 @@ function continueChatCallback() {
|
||||
|
||||
// Prevent infinite recursion
|
||||
$('#send_textarea').val('').trigger('input');
|
||||
$('#option_continue').trigger('click', { fromSlashCommand: true });
|
||||
$('#option_continue').trigger('click', { fromSlashCommand: true, additionalPrompt: prompt });
|
||||
}, 1);
|
||||
|
||||
return '';
|
||||
|
||||
@ -35,6 +35,20 @@ export const textgen_types = {
|
||||
};
|
||||
|
||||
const { MANCER, APHRODITE, TABBY, TOGETHERAI, OOBA, OLLAMA, LLAMACPP, INFERMATICAI } = textgen_types;
|
||||
const OOBA_DEFAULT_ORDER = [
|
||||
'temperature',
|
||||
'dynamic_temperature',
|
||||
'quadratic_sampling',
|
||||
'top_k',
|
||||
'top_p',
|
||||
'typical_p',
|
||||
'epsilon_cutoff',
|
||||
'eta_cutoff',
|
||||
'tfs',
|
||||
'top_a',
|
||||
'min_p',
|
||||
'mirostat',
|
||||
];
|
||||
const BIAS_KEY = '#textgenerationwebui_api-settings';
|
||||
|
||||
// Maybe let it be configurable in the future?
|
||||
@ -98,6 +112,7 @@ const settings = {
|
||||
negative_prompt: '',
|
||||
grammar_string: '',
|
||||
banned_tokens: '',
|
||||
sampler_priority: OOBA_DEFAULT_ORDER,
|
||||
//n_aphrodite: 1,
|
||||
//best_of_aphrodite: 1,
|
||||
ignore_eos_token_aphrodite: false,
|
||||
@ -173,6 +188,7 @@ const setting_names = [
|
||||
//'log_probs_aphrodite',
|
||||
//'prompt_log_probs_aphrodite'
|
||||
'sampler_order',
|
||||
'sampler_priority',
|
||||
'n',
|
||||
'logit_bias',
|
||||
'custom_model',
|
||||
@ -429,7 +445,7 @@ function loadTextGenSettings(data, loadedSettings) {
|
||||
* Sorts the sampler items by the given order.
|
||||
* @param {any[]} orderArray Sampler order array.
|
||||
*/
|
||||
function sortItemsByOrder(orderArray) {
|
||||
function sortKoboldItemsByOrder(orderArray) {
|
||||
console.debug('Preset samplers order: ' + orderArray);
|
||||
const $draggableItems = $('#koboldcpp_order');
|
||||
|
||||
@ -440,6 +456,16 @@ function sortItemsByOrder(orderArray) {
|
||||
}
|
||||
}
|
||||
|
||||
function sortOobaItemsByOrder(orderArray) {
|
||||
console.debug('Preset samplers order: ', orderArray);
|
||||
const $container = $('#sampler_priority_container');
|
||||
|
||||
orderArray.forEach((name) => {
|
||||
const $item = $container.find(`[data-name="${name}"]`).detach();
|
||||
$container.append($item);
|
||||
});
|
||||
}
|
||||
|
||||
jQuery(function () {
|
||||
$('#koboldcpp_order').sortable({
|
||||
delay: getSortableDelay(),
|
||||
@ -456,7 +482,27 @@ jQuery(function () {
|
||||
|
||||
$('#koboldcpp_default_order').on('click', function () {
|
||||
settings.sampler_order = KOBOLDCPP_ORDER;
|
||||
sortItemsByOrder(settings.sampler_order);
|
||||
sortKoboldItemsByOrder(settings.sampler_order);
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
$('#sampler_priority_container').sortable({
|
||||
delay: getSortableDelay(),
|
||||
stop: function () {
|
||||
const order = [];
|
||||
$('#sampler_priority_container').children().each(function () {
|
||||
order.push($(this).data('name'));
|
||||
});
|
||||
settings.sampler_priority = order;
|
||||
console.log('Samplers reordered:', settings.sampler_priority);
|
||||
saveSettingsDebounced();
|
||||
},
|
||||
});
|
||||
|
||||
$('#textgenerationwebui_default_order').on('click', function () {
|
||||
sortOobaItemsByOrder(OOBA_DEFAULT_ORDER);
|
||||
settings.sampler_priority = OOBA_DEFAULT_ORDER;
|
||||
console.log('Default samplers order loaded:', settings.sampler_priority);
|
||||
saveSettingsDebounced();
|
||||
});
|
||||
|
||||
@ -543,6 +589,7 @@ jQuery(function () {
|
||||
'penalty_alpha_textgenerationwebui': 0,
|
||||
'typical_p_textgenerationwebui': 1, // Added entry
|
||||
'guidance_scale_textgenerationwebui': 1,
|
||||
'smoothing_factor_textgenerationwebui': 0,
|
||||
};
|
||||
|
||||
for (const [id, value] of Object.entries(inputs)) {
|
||||
@ -622,11 +669,18 @@ function setSettingByName(setting, value, trigger) {
|
||||
|
||||
if ('sampler_order' === setting) {
|
||||
value = Array.isArray(value) ? value : KOBOLDCPP_ORDER;
|
||||
sortItemsByOrder(value);
|
||||
sortKoboldItemsByOrder(value);
|
||||
settings.sampler_order = value;
|
||||
return;
|
||||
}
|
||||
|
||||
if ('sampler_priority' === setting) {
|
||||
value = Array.isArray(value) ? value : OOBA_DEFAULT_ORDER;
|
||||
sortOobaItemsByOrder(value);
|
||||
settings.sampler_priority = value;
|
||||
return;
|
||||
}
|
||||
|
||||
if ('logit_bias' === setting) {
|
||||
settings.logit_bias = Array.isArray(value) ? value : [];
|
||||
return;
|
||||
@ -838,6 +892,7 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate,
|
||||
'dynatemp_range': settings.dynatemp ? (settings.max_temp - settings.min_temp) / 2 : 0,
|
||||
'dynatemp_exponent': settings.dynatemp ? settings.dynatemp_exponent : 1,
|
||||
'smoothing_factor': settings.smoothing_factor,
|
||||
'sampler_priority': settings.type === OOBA ? settings.sampler_priority : undefined,
|
||||
'stopping_strings': getStoppingStrings(isImpersonate, isContinue),
|
||||
'stop': getStoppingStrings(isImpersonate, isContinue),
|
||||
'truncation_length': max_context,
|
||||
|
||||
@ -1,15 +1,37 @@
|
||||
const fs = require('fs');
|
||||
|
||||
const encode = require('png-chunks-encode');
|
||||
const extract = require('png-chunks-extract');
|
||||
const PNGtext = require('png-chunk-text');
|
||||
|
||||
const parse = async (cardUrl, format) => {
|
||||
let fileFormat = format === undefined ? 'png' : format;
|
||||
/**
|
||||
* Writes Character metadata to a PNG image buffer.
|
||||
* @param {Buffer} image PNG image buffer
|
||||
* @param {string} data Character data to write
|
||||
* @returns {Buffer} PNG image buffer with metadata
|
||||
*/
|
||||
const write = (image, data) => {
|
||||
const chunks = extract(image);
|
||||
const tEXtChunks = chunks.filter(chunk => chunk.name === 'tEXt');
|
||||
|
||||
switch (fileFormat) {
|
||||
case 'png': {
|
||||
const buffer = fs.readFileSync(cardUrl);
|
||||
const chunks = extract(buffer);
|
||||
// Remove all existing tEXt chunks
|
||||
for (let tEXtChunk of tEXtChunks) {
|
||||
chunks.splice(chunks.indexOf(tEXtChunk), 1);
|
||||
}
|
||||
// Add new chunks before the IEND chunk
|
||||
const base64EncodedData = Buffer.from(data, 'utf8').toString('base64');
|
||||
chunks.splice(-1, 0, PNGtext.encode('chara', base64EncodedData));
|
||||
const newBuffer = Buffer.from(encode(chunks));
|
||||
return newBuffer;
|
||||
};
|
||||
|
||||
/**
|
||||
* Reads Character metadata from a PNG image buffer.
|
||||
* @param {Buffer} image PNG image buffer
|
||||
* @returns {string} Character data
|
||||
*/
|
||||
const read = (image) => {
|
||||
const chunks = extract(image);
|
||||
|
||||
const textChunks = chunks.filter(function (chunk) {
|
||||
return chunk.name === 'tEXt';
|
||||
@ -30,12 +52,29 @@ const parse = async (cardUrl, format) => {
|
||||
}
|
||||
|
||||
return Buffer.from(textChunks[index].text, 'base64').toString('utf8');
|
||||
};
|
||||
|
||||
/**
|
||||
* Parses a card image and returns the character metadata.
|
||||
* @param {string} cardUrl Path to the card image
|
||||
* @param {string} format File format
|
||||
* @returns {string} Character data
|
||||
*/
|
||||
const parse = (cardUrl, format) => {
|
||||
let fileFormat = format === undefined ? 'png' : format;
|
||||
|
||||
switch (fileFormat) {
|
||||
case 'png': {
|
||||
const buffer = fs.readFileSync(cardUrl);
|
||||
return read(buffer);
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
throw new Error('Unsupported format');
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
parse: parse,
|
||||
parse,
|
||||
write,
|
||||
read,
|
||||
};
|
||||
|
||||
@ -7,9 +7,6 @@ const writeFileAtomicSync = require('write-file-atomic').sync;
|
||||
const yaml = require('yaml');
|
||||
const _ = require('lodash');
|
||||
|
||||
const encode = require('png-chunks-encode');
|
||||
const extract = require('png-chunks-extract');
|
||||
const PNGtext = require('png-chunk-text');
|
||||
const jimp = require('jimp');
|
||||
|
||||
const { DIRECTORIES, UPLOADS_PATH, AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants');
|
||||
@ -33,7 +30,7 @@ const characterDataCache = new Map();
|
||||
* @param {string} input_format - 'png'
|
||||
* @returns {Promise<string | undefined>} - Character card data
|
||||
*/
|
||||
async function charaRead(img_url, input_format) {
|
||||
async function charaRead(img_url, input_format = 'png') {
|
||||
const stat = fs.statSync(img_url);
|
||||
const cacheKey = `${img_url}-${stat.mtimeMs}`;
|
||||
if (characterDataCache.has(cacheKey)) {
|
||||
@ -59,22 +56,12 @@ async function charaWrite(img_url, data, target_img, response = undefined, mes =
|
||||
}
|
||||
}
|
||||
// Read the image, resize, and save it as a PNG into the buffer
|
||||
const image = await tryReadImage(img_url, crop);
|
||||
const inputImage = await tryReadImage(img_url, crop);
|
||||
|
||||
// Get the chunks
|
||||
const chunks = extract(image);
|
||||
const tEXtChunks = chunks.filter(chunk => chunk.name === 'tEXt');
|
||||
const outputImage = characterCardParser.write(inputImage, data);
|
||||
|
||||
// Remove all existing tEXt chunks
|
||||
for (let tEXtChunk of tEXtChunks) {
|
||||
chunks.splice(chunks.indexOf(tEXtChunk), 1);
|
||||
}
|
||||
// Add new chunks before the IEND chunk
|
||||
const base64EncodedData = Buffer.from(data, 'utf8').toString('base64');
|
||||
chunks.splice(-1, 0, PNGtext.encode('chara', base64EncodedData));
|
||||
//chunks.splice(-1, 0, text.encode('lorem', 'ipsum'));
|
||||
|
||||
writeFileAtomicSync(DIRECTORIES.characters + target_img + '.png', Buffer.from(encode(chunks)));
|
||||
writeFileAtomicSync(DIRECTORIES.characters + target_img + '.png', outputImage);
|
||||
if (response !== undefined) response.send(mes);
|
||||
return true;
|
||||
} catch (err) {
|
||||
@ -152,13 +139,13 @@ const processCharacter = async (item, i) => {
|
||||
const img_data = await charaRead(DIRECTORIES.characters + item);
|
||||
if (img_data === undefined) throw new Error('Failed to read character file');
|
||||
|
||||
let jsonObject = getCharaCardV2(JSON.parse(img_data));
|
||||
let jsonObject = getCharaCardV2(JSON.parse(img_data), false);
|
||||
jsonObject.avatar = item;
|
||||
characters[i] = jsonObject;
|
||||
characters[i]['json_data'] = img_data;
|
||||
const charStat = fs.statSync(path.join(DIRECTORIES.characters, item));
|
||||
characters[i]['date_added'] = charStat.birthtimeMs;
|
||||
characters[i]['create_date'] = jsonObject['create_date'] || humanizedISO8601DateTime(charStat.birthtimeMs);
|
||||
characters[i]['date_added'] = charStat.ctimeMs;
|
||||
characters[i]['create_date'] = jsonObject['create_date'] || humanizedISO8601DateTime(charStat.ctimeMs);
|
||||
const char_dir = path.join(DIRECTORIES.chats, item.replace('.png', ''));
|
||||
|
||||
const { chatSize, dateLastChat } = calculateChatSize(char_dir);
|
||||
@ -183,15 +170,30 @@ const processCharacter = async (item, i) => {
|
||||
}
|
||||
};
|
||||
|
||||
function getCharaCardV2(jsonObject) {
|
||||
/**
|
||||
* Convert a character object to Spec V2 format.
|
||||
* @param {object} jsonObject Character object
|
||||
* @param {boolean} hoistDate Will set the chat and create_date fields to the current date if they are missing
|
||||
* @returns {object} Character object in Spec V2 format
|
||||
*/
|
||||
function getCharaCardV2(jsonObject, hoistDate = true) {
|
||||
if (jsonObject.spec === undefined) {
|
||||
jsonObject = convertToV2(jsonObject);
|
||||
|
||||
if (hoistDate && !jsonObject.create_date) {
|
||||
jsonObject.create_date = humanizedISO8601DateTime();
|
||||
}
|
||||
} else {
|
||||
jsonObject = readFromV2(jsonObject);
|
||||
}
|
||||
return jsonObject;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a character object to Spec V2 format.
|
||||
* @param {object} char Character object
|
||||
* @returns {object} Character object in Spec V2 format
|
||||
*/
|
||||
function convertToV2(char) {
|
||||
// Simulate incoming data from frontend form
|
||||
const result = charaFormatData({
|
||||
@ -212,7 +214,8 @@ function convertToV2(char) {
|
||||
});
|
||||
|
||||
result.chat = char.chat ?? humanizedISO8601DateTime();
|
||||
result.create_date = char.create_date ?? humanizedISO8601DateTime();
|
||||
result.create_date = char.create_date;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@ -10,6 +10,7 @@ const contentLogPath = path.join(contentDirectory, 'content.log');
|
||||
const contentIndexPath = path.join(contentDirectory, 'index.json');
|
||||
const { DIRECTORIES } = require('../constants');
|
||||
const presetFolders = [DIRECTORIES.koboldAI_Settings, DIRECTORIES.openAI_Settings, DIRECTORIES.novelAI_Settings, DIRECTORIES.textGen_Settings];
|
||||
const characterCardParser = require('../character-card-parser.js');
|
||||
|
||||
/**
|
||||
* Gets the default presets from the content directory.
|
||||
@ -219,6 +220,56 @@ async function downloadChubCharacter(id) {
|
||||
return { buffer, fileName, fileType };
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads a character card from the Pygsite.
|
||||
* @param {string} id UUID of the character
|
||||
* @returns {Promise<{buffer: Buffer, fileName: string, fileType: string}>}
|
||||
*/
|
||||
async function downloadPygmalionCharacter(id) {
|
||||
const result = await fetch(`https://server.pygmalion.chat/api/export/character/${id}/v2`);
|
||||
|
||||
if (!result.ok) {
|
||||
const text = await result.text();
|
||||
console.log('Pygsite returned error', result.status, text);
|
||||
throw new Error('Failed to download character');
|
||||
}
|
||||
|
||||
const jsonData = await result.json();
|
||||
const characterData = jsonData?.character;
|
||||
|
||||
if (!characterData || typeof characterData !== 'object') {
|
||||
console.error('Pygsite returned invalid character data', jsonData);
|
||||
throw new Error('Failed to download character');
|
||||
}
|
||||
|
||||
try {
|
||||
const avatarUrl = characterData?.data?.avatar;
|
||||
|
||||
if (!avatarUrl) {
|
||||
console.error('Pygsite character does not have an avatar', characterData);
|
||||
throw new Error('Failed to download avatar');
|
||||
}
|
||||
|
||||
const avatarResult = await fetch(avatarUrl);
|
||||
const avatarBuffer = await avatarResult.buffer();
|
||||
|
||||
const cardBuffer = characterCardParser.write(avatarBuffer, JSON.stringify(characterData));
|
||||
|
||||
return {
|
||||
buffer: cardBuffer,
|
||||
fileName: `${sanitize(id)}.png`,
|
||||
fileType: 'image/png',
|
||||
};
|
||||
} catch (e) {
|
||||
console.error('Failed to download avatar, using JSON instead', e);
|
||||
return {
|
||||
buffer: Buffer.from(JSON.stringify(jsonData)),
|
||||
fileName: `${sanitize(id)}.json`,
|
||||
fileType: 'application/json',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {String} str
|
||||
@ -294,7 +345,7 @@ async function downloadJannyCharacter(uuid) {
|
||||
* @param {String} url
|
||||
* @returns {String | null } UUID of the character
|
||||
*/
|
||||
function parseJannyUrl(url) {
|
||||
function getUuidFromUrl(url) {
|
||||
// Extract UUID from URL
|
||||
const uuidRegex = /[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/;
|
||||
const matches = url.match(uuidRegex);
|
||||
@ -317,8 +368,18 @@ router.post('/import', jsonParser, async (request, response) => {
|
||||
let type;
|
||||
|
||||
const isJannnyContent = url.includes('janitorai');
|
||||
if (isJannnyContent) {
|
||||
const uuid = parseJannyUrl(url);
|
||||
const isPygmalionContent = url.includes('pygmalion.chat');
|
||||
|
||||
if (isPygmalionContent) {
|
||||
const uuid = getUuidFromUrl(url);
|
||||
if (!uuid) {
|
||||
return response.sendStatus(404);
|
||||
}
|
||||
|
||||
type = 'character';
|
||||
result = await downloadPygmalionCharacter(uuid);
|
||||
} else if (isJannnyContent) {
|
||||
const uuid = getUuidFromUrl(url);
|
||||
if (!uuid) {
|
||||
return response.sendStatus(404);
|
||||
}
|
||||
|
||||
61
src/util.js
61
src/util.js
@ -365,7 +365,7 @@ function getImages(path) {
|
||||
/**
|
||||
* Pipe a fetch() response to an Express.js Response, including status code.
|
||||
* @param {import('node-fetch').Response} from The Fetch API response to pipe from.
|
||||
* @param {Express.Response} to The Express response to pipe to.
|
||||
* @param {import('express').Response} to The Express response to pipe to.
|
||||
*/
|
||||
function forwardFetchResponse(from, to) {
|
||||
let statusCode = from.status;
|
||||
@ -399,6 +399,64 @@ function forwardFetchResponse(from, to) {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes an HTTP/2 request to the specified endpoint.
|
||||
*
|
||||
* @deprecated Use `node-fetch` if possible.
|
||||
* @param {string} endpoint URL to make the request to
|
||||
* @param {string} method HTTP method to use
|
||||
* @param {string} body Request body
|
||||
* @param {object} headers Request headers
|
||||
* @returns {Promise<string>} Response body
|
||||
*/
|
||||
function makeHttp2Request(endpoint, method, body, headers) {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
const http2 = require('http2');
|
||||
const url = new URL(endpoint);
|
||||
const client = http2.connect(url.origin);
|
||||
|
||||
const req = client.request({
|
||||
':method': method,
|
||||
':path': url.pathname,
|
||||
...headers,
|
||||
});
|
||||
req.setEncoding('utf8');
|
||||
|
||||
req.on('response', (headers) => {
|
||||
const status = Number(headers[':status']);
|
||||
|
||||
if (status < 200 || status >= 300) {
|
||||
reject(new Error(`Request failed with status ${status}`));
|
||||
}
|
||||
|
||||
let data = '';
|
||||
|
||||
req.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
req.on('end', () => {
|
||||
console.log(data);
|
||||
resolve(data);
|
||||
});
|
||||
});
|
||||
|
||||
req.on('error', (err) => {
|
||||
reject(err);
|
||||
});
|
||||
|
||||
if (body) {
|
||||
req.write(body);
|
||||
}
|
||||
|
||||
req.end();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds YAML-serialized object to the object.
|
||||
* @param {object} obj Object
|
||||
@ -547,4 +605,5 @@ module.exports = {
|
||||
excludeKeysByYaml,
|
||||
trimV1,
|
||||
Cache,
|
||||
makeHttp2Request,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user