rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: implement miHoYo scraper

This commit is contained in:
Bronya-Rand 2024-04-22 19:11:00 +01:00
parent 6d1933c8f3
commit 0f0895f345
4 changed files with 193 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
public/img/mihoyo.svg Normal file
View File

@ -0,0 +1,3 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="19.998" height="19.998" viewBox="0 0 99 99">
<image width="99" height="99" xlink:href="data:img/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAADbUlEQVQ4jVzLsQkAIADEwFPBAdx/ScH+bQTFQKqQkqS65PjSMNCxML9+f2wAAAD//2LBY5gRAwODLwMDgxUDA4MM1MBPDAwM5xgYGLYyMDBshKr/BzeUgYEBAAAA//9iALnw////IJeC2CAs8P///77///9/+48f7Pj//78ukj6m////MwEAAAD//wIbhAQUGRgYFjEwMNhAxR4zMDBchNIgoMDAwKALdTEI/GFgYAhiYGDYzMDAwMjAwMAIAAAA//9M0CEVwCAAQMHTcyj8KiBogiQehgBYSiwKAjeDwP934t9gwEDGQkfDh32aBwkFEe95XDHhBwAA//9CNnAyAwNDDgMDw3soezYDA8MTtAiAAZAr/RgYGB4yMDBwQ126kIGB4Q8AAAD//3TQsQ0AEBRAwRhDIrGBTmPDX9vXaUg0ijfAu/vfsQ7ZRHlsfg0EGjIq0gYAAP//gsVOKDQM7kBjD+YykBhIDQjD2CAaBI4xMDC8YGBgKGBgYDBkYGDgYWBg+A8AAAD//wIpAIWLNVTRaQYGhutIXoNpRo45cOBD2asYGBjYGRgYYsERysDACgAAAP//AqVDMQYGBhGoggcMDAw/0MIL2TD0RA9K5O8YGBg+MDAwsDAwMMgCAAAA//+CeQFm418ckYALgPSBDAPhHwwMDMwAAAAA//8CGfgaGrMgIAlyNpqLYBYiWwwDIFeB1ED0MTC8AgAAAP//AhkIyp+gsAMBU2jixQeQsygoW4IiA5R0lBkYGD4CAAAA//+CxfJaKK3DwMDggxSmsLyKjGEAVGDYQmP6BgMDw1MGBgYGAAAAAP//ghl4kIGBYT00WYBSPSjWQBpwAZCLMhgYGPQYGBg+MzAwPGNgYNjBwMDAAAAAAP//TNKxCcAgFEBBBwg6RMBenMslnNSQKlOcEH5h8dpX3Yn2xhu4n0DbkVGiCw0THxYG6v8gbQAAAP//AmU9UEAjh8lyBgYGOSj/JAMDw2VopIGyFx8DA4MBUroFZc9GmHcZGBgYAQAAAP//ApmKXHSBsPr///83Eyi6QGDJ////BZH0Mf7//58RAAAA//+CuRA94YLEQJk/GFr6SEDFQRFwnIGBYR0DA8MmJLWQ5MXAwAAAAAD//0I3ENlQGADFOKioAoFX0HQLA6h6GRj+AwAAAP//AwA0SINHgVxAugAAAABJRU5ErkJggg=="/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.4 KiB

8
public/scripts/chats.js
View File

@ -789,7 +789,13 @@ async function openAttachmentManager() {
}
const buttonTemplate = template.find('.actionButtonTemplate .actionButton').clone();
buttonTemplate.find('.actionButtonIcon').addClass(scraper.iconClass);
if (scraper.iconAvailable) {
buttonTemplate.find('.actionButtonIcon').addClass(scraper.iconClass);
buttonTemplate.find('.actionButtonImg').remove();
} else {
buttonTemplate.find('.actionButtonImg').attr('src', scraper.iconClass);
buttonTemplate.find('.actionButtonIcon').remove();
}
buttonTemplate.find('.actionButtonText').text(scraper.name);
buttonTemplate.attr('title', scraper.description);
buttonTemplate.on('click', () => {

27
public/scripts/extensions/attachments/mihoyo-scrape.html Normal file
View File

@ -0,0 +1,27 @@
<div class="flexFlowColumn flex-container">
<div class="range-block-title">
<h3 data-i18n="miHoYo/HoYoverse HoYoLAB Scraper">miHoYo/HoYoverse HoYoLAB Scraper</h3>
</div>
<h4 data-i18n="Select a Wiki to parse through.">Select a Wiki to parse through.</h4>
<div class="range-block-range wide100p">
<select id="mihoyoScrapeWikiDropdown" name="mihoyoScrapeWikiDropdown" class="wide100p">
<option value="">--- None ---</option>
<option value="hsr" data-i18n="Honkai: Star Rail (H:SR)">Honkai: Star Rail (H:SR)</option>
<option value="genshin" data-i18n="Genshin Impact (GI)">Genshin Impact (GI)</option>
</select>
</div>
<div class="range-block-title">
<h4>
<span data-i18n="Enter the Wiki Page ID.">Enter the Wiki Page ID.</span>
</h4>
</div>
<div class="range-block-counter justifyCenter flex-container flexFlowColumn margin-bot-10px">
<span data-i18n="This is the last digit in the HoYoLAB URL i.e.">This is the last digit in the HoYoLAB URL i.e.</span>
<code>https://wiki.hoyolab.com/pc/hsr/entry/X</code>
<small>
<span data-i18n="Example:">Example:</span>
<code>14</code>
</small>
</div>
<input type="text" id="mihoyoScrapeWikiID" name="mihoyoScrapeWikiID" class="text_pole" placeholder="14">
</div>

157
public/scripts/scrapers.js
View File

@ -9,6 +9,7 @@ import { isValidUrl } from './utils.js';
* @property {string} name
* @property {string} description
* @property {string} iconClass
* @property {boolean} iconAvailable
* @property {() => Promise<boolean>} isAvailable
* @property {() => Promise<File[]>} scrape
*/
@ -19,6 +20,7 @@ import { isValidUrl } from './utils.js';
* @property {string} name
* @property {string} description
* @property {string} iconClass
* @property {boolean} iconAvailable
*/
export class ScraperManager {
@ -45,7 +47,7 @@ export class ScraperManager {
* @returns {ScraperInfo[]} List of scrapers available for the Data Bank
*/
static getDataBankScrapers() {
return ScraperManager.#scrapers.map(s => ({ id: s.id, name: s.name, description: s.description, iconClass: s.iconClass }));
return ScraperManager.#scrapers.map(s => ({ id: s.id, name: s.name, description: s.description, iconClass: s.iconClass, iconAvailable: s.iconAvailable}));
}
/**
@ -87,6 +89,7 @@ class Notepad {
this.name = 'Notepad';
this.description = 'Create a text file from scratch.';
this.iconClass = 'fa-solid fa-note-sticky';
this.iconAvailable = true;
}
/**
@ -133,6 +136,7 @@ class WebScraper {
this.name = 'Web';
this.description = 'Download a page from the web.';
this.iconClass = 'fa-solid fa-globe';
this.iconAvailable = true;
}
/**
@ -207,6 +211,7 @@ class FileScraper {
this.name = 'File';
this.description = 'Upload a file from your computer.';
this.iconClass = 'fa-solid fa-upload';
this.iconAvailable = true;
}
/**
@ -243,6 +248,7 @@ class FandomScraper {
this.name = 'Fandom';
this.description = 'Download a page from the Fandom wiki.';
this.iconClass = 'fa-solid fa-fire';
this.iconAvailable = true;
}
/**
@ -339,6 +345,153 @@ class FandomScraper {
}
}
/**
* Scrapes data from the miHoYo/HoYoverse HoYoLAB wiki.
* @implements {Scraper}
*/
class miHoYoScraper {
constructor() {
this.id = 'mihoyo';
this.name = 'miHoYo';
this.description = 'Scrapes a page from the miHoYo/HoYoverse HoYoLAB wiki.';
this.iconClass = 'img/mihoyo.svg';
this.iconAvailable = false; // There is no miHoYo icon in Font Awesome
}
/**
* Check if the scraper is available.
* @returns {Promise<boolean>}
*/
async isAvailable() {
try {
const result = await fetch('/api/plugins/hoyoverse/probe', {
method: 'POST',
headers: getRequestHeaders(),
});
return result.ok;
} catch (error) {
console.debug('Could not probe miHoYo plugin', error);
return false;
}
}
/**
* Outputs Data Information in a human-readable format.
* @param {Object} m Data to be parsed
* @returns {string} Human-readable format of the data
*/
parseOutput(m) {
let temp = '';
for (const d in m) {
if (m[d].key === "") {
temp += `- ${m[d].value}\n`;
continue;
}
temp += `- ${m[d].key}: ${m[d].value}\n`;
}
return temp;
}
/** Scrape data from the miHoYo/HoYoverse HoYoLAB wiki.
* @returns {Promise<File[]>} File attachments scraped from the wiki.
*/
async scrape() {
let miHoYoWiki = '';
let miHoYoWikiID = '';
const template = $(await renderExtensionTemplateAsync('attachments', 'mihoyo-scrape', {}));
template.find('select[name="mihoyoScrapeWikiDropdown"]').on('change', function () {
miHoYoWiki = String($(this).val());
});
template.find('input[name="mihoyoScrapeWikiID"]').on('input', function () {
miHoYoWikiID = String($(this).val());
});
const confirm = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: false, large: false });
if (confirm !== POPUP_RESULT.AFFIRMATIVE) {
return;
}
if (!miHoYoWiki) {
toastr.error('A specific HoYoLab wiki is required');
return;
}
if (!miHoYoWikiID) {
toastr.error('A specific HoYoLab wiki ID is required');
return;
}
if (miHoYoWiki === 'genshin') {
toastr.error('The Genshin Impact parser has not been implemented *yet*');
return;
}
let toast;
if (miHoYoWiki === 'hsr') {
toast = toastr.info(`Scraping the Honkai: Star Rail HoYoLAB wiki for Wiki Entry ID: ${miHoYoWikiID}`);
} else {
toast = toastr.info(`Scraping the Genshin Impact wiki for Wiki Entry ID: ${miHoYoWikiID}`);
}
let result;
if (miHoYoWiki === 'hsr') {
result = await fetch('/api/plugins/hoyoverse/silver-wolf', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ miHoYoWiki, miHoYoWikiID }),
});
} else if (miHoYoWiki === 'genshin') {
result = await fetch('/api/plugins/hoyoverse/furina', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ miHoYoWiki, miHoYoWikiID }),
});
} else {
throw new Error('Unknown wiki name identifier');
}
if (!result.ok) {
const error = await result.text();
throw new Error(error);
}
const data = await result.json();
toastr.clear(toast);
const fileName = data[0].name;
const dataContent = data[0].content;
//parse the data as a long string of data
let combinedContent = '';
combinedContent += `Name: ${data[0].name}\n`;
if (dataContent.description !== "") {
combinedContent += `Description: ${dataContent.description}\n\n`;
}
if (dataContent.modules != []) {
for (const m in dataContent.modules) {
if (dataContent.modules[m].data.length === 0) {
continue;
}
combinedContent += dataContent.modules[m].name + '\n';
combinedContent += this.parseOutput(dataContent.modules[m].data);
combinedContent += '\n';
}
}
const file = new File([combinedContent], `${fileName}.txt`, { type: 'text/plain' });
return [file];
}
}
/**
* Scrape transcript from a YouTube video.
* @implements {Scraper}
@ -349,6 +502,7 @@ class YouTubeScraper {
this.name = 'YouTube';
this.description = 'Download a transcript from a YouTube video.';
this.iconClass = 'fa-solid fa-closed-captioning';
this.iconAvailable = true;
}
/**
@ -413,4 +567,5 @@ ScraperManager.registerDataBankScraper(new FileScraper());
ScraperManager.registerDataBankScraper(new Notepad());
ScraperManager.registerDataBankScraper(new WebScraper());
ScraperManager.registerDataBankScraper(new FandomScraper());
ScraperManager.registerDataBankScraper(new miHoYoScraper());
ScraperManager.registerDataBankScraper(new YouTubeScraper());