Updated game script to align with GameDB API on core. Updated license files to match Github repository. (#82)
This commit is contained in:
Flame Sage
GitHub
@ -1,26 +1,18 @@
|
||||
const fs = require('fs');
|
||||
const fsextra = require('fs-extra');
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
const util = require('util');
|
||||
const logger = require('winston');
|
||||
|
||||
const sanitizeHtml = require('sanitize-html');
|
||||
|
||||
const request = require('request-promise');
|
||||
const toml = require('toml');
|
||||
const tomlify = require('tomlify-j0.4');
|
||||
const blackfriday = require('./blackfriday.js');
|
||||
|
||||
const del = require('delete');
|
||||
const exec = require('sync-exec');
|
||||
|
||||
const fsPathCode = './citra-games-wiki/games';
|
||||
const fsPathWiki = './citra-games-wiki.wiki';
|
||||
const fsPathHugoContent = '../../site/content/game';
|
||||
const fsPathHugoBoxart = '../../site/static/images/game/boxart';
|
||||
const fsPathHugoIcon = '../../site/static/images/game/icons';
|
||||
const fsPathHugoScreenshots = '../../site/static/images/screenshots0';
|
||||
const fsPathHugoSavefiles = '../../site/static/savefiles/';
|
||||
const fsPathCode = './citra-games-wiki/games'
|
||||
const fsPathWiki = './citra-games-wiki.wiki'
|
||||
const fsPathHugoContent = '../../site/content/game'
|
||||
const fsPathHugoBoxart = '../../site/static/images/game/boxart'
|
||||
const fsPathHugoIcon = '../../site/static/images/game/icons'
|
||||
const fsPathHugoScreenshots = '../../site/static/images/screenshots0'
|
||||
const fsPathHugoSavefiles = '../../site/static/savefiles/'
|
||||
|
||||
process.on('unhandledRejection', err => {
|
||||
logger.error('Unhandled rejection on process.');
|
||||
@ -30,290 +22,109 @@ process.on('unhandledRejection', err => {
|
||||
|
||||
function gitPull(directory, repository) {
|
||||
if (fs.existsSync(directory)) {
|
||||
logger.info(`Fetching latest from Github : ${directory}`);
|
||||
logger.info(`git --git-dir=${directory} pull`);
|
||||
exec(`git --git-dir=${directory} pull`);
|
||||
logger.info(`Fetching latest from Github : ${directory}`);
|
||||
logger.info(`git --git-dir=${directory} pull`);
|
||||
exec(`git --git-dir=${directory} pull`);
|
||||
} else {
|
||||
logger.info(`Cloning repository from Github : ${directory}`);
|
||||
logger.info(`git clone ${repository}`);
|
||||
exec(`git clone ${repository}`);
|
||||
logger.info(`Cloning repository from Github : ${directory}`);
|
||||
logger.info(`git clone ${repository}`);
|
||||
exec(`git clone ${repository}`);
|
||||
}
|
||||
}
|
||||
|
||||
function getDirectories (srcpath) {
|
||||
return fs.readdirSync(srcpath)
|
||||
.filter(file => fs.lstatSync(path.join(srcpath, file)).isDirectory())
|
||||
async function getDirectories(x) {
|
||||
return fs.readdir(x).filter(file => fs.lstatSync(path.join(x, file)).isDirectory())
|
||||
}
|
||||
|
||||
async function getGithubIssues() {
|
||||
var results = [];
|
||||
|
||||
// Only loop through the first x pages to prevent API limiting.
|
||||
for (var page = 0; page <= 15; page++) {
|
||||
let options = {
|
||||
url: `https://api.github.com/repos/citra-emu/citra/issues?per_page=99&page=${page}`,
|
||||
headers: { 'User-Agent': 'citrabot' }
|
||||
};
|
||||
|
||||
logger.verbose(`Requesting Github Issue Page ${page}.`);
|
||||
var body = await request.get(options);
|
||||
var obj = JSON.parse(body);
|
||||
|
||||
if (obj == null || obj.length == 0) {
|
||||
logger.verbose(`No more Github issues found -- on page ${page}.`);
|
||||
break;
|
||||
} else {
|
||||
results = results.concat(obj);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async function getTestcases() {
|
||||
let options = {
|
||||
url: 'https://api.citra-emu.org/telemetry/testcase/',
|
||||
headers: { 'User-Agent': 'citrabot' },
|
||||
};
|
||||
|
||||
var body = await request.get(options);
|
||||
return JSON.parse(body).map(x => {
|
||||
return {
|
||||
id: x.id,
|
||||
title: x.title,
|
||||
compatibility: x.compatibility.toString(),
|
||||
date: x.date,
|
||||
version: x.version,
|
||||
buildName: x.buildName,
|
||||
buildDate: x.buildDate,
|
||||
cpu: x.cpu,
|
||||
gpu: x.gpu,
|
||||
os: x.os,
|
||||
author: x.author
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Fetch game information stored in Github repository.
|
||||
gitPull('./citra-games-wiki', 'https://github.com/citra-emu/citra-games-wiki.git');
|
||||
|
||||
// Fetch game articles stored in Github wiki.
|
||||
gitPull('./citra-games-wiki.wiki', 'https://github.com/citra-emu/citra-games-wiki.wiki.git');
|
||||
|
||||
// Fetch all issues from Github.
|
||||
var githubIssues = [];
|
||||
var wikiEntries = {};
|
||||
var testcases = [];
|
||||
|
||||
async function setup() {
|
||||
githubIssues = await getGithubIssues();
|
||||
logger.info(`Imported ${githubIssues.length} issues from Github.`);
|
||||
|
||||
testcases = await getTestcases();
|
||||
logger.info(`Obtained ${testcases.length} testcases from Telemetry API.`);
|
||||
};
|
||||
|
||||
setup().then(function() {
|
||||
async function run() {
|
||||
// Make sure the output directories in Hugo exist.
|
||||
[fsPathHugoContent, fsPathHugoBoxart, fsPathHugoIcon, fsPathHugoSavefiles, fsPathHugoScreenshots].forEach(function (path) {
|
||||
if (fs.existsSync(path) == false) {
|
||||
logger.info(`Creating Hugo output directory: ${path}`);
|
||||
fs.mkdirSync(path);
|
||||
logger.info(`Creating Hugo output directory: ${path}`);
|
||||
fs.ensureDirSync(path);
|
||||
}
|
||||
});
|
||||
}).then(function() {
|
||||
// Transform wiki entries to lowercase
|
||||
const files = fs.readdirSync(fsPathWiki);
|
||||
|
||||
logger.info(`Generating wiki database...`);
|
||||
files.forEach((file) => {
|
||||
wikiEntries[file.toLowerCase()] = file;
|
||||
});
|
||||
// Fetch game files stored in games-wiki repository.
|
||||
gitPull('./citra-games-wiki', 'https://github.com/citra-emu/citra-games-wiki.git');
|
||||
|
||||
// Loop through each game and process it.
|
||||
getDirectories(fsPathCode).forEach(function(game) {
|
||||
processGame(game);
|
||||
});
|
||||
}).catch(function(err) {
|
||||
logger.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
let games = await request.get({ uri: 'https://api.citra-emu.org/gamedb/websiteFeed/', json: true })
|
||||
await Promise.all(games.map(async (x) => {
|
||||
try {
|
||||
logger.info(`Processing game: ${x.id}`);
|
||||
|
||||
function processGame(game) {
|
||||
try {
|
||||
if (game == '.git' || game == '_validation') { return; }
|
||||
// Set metadata.
|
||||
x.date = `${new Date().toISOString()}`
|
||||
|
||||
logger.info(`Processing game: ${game}`);
|
||||
// In Hugo, data keys need to be strings.
|
||||
x.compatibility = x.compatibility.toString()
|
||||
|
||||
// Copy the boxart for the game.
|
||||
fsextra.copySync(`${fsPathCode}/${game}/boxart.png`, `${fsPathHugoBoxart}/${game}.png`);
|
||||
x.testcases.forEach(x => x.compatibility = x.compatibility.toString())
|
||||
|
||||
// Copy the icon for the game.
|
||||
fsextra.copySync(`${fsPathCode}/${game}/icon.png`, `${fsPathHugoIcon}/${game}.png`);
|
||||
// Reverse the testcases so the most recent ones show up top.
|
||||
x.testcases = x.testcases.reverse()
|
||||
x.issues = x.issues || []
|
||||
|
||||
var model = toml.parse(fs.readFileSync(`${fsPathCode}/${game}/game.dat`, 'utf8'));
|
||||
let currentDate = new Date();
|
||||
model.date = `${currentDate.toISOString()}`;
|
||||
// Copy the boxart for the game.
|
||||
fs.copySync(`${fsPathCode}/${x.id}/boxart.png`, `${fsPathHugoBoxart}/${x.id}.png`);
|
||||
|
||||
// Look up all testcases associated with the Title IDs.
|
||||
let releases = model.releases.map(y => y.title);
|
||||
let foundTestcases = testcases.filter(x => {
|
||||
return releases.includes(x.title);
|
||||
});
|
||||
|
||||
// If no testcases exist in the toml file, create a blank array.
|
||||
if (!model.testcases) {
|
||||
model.testcases = [];
|
||||
}
|
||||
// Copy the icon for the game.
|
||||
fs.copySync(`${fsPathCode}/${x.id}/icon.png`, `${fsPathHugoIcon}/${x.id}.png`);
|
||||
|
||||
logger.info(`Found ${foundTestcases.length} testcases from telemetry, found ${model.testcases.length} in toml file.`);
|
||||
|
||||
model.testcases = model.testcases.concat(foundTestcases);
|
||||
|
||||
// Sort the testcases from most recent to least recent.
|
||||
model.testcases.sort(function(a, b){
|
||||
// Turn your strings into dates, and then subtract them
|
||||
// to get a value that is either negative, positive, or zero.
|
||||
return new Date(b.date) - new Date(a.date);
|
||||
});
|
||||
|
||||
// SHORTCUTS BLOCK
|
||||
// Parse testcase information out of the dat to reinject as shortcut values.
|
||||
if (model.testcases == null || model.testcases.length == 0) {
|
||||
model.compatibility = "99";
|
||||
model.testcase_date = "2000-01-01";
|
||||
} else {
|
||||
let recent = model.testcases[0];
|
||||
// The displayed compatibility rating is a weighted arithmetic mean of the submitted ratings.
|
||||
let numerator = 0;
|
||||
let denominator = 0;
|
||||
model.testcases.forEach(testcase => {
|
||||
// Build date is a better metric but testcases from the TOML files only have a submission date.
|
||||
let weigh_date = new Date(testcase.buildDate == undefined ? testcase.date : testcase.buildDate);
|
||||
// The test case is weighted on an exponential decay curve such that a test case is half as relevant as each month (30 days) passes.
|
||||
// The exponent is obtained by dividing the time in millisecond between the current date and date of testing by the number of milliseconds in 30 days.
|
||||
let weight = Math.pow(.5, (currentDate - weigh_date) / (30 * 1000 * 3600 * 24));
|
||||
numerator += testcase.compatibility * weight;
|
||||
denominator += weight;
|
||||
});
|
||||
model.compatibility = Math.round(numerator / denominator).toString();
|
||||
model.testcase_date = recent.date;
|
||||
}
|
||||
|
||||
const toTrim = ["the", "a", "an"];
|
||||
|
||||
let trimmedTitle = model.title.toLowerCase();
|
||||
toTrim.forEach(trim => {
|
||||
if (trimmedTitle.startsWith(trim + " ")) {
|
||||
trimmedTitle = trimmedTitle.substr(trim.length + 1);
|
||||
}
|
||||
});
|
||||
|
||||
let section_id = `${trimmedTitle[0]}`;
|
||||
if (!section_id.match(/[a-z]+/)) {
|
||||
section_id = "#";
|
||||
}
|
||||
|
||||
model.section_id = section_id;
|
||||
// END SHORTCUTS BLOCK
|
||||
|
||||
// SAVEFILE BLOCK
|
||||
var fsPathCodeSavefilesGame = `${fsPathCode}/${game}/savefiles/`;
|
||||
var fsPathHugoSavefilesGame = `${fsPathHugoSavefiles}/${game}/`;
|
||||
if (fs.existsSync(fsPathCodeSavefilesGame)) {
|
||||
// Create the savefile directory for the game.
|
||||
if (fs.existsSync(fsPathHugoSavefilesGame) == false) {
|
||||
// SAVEFILE BLOCK
|
||||
var fsPathCodeSavefilesGame = `${fsPathCode}/${x.id}/savefiles/`;
|
||||
var fsPathHugoSavefilesGame = `${fsPathHugoSavefiles}/${x.id}/`;
|
||||
if (fs.existsSync(fsPathCodeSavefilesGame)) {
|
||||
// Create the savefile directory for the game.
|
||||
if (fs.existsSync(fsPathHugoSavefilesGame) == false) {
|
||||
fs.mkdirSync(fsPathHugoSavefilesGame);
|
||||
}
|
||||
|
||||
// Copy all savefiles into the output folder, and read their data.
|
||||
fs.readdirSync(fsPathCodeSavefilesGame).forEach(file => {
|
||||
if (path.extname(file) == '.zip') {
|
||||
fs.copySync(`${fsPathCodeSavefilesGame}/${file}`, `${fsPathHugoSavefilesGame}/${file}`);
|
||||
}
|
||||
});
|
||||
// Finished copying all savefiles into the output folder, and reading their data.
|
||||
}
|
||||
// END SAVEFILE BLOCK
|
||||
|
||||
// Copy all savefiles into the output folder, and read their data.
|
||||
model.savefiles = [];
|
||||
fs.readdirSync(fsPathCodeSavefilesGame).forEach(file => {
|
||||
if (path.extname(file) == '.zip') {
|
||||
fsextra.copySync(`${fsPathCodeSavefilesGame}/${file}`, `${fsPathHugoSavefilesGame}/${file}`);
|
||||
} else if (path.extname(file) == '.dat') {
|
||||
// Read the data file into an object.
|
||||
let savefile = toml.parse(fs.readFileSync(`${fsPathCodeSavefilesGame}/${file}`, 'utf8'));
|
||||
|
||||
let stats = fs.statSync(`${fsPathCodeSavefilesGame}/${file}`);
|
||||
|
||||
// Store the contents of the file in memory for adding it into the markdown later.
|
||||
model.savefiles.push({
|
||||
date: new Date(util.inspect(stats.mtime)),
|
||||
filename: file.replace('.dat', '.zip'),
|
||||
title: savefile.title,
|
||||
description: savefile.description,
|
||||
author: savefile.author,
|
||||
title_id: savefile.title_id
|
||||
});
|
||||
}
|
||||
});
|
||||
// Finished copying all savefiles into the output folder, and reading their data.
|
||||
}
|
||||
// END SAVEFILE BLOCK
|
||||
|
||||
// GITHUB ISSUES BLOCK
|
||||
model.issues = [];
|
||||
model.closed_issues = [];
|
||||
|
||||
if (model.github_issues != null && model.github_issues.length > 0) {
|
||||
model.github_issues.forEach(function(number) {
|
||||
let issue = githubIssues.find(x => x.number == number);
|
||||
if (issue == null) {
|
||||
model.closed_issues.push(number);
|
||||
} else {
|
||||
model.issues.push({
|
||||
number: issue.number.toString(),
|
||||
title: issue.title,
|
||||
state: issue.state,
|
||||
created_at: issue.created_at,
|
||||
updated_at: issue.updated_at,
|
||||
labels: issue.labels.map(function(x) { return { name: x.name, color: x.color } })
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
github_issues = null;
|
||||
}
|
||||
// END GITHUB ISSUES BLOCK
|
||||
|
||||
// Copy the screenshots for the game.
|
||||
let fsPathScreenshotInputGame = `${fsPathCode}/${game}/screenshots/`;
|
||||
let fsPathScreenshotOutputGame = `${fsPathHugoScreenshots}/${game}/`;
|
||||
if (fs.existsSync(fsPathScreenshotInputGame)) {
|
||||
// Create the savefile directory for each game.
|
||||
if (fs.existsSync(fsPathScreenshotOutputGame) == false) {
|
||||
// Copy the screenshots for the game.
|
||||
let fsPathScreenshotInputGame = `${fsPathCode}/${x.id}/screenshots/`;
|
||||
let fsPathScreenshotOutputGame = `${fsPathHugoScreenshots}/${x.id}/`;
|
||||
if (fs.existsSync(fsPathScreenshotInputGame)) {
|
||||
// Create the savefile directory for each game.
|
||||
if (fs.existsSync(fsPathScreenshotOutputGame) == false) {
|
||||
fs.mkdirSync(fsPathScreenshotOutputGame);
|
||||
}
|
||||
|
||||
// Copy all screenshots into the output folder.
|
||||
fs.readdirSync(fsPathScreenshotInputGame).forEach(file => {
|
||||
if (path.extname(file) == '.png') {
|
||||
fs.copySync(`${fsPathScreenshotInputGame}/${file}`, `${fsPathScreenshotOutputGame}/${file}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Copy all screenshots into the output folder.
|
||||
fs.readdirSync(fsPathScreenshotInputGame).forEach(file => {
|
||||
if (path.extname(file) == '.png') {
|
||||
fsextra.copySync(`${fsPathScreenshotInputGame}/${file}`, `${fsPathScreenshotOutputGame}/${file}`);
|
||||
}
|
||||
});
|
||||
// Clear out the wiki markdown so it won't be stored with the metadata.
|
||||
let wikiText = x.wiki_markdown
|
||||
x.wiki_markdown = null
|
||||
|
||||
let meta = tomlify(x, null, 2);
|
||||
let contentOutput = `+++\r\n${meta}\r\n+++\r\n\r\n${wikiText}\r\n`;
|
||||
|
||||
await fs.writeFile(`${fsPathHugoContent}/${x.id}.md`, contentOutput);
|
||||
|
||||
return x
|
||||
} catch (ex) {
|
||||
logger.warn(`${x.id} ${x.title} failed to generate: ${ex}`);
|
||||
logger.error(ex);
|
||||
throw ex
|
||||
}
|
||||
|
||||
// WIKI BLOCK
|
||||
var wikiText = "";
|
||||
let fsPathWikiGame = `${fsPathWiki}/${wikiEntries[game.toLowerCase() + ".md"]}`;
|
||||
if (fs.existsSync(fsPathWikiGame)) {
|
||||
wikiText = fs.readFileSync(fsPathWikiGame, 'utf8');
|
||||
|
||||
// Fix Blackfriday markdown rendering differences.
|
||||
wikiText = blackfriday.fixLists(wikiText);
|
||||
wikiText = blackfriday.fixLinks(wikiText);
|
||||
} else {
|
||||
wikiText = "## No wiki exists yet for this game.";
|
||||
}
|
||||
// END WIKI BLOCK
|
||||
|
||||
let modelText = tomlify(model, null, 2);
|
||||
|
||||
let contentOutput = `+++\r\n${modelText}\r\n+++\r\n\r\n${wikiText}\r\n`;
|
||||
fs.writeFileSync(`${fsPathHugoContent}/${game}.md`, contentOutput);
|
||||
} catch (ex) {
|
||||
logger.warn(`${game} failed to generate: ${ex}`);
|
||||
logger.error(ex);
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
// Script start
|
||||
run()
|
||||
|
||||
Reference in New Issue
Block a user