2022-12-12 23:40:40 +01:00
|
|
|
#!/usr/bin/env -S node --experimental-fetch
|
2023-05-07 16:35:05 +02:00
|
|
|
require('./Lib/Syncers.js').importAll();
|
2022-12-12 23:40:40 +01:00
|
|
|
const JSDOM = require('jsdom').JSDOM;
|
|
|
|
|
|
2022-12-14 18:29:28 +01:00
|
|
|
const BlogURL = 'https://listed.to/@u8'; // Full base URL of the Listed blog (any server)
|
2022-12-12 23:40:40 +01:00
|
|
|
const SiteName = 'sitoctt';
|
|
|
|
|
//const DefaultMode = 'Include' // 'Include' or 'Exclude' | Not implemented
|
2022-12-14 18:29:28 +01:00
|
|
|
const PostsFileDate = true; // Append dates (YYYY-MM-DD) to posts file names
|
2022-12-26 15:34:37 +01:00
|
|
|
let Replacements = { // Format: { ReplaceWithString: [ToFindString] }
|
2022-12-17 10:54:21 +01:00
|
|
|
"<h2>[:HNotesRefsHTML:]</h2>": "<h2>🏷️ Note e Riferimenti</h2>",
|
|
|
|
|
'<div class="footnotes">': ['<div class="footnotes"><hr>', '<div class="footnotes">\n<hr>'],
|
2022-12-24 16:17:26 +01:00
|
|
|
'"><a class="footnote-ref" href="#fn': '"><a href="#fn',
|
|
|
|
|
" href=\"[staticoso:CustomPath:Assets]/": " href=\"https://sitoctt-assets.octt.eu.org/",
|
|
|
|
|
" src=\"[staticoso:CustomPath:Assets]/": " src=\"https://sitoctt-assets.octt.eu.org/",
|
2023-03-19 15:55:54 +01:00
|
|
|
'<div class="highlight CodeScroll">': '<div class="highlight">',
|
2023-03-20 23:42:22 +01:00
|
|
|
//'<iframe src="[:YouTubeURL:]/' :'<iframe src="https://www.youtube-nocookie.com/',
|
|
|
|
|
'<video src="[:YouTube360:]' :'<iframe src="https://www.youtube-nocookie.com/embed/',
|
|
|
|
|
'></video>': ' class="YouTube"></iframe>',
|
2022-12-14 18:29:28 +01:00
|
|
|
// TODO: Fix anchor rels
|
2022-12-12 23:40:40 +01:00
|
|
|
};
|
2022-12-26 15:34:37 +01:00
|
|
|
const TestURL = 'https://listed.to/p/hDaMhJ2ts7';
|
2022-12-12 23:40:40 +01:00
|
|
|
|
2022-12-14 18:29:28 +01:00
|
|
|
const MetadataBlockSelect = '.MetadataBlock, .MetadataBlock + :Where(Div, Pre, Code)';
|
2022-12-26 15:34:37 +01:00
|
|
|
const ReplacementsBlockSelect = '.ReplacementsBlock, .ReplacementsBlock + :Where(Div, Pre, Code)';
|
2022-12-14 18:29:28 +01:00
|
|
|
const ExtractCodeBlockSelect = '.ExtractCodeBlock, .ExtractCodeBlock + :Where(Div, Pre, Code)';
|
2022-12-26 15:34:37 +01:00
|
|
|
const DeleteElementBlockSelect = '.DeleteElementBlock';
|
2022-12-14 18:29:28 +01:00
|
|
|
|
2022-12-12 23:40:40 +01:00
|
|
|
const TryReadFileSync = Path => {
|
2023-05-07 16:35:05 +02:00
|
|
|
if (Fs.existsSync(Path)) {
|
|
|
|
|
return Fs.readFileSync(Path, 'utf8');
|
2022-12-12 23:40:40 +01:00
|
|
|
};
|
|
|
|
|
};
|
2022-12-17 10:54:21 +01:00
|
|
|
|
2022-12-12 23:40:40 +01:00
|
|
|
const TryMkdirSync = Path => {
|
2023-05-07 16:35:05 +02:00
|
|
|
if (!Fs.existsSync(Path)) {
|
|
|
|
|
return Fs.mkdirSync(Path, {recursive: true});
|
2022-12-12 23:40:40 +01:00
|
|
|
};
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const GetPath = URL => {
|
|
|
|
|
if (URL.startsWith('http://') || URL.startsWith('https://')) {
|
|
|
|
|
URL = URL.split('/').slice(3).join('/');
|
|
|
|
|
};
|
|
|
|
|
return URL;
|
|
|
|
|
};
|
|
|
|
|
|
2022-12-14 18:29:28 +01:00
|
|
|
const GetFragHTML = Frag => {
|
|
|
|
|
let Dom = new JSDOM('<body></body>');
|
|
|
|
|
Dom.window.document.body.appendChild(Frag);
|
2022-12-17 10:54:21 +01:00
|
|
|
return Dom.window.document.body.innerHTML.trim();
|
2022-12-14 18:29:28 +01:00
|
|
|
};
|
|
|
|
|
|
2022-12-26 15:34:37 +01:00
|
|
|
const CSSFirstTokenSelector = Select => {
|
|
|
|
|
return Select.trim().replaceAll('.', '').replaceAll(',', '').split(' ')[0];
|
|
|
|
|
};
|
|
|
|
|
|
2022-12-17 10:54:21 +01:00
|
|
|
const CheckDownsync = Body => {
|
|
|
|
|
if (Body) {
|
|
|
|
|
const Lines = Body.trim().toLowerCase().split('\n');
|
2022-12-12 23:40:40 +01:00
|
|
|
for (let i=0; i<Lines.length; i++) {
|
2022-12-17 10:54:21 +01:00
|
|
|
const Line = Lines[i].trim()
|
|
|
|
|
const CheckLine = Line.replaceAll(' ', ' ').replaceAll(':', ' : ').replaceAll('=', ' = ');
|
|
|
|
|
if (CheckLine.startsWith('// ')) {
|
|
|
|
|
const Tokens = CheckLine.split(' ').filter(i => {return i != ''});
|
|
|
|
|
if (Tokens[1] == '%' && Tokens[2] == 'downsync' && [':', '='].includes(Tokens[3])) {
|
2022-12-24 16:17:26 +01:00
|
|
|
if (['false', 'disabled', 'off', 'no', '0'].includes(Tokens[4])) {
|
2022-12-17 10:54:21 +01:00
|
|
|
return false;
|
|
|
|
|
} else if (Tokens[4].startsWith('/')) {
|
|
|
|
|
return Line.substring(Line.indexOf('/', 2));
|
|
|
|
|
};
|
2022-12-12 23:40:40 +01:00
|
|
|
};
|
|
|
|
|
};
|
|
|
|
|
};
|
|
|
|
|
};
|
2022-12-17 10:54:21 +01:00
|
|
|
return true;
|
2022-12-12 23:40:40 +01:00
|
|
|
};
|
|
|
|
|
|
2022-12-14 18:29:28 +01:00
|
|
|
const GetLinkElem = Dom => {
|
2022-12-12 23:40:40 +01:00
|
|
|
let Elem;
|
|
|
|
|
Elem = Dom.querySelector(`.Mirror-${SiteName}`);
|
|
|
|
|
if (!Elem) {
|
|
|
|
|
Elem = Dom.querySelector(`.Mirror-${SiteName}-Include`);
|
|
|
|
|
};
|
2022-12-14 18:29:28 +01:00
|
|
|
return Elem;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const MakeMetaStr = Post => {
|
|
|
|
|
let Str = '';
|
2023-05-07 16:35:05 +02:00
|
|
|
['Meta', 'Macros'].forEach((Type) => {
|
|
|
|
|
const Marks = { Meta: "%", Macros: "$", };
|
|
|
|
|
Object.keys(Post[Type]).forEach((Key) => {
|
|
|
|
|
Str += `// ${Marks[Type]} ${Key} = ${Post[Type][Key]}\n`;
|
|
|
|
|
});
|
|
|
|
|
});
|
2022-12-14 18:29:28 +01:00
|
|
|
return Str;
|
|
|
|
|
};
|
|
|
|
|
|
2022-12-26 15:34:37 +01:00
|
|
|
const HandlePost = (PostSrc, Output) => {
|
2022-12-17 10:54:21 +01:00
|
|
|
let ContentDom, LinkPath;
|
|
|
|
|
let Post = {'Meta': {}, 'Macros': {}};
|
2022-12-14 18:29:28 +01:00
|
|
|
|
|
|
|
|
Post.Meta.Title = PostSrc.title;
|
|
|
|
|
Post.Meta.CreatedOn = PostSrc.created_at.split('T')[0];
|
|
|
|
|
Post.Content = PostSrc.rendered_text;
|
|
|
|
|
console.log(`[I] => [${Post.Meta.CreatedOn}] ${Post.Meta.Title}`);
|
|
|
|
|
|
|
|
|
|
ContentDom = JSDOM.fragment(Post.Content);
|
|
|
|
|
|
2022-12-26 15:34:37 +01:00
|
|
|
// Handle .MetadataBlock elements
|
2023-05-07 16:35:05 +02:00
|
|
|
Array.from(ContentDom.querySelectorAll(MetadataBlockSelect)).forEach((Elem) => {
|
2022-12-17 10:54:21 +01:00
|
|
|
if (Elem.textContent) {
|
|
|
|
|
const Meta = ParseMeta(Elem.textContent);
|
|
|
|
|
Post.Meta = Object.assign(Post.Meta, Meta.Meta);
|
|
|
|
|
Post.Macros = Object.assign(Post.Macros, Meta.Macros);
|
|
|
|
|
};
|
2023-05-07 16:35:05 +02:00
|
|
|
Elem.outerHTML = '';
|
|
|
|
|
});
|
2022-12-17 10:54:21 +01:00
|
|
|
// NOTE: Maybe would be better to first do string replacements?
|
|
|
|
|
|
|
|
|
|
let LinkElem = GetLinkElem(ContentDom);
|
|
|
|
|
if (LinkElem) {
|
|
|
|
|
LinkPath = GetPath(JSDOM.fragment(LinkElem.outerHTML).querySelector('[href]').href);
|
|
|
|
|
LinkElem.outerHTML = '';
|
|
|
|
|
} else {
|
|
|
|
|
let Check = Post.Meta.Downsync;
|
|
|
|
|
if (typeof(Check) == 'string' && Check.startsWith('/')) {
|
|
|
|
|
LinkPath = Check.substring(1);
|
|
|
|
|
} else {
|
|
|
|
|
console.log(`[I] : No Downsync flag set with URL in source body; Skipping!`);
|
|
|
|
|
return;
|
2022-12-12 23:40:40 +01:00
|
|
|
};
|
|
|
|
|
};
|
2022-12-17 10:54:21 +01:00
|
|
|
|
2022-12-26 15:34:37 +01:00
|
|
|
// Handle .ReplacementsBlock elements: Add replacements to do to the default ones or override them.
|
2023-05-07 16:35:05 +02:00
|
|
|
Array.from(ContentDom.querySelectorAll(ReplacementsBlockSelect)).forEach((Elem) => {
|
2022-12-26 15:34:37 +01:00
|
|
|
let Text = Elem.textContent.trim();
|
|
|
|
|
if (Text) {
|
|
|
|
|
if (!(Text.startsWith('{') && Text.endsWith('}'))) {
|
|
|
|
|
Text = `{${Text}}`;
|
|
|
|
|
};
|
|
|
|
|
try {
|
|
|
|
|
Replacements = Object.assign(Replacements, JSON.parse(Text));
|
|
|
|
|
} catch(e) {
|
|
|
|
|
console.log(`[W] : Problem parsing JSON in a ReplacementsBlock; Ignoring!`);
|
|
|
|
|
};
|
|
|
|
|
};
|
2023-05-07 16:35:05 +02:00
|
|
|
Elem.outerHTML = '';
|
|
|
|
|
});
|
2022-12-26 15:34:37 +01:00
|
|
|
|
2022-12-17 10:54:21 +01:00
|
|
|
Post.Content = GetFragHTML(ContentDom);
|
|
|
|
|
|
|
|
|
|
const PathFile = LinkPath.split('/').slice(-1)[0];
|
|
|
|
|
const PathDir = LinkPath.split('/').slice(0, (LinkPath.split('/').length - 1)).join('/');
|
|
|
|
|
const DatePrefix = PostsFileDate ? Post.Meta.CreatedOn + '-' : '';
|
|
|
|
|
const FinalFilePath = `${PathDir}/${DatePrefix}${PathFile.substring(0, (PathFile.length - 4))}md`;
|
|
|
|
|
if (!CheckDownsync(TryReadFileSync(FinalFilePath))) {
|
|
|
|
|
console.log(`[I] : Downsync disabled in destination body; Skipping!`);
|
|
|
|
|
return;
|
|
|
|
|
};
|
2022-12-12 23:40:40 +01:00
|
|
|
|
2022-12-14 18:29:28 +01:00
|
|
|
// Do string replacements
|
2022-12-17 10:54:21 +01:00
|
|
|
// TODO: Replacements written in post body?
|
2023-05-07 16:35:05 +02:00
|
|
|
Object.keys(Replacements).forEach((To) => {
|
|
|
|
|
let FromList = Replacements[To];
|
|
|
|
|
if (typeof(FromList) != 'object') {
|
|
|
|
|
FromList = [FromList];
|
2022-12-12 23:40:40 +01:00
|
|
|
};
|
2023-05-07 16:35:05 +02:00
|
|
|
FromList.forEach((From) => {
|
|
|
|
|
Post.Content = Post.Content.replaceAll(From, To);
|
|
|
|
|
});
|
|
|
|
|
});
|
2022-12-14 18:29:28 +01:00
|
|
|
|
|
|
|
|
ContentDom = JSDOM.fragment(Post.Content);
|
|
|
|
|
|
2022-12-26 15:34:37 +01:00
|
|
|
// Handle .DeleteElementBlock elements: Elements that must be visible on Listed but deleted here.
|
2023-05-07 16:35:05 +02:00
|
|
|
Array.from(ContentDom.querySelectorAll(DeleteElementBlockSelect)).forEach((Elem) => {
|
2022-12-26 15:34:37 +01:00
|
|
|
if (!Elem.textContent) {
|
2023-05-07 16:35:05 +02:00
|
|
|
Elem.nextElementSibling.outerHTML = '';
|
2022-12-26 15:34:37 +01:00
|
|
|
};
|
2023-05-07 16:35:05 +02:00
|
|
|
Elem.outerHTML = '';
|
|
|
|
|
});
|
2022-12-26 15:34:37 +01:00
|
|
|
|
|
|
|
|
// Handle .ExtractCodeBlock elements: Allow for text to be treated as plain on Listed, and then extracted here.
|
2023-05-07 16:35:05 +02:00
|
|
|
Array.from(ContentDom.querySelectorAll(ExtractCodeBlockSelect)).forEach((Elem) => {
|
2022-12-26 15:34:37 +01:00
|
|
|
const Find = CSSFirstTokenSelector(ExtractCodeBlockSelect);
|
2022-12-14 18:29:28 +01:00
|
|
|
if (Array.from(Elem.classList).includes(Find)) {
|
2023-05-07 16:35:05 +02:00
|
|
|
// Remove the ExtractCodeBlock upper-marker
|
|
|
|
|
Elem.outerHTML = '';
|
2022-12-14 18:29:28 +01:00
|
|
|
} else {
|
2023-05-07 16:35:05 +02:00
|
|
|
// Extract the marker's text as raw HTML
|
|
|
|
|
Elem.outerHTML = Elem.textContent;
|
2022-12-12 23:40:40 +01:00
|
|
|
};
|
2023-05-07 16:35:05 +02:00
|
|
|
});
|
2022-12-12 23:40:40 +01:00
|
|
|
|
2022-12-17 10:54:21 +01:00
|
|
|
Post.Content = GetFragHTML(ContentDom);
|
2022-12-14 18:29:28 +01:00
|
|
|
|
2022-12-26 15:34:37 +01:00
|
|
|
if (Output == 'file') {
|
|
|
|
|
TryMkdirSync(PathDir);
|
2023-05-07 16:35:05 +02:00
|
|
|
Fs.writeFileSync(FinalFilePath, `\
|
2023-03-19 00:22:52 +01:00
|
|
|
<!-- < Autogenerated by ListedDownsync.js. Do not edit (unless also set "% Downsync = False") - it would be overwritten. > -->
|
2022-12-14 18:29:28 +01:00
|
|
|
${MakeMetaStr(Post)}
|
2022-12-17 10:54:21 +01:00
|
|
|
<h1>${Post.Meta.HTMLTitle ? Post.Meta.HTMLTitle : Post.Meta.Title}</h1>
|
2022-12-12 23:40:40 +01:00
|
|
|
|
2022-12-14 18:29:28 +01:00
|
|
|
${Post.Content}
|
2022-12-12 23:40:40 +01:00
|
|
|
`);
|
2022-12-26 15:34:37 +01:00
|
|
|
} else if (Output == 'stdout') {
|
|
|
|
|
console.log(Post.Content);
|
|
|
|
|
};
|
2022-12-12 23:40:40 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const Main = _ => {
|
|
|
|
|
console.log('[I] Downloading...');
|
|
|
|
|
fetch(`${BlogURL}/all`).then(Response => Response.text()).then(Data => {
|
|
|
|
|
console.log('[I] Parsing...');
|
|
|
|
|
const Elem = JSDOM.fragment(Data).querySelector('script[data-component-name="AuthorAll"]');
|
|
|
|
|
const Posts = JSON.parse(Elem.childNodes[0].data).posts;
|
|
|
|
|
for (let i=0; i<Posts.length; i++) {
|
2022-12-26 15:34:37 +01:00
|
|
|
HandlePost(Posts[i], 'file');
|
2022-12-12 23:40:40 +01:00
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
};
|
|
|
|
|
|
2022-12-26 15:34:37 +01:00
|
|
|
const Test = _ => {
|
|
|
|
|
console.log('[I] Testing...');
|
|
|
|
|
fetch(TestURL).then(Response => Response.text()).then(Data => {
|
|
|
|
|
const Elem = JSDOM.fragment(Data).querySelector('script[data-component-name="PostShow"]');
|
|
|
|
|
const Post = JSON.parse(Elem.childNodes[0].data).post;
|
|
|
|
|
HandlePost(Post, 'stdout');
|
|
|
|
|
});
|
|
|
|
|
};
|
|
|
|
|
|
2022-12-12 23:40:40 +01:00
|
|
|
Main();
|
2022-12-26 15:34:37 +01:00
|
|
|
//Test();
|
