octospacc/sitoctt
1
0
Fork 0
mirror of https://gitlab.com/octtspacc/sitoctt synced 2025-06-05 22:09:20 +02:00
Code Issues Projects Releases Wiki Activity

Agg. Posts/Blogging-Rapido-con-Standard-Notes.md, ListedDownsync.js

Browse Source
This commit is contained in:
octospacc
2022-12-14 18:29:28 +01:00
parent 78c7abe658
commit 84638a9ed6
3 changed files with 177 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
Scripts/Lib/config-ini-parser.js Normal file
View File

@ -0,0 +1,9 @@
/**
* Created by Erxin, Shang(Edwin) on 6/21/2016.
* JavaScript Configuration file(.ini) content parser, similar to python ConfigParser without I/O operations
* The license is under GPL-3.0
* Git repo:https://github.com/shangerxin/config-ini
* Author homepage: http://www.shangerxin.com
* Version, 1.6.1
*/
!function(t){var n=Object.assign(new Error,{name:"ConfigIniParser Error",message:"Parse config ini file error"}),r=Object.assign(new Error,{name:"ConfigIniParser Error",message:"The specify section not found"}),e=Object.assign(new Error,{name:"ConfigIniParser Error",message:"The specify option not found"}),i=Object.assign(new Error,{name:"ConfigIniParser Error",message:"Found duplicated section in the given ini file"}),o=Object.assign(new Error,{name:"ConfigIniParser Error",message:"Multiple call parse on the same parser instance"}),s=Object.assign(new TypeError,{name:"ConfigIniParser Error",message:"The argument type is not correct"}),a="__DEFAULT_SECTION__",u=/^\s*\[\s*([^\]]+?)\s*\]\s*$/,p=/\s*([^=:\s]+)\s*[=:]\s*(.*)\s*/,f=/^\s*[#;].*/,c=/^\s*$/,h=-1;function l(t,n){for(var r=t.sections,e=0;e<r.length;e++){var i=r[e];if(i.name==n)return i}}function m(t,n){for(var r=t.options,e=0;e<r.length;e++){var i=r[e];if(i.name==n)return i}}function g(t){return{name:t,options:[]}}function v(t,n){return{name:t,value:n}}var y=function(t){this.delimiter=t||"\n",this._calledParse=!1,this._ini={sections:[]},this._ini.sections.push(g(a))};y.prototype.addSection=function(t){if(l(this._ini,t))throw i;var n=g(t);return this._ini.sections.push(n),this},y.prototype.get=function(t,n,r){var i=l(this._ini,t||a);if(i){var o=m(i,n);if(o)return o.value}if(void 0===r)throw e;return r},y.prototype.getOptionFromDefaultSection=function(t,n){return this.get(null,t,n)},y.prototype.getBoolean=function(t,n){var r=this.get(t||a,n);return isNaN(r)?"true"==String(r).toLowerCase():0!=r},y.prototype.getBooleanFromDefaultSection=function(t){return this.getBoolean(null,t)},y.prototype.getNumber=function(t,n){return+this.get(t||a,n)},y.prototype.getNumberFromDefaultSection=function(t){return this.getNumber(null,t)},y.prototype.isHaveSection=function(t){if(!t)throw s;return!!l(this._ini,t)},y.prototype.isHaveOption=function(t,n){var r=l(this._ini,t||a);if(r&&m(r,n))return!0;return!1},y.prototype.isHaveOptionInDefaultSection=function(t){return this.isHaveOption(null,t)},y.prototype.items=function(t){for(var n=l(this._ini,t||a),r=[],e=0;e<n.options.length;e++){var i=n.options[e];r.push([i.name,i.value])}return r},y.prototype.options=function(t){var n=l(this._ini,t||a);if(n){for(var e,i=[],o=n.options,s=0;s<o.length;s++)e=o[s],i.push(e.name);return i}throw r},y.prototype.removeOption=function(t,n){var r=l(this._ini,t||a);if(r){var e=function(t,n){for(var r=t.options,e=0;e<r.length;e++)if(r[e].name==n)return e;return h}(r,n);if(e!=h)return r.options.splice(e,1),!0}return!1},y.prototype.removeOptionFromDefaultSection=function(t){return this.removeOption(null,t)},y.prototype.removeSection=function(t){var n=function(t,n){for(var r=t.sections,e=0;e<r.length;e++)if(r[e].name==n)return e;return h}(this._ini,t||a);return n!=h&&(this._ini.sections.splice(n,1),t===a&&this._ini.sections.push(g(a)),!0)},y.prototype.sections=function(){for(var t,n=[],r=this._ini.sections,e=0;e<r.length;e++)(t=r[e]).name!=a&&n.push(t.name);return n},y.prototype.set=function(t,n,e){var i,o=l(this._ini,t||a);if(o)return(i=m(o,n))?(i.value=e,this):(i=v(n,e),o.options.push(i),this);throw r},y.prototype.setOptionInDefaultSection=function(t,n){return this.set(null,t,n)},y.prototype.stringify=function(t){for(var n,r,e,i=[],o=this._ini.sections,s=0;s<o.length;s++){(n=o[s]).name!=a&&i.push("["+n.name+"]"),r=n.options;for(var u=0;u<r.length;u++)e=r[u],i.push(e.name+"="+e.value);i.length>0&&i.push("")}return i.join(t||this.delimiter)},y.prototype.parse=function(t){if(this._calledParse)throw o;this._calledParse=!0;for(var r=t.split(this.delimiter),e=l(this._ini,a),s=0;s<r.length;s++){var h=r[s];if(!h.match(f)&&!h.match(c)){var m=h.match(u);if(m){var y=m[1];if(l(this._ini,y))throw i;e=g(y),this._ini.sections.push(e)}else{var _=h.match(p);if(!_)throw n;var E=v(_[1],_[2]);e.options.push(E)}}}return this},y.Errors={Error:n,ErrorNoSection:r,ErrorNoOption:e,ErrorDuplicateSectionError:i,ErrorCallParseMultipleTimes:o,ErrorIncorrectArgumentType:s},t.ConfigIniParser=y}("undefined"!=typeof exports?exports:"undefined"!=typeof window?window:{});

176
Scripts/ListedDownsync.js
View File

@ -1,20 +1,21 @@
#!/usr/bin/env -S node --experimental-fetch
const fs = require('fs');
const JSDOM = require('jsdom').JSDOM;
const ConfigParser = require("./Lib/config-ini-parser").ConfigIniParser;
const BlogURL = 'https://listed.to/@u8';
const BlogURL = 'https://listed.to/@u8'; // Full base URL of the Listed blog (any server)
const SiteName = 'sitoctt';
//const DefaultMode = 'Include' // 'Include' or 'Exclude' | Not implemented
const PostsFileDate = true; // Append date to posts file names
const Replacements = {
const PostsFileDate = true; // Append dates (YYYY-MM-DD) to posts file names
const Replacements = { // Format: { ReplaceWithString: [ToFindString] }
"<a href=\"[staticoso:CustomPath:Assets]/": "<a href=\"https://sitoctt-assets.octt.eu.org/",
"<img src=\"[staticoso:CustomPath:Assets]/": "<img src=\"https://sitoctt-assets.octt.eu.org/",
//"[staticoso:CustomPath:Assets]/": [
// "<a href=\"https://sitoctt-assets.octt.eu.org/",
// "<img src=\"https://sitoctt-assets.octt.eu.org/"
//]
// TODO: Fix anchor rels
};
const MetadataBlockSelect = '.MetadataBlock, .MetadataBlock + :Where(Div, Pre, Code)';
const ExtractCodeBlockSelect = '.ExtractCodeBlock, .ExtractCodeBlock + :Where(Div, Pre, Code)';
const TryReadFileSync = Path => {
if (fs.existsSync(Path)) {
return fs.readFileSync(Path, 'utf8');
@ -22,7 +23,7 @@ const TryReadFileSync = Path => {
};
const TryMkdirSync = Path => {
if (!fs.existsSync(Path)) {
return fs.mkdirSync(Path, {recursive:true});
return fs.mkdirSync(Path, {recursive: true});
};
};
@ -33,6 +34,12 @@ const GetPath = URL => {
return URL;
};
const GetFragHTML = Frag => {
let Dom = new JSDOM('<body></body>');
Dom.window.document.body.appendChild(Frag);
return Dom.window.document.body.innerHTML;
};
const CheckDoDownsync = File => {
let DoDownsync = true;
const TryFile = TryReadFileSync(File);
@ -41,11 +48,10 @@ const CheckDoDownsync = File => {
for (let i=0; i<Lines.length; i++) {
const Line = Lines[i].trim().replaceAll(' ', ' ').replaceAll(':', ' : ').replaceAll('=', ' = ');
if (Line.startsWith('// ')) {
const Tokens = Line.split(' ').filter(e => {return e != ''});
if (Tokens[1] == '%' && Tokens[2] == 'downsync') {
if (['false', 'disabled', 'off', 'no'].includes(Tokens[4])) {
DoDownsync = false;
};
const Tokens = Line.split(' ').filter(i => {return i != ''});
if (Tokens[1] == '%' && Tokens[2] == 'downsync' && [':', '='].includes(Tokens[3]) && ['false', 'disabled', 'off', 'no'].includes(Tokens[4])) {
DoDownsync = false;
break;
};
};
};
@ -53,27 +59,89 @@ const CheckDoDownsync = File => {
return DoDownsync;
};
const HandlePost = PostSrc => {
const GetLinkElem = Dom => {
let Elem;
let Post = {};
Post['Title'] = PostSrc.title;
Post['CreatedOn'] = PostSrc.created_at.split('T')[0];
console.log(`[I] => [${Post['CreatedOn']}] ${Post['Title']}`);
const Dom = JSDOM.fragment(PostSrc.rendered_text);
Elem = Dom.querySelector(`.Mirror-${SiteName}`);
if (!Elem) {
Elem = Dom.querySelector(`.Mirror-${SiteName}-Include`);
};
if (!Elem) { // Post content has no mirror-flagging element, skip it
console.log(`[I] : No mirror flag in source body; Skipping!`);
return; // TODO: Exclusion mode instead of inclusion? Aka automatically handle posts without the element
};
return Elem;
};
const Path = GetPath(JSDOM.fragment(Elem.outerHTML).querySelector('[href]').href);
Post['Categories'] = '';
const Classes = Elem.classList;
const ParseMeta = Raw => {
let Mid = {"Meta": "", "Macros": ""};
let Data = {"Meta": {}, "Macros": {}};
const Lines = Raw.trim().split('\n');
for (let i=0; i<Lines.length; i++) {
let Type;
const Line = Lines[i].trim();
if (Line.startsWith('%')) {
Type = 'Meta';
} else if (Line.startsWith('$')) {
Type = 'Macros';
} else {
continue;
};
Mid[Type] += Line.substring(1).trim() + '\n';
};
const Types = Object.keys(Mid);
for (let i=0; i<Types.length; i++) {
const Type = Types[i];
Parser = new ConfigParser();
Parser.parse(Mid[Type]);
const Items = Parser.items();
for (let i=0; i<Items.length; i++) {
const Item = Items[i];
Data[Type][Item[0]] = Item[1];
};
};
return Data;
};
const MakeMetaStr = Post => {
let Str = '';
const Types = ['Meta', 'Macros'];
for (let i=0; i<Types.length; i++) {
let Mark;
const Type = Types[i];
if (Type == 'Meta') {
Mark = '%';
} else if (Type == 'Macros') {
Mark = '$';
};
const Keys = Object.keys(Post[Type]);
for (let i=0; i<Keys.length; i++) {
const Key = Keys[i];
Str += `// ${Mark} ${Key} = ${Post[Type][Key]}\n`
};
};
return Str;
};
const HandlePost = PostSrc => {
let LinkElem, ContentDom;
let Post = {"Meta": {}, "Macros": {}};
Post.Meta.Title = PostSrc.title;
Post.Meta.CreatedOn = PostSrc.created_at.split('T')[0];
Post.Content = PostSrc.rendered_text;
console.log(`[I] => [${Post.Meta.CreatedOn}] ${Post.Meta.Title}`);
ContentDom = JSDOM.fragment(Post.Content);
LinkElem = GetLinkElem(ContentDom);
if (!LinkElem) { // Post content has no mirror-flagging element, skip it
// TODO: Exclusion mode instead of inclusion? Aka automatically handle posts without the element
// TODO: Check flagging via MetadataBlock?
console.log(`[I] : No mirror flag in source body; Skipping!`);
return;
};
const LinkPath = GetPath(JSDOM.fragment(LinkElem.outerHTML).querySelector('[href]').href);
/*
// Get post categories
Post.Categories = '';
const Classes = LinkElem.classList;
for (let i=0; i<Classes.length; i++) {
const Class = Classes[i];
const Key = `Mirror-${SiteName}-Categories-`;
@ -81,8 +149,9 @@ const HandlePost = PostSrc => {
Post['Categories'] = '// % Categories = ' + Class.substring(Key.length).replaceAll('|', ' ');
};
};
*/
Post['Content'] = PostSrc.rendered_text.replace(Elem.outerHTML, '');
// Do string replacements
const ReplacementsKeys = Object.keys(Replacements);
for (let i=0; i<ReplacementsKeys.length; i++) {
const To = ReplacementsKeys[i];
@ -91,25 +160,56 @@ const HandlePost = PostSrc => {
FromItems = [FromItems];
};
for (let i=0; i<FromItems.length; i++) {
Post['Content'] = Post['Content'].replaceAll(FromItems[i], To);
Post.Content = Post.Content.replaceAll(FromItems[i], To);
};
};
const PathFile = Path.split('/').slice(-1)[0];
const PathDir = Path.split('/').slice(0, (Path.split('/').length - 1)).join('/');
const FinalFilePath = `${PathDir}/${PostsFileDate ? Post['CreatedOn']+'-' : ''}${PathFile.substring(0, (PathFile.length - 4))}md`;
ContentDom = JSDOM.fragment(Post.Content);
LinkElem = GetLinkElem(ContentDom);
LinkElem.outerHTML = '';
// Handle MetadataBlock elements
let MetadataBlocks = ContentDom.querySelectorAll(MetadataBlockSelect);
for (let i=0; i<MetadataBlocks.length; i++) {
const Elem = MetadataBlocks[i];
if (Elem.textContent) {
const Meta = ParseMeta(Elem.textContent);
Post.Meta = Object.assign(Post.Meta, Meta.Meta);
Post.Macros = Object.assign(Post.Macros, Meta.Macros);
};
MetadataBlocks[i].outerHTML = '';
};
// Handle ExtractCodeBlock elements
let ExtCodeBlocks = ContentDom.querySelectorAll(ExtractCodeBlockSelect);
for (let i=0; i<ExtCodeBlocks.length; i++) {
const Elem = ExtCodeBlocks[i];
const Find = ExtractCodeBlockSelect.trim().replaceAll('.', '').replaceAll(',', '').split(' ')[0];
if (Array.from(Elem.classList).includes(Find)) {
ExtCodeBlocks[i].outerHTML = ''; // Remove the ExtractCodeBlock upper-marker
} else {
ExtCodeBlocks[i].outerHTML = Elem.textContent; // Extract the marker's text as raw HTML
};
};
Post.Content = GetFragHTML(ContentDom).trim();
const PathFile = LinkPath.split('/').slice(-1)[0];
const PathDir = LinkPath.split('/').slice(0, (LinkPath.split('/').length - 1)).join('/');
const FinalFilePath = `${PathDir}/${PostsFileDate ? Post.Meta.CreatedOn + '-' : ''}${PathFile.substring(0, (PathFile.length - 4))}md`;
if (!CheckDoDownsync(FinalFilePath)) {
console.log(`[I] : Downsync disabled in destination body; Skipping!`);
return;
};
TryMkdirSync(PathDir);
fs.writeFileSync(FinalFilePath, `\
${Post['Categories']}
// % CreatedOn = ${Post['CreatedOn']}
${MakeMetaStr(Post)}
# ${Post.Meta.Title}
# ${Post['Title']}
${Post['Content']}
${Post.Content}
`);
};