Scripts updates

This commit is contained in:
octospacc 2023-05-07 16:35:05 +02:00
parent 76fee48000
commit 11f11068e3
3 changed files with 145 additions and 96 deletions

36
Scripts/Lib/Syncers.js Normal file
View File

@ -0,0 +1,36 @@
const Exp = {};
Exp.Fs = require('fs');
Exp.ConfigParser = require('./config-ini-parser').ConfigIniParser;
// https://stackoverflow.com/a/69049676
Exp.importAll = function importAll(){ delete this.importAll; Object.assign(global, this); };
Exp.ParseMeta = (Raw) => {
let Mid = { Meta: "", Macros: "", };
let Data = { Meta: {}, Macros: {}, };
const Lines = Raw.trim().split('\n');
for (let i=0; i<Lines.length; i++) {
let Type;
let Line = Lines[i].trim();
if (Line.startsWith('//')) {
Line = Line.slice('2').trim();
};
if (Line.startsWith('%')) {
Type = 'Meta';
} else if (Line.startsWith('$')) {
Type = 'Macros';
} else {
continue;
};
Mid[Type] += Line.substring(1).trim() + '\n';
};
Object.keys(Mid).forEach((Type) => {
const Items = new ConfigParser().parse(Mid[Type]).items();
Items.forEach((Item) => {
Data[Type][Item[0]] = Item[1];
});
});
return Data;
};
module.exports = Exp;

View File

@ -1,7 +1,6 @@
#!/usr/bin/env -S node --experimental-fetch #!/usr/bin/env -S node --experimental-fetch
const fs = require('fs'); require('./Lib/Syncers.js').importAll();
const JSDOM = require('jsdom').JSDOM; const JSDOM = require('jsdom').JSDOM;
const ConfigParser = require('./Lib/config-ini-parser').ConfigIniParser;
const BlogURL = 'https://listed.to/@u8'; // Full base URL of the Listed blog (any server) const BlogURL = 'https://listed.to/@u8'; // Full base URL of the Listed blog (any server)
const SiteName = 'sitoctt'; const SiteName = 'sitoctt';
@ -27,14 +26,14 @@ const ExtractCodeBlockSelect = '.ExtractCodeBlock, .ExtractCodeBlock + :Where(Di
const DeleteElementBlockSelect = '.DeleteElementBlock'; const DeleteElementBlockSelect = '.DeleteElementBlock';
const TryReadFileSync = Path => { const TryReadFileSync = Path => {
if (fs.existsSync(Path)) { if (Fs.existsSync(Path)) {
return fs.readFileSync(Path, 'utf8'); return Fs.readFileSync(Path, 'utf8');
}; };
}; };
const TryMkdirSync = Path => { const TryMkdirSync = Path => {
if (!fs.existsSync(Path)) { if (!Fs.existsSync(Path)) {
return fs.mkdirSync(Path, {recursive: true}); return Fs.mkdirSync(Path, {recursive: true});
}; };
}; };
@ -85,53 +84,14 @@ const GetLinkElem = Dom => {
return Elem; return Elem;
}; };
const ParseMeta = Raw => {
let Mid = {'Meta': '', 'Macros': ''};
let Data = {'Meta': {}, 'Macros': {}};
const Lines = Raw.trim().split('\n');
for (let i=0; i<Lines.length; i++) {
let Type;
const Line = Lines[i].trim();
if (Line.startsWith('%')) {
Type = 'Meta';
} else if (Line.startsWith('$')) {
Type = 'Macros';
} else {
continue;
};
Mid[Type] += Line.substring(1).trim() + '\n';
};
const Types = Object.keys(Mid);
for (let i=0; i<Types.length; i++) {
const Type = Types[i];
Parser = new ConfigParser();
Parser.parse(Mid[Type]);
const Items = Parser.items();
for (let i=0; i<Items.length; i++) {
const Item = Items[i];
Data[Type][Item[0]] = Item[1];
};
};
return Data;
};
const MakeMetaStr = Post => { const MakeMetaStr = Post => {
let Str = ''; let Str = '';
const Types = ['Meta', 'Macros']; ['Meta', 'Macros'].forEach((Type) => {
for (let i=0; i<Types.length; i++) { const Marks = { Meta: "%", Macros: "$", };
let Mark; Object.keys(Post[Type]).forEach((Key) => {
const Type = Types[i]; Str += `// ${Marks[Type]} ${Key} = ${Post[Type][Key]}\n`;
if (Type == 'Meta') { });
Mark = '%'; });
} else if (Type == 'Macros') {
Mark = '$';
};
const Keys = Object.keys(Post[Type]);
for (let i=0; i<Keys.length; i++) {
const Key = Keys[i];
Str += `// ${Mark} ${Key} = ${Post[Type][Key]}\n`
};
};
return Str; return Str;
}; };
@ -147,16 +107,14 @@ const HandlePost = (PostSrc, Output) => {
ContentDom = JSDOM.fragment(Post.Content); ContentDom = JSDOM.fragment(Post.Content);
// Handle .MetadataBlock elements // Handle .MetadataBlock elements
let MetadataBlocks = ContentDom.querySelectorAll(MetadataBlockSelect); Array.from(ContentDom.querySelectorAll(MetadataBlockSelect)).forEach((Elem) => {
for (let i=0; i<MetadataBlocks.length; i++) {
const Elem = MetadataBlocks[i];
if (Elem.textContent) { if (Elem.textContent) {
const Meta = ParseMeta(Elem.textContent); const Meta = ParseMeta(Elem.textContent);
Post.Meta = Object.assign(Post.Meta, Meta.Meta); Post.Meta = Object.assign(Post.Meta, Meta.Meta);
Post.Macros = Object.assign(Post.Macros, Meta.Macros); Post.Macros = Object.assign(Post.Macros, Meta.Macros);
}; };
MetadataBlocks[i].outerHTML = ''; Elem.outerHTML = '';
}; });
// NOTE: Maybe would be better to first do string replacements? // NOTE: Maybe would be better to first do string replacements?
let LinkElem = GetLinkElem(ContentDom); let LinkElem = GetLinkElem(ContentDom);
@ -174,9 +132,7 @@ const HandlePost = (PostSrc, Output) => {
}; };
// Handle .ReplacementsBlock elements: Add replacements to do to the default ones or override them. // Handle .ReplacementsBlock elements: Add replacements to do to the default ones or override them.
let ReplBlocks = ContentDom.querySelectorAll(ReplacementsBlockSelect); Array.from(ContentDom.querySelectorAll(ReplacementsBlockSelect)).forEach((Elem) => {
for (let i=0; i<ReplBlocks.length; i++) {
const Elem = ReplBlocks[i];
let Text = Elem.textContent.trim(); let Text = Elem.textContent.trim();
if (Text) { if (Text) {
if (!(Text.startsWith('{') && Text.endsWith('}'))) { if (!(Text.startsWith('{') && Text.endsWith('}'))) {
@ -188,8 +144,8 @@ const HandlePost = (PostSrc, Output) => {
console.log(`[W] : Problem parsing JSON in a ReplacementsBlock; Ignoring!`); console.log(`[W] : Problem parsing JSON in a ReplacementsBlock; Ignoring!`);
}; };
}; };
ReplBlocks[i].outerHTML = ''; Elem.outerHTML = '';
}; });
Post.Content = GetFragHTML(ContentDom); Post.Content = GetFragHTML(ContentDom);
@ -204,47 +160,43 @@ const HandlePost = (PostSrc, Output) => {
// Do string replacements // Do string replacements
// TODO: Replacements written in post body? // TODO: Replacements written in post body?
const ReplacementsKeys = Object.keys(Replacements); Object.keys(Replacements).forEach((To) => {
for (let i=0; i<ReplacementsKeys.length; i++) { let FromList = Replacements[To];
const To = ReplacementsKeys[i]; if (typeof(FromList) != 'object') {
let FromItems = Replacements[To]; FromList = [FromList];
if (typeof(FromItems) != 'object') {
FromItems = [FromItems];
};
for (let i=0; i<FromItems.length; i++) {
Post.Content = Post.Content.replaceAll(FromItems[i], To);
};
}; };
FromList.forEach((From) => {
Post.Content = Post.Content.replaceAll(From, To);
});
});
ContentDom = JSDOM.fragment(Post.Content); ContentDom = JSDOM.fragment(Post.Content);
// Handle .DeleteElementBlock elements: Elements that must be visible on Listed but deleted here. // Handle .DeleteElementBlock elements: Elements that must be visible on Listed but deleted here.
let DelElemBlocks = ContentDom.querySelectorAll(DeleteElementBlockSelect); Array.from(ContentDom.querySelectorAll(DeleteElementBlockSelect)).forEach((Elem) => {
for (let i=0; i<DelElemBlocks.length; i++) {
const Elem = DelElemBlocks[i];
if (!Elem.textContent) { if (!Elem.textContent) {
DelElemBlocks[i].nextElementSibling.outerHTML = ''; Elem.nextElementSibling.outerHTML = '';
};
DelElemBlocks[i].outerHTML = '';
}; };
Elem.outerHTML = '';
});
// Handle .ExtractCodeBlock elements: Allow for text to be treated as plain on Listed, and then extracted here. // Handle .ExtractCodeBlock elements: Allow for text to be treated as plain on Listed, and then extracted here.
let ExtCodeBlocks = ContentDom.querySelectorAll(ExtractCodeBlockSelect); Array.from(ContentDom.querySelectorAll(ExtractCodeBlockSelect)).forEach((Elem) => {
for (let i=0; i<ExtCodeBlocks.length; i++) {
const Elem = ExtCodeBlocks[i];
const Find = CSSFirstTokenSelector(ExtractCodeBlockSelect); const Find = CSSFirstTokenSelector(ExtractCodeBlockSelect);
if (Array.from(Elem.classList).includes(Find)) { if (Array.from(Elem.classList).includes(Find)) {
ExtCodeBlocks[i].outerHTML = ''; // Remove the ExtractCodeBlock upper-marker // Remove the ExtractCodeBlock upper-marker
Elem.outerHTML = '';
} else { } else {
ExtCodeBlocks[i].outerHTML = Elem.textContent; // Extract the marker's text as raw HTML // Extract the marker's text as raw HTML
}; Elem.outerHTML = Elem.textContent;
}; };
});
Post.Content = GetFragHTML(ContentDom); Post.Content = GetFragHTML(ContentDom);
if (Output == 'file') { if (Output == 'file') {
TryMkdirSync(PathDir); TryMkdirSync(PathDir);
fs.writeFileSync(FinalFilePath, `\ Fs.writeFileSync(FinalFilePath, `\
<!-- < Autogenerated by ListedDownsync.js. Do not edit (unless also set "% Downsync = False") - it would be overwritten. > --> <!-- < Autogenerated by ListedDownsync.js. Do not edit (unless also set "% Downsync = False") - it would be overwritten. > -->
${MakeMetaStr(Post)} ${MakeMetaStr(Post)}
<h1>${Post.Meta.HTMLTitle ? Post.Meta.HTMLTitle : Post.Meta.Title}</h1> <h1>${Post.Meta.HTMLTitle ? Post.Meta.HTMLTitle : Post.Meta.Title}</h1>

View File

@ -1,8 +1,16 @@
#!/usr/bin/env node #!/usr/bin/env node
require('./Lib/Syncers.js').importAll();
const Path = require('path');
const Http = require('http'); const Http = require('http');
const Https = require('https'); const Https = require('https');
// true: Reuse the local directory structure for URL slugs
// false: Keep a flat URL slug composing only of the file name
const KeepTree = false;
// Local OAuth server
const Host = '127.0.0.1'; const Host = '127.0.0.1';
let [Auth, Session] = [ let [Auth, Session] = [
JSON.parse(process.env.WordpressAuth || '{}'), JSON.parse(process.env.WordpressAuth || '{}'),
JSON.parse(process.env.WordpressSession || '{}'), JSON.parse(process.env.WordpressSession || '{}'),
@ -23,12 +31,29 @@ const AuthHeaders = () => {
}; };
}; };
let [LocalPosts, RemotePosts] = [[], []];
let FetchingRemotePosts = false;
// https://stackoverflow.com/a/73594511
Fs.walkSync = (Dir, Files = []) => {
const dirFiles = Fs.readdirSync(Dir);
for (const f of dirFiles) {
const stat = Fs.lstatSync(Dir + Path.sep + f)
if (stat.isDirectory()) {
Fs.walkSync(Dir + Path.sep + f, Files);
} else {
Files.push(Dir + Path.sep + f);
};
};
return Files;
};
// https://developer.wordpress.com/docs/oauth2/ // https://developer.wordpress.com/docs/oauth2/
const AuthServer = () => { const AuthServer = () => {
const Serv = Http.createServer((Req, Res) => { const Serv = Http.createServer((Req, Res) => {
Res.setHeader('Content-Type', 'text/plain'); Res.setHeader('Content-Type', 'text/plain');
let Query = new URLSearchParams(Req.url.slice(1).replaceAll('?', '')); const Query = new URLSearchParams(Req.url.slice(1).replaceAll('?', ''));
let AuthCode = Query.get('code'); const AuthCode = Query.get('code');
if (AuthCode) { if (AuthCode) {
Res.statusCode = 200; Res.statusCode = 200;
Res.end('This window can now be closed.'); Res.end('This window can now be closed.');
@ -63,18 +88,54 @@ const AuthServer = () => {
}); });
}; };
// TODO const GetRemotePosts = (Page) => {
const TryUpsync = () => { FetchingRemotePosts = true;
// https://developer.wordpress.com/docs/api/1.1/get/sites/%24site/posts/ const Opts = {
Https.get(`https://${ApiBase}/rest/v1.1/sites/${Session.blog_id}/posts/?&fields=ID,slug,status,categories,tags,title,content&context=edit`, AuthHeaders(), Url: `https://${ApiBase}/rest/v1.1/sites/${Session.blog_id}/posts/`,
fields: "ID,slug,status,categories,tags,title,content",
page_handle: encodeURIComponent(Page || ''),
};
Https.get(`${Opts.Url}?&fields=${Opts.fields}&context=edit&page_handle=${Opts.page_handle}`, AuthHeaders(),
(Res) => { (Res) => {
let Data = ''; let Data = '';
Res.on('data', (Frag) => { Res.on('data', (Frag) => {
Data += Frag; Data += Frag;
}).on('end', () => { }).on('end', () => {
let RemotePosts = JSON.parse(Data); const JsonData = JSON.parse(Data);
const NextPage = JsonData.meta.next_page;
RemotePosts = RemotePosts.concat(JsonData.posts);
if (NextPage) {
GetRemotePosts(NextPage);
process.stdout.write('.');
} else {
FetchingRemotePosts = false;
};
});
});
};
const TryUpsync = () => {
console.log('[I] Reading local posts...');
Fs.walkSync('./Posts').forEach((File) => {
let Content = Fs.readFileSync(File, 'utf8').trim();
const Meta = ParseMeta(Content.split('\n\n')[0]);
if (Meta.Meta.Type === 'Post' || !Meta.Meta.Type) {
Content = Content.split('\n\n').slice('1').join('\n\n');
const Slug = KeepTree
? File.slice('./Posts/'.length)
: File.split('/').slice(-1)[0];
const Obj = { Path: Slug, Meta: Meta, Content: Content, };
LocalPosts.push(Obj);
//LocalPosts[Slug] = Obj;
};
});
console.log(LocalPosts);
console.log('[I] Fetching remote posts...');
GetRemotePosts();
var Interv = setInterval(() => {
if (!FetchingRemotePosts) {
clearInterval(Interv);
console.log(RemotePosts); console.log(RemotePosts);
// Read existing posts from ./Posts/ here
// Find out which posts exist on remote and which not // Find out which posts exist on remote and which not
// Exist on remote and local: // Exist on remote and local:
// Check if remote data is same as local, update remote if not // Check if remote data is same as local, update remote if not
@ -84,8 +145,8 @@ const TryUpsync = () => {
// https://developer.wordpress.com/docs/api/1.1/post/sites/%24site/posts/new/ // https://developer.wordpress.com/docs/api/1.1/post/sites/%24site/posts/new/
// Exist on remote only: // Exist on remote only:
// Was probably deleted or moved locally, what to do here? // Was probably deleted or moved locally, what to do here?
}); };
}); }, 50);
}; };
if (!Session.access_token) { if (!Session.access_token) {