From 9c4cda7b4215110e1158529fc4b8d639d8903543 Mon Sep 17 00:00:00 2001 From: StevenNMeza Date: Mon, 21 Dec 2020 12:52:48 +0100 Subject: [PATCH] Serve the flair images from own server --- config.js.template | 2 +- inc/commons.js | 8 ++++---- inc/downloadAndSave.js | 27 ++++++++++++++++++--------- inc/processJsonPost.js | 18 +++++++++--------- inc/processJsonSubreddit.js | 4 ++-- inc/processJsonUser.js | 4 ++-- 6 files changed, 36 insertions(+), 27 deletions(-) diff --git a/config.js.template b/config.js.template index b9cd00e..e9e7c53 100644 --- a/config.js.template +++ b/config.js.template @@ -34,7 +34,7 @@ const config = { shorts: 60 * 60 * 24 * 31 }, post_comments_sort: 'confidence', // "confidence" is the default sorting in Reddit. Must be one of: confidence, top, new, controversial, old, random, qa, live. - valid_media_domains: ['preview.redd.it', 'external-preview.redd.it', 'i.redd.it', 'v.redd.it', 'a.thumbs.redditmedia.com', 'b.thumbs.redditmedia.com', 'thumbs.gfycat.com', 'i.ytimg.com'], + valid_media_domains: ['preview.redd.it', 'external-preview.redd.it', 'i.redd.it', 'v.redd.it', 'a.thumbs.redditmedia.com', 'b.thumbs.redditmedia.com', 'emoji.redditmedia.com', 'thumbs.gfycat.com', 'i.ytimg.com'], reddit_api_error_text: `Seems like your instance is either blocked (e.g. due to API rate limiting), reddit is currently down, or your API key is expired and not renewd properly. This can also happen for other reasons.` }; diff --git a/inc/commons.js b/inc/commons.js index c3b6184..b0e7aa3 100644 --- a/inc/commons.js +++ b/inc/commons.js @@ -198,7 +198,7 @@ module.exports = function(request, fs) { } } - this.formatLinkFlair = (post) => { + this.formatLinkFlair = async (post) => { const wrap = (inner) => `${inner}` if (post.link_flair_text === null) @@ -213,7 +213,7 @@ module.exports = function(request, fs) { if (fragment.e === 'text') flair += fragment.t else if (fragment.e === 'emoji') - flair += `` + flair += `` } return wrap(flair) } @@ -221,7 +221,7 @@ module.exports = function(request, fs) { return '' } - this.formatUserFlair = (post) => { + this.formatUserFlair = async (post) => { // Generate the entire HTML here for consistency in both pug and HTML const wrap = (inner) => `${inner}` @@ -238,7 +238,7 @@ module.exports = function(request, fs) { if (fragment.e === 'text') flair += fragment.t // `t` is the text else if (fragment.e === 'emoji') - flair += `` // `u` is the emoji URL + flair += `` // `u` is the emoji URL } return wrap(flair) } diff --git a/inc/downloadAndSave.js b/inc/downloadAndSave.js index fdd83e3..d0224db 100644 --- a/inc/downloadAndSave.js +++ b/inc/downloadAndSave.js @@ -1,12 +1,12 @@ -module.exports = function(tools) { +module.exports = function(tools) { const config = require('../config') const {spawn} = require('child_process') const fs = require('fs') this.downloadAndSave = (url, file_prefix = '', gifmp4, isYouTubeThumbnail) => { - /** + /** * This function downloads media (video or image) to disk. * Returns a localized URL - * + * * For example for images: * https://external-preview.redd.it/DiaeK_j5fqpBqbatvo7GZzbHNJY2oxEym93B_3.jpg * => @@ -32,21 +32,23 @@ module.exports = function(tools) { if(gifmp4) { file_ext = 'mp4' } else { - if(!pathname.includes('.')) { - /** + if (file_prefix === 'flair_') { + // Flair emojis end in the name without a file extension + file_ext = 'png' + } else if(!pathname.includes('.')) { /** * Sometimes reddit API returns video without extension, like * "DASH_480" and not "DASH_480.mp4". */ file_ext = 'mp4' has_extension = false - } else { + } else { file_ext = pathname.substring(pathname.lastIndexOf('.') + 1) } } - + if(file_prefix === 'thumb_') dir = 'thumbs/' - if(file_prefix === 'flair') + if(file_prefix === 'flair_') dir = 'flairs/' if(valid_video_extensions.includes(file_ext) || gifmp4) { @@ -130,7 +132,14 @@ module.exports = function(tools) { if(temp_url.searchParams.get('width')) { width = temp_url.searchParams.get('width') } - filename = `${file_prefix}w:${temp_url.searchParams.get('width')}_${temp_url.pathname.split('/').slice(-1)}` + if(file_prefix === 'flair_') { + // Flair emojis have a full path of `UUID/name`, + // so we need to incorporate the UUID to avoid duplicates + // since names alone are not unique across all of reddit + filename = `${pathname.slice(1).replace('/', '_')}.png` // Only first replacement is fine + } else { + filename = `${file_prefix}w:${temp_url.searchParams.get('width')}_${temp_url.pathname.split('/').slice(-1)}` + } } path = `./dist/pics/${dir}${filename}` if(!fs.existsSync(path)) { diff --git a/inc/processJsonPost.js b/inc/processJsonPost.js index d692363..467009f 100644 --- a/inc/processJsonPost.js +++ b/inc/processJsonPost.js @@ -33,8 +33,8 @@ module.exports = function(fetch) { images: null, crosspost: false, selftext: unescape(post.selftext_html), - link_flair: formatLinkFlair(post), - user_flair: formatUserFlair(post) + link_flair: await formatLinkFlair(post), + user_flair: await formatUserFlair(post) } let validEmbedDomains = ['gfycat.com', 'youtube.com'] @@ -87,7 +87,7 @@ module.exports = function(fetch) { selftext: unescape(post.selftext_html), selftext_crosspost: unescape(post.crosspost.selftext_html), is_crosspost: true, - user_flair: formatUserFlair(post) + user_flair: await formatUserFlair(post) } } @@ -147,7 +147,7 @@ module.exports = function(fetch) { edited: comment.edited, replies: [], depth: 0, - user_flair: formatUserFlair(comment) + user_flair: await formatUserFlair(comment) } } else { obj = { @@ -163,7 +163,7 @@ module.exports = function(fetch) { if(comment.replies && kind !== 'more') { if(comment.replies.data) { if(comment.replies.data.children.length > 0) { - obj.replies = processReplies(comment.replies.data.children, post_id, 1) + obj.replies = await processReplies(comment.replies.data.children, post_id, 1) } } } @@ -202,7 +202,7 @@ module.exports = function(fetch) { return { post_data: post_data, comments: comments_html } } - this.processReplies = (data, post_id, depth) => { + this.processReplies = async (data, post_id, depth) => { let return_replies = [] for(var i = 0; i < data.length; i++) { let kind = data[i].kind @@ -225,7 +225,7 @@ module.exports = function(fetch) { edited: reply.edited, replies: [], depth: depth, - user_flair: formatUserFlair(reply) + user_flair: await formatUserFlair(reply) } } else { obj = { @@ -261,7 +261,7 @@ module.exports = function(fetch) { distinguished: comment.edited, replies: [], depth: depth + 1, - user_flair: formatUserFlair(comment) + user_flair: await formatUserFlair(comment) } } else { objct = { @@ -283,7 +283,7 @@ module.exports = function(fetch) { if(comment.replies) { if(comment.replies.data) { if(comment.replies.data.children.length > 0) { - objct.replies = processReplies(comment.replies.data.children, post_id, depth ) + objct.replies = await processReplies(comment.replies.data.children, post_id, depth ) } } } diff --git a/inc/processJsonSubreddit.js b/inc/processJsonSubreddit.js index 1dc8e42..1b4c04c 100644 --- a/inc/processJsonSubreddit.js +++ b/inc/processJsonSubreddit.js @@ -74,8 +74,8 @@ module.exports = function() { stickied: data.stickied, is_self_link: is_self_link, subreddit_front: subreddit_front, - link_flair: formatLinkFlair(data), - user_flair: formatUserFlair(data) + link_flair: await formatLinkFlair(data), + user_flair: await formatUserFlair(data) } ret.links.push(obj) } diff --git a/inc/processJsonUser.js b/inc/processJsonUser.js index 4114fb8..fe86322 100644 --- a/inc/processJsonUser.js +++ b/inc/processJsonUser.js @@ -63,7 +63,7 @@ module.exports = function() { selftext_html: unescape(post.selftext_html), num_comments: post.num_comments, permalink: post.permalink, - user_flair: formatUserFlair(post) + user_flair: await formatUserFlair(post) } } if(type === 't1') { @@ -81,7 +81,7 @@ module.exports = function() { permalink: post.permalink, link_author: post.link_author, link_title: post.link_title, - user_flair: formatUserFlair(post) + user_flair: await formatUserFlair(post) } } posts.push(obj)