make boosts work. It's kinda slow.
This commit is contained in:
parent
de73654554
commit
3b682a12f4
117
lib/convertv2.js
117
lib/convertv2.js
|
@ -4,20 +4,38 @@ var template = ejs.compile(fs.readFileSync('./lib/template.ejs', 'utf8'));
|
||||||
var timeAgo = require('timeago.js');
|
var timeAgo = require('timeago.js');
|
||||||
|
|
||||||
// TODO try https://www.npmjs.com/package/request-promise-cache for the requests
|
// TODO try https://www.npmjs.com/package/request-promise-cache for the requests
|
||||||
var request = require('request-promise-native')
|
var request = require('request-promise-cache')
|
||||||
|
|
||||||
|
const hour = 3600000;
|
||||||
|
|
||||||
// get JSON for an AP URL
|
// get JSON for an AP URL
|
||||||
// TODO make it reject on HTTP 4xx or 5xx
|
// TODO make it reject on HTTP 4xx or 5xx
|
||||||
async function apGet(url) {
|
async function apGet(url,ttl) {
|
||||||
return request.get( {
|
return new Promise(function(resolve,reject){
|
||||||
uri:url,
|
|
||||||
headers: {
|
request( {
|
||||||
"accept": "application/activity+json"
|
uri:url,
|
||||||
},
|
cacheKey:url,
|
||||||
transform: function (body) {
|
cacheTTL:ttl || 24 * hour,
|
||||||
return JSON.parse(body);
|
headers: {
|
||||||
}
|
"accept": "application/activity+json"
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.then(body=>JSON.parse(body))
|
||||||
|
.then(resolve)
|
||||||
|
.catch(reject)
|
||||||
|
|
||||||
})
|
})
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// never rejects, instead it returns an object with a status, error and/or result.
|
||||||
|
async function apGetNoReject(...args){
|
||||||
|
return new Promise(function(resolve,reject){
|
||||||
|
apGet(...args)
|
||||||
|
.then(res=>resolve({status:true,result:res}))
|
||||||
|
.catch(e=>resolve({status:false,error:e}))
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = async function (opts) {
|
module.exports = async function (opts) {
|
||||||
|
@ -33,25 +51,25 @@ module.exports = async function (opts) {
|
||||||
|
|
||||||
var user, feed;
|
var user, feed;
|
||||||
|
|
||||||
// get user and feed in parallel if I have both URLs
|
// get user and feed in parallel if I have both URLs.
|
||||||
|
// can cache feed aggressively since it is a specific start and end.
|
||||||
if (userUrl && feedUrl){
|
if (userUrl && feedUrl){
|
||||||
[user, feed] = await Promise.all([ apGet(userUrl), apGet(feedUrl) ]);
|
[user, feed] = await Promise.all([ apGet(userUrl), apGet(feedUrl) ]);
|
||||||
}else{
|
}else{
|
||||||
// get user, then outbox, then feed
|
// get user, then outbox, then feed
|
||||||
|
|
||||||
user = await apGet(userUrl);
|
user = await apGet(userUrl,24 * hour);
|
||||||
isIndex = true;
|
isIndex = true;
|
||||||
var outbox = await apGet(user.outbox);
|
var outbox = await apGet(user.outbox,24 * hour);
|
||||||
feedUrl = outbox.first;
|
feedUrl = outbox.first;
|
||||||
feed = await apGet(feedUrl);
|
feed = await apGet(feedUrl,hour/6);// 10 mins. Because the base feed URL can get new toots quickly.
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
var templateData = {
|
var templateData = {
|
||||||
opts: opts,// from the request
|
opts: opts,// from the request
|
||||||
meta: metaForUser(user),
|
meta: metaForUser(user),
|
||||||
items: itemsForFeed(user,feed),
|
items: await itemsForFeed(opts,user,feed),
|
||||||
nextPageLink: getNextPage(opts,user,feed),
|
nextPageLink: getNextPage(opts,user,feed),
|
||||||
isIndex: isIndex
|
isIndex: isIndex
|
||||||
};
|
};
|
||||||
|
@ -70,11 +88,59 @@ function metaForUser(user) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO make function
|
// TODO make function
|
||||||
function itemsForFeed(user,feed) {
|
async function itemsForFeed(opts,user,feed) {
|
||||||
|
|
||||||
return feed.orderedItems.filter((item)=>{
|
var items = feed.orderedItems;
|
||||||
|
|
||||||
|
if (opts.boosts){
|
||||||
|
// yes, I have to fetch all the fucking boosts for this whole feed apparently >:/
|
||||||
|
var boostData = [];
|
||||||
|
var boostUrls = feed.orderedItems.filter(i=>i.type=="Announce").map(i=>i.object);
|
||||||
|
// console.log(boostUrls);
|
||||||
|
boostData = await Promise.all(boostUrls.map(apGetNoReject));
|
||||||
|
boostData = boostData.map(d=>d.result||{});
|
||||||
|
|
||||||
|
// now get user data for each of those
|
||||||
|
let userData = await Promise.all(boostData.map(d=>d.attributedTo||'').map(apGetNoReject));
|
||||||
|
|
||||||
|
// put a ._userdata key on the item object if this is a boost etc
|
||||||
|
for (var i = 0; i < boostData.length; i ++){
|
||||||
|
if (userData[i].status){
|
||||||
|
boostData[i]._userdata = userData[i].result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// some URLs may have failed but IDGAF
|
||||||
|
|
||||||
|
// console.log(boostData[0]);
|
||||||
|
|
||||||
|
boostData.forEach((boostToot)=>{
|
||||||
|
|
||||||
|
// inject in-place into items
|
||||||
|
|
||||||
|
var index = -1;
|
||||||
|
for (var i = 0; i < items.length; i ++){
|
||||||
|
if (items[i].object == boostToot.id){
|
||||||
|
index = i;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (index == -1){
|
||||||
|
console.warn("warning: couldn't match boost to item: ",boostToot.id)
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
boostToot.object = boostToot;// this lets the later stage parser access object without errors :)
|
||||||
|
items[i] = boostToot;
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
return items.filter((item)=>{
|
||||||
// this is temporary, don't handle boosts (TODO)
|
// this is temporary, don't handle boosts (TODO)
|
||||||
return item.type == "Create" && item.object && item.object.type=="Note";
|
// return item.type == "Create" && item.object && item.object.type=="Note";
|
||||||
|
return typeof item.object == 'object';// handle weird cases
|
||||||
}).map((item)=>{
|
}).map((item)=>{
|
||||||
|
|
||||||
var enclosures = (item.object.attachment||[]).filter((a)=>{
|
var enclosures = (item.object.attachment||[]).filter((a)=>{
|
||||||
|
@ -87,10 +153,11 @@ function itemsForFeed(user,feed) {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
var op = item._userdata?item._userdata:user;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
isBoost:false,
|
isBoost:!!item._userdata,
|
||||||
title:'New Status by '+user.preferredUsername,
|
title:item._userdata?user.preferredUsername+' shared a status by '+op.preferredUsername:'',
|
||||||
isReply:!!(item.object && item.object.inReplyTo),
|
isReply:!!(item.object && item.object.inReplyTo),
|
||||||
hasCw:item.object.sensitive||false,
|
hasCw:item.object.sensitive||false,
|
||||||
cw:item.object.summary,
|
cw:item.object.summary,
|
||||||
|
@ -99,10 +166,10 @@ function itemsForFeed(user,feed) {
|
||||||
enclosures:enclosures,
|
enclosures:enclosures,
|
||||||
stringDate:item.published?getTimeDisplay(Date.parse(item.published)):'',
|
stringDate:item.published?getTimeDisplay(Date.parse(item.published)):'',
|
||||||
author:{
|
author:{
|
||||||
uri:user.url,// link to author page
|
uri:op.url,// link to author page
|
||||||
avatar:user.icon&&user.icon.url?user.icon.url:'',
|
avatar:op.icon&&op.icon.url?op.icon.url:'',
|
||||||
displayName:user.name,
|
displayName:op.name,
|
||||||
fullName:user.preferredUsername+'@'+(new URL(user.url).hostname),
|
fullName:op.preferredUsername+'@'+(new URL(op.url).hostname),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
|
@ -451,6 +451,11 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"extend": {
|
||||||
|
"version": "3.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
|
||||||
|
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
|
||||||
|
},
|
||||||
"extsprintf": {
|
"extsprintf": {
|
||||||
"version": "1.0.2",
|
"version": "1.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz",
|
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz",
|
||||||
|
@ -1008,6 +1013,14 @@
|
||||||
"integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw==",
|
"integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw==",
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
|
"nano-cache": {
|
||||||
|
"version": "1.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/nano-cache/-/nano-cache-1.1.2.tgz",
|
||||||
|
"integrity": "sha512-tqs6EYyIq8oCQMc1L+QSUdZYOk/STwWCmIqyGO6cXI4QhejSb586WPBgO3nplzD5L3+aJCAITMVtMTFFLAdb7w==",
|
||||||
|
"requires": {
|
||||||
|
"extend": "~3.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"negotiator": {
|
"negotiator": {
|
||||||
"version": "0.6.2",
|
"version": "0.6.2",
|
||||||
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
|
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
|
||||||
|
@ -1402,6 +1415,15 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"request-promise-cache": {
|
||||||
|
"version": "2.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/request-promise-cache/-/request-promise-cache-2.0.1.tgz",
|
||||||
|
"integrity": "sha512-y+me4+M3wpQ1Sj4WPr3Ywg459UjkP+uyh0JlM0FQxjZtNNqixQxbqhmo5M3t+e2mjglMpfm6UPDhSo40kLWsmA==",
|
||||||
|
"requires": {
|
||||||
|
"nano-cache": "^1.1.2",
|
||||||
|
"request": "^2.88.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"request-promise-core": {
|
"request-promise-core": {
|
||||||
"version": "1.1.2",
|
"version": "1.1.2",
|
||||||
"resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.2.tgz",
|
"resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.2.tgz",
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
"express": "^4.16.4",
|
"express": "^4.16.4",
|
||||||
"feedparser": "^2.2.9",
|
"feedparser": "^2.2.9",
|
||||||
"request": "^2.88.0",
|
"request": "^2.88.0",
|
||||||
|
"request-promise-cache": "^2.0.1",
|
||||||
"request-promise-native": "^1.0.7",
|
"request-promise-native": "^1.0.7",
|
||||||
"serve-static": "^1.13.2",
|
"serve-static": "^1.13.2",
|
||||||
"timeago.js": "^3.0.2"
|
"timeago.js": "^3.0.2"
|
||||||
|
|
Loading…
Reference in New Issue