make boosts work. It's kinda slow.
This commit is contained in:
fenwick67
parent
de73654554
commit
3b682a12f4
119
lib/convertv2.js
119
lib/convertv2.js
|
|
@ -4,20 +4,38 @@ var template = ejs.compile(fs.readFileSync('./lib/template.ejs', 'utf8'));
|
|||
var timeAgo = require('timeago.js');
|
||||
|
||||
// TODO try https://www.npmjs.com/package/request-promise-cache for the requests
|
||||
var request = require('request-promise-native')
|
||||
var request = require('request-promise-cache')
|
||||
|
||||
const hour = 3600000;
|
||||
|
||||
// get JSON for an AP URL
|
||||
// TODO make it reject on HTTP 4xx or 5xx
|
||||
async function apGet(url) {
|
||||
return request.get( {
|
||||
uri:url,
|
||||
headers: {
|
||||
"accept": "application/activity+json"
|
||||
},
|
||||
transform: function (body) {
|
||||
return JSON.parse(body);
|
||||
}
|
||||
async function apGet(url,ttl) {
|
||||
return new Promise(function(resolve,reject){
|
||||
|
||||
request( {
|
||||
uri:url,
|
||||
cacheKey:url,
|
||||
cacheTTL:ttl || 24 * hour,
|
||||
headers: {
|
||||
"accept": "application/activity+json"
|
||||
}
|
||||
})
|
||||
.then(body=>JSON.parse(body))
|
||||
.then(resolve)
|
||||
.catch(reject)
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// never rejects, instead it returns an object with a status, error and/or result.
|
||||
async function apGetNoReject(...args){
|
||||
return new Promise(function(resolve,reject){
|
||||
apGet(...args)
|
||||
.then(res=>resolve({status:true,result:res}))
|
||||
.catch(e=>resolve({status:false,error:e}))
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = async function (opts) {
|
||||
|
|
@ -33,25 +51,25 @@ module.exports = async function (opts) {
|
|||
|
||||
var user, feed;
|
||||
|
||||
// get user and feed in parallel if I have both URLs
|
||||
// get user and feed in parallel if I have both URLs.
|
||||
// can cache feed aggressively since it is a specific start and end.
|
||||
if (userUrl && feedUrl){
|
||||
[user, feed] = await Promise.all([ apGet(userUrl), apGet(feedUrl) ]);
|
||||
}else{
|
||||
// get user, then outbox, then feed
|
||||
|
||||
user = await apGet(userUrl);
|
||||
user = await apGet(userUrl,24 * hour);
|
||||
isIndex = true;
|
||||
var outbox = await apGet(user.outbox);
|
||||
var outbox = await apGet(user.outbox,24 * hour);
|
||||
feedUrl = outbox.first;
|
||||
feed = await apGet(feedUrl);
|
||||
feed = await apGet(feedUrl,hour/6);// 10 mins. Because the base feed URL can get new toots quickly.
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
var templateData = {
|
||||
opts: opts,// from the request
|
||||
meta: metaForUser(user),
|
||||
items: itemsForFeed(user,feed),
|
||||
items: await itemsForFeed(opts,user,feed),
|
||||
nextPageLink: getNextPage(opts,user,feed),
|
||||
isIndex: isIndex
|
||||
};
|
||||
|
|
@ -70,11 +88,59 @@ function metaForUser(user) {
|
|||
}
|
||||
|
||||
// TODO make function
|
||||
function itemsForFeed(user,feed) {
|
||||
async function itemsForFeed(opts,user,feed) {
|
||||
|
||||
return feed.orderedItems.filter((item)=>{
|
||||
var items = feed.orderedItems;
|
||||
|
||||
if (opts.boosts){
|
||||
// yes, I have to fetch all the fucking boosts for this whole feed apparently >:/
|
||||
var boostData = [];
|
||||
var boostUrls = feed.orderedItems.filter(i=>i.type=="Announce").map(i=>i.object);
|
||||
// console.log(boostUrls);
|
||||
boostData = await Promise.all(boostUrls.map(apGetNoReject));
|
||||
boostData = boostData.map(d=>d.result||{});
|
||||
|
||||
// now get user data for each of those
|
||||
let userData = await Promise.all(boostData.map(d=>d.attributedTo||'').map(apGetNoReject));
|
||||
|
||||
// put a ._userdata key on the item object if this is a boost etc
|
||||
for (var i = 0; i < boostData.length; i ++){
|
||||
if (userData[i].status){
|
||||
boostData[i]._userdata = userData[i].result;
|
||||
}
|
||||
}
|
||||
|
||||
// some URLs may have failed but IDGAF
|
||||
|
||||
// console.log(boostData[0]);
|
||||
|
||||
boostData.forEach((boostToot)=>{
|
||||
|
||||
// inject in-place into items
|
||||
|
||||
var index = -1;
|
||||
for (var i = 0; i < items.length; i ++){
|
||||
if (items[i].object == boostToot.id){
|
||||
index = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (index == -1){
|
||||
console.warn("warning: couldn't match boost to item: ",boostToot.id)
|
||||
return;
|
||||
}
|
||||
|
||||
boostToot.object = boostToot;// this lets the later stage parser access object without errors :)
|
||||
items[i] = boostToot;
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
return items.filter((item)=>{
|
||||
// this is temporary, don't handle boosts (TODO)
|
||||
return item.type == "Create" && item.object && item.object.type=="Note";
|
||||
// return item.type == "Create" && item.object && item.object.type=="Note";
|
||||
return typeof item.object == 'object';// handle weird cases
|
||||
}).map((item)=>{
|
||||
|
||||
var enclosures = (item.object.attachment||[]).filter((a)=>{
|
||||
|
|
@ -87,10 +153,11 @@ function itemsForFeed(user,feed) {
|
|||
}
|
||||
});
|
||||
|
||||
var op = item._userdata?item._userdata:user;
|
||||
|
||||
return {
|
||||
isBoost:false,
|
||||
title:'New Status by '+user.preferredUsername,
|
||||
isBoost:!!item._userdata,
|
||||
title:item._userdata?user.preferredUsername+' shared a status by '+op.preferredUsername:'',
|
||||
isReply:!!(item.object && item.object.inReplyTo),
|
||||
hasCw:item.object.sensitive||false,
|
||||
cw:item.object.summary,
|
||||
|
|
@ -99,10 +166,10 @@ function itemsForFeed(user,feed) {
|
|||
enclosures:enclosures,
|
||||
stringDate:item.published?getTimeDisplay(Date.parse(item.published)):'',
|
||||
author:{
|
||||
uri:user.url,// link to author page
|
||||
avatar:user.icon&&user.icon.url?user.icon.url:'',
|
||||
displayName:user.name,
|
||||
fullName:user.preferredUsername+'@'+(new URL(user.url).hostname),
|
||||
uri:op.url,// link to author page
|
||||
avatar:op.icon&&op.icon.url?op.icon.url:'',
|
||||
displayName:op.name,
|
||||
fullName:op.preferredUsername+'@'+(new URL(op.url).hostname),
|
||||
}
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -451,6 +451,11 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"extend": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
|
||||
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
|
||||
},
|
||||
"extsprintf": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz",
|
||||
|
|
@ -1008,6 +1013,14 @@
|
|||
"integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw==",
|
||||
"dev": true
|
||||
},
|
||||
"nano-cache": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/nano-cache/-/nano-cache-1.1.2.tgz",
|
||||
"integrity": "sha512-tqs6EYyIq8oCQMc1L+QSUdZYOk/STwWCmIqyGO6cXI4QhejSb586WPBgO3nplzD5L3+aJCAITMVtMTFFLAdb7w==",
|
||||
"requires": {
|
||||
"extend": "~3.0.0"
|
||||
}
|
||||
},
|
||||
"negotiator": {
|
||||
"version": "0.6.2",
|
||||
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
|
||||
|
|
@ -1402,6 +1415,15 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"request-promise-cache": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/request-promise-cache/-/request-promise-cache-2.0.1.tgz",
|
||||
"integrity": "sha512-y+me4+M3wpQ1Sj4WPr3Ywg459UjkP+uyh0JlM0FQxjZtNNqixQxbqhmo5M3t+e2mjglMpfm6UPDhSo40kLWsmA==",
|
||||
"requires": {
|
||||
"nano-cache": "^1.1.2",
|
||||
"request": "^2.88.0"
|
||||
}
|
||||
},
|
||||
"request-promise-core": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.2.tgz",
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
"express": "^4.16.4",
|
||||
"feedparser": "^2.2.9",
|
||||
"request": "^2.88.0",
|
||||
"request-promise-cache": "^2.0.1",
|
||||
"request-promise-native": "^1.0.7",
|
||||
"serve-static": "^1.13.2",
|
||||
"timeago.js": "^3.0.2"
|
||||
|
|
|
|||
Loading…
Reference in New Issue