FreshRSS/rssguard
1
0
Fork 0
mirror of https://github.com/martinrotter/rssguard.git synced 2025-01-24 14:41:23 +01:00
Code Issues Packages Projects Releases Wiki Activity

better twitter

Browse Source
This commit is contained in:
Martin Rotter 2021-04-07 10:12:10 +02:00
parent e5dc5fc6e7
commit d75d65cd1d
1 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
resources/scripts/scrapers/twitter.py
View File

@ -1,31 +1,35 @@
# Generates JSON feed from Twitter timeline URL. # Generates JSON feed from Twitter timeline URL.
# #
# This script expects two input parameters: # This script expects one input parameter:
# twitter.py [twitter-user-name] [twitter-user-id] # twitter.py [twitter-user-name]
# #
# For example: # For example:
# twitter.py 'NASA' '11348282' # twitter.py 'NASA'
import json import json
import re import re
import sys import sys
import time import time
import html import html
import urllib.request
import requests import requests
import distutils.util import distutils.util
from datetime import datetime from datetime import datetime
twitter_url = "https://twitter.com/" + sys.argv[1] twitter_url = "https://twitter.com/" + sys.argv[1]
twitter_id = sys.argv[2]
twitter_username = twitter_url[twitter_url.rfind("/") + 1:] twitter_username = twitter_url[twitter_url.rfind("/") + 1:]
twitter_bearer = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" twitter_bearer = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
# Download RAW Twitter HTML data and extract token. # Download RAW Twitter HTML data and extract token.
url_response = urllib.request.urlopen(twitter_url) url_response = url_response = requests.get(twitter_url)
twitter_html = url_response.read().decode("utf-8") twitter_html = url_response.text
twitter_token = re.search("gt=(\d+);", twitter_html).group(1) twitter_token = re.search("gt=(\d+);", twitter_html).group(1)
url_response = requests.get("https://twitter.com/i/api/graphql/hc-pka9A7gyS3xODIafnrQ/UserByScreenName?variables=%7B%22screen_name%22%3A%22sportscenter%22%2C%22withHighlightedLabel%22%3Atrue%7D", headers = {
"x-guest-token": twitter_token,
"Authorization": twitter_bearer
})
twitter_id = json.loads(url_response.text)["data"]["user"]["rest_id"]
# Obtain JSON Twitter data with token. # Obtain JSON Twitter data with token.
twitter_json_url = "https://twitter.com/i/api/2/timeline/profile/{user_id}.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweet=true&include_tweet_replies=false&count=50&userId={user_id}&ext=mediaStats&2ChighlightedLabel".format(user_id = twitter_id) twitter_json_url = "https://twitter.com/i/api/2/timeline/profile/{user_id}.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweet=true&include_tweet_replies=false&count=50&userId={user_id}&ext=mediaStats&2ChighlightedLabel".format(user_id = twitter_id)