38 lines
1.3 KiB
Python
38 lines
1.3 KiB
Python
# Obtains Wikipedia's "In the news" today's articles.
|
|
|
|
import urllib.request
|
|
import re
|
|
import json
|
|
from newspaper import Article
|
|
|
|
url = "https://en.wikipedia.org/wiki/Main_Page"
|
|
response = urllib.request.urlopen(url)
|
|
text = response.read().decode("utf-8")
|
|
|
|
text_li = re.search("In the news[\S\n\t\v ]+?<ul>([\S\n\t\v ]+?)<\/ul>", text).group(1)
|
|
articles_li = re.findall("<li>([\S\n\t\v ]+?)<\/li>", text_li)
|
|
|
|
# Iterate all articles and generate JSON feed entries.
|
|
wiki_base_url = "https://en.wikipedia.org"
|
|
|
|
|
|
json_feed = "{{\"title\": \"Wikipedia - In the news\", \"items\": [{items}]}}"
|
|
items = list()
|
|
|
|
for article in articles_li:
|
|
article_url = wiki_base_url + re.search("^.+?href=\"(.+?)\"", article).group(1)
|
|
|
|
f = Article(article_url, keep_article_html = True)
|
|
f.download()
|
|
f.parse()
|
|
|
|
article_url = json.dumps(article_url)
|
|
article_title = json.dumps(f.title)
|
|
article_html = json.dumps(f.article_html)
|
|
items.append("{{\"title\": {title}, \"content_html\": {html}, \"url\": {url}}}".format(title=article_title,
|
|
html=article_html,
|
|
url=article_url))
|
|
|
|
json_feed = json_feed.format(items=", ".join(items))
|
|
|
|
print(json_feed) |