2021-02-03 01:06:44 +01:00
|
|
|
#!/usr/bin/python3
|
|
|
|
# -*- coding: UTF-8 -*-
|
2021-02-03 12:20:28 +01:00
|
|
|
|
2021-02-03 01:06:44 +01:00
|
|
|
import json
|
|
|
|
import sqlite3
|
|
|
|
import requests
|
|
|
|
import xmltodict
|
|
|
|
import time
|
2021-02-03 12:20:28 +01:00
|
|
|
from misskey import Misskey
|
2021-02-03 01:06:44 +01:00
|
|
|
|
2021-02-03 04:56:54 +01:00
|
|
|
time_start = time.time()
|
|
|
|
print(
|
|
|
|
''' __ __ _ _ __ __ _ _ \n | \/ (_) | | / //_ | | | | \n | \ / |_ ___ ___| | _____ _ _ __ __/ /_ | | |__ ___ | |_ \n | |\/| | / __/ __| |/ / _ \ | | | \ \/ / '_ \| | '_ \ / _ \| __|\n | | | | \__ \__ \ < __/ |_| | > <| (_) | | |_) | (_) | |_ \n |_| |_|_|___/___/_|\_\___|\__, | /_/\_\\___/|_|_.__/ \___/ \__|\n __/ | \n |___/ ''')
|
2021-02-03 01:06:44 +01:00
|
|
|
conn = sqlite3.connect('DB.sqlite3')
|
|
|
|
print("Opened database successfully")
|
|
|
|
|
|
|
|
|
2021-02-03 12:20:28 +01:00
|
|
|
def json_read(file):
|
|
|
|
config_file = open(file, 'r')
|
|
|
|
config = json.loads(config_file.read())
|
|
|
|
config_file.close()
|
|
|
|
return config
|
|
|
|
|
2021-02-03 01:06:44 +01:00
|
|
|
def xml_to_json(xml):
|
|
|
|
pars = xmltodict.parse(xml)
|
|
|
|
return json.dumps(pars)
|
|
|
|
|
|
|
|
|
|
|
|
def spider(rule_name, rss_url):
|
2021-02-03 12:20:28 +01:00
|
|
|
print("Fetch RSS: [" + rule_name + "] ", rss_url)
|
|
|
|
start = time.time()
|
2021-02-03 01:06:44 +01:00
|
|
|
c = conn.cursor()
|
2021-02-03 04:56:54 +01:00
|
|
|
fetch = requests.get(rss_url)
|
|
|
|
if fetch.status_code != 200:
|
|
|
|
print("Failed to fetch")
|
|
|
|
return False
|
|
|
|
result = xmltodict.parse(fetch.content)
|
2021-02-03 01:06:44 +01:00
|
|
|
c.execute('INSERT INTO "main"."spider_log" ("rule_name", "rss_url", "result_json", "timestamp") '
|
|
|
|
'VALUES (?, ?, ?, ?)', (rule_name, rss_url, json.dumps(result), time.time()))
|
|
|
|
c.close()
|
2021-02-03 12:20:28 +01:00
|
|
|
end = time.time()
|
|
|
|
print("Fetch done in", end - start, "s")
|
2021-02-03 01:06:44 +01:00
|
|
|
return result
|
|
|
|
|
|
|
|
|
2021-02-03 12:20:28 +01:00
|
|
|
def fetch_detail(url):
|
|
|
|
print()
|
|
|
|
|
|
|
|
|
2021-02-03 01:06:44 +01:00
|
|
|
if __name__ == '__main__':
|
2021-02-03 04:56:54 +01:00
|
|
|
print("Misskey X61 RSS Bot initialized")
|
2021-02-03 12:20:28 +01:00
|
|
|
spider("DuoWei", "https://rsshub.app/dwnews/yaowen/global")
|
|
|
|
|
|
|
|
config = json_read("config.json")
|
|
|
|
rules = json_read("rules.json")
|
|
|
|
|
|
|
|
Misskey.config = config['misskey.io']
|
2021-02-03 04:56:54 +01:00
|
|
|
|
2021-02-03 12:20:28 +01:00
|
|
|
req = Misskey.post(baseurl="https://misskey.io", content="Beep.. Beep Beep! Beep:" + str(time.time()), self=Misskey)
|
2021-02-03 01:06:44 +01:00
|
|
|
conn.commit()
|
|
|
|
conn.close()
|
2021-02-03 04:56:54 +01:00
|
|
|
|
|
|
|
time_end = time.time()
|
|
|
|
print("X61 bot: done in", time_end - time_start, "s")
|