1
0
mirror of https://github.com/hughrun/ephemetoot synced 2025-01-30 13:24:46 +01:00

move check_batch to top level function

This commit is contained in:
Hugh Rundle 2020-08-31 15:10:18 +10:00
parent 0a01f18e44
commit 43a792e44d
2 changed files with 228 additions and 238 deletions

View File

@ -122,7 +122,7 @@ else:
def main(): def main():
''' '''
Call ephemetoot.checkToots() on each user in the config file, with options set via flags from command line. Call ephemetoot.check_toots() on each user in the config file, with options set via flags from command line.
''' '''
try: try:
@ -149,7 +149,7 @@ def main():
with open(config_file) as config: with open(config_file) as config:
for accounts in yaml.safe_load_all(config): for accounts in yaml.safe_load_all(config):
for user in accounts: for user in accounts:
func.checkToots(user, options) func.check_toots(user, options)
except FileNotFoundError as err: except FileNotFoundError as err:

View File

@ -218,17 +218,10 @@ def datestamp_now():
) )
) )
# TODO: move this out of checkToots and pass through all needed arg def check_batch(config, options, mastodon, user_id, timeline, deleted_count=0):
# def checkBatch(): """
# pass Check a batch of up to 40 toots. This is usually triggered by check_toots, and then recursively calls itself until all toots within the time period specified have been checked.
"""
# TODO: rename to check_toots
def checkToots(config, options, retry_count=0):
'''
The main function, uses the Mastodon API to check all toots in the user timeline, and delete any that do not meet any of the exclusion criteria from the config file.
'''
keep_pinned = "keep_pinned" in config and config["keep_pinned"] keep_pinned = "keep_pinned" in config and config["keep_pinned"]
toots_to_keep = config["toots_to_keep"] if "toots_to_keep" in config else [] toots_to_keep = config["toots_to_keep"] if "toots_to_keep" in config else []
visibility_to_keep = ( visibility_to_keep = (
@ -238,17 +231,8 @@ def checkToots(config, options, retry_count=0):
set(config["hashtags_to_keep"]) if "hashtags_to_keep" in config else set() set(config["hashtags_to_keep"]) if "hashtags_to_keep" in config else set()
) )
days_to_keep = config["days_to_keep"] if "days_to_keep" in config else 365 days_to_keep = config["days_to_keep"] if "days_to_keep" in config else 365
cutoff_date = datetime.now(timezone.utc) - timedelta(days=days_to_keep)
try:
print(
"Fetching account details for @"
+ config["username"]
+ "@"
+ config["base_url"]
)
# TODO: rename this to check_batch
def checkBatch(timeline, deleted_count=0):
for toot in timeline: for toot in timeline:
# TODO: move all this into a new testable function process_toot() # TODO: move all this into a new testable function process_toot()
if "id" in toot and "archive" in config: if "id" in toot and "archive" in config:
@ -448,12 +432,12 @@ def checkToots(config, options, retry_count=0):
# the account_statuses call is paginated with a 40-toot limit # the account_statuses call is paginated with a 40-toot limit
# get the id of the last toot to include as 'max_id' in the next API call. # get the id of the last toot to include as 'max_id' in the next API call.
# then keep triggering new rounds of checkToots() until there are no more toots to check # then keep triggering new rounds of check_toots() until there are no more toots to check
try: try:
max_id = timeline[-1:][0].id max_id = timeline[-1:][0].id
next_batch = mastodon.account_statuses(user_id, limit=40, max_id=max_id) next_batch = mastodon.account_statuses(user_id, limit=40, max_id=max_id)
if len(next_batch) > 0: if len(next_batch) > 0:
checkBatch(next_batch, deleted_count) check_batch(config, options, mastodon, user_id, next_batch, deleted_count)
else: else:
if options.test: if options.test:
if options.datestamp: if options.datestamp:
@ -489,31 +473,37 @@ def checkToots(config, options, retry_count=0):
except Exception as e: except Exception as e:
print("ERROR: " + str(e.args[0])) print("ERROR: " + str(e.args[0]))
def check_toots(config, options, retry_count=0):
'''
The main function, uses the Mastodon API to check all toots in the user timeline, and delete any that do not meet any of the exclusion criteria from the config file.
'''
try:
print("Fetching account details for @", config["username"], "@", config["base_url"], sep="")
if options.pace: if options.pace:
mastodon = Mastodon( mastodon = Mastodon(
access_token=config["access_token"], access_token=config["access_token"],
api_base_url="https://" + config["base_url"], api_base_url="https://" + config["base_url"],
ratelimit_method="pace", ratelimit_method="pace",
) )
else: else:
mastodon = Mastodon( mastodon = Mastodon(
access_token=config["access_token"], access_token=config["access_token"],
api_base_url="https://" + config["base_url"], api_base_url="https://" + config["base_url"],
ratelimit_method="wait", ratelimit_method="wait",
) )
# STARTS HERE user_id = mastodon.account_verify_credentials().id # verify user and get ID
cutoff_date = datetime.now(timezone.utc) - timedelta(days=days_to_keep) account = mastodon.account(user_id) # get the account
user_id = mastodon.account_verify_credentials().id timeline = mastodon.account_statuses(user_id, limit=40) # initial batch
account = mastodon.account(user_id)
timeline = mastodon.account_statuses(user_id, limit=40)
if not options.quiet: if not options.quiet:
print("Checking " + str(account.statuses_count) + " toots") print("Checking", str(account.statuses_count), "toots")
checkBatch(timeline) # check first batch, check_batch then recursively keeps looping until all toots have been checked
check_batch(config, options, mastodon, user_id, timeline)
except KeyError as val: except KeyError as val:
print("\n⚠️ error with in your config.yaml file!") print("\n⚠️ error with in your config.yaml file!")
@ -537,6 +527,6 @@ def checkToots(config, options, retry_count=0):
time.sleep(60 * options.retry_mins) time.sleep(60 * options.retry_mins)
retry_count += 1 retry_count += 1
print("Attempt " + str(retry_count + 1)) print("Attempt " + str(retry_count + 1))
checkToots(config, options, retry_count) check_toots(config, options, retry_count)
else: else:
print("Gave up waiting for network\n") print("Gave up waiting for network\n")