Merge pull request #47 from hughrun/3alpha1

3 beta 0

.gitignore

@@ -1,4 +1,5 @@
 config.yaml
 _assets
 pypi-readme.md
+archive
 dist

docs/index.md

@@ -2,7 +2,11 @@
 
 **ephemetoot** is a Python command line tool for deleting old toots.
 
-These docs apply to Version 3.
+These docs apply to `ephemetoot` version 3.
+
+Note that throughout these docs the `pip` command is referred to as `pip3`. This is to help new Python users on systems running both Python 2 and Python 3, which is currently still common and a frequent source of confusion. On some systems, `pip` will be the appropriate command, as it points to Python 3 environments.
+
+If you are upgrading from an `ephemetoot` version prior to v3.0.0 please see the [upgrading](./upgrade.md) instructions and note that you need to manually uninstall the old version first.
 
 * [Installation](./install.md)
 * [Options](./options.md)

docs/install.md

@@ -14,7 +14,7 @@ pip3 install ephemetoot
 If you do not have permission to install python modules, you may need to use the `--user` flag. Generally this is not advisable, since you will need to run ephemetoot with the same user since it will only be installed for that user and not globally:
 
 ```shell
-pip3 install . --user
+pip3 install ephemetoot --user
 ```
 
 ## Obtain an access token
@@ -27,7 +27,7 @@ Now you've installed `ephemetoot`, in order to actually use it you will need an
 4. Enter an application name (e.g. 'ephemetoot'), and give the app both 'read' and 'write' Scopes
 5. Click `SUBMIT`
 6. Click on the name of the new app, which should be a link
-7. Copy the `Your access token` string
+7. Copy the `Your access token` string - you will need this for your configuration file (see below)
 
 **NOTE**: Anyone who has your access token and the domain name of your Mastodon server will be able to:
 * read all your private and direct toots, 

docs/upgrade.md

@@ -2,14 +2,15 @@
 
 ## Upgrading
 
-### Note for users upgrading from Version 2 to Version 3
+### Note for users upgrading to Version 3
 
-To upgrade from Version 2.x to Version 3.x you should be able to simply follow the _Upgrading with pypi_ instructions below. However the safest procedure is:
+To upgrade from an earlier version to Version 3.x you will need to remove your existing install.
 
 1. save a copy of your `config.yaml` file somewhere safe
-2. run `pip uninstall ephemetoot`
-3. run `pip install ephemetoot`
-4. do a test run with `ephemetoot --test`
+2. run `pip3 uninstall ephemetoot`
+3. run `pip3 install ephemetoot`
+4. check your config file is in the current directory
+5. check everythign is working with `ephemetoot --test` or `ephemetoot --version`
 
 ### Upgrading with pypi
 To upgrade to a new version, the easiest way is to use pip to download the latest version from pypi (remembering that for your machine you may need to substitute `pip` for `pip3`):
@@ -33,7 +34,7 @@ To upgrade without using git or pypi:
 * put your config file somewhere safe
 * download and unzip the zip file into your `ephemetoot` directory over the top of your existing installation
 * move your config file back in to the ephemetoot directory
-* run `pip install .` from within the directory
+* run `pip3 install .` from within the directory
 
 ## Uninstalling
 

ephemetoot/console.py

@@ -30,7 +30,7 @@ import yaml
 from argparse import ArgumentParser
 from datetime import datetime, timezone
 import os
-import pkg_resources 
+import pkg_resources
 
 # import funtions
 from ephemetoot import ephemetoot as func
@@ -40,70 +40,127 @@ vnum = pkg_resources.require("ephemetoot")[0].version
 
 parser = ArgumentParser()
 parser.add_argument(
-    "--archive-deleted", action="store_true", help="Only archive toots that are being deleted"
+    "--archive-deleted",
+    action="store_true",
+    help="Only archive toots that are being deleted",
 )
 parser.add_argument(
-    "--config", action="store", metavar="filepath", default="config.yaml", help="Filepath of your config file, absolute or relative to the current directory. If no --config path is provided, ephemetoot will use 'config.yaml'."
+    "--config",
+    action="store",
+    metavar="filepath",
+    default="config.yaml",
+    help="Filepath of your config file, absolute or relative to the current directory. If no --config path is provided, ephemetoot will use 'config.yaml'in the current directory",
 )
 parser.add_argument(
-    "--datestamp", action="store_true", help="Include a datetime stamp for every action (e.g. deleting a toot)"
+    "--datestamp",
+    action="store_true",
+    help="Include a datetime stamp for every action (e.g. deleting a toot)",
 )
 parser.add_argument(
-    "--hide-skipped", "--hide_skipped", action="store_true", help="Do not write to log when skipping saved toots"
+    "--hide-skipped",
+    "--hide_skipped",
+    action="store_true",
+    help="Do not write to log when skipping saved toots",
 )
 parser.add_argument(
-    "--init", action="store_true", help="Initialise creation of a config file saved in the current directory."
+    "--init",
+    action="store_true",
+    help="Create a config file that is saved in the current directory",
 )
 parser.add_argument(
-    "--pace", action="store_true", help="Slow deletion actions to match API rate limit to avoid pausing"
+    "--pace",
+    action="store_true",
+    help="Slow deletion actions to match API rate limit to avoid pausing",
+)
+parser.add_argument("--quiet", action="store_true", help="Suppress most logging")
+parser.add_argument(
+    "--retry-mins",
+    action="store",
+    metavar="minutes",
+    nargs="?",
+    const="1",
+    default="1",
+    type=int,
+    help="Number of minutes to wait between retries, when an error is thrown",
 )
 parser.add_argument(
-    "--quiet", action="store_true", help="Suppress most logging"
-)
-parser.add_argument(
-    "--retry-mins", action="store", metavar="minutes", nargs="?", const="1", default="1", type=int, help="Number of minutes to wait between retries, when an error is thrown"
-)
-parser.add_argument(
-    "--schedule", action="store", metavar="filepath", nargs="?", const=".", help="Save and load plist file on MacOS"
+    "--schedule",
+    action="store",
+    metavar="filepath",
+    nargs="?",
+    const=".",
+    help="Save and load plist file on MacOS",
 )
 parser.add_argument(
     "--test", action="store_true", help="Do a test run without deleting any toots"
 )
 parser.add_argument(
-    "--time", action="store", metavar=('hour', 'minute'), nargs="*", help="Hour and minute to schedule: e.g. 9 30 for 9.30am"
+    "--time",
+    action="store",
+    metavar=("hour", "minute"),
+    nargs="*",
+    help="Hour and minute to schedule: e.g. 9 30 for 9.30am",
 )
 parser.add_argument(
-    "--version", action="store_true", help="Display the version number"
+    "--version",
+    action="store_true",
+    help="Display the version numbers of the installed and latest versions",
 )
 
 options = parser.parse_args()
-if options.config[0] == '~':
+if options.config[0] == "~":
     config_file = os.path.expanduser(options.config)
-elif options.config[0] == '/':
-    config_file = options.config
-else: 
-    config_file = os.path.join( os.getcwd(), options.config )
+elif options.config[0] == "/":
+    # make sure user isn't passing in something dodgy
+    if os.path.exists(options.config):
+        config_file = options.config
+    else:
+        config_file = ""
+else:
+    config_file = os.path.join(os.getcwd(), options.config)
+
 
 def main():
-    if options.init:
-        func.init()
-    elif options.version:
-        func.version(vnum)
-    elif options.schedule:
-        func.schedule(options)
-    else:
-        if not options.quiet:
-            print('')
-            print('============= EPHEMETOOT v' + vnum + ' ================')
-            print('Running at ' + str( datetime.now(timezone.utc).strftime('%a %d %b %Y %H:%M:%S %z') ))
-            print('================================================')
-            print('')
-        if options.test:
-            print("This is a test run...\n")
-        with open(config_file) as config:
-            for accounts in yaml.safe_load_all(config):
-                for user in accounts:
-                    func.checkToots(user, options)
+    '''
+    Call ephemetoot.check_toots() on each user in the config file, with options set via flags from command line.
+    '''
+    try:
 
-if __name__ == '__main__':
-    main()
+        if options.init:
+            func.init()
+        elif options.version:
+            func.version(vnum)
+        elif options.schedule:
+            func.schedule(options)
+        else:
+            if not options.quiet:
+                print("")
+                print("============= EPHEMETOOT v" + vnum + " ================")
+                print(
+                    "Running at "
+                    + str(
+                        datetime.now(timezone.utc).strftime("%a %d %b %Y %H:%M:%S %z")
+                    )
+                )
+                print("================================================")
+                print("")
+            if options.test:
+                print("This is a test run...\n")
+            with open(config_file) as config:
+                for accounts in yaml.safe_load_all(config):
+                    for user in accounts:
+                        func.check_toots(user, options)
+
+    except FileNotFoundError as err:
+
+        if err.filename == config_file:
+            print("🕵️  Missing config file")
+            print("Run \033[92mephemetoot --init\033[0m to create a new one\n")
+
+        else:
+            print("\n🤷‍♂️  The archive directory in your config file does not exist")
+            print("Create the directory or correct your config before trying again\n")
+
+
+if __name__ == "__main__":
+    main()

ephemetoot/ephemetoot.py

@@ -19,140 +19,124 @@ import requests
 # local
 from ephemetoot import plist
 
+def compulsory_input(tags, name, example):
+
+    value = ""
+    while len(value) < 1:
+        if example:
+            value = input(tags[0] + name + tags[1] + example + tags[2])
+        else:
+            value = input(tags[0] + name + tags[2])
+
+    return value
+
+def digit_input(tags, name, example):
+
+    value = ""
+    while value.isdigit() == False:
+        if example:
+            value = input(tags[0] + name + tags[1] + example + tags[2])
+        else:
+            value = input(tags[0] + name + tags[2])
+
+    return value
+
+def yes_no_input(tags, name):
+    value = ""
+    while value not in ["y", "n"]:
+        value = input(
+            tags[0] + name + tags[1] + "(y or n):" + tags[2]
+        )
+    return_val = "true" if value == "y" else "false"
+    return return_val
+
+def optional_input(tags, name, example):
+    value = input(tags[0] + name + tags[1] + example + tags[2])
+    return value
+        
 def init():
+    '''
+    Creates a config.yaml file in the current directory, based on user input.
+    '''
+    try:
 
-    init_start = "\033[96m"
-    init_end = "\033[0m"
-    init_eg = "\033[2m"
+        # text colour markers (beginning, example, end)
+        tags = ("\033[96m", "\033[2m", "\033[0m")
 
-    conf_token = ""
-    while len(conf_token) < 1:
-        conf_token = input(init_start + "Access token: " + init_end)
+        conf_token = compulsory_input(tags, "Access token: ", None)
+        conf_user = compulsory_input(tags, "Username", "(without the '@' - e.g. alice):")
+        conf_url = compulsory_input(tags, "Base URL", "(e.g. example.social):")
+        conf_days = digit_input(tags, "Days to keep", "(default 365):")
+        conf_pinned = yes_no_input(tags, "Keep pinned toots?")
+        conf_keep_toots = optional_input(tags, "Toots to keep", "(optional list of IDs separated by commas):")
+        conf_keep_hashtags = optional_input(tags, "Hashtags to keep", "(optional list separated by commas):")
+        conf_keep_visibility = optional_input(tags, "Visibility to keep", "(optional list separated by commas):")
+        conf_archive = optional_input(tags, "Archive path", "(optional filepath for archive):")
 
-    conf_user = ""
-    while len(conf_user) < 1:
-        conf_user = input(
-          init_start
-          + "Username"
-          + init_eg
-          + "(without the '@' - e.g. alice):"
-          + init_end
-        )
+        # write out the config file
+        with open("config.yaml", "w") as configfile:
 
-    conf_url = ""
-    while len(conf_url) < 1:
-        conf_url = input(
-          init_start
-          + "Base URL"
-          + init_eg
-          + "(e.g. example.social):"
-          + init_end
-        )
+            configfile.write("-")
+            configfile.write("\n  access_token: " + conf_token)
+            configfile.write("\n  username: " + conf_user)
+            configfile.write("\n  base_url: " + conf_url)
+            configfile.write("\n  days_to_keep: " + conf_days)
+            configfile.write("\n  keep_pinned: " + conf_pinned)
 
-    conf_days = ""
-    while conf_days.isdigit() == False:
-        conf_days = input(
-          init_start
-          + "Days to keep"
-          + init_eg
-          + "(default 365):"
-          + init_end
-        )
+            if len(conf_keep_toots) > 0:
+                keep_list = conf_keep_toots.split(",")
+                configfile.write("\n  toots_to_keep:")
+                for toot in keep_list:
+                    configfile.write("\n    - " + toot.strip())
 
-    conf_keep_pinned = ""
-    while conf_keep_pinned not in ["y", "n"]:
-        conf_keep_pinned = input(
-            init_start 
-            + "Keep pinned toots?"
-            + init_eg 
-            + "(y or n):"
-            + init_end
-        )
+            if len(conf_keep_hashtags) > 0:
+                tag_list = conf_keep_hashtags.split(",")
+                configfile.write("\n  hashtags_to_keep:")
+                for tag in tag_list:
+                    configfile.write("\n    - " + tag.strip())
 
-    conf_pinned = "true" if conf_keep_pinned == "y" else "false"
+            if len(conf_keep_visibility) > 0:
+                viz_list = conf_keep_visibility.split(",")
+                configfile.write("\n  visibility_to_keep:")
+                for mode in viz_list:
+                    configfile.write("\n    - " + mode.strip())
 
-    conf_keep_toots = input(
-        init_start 
-        + "Toots to keep"
-        + init_eg 
-        + " (optional list of IDs separated by commas):"
-        + init_end
-      )
+            if len(conf_archive) > 0:
+                configfile.write("\n  archive: " + conf_archive)
 
-    conf_keep_hashtags = input(
-        init_start 
-        + "Hashtags to keep"
-        + init_eg 
-        + " (optional list separated by commas):"
-        + init_end
-      )
+            configfile.close()
 
-    conf_keep_visibility = input(
-        init_start 
-        + "Visibility to keep"
-        + init_eg 
-        + " (optional list separated by commas):" 
-        + init_end
-      )
-
-    conf_archive = input(
-        init_start 
-        + "Archive path"
-        + init_eg 
-        + " (optional filepath for archive):"
-        + init_end
-      )
-
-    # write out the config file
-    with open("config.yaml", "w") as configfile:
-
-        configfile.write("-")
-        configfile.write("\n  access_token: " + conf_token)
-        configfile.write("\n  username: " + conf_user)
-        configfile.write("\n  base_url: " + conf_url)
-        configfile.write("\n  days_to_keep: " + conf_days)
-        configfile.write("\n  keep_pinned: " + conf_pinned)
-
-        if len(conf_keep_toots) > 0:
-            keep_list = conf_keep_toots.split(',')
-            configfile.write("\n  toots_to_keep:")
-            for toot in keep_list:
-              configfile.write("\n    - " + toot.strip())
-
-        if len(conf_keep_hashtags) > 0:
-            tag_list = conf_keep_hashtags.split(',')
-            configfile.write("\n  hashtags_to_keep:") 
-            for tag in tag_list:
-              configfile.write("\n    - " + tag.strip())
-
-        if len(conf_keep_visibility) > 0:
-            viz_list = conf_keep_visibility.split(',')
-            configfile.write("\n  visibility_to_keep:") 
-            for mode in viz_list:
-              configfile.write("\n    - " + mode.strip())
-
-        if len(conf_archive) > 0:
-            configfile.write("\n  archive: " + conf_archive)
-
-        configfile.close()
+    except Exception as e:
+        print(e)
 
 def version(vnum):
+    '''
+    Prints current and latest version numbers to console.
+    '''
+
     try:
         latest = requests.get(
             "https://api.github.com/repos/hughrun/ephemetoot/releases/latest"
         )
         res = latest.json()
-        latest_version = res["name"]
+        latest_version = res["tag_name"]
         print("\nephemetoot ==> 🥳 ==> 🧼 ==> 😇")
         print("-------------------------------")
-        print("Using:  \033[92mVersion " + vnum + "\033[0m")
-        print("Latest: \033[92m" + latest_version + "\033[0m")
-        print("To upgrade to the most recent version run \033[92mpip3 install --update ephemetoot\033[0m")
+        print("You are using release: \033[92mv", vnum, "\033[0m", sep="")
+        print("The latest release is: \033[92m" + latest_version + "\033[0m")
+        print(
+            "To upgrade to the most recent version run \033[92mpip3 install --update ephemetoot\033[0m"
+        )
 
     except Exception as e:
         print("Something went wrong:")
 
+
 def schedule(options):
+
+    '''
+    Creates and loads a plist file for scheduled running with launchd. If --time flag is used, the scheduled time is set accordingly. Note that this is designed for use on MacOS.
+    '''
     try:
 
         if options.schedule == ".":
@@ -172,7 +156,13 @@ def schedule(options):
             lines[23] = "    <integer>" + options.time[1] + "</integer>"
 
         # write out file directly to ~/Library/LaunchAgents
-        f = open(os.path.expanduser("~/Library/LaunchAgents/") + "ephemetoot.scheduler.plist", mode="w")
+        f = open(
+            os.path.join(
+              os.path.expanduser("~/Library/LaunchAgents"),
+              "ephemetoot.scheduler.plist"
+            ),
+            mode="w",
+        )
         for line in lines:
             if line == lines[-1]:
                 f.write(line)
@@ -185,19 +175,90 @@ def schedule(options):
             ["launchctl unload ~/Library/LaunchAgents/ephemetoot.scheduler.plist"],
             stdout=subprocess.DEVNULL,
             stderr=subprocess.DEVNULL,
-            shell=True
+            shell=True,
         )
         # load the new file
         subprocess.run(
             ["launchctl load ~/Library/LaunchAgents/ephemetoot.scheduler.plist"],
-            shell=True
+            shell=True,
         )
         print("⏰ Scheduled!")
     except Exception as e:
-        print("🙁 Scheduling failed.")
+        print("🙁 Scheduling failed.", e)
 
+def archive_toot(config, toot):
+    # define archive path
+    if config["archive"][0] == "~":
+        archive_path = os.path.expanduser(config["archive"])
+    elif config["archive"][0] == "/":
+        archive_path = config["archive"]
+    else:
+        archive_path = os.path.join(os.getcwd(), config["archive"])
+    if archive_path[-1] != "/":
+        archive_path += "/"
 
-def checkToots(config, options, retry_count=0):
+    filename = os.path.join(archive_path, str(toot.id) + ".json")
+
+    # write to file
+    with open(filename, "w") as f:
+        f.write(json.dumps(toot, indent=4, default=jsondefault))
+        f.close()
+
+def jsondefault(obj):
+    if isinstance(obj, (date, datetime)):
+        return obj.isoformat()
+
+def tooted_date(toot):
+    return toot.created_at.strftime("%d %b %Y")
+
+def datestamp_now():
+    return str(
+        datetime.now(timezone.utc).strftime(
+            "%a %d %b %Y %H:%M:%S %z"
+        )
+    )
+
+def console_print(msg, options, skip):
+
+    skip_announcement = True if (options.hide_skipped and skip) else False
+    if not (skip_announcement or options.quiet):
+
+        if options.datestamp:
+            msg = datestamp_now() + " : " + msg
+
+        print(msg)
+
+def print_rate_limit_message(mastodon):
+
+    now = time.time()
+    diff = mastodon.ratelimit_reset - now
+
+    print(
+        "\nRate limit reached at",
+        datestamp_now(),
+        "- next reset due in",
+        str(format(diff / 60, ".0f")),
+        "minutes.\n"
+    )
+
+def retry_on_error(options, mastodon, toot, attempts):
+
+    if attempts < 6:
+        try:
+            console_print(
+              "Attempt " + str(attempts) + " at " + datestamp_now(),
+              options,
+              False
+            )
+            mastodon.status_delete(toot)
+        except:
+            attempts += 1
+            time.sleep(60 * options.retry_mins)
+            retry_on_error(options, mastodon, toot, attempts)
+    else:
+        raise TimeoutError("Gave up after 5 attempts")
+
+def process_toot(config, options, mastodon, deleted_count, toot):
 
     keep_pinned = "keep_pinned" in config and config["keep_pinned"]
     toots_to_keep = config["toots_to_keep"] if "toots_to_keep" in config else []
@@ -208,334 +269,176 @@ def checkToots(config, options, retry_count=0):
         set(config["hashtags_to_keep"]) if "hashtags_to_keep" in config else set()
     )
     days_to_keep = config["days_to_keep"] if "days_to_keep" in config else 365
+    cutoff_date = datetime.now(timezone.utc) - timedelta(days=days_to_keep)
+
+    if toot.id and "archive" in config:
+
+        if not options.archive_deleted:
+            # write toot to archive
+            archive_toot(config, toot)
+
+    toot_tags = set()
+    for tag in toot.tags:
+        toot_tags.add(tag.name)
 
     try:
+        if keep_pinned and hasattr(toot, "pinned") and toot.pinned:
+            console_print(
+              "📌 skipping pinned toot - " + str(toot.id), 
+              options, 
+              True
+            )
+
+        elif toot.id in toots_to_keep:
+            console_print(
+              "💾 skipping saved toot - " + str(toot.id), 
+              options, 
+              True
+            )
+
+        elif toot.visibility in visibility_to_keep:
+            console_print(
+              "👀 skipping " + toot.visibility + " toot - " + str(toot.id), 
+              options, 
+              True
+            )
+
+        elif len(hashtags_to_keep.intersection(toot_tags)) > 0:
+            console_print(
+              "#️⃣  skipping toot with hashtag - " + str(toot.id), 
+              options, 
+              True
+            )
+
+        elif cutoff_date > toot.created_at:
+            if hasattr(toot, "reblog") and toot.reblog:
+                console_print( 
+                  "👎 unboosting toot " + str(toot.id) + " boosted " + tooted_date(toot), 
+                  options, 
+                  False
+                )
+
+                deleted_count += 1
+                # unreblog the original toot (their toot), not the toot created by boosting (your toot)
+                if not options.test:
+                    if mastodon.ratelimit_remaining == 0:
+                        console_print(
+                          "Rate limit reached. Waiting for a rate limit reset", 
+                          options, 
+                          False
+                        )
+
+                    # check for --archive-deleted
+                    if (options.archive_deleted and "id" in toot and "archive" in config):
+                        # write toot to archive
+                        archive_toot(config, toot)
+
+                    mastodon.status_unreblog(toot.reblog)
+
+            else:
+                console_print(
+                  "❌ deleting toot " + str(toot.id) + " tooted " + tooted_date(toot),
+                  options,
+                  False
+                )
+
+                deleted_count += 1
+                time.sleep(2)  # wait 2 secs between deletes to be a bit nicer to the server
+
+                if not options.test:
+                    # deal with rate limits
+                    if (mastodon.ratelimit_remaining == 0 and not options.quiet):
+                        print_rate_limit_message(mastodon)
+
+                    # check for --archive-deleted
+                    if (options.archive_deleted and "id" in toot and "archive" in config):
+                        archive_toot(config, toot)
+
+                    # finally we actually delete the toot
+                    mastodon.status_delete(toot)
+
+        # return the deleted_count back so that it can be tallied within check_batch()
+        return deleted_count
+
+    except MastodonRatelimitError:
+
+        print_rate_limit_message(mastodon)
+        time.sleep(diff + 1)  # wait for rate limit to reset
+
+    except MastodonError as e:
+
+        print( "🛑 ERROR deleting toot -", str(toot.id), "-", str(e.args[0]), "-", str(e.args[3]) )
+        console_print(
+          "Waiting " + str(options.retry_mins) + " minutes before re-trying",
+          options,
+          False
+        )
+        time.sleep(60 * options.retry_mins)
+        retry_on_error(options, mastodon, toot, attempts=2)
+
+    except KeyboardInterrupt:
+        print("Operation aborted.")
+
+    except KeyError as e:
         print(
-            "Fetching account details for @"
-            + config["username"]
-            + "@"
-            + config["base_url"]
+            "⚠️  There is an error in your config.yaml file. Please add a value for",
+            str(e),
+            "and try again."
         )
 
-        def jsondefault(obj):
-            if isinstance(obj, (date, datetime)):
-                return obj.isoformat()
+    except:
+        e = sys.exc_info()
+        print( "🛑 Unknown ERROR deleting toot -", str(toot.id) )
+        print( "ERROR:", str(e[0]),"-", str(e[1]) )
 
-        def checkBatch(timeline, deleted_count=0):
-            for toot in timeline:
-                if "id" in toot and "archive" in config:
 
-                    # define archive path
-                    if config["archive"][0] == "~":
-                        archive_path = os.path.expanduser(config["archive"])
-                    elif config["archive"][0] == "/":
-                        archive_path = config["archive"]
-                    else:
-                        archive_path = os.path.join(os.getcwd(), config["archive"])
-                    if archive_path[-1] != "/":
-                        archive_path += "/"
+def check_batch(config, options, mastodon, user_id, timeline, deleted_count=0):
+    """
+    Check a batch of up to 40 toots. This is usually triggered by check_toots, and then recursively calls itself until all toots within the time period specified have been checked.
+    """
 
-                    filename = os.path.join(archive_path, str(toot["id"]) + ".json")
+    try:
+        for toot in timeline:
+            # process_toot returns the value of the deleted_count so we can keep track here
+            deleted_count = process_toot(config, options, mastodon, deleted_count, toot)
 
-                    if not options.archive_deleted:
-                        # write toot to archive
-                        with open(filename, "w") as f:
-                            f.write(json.dumps(toot, indent=4, default=jsondefault))
-                            f.close()
+        # the account_statuses call is paginated with a 40-toot limit
+        # get the id of the last toot to include as 'max_id' in the next API call.
+        # then keep triggering new rounds of check_toots() until there are no more toots to check
+        max_id = timeline[-1:][0].id
+        next_batch = mastodon.account_statuses(user_id, limit=40, max_id=max_id)
+        if len(next_batch) > 0:
+            check_batch(config, options, mastodon, user_id, next_batch, deleted_count)
+        else:
+            if options.test:
+                if options.datestamp:
+                    print( "\n", datestamp_now(), sep="", end=" : ")
 
-                toot_tags = set()
-                for tag in toot.tags:
-                    toot_tags.add(tag.name)
-                try:
-                    if keep_pinned and hasattr(toot, "pinned") and toot.pinned:
-                        if not (options.hide_skipped or options.quiet):
-                            if options.datestamp:
-                                print(
-                                    str(
-                                        datetime.now(timezone.utc).strftime(
-                                            "%a %d %b %Y %H:%M:%S %z"
-                                        )
-                                    ),
-                                    end=" : ",
-                                )
+                print(
+                    "Test run completed. This would have removed", str(deleted_count), "toots.\n")
+            else:
+                if options.datestamp:
+                    print( "\n", datestamp_now(), end=" : ")
 
-                            print("📌 skipping pinned toot - " + str(toot.id))
-                    elif toot.id in toots_to_keep:
-                        if not (options.hide_skipped or options.quiet):
-                            if options.datestamp:
-                                print(
-                                    str(
-                                        datetime.now(timezone.utc).strftime(
-                                            "%a %d %b %Y %H:%M:%S %z"
-                                        )
-                                    ),
-                                    end=" : ",
-                                )
+                print("Removed " + str(deleted_count) + " toots.\n")
 
-                            print("💾 skipping saved toot - " + str(toot.id))
-                    elif toot.visibility in visibility_to_keep:
-                        if not (options.hide_skipped or options.quiet):
-                            if options.datestamp:
-                                print(
-                                    str(
-                                        datetime.now(timezone.utc).strftime(
-                                            "%a %d %b %Y %H:%M:%S %z"
-                                        )
-                                    ),
-                                    end=" : ",
-                                )
+            if not options.quiet:
+                print("---------------------------------------")
+                print("🥳 ==> 🧼 ==> 😇 User cleanup complete!")
+                print("---------------------------------------\n")
 
-                            print(
-                                "👀 skipping "
-                                + toot.visibility
-                                + " toot - "
-                                + str(toot.id)
-                            )
-                    elif len(hashtags_to_keep.intersection(toot_tags)) > 0:
-                        if not (options.hide_skipped or options.quiet):
-                            if options.datestamp:
-                                print(
-                                    str(
-                                        datetime.now(timezone.utc).strftime(
-                                            "%a %d %b %Y %H:%M:%S %z"
-                                        )
-                                    ),
-                                    end=" : ",
-                                )
+    except IndexError:
+        print("No toots found!\n")
 
-                            print("#️⃣  skipping toot with hashtag - " + str(toot.id))
-                    elif cutoff_date > toot.created_at:
-                        if hasattr(toot, "reblog") and toot.reblog:
-                            if not options.quiet:
-                                if options.datestamp:
-                                    print(
-                                        str(
-                                            datetime.now(timezone.utc).strftime(
-                                                "%a %d %b %Y %H:%M:%S %z"
-                                            )
-                                        ),
-                                        end=" : ",
-                                    )
+    except Exception as e:
+        print("ERROR:", str(e.args[0]), "\n")
 
-                                print(
-                                    "👎 unboosting toot "
-                                    + str(toot.id)
-                                    + " boosted "
-                                    + toot.created_at.strftime("%d %b %Y")
-                                )
-                            deleted_count += 1
-                            # unreblog the original toot (their toot), not the toot created by boosting (your toot)
-                            if not options.test:
-                                if mastodon.ratelimit_remaining == 0:
-                                    if not options.quiet:
-                                        print(
-                                            "Rate limit reached. Waiting for a rate limit reset"
-                                        )
-                                # check for --archive-deleted
-                                if (
-                                    options.archive_deleted
-                                    and "id" in toot
-                                    and "archive" in config
-                                ):
-                                    # write toot to archive
-                                    with open(filename, "w") as f:
-                                        f.write(
-                                            json.dumps(
-                                                toot, indent=4, default=jsondefault
-                                            )
-                                        )
-                                        f.close()
-                                mastodon.status_unreblog(toot.reblog)
-                        else:
-                            if not options.quiet:
-                                if options.datestamp:
-                                    print(
-                                        str(
-                                            datetime.now(timezone.utc).strftime(
-                                                "%a %d %b %Y %H:%M:%S %z"
-                                            )
-                                        ),
-                                        end=" : ",
-                                    )
-
-                                print(
-                                    "❌ deleting toot "
-                                    + str(toot.id)
-                                    + " tooted "
-                                    + toot.created_at.strftime("%d %b %Y")
-                                )
-                            deleted_count += 1
-                            time.sleep(
-                                2
-                            )  # wait 2 secs between deletes to be a bit nicer to the server
-                            if not options.test:
-                                if (
-                                    mastodon.ratelimit_remaining == 0
-                                    and not options.quiet
-                                ):
-
-                                    now = time.time()
-                                    diff = mastodon.ratelimit_reset - now
-
-                                    print(
-                                        "\nRate limit reached at "
-                                        + str(
-                                            datetime.now(timezone.utc).strftime(
-                                                "%a %d %b %Y %H:%M:%S %z"
-                                            )
-                                        )
-                                        + " - next reset due in "
-                                        + str(format(diff / 60, ".0f"))
-                                        + " minutes.\n"
-                                    )
-                                # check for --archive-deleted
-                                if (
-                                    options.archive_deleted
-                                    and "id" in toot
-                                    and "archive" in config
-                                ):
-                                    # write toot to archive
-                                    with open(filename, "w") as f:
-                                        f.write(
-                                            json.dumps(
-                                                toot, indent=4, default=jsondefault
-                                            )
-                                        )
-                                        f.close()
-
-                                mastodon.status_delete(toot)
-
-                except MastodonRatelimitError:
-
-                    now = time.time()
-                    diff = mastodon.ratelimit_reset - now
-
-                    print(
-                        "\nRate limit reached at "
-                        + str(
-                            datetime.now(timezone.utc).strftime(
-                                "%a %d %b %Y %H:%M:%S %z"
-                            )
-                        )
-                        + " - waiting for next reset due in "
-                        + str(format(diff / 60, ".0f"))
-                        + " minutes.\n"
-                    )
-
-                    time.sleep(diff + 1)  # wait for rate limit to reset
-
-                except MastodonError as e:
-
-                    def retry_on_error(attempts):
-
-                        if attempts < 6:
-                            try:
-                                if not options.quiet:
-                                    print(
-                                        "Attempt "
-                                        + str(attempts)
-                                        + " at "
-                                        + str(
-                                            datetime.now(timezone.utc).strftime(
-                                                "%a %d %b %Y %H:%M:%S %z"
-                                            )
-                                        )
-                                    )
-                                mastodon.status_delete(toot)
-                                time.sleep(
-                                    2
-                                )  # wait 2 secs between deletes to be a bit nicer to the server
-                            except:
-                                attempts += 1
-                                time.sleep(60 * options.retry_mins)
-                                retry_on_error(attempts)
-                        else:
-                            raise TimeoutError("Gave up after 5 attempts")
-
-                    print(
-                        "🛑 ERROR deleting toot - "
-                        + str(toot.id)
-                        + " - "
-                        + str(e.args[0])
-                        + " - "
-                        + str(e.args[3])
-                    )
-                    if not options.quiet:
-                        print(
-                            "Waiting "
-                            + str(options.retry_mins)
-                            + " minutes before re-trying"
-                        )
-                    time.sleep(60 * options.retry_mins)
-                    retry_on_error(attempts=2)
-
-                except KeyboardInterrupt:
-                    print("Operation aborted.")
-                    break
-                except KeyError as e:
-                    print(
-                        "⚠️  There is an error in your config.yaml file. Please add a value for "
-                        + str(e)
-                        + " and try again."
-                    )
-                    break
-                except:
-                    e = sys.exc_info()
-
-                    print("🛑 Unknown ERROR deleting toot - " + str(toot.id))
-
-                    print("ERROR: " + str(e[0]) + " - " + str(e[1]))
-
-            # the account_statuses call is paginated with a 40-toot limit
-            # get the id of the last toot to include as 'max_id' in the next API call.
-            # then keep triggering new rounds of checkToots() until there are no more toots to check
-            try:
-                max_id = timeline[-1:][0].id
-                next_batch = mastodon.account_statuses(user_id, limit=40, max_id=max_id)
-                if len(next_batch) > 0:
-                    checkBatch(next_batch, deleted_count)
-                else:
-                    if options.test:
-                        if options.datestamp:
-                            print(
-                                "\n\n"
-                                + str(
-                                    datetime.now(timezone.utc).strftime(
-                                        "%a %d %b %Y %H:%M:%S %z"
-                                    )
-                                ),
-                                end=" : ",
-                            )
-
-                        print(
-                            "Test run completed. This would have removed "
-                            + str(deleted_count)
-                            + " toots."
-                        )
-                    else:
-                        if options.datestamp:
-                            print(
-                                "\n\n"
-                                + str(
-                                    datetime.now(timezone.utc).strftime(
-                                        "%a %d %b %Y %H:%M:%S %z"
-                                    )
-                                ),
-                                end=" : ",
-                            )
-
-                        print("Removed " + str(deleted_count) + " toots.")
-
-                    if not options.quiet:
-                        print("\n---------------------------------------")
-                        print("🥳 ==> 🧼 ==> 😇 User cleanup complete!")
-                        print("---------------------------------------\n")
-
-            except IndexError:
-                print("No toots found!")
-
-            except Exception as e:
-                print("ERROR: " + str(e.args[0]))
+def check_toots(config, options, retry_count=0):
+    '''
+    The main function, uses the Mastodon API to check all toots in the user timeline, and delete any that do not meet any of the exclusion criteria from the config file.
+    '''
+    try:
+        print("Fetching account details for @", config["username"], "@", config["base_url"], sep="")
 
         if options.pace:
             mastodon = Mastodon(
@@ -543,25 +446,23 @@ def checkToots(config, options, retry_count=0):
                 api_base_url="https://" + config["base_url"],
                 ratelimit_method="pace",
             )
-
         else:
-
             mastodon = Mastodon(
                 access_token=config["access_token"],
                 api_base_url="https://" + config["base_url"],
                 ratelimit_method="wait",
             )
 
-        # STARTS HERE
-        cutoff_date = datetime.now(timezone.utc) - timedelta(days=days_to_keep)
-        user_id = mastodon.account_verify_credentials().id
-        account = mastodon.account(user_id)
-        timeline = mastodon.account_statuses(user_id, limit=40)
+        user_id = mastodon.account_verify_credentials().id # verify user and get ID
+        account = mastodon.account(user_id) # get the account
+        timeline = mastodon.account_statuses(user_id, limit=40) # initial batch
 
         if not options.quiet:
-            print("Checking " + str(account.statuses_count) + " toots")
+            print("Checking", str(account.statuses_count), "toots")
 
-        checkBatch(timeline)
+        # check first batch 
+        # check_batch() then recursively keeps looping until all toots have been checked
+        check_batch(config, options, mastodon, user_id, timeline) 
 
     except KeyError as val:
         print("\n⚠️  error with in your config.yaml file!")
@@ -569,24 +470,22 @@ def checkToots(config, options, retry_count=0):
 
     except MastodonAPIError as e:
         if e.args[1] == 401:
-            print("\n🙅  User and/or access token does not exist or has been deleted (401)")
+            print(
+                "\n🙅  User and/or access token does not exist or has been deleted (401)\n"
+            )
         elif e.args[1] == 404:
-            print("\n🔭  Can't find that server (404)")
+            print("\n🔭  Can't find that server (404)\n")
         else:
-            print("\n😕  Server has returned an error (5xx)")
+            print("\n😕  Server has returned an error (5xx)\n")
 
     except MastodonNetworkError:
         if retry_count == 0:
             print("\n📡  ephemetoot cannot connect to the server - are you online?")
         if retry_count < 4:
-            print(
-                "Waiting "
-                + str(options.retry_mins)
-                + " minutes before trying again"
-            )
+            print("Waiting " + str(options.retry_mins) + " minutes before trying again")
             time.sleep(60 * options.retry_mins)
             retry_count += 1
             print("Attempt " + str(retry_count + 1))
-            checkToots(config, options, retry_count)
+            check_toots(config, options, retry_count)
         else:
-            print("Gave up waiting for network")
+            print("Gave up waiting for network\n")

ephemetoot/plist.py

@@ -1,4 +1,4 @@
-default_file = '''<?xml version="1.0" encoding="UTF-8"?>
+default_file = """<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
@@ -24,4 +24,4 @@ default_file = '''<?xml version="1.0" encoding="UTF-8"?>
     <integer>00</integer>
   </dict>
 </dict>
-</plist>'''
+</plist>"""

poetry.lock

@@ -1,3 +1,24 @@
+[[package]]
+category = "dev"
+description = "Atomic file writes."
+name = "atomicwrites"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "1.4.0"
+
+[[package]]
+category = "dev"
+description = "Classes Without Boilerplate"
+name = "attrs"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "20.1.0"
+
+[package.extras]
+dev = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"]
+docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"]
+tests = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
+
 [[package]]
 category = "main"
 description = "Pure-Python implementation of the blurhash algorithm."
@@ -25,6 +46,14 @@ optional = false
 python-versions = "*"
 version = "3.0.4"
 
+[[package]]
+category = "dev"
+description = "Cross-platform colored terminal text."
+name = "colorama"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+version = "0.4.3"
+
 [[package]]
 category = "main"
 description = "Decorators for Humans"
@@ -41,6 +70,29 @@ optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 version = "2.10"
 
+[[package]]
+category = "dev"
+description = "Read metadata from Python packages"
+name = "importlib-metadata"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
+version = "1.7.0"
+
+[package.dependencies]
+zipp = ">=0.5"
+
+[package.extras]
+docs = ["sphinx", "rst.linker"]
+testing = ["packaging", "pep517", "importlib-resources (>=1.3)"]
+
+[[package]]
+category = "dev"
+description = "iniconfig: brain-dead simple config-ini parsing"
+name = "iniconfig"
+optional = false
+python-versions = "*"
+version = "1.0.1"
+
 [[package]]
 category = "main"
 description = "Python wrapper for the Mastodon API"
@@ -63,6 +115,80 @@ blurhash = ["blurhash (>=1.1.4)"]
 test = ["blurhash (>=1.1.4)", "cryptography (>=1.6.0)", "http-ece (>=1.0.5)", "pytest", "pytest-cov", "pytest-mock", "pytest-runner", "pytest-vcr", "requests-mock", "vcrpy"]
 webpush = ["cryptography (>=1.6.0)", "http-ece (>=1.0.5)"]
 
+[[package]]
+category = "dev"
+description = "More routines for operating on iterables, beyond itertools"
+name = "more-itertools"
+optional = false
+python-versions = ">=3.5"
+version = "8.5.0"
+
+[[package]]
+category = "dev"
+description = "Core utilities for Python packages"
+name = "packaging"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "20.4"
+
+[package.dependencies]
+pyparsing = ">=2.0.2"
+six = "*"
+
+[[package]]
+category = "dev"
+description = "plugin and hook calling mechanisms for python"
+name = "pluggy"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "0.13.1"
+
+[package.dependencies]
+importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
+
+[package.extras]
+dev = ["pre-commit", "tox"]
+
+[[package]]
+category = "dev"
+description = "library with cross-python path, ini-parsing, io, code, log facilities"
+name = "py"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "1.9.0"
+
+[[package]]
+category = "dev"
+description = "Python parsing module"
+name = "pyparsing"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+version = "2.4.7"
+
+[[package]]
+category = "dev"
+description = "pytest: simple powerful testing with Python"
+name = "pytest"
+optional = false
+python-versions = ">=3.5"
+version = "6.0.1"
+
+[package.dependencies]
+attrs = ">=17.4.0"
+iniconfig = "*"
+more-itertools = ">=4.0.0"
+packaging = "*"
+pluggy = ">=0.12,<1.0"
+py = ">=1.8.2"
+toml = "*"
+atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""}
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
+
+[package.extras]
+checkqa_mypy = ["mypy (0.780)"]
+testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
+
 [[package]]
 category = "main"
 description = "Extensions to the standard Python datetime module"
@@ -124,6 +250,14 @@ optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
 version = "1.15.0"
 
+[[package]]
+category = "dev"
+description = "Python Library for Tom's Obvious, Minimal Language"
+name = "toml"
+optional = false
+python-versions = "*"
+version = "0.10.1"
+
 [[package]]
 category = "main"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
@@ -137,12 +271,32 @@ brotli = ["brotlipy (>=0.6.0)"]
 secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"]
 socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"]
 
+[[package]]
+category = "dev"
+description = "Backport of pathlib-compatible object wrapper for zip files"
+name = "zipp"
+optional = false
+python-versions = ">=3.6"
+version = "3.1.0"
+
+[package.extras]
+docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
+testing = ["jaraco.itertools", "func-timeout"]
+
 [metadata]
-content-hash = "35d07702bbbc789f7e6dc248539c816144f6b4a294d7f5e277ab4aed3c094584"
+content-hash = "fefbdaa7cc9c44fec27e8a8a10c673b7d72e00316507560ceef047285bcdfb25"
 lock-version = "1.1"
 python-versions = "^3.6"
 
 [metadata.files]
+atomicwrites = [
+    {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"},
+    {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"},
+]
+attrs = [
+    {file = "attrs-20.1.0-py2.py3-none-any.whl", hash = "sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff"},
+    {file = "attrs-20.1.0.tar.gz", hash = "sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a"},
+]
 blurhash = [
     {file = "blurhash-1.1.4-py2.py3-none-any.whl", hash = "sha256:7611c1bc41383d2349b6129208587b5d61e8792ce953893cb49c38beeb400d1d"},
     {file = "blurhash-1.1.4.tar.gz", hash = "sha256:da56b163e5a816e4ad07172f5639287698e09d7f3dc38d18d9726d9c1dbc4cee"},
@@ -155,6 +309,10 @@ chardet = [
     {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
     {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"},
 ]
+colorama = [
+    {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"},
+    {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"},
+]
 decorator = [
     {file = "decorator-4.4.2-py2.py3-none-any.whl", hash = "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760"},
     {file = "decorator-4.4.2.tar.gz", hash = "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7"},
@@ -163,10 +321,42 @@ idna = [
     {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},
     {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"},
 ]
+importlib-metadata = [
+    {file = "importlib_metadata-1.7.0-py2.py3-none-any.whl", hash = "sha256:dc15b2969b4ce36305c51eebe62d418ac7791e9a157911d58bfb1f9ccd8e2070"},
+    {file = "importlib_metadata-1.7.0.tar.gz", hash = "sha256:90bb658cdbbf6d1735b6341ce708fc7024a3e14e99ffdc5783edea9f9b077f83"},
+]
+iniconfig = [
+    {file = "iniconfig-1.0.1-py3-none-any.whl", hash = "sha256:80cf40c597eb564e86346103f609d74efce0f6b4d4f30ec8ce9e2c26411ba437"},
+    {file = "iniconfig-1.0.1.tar.gz", hash = "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69"},
+]
 "mastodon.py" = [
     {file = "Mastodon.py-1.5.1-py2.py3-none-any.whl", hash = "sha256:cc454cac0ed1ae4f105f7399ea53f5b31a1be5075d1882f47162d2e78a9e4064"},
     {file = "Mastodon.py-1.5.1.tar.gz", hash = "sha256:2afddbad8b5d7326fcc8a8f8c62bfe956e34627f516b06c6694fc8c8fedc33ee"},
 ]
+more-itertools = [
+    {file = "more-itertools-8.5.0.tar.gz", hash = "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20"},
+    {file = "more_itertools-8.5.0-py3-none-any.whl", hash = "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c"},
+]
+packaging = [
+    {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"},
+    {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"},
+]
+pluggy = [
+    {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"},
+    {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
+]
+py = [
+    {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"},
+    {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"},
+]
+pyparsing = [
+    {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"},
+    {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"},
+]
+pytest = [
+    {file = "pytest-6.0.1-py3-none-any.whl", hash = "sha256:8b6007800c53fdacd5a5c192203f4e531eb2a1540ad9c752e052ec0f7143dbad"},
+    {file = "pytest-6.0.1.tar.gz", hash = "sha256:85228d75db9f45e06e57ef9bf4429267f81ac7c0d742cc9ed63d09886a9fe6f4"},
+]
 python-dateutil = [
     {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"},
     {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"},
@@ -200,7 +390,15 @@ six = [
     {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
     {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},
 ]
+toml = [
+    {file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"},
+    {file = "toml-0.10.1.tar.gz", hash = "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f"},
+]
 urllib3 = [
     {file = "urllib3-1.25.10-py2.py3-none-any.whl", hash = "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"},
     {file = "urllib3-1.25.10.tar.gz", hash = "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a"},
 ]
+zipp = [
+    {file = "zipp-3.1.0-py3-none-any.whl", hash = "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b"},
+    {file = "zipp-3.1.0.tar.gz", hash = "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"},
+]

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ephemetoot"
-version = "3.0.0-alpha.0"
+version = "3.0.0-beta.0"
 description = "A command line tool to delete your old toots"
 authors = ["Hugh Rundle <ephemetoot@hugh.run>"]
 license = "GPL-3.0-or-later"
@@ -24,7 +24,7 @@ requests = "^2.22.0"
 pyyaml = "^5.0"
 
 [tool.poetry.dev-dependencies]
-# TODO - tests!
+pytest = "^6"
 
 [tool.poetry.scripts]
 ephemetoot = 'ephemetoot.console:main'

tests/test_ephemetoot.py

@@ -0,0 +1,446 @@
+import datetime
+from datetime import timezone
+from dateutil.tz import tzutc
+import json
+import os
+import subprocess
+import sys
+
+import pytest
+import requests
+
+from ephemetoot import ephemetoot
+
+
+########################
+#         MOCKS        #
+########################
+
+toot_dict = {
+  'id': 104136090490756999, 
+  'created_at': datetime.datetime(2020, 5, 9, 2, 17, 18, 598000, tzinfo=tzutc()), 
+  'in_reply_to_id': None, 
+  'in_reply_to_account_id': None, 
+  'sensitive': False, 
+  'spoiler_text': '', 
+  'visibility': 'public', 
+  'language': 'en', 
+  'uri': 'https://example.social/users/testbot/statuses/104136090490756503', 
+  'url': 'https://example.social/@testbot/104136090490756503', 
+  'replies_count': 0, 
+  'reblogs_count': 0, 
+  'favourites_count': 0, 
+  'favourited': False, 
+  'reblogged': False, 
+  'muted': False, 
+  'bookmarked': False, 
+  'pinned': True, 
+  'content': '<p>hello I am testing</p>', 
+  'reblog': None, 
+  'application': None, 
+  'account': {
+    'id': 16186, 
+    'username': 'testbot', 
+    'acct': 'testbot', 
+    'display_name': 'ephemtoot Testing Bot', 
+    'locked': True, 
+    'bot': True, 
+    'discoverable': False, 
+    'group': False, 
+    'created_at': datetime.datetime(2018, 11, 16, 23, 15, 15, 718000, tzinfo=tzutc()), 
+    'note': '<p>Liable to explode at any time, handle with care.</p>', 
+    'url': 'https://example.social/@testbot', 
+    'avatar': 'https://example.social/system/accounts/avatars/000/016/186/original/66d11c4191332e7a.png?1542410869', 
+    'avatar_static': 'https://example.social/system/accounts/avatars/000/016/186/original/66d11c4191332e7a.png?1542410869', 
+    'header': 'https://example.social/headers/original/header.png', 
+    'header_static': 'https://example.social/headers/original/header.png', 
+    'followers_count': 100, 
+    'following_count': 10, 
+    'statuses_count': 99, 
+    'last_status_at': datetime.datetime(2020, 8, 17, 0, 0), 
+    'emojis': [], 
+    'fields': [
+      {
+        'name': 'Fully', 
+        'value': 'Automated', 
+        'verified_at': None
+      }, 
+      {
+        'name': 'Luxury', 
+        'value': 'Communism', 
+        'verified_at': None
+      }
+    ]
+  }, 
+  'media_attachments': [], 
+  'mentions': [], 
+  'tags': [], 
+  'emojis': [], 
+  'card': None, 
+  'poll': None
+}
+
+# Turn dict into object needed by mastodon.py
+# Use this in tests after making any changes
+# you need to your dict object
+# NOTE: ensure values in the dict object are what you need: 
+# it can be mutated by any test before your test runs
+
+def dict2obj(d): 
+    # checking whether object d is a 
+    # instance of class list 
+    if isinstance(d, list): 
+           d = [dict2obj(x) for x in d]  
+  
+    # if d is not a instance of dict then 
+    # directly object is returned 
+    if not isinstance(d, dict): 
+           return d 
+   
+    # declaring a class 
+    class C: 
+        pass
+   
+    # constructor of the class passed to obj 
+    obj = C() 
+   
+    for k in d: 
+        obj.__dict__[k] = dict2obj(d[k]) 
+   
+    return obj 
+
+# here is our toot object - use this in tests
+toot = dict2obj(toot_dict)
+
+# config file after being parsed by yaml.safe_load
+config_file = {
+  'access_token': 'abcd_1234', 
+  'username': 'alice', 
+  'base_url': 'test.social', 
+  'hashtags_to_keep': ['ephemetoot'], 
+  'days_to_keep': 14, 
+  'keep_pinned': True, 
+  'toots_to_keep': [103996285277439262, 103976473612749097, 103877521458738491], 
+  'visibility_to_keep': [None],
+  'archive': 'archive'
+}
+
+# mock GitHub API call for the version number
+class MockGitHub:
+    @staticmethod
+    def json():
+        return {"tag_name": "vLATEST_VERSION"}
+
+# mock Mastodon
+class Mocktodon:
+    def __init__(self):
+        return None
+
+    def status_delete(self, t=toot):
+        return None
+
+    def status_unreblog(self, t=toot):
+        return None
+
+    def ratelimit_remaining(self):
+        return 100
+
+    def account_statuses(self, user_id=None, limit=None, max_id=None):
+        # create 10 statuses
+        # the first 2 will be checked in the first batch (in production it would be 40)
+        user_toots = []
+        def make_toot(i=1):
+            if i < 11:
+                keys = ("id", "created_at", "reblog", "tags", "visibility")
+                vals = (
+                  i, 
+                  datetime.datetime(2018, 11, i, 23, 15, 15, 718000, tzinfo=tzutc()),
+                  False,
+                  [],
+                  "public"
+                )
+                user_toot = dict(zip(keys, vals))
+                user_toots.append(user_toot)
+                total = i + 1
+                make_toot(total)
+
+        user_toots.sort(reverse=True)
+        make_toot(1) # make the batch of toots
+        # ignore user_id
+        # filter for toots with id smaller than max_id
+        this_batch = []
+        # use dict_to_obj to make a toot for each toot in the obj then a list from that
+        this_batch = [ dict2obj(t) for t in user_toots if t["id"] > max_id][:limit]
+        return this_batch
+
+# mock argparse objects (options)
+class Namespace:
+    def __init__(self, archive_deleted=False, datestamp=False, hide_skipped=False, retry_mins=1, schedule=False, test=False, time=False,  quiet=False):
+        self.archive_deleted = archive_deleted
+        self.datestamp = datestamp
+        self.schedule = schedule
+        self.time = time
+        self.test = test
+        self.hide_skipped = hide_skipped
+        self.quiet = quiet
+        self.retry_mins = retry_mins
+
+@pytest.fixture
+def mock_github_response(monkeypatch):
+
+    def mock_get(*args, **kwargs):
+        return MockGitHub()
+
+    monkeypatch.setattr(requests, "get", mock_get)
+
+########################
+#         TESTS        #
+########################
+
+def test_archive_toot(tmpdir):
+    p = tmpdir.mkdir("archive")
+    config_file['archive'] = str(p) # make archive directory a temp test dir
+
+    ephemetoot.archive_toot(config_file, toot)
+
+    file_exists = os.path.exists(p + "/104136090490756999.json")
+    assert file_exists
+
+def test_check_batch(capfd, monkeypatch):
+    config = config_file
+    options = Namespace(archive_deleted=False)
+    mastodon = Mocktodon()
+    user_id = "test_user_id"
+    # limit to 2 so check_batch calls itself for the last 8 toots
+    timeline = mastodon.account_statuses(user_id=user_id, limit=2, max_id=0)
+    # monkeypatch process_toot to add 1 to deleted_count and return
+    # this simulates what would happen if the toot was being deleted
+    monkeypatch.setattr(
+      "ephemetoot.ephemetoot.process_toot", 
+      lambda config, options, mastodon, deleted_count, toot: deleted_count + 1
+    )
+    # run check_batch
+    ephemetoot.check_batch(config, options, mastodon, user_id, timeline, 0)
+    # deleted_count should be 10
+    output = capfd.readouterr().out.split("\n")
+    assert output[0] == "Removed 10 toots."
+
+def test_console_print(capfd):
+    ephemetoot.console_print("test123", Namespace(test=False, hide_skipped=False, quiet=False), False)
+    assert capfd.readouterr().out == "test123\n"
+
+def test_console_print_quiet():
+    result = ephemetoot.console_print("test123", Namespace(test=False, hide_skipped=False, quiet=True), False)
+    assert result == None
+
+def test_console_print_skip():
+    result = ephemetoot.console_print("test123", Namespace(test=False, hide_skipped=True, quiet=False), True)
+    assert result == None
+
+def test_datestamp_now():
+    datestamp = ephemetoot.datestamp_now()
+    date_object = datetime.datetime.strptime(datestamp, "%a %d %b %Y %H:%M:%S %z")
+    # use timetuple() to exclude differences in milliseconds
+    assert datetime.datetime.now(timezone.utc).timetuple()  == date_object.timetuple()
+
+def test_init(monkeypatch, tmpdir):
+
+    # monkeypatch current directory
+    current_dir = tmpdir.mkdir("current_dir") # temporary directory for testing
+    monkeypatch.chdir(current_dir)
+
+    # monkeypatch input ...outputs
+    monkeypatch.setattr("ephemetoot.ephemetoot.compulsory_input", lambda a, b, c: "compulsory")
+    monkeypatch.setattr("ephemetoot.ephemetoot.digit_input", lambda a, b, c: "14")
+    monkeypatch.setattr("ephemetoot.ephemetoot.yes_no_input", lambda a, b: "false")
+    monkeypatch.setattr("ephemetoot.ephemetoot.optional_input", lambda a, b, c: "optional")
+
+    # run init
+    ephemetoot.init()
+    assert os.path.exists( os.path.join(current_dir, "config.yaml"))
+
+def test_jsondefault():
+    d = ephemetoot.jsondefault(toot.created_at)
+    assert d == "2020-05-09T02:17:18.598000+00:00"
+
+def test_process_toot(capfd, tmpdir, monkeypatch):
+    # config uses config_listed at top of this tests file
+    p = tmpdir.mkdir("archive") # use temporary test directory
+    config_file["archive"] = str(p)
+    config_file["keep_pinned"] = False
+    config_file["toots_to_keep"] = [] 
+    config_file["visibility_to_keep"] = []
+    options = Namespace(archive_deleted=False)
+    mastodon = Mocktodon()
+    toot_dict["pinned"] = False 
+    toot_dict["visibility"] = "public"
+    toot_dict["reblog"] = False
+    toot = dict2obj(toot_dict)
+    ephemetoot.process_toot(config_file, options, mastodon, 0, toot)
+    assert capfd.readouterr().out == "❌ deleting toot 104136090490756999 tooted 09 May 2020\n"
+
+def test_process_toot_pinned(capfd, tmpdir):
+    # config uses config_listed at top of this tests file
+    p = tmpdir.mkdir("archive") # use temporary test directory
+    config_file["archive"] = str(p)
+    config_file["keep_pinned"] = True
+    options = Namespace(archive_deleted=False)
+    mastodon = Mocktodon()
+    toot_dict["pinned"] = True
+    toot = dict2obj(toot_dict)
+    ephemetoot.process_toot(config_file, options, mastodon, 0, toot)
+    assert capfd.readouterr().out == "📌 skipping pinned toot - 104136090490756999\n"
+def test_process_toot_saved(capfd, tmpdir):
+    # config uses config_listed at top of this tests file
+    p = tmpdir.mkdir("archive") # use temporary test directory
+    config_file["archive"] = str(p)
+    config_file["keep_pinned"] = False
+    config_file["toots_to_keep"].append(104136090490756999)
+    options = Namespace(archive_deleted=False)
+    mastodon = Mocktodon()
+    toot_dict["pinned"] = False
+    toot = dict2obj(toot_dict)
+    ephemetoot.process_toot(config_file, options, mastodon, 0, toot)
+    assert capfd.readouterr().out == "💾 skipping saved toot - 104136090490756999\n"
+
+def test_process_toot_visibility(capfd, tmpdir):
+    # config uses config_listed at top of this tests file
+    p = tmpdir.mkdir("archive") # use temporary test directory
+    config_file["archive"] = str(p)
+    config_file["keep_pinned"] = False # is true above so make false
+    config_file["toots_to_keep"].remove(104136090490756999) # don't keep this toot
+    config_file["visibility_to_keep"].append("testing")
+    options = Namespace(archive_deleted=False)
+    mastodon = Mocktodon()
+    toot_dict["pinned"] = False # is true above so make false
+    toot_dict["visibility"] = "testing"
+    toot = dict2obj(toot_dict)
+    ephemetoot.process_toot(config_file, options, mastodon, 0, toot)
+    assert capfd.readouterr().out == "👀 skipping testing toot - 104136090490756999\n"
+
+def test_process_toot_hashtag(capfd, tmpdir, monkeypatch):
+    # config uses config_listed at top of this tests file
+    p = tmpdir.mkdir("archive") # use temporary test directory
+    config_file["archive"] = str(p)
+    config_file["keep_pinned"] = False
+    config_file["toots_to_keep"] = [] 
+    config_file["visibility_to_keep"] = []
+    options = Namespace(archive_deleted=False)
+    mastodon = Mocktodon()
+    toot_dict["pinned"] = False 
+    toot_dict["visibility"] = "public"
+    toot_dict["reblog"] = True
+    toot = dict2obj(toot_dict)
+
+    ephemetoot.process_toot(config_file, options, mastodon, 0, toot)
+    assert capfd.readouterr().out == "👎 unboosting toot 104136090490756999 boosted 09 May 2020\n"
+
+def test_retry_on_error():
+    # Namespace object constructed from top of tests (representing options)
+    # toot comes from variable at top of test
+    mastodon = Mocktodon()
+    toot = dict2obj(toot_dict)
+    retry = ephemetoot.retry_on_error(Namespace(retry_mins=True), mastodon, toot, 5 )
+    assert retry == None # should not return an error
+
+def test_retry_on_error_max_tries():
+    # Namespace object constructed from top of tests (representing options)
+    # toot and mastodon come from objects at top of test
+    with pytest.raises(TimeoutError):
+        mastodon = Mocktodon()
+        toot = dict2obj(toot_dict)
+        retry = ephemetoot.retry_on_error(Namespace(retry_mins=True), mastodon, toot, 7 )
+
+def test_schedule(monkeypatch, tmpdir):
+
+    home = tmpdir.mkdir("current_dir") # temporary directory for testing
+    launch = tmpdir.mkdir("TestAgents") # temporary directory for testing
+
+    # monkeypatch directories and suppress the plist loading process
+    # NOTE: it may be possible to test the plist loading process
+    # but I can't work out how to do it universally / consistently
+
+    def mock_current_dir():
+        return str(home)
+
+    def mock_home_dir_expansion(arg):
+        return str(launch)
+
+    def suppress_subprocess(args, stdout=None, stderr=None, shell=None):
+        return None
+
+    monkeypatch.setattr(os, "getcwd", mock_current_dir)
+    monkeypatch.setattr(os.path, "expanduser", mock_home_dir_expansion)
+    monkeypatch.setattr(subprocess, "run", suppress_subprocess)
+
+    # now we run the function we're testing
+    ephemetoot.schedule(Namespace(schedule=".", time=None))
+
+    # assert the plist file was created
+    plist_file = os.path.join(launch, "ephemetoot.scheduler.plist")
+    assert os.path.lexists(plist_file)
+
+    # check that correct values were modified in the file
+    f = open(plist_file, "r")
+    plist = f.readlines()
+    assert plist[7] == "  <string>" + str(home) + "</string>\n"
+    assert plist[7] == "  <string>" + str(home) + "</string>\n"
+    assert plist[10] == "    <string>" + sys.argv[0] + "</string>\n"
+    assert plist[12] == "    <string>" + str(home) + "/config.yaml</string>\n"
+    assert plist[15] == "  <string>" + str(home) + "/ephemetoot.log</string>\n"
+    assert plist[17] == "  <string>" + str(home) + "/ephemetoot.error.log</string>\n"
+
+def test_schedule_with_time(monkeypatch, tmpdir):
+
+    home = tmpdir.mkdir("current_dir") # temporary directory for testing
+    launch = tmpdir.mkdir("TestAgents") # temporary directory for testing
+
+    # monkeypatch directories and suppress the plist loading process
+    # NOTE: it may be possible to test the plist loading process
+    # but I can't work out how to do it universally / consistently
+
+    def mock_current_dir():
+        return str(home)
+
+    def mock_home_dir_expansion(arg):
+        return str(launch)
+
+    def suppress_subprocess(args, stdout=None, stderr=None, shell=None):
+        return None
+
+    monkeypatch.setattr(os, "getcwd", mock_current_dir)
+    monkeypatch.setattr(os.path, "expanduser", mock_home_dir_expansion)
+    monkeypatch.setattr(subprocess, "run", suppress_subprocess)
+
+    # now we run the function we're testing
+    ephemetoot.schedule(Namespace(schedule=".", time=["10", "30"]))
+
+    # assert the plist file was created
+    plist_file = os.path.join(launch, "ephemetoot.scheduler.plist")
+    assert os.path.lexists(plist_file)
+
+    # assert that correct values were modified in the file
+    f = open(plist_file, "r")
+    plist = f.readlines()
+
+    assert plist[21] == "    <integer>10</integer>\n"
+    assert plist[23] == "    <integer>30</integer>\n"
+
+def test_tooted_date():
+    string = ephemetoot.tooted_date(toot)
+    created = datetime.datetime(2020, 5, 9, 2, 17, 18, 598000, tzinfo=timezone.utc)
+    test_string = created.strftime("%d %b %Y")
+
+    assert string == test_string
+
+def test_version(mock_github_response, capfd):
+    ephemetoot.version("TEST_VERSION")
+    output = capfd.readouterr().out
+    msg = """
+ephemetoot ==> 🥳 ==> 🧼 ==> 😇
+-------------------------------
+You are using release: \033[92mvTEST_VERSION\033[0m
+The latest release is: \033[92mvLATEST_VERSION\033[0m
+To upgrade to the most recent version run \033[92mpip3 install --update ephemetoot\033[0m\n"""
+
+    assert output == msg