Merge branch 'master' into remove-yarn

CHANGELOG.markdown

@@ -1,3 +1,7 @@
+## v1.4.0
+
+* added warning when it looks like an archive is a full "Your Twitter data" archive
+
 ## v1.3.0
 
 Released 2018-07-06

Pipfile.lock

@@ -133,7 +133,6 @@
                 "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f",
                 "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
             "version": "==1.11.5"
         },
         "chardet": {
@@ -145,10 +144,11 @@
         },
         "click": {
             "hashes": [
-                "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d",
+                "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
-                "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b"
+                "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
             ],
-            "version": "==6.7"
+            "markers": "python_version >= '2.7' and python_version != '3.3.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*'",
+            "version": "==7.0"
         },
         "cloudpickle": {
             "hashes": [
@@ -179,7 +179,7 @@
                 "sha256:e79ab4485b99eacb2166f3212218dd858258f374855e1568f728462b0e6ee0d9",
                 "sha256:f995d3667301e1754c57b04e0bae6f0fa9d710697a9f8d6712e8cca02550910f"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
+            "markers": "python_version >= '2.7' and python_version != '3.3.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*'",
             "version": "==2.3.1"
         },
         "csscompressor": {
@@ -248,7 +248,6 @@
             "hashes": [
                 "sha256:2f31a0640c31a0c2934ab1e37005dd9a559ae854a16304f9b839e062074106cc"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
             "version": "==1.0.5"
         },
         "idna": {
@@ -300,39 +299,39 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:00def5b638994f888d1058e4d17c86dec8e1113c3741a0a8a659039aec59a83a",
+                "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360",
-                "sha256:026449b64e559226cdb8e6d8c931b5965d8fc90ec18ebbb0baa04c5b36503c72",
+                "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7",
-                "sha256:03dbb224ee196ef30ed2156d41b579143e1efeb422974719a5392fc035e4f574",
+                "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e",
-                "sha256:03eb0e04f929c102ae24bc436bf1c0c60a4e63b07ebd388e84d8b219df3e6acd",
+                "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93",
-                "sha256:1be66b9a89e367e7d20d6cae419794997921fe105090fafd86ef39e20a3baab2",
+                "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c",
-                "sha256:1e977a3ed998a599bda5021fb2c2889060617627d3ae228297a529a082a3cd5c",
+                "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8",
-                "sha256:22cf3406d135cfcc13ec6228ade774c8461e125c940e80455f500638429be273",
+                "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b",
-                "sha256:24adccf1e834f82718c7fc8e3ec1093738da95144b8b1e44c99d5fc7d3e9c554",
+                "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0",
-                "sha256:2a3e362c97a5e6a259ee9cd66553292a1f8928a5bdfa3622fdb1501570834612",
+                "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd",
-                "sha256:3832e26ecbc9d8a500821e3a1d3765bda99d04ae29ffbb2efba49f5f788dc934",
+                "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb",
-                "sha256:4fd1f0c2dc02aaec729d91c92cd85a2df0289d88e9f68d1e8faba750bb9c4786",
+                "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581",
-                "sha256:4fda62030f2c515b6e2e673c57caa55cb04026a81968f3128aae10fc28e5cc27",
+                "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64",
-                "sha256:5044d75a68b49ce36a813c82d8201384207112d5d81643937fc758c05302f05b",
+                "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace",
-                "sha256:522184556921512ec484cb93bd84e0bab915d0ac5a372d49571c241a7f73db62",
+                "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956",
-                "sha256:5914cff11f3e920626da48e564be6818831713a3087586302444b9c70e8552d9",
+                "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60",
-                "sha256:6661a7908d68c4a133e03dac8178287aa20a99f841ea90beeb98a233ae3fd710",
+                "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16",
-                "sha256:79258a8df3e309a54c7ef2ef4a59bb8e28f7e4a8992a3ad17c24b1889ced44f3",
+                "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37",
-                "sha256:7d74c20b8f1c3e99d3f781d3b8ff5abfefdd7363d61e23bdeba9992ff32cc4b4",
+                "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae",
-                "sha256:81918afeafc16ba5d9d0d4e9445905f21aac969a4ebb6f2bff4b9886da100f4b",
+                "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a",
-                "sha256:8194d913ca1f459377c8a4ed8f9b7ad750068b8e0e3f3f9c6963fcc87a84515f",
+                "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073",
-                "sha256:84d5d31200b11b3c76fab853b89ac898bf2d05c8b3da07c1fcc23feb06359d6e",
+                "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8",
-                "sha256:989981db57abffb52026b114c9a1f114c7142860a6d30a352d28f8cbf186500b",
+                "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb",
-                "sha256:a3d7511d3fad1618a82299aab71a5fceee5c015653a77ffea75ced9ef917e71a",
+                "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409",
-                "sha256:b3ef168d4d6fd4fa6685aef7c91400f59f7ab1c0da734541f7031699741fb23f",
+                "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f",
-                "sha256:c1c5792b6e74bbf2af0f8e892272c2a6c48efa895903211f11b8342e03129fea",
+                "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2",
-                "sha256:c5dcb5a56aebb8a8f2585042b2f5c496d7624f0bcfe248f0cc33ceb2fd8d39e7",
+                "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4",
-                "sha256:e2bed4a04e2ca1050bb5f00865cf2f83c0b92fd62454d9244f690fcd842e27a4",
+                "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74",
-                "sha256:e87a527c06319428007e8c30511e1f0ce035cb7f14bb4793b003ed532c3b9333",
+                "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e",
-                "sha256:f63e420180cbe22ff6e32558b612e75f50616fc111c5e095a4631946c782e109",
+                "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1",
-                "sha256:f8b3d413c5a8f84b12cd4c5df1d8e211777c9852c6be3ee9c094b626644d3eab"
+                "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888"
             ],
             "index": "pypi",
-            "version": "==5.2.0"
+            "version": "==5.3.0"
         },
         "psycopg2": {
             "hashes": [
@@ -374,7 +373,7 @@
             "hashes": [
                 "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"
             ],
-            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.3.*'",
+            "markers": "python_version >= '2.7' and python_version != '3.3.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*'",
             "version": "==2.19"
         },
         "pyinotify": {
@@ -455,7 +454,7 @@
                 "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf",
                 "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5"
             ],
-            "markers": "python_version < '4' and python_version != '3.3.*' and python_version != '3.0.*' and python_version >= '2.6' and python_version != '3.1.*' and python_version != '3.2.*'",
+            "markers": "python_version != '3.3.*' and python_version != '3.2.*' and python_version >= '2.6' and python_version < '4' and python_version != '3.1.*' and python_version != '3.0.*'",
             "version": "==1.23"
         },
         "vine": {
@@ -479,7 +478,7 @@
                 "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0",
                 "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee"
             ],
-            "markers": "python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*'",
+            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.3.*' and python_version != '3.0.*'",
             "version": "==1.2.1"
         },
         "attrs": {
@@ -560,7 +559,7 @@
                 "sha256:27390c35495b0b3eab4720f3e186fac57c520104b84d7016cdc5f70a409f3f0c",
                 "sha256:4d246c7621b877e284c5b0a2cdf0b62406560ff548453ea759aedd11dfca4004"
             ],
-            "markers": "python_version != '3.3.*' and python_version != '3.0.*' and python_version >= '2.6' and python_version != '3.1.*' and python_version != '3.2.*'",
+            "markers": "python_version != '3.3.*' and python_version != '3.2.*' and python_version >= '2.6' and python_version != '3.1.*' and python_version != '3.0.*'",
             "version": "==1.0.0"
         },
         "more-itertools": {
@@ -576,7 +575,7 @@
                 "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1",
                 "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1"
             ],
-            "markers": "python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*'",
+            "markers": "python_version != '3.3.*' and python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.0.*'",
             "version": "==0.7.1"
         },
         "port-for": {
@@ -598,7 +597,7 @@
                 "sha256:cea2557ee6a9faa2c100947637ded68414e12b851633c4ce26e0311b2a2ed539",
                 "sha256:d081707ef0081920533db30200a2d30d5c0ea9cf6afa7cf8881ae4516cc69c48"
             ],
-            "markers": "python_version != '3.3.*' and python_version != '3.0.*' and python_version >= '2.6' and python_version != '3.1.*' and python_version != '3.2.*'",
+            "markers": "python_version >= '2.6' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.3.*' and python_version != '3.0.*'",
             "version": "==5.4.7"
         },
         "py": {
@@ -606,16 +605,16 @@
                 "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1",
                 "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6"
             ],
-            "markers": "python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*'",
+            "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.3.*' and python_version != '3.0.*'",
             "version": "==1.6.0"
         },
         "pytest": {
             "hashes": [
-                "sha256:0a72d8a9f559c006ba153e0c9b4838efd7b656cf1f993747ba7128770d6eb12c",
+                "sha256:7e258ee50338f4e46957f9e09a0f10fb1c2d05493fa901d113a8dafd0790de4e",
-                "sha256:95529588ff4e85114a0b0ad8e9cf0131ca47d46b28230e25366c5aba66b1d854"
+                "sha256:9332147e9af2dcf46cd7ceb14d5acadb6564744ddff1fe8c17f0ce60ece7d9a2"
             ],
             "index": "pypi",
-            "version": "==3.8.1"
+            "version": "==3.8.2"
         },
         "pytest-cov": {
             "hashes": [
@@ -661,7 +660,7 @@
                 "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf",
                 "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5"
             ],
-            "markers": "python_version < '4' and python_version != '3.3.*' and python_version != '3.0.*' and python_version >= '2.6' and python_version != '3.1.*' and python_version != '3.2.*'",
+            "markers": "python_version != '3.3.*' and python_version != '3.2.*' and python_version >= '2.6' and python_version < '4' and python_version != '3.1.*' and python_version != '3.0.*'",
             "version": "==1.23"
         },
         "versioneer": {

assets/settings.js

@@ -1,4 +1,5 @@
 import Banner from '../components/Banner.html';
+import ArchiveForm from '../components/ArchiveForm.html';
 
 (function settings_init(){
     if(!('fetch' in window)){
@@ -188,4 +189,17 @@ import Banner from '../components/Banner.html';
             reason_banner.parentElement.removeChild(reason_banner);
         })
     }
+
+    let archive_form_el = document.querySelector('#archive-form');
+    if(archive_form_el){
+        let csrf_token = archive_form_el.querySelector('input[name=csrf-token]').value;
+        let archive_form = new ArchiveForm({
+            target: archive_form_el,
+            hydrate: true,
+            data: {
+                action: archive_form_el.action,
+                csrf_token: csrf_token,
+            },
+        })
+    }
 })();

assets/styles.css

@@ -76,6 +76,16 @@ img.avatar.mastodon {
     padding-top: .6rem;
     padding-bottom: .6rem;
     margin-bottom: 1rem;
+    padding-left: 1rem;
+    padding-right: 1rem;
+}
+
+.banner p {
+    margin: .3rem 0;
+}
+
+.banner p ~ p {
+    margin-top: 1.3rem;
 }
 
 body > section > .banner,

components/ArchiveForm.html

@@ -0,0 +1,49 @@
+<form action={action} method='post' enctype='multipart/form-data'>
+    {#if file_too_big}
+        <div class="banner warning">
+            <p>The file you have selected is very large.</p>
+
+            <p>Twitter has two types of archives, <b>Forget does not support
+                "Your Twitter data" archives, only "Tweet archives"</b>
+            available from
+            <a href="https://twitter.com/settings/account#export_request">your account settings</a>.
+            Please make sure you have the right type of archive.</p>
+        </div>
+    {/if}
+    <input type="file" name="file" accept="application/zip,.zip"
+        on:change="take_file(this.files)">
+    <input type="submit" value="Upload">
+    <input type='hidden' name='csrf-token' value={csrf_token}>
+</form>
+
+<script>
+function get_file_size(file_list){
+    /* returns size of selected file given an <input type="file">
+        or 0 if no file is selected */
+    let size = 0;
+    for(let i = 0; i < file_list.length; i++){
+        size += file_list[i].size;
+    }
+    return size;
+}
+
+
+export default {
+    data(){
+        return { file_size: 0 }
+    },
+
+    oncreate(){
+    },
+
+    methods: {
+        take_file(file_list){
+            this.set({file_size: get_file_size(file_list)});
+        },
+    },
+
+    computed: {
+        file_too_big: ({file_size}) => file_size > 10000000, // 10 MB
+    },
+}
+</script>

dodo.py

@@ -125,7 +125,7 @@ def task_rollup():
                 file_dep=list(chain(
                     # fuck it
                     glob('assets/*.js'),
-                    glob('components/*.html'))),
+                    glob('components/*.html'))) + ['rollup.config.js'],
                 targets=[dst],
                 clean=True,
                 actions=[

requirements-dev.txt

@@ -1,22 +1,22 @@
--i https://pypi.python.org/simple/
+-i https://pypi.python.org/simple
-atomicwrites==1.2.1; python_version != '3.2.*'
+atomicwrites==1.2.1; python_version != '3.0.*'
 attrs==18.2.0
 certifi==2018.8.24
 chardet==3.0.4
 codecov==2.0.15
 coverage==4.5.1
 idna==2.7
-mirakuru==1.0.0; python_version != '3.2.*'
+mirakuru==1.0.0; python_version != '3.0.*'
 more-itertools==4.3.0
-pluggy==0.7.1; python_version != '3.2.*'
+pluggy==0.7.1; python_version != '3.0.*'
 port-for==0.4
-psutil==5.4.7; python_version != '3.2.*'
+psutil==5.4.7; python_version != '3.0.*'
-py==1.6.0; python_version != '3.2.*'
+py==1.6.0; python_version != '3.0.*'
 pytest-cov==2.6.0
 pytest-redis==1.3.2
-pytest==3.8.1
+pytest==3.8.2
 redis==2.10.6
 requests==2.19.1
 six==1.11.0
-urllib3==1.23; python_version != '3.2.*'
+urllib3==1.23; python_version != '3.0.*'
 versioneer==0.18

requirements.txt

@@ -1,4 +1,4 @@
--i https://pypi.python.org/simple/
+-i https://pypi.python.org/simple
 alembic==1.0.0
 amqp==2.3.2
 asn1crypto==0.24.0
@@ -7,11 +7,11 @@ blinker==1.4
 brotli==1.0.6
 celery==4.2.1
 certifi==2018.8.24
-cffi==1.11.5; python_version != '3.3.*'
+cffi==1.11.5
 chardet==3.0.4
-click==6.7
+click==7.0; python_version != '3.2.*'
 cloudpickle==0.5.6
-cryptography==2.3.1; python_version != '3.3.*'
+cryptography==2.3.1; python_version != '3.2.*'
 csscompressor==0.9.5
 decorator==4.3.0
 doit==0.31.1
@@ -20,7 +20,7 @@ flask-sqlalchemy==2.3.2
 flask==1.0.2
 gunicorn==19.9.0
 honcho==1.0.1
-http-ece==1.0.5; python_version != '3.3.*'
+http-ece==1.0.5
 idna==2.7
 itsdangerous==0.24
 jinja2==2.10
@@ -28,9 +28,9 @@ kombu==4.2.1
 mako==1.0.7
 markupsafe==1.0
 mastodon.py==1.3.1
-pillow==5.2.0
+pillow==5.3.0
 psycopg2==2.7.5
-pycparser==2.19; python_version != '3.3.*'
+pycparser==2.19; python_version != '3.2.*'
 pyinotify==0.9.6; sys_platform == 'linux'
 python-dateutil==2.7.3
 python-editor==1.0.3
@@ -41,6 +41,6 @@ requests==2.19.1
 six==1.11.0
 sqlalchemy==1.2.12
 twitter==1.18.0
-urllib3==1.23; python_version != '3.2.*'
+urllib3==1.23; python_version != '3.0.*'
 vine==1.1.4
 werkzeug==0.14.1

rollup.config.js

@@ -7,6 +7,7 @@ export default {
     plugins: [
         svelte({
             include: 'components/**/*.html',
+            hydratable: true,
         }),
     ]
 }

templates/logged_in.html

@@ -139,8 +139,8 @@
 
     <h2 id='tweet_archive_import'>Tweet archive import</h2>
     <p>
-    Twitter's API only lets us access up to 3200 of your most recent tweets. If you have more tweets than that, you can request a zip archive of your tweets from
+    Twitter's API only lets us access up to 3200 of your most recent tweets. If you have more tweets than that, you can request a tweet archive from
-    <a href="https://twitter.com/settings/account">Twitter's settings page</a>,
+    <a href="https://twitter.com/settings/account#export_request">your account settings</a>,
     and upload it here.
     </p>
 
@@ -148,7 +148,7 @@
     <div class="banner error">The file you uploaded is not a valid tweet archive. No posts have been imported.</div>
     {% endif %}
 
-    <form action='{{url_for('upload_tweet_archive')}}' method='post' enctype='multipart/form-data'>
+    <form action='{{url_for('upload_tweet_archive')}}' method='post' enctype='multipart/form-data' id='archive-form'>
         <input type="file" name='file' accept='application/zip,.zip'><input type="submit" value="Upload">
         <input type='hidden' name='csrf-token' value='{{g.viewer.csrf_token}}'>
     </form>