diff --git a/lib/twitter.py b/lib/twitter.py index 16da75c..a0c6efd 100644 --- a/lib/twitter.py +++ b/lib/twitter.py @@ -4,6 +4,7 @@ from model import OAuthToken, Account, Post from app import db from math import inf from datetime import datetime +import locale def get_login_url(callback='oob', consumer_key=None, consumer_secret=None): twitter = Twitter( @@ -50,8 +51,31 @@ def get_twitter_for_acc(account, consumer_key=None, consumer_secret=None): auth=OAuth(token.token, token.token_secret, consumer_key, consumer_secret)) return t -import locale -locale.setlocale(locale.LC_TIME, 'C') # jeez i hate that i have to do this +locale.setlocale(locale.LC_TIME, 'C') + +def csv_tweet_to_json_tweet(tweet, account): + tweet.update({ + 'id': int(tweet['tweet_id']), + 'id_str': tweet['tweet_id'], + 'created_at': datetime.strptime(tweet['timestamp'], + '%Y-%m-%d %H:%M:%S %z')\ + .strftime('%a %b %d %H:%M:%S %z %Y'), + 'user': { + 'id': int(account.twitter_id), + 'id_str': account.twitter_id + } + }) + return tweet + +def tweet_to_post(tweet): + post = Post(twitter_id=tweet['id_str']) + post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y') + if 'full_text' in tweet: + post.body = tweet['full_text'] + else: + post.body = tweet['text'] + post.author_id = 'twitter:{}'.format(tweet['user']['id_str']) + return post def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None): t = get_twitter_for_acc(account, consumer_key=consumer_key, consumer_secret=consumer_secret) @@ -79,11 +103,7 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None): kwargs['max_id'] = +inf for tweet in tweets: - post = Post(twitter_id=tweet['id_str']) - post = db.session.merge(post) - post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y') - post.body = tweet['full_text'] - post.author = account + import_tweet(tweet, account, db.session) kwargs['max_id'] = min(tweet['id'] - 1, kwargs['max_id']) else: diff --git a/migrations/versions/0cb99099c2dd_add_twitter_archives.py b/migrations/versions/0cb99099c2dd_add_twitter_archives.py new file mode 100644 index 0000000..277029b --- /dev/null +++ b/migrations/versions/0cb99099c2dd_add_twitter_archives.py @@ -0,0 +1,32 @@ +"""add twitter archives + +Revision ID: 0cb99099c2dd +Revises: 92ffc9941fd9 +Create Date: 2017-07-30 23:13:48.949949 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '0cb99099c2dd' +down_revision = '92ffc9941fd9' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('twitter_archives', + sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('account_id', sa.String(), nullable=False), + sa.Column('body', sa.LargeBinary(), nullable=False), + sa.ForeignKeyConstraint(['account_id'], ['accounts.id'], name=op.f('fk_twitter_archives_account_id_accounts')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_twitter_archives')) + ) + + +def downgrade(): + op.drop_table('twitter_archives') diff --git a/model.py b/model.py index 662a5a7..78424c7 100644 --- a/model.py +++ b/model.py @@ -54,6 +54,9 @@ class Account(db.Model, TimestampMixin, RemoteIDMixin): def __repr__(self): return f"" + def post_count(self): + return Post.query.filter(Post.author_id == self.id).count() + class OAuthToken(db.Model, TimestampMixin): __tablename__ = 'oauth_tokens' @@ -79,3 +82,11 @@ class Post(db.Model, TimestampMixin, RemoteIDMixin): author_id = db.Column(db.String, db.ForeignKey('accounts.id')) author = db.relationship(Account) + +class TwitterArchive(db.Model, TimestampMixin): + __tablename__ = 'twitter_archives' + + id = db.Column(db.Integer, primary_key=True) + account_id = db.Column(db.String, db.ForeignKey('accounts.id'), nullable=False) + account = db.relationship(Account) + body = db.Column(db.LargeBinary, nullable=False) diff --git a/routes.py b/routes.py index 3debd3e..306fb31 100644 --- a/routes.py +++ b/routes.py @@ -2,7 +2,7 @@ from app import app from flask import render_template, url_for, redirect, request, g, Response from datetime import datetime import lib.twitter -from model import Account, Session, Post +from model import Account, Session, Post, TwitterArchive from app import db import tasks @@ -47,13 +47,28 @@ def twitter_login_step2(): tasks.fetch_acc.s(token.account_id).delay() - resp = Response(status=301, headers={"location": url_for('index')}) + resp = Response(status=302, headers={"location": url_for('index')}) resp.set_cookie('forget_sid', session.id, max_age=60*60*48, httponly=True, secure=app.config.get("HTTPS")) return resp +@app.route('/upload_twitter_archive', methods=('POST',)) +def upload_twitter_archive(): + if not g.viewer or 'file' not in request.files: + return "no" + return redirect(url_for('index')) + + ta = TwitterArchive(account = g.viewer.account, + body = request.files['file'].read()) + db.session.add(ta) + db.session.commit() + + tasks.import_twitter_archive.s(ta.id).apply_async() + + return "cool. your file's being processed probably" + @app.route('/logout') def logout(): if(g.viewer): diff --git a/tasks.py b/tasks.py index 3625804..4011fae 100644 --- a/tasks.py +++ b/tasks.py @@ -2,11 +2,14 @@ from celery import Celery from app import app as flaskapp from app import db -from model import Session, Account +from model import Session, Account, TwitterArchive import lib.twitter from twitter import TwitterError from urllib.error import URLError from datetime import timedelta, datetime +from zipfile import ZipFile +from io import BytesIO, TextIOWrapper +import csv app = Celery('tasks', broker=flaskapp.config['CELERY_BROKER'], task_serializer='pickle') @@ -46,6 +49,21 @@ def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limi app.add_periodic_task(10*60, remove_old_sessions) app.add_periodic_task(60, queue_fetch_for_most_stale_accounts) +@app.task +def import_twitter_archive(id): + ta = TwitterArchive.query.get(id) + + with ZipFile(BytesIO(ta.body), 'r') as zipfile: + tweetscsv = TextIOWrapper(zipfile.open('tweets.csv', 'r')) + + for tweet in csv.DictReader(tweetscsv): + tweet = lib.twitter.csv_tweet_to_json_tweet(tweet, ta.account) + post = lib.twitter.tweet_to_post(tweet) + db.session.merge(post) + db.session.commit() + + db.session.delete(ta) + db.session.commit() diff --git a/templates/index.html b/templates/index.html index 894f8a9..a2d4d5c 100644 --- a/templates/index.html +++ b/templates/index.html @@ -2,7 +2,13 @@

Hello, {{g.viewer.account.remote_display_name}}! Log out

-

your posts:

+ +
+ Upload your tweet archive: + +
+ +

your posts ({{g.viewer.account.post_count()}}):

{% for post in posts %}

{{post.body}}

{% else %}