ahhhhhhhhhhhhhh!!!!!!!!!!!!!!!!!!!!!!!!! posts
This commit is contained in:
parent
d58073d1be
commit
9722dbde26
|
@ -20,6 +20,14 @@ def get_login_url(callback='oob', consumer_key=None, consumer_secret=None):
|
||||||
|
|
||||||
return "https://api.twitter.com/oauth/authenticate?oauth_token=%s" % (oauth_token,)
|
return "https://api.twitter.com/oauth/authenticate?oauth_token=%s" % (oauth_token,)
|
||||||
|
|
||||||
|
def account_from_api_user_object(obj):
|
||||||
|
return Account(
|
||||||
|
twitter_id = obj['id_str'],
|
||||||
|
display_name = obj['name'],
|
||||||
|
screen_name = obj['screen_name'],
|
||||||
|
avatar_url = obj['profile_image_url_https'],
|
||||||
|
reported_post_count = obj['statuses_count'])
|
||||||
|
|
||||||
def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_secret=None):
|
def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_secret=None):
|
||||||
temp_token = OAuthToken.query.get(oauth_token)
|
temp_token = OAuthToken.query.get(oauth_token)
|
||||||
if not temp_token:
|
if not temp_token:
|
||||||
|
@ -34,12 +42,9 @@ def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_se
|
||||||
new_twitter = Twitter(
|
new_twitter = Twitter(
|
||||||
auth=OAuth(new_token.token, new_token.token_secret, consumer_key, consumer_secret))
|
auth=OAuth(new_token.token, new_token.token_secret, consumer_key, consumer_secret))
|
||||||
remote_acct = new_twitter.account.verify_credentials()
|
remote_acct = new_twitter.account.verify_credentials()
|
||||||
acct = Account(twitter_id = remote_acct['id_str'])
|
acct = account_from_api_user_object(remote_acct)
|
||||||
acct = db.session.merge(acct)
|
acct = db.session.merge(acct)
|
||||||
|
|
||||||
acct.display_name = remote_acct['name']
|
|
||||||
acct.screen_name = remote_acct['screen_name']
|
|
||||||
acct.avatar_url = remote_acct['profile_image_url_https']
|
|
||||||
new_token.account = acct
|
new_token.account = acct
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
|
|
||||||
|
@ -57,7 +62,7 @@ def get_twitter_for_acc(account):
|
||||||
|
|
||||||
locale.setlocale(locale.LC_TIME, 'C')
|
locale.setlocale(locale.LC_TIME, 'C')
|
||||||
|
|
||||||
def tweet_to_post(tweet, post=None):
|
def post_from_api_tweet_object(tweet, post=None):
|
||||||
if not post:
|
if not post:
|
||||||
post = Post()
|
post = Post()
|
||||||
post.twitter_id = tweet['id_str']
|
post.twitter_id = tweet['id_str']
|
||||||
|
@ -79,13 +84,11 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
|
||||||
t = get_twitter_for_acc(account)
|
t = get_twitter_for_acc(account)
|
||||||
|
|
||||||
user = t.account.verify_credentials()
|
user = t.account.verify_credentials()
|
||||||
|
db.session.merge(account_from_api_user_object(user))
|
||||||
account.remote_display_name = user['name']
|
|
||||||
account.remote_screen_name = user['screen_name']
|
|
||||||
account.remote_avatar_url = user['profile_image_url_https']
|
|
||||||
|
|
||||||
kwargs = { 'user_id': account.twitter_id, 'count': 200, 'trim_user': True, 'tweet_mode': 'extended' }
|
kwargs = { 'user_id': account.twitter_id, 'count': 200, 'trim_user': True, 'tweet_mode': 'extended' }
|
||||||
kwargs.update(cursor or {})
|
if cursor:
|
||||||
|
kwargs.update(cursor)
|
||||||
|
|
||||||
if 'max_id' not in kwargs:
|
if 'max_id' not in kwargs:
|
||||||
most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.id).first()
|
most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.id).first()
|
||||||
|
@ -101,7 +104,7 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
|
||||||
kwargs['max_id'] = +inf
|
kwargs['max_id'] = +inf
|
||||||
|
|
||||||
for tweet in tweets:
|
for tweet in tweets:
|
||||||
db.session.merge(tweet_to_post(tweet))
|
db.session.merge(post_from_api_tweet_object(tweet))
|
||||||
kwargs['max_id'] = min(tweet['id'] - 1, kwargs['max_id'])
|
kwargs['max_id'] = min(tweet['id'] - 1, kwargs['max_id'])
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
@ -125,7 +128,7 @@ def refresh_posts(posts):
|
||||||
if not tweet:
|
if not tweet:
|
||||||
session.delete(post)
|
session.delete(post)
|
||||||
else:
|
else:
|
||||||
post = db.session.merge(tweet_to_post(tweet))
|
post = db.session.merge(post_from_api_tweet_object(tweet))
|
||||||
refreshed_posts.append(post)
|
refreshed_posts.append(post)
|
||||||
|
|
||||||
return refreshed_posts
|
return refreshed_posts
|
||||||
|
|
|
@ -0,0 +1,24 @@
|
||||||
|
"""add reported post count column to account
|
||||||
|
|
||||||
|
Revision ID: 0983f1227366
|
||||||
|
Revises: 7afc95e24778
|
||||||
|
Create Date: 2017-08-03 19:16:55.883575
|
||||||
|
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = '0983f1227366'
|
||||||
|
down_revision = '7afc95e24778'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.add_column('accounts', sa.Column('reported_post_count', sa.Integer(), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_column('accounts', 'reported_post_count')
|
|
@ -0,0 +1,28 @@
|
||||||
|
"""add policy defaults
|
||||||
|
|
||||||
|
Revision ID: 2e3a2882e5a4
|
||||||
|
Revises: 0983f1227366
|
||||||
|
Create Date: 2017-08-03 19:54:20.643627
|
||||||
|
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = '2e3a2882e5a4'
|
||||||
|
down_revision = '0983f1227366'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.alter_column('accounts', 'policy_keep_latest', server_default='100')
|
||||||
|
op.alter_column('accounts', 'policy_keep_younger', server_default='365 days')
|
||||||
|
op.alter_column('accounts', 'policy_delete_every', server_default='30 minutes')
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.alter_column('accounts', 'policy_keep_latest', server_default='0')
|
||||||
|
op.alter_column('accounts', 'policy_keep_younger', server_default='0')
|
||||||
|
op.alter_column('accounts', 'policy_delete_every', server_default='0')
|
8
model.py
8
model.py
|
@ -42,14 +42,15 @@ class Account(TimestampMixin, RemoteIDMixin):
|
||||||
id = db.Column(db.String, primary_key=True)
|
id = db.Column(db.String, primary_key=True)
|
||||||
|
|
||||||
policy_enabled = db.Column(db.Boolean, server_default='FALSE', nullable=False)
|
policy_enabled = db.Column(db.Boolean, server_default='FALSE', nullable=False)
|
||||||
policy_keep_latest = db.Column(db.Integer, server_default='0', nullable=False)
|
policy_keep_latest = db.Column(db.Integer, server_default='100', nullable=False)
|
||||||
policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE', nullable=False)
|
policy_keep_favourites = db.Column(db.Boolean, server_default='TRUE', nullable=False)
|
||||||
policy_delete_every = db.Column(db.Interval, server_default='0', nullable=False)
|
policy_delete_every = db.Column(db.Interval, server_default='30 minutes', nullable=False)
|
||||||
policy_keep_younger = db.Column(db.Interval, server_default='0', nullable=False)
|
policy_keep_younger = db.Column(db.Interval, server_default='365 days', nullable=False)
|
||||||
|
|
||||||
display_name = db.Column(db.String)
|
display_name = db.Column(db.String)
|
||||||
screen_name = db.Column(db.String)
|
screen_name = db.Column(db.String)
|
||||||
avatar_url = db.Column(db.String)
|
avatar_url = db.Column(db.String)
|
||||||
|
reported_post_count = db.Column(db.Integer)
|
||||||
|
|
||||||
last_fetch = db.Column(db.DateTime, server_default='epoch')
|
last_fetch = db.Column(db.DateTime, server_default='epoch')
|
||||||
last_delete = db.Column(db.DateTime, server_default='epoch')
|
last_delete = db.Column(db.DateTime, server_default='epoch')
|
||||||
|
@ -77,6 +78,7 @@ class Account(TimestampMixin, RemoteIDMixin):
|
||||||
return Post.query.with_parent(self).count()
|
return Post.query.with_parent(self).count()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class Account(Account, db.Model):
|
class Account(Account, db.Model):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
|
@ -63,9 +63,9 @@ def upload_tweet_archive():
|
||||||
db.session.add(ta)
|
db.session.add(ta)
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
|
|
||||||
tasks.import_twitter_archive.s(ta.id).apply_async(routing_key='high')
|
tasks.chunk_twitter_archive(ta.id)
|
||||||
|
|
||||||
return redirect(url_for('index'))
|
return redirect(url_for('index', _anchor='recent_archives'))
|
||||||
|
|
||||||
@app.route('/settings', methods=('POST',))
|
@app.route('/settings', methods=('POST',))
|
||||||
@require_auth
|
@require_auth
|
||||||
|
|
4
tasks.py
4
tasks.py
|
@ -49,7 +49,7 @@ def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limi
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
|
|
||||||
@app.task
|
@app.task
|
||||||
def import_twitter_archive(archive_id):
|
def chunk_twitter_archive(archive_id):
|
||||||
ta = TwitterArchive.query.get(archive_id)
|
ta = TwitterArchive.query.get(archive_id)
|
||||||
|
|
||||||
with ZipFile(BytesIO(ta.body), 'r') as zipfile:
|
with ZipFile(BytesIO(ta.body), 'r') as zipfile:
|
||||||
|
@ -79,7 +79,7 @@ def import_twitter_archive_month(archive_id, month_path):
|
||||||
tweets = json.load(f)
|
tweets = json.load(f)
|
||||||
|
|
||||||
for tweet in tweets:
|
for tweet in tweets:
|
||||||
post = lib.twitter.tweet_to_post(tweet)
|
post = lib.twitter.post_from_api_tweet_object(tweet)
|
||||||
existing_post = db.session.query(Post).get(post.id)
|
existing_post = db.session.query(Post).get(post.id)
|
||||||
|
|
||||||
if post.author_id != ta.account_id \
|
if post.author_id != ta.account_id \
|
||||||
|
|
|
@ -20,7 +20,13 @@
|
||||||
</form>
|
</form>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<p>Currently keeping track of {{g.viewer.account.post_count()}} posts</p>
|
{% set post_count = g.viewer.account.post_count() %}
|
||||||
|
<p>Currently keeping track of {{ post_count }} posts</p>
|
||||||
|
{% if g.viewer.account.service == 'twitter'
|
||||||
|
and post_count < g.viewer.account.reported_post_count * 3/4
|
||||||
|
and g.viewer.account.reported_post_count > 3200 -%}
|
||||||
|
<div class='banner warning'>Are we missing some of your posts? Try uploading a tweet archive below</div>
|
||||||
|
{% endif %}
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section>
|
<section>
|
||||||
|
@ -64,7 +70,7 @@
|
||||||
</form>
|
</form>
|
||||||
|
|
||||||
{% if g.viewer.account.twitter_archives %}
|
{% if g.viewer.account.twitter_archives %}
|
||||||
<h3 >Recent imports</h3>
|
<h3 name="recent_archives">Recent archives</h3>
|
||||||
<ul>
|
<ul>
|
||||||
{% for archive in g.viewer.account.twitter_archives %}
|
{% for archive in g.viewer.account.twitter_archives %}
|
||||||
<li>{{archive.created_at.strftime('%Y-%m-%d %H:%M')}}<br>
|
<li>{{archive.created_at.strftime('%Y-%m-%d %H:%M')}}<br>
|
||||||
|
|
Loading…
Reference in New Issue