Mastodon
/
forget-cancellare-vecchi-toot
mirror of https://github.com/codl/forget
2
0
Fork 0
Code Issues Releases Wiki Activity

words are like 

the middle class
a drinking glass
a mask

words are like
a spanish town
a wedding gown
a crown
This commit is contained in:
codl 2017-07-29 17:43:09 +02:00
parent dab2f8fc0f
commit 079cb51b7f
No known key found for this signature in database
GPG Key ID: 6CD7C8891ED1233A
8 changed files with 53 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
lib/twitter.py
View File

@ -33,7 +33,7 @@ def receive_verifier(oauth_token, oauth_verifier, consumer_key=None, consumer_se
new_twitter = Twitter(
auth=OAuth(new_token.token, new_token.token_secret, consumer_key, consumer_secret))
remote_acct = new_twitter.account.verify_credentials()
acct = Account(remote_id = remote_acct['id_str'])
acct = Account(twitter_id = remote_acct['id_str'])
acct = db.session.merge(acct)
acct.remote_display_name = remote_acct['name']
@ -62,15 +62,13 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
account.remote_screen_name = user['screen_name']
account.remote_avatar_url = user['profile_image_url_https']
kwargs = { 'user_id': account.remote_id, 'count': 200, 'trim_user': True }
kwargs = { 'user_id': account.twitter_id, 'count': 200, 'trim_user': True }
kwargs.update(cursor or {})
if 'max_id' not in kwargs:
most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.remote_id).first()
most_recent_post = Post.query.order_by(db.desc(Post.created_at)).filter(Post.author_id == account.twitter_id).first()
if most_recent_post:
kwargs['since_id'] = most_recent_post.remote_id
print(kwargs)
kwargs['since_id'] = most_recent_post.twitter_id
tweets = t.statuses.user_timeline(**kwargs)
@ -81,8 +79,7 @@ def fetch_acc(account, cursor, consumer_key=None, consumer_secret=None):
kwargs['max_id'] = +inf
for tweet in tweets:
print("TWEET ", tweet['text'])
post = Post(remote_id=tweet['id_str'])
post = Post(twitter_id=tweet['id_str'])
post = db.session.merge(post)
post.created_at = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S %z %Y')
post.body = tweet['text']

24
migrations/versions/1003f9df0ae0_.py
View File

@ -1,24 +0,0 @@
"""empty message
Revision ID: 1003f9df0ae0
Revises: 8c2ce3a66650
Create Date: 2017-07-28 12:32:54.375901
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '1003f9df0ae0'
down_revision = '8c2ce3a66650'
branch_labels = None
depends_on = None
def upgrade():
op.alter_column('accounts', 'last_post_fetch', new_column_name='last_fetch')
def downgrade():
op.alter_column('accounts', 'last_fetch', new_column_name='last_post_fetch')

24
migrations/versions/1727266feaff_.py
View File

@ -1,24 +0,0 @@
"""empty message
Revision ID: 1727266feaff
Revises: 1003f9df0ae0
Create Date: 2017-07-29 11:09:02.743619
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '1727266feaff'
down_revision = '1003f9df0ae0'
branch_labels = None
depends_on = None
def upgrade():
op.add_column('accounts', sa.Column('remote_screen_name', sa.String(), nullable=True))
def downgrade():
op.drop_column('accounts', 'remote_screen_name')

28
migrations/versions/8c2ce3a66650_.py
View File

@ -1,28 +0,0 @@
"""empty message
Revision ID: 8c2ce3a66650
Revises: a5718ca3ead1
Create Date: 2017-07-27 23:35:48.842519
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '8c2ce3a66650'
down_revision = 'a5718ca3ead1'
branch_labels = None
depends_on = None
def upgrade():
op.drop_constraint('fk_posts_author_remote_id_accounts', 'posts', type_='foreignkey')
op.alter_column('posts', 'author_remote_id', new_column_name='author_id')
op.create_foreign_key(op.f('fk_posts_author_id_accounts'), 'posts', 'accounts', ['author_id'], ['remote_id'])
def downgrade():
op.drop_constraint(op.f('fk_posts_author_id_accounts'), 'posts', type_='foreignkey')
op.alter_column('posts', 'author_id', new_column_name='author_remote_id')
op.create_foreign_key('fk_posts_author_remote_id_accounts', 'posts', 'accounts', ['author_remote_id'], ['remote_id'])

27
migrations/versions/a5718ca3ead1_.py → migrations/versions/92ffc9941fd9_.py
View File

@ -1,8 +1,8 @@
"""empty message
Revision ID: a5718ca3ead1
Revision ID: 92ffc9941fd9
Revises:
Create Date: 2017-07-27 20:12:42.873090
Create Date: 2017-07-29 12:31:47.687234
"""
from alembic import op
@ -10,7 +10,7 @@ import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'a5718ca3ead1'
revision = '92ffc9941fd9'
down_revision = None
branch_labels = None
depends_on = None
@ -21,36 +21,37 @@ def upgrade():
op.create_table('accounts',
sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
sa.Column('remote_id', sa.String(), nullable=False),
sa.Column('id', sa.String(), nullable=False),
sa.Column('remote_display_name', sa.String(), nullable=True),
sa.Column('remote_screen_name', sa.String(), nullable=True),
sa.Column('remote_avatar_url', sa.String(), nullable=True),
sa.Column('last_post_fetch', sa.DateTime(), server_default='epoch', nullable=True),
sa.PrimaryKeyConstraint('remote_id', name=op.f('pk_accounts'))
sa.Column('last_fetch', sa.DateTime(), server_default='epoch', nullable=True),
sa.PrimaryKeyConstraint('id', name=op.f('pk_accounts'))
)
op.create_table('oauth_tokens',
sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
sa.Column('token', sa.String(), nullable=False),
sa.Column('token_secret', sa.String(), nullable=False),
sa.Column('remote_id', sa.String(), nullable=True),
sa.ForeignKeyConstraint(['remote_id'], ['accounts.remote_id'], name=op.f('fk_oauth_tokens_remote_id_accounts')),
sa.Column('account_id', sa.String(), nullable=True),
sa.ForeignKeyConstraint(['account_id'], ['accounts.id'], name=op.f('fk_oauth_tokens_account_id_accounts')),
sa.PrimaryKeyConstraint('token', name=op.f('pk_oauth_tokens'))
)
op.create_table('posts',
sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
sa.Column('remote_id', sa.String(), nullable=False),
sa.Column('id', sa.String(), nullable=False),
sa.Column('body', sa.String(), nullable=True),
sa.Column('author_remote_id', sa.String(), nullable=True),
sa.ForeignKeyConstraint(['author_remote_id'], ['accounts.remote_id'], name=op.f('fk_posts_author_remote_id_accounts')),
sa.PrimaryKeyConstraint('remote_id', name=op.f('pk_posts'))
sa.Column('author_id', sa.String(), nullable=True),
sa.ForeignKeyConstraint(['author_id'], ['accounts.id'], name=op.f('fk_posts_author_id_accounts')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_posts'))
)
op.create_table('sessions',
sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
sa.Column('id', sa.String(), nullable=False),
sa.Column('account_id', sa.String(), nullable=True),
sa.ForeignKeyConstraint(['account_id'], ['accounts.remote_id'], name=op.f('fk_sessions_account_id_accounts')),
sa.ForeignKeyConstraint(['account_id'], ['accounts.id'], name=op.f('fk_sessions_account_id_accounts')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_sessions'))
)
# ### end Alembic commands ###

34
model.py
View File

@ -12,10 +12,28 @@ class TimestampMixin(object):
def touch(self):
self.updated_at=db.func.now()
class RemoteIDMixin(object):
@property
def service(self):
if not self.id:
return None
return self.id.split(":")[0]
class Account(db.Model, TimestampMixin):
@property
def twitter_id(self):
if self.service != "twitter":
raise Exception("wrong service bucko")
return self.id.split(":")[1]
@twitter_id.setter
def twitter_id(self, id):
self.id = "twitter:{}".format(id)
class Account(db.Model, TimestampMixin, RemoteIDMixin):
__tablename__ = 'accounts'
remote_id = db.Column(db.String, primary_key=True)
id = db.Column(db.String, primary_key=True)
# policy_enabled = db.Column(db.Boolean, server_default='FALSE', nullable=False)
# policy_keep_younger = db.Column(db.Interval)
@ -32,7 +50,7 @@ class Account(db.Model, TimestampMixin):
# backref: tokens
def __repr__(self):
return f"<Account({self.remote_id}, {self.remote_screen_name}, {self.remote_display_name})>"
return f"<Account({self.id}, {self.remote_screen_name}, {self.remote_display_name})>"
class OAuthToken(db.Model, TimestampMixin):
__tablename__ = 'oauth_tokens'
@ -40,7 +58,7 @@ class OAuthToken(db.Model, TimestampMixin):
token = db.Column(db.String, primary_key=True)
token_secret = db.Column(db.String, nullable=False)
remote_id = db.Column(db.String, db.ForeignKey('accounts.remote_id'))
account_id = db.Column(db.String, db.ForeignKey('accounts.id'))
account = db.relationship(Account, backref=db.backref('tokens', order_by=lambda: db.desc(OAuthToken.created_at)))
class Session(db.Model, TimestampMixin):
@ -48,14 +66,14 @@ class Session(db.Model, TimestampMixin):
id = db.Column(db.String, primary_key=True, default=lambda: secrets.token_urlsafe())
account_id = db.Column(db.String, db.ForeignKey('accounts.remote_id'))
account_id = db.Column(db.String, db.ForeignKey('accounts.id'))
account = db.relationship(Account, lazy='joined')
class Post(db.Model, TimestampMixin):
class Post(db.Model, TimestampMixin, RemoteIDMixin):
__tablename__ = 'posts'
remote_id = db.Column(db.String, primary_key=True)
id = db.Column(db.String, primary_key=True)
body = db.Column(db.String)
author_id = db.Column(db.String, db.ForeignKey('accounts.remote_id'))
author_id = db.Column(db.String, db.ForeignKey('accounts.id'))
author = db.relationship(Account)

2
routes.py
View File

@ -39,7 +39,7 @@ def twitter_login_step2():
oauth_token = request.args['oauth_token']
oauth_verifier = request.args['oauth_verifier']
token = lib.twitter.receive_verifier(oauth_token, oauth_verifier, **app.config.get_namespace("TWITTER_"))
session = Session(account_id = token.remote_id)
session = Session(account_id = token.account_id)
db.session.add(session)
db.session.commit()
resp = Response(status=301, headers={"location": url_for('index')})

13
tasks.py
View File

@ -17,13 +17,14 @@ def remove_old_sessions():
db.session.commit()
@app.task(autoretry_for=(TwitterError, URLError))
def fetch_acc(remote_id, cursor=None):
acc = Account.query.get(remote_id)
def fetch_acc(id, cursor=None):
acc = Account.query.get(id)
print(f'fetching {acc}')
try:
cursor = lib.twitter.fetch_acc(acc, cursor, **flaskapp.config.get_namespace("TWITTER_"))
if cursor:
fetch_acc.si(remote_id, cursor).apply_async()
if(acc.service == 'twitter'):
cursor = lib.twitter.fetch_acc(acc, cursor, **flaskapp.config.get_namespace("TWITTER_"))
if cursor:
fetch_acc.si(id, cursor).apply_async()
finally:
db.session.rollback()
acc.last_fetch = db.func.now()
@ -38,7 +39,7 @@ def queue_fetch_for_most_stale_accounts(min_staleness=timedelta(minutes=5), limi
.order_by(db.asc(Account.last_fetch))\
.limit(limit)
for acc in accs:
fetch_acc.s(acc.remote_id).delay()
fetch_acc.s(acc.id).delay()
acc.last_fetch = db.func.now()
db.session.commit()