remove dependency on persistentlist because unstable and use own storage

This commit is contained in:
Carl Chenet 2017-04-12 11:38:22 +02:00
parent 770b3e3e1e
commit dffd5c8c36
3 changed files with 57 additions and 5 deletions

52
feed2toot/feedcache.py Normal file
View File

@ -0,0 +1,52 @@
# vim:ts=4:sw=4:ft=python:fileencoding=utf-8
# Copyright © 2015-2017 Carl Chenet <carl.chenet@ohmytux.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>
'''Manage a cache with the ids of the feed entries'''
# standard libraires imports
from collections import deque
import os
import os.path
class FeedCache:
'''FeedCache class'''
def __init__(self, options):
'''Constructore of the FeedCache class'''
self.options = options
self.main()
def getdeque(self):
'''return the deque'''
return self.dbfeed
def main(self):
'''Main of the FeedCache class'''
if os.path.exists(self.options['cachefile']):
with open(self.options['cachefile']) as dbdsc:
dbfromfile = dbdsc.readlines()
dblist = [i.strip() for i in dbfromfile]
self.dbfeed = deque(dblist, self.options['cache_limit'] )
else:
self.dbfeed = deque([], self.options['cache_limit'] )
def append(self, rssid):
'''Append a rss id to the cache'''
self.dbfeed.append(rssid)
def close(self):
'''Close the cache'''
with open(self.options['cachefile'], 'w') as dbdsc:
dbdsc.writelines((''.join([i, os.linesep]) for i in self.dbfeed))

View File

@ -26,7 +26,6 @@ import sys
# 3rd party libraries imports
import feedparser
from persistentlist import PersistentList
# app libraries imports
from feed2toot.addtags import AddTags
@ -35,6 +34,7 @@ from feed2toot.confparse import ConfParse
from feed2toot.filterentry import FilterEntry
from feed2toot.removeduplicates import RemoveDuplicates
from feed2toot.tootpost import TootPost
from feed2toot.feedcache import FeedCache
class Main(object):
'''Main class of Feed2toot'''
@ -73,8 +73,8 @@ class Main(object):
tweetformat = conf[2]
feeds = conf[3]
plugins = conf[4]
# open the persistent list
cache = PersistentList(options['cachefile'][0:-3], options['cache_limit'])
# create link to the persistent list
cache = FeedCache(options)
if options['hashtaglist']:
severalwordshashtags = codecs.open(options['hashtaglist'],
encoding='utf-8').readlines()
@ -97,7 +97,7 @@ class Main(object):
# cache the ids of last rss feeds
if not clioptions.all:
for i in entries:
if 'id' in i and i['id'] not in cache:
if 'id' in i and i['id'] not in cache.getdeque():
totweet.append(i)
else:
totweet = entries

View File

@ -42,7 +42,7 @@ setup(
download_url='https://github.com/chaica/feed2toot',
packages=find_packages(),
scripts=['scripts/feed2toot', 'scripts/register_feed2toot_app'],
install_requires=['feedparser', 'persistentlist>=0.4', 'Mastodon.py'],
install_requires=['feedparser', 'Mastodon.py'],
extras_require={
'influxdb': ["influxdb"]
}