FreshRSS/Readrops
1
0
Fork 0
mirror of https://github.com/readrops/Readrops.git synced 2025-02-03 12:07:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

Starting re-implementing feed and item insertion process, for better flexibility and control over results

Browse Source
This commit is contained in:
Shinokuni 2019-03-11 19:12:30 +01:00
parent 5aa6496d1e
commit 0869631d95
10 changed files with 193 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
app/src/main/java/com/readrops/app/activities/AddFeedActivity.java
View File

@ -1,7 +1,5 @@
package com.readrops.app.activities;
import android.arch.lifecycle.AndroidViewModel;
import android.arch.lifecycle.ViewModelProvider;
import android.arch.lifecycle.ViewModelProviders;
import android.support.design.widget.TextInputEditText;
import android.support.v7.app.AppCompatActivity;
@ -10,32 +8,25 @@ import android.support.v7.widget.DividerItemDecoration;
import android.support.v7.widget.LinearLayoutManager;
import android.support.v7.widget.RecyclerView;
import android.util.Patterns;
import android.util.SparseBooleanArray;
import android.view.MenuItem;
import android.view.View;
import android.widget.Button;
import android.widget.ProgressBar;
import android.widget.Toast;
import com.mikepenz.fastadapter.FastAdapter;
import com.mikepenz.fastadapter.adapters.ItemAdapter;
import com.readrops.app.R;
import com.readrops.app.utils.HtmlParser;
import com.readrops.app.database.entities.Feed;
import com.readrops.app.utils.Utils;
import com.readrops.app.utils.ParsingResult;
import com.readrops.app.viewmodels.AddFeedsViewModel;
import com.readrops.readropslibrary.localfeed.RSSNetwork;
import java.util.ArrayList;
import java.util.List;
import io.reactivex.Scheduler;
import io.reactivex.Single;
import io.reactivex.SingleObserver;
import io.reactivex.SingleOnSubscribe;
import io.reactivex.android.schedulers.AndroidSchedulers;
import io.reactivex.disposables.Disposable;
import io.reactivex.observers.DisposableCompletableObserver;
import io.reactivex.schedulers.Schedulers;
public class AddFeedActivity extends AppCompatActivity implements View.OnClickListener {
@ -125,15 +116,20 @@ public class AddFeedActivity extends AppCompatActivity implements View.OnClickLi
viewModel.addFeeds(feedsToInsert)
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe(new DisposableCompletableObserver() {
.subscribe(new SingleObserver<List<Feed>>() {
@Override
public void onComplete() {
Toast.makeText(getApplication(), "feeds inserted", Toast.LENGTH_LONG).show();
public void onSubscribe(Disposable d) {
}
@Override
public void onSuccess(List<Feed> feeds) {
}
@Override
public void onError(Throwable e) {
Toast.makeText(getApplication(), "error on feed insertion", Toast.LENGTH_LONG).show();
}
});
}

9
app/src/main/java/com/readrops/app/database/entities/Item.java
View File

@ -10,11 +10,13 @@ import com.readrops.readropslibrary.localfeed.rss.RSSEnclosure;
import com.readrops.readropslibrary.localfeed.rss.RSSItem;
import com.readrops.readropslibrary.localfeed.rss.RSSMediaContent;
import org.joda.time.DateTime;
import org.joda.time.LocalDateTime;
import org.jsoup.Jsoup;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.regex.Pattern;
@ -23,7 +25,7 @@ import static android.arch.persistence.room.ForeignKey.CASCADE;
@Entity
(foreignKeys = @ForeignKey(entity = Feed.class, parentColumns = "id", childColumns = "feed_id", onDelete = CASCADE))
public class Item {
public class Item implements Comparable<Item> {
@PrimaryKey(autoGenerate = true)
private int id;
@ -269,4 +271,9 @@ public class Item {
return dbItems;
}
@Override
public int compareTo(Item o) {
return this.pubDate.compareTo(o.getPubDate());
}
}

5
app/src/main/java/com/readrops/app/repositories/ARepository.java
View File

@ -16,6 +16,7 @@ import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import io.reactivex.Completable;
import io.reactivex.Single;
public abstract class ARepository {
@ -33,11 +34,11 @@ public abstract class ARepository {
this.callback = callback;
}
public abstract Completable sync();
public abstract Completable sync(List<Feed> feeds);
public abstract void addFeed(ParsingResult result);
public abstract Completable addFeeds(List<ParsingResult> results);
public abstract Single<List<Feed>> addFeeds(List<ParsingResult> results);
public abstract void updateFeed(Feed feed);

106
app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java
View File

@ -17,7 +17,8 @@ import com.readrops.app.utils.ParsingResult;
import com.readrops.readropslibrary.QueryCallback;
import com.readrops.readropslibrary.Utils.LibUtils;
import com.readrops.readropslibrary.localfeed.AFeed;
import com.readrops.readropslibrary.localfeed.RSSNetwork;
import com.readrops.readropslibrary.localfeed.RSSQuery;
import com.readrops.readropslibrary.localfeed.RSSQueryResult;
import com.readrops.readropslibrary.localfeed.atom.ATOMFeed;
import com.readrops.readropslibrary.localfeed.json.JSONFeed;
import com.readrops.readropslibrary.localfeed.rss.RSSFeed;
@ -26,12 +27,16 @@ import org.joda.time.LocalDateTime;
import org.jsoup.Jsoup;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.TreeMap;
import io.reactivex.Completable;
import io.reactivex.Single;
public class LocalFeedRepository extends ARepository implements QueryCallback {
@ -50,11 +55,17 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
}
@Override
public Completable sync() {
public Completable sync(List<Feed> feeds) {
return Completable.create(emitter -> {
RSSNetwork rssNet = new RSSNetwork();
rssNet.setCallback(this);
List<Feed> feedList = database.feedDao().getAllFeeds();
List<Feed> feedList;
if (feeds == null || feeds.size() == 0)
feedList = database.feedDao().getAllFeeds();
else
feedList = new ArrayList<>(feeds);
RSSQuery rssQuery = new RSSQuery();
rssQuery.setCallback(this);
for (Feed feed : feedList) {
try {
@ -64,8 +75,11 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
if (feed.getLastModified() != null)
headers.put(LibUtils.IF_MODIFIED_HEADER, feed.getLastModified());
rssNet.requestUrl(feed.getUrl(), headers);
} catch (Exception e) {
RSSQueryResult queryResult = rssQuery.queryUrl(feed.getUrl(), headers);
if (queryResult != null && queryResult.getException() == null)
insertNewItems(queryResult.getFeed(), queryResult.getRssType());
} catch (ParseException e) {
emitter.onError(e);
}
}
@ -78,9 +92,13 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
public void addFeed(ParsingResult result) {
executor.execute(() -> {
try {
RSSNetwork rssNet = new RSSNetwork();
RSSQuery rssNet = new RSSQuery();
rssNet.setCallback(this);
rssNet.requestUrl(result.getUrl(), new HashMap<>());
RSSQueryResult queryResult = rssNet.queryUrl(result.getUrl(), new HashMap<>());
if (queryResult != null && queryResult.getException() == null) {
insertFeed(queryResult.getFeed(), queryResult.getRssType());
}
postCallBackSuccess();
} catch (Exception e) {
@ -90,15 +108,21 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
}
@Override
public Completable addFeeds(List<ParsingResult> results) {
return Completable.create(emitter -> {
for (ParsingResult result : results) {
RSSNetwork rssNet = new RSSNetwork();
rssNet.setCallback(this);
rssNet.requestUrl(result.getUrl(), new HashMap<>());
}
public Single<List<Feed>> addFeeds(List<ParsingResult> results) {
return Single.create(emitter -> {
List<Feed> insertedFeeds = new ArrayList<>();
emitter.onComplete();
for (ParsingResult result : results) {
RSSQuery rssNet = new RSSQuery();
rssNet.setCallback(this);
RSSQueryResult queryResult = rssNet.queryUrl(result.getUrl(), new HashMap<>());
if (queryResult != null && queryResult.getException() == null) {
insertedFeeds.add(insertFeed(queryResult.getFeed(), queryResult.getRssType()));
}
}
emitter.onSuccess(insertedFeeds);
});
}
@ -138,7 +162,7 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
}
@Override
public void onSyncSuccess(AFeed feed, RSSNetwork.RSSType type) {
public void onSyncSuccess(AFeed feed, RSSQuery.RSSType type) {
switch (type) {
case RSS_2:
parseRSSItems(((RSSFeed) feed));
@ -157,6 +181,52 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
failureCallBackInMainThread(e);
}
private void insertNewItems(AFeed feed, RSSQuery.RSSType type) throws ParseException {
Feed dbFeed = null;
List<Item> items = null;
switch (type) {
case RSS_2:
dbFeed = database.feedDao().getFeedByUrl(((RSSFeed) feed).getChannel().getFeedUrl());
items = Item.itemsFromRSS(((RSSFeed) feed).getChannel().getItems(), dbFeed);
break;
case RSS_ATOM:
dbFeed = database.feedDao().getFeedByUrl(((ATOMFeed) feed).getUrl());
items = Item.itemsFromATOM(((ATOMFeed) feed).getEntries(), dbFeed);
break;
case RSS_JSON:
dbFeed = database.feedDao().getFeedByUrl(((JSONFeed) feed).getFeedUrl());
items = Item.itemsFromJSON(((JSONFeed) feed).getItems(), dbFeed);
break;
}
database.feedDao().updateHeaders(dbFeed.getEtag(), dbFeed.getLastModified(), dbFeed.getId());
Collections.sort(items, Item::compareTo);
insertItems(items, dbFeed);
}
private Feed insertFeed(AFeed feed, RSSQuery.RSSType type) throws IOException {
Feed dbFeed = null;
switch (type) {
case RSS_2:
dbFeed = Feed.feedFromRSS((RSSFeed) feed);
break;
case RSS_ATOM:
dbFeed = Feed.feedFromATOM((ATOMFeed) feed);
break;
case RSS_JSON:
dbFeed = Feed.feedFromJSON((JSONFeed) feed);
break;
}
setFavIconUtils(dbFeed);
dbFeed.setId((int)(database.feedDao().insert(dbFeed)));
return dbFeed;
}
private void parseRSSItems(RSSFeed rssFeed) {
try {
Feed dbFeed = database.feedDao().getFeedByUrl(rssFeed.getChannel().getFeedUrl());

9
app/src/main/java/com/readrops/app/viewmodels/AddFeedsViewModel.java
View File

@ -4,17 +4,16 @@ import android.app.Application;
import android.arch.lifecycle.AndroidViewModel;
import android.support.annotation.NonNull;
import com.readrops.app.database.entities.Feed;
import com.readrops.app.repositories.LocalFeedRepository;
import com.readrops.app.utils.HtmlParser;
import com.readrops.app.utils.ParsingResult;
import com.readrops.readropslibrary.localfeed.RSSNetwork;
import com.readrops.readropslibrary.localfeed.RSSQuery;
import java.util.ArrayList;
import java.util.List;
import io.reactivex.Completable;
import io.reactivex.Single;
import io.reactivex.SingleOnSubscribe;
public class AddFeedsViewModel extends AndroidViewModel {
@ -26,13 +25,13 @@ public class AddFeedsViewModel extends AndroidViewModel {
repository = new LocalFeedRepository(application);
}
public Completable addFeeds(List<ParsingResult> results) {
public Single<List<Feed>> addFeeds(List<ParsingResult> results) {
return repository.addFeeds(results);
}
public Single<List<ParsingResult>> parseUrl(String url) {
return Single.create(emitter -> {
RSSNetwork rssApi = new RSSNetwork();
RSSQuery rssApi = new RSSQuery();
List<ParsingResult> results = new ArrayList<>();
if (rssApi.isUrlFeedLink(url)) {

3
app/src/main/java/com/readrops/app/viewmodels/MainViewModel.java
View File

@ -5,6 +5,7 @@ import android.arch.lifecycle.AndroidViewModel;
import android.arch.lifecycle.LiveData;
import android.support.annotation.NonNull;
import com.readrops.app.database.entities.Feed;
import com.readrops.app.database.pojo.ItemWithFeed;
import com.readrops.app.repositories.LocalFeedRepository;
import com.readrops.app.views.SimpleCallback;
@ -36,7 +37,7 @@ public class MainViewModel extends AndroidViewModel {
}
public Completable sync() {
return repository.sync();
return repository.sync(null);
}
public void addFeed(ParsingResult parsingResult) {

4
app/src/main/java/com/readrops/app/views/AddFeedDialog.java
View File

@ -23,7 +23,7 @@ import com.readrops.app.activities.MainActivity;
import com.readrops.app.utils.Utils;
import com.readrops.app.utils.HtmlParser;
import com.readrops.app.utils.ParsingResult;
import com.readrops.readropslibrary.localfeed.RSSNetwork;
import com.readrops.readropslibrary.localfeed.RSSQuery;
import java.net.UnknownHostException;
import java.util.List;
@ -99,7 +99,7 @@ public class AddFeedDialog extends Dialog implements View.OnClickListener {
Executors.newSingleThreadExecutor().execute(() -> {
try {
RSSNetwork rssApi = new RSSNetwork();
RSSQuery rssApi = new RSSQuery();
if (rssApi.isUrlFeedLink(finalUrl)) {
ParsingResult parsingResult = new ParsingResult(finalUrl, null);
handler.post(() -> ((MainActivity) getOwnerActivity()).insertNewFeed(parsingResult));

4
readropslibrary/src/main/java/com/readrops/readropslibrary/QueryCallback.java
View File

@ -1,11 +1,11 @@
package com.readrops.readropslibrary;
import com.readrops.readropslibrary.localfeed.AFeed;
import com.readrops.readropslibrary.localfeed.RSSNetwork;
import com.readrops.readropslibrary.localfeed.RSSQuery;
public interface QueryCallback {
void onSyncSuccess(AFeed feed, RSSNetwork.RSSType type);
void onSyncSuccess(AFeed feed, RSSQuery.RSSType type);
void onSyncFailure(Exception e);

59
readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/RSSNetwork.java → readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/RSSQuery.java
View File

@ -24,9 +24,9 @@ import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
public class RSSNetwork {
public class RSSQuery {
private static final String TAG = RSSNetwork.class.getSimpleName();
private static final String TAG = RSSQuery.class.getSimpleName();
private static final String RSS_CONTENT_TYPE_REGEX = "([^;]+)";
@ -40,26 +40,24 @@ public class RSSNetwork {
* @param url url to request
* @throws Exception
*/
public void requestUrl(String url, Map<String, String> headers) throws Exception {
if (callback == null)
throw new NullPointerException("Callback can't be null");
public RSSQueryResult queryUrl(String url, Map<String, String> headers) throws Exception {
Response response = query(url, headers);
RSSQueryResult queryResult;
if (response.isSuccessful()) {
String header = response.header(LibUtils.CONTENT_TYPE_HEADER);
RSSType type = getRSSType(header);
if (type == null) {
callback.onSyncFailure(new IllegalArgumentException("bad content type : " + header + "for " + url));
return;
queryResult = new RSSQueryResult(new IllegalArgumentException("bad content type : " + header + "for " + url));
return queryResult;
}
parseFeed(response.body().byteStream(), type, response);
return parseFeed(response.body().byteStream(), type, response);
} else if (response.code() == 304)
return;
return null;
else
callback.onSyncFailure(new Exception("Error " + response.code() + " when requesting url " + url));
return new RSSQueryResult(new Exception("Error " + response.code() + " when requesting url " + url));
}
public boolean isUrlFeedLink(String url) throws IOException {
@ -128,7 +126,7 @@ public class RSSNetwork {
* @param response query response
* @throws Exception
*/
private void parseFeed(InputStream stream, RSSType type, Response response) throws Exception {
private RSSQueryResult parseFeed(InputStream stream, RSSType type, Response response) throws Exception {
String xml = LibUtils.inputStreamToString(stream);
Serializer serializer = new Persister();
@ -136,43 +134,50 @@ public class RSSNetwork {
RSSType contentType = getContentRSSType(xml);
if (contentType == RSSType.RSS_UNKNOWN) {
callback.onSyncFailure(new Exception("Unknown content format"));
return;
return null;
} else
type = contentType;
}
String etag = response.header(LibUtils.ETAG_HEADER);
String lastModified = response.header(LibUtils.LAST_MODIFIED_HEADER);
AFeed feed;
RSSQueryResult queryResult = new RSSQueryResult();
switch (type) {
case RSS_2:
RSSFeed rssFeed = serializer.read(RSSFeed.class, xml);
if (rssFeed.getChannel().getFeedUrl() == null) // workaround si the channel does not have any atom:link tag
rssFeed.getChannel().getLinks().add(new RSSLink(null, response.request().url().toString()));
rssFeed.setEtag(etag);
rssFeed.setLastModified(lastModified);
feed = serializer.read(RSSFeed.class, xml);
if (((RSSFeed)feed).getChannel().getFeedUrl() == null) // workaround si the channel does not have any atom:link tag
((RSSFeed)feed).getChannel().getLinks().add(new RSSLink(null, response.request().url().toString()));
feed.setEtag(etag);
feed.setLastModified(lastModified);
callback.onSyncSuccess(rssFeed, type);
queryResult.setFeed(feed);
queryResult.setRssType(type);
break;
case RSS_ATOM:
ATOMFeed atomFeed = serializer.read(ATOMFeed.class, xml);
atomFeed.setWebsiteUrl(response.request().url().scheme() + "://" + response.request().url().host());
atomFeed.setUrl(response.request().url().toString());
atomFeed.setEtag(etag);
atomFeed.setLastModified(etag);
feed = serializer.read(ATOMFeed.class, xml);
((ATOMFeed)feed).setWebsiteUrl(response.request().url().scheme() + "://" + response.request().url().host());
((ATOMFeed)feed).setUrl(response.request().url().toString());
feed.setEtag(etag);
feed.setLastModified(etag);
callback.onSyncSuccess(atomFeed, type);
queryResult.setFeed(feed);
queryResult.setRssType(type);
break;
case RSS_JSON:
Gson gson = new Gson();
JSONFeed feed = gson.fromJson(xml, JSONFeed.class);
feed = gson.fromJson(xml, JSONFeed.class);
feed.setEtag(etag);
feed.setLastModified(lastModified);
callback.onSyncSuccess(feed, type);
queryResult.setFeed(feed);
queryResult.setRssType(type);
break;
}
return queryResult;
}
private RSSType getContentRSSType(String content) {

42
readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/RSSQueryResult.java Normal file
View File

@ -0,0 +1,42 @@
package com.readrops.readropslibrary.localfeed;
public class RSSQueryResult {
private AFeed feed;
private RSSQuery.RSSType rssType;
private Exception exception;
public RSSQueryResult(Exception exception) {
this.exception = exception;
}
public RSSQueryResult() {
}
public AFeed getFeed() {
return feed;
}
public void setFeed(AFeed feed) {
this.feed = feed;
}
public RSSQuery.RSSType getRssType() {
return rssType;
}
public void setRssType(RSSQuery.RSSType rssType) {
this.rssType = rssType;
}
public void setException(Exception exception) {
this.exception = exception;
}
public Exception getException() {
return exception;
}
}