FreshRSS
/
Readrops
mirror of https://github.com/readrops/Readrops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implementing Optional GET with If-None-Match and If-Modified-Since headers

This commit is contained in:
Shinokuni 2019-02-06 21:01:51 +00:00
parent f384d5b9c8
commit 098ae50044
7 changed files with 145 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
app/src/main/java/com/readrops/app/LocalFeedRepository.java
View File

@ -15,6 +15,7 @@ import com.readrops.app.database.entities.Item;
import com.readrops.readropslibrary.HtmlParser;
import com.readrops.readropslibrary.ParsingResult;
import com.readrops.readropslibrary.QueryCallback;
import com.readrops.readropslibrary.Utils.LibUtils;
import com.readrops.readropslibrary.localfeed.AFeed;
import com.readrops.readropslibrary.localfeed.RSSNetwork;
import com.readrops.readropslibrary.localfeed.atom.ATOMFeed;
@ -25,6 +26,7 @@ import org.jsoup.Jsoup;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.List;
import okhttp3.OkHttpClient;
@ -51,11 +53,18 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
public void sync() {
executor.execute(() -> {
RSSNetwork rssNet = new RSSNetwork();
rssNet.setCallback(this);
List<Feed> feedList = database.feedDao().getAllFeeds();
for (Feed feed : feedList) {
try {
rssNet.request(feed.getUrl(), this);
HashMap<String, String> headers = new HashMap<>();
if (feed.getEtag() != null)
headers.put(LibUtils.IF_NONE_MATCH_HEADER, feed.getEtag());
if (feed.getLastModified() != null)
headers.put(LibUtils.IF_MODIFIED_HEADER, feed.getLastModified());
rssNet.request(feed.getUrl(), headers);
} catch (Exception e) {
failureCallBackInMainThread(e);
}
@ -70,7 +79,8 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
executor.execute(() -> {
try {
RSSNetwork rssNet = new RSSNetwork();
rssNet.request(result.getUrl(), this);
rssNet.setCallback(this);
rssNet.request(result.getUrl(), new HashMap<>());
postCallBackSuccess();
} catch (Exception e) {
@ -116,11 +126,12 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
try {
Feed dbFeed = database.feedDao().getFeedByUrl(rssFeed.getChannel().getFeedUrl());
if (dbFeed == null) {
dbFeed = Feed.feedFromRSS(rssFeed.getChannel());
dbFeed = Feed.feedFromRSS(rssFeed);
setFavIconUtils(dbFeed);
dbFeed.setId((int)(database.feedDao().insert(dbFeed)));
}
} else
database.feedDao().updateHeaders(rssFeed.getEtag(), rssFeed.getLastModified(), dbFeed.getId());
List<Item> dbItems = Item.itemsFromRSS(rssFeed.getChannel().getItems(), dbFeed);
insertItems(dbItems, dbFeed);
@ -139,7 +150,8 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
setFavIconUtils(dbFeed);
dbFeed.setId((int)(database.feedDao().insert(dbFeed)));
}
} else
database.feedDao().updateHeaders(feed.getEtag(), feed.getLastModified(), dbFeed.getId());
List<Item> dbItems = Item.itemsFromATOM(feed.getEntries(), dbFeed);
insertItems(dbItems, dbFeed);
@ -157,7 +169,8 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
setFavIconUtils(dbFeed);
dbFeed.setId((int)(database.feedDao().insert(dbFeed)));
}
} else
database.feedDao().updateHeaders(feed.getEtag(), feed.getLastModified(), dbFeed.getId());
List<Item> dbItems = Item.itemsFromJSON(feed.getItems(), dbFeed);
insertItems(dbItems, dbFeed);

4
app/src/main/java/com/readrops/app/database/dao/FeedDao.java
View File

@ -4,6 +4,7 @@ package com.readrops.app.database.dao;
import android.arch.persistence.room.Dao;
import android.arch.persistence.room.Insert;
import android.arch.persistence.room.Query;
import android.arch.persistence.room.Update;
import com.readrops.app.database.entities.Feed;
@ -27,4 +28,7 @@ public interface FeedDao {
@Query("Select id from Feed Where url = :feedUrl")
int getFeedIdByUrl(String feedUrl);
@Query("Update Feed set etag = :etag, last_modified = :lastModified Where id = :feedId")
void updateHeaders(String etag, String lastModified, int feedId);
}

36
app/src/main/java/com/readrops/app/database/entities/Feed.java
View File

@ -7,6 +7,9 @@ import android.support.annotation.ColorInt;
import com.readrops.readropslibrary.localfeed.atom.ATOMFeed;
import com.readrops.readropslibrary.localfeed.json.JSONFeed;
import com.readrops.readropslibrary.localfeed.rss.RSSChannel;
import com.readrops.readropslibrary.localfeed.rss.RSSFeed;
import java.nio.channels.Channel;
@Entity
public class Feed {
@ -29,6 +32,11 @@ public class Feed {
@ColumnInfo(name = "icon_url")
private String iconUrl;
private String etag;
@ColumnInfo(name = "last_modified")
private String lastModified;
public Feed() {
}
@ -104,8 +112,25 @@ public class Feed {
this.iconUrl = iconUrl;
}
public static Feed feedFromRSS(RSSChannel channel) {
public String getEtag() {
return etag;
}
public void setEtag(String etag) {
this.etag = etag;
}
public String getLastModified() {
return lastModified;
}
public void setLastModified(String lastModified) {
this.lastModified = lastModified;
}
public static Feed feedFromRSS(RSSFeed rssFeed) {
Feed feed = new Feed();
RSSChannel channel = rssFeed.getChannel();
feed.setName(channel.getTitle());
feed.setUrl(channel.getFeedUrl());
@ -113,6 +138,9 @@ public class Feed {
feed.setDescription(channel.getDescription());
feed.setLastUpdated(channel.getLastUpdated());
feed.setEtag(rssFeed.getEtag());
feed.setLastModified(rssFeed.getLastModified());
return feed;
}
@ -126,6 +154,9 @@ public class Feed {
feed.setDescription(atomFeed.getSubtitle());
feed.setLastUpdated(atomFeed.getUpdated());
feed.setEtag(atomFeed.getEtag());
feed.setLastModified(atomFeed.getLastModified());
return feed;
}
@ -137,6 +168,9 @@ public class Feed {
feed.setDescription(jsonFeed.getDescription());
//feed.setLastUpdated(jsonFeed.); maybe later ?
feed.setEtag(jsonFeed.getEtag());
feed.setLastModified(jsonFeed.getLastModified());
return feed;
}
}

12
readropslibrary/src/main/java/com/readrops/readropslibrary/HtmlParser.java
View File

@ -2,7 +2,7 @@ package com.readrops.readropslibrary;
import android.util.Log;
import com.readrops.readropslibrary.Utils.Utils;
import com.readrops.readropslibrary.Utils.LibUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
@ -45,11 +45,11 @@ public final class HtmlParser {
}
private static boolean isTypeRssFeed(String type) {
return type.equals(Utils.RSS_DEFAULT_CONTENT_TYPE) ||
type.equals(Utils.ATOM_CONTENT_TYPE) ||
type.equals(Utils.JSON_CONTENT_TYPE) ||
type.equals(Utils.RSS_TEXT_CONTENT_TYPE) ||
type.equals(Utils.RSS_APPLICATION_CONTENT_TYPE);
return type.equals(LibUtils.RSS_DEFAULT_CONTENT_TYPE) ||
type.equals(LibUtils.ATOM_CONTENT_TYPE) ||
type.equals(LibUtils.JSON_CONTENT_TYPE) ||
type.equals(LibUtils.RSS_TEXT_CONTENT_TYPE) ||
type.equals(LibUtils.RSS_APPLICATION_CONTENT_TYPE);
}
/**

8
readropslibrary/src/main/java/com/readrops/readropslibrary/Utils/Utils.java → readropslibrary/src/main/java/com/readrops/readropslibrary/Utils/LibUtils.java
View File

@ -3,7 +3,7 @@ package com.readrops.readropslibrary.Utils;
import java.io.InputStream;
import java.util.Scanner;
public final class Utils {
public final class LibUtils {
public static final String RSS_DEFAULT_CONTENT_TYPE = "application/rss+xml";
public static final String RSS_TEXT_CONTENT_TYPE = "text/xml";
@ -12,6 +12,12 @@ public final class Utils {
public static final String JSON_CONTENT_TYPE = "application/json";
public static final String HTML_CONTENT_TYPE = "text/html";
public static final String CONTENT_TYPE_HEADER = "content-type";
public static final String ETAG_HEADER = "ETag";
public static final String IF_NONE_MATCH_HEADER = "If-None-Match";
public static final String LAST_MODIFIED_HEADER = "Last-Modified";
public static final String IF_MODIFIED_HEADER = "If-Modified-Since";
public static String inputStreamToString(InputStream input) {
Scanner scanner = new Scanner(input).useDelimiter("\\A");
return scanner.hasNext() ? scanner.next() : "";

20
readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/AFeed.java
View File

@ -4,4 +4,24 @@ package com.readrops.readropslibrary.localfeed;
A simple class to give an abstract level to rss/atom/json feed classes
*/
public abstract class AFeed {
protected String etag;
protected String lastModified;
public String getEtag() {
return etag;
}
public void setEtag(String etag) {
this.etag = etag;
}
public String getLastModified() {
return lastModified;
}
public void setLastModified(String lastModified) {
this.lastModified = lastModified;
}
}

81
readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/RSSNetwork.java
View File

@ -4,7 +4,7 @@ import android.util.Log;
import com.google.gson.Gson;
import com.readrops.readropslibrary.QueryCallback;
import com.readrops.readropslibrary.Utils.Utils;
import com.readrops.readropslibrary.Utils.LibUtils;
import com.readrops.readropslibrary.localfeed.atom.ATOMFeed;
import com.readrops.readropslibrary.localfeed.json.JSONFeed;
import com.readrops.readropslibrary.localfeed.rss.RSSFeed;
@ -14,6 +14,7 @@ import org.simpleframework.xml.Serializer;
import org.simpleframework.xml.core.Persister;
import java.io.InputStream;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -29,37 +30,48 @@ public class RSSNetwork {
private static final String RSS_2_REGEX = "rss.*version=\"2.0\"";
private QueryCallback callback;
/**
* Request the url given in parameter.
* This method is synchronous, <b>it has to be called from another thread than the main one</b>.
* @param url url to request
* @param callback result callback if success or error
* @throws Exception
*/
public void request(String url, final QueryCallback callback) throws Exception {
public void request(String url, Map<String, String> headers) throws Exception {
if (callback == null)
throw new NullPointerException("Callback can't be null");
OkHttpClient okHttpClient = new OkHttpClient();
Request request = new Request.Builder().url(url).build();
Request.Builder builder = new Request.Builder().url(url);
for (String header : headers.keySet()) {
String value = headers.get(header);
builder.addHeader(header, value);
}
Request request = builder.build();
Response response = okHttpClient.newCall(request).execute();
Pattern pattern = Pattern.compile(RSS_CONTENT_TYPE_REGEX);
Matcher matcher = pattern.matcher(response.header("content-type"));
String header;
if (matcher.find())
header = matcher.group(0);
else
header = response.header("content-type");
if (response.isSuccessful()) {
Pattern pattern = Pattern.compile(RSS_CONTENT_TYPE_REGEX);
Matcher matcher = pattern.matcher(response.header(LibUtils.CONTENT_TYPE_HEADER));
String header;
if (matcher.find())
header = matcher.group(0);
else
header = response.header(LibUtils.CONTENT_TYPE_HEADER);
RSSType type = getRSSType(header);
if (type == null) {
callback.onSyncFailure(new IllegalArgumentException("bad content type : " + header + "for " + url));
return;
}
parseFeed(response.body().byteStream(), type, callback, url);
} else
parseFeed(response.body().byteStream(), type, response);
} else if (response.code() == 304)
return;
else
callback.onSyncFailure(new Exception("Error " + response.code() + " when requesting url " + url));
}
@ -67,12 +79,11 @@ public class RSSNetwork {
* Parse input feed
* @param stream inputStream to parse
* @param type rss type, important to know the format
* @param callback success callback
* @param url feed url
* @param response query response
* @throws Exception
*/
private void parseFeed(InputStream stream, RSSType type, QueryCallback callback, String url) throws Exception {
String xml = Utils.inputStreamToString(stream);
private void parseFeed(InputStream stream, RSSType type, Response response) throws Exception {
String xml = LibUtils.inputStreamToString(stream);
Serializer serializer = new Persister();
if (type == RSSType.RSS_UNKNOWN) {
@ -86,21 +97,33 @@ public class RSSNetwork {
}
}
String etag = response.header(LibUtils.ETAG_HEADER);
String lastModified = response.header(LibUtils.LAST_MODIFIED_HEADER);
switch (type) {
case RSS_2:
RSSFeed rssFeed = serializer.read(RSSFeed.class, xml);
if (rssFeed.getChannel().getFeedUrl() == null) // workaround si the channel does not have any atom:link tag
rssFeed.getChannel().getLinks().add(new RSSLink(null, url));
rssFeed.getChannel().getLinks().add(new RSSLink(null, response.request().url().toString()));
rssFeed.setEtag(etag);
rssFeed.setLastModified(lastModified);
callback.onSyncSuccess(rssFeed, type);
break;
case RSS_ATOM:
ATOMFeed atomFeed = serializer.read(ATOMFeed.class, xml);
atomFeed.setEtag(etag);
atomFeed.setLastModified(etag);
callback.onSyncSuccess(atomFeed, type);
break;
case RSS_JSON:
Gson gson = new Gson();
JSONFeed feed = gson.fromJson(xml, JSONFeed.class);
feed.setEtag(etag);
feed.setLastModified(lastModified);
callback.onSyncSuccess(feed, type);
break;
}
@ -113,17 +136,17 @@ public class RSSNetwork {
*/
private RSSType getRSSType(String contentType) {
switch (contentType) {
case Utils.RSS_DEFAULT_CONTENT_TYPE:
case LibUtils.RSS_DEFAULT_CONTENT_TYPE:
return RSSType.RSS_2;
case Utils.RSS_TEXT_CONTENT_TYPE:
case LibUtils.RSS_TEXT_CONTENT_TYPE:
return RSSType.RSS_UNKNOWN;
case Utils.RSS_APPLICATION_CONTENT_TYPE:
case LibUtils.RSS_APPLICATION_CONTENT_TYPE:
return RSSType.RSS_UNKNOWN;
case Utils.ATOM_CONTENT_TYPE:
case LibUtils.ATOM_CONTENT_TYPE:
return RSSType.RSS_ATOM;
case Utils.JSON_CONTENT_TYPE:
case LibUtils.JSON_CONTENT_TYPE:
return RSSType.RSS_JSON;
case Utils.HTML_CONTENT_TYPE:
case LibUtils.HTML_CONTENT_TYPE:
return RSSType.RSS_UNKNOWN;
default:
Log.d(TAG, "bad content type : " + contentType);
@ -132,6 +155,10 @@ public class RSSNetwork {
}
}
public void setCallback(QueryCallback callback) {
this.callback = callback;
}
public enum RSSType {
RSS_2("rss"),
RSS_ATOM("atom"),