diff --git a/app/src/main/java/com/readrops/app/LocalFeedRepository.java b/app/src/main/java/com/readrops/app/LocalFeedRepository.java index 51c1f8de..f136d12e 100644 --- a/app/src/main/java/com/readrops/app/LocalFeedRepository.java +++ b/app/src/main/java/com/readrops/app/LocalFeedRepository.java @@ -15,6 +15,7 @@ import com.readrops.app.database.entities.Item; import com.readrops.readropslibrary.HtmlParser; import com.readrops.readropslibrary.ParsingResult; import com.readrops.readropslibrary.QueryCallback; +import com.readrops.readropslibrary.Utils.LibUtils; import com.readrops.readropslibrary.localfeed.AFeed; import com.readrops.readropslibrary.localfeed.RSSNetwork; import com.readrops.readropslibrary.localfeed.atom.ATOMFeed; @@ -25,6 +26,7 @@ import org.jsoup.Jsoup; import java.io.IOException; import java.io.InputStream; +import java.util.HashMap; import java.util.List; import okhttp3.OkHttpClient; @@ -51,11 +53,18 @@ public class LocalFeedRepository extends ARepository implements QueryCallback { public void sync() { executor.execute(() -> { RSSNetwork rssNet = new RSSNetwork(); + rssNet.setCallback(this); List feedList = database.feedDao().getAllFeeds(); for (Feed feed : feedList) { try { - rssNet.request(feed.getUrl(), this); + HashMap headers = new HashMap<>(); + if (feed.getEtag() != null) + headers.put(LibUtils.IF_NONE_MATCH_HEADER, feed.getEtag()); + if (feed.getLastModified() != null) + headers.put(LibUtils.IF_MODIFIED_HEADER, feed.getLastModified()); + + rssNet.request(feed.getUrl(), headers); } catch (Exception e) { failureCallBackInMainThread(e); } @@ -70,7 +79,8 @@ public class LocalFeedRepository extends ARepository implements QueryCallback { executor.execute(() -> { try { RSSNetwork rssNet = new RSSNetwork(); - rssNet.request(result.getUrl(), this); + rssNet.setCallback(this); + rssNet.request(result.getUrl(), new HashMap<>()); postCallBackSuccess(); } catch (Exception e) { @@ -116,11 +126,12 @@ public class LocalFeedRepository extends ARepository implements QueryCallback { try { Feed dbFeed = database.feedDao().getFeedByUrl(rssFeed.getChannel().getFeedUrl()); if (dbFeed == null) { - dbFeed = Feed.feedFromRSS(rssFeed.getChannel()); + dbFeed = Feed.feedFromRSS(rssFeed); setFavIconUtils(dbFeed); dbFeed.setId((int)(database.feedDao().insert(dbFeed))); - } + } else + database.feedDao().updateHeaders(rssFeed.getEtag(), rssFeed.getLastModified(), dbFeed.getId()); List dbItems = Item.itemsFromRSS(rssFeed.getChannel().getItems(), dbFeed); insertItems(dbItems, dbFeed); @@ -139,7 +150,8 @@ public class LocalFeedRepository extends ARepository implements QueryCallback { setFavIconUtils(dbFeed); dbFeed.setId((int)(database.feedDao().insert(dbFeed))); - } + } else + database.feedDao().updateHeaders(feed.getEtag(), feed.getLastModified(), dbFeed.getId()); List dbItems = Item.itemsFromATOM(feed.getEntries(), dbFeed); insertItems(dbItems, dbFeed); @@ -157,7 +169,8 @@ public class LocalFeedRepository extends ARepository implements QueryCallback { setFavIconUtils(dbFeed); dbFeed.setId((int)(database.feedDao().insert(dbFeed))); - } + } else + database.feedDao().updateHeaders(feed.getEtag(), feed.getLastModified(), dbFeed.getId()); List dbItems = Item.itemsFromJSON(feed.getItems(), dbFeed); insertItems(dbItems, dbFeed); diff --git a/app/src/main/java/com/readrops/app/database/dao/FeedDao.java b/app/src/main/java/com/readrops/app/database/dao/FeedDao.java index 727b16e1..ad8fff93 100644 --- a/app/src/main/java/com/readrops/app/database/dao/FeedDao.java +++ b/app/src/main/java/com/readrops/app/database/dao/FeedDao.java @@ -4,6 +4,7 @@ package com.readrops.app.database.dao; import android.arch.persistence.room.Dao; import android.arch.persistence.room.Insert; import android.arch.persistence.room.Query; +import android.arch.persistence.room.Update; import com.readrops.app.database.entities.Feed; @@ -27,4 +28,7 @@ public interface FeedDao { @Query("Select id from Feed Where url = :feedUrl") int getFeedIdByUrl(String feedUrl); + @Query("Update Feed set etag = :etag, last_modified = :lastModified Where id = :feedId") + void updateHeaders(String etag, String lastModified, int feedId); + } diff --git a/app/src/main/java/com/readrops/app/database/entities/Feed.java b/app/src/main/java/com/readrops/app/database/entities/Feed.java index f52015c8..7026a942 100644 --- a/app/src/main/java/com/readrops/app/database/entities/Feed.java +++ b/app/src/main/java/com/readrops/app/database/entities/Feed.java @@ -7,6 +7,9 @@ import android.support.annotation.ColorInt; import com.readrops.readropslibrary.localfeed.atom.ATOMFeed; import com.readrops.readropslibrary.localfeed.json.JSONFeed; import com.readrops.readropslibrary.localfeed.rss.RSSChannel; +import com.readrops.readropslibrary.localfeed.rss.RSSFeed; + +import java.nio.channels.Channel; @Entity public class Feed { @@ -29,6 +32,11 @@ public class Feed { @ColumnInfo(name = "icon_url") private String iconUrl; + private String etag; + + @ColumnInfo(name = "last_modified") + private String lastModified; + public Feed() { } @@ -104,8 +112,25 @@ public class Feed { this.iconUrl = iconUrl; } - public static Feed feedFromRSS(RSSChannel channel) { + public String getEtag() { + return etag; + } + + public void setEtag(String etag) { + this.etag = etag; + } + + public String getLastModified() { + return lastModified; + } + + public void setLastModified(String lastModified) { + this.lastModified = lastModified; + } + + public static Feed feedFromRSS(RSSFeed rssFeed) { Feed feed = new Feed(); + RSSChannel channel = rssFeed.getChannel(); feed.setName(channel.getTitle()); feed.setUrl(channel.getFeedUrl()); @@ -113,6 +138,9 @@ public class Feed { feed.setDescription(channel.getDescription()); feed.setLastUpdated(channel.getLastUpdated()); + feed.setEtag(rssFeed.getEtag()); + feed.setLastModified(rssFeed.getLastModified()); + return feed; } @@ -126,6 +154,9 @@ public class Feed { feed.setDescription(atomFeed.getSubtitle()); feed.setLastUpdated(atomFeed.getUpdated()); + feed.setEtag(atomFeed.getEtag()); + feed.setLastModified(atomFeed.getLastModified()); + return feed; } @@ -137,6 +168,9 @@ public class Feed { feed.setDescription(jsonFeed.getDescription()); //feed.setLastUpdated(jsonFeed.); maybe later ? + feed.setEtag(jsonFeed.getEtag()); + feed.setLastModified(jsonFeed.getLastModified()); + return feed; } } diff --git a/readropslibrary/src/main/java/com/readrops/readropslibrary/HtmlParser.java b/readropslibrary/src/main/java/com/readrops/readropslibrary/HtmlParser.java index fc87447b..0ee6afc1 100644 --- a/readropslibrary/src/main/java/com/readrops/readropslibrary/HtmlParser.java +++ b/readropslibrary/src/main/java/com/readrops/readropslibrary/HtmlParser.java @@ -2,7 +2,7 @@ package com.readrops.readropslibrary; import android.util.Log; -import com.readrops.readropslibrary.Utils.Utils; +import com.readrops.readropslibrary.Utils.LibUtils; import org.jsoup.Connection; import org.jsoup.Jsoup; @@ -45,11 +45,11 @@ public final class HtmlParser { } private static boolean isTypeRssFeed(String type) { - return type.equals(Utils.RSS_DEFAULT_CONTENT_TYPE) || - type.equals(Utils.ATOM_CONTENT_TYPE) || - type.equals(Utils.JSON_CONTENT_TYPE) || - type.equals(Utils.RSS_TEXT_CONTENT_TYPE) || - type.equals(Utils.RSS_APPLICATION_CONTENT_TYPE); + return type.equals(LibUtils.RSS_DEFAULT_CONTENT_TYPE) || + type.equals(LibUtils.ATOM_CONTENT_TYPE) || + type.equals(LibUtils.JSON_CONTENT_TYPE) || + type.equals(LibUtils.RSS_TEXT_CONTENT_TYPE) || + type.equals(LibUtils.RSS_APPLICATION_CONTENT_TYPE); } /** diff --git a/readropslibrary/src/main/java/com/readrops/readropslibrary/Utils/Utils.java b/readropslibrary/src/main/java/com/readrops/readropslibrary/Utils/LibUtils.java similarity index 66% rename from readropslibrary/src/main/java/com/readrops/readropslibrary/Utils/Utils.java rename to readropslibrary/src/main/java/com/readrops/readropslibrary/Utils/LibUtils.java index d50c4eed..dcd3d7d0 100644 --- a/readropslibrary/src/main/java/com/readrops/readropslibrary/Utils/Utils.java +++ b/readropslibrary/src/main/java/com/readrops/readropslibrary/Utils/LibUtils.java @@ -3,7 +3,7 @@ package com.readrops.readropslibrary.Utils; import java.io.InputStream; import java.util.Scanner; -public final class Utils { +public final class LibUtils { public static final String RSS_DEFAULT_CONTENT_TYPE = "application/rss+xml"; public static final String RSS_TEXT_CONTENT_TYPE = "text/xml"; @@ -12,6 +12,12 @@ public final class Utils { public static final String JSON_CONTENT_TYPE = "application/json"; public static final String HTML_CONTENT_TYPE = "text/html"; + public static final String CONTENT_TYPE_HEADER = "content-type"; + public static final String ETAG_HEADER = "ETag"; + public static final String IF_NONE_MATCH_HEADER = "If-None-Match"; + public static final String LAST_MODIFIED_HEADER = "Last-Modified"; + public static final String IF_MODIFIED_HEADER = "If-Modified-Since"; + public static String inputStreamToString(InputStream input) { Scanner scanner = new Scanner(input).useDelimiter("\\A"); return scanner.hasNext() ? scanner.next() : ""; diff --git a/readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/AFeed.java b/readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/AFeed.java index ad141438..e84102fa 100644 --- a/readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/AFeed.java +++ b/readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/AFeed.java @@ -4,4 +4,24 @@ package com.readrops.readropslibrary.localfeed; A simple class to give an abstract level to rss/atom/json feed classes */ public abstract class AFeed { + + protected String etag; + + protected String lastModified; + + public String getEtag() { + return etag; + } + + public void setEtag(String etag) { + this.etag = etag; + } + + public String getLastModified() { + return lastModified; + } + + public void setLastModified(String lastModified) { + this.lastModified = lastModified; + } } diff --git a/readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/RSSNetwork.java b/readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/RSSNetwork.java index dfc7ba85..dbdf558e 100644 --- a/readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/RSSNetwork.java +++ b/readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/RSSNetwork.java @@ -4,7 +4,7 @@ import android.util.Log; import com.google.gson.Gson; import com.readrops.readropslibrary.QueryCallback; -import com.readrops.readropslibrary.Utils.Utils; +import com.readrops.readropslibrary.Utils.LibUtils; import com.readrops.readropslibrary.localfeed.atom.ATOMFeed; import com.readrops.readropslibrary.localfeed.json.JSONFeed; import com.readrops.readropslibrary.localfeed.rss.RSSFeed; @@ -14,6 +14,7 @@ import org.simpleframework.xml.Serializer; import org.simpleframework.xml.core.Persister; import java.io.InputStream; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -29,37 +30,48 @@ public class RSSNetwork { private static final String RSS_2_REGEX = "rss.*version=\"2.0\""; + private QueryCallback callback; + /** * Request the url given in parameter. * This method is synchronous, it has to be called from another thread than the main one. * @param url url to request - * @param callback result callback if success or error * @throws Exception */ - public void request(String url, final QueryCallback callback) throws Exception { + public void request(String url, Map headers) throws Exception { + if (callback == null) + throw new NullPointerException("Callback can't be null"); + OkHttpClient okHttpClient = new OkHttpClient(); - Request request = new Request.Builder().url(url).build(); + Request.Builder builder = new Request.Builder().url(url); + for (String header : headers.keySet()) { + String value = headers.get(header); + builder.addHeader(header, value); + } + + Request request = builder.build(); Response response = okHttpClient.newCall(request).execute(); - Pattern pattern = Pattern.compile(RSS_CONTENT_TYPE_REGEX); - Matcher matcher = pattern.matcher(response.header("content-type")); - - String header; - if (matcher.find()) - header = matcher.group(0); - else - header = response.header("content-type"); - if (response.isSuccessful()) { + Pattern pattern = Pattern.compile(RSS_CONTENT_TYPE_REGEX); + Matcher matcher = pattern.matcher(response.header(LibUtils.CONTENT_TYPE_HEADER)); + + String header; + if (matcher.find()) + header = matcher.group(0); + else + header = response.header(LibUtils.CONTENT_TYPE_HEADER); + RSSType type = getRSSType(header); if (type == null) { callback.onSyncFailure(new IllegalArgumentException("bad content type : " + header + "for " + url)); return; } - - parseFeed(response.body().byteStream(), type, callback, url); - } else + parseFeed(response.body().byteStream(), type, response); + } else if (response.code() == 304) + return; + else callback.onSyncFailure(new Exception("Error " + response.code() + " when requesting url " + url)); } @@ -67,12 +79,11 @@ public class RSSNetwork { * Parse input feed * @param stream inputStream to parse * @param type rss type, important to know the format - * @param callback success callback - * @param url feed url + * @param response query response * @throws Exception */ - private void parseFeed(InputStream stream, RSSType type, QueryCallback callback, String url) throws Exception { - String xml = Utils.inputStreamToString(stream); + private void parseFeed(InputStream stream, RSSType type, Response response) throws Exception { + String xml = LibUtils.inputStreamToString(stream); Serializer serializer = new Persister(); if (type == RSSType.RSS_UNKNOWN) { @@ -86,21 +97,33 @@ public class RSSNetwork { } } + String etag = response.header(LibUtils.ETAG_HEADER); + String lastModified = response.header(LibUtils.LAST_MODIFIED_HEADER); + switch (type) { case RSS_2: RSSFeed rssFeed = serializer.read(RSSFeed.class, xml); if (rssFeed.getChannel().getFeedUrl() == null) // workaround si the channel does not have any atom:link tag - rssFeed.getChannel().getLinks().add(new RSSLink(null, url)); + rssFeed.getChannel().getLinks().add(new RSSLink(null, response.request().url().toString())); + rssFeed.setEtag(etag); + rssFeed.setLastModified(lastModified); callback.onSyncSuccess(rssFeed, type); break; case RSS_ATOM: ATOMFeed atomFeed = serializer.read(ATOMFeed.class, xml); + atomFeed.setEtag(etag); + atomFeed.setLastModified(etag); + callback.onSyncSuccess(atomFeed, type); break; case RSS_JSON: Gson gson = new Gson(); JSONFeed feed = gson.fromJson(xml, JSONFeed.class); + + feed.setEtag(etag); + feed.setLastModified(lastModified); + callback.onSyncSuccess(feed, type); break; } @@ -113,17 +136,17 @@ public class RSSNetwork { */ private RSSType getRSSType(String contentType) { switch (contentType) { - case Utils.RSS_DEFAULT_CONTENT_TYPE: + case LibUtils.RSS_DEFAULT_CONTENT_TYPE: return RSSType.RSS_2; - case Utils.RSS_TEXT_CONTENT_TYPE: + case LibUtils.RSS_TEXT_CONTENT_TYPE: return RSSType.RSS_UNKNOWN; - case Utils.RSS_APPLICATION_CONTENT_TYPE: + case LibUtils.RSS_APPLICATION_CONTENT_TYPE: return RSSType.RSS_UNKNOWN; - case Utils.ATOM_CONTENT_TYPE: + case LibUtils.ATOM_CONTENT_TYPE: return RSSType.RSS_ATOM; - case Utils.JSON_CONTENT_TYPE: + case LibUtils.JSON_CONTENT_TYPE: return RSSType.RSS_JSON; - case Utils.HTML_CONTENT_TYPE: + case LibUtils.HTML_CONTENT_TYPE: return RSSType.RSS_UNKNOWN; default: Log.d(TAG, "bad content type : " + contentType); @@ -132,6 +155,10 @@ public class RSSNetwork { } } + public void setCallback(QueryCallback callback) { + this.callback = callback; + } + public enum RSSType { RSS_2("rss"), RSS_ATOM("atom"),