Implementing Optional GET with If-None-Match and If-Modified-Since headers

app/src/main/java/com/readrops/app/LocalFeedRepository.java

@@ -15,6 +15,7 @@ import com.readrops.app.database.entities.Item;
 import com.readrops.readropslibrary.HtmlParser;
 import com.readrops.readropslibrary.ParsingResult;
 import com.readrops.readropslibrary.QueryCallback;
+import com.readrops.readropslibrary.Utils.LibUtils;
 import com.readrops.readropslibrary.localfeed.AFeed;
 import com.readrops.readropslibrary.localfeed.RSSNetwork;
 import com.readrops.readropslibrary.localfeed.atom.ATOMFeed;
@@ -25,6 +26,7 @@ import org.jsoup.Jsoup;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.HashMap;
 import java.util.List;
 
 import okhttp3.OkHttpClient;
@@ -51,11 +53,18 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
     public void sync() {
         executor.execute(() -> {
             RSSNetwork rssNet = new RSSNetwork();
+            rssNet.setCallback(this);
             List<Feed> feedList = database.feedDao().getAllFeeds();
 
             for (Feed feed : feedList) {
                 try {
-                    rssNet.request(feed.getUrl(), this);
+                    HashMap<String, String> headers = new HashMap<>();
+                    if (feed.getEtag() != null)
+                        headers.put(LibUtils.IF_NONE_MATCH_HEADER, feed.getEtag());
+                    if (feed.getLastModified() != null)
+                        headers.put(LibUtils.IF_MODIFIED_HEADER, feed.getLastModified());
+
+                    rssNet.request(feed.getUrl(), headers);
                 } catch (Exception e) {
                     failureCallBackInMainThread(e);
                 }
@@ -70,7 +79,8 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
         executor.execute(() -> {
             try {
                 RSSNetwork rssNet = new RSSNetwork();
-                rssNet.request(result.getUrl(), this);
+                rssNet.setCallback(this);
+                rssNet.request(result.getUrl(), new HashMap<>());
 
                 postCallBackSuccess();
             } catch (Exception e) {
@@ -116,11 +126,12 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
         try {
             Feed dbFeed = database.feedDao().getFeedByUrl(rssFeed.getChannel().getFeedUrl());
             if (dbFeed == null) {
-                dbFeed = Feed.feedFromRSS(rssFeed.getChannel());
+                dbFeed = Feed.feedFromRSS(rssFeed);
 
                 setFavIconUtils(dbFeed);
                 dbFeed.setId((int)(database.feedDao().insert(dbFeed)));
-            }
+            } else
+                database.feedDao().updateHeaders(rssFeed.getEtag(), rssFeed.getLastModified(), dbFeed.getId());
 
             List<Item> dbItems = Item.itemsFromRSS(rssFeed.getChannel().getItems(), dbFeed);
             insertItems(dbItems, dbFeed);
@@ -139,7 +150,8 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
 
                 setFavIconUtils(dbFeed);
                 dbFeed.setId((int)(database.feedDao().insert(dbFeed)));
-            }
+            } else
+                database.feedDao().updateHeaders(feed.getEtag(), feed.getLastModified(), dbFeed.getId());
 
             List<Item> dbItems = Item.itemsFromATOM(feed.getEntries(), dbFeed);
             insertItems(dbItems, dbFeed);
@@ -157,7 +169,8 @@ public class LocalFeedRepository extends ARepository implements QueryCallback {
 
                 setFavIconUtils(dbFeed);
                 dbFeed.setId((int)(database.feedDao().insert(dbFeed)));
-            }
+            } else
+                database.feedDao().updateHeaders(feed.getEtag(), feed.getLastModified(), dbFeed.getId());
 
             List<Item> dbItems = Item.itemsFromJSON(feed.getItems(), dbFeed);
             insertItems(dbItems, dbFeed);

app/src/main/java/com/readrops/app/database/dao/FeedDao.java

@@ -4,6 +4,7 @@ package com.readrops.app.database.dao;
 import android.arch.persistence.room.Dao;
 import android.arch.persistence.room.Insert;
 import android.arch.persistence.room.Query;
+import android.arch.persistence.room.Update;
 
 import com.readrops.app.database.entities.Feed;
 
@@ -27,4 +28,7 @@ public interface FeedDao {
     @Query("Select id from Feed Where url = :feedUrl")
     int getFeedIdByUrl(String feedUrl);
 
+    @Query("Update Feed set etag = :etag, last_modified = :lastModified Where id = :feedId")
+    void updateHeaders(String etag, String lastModified, int feedId);
+
 }

app/src/main/java/com/readrops/app/database/entities/Feed.java

@@ -7,6 +7,9 @@ import android.support.annotation.ColorInt;
 import com.readrops.readropslibrary.localfeed.atom.ATOMFeed;
 import com.readrops.readropslibrary.localfeed.json.JSONFeed;
 import com.readrops.readropslibrary.localfeed.rss.RSSChannel;
+import com.readrops.readropslibrary.localfeed.rss.RSSFeed;
+
+import java.nio.channels.Channel;
 
 @Entity
 public class Feed {
@@ -29,6 +32,11 @@ public class Feed {
     @ColumnInfo(name = "icon_url")
     private String iconUrl;
 
+    private String etag;
+
+    @ColumnInfo(name = "last_modified")
+    private String lastModified;
+
     public Feed() {
 
     }
@@ -104,8 +112,25 @@ public class Feed {
         this.iconUrl = iconUrl;
     }
 
-    public static Feed feedFromRSS(RSSChannel channel) {
+    public String getEtag() {
+        return etag;
+    }
+
+    public void setEtag(String etag) {
+        this.etag = etag;
+    }
+
+    public String getLastModified() {
+        return lastModified;
+    }
+
+    public void setLastModified(String lastModified) {
+        this.lastModified = lastModified;
+    }
+
+    public static Feed feedFromRSS(RSSFeed rssFeed) {
         Feed feed = new Feed();
+        RSSChannel channel = rssFeed.getChannel();
 
         feed.setName(channel.getTitle());
         feed.setUrl(channel.getFeedUrl());
@@ -113,6 +138,9 @@ public class Feed {
         feed.setDescription(channel.getDescription());
         feed.setLastUpdated(channel.getLastUpdated());
 
+        feed.setEtag(rssFeed.getEtag());
+        feed.setLastModified(rssFeed.getLastModified());
+
         return feed;
     }
 
@@ -126,6 +154,9 @@ public class Feed {
         feed.setDescription(atomFeed.getSubtitle());
         feed.setLastUpdated(atomFeed.getUpdated());
 
+        feed.setEtag(atomFeed.getEtag());
+        feed.setLastModified(atomFeed.getLastModified());
+
         return feed;
     }
 
@@ -137,6 +168,9 @@ public class Feed {
         feed.setDescription(jsonFeed.getDescription());
         //feed.setLastUpdated(jsonFeed.); maybe later ?
 
+        feed.setEtag(jsonFeed.getEtag());
+        feed.setLastModified(jsonFeed.getLastModified());
+
         return feed;
     }
 }

readropslibrary/src/main/java/com/readrops/readropslibrary/HtmlParser.java

@@ -2,7 +2,7 @@ package com.readrops.readropslibrary;
 
 import android.util.Log;
 
-import com.readrops.readropslibrary.Utils.Utils;
+import com.readrops.readropslibrary.Utils.LibUtils;
 
 import org.jsoup.Connection;
 import org.jsoup.Jsoup;
@@ -45,11 +45,11 @@ public final class HtmlParser {
     }
 
     private static boolean isTypeRssFeed(String type) {
-        return type.equals(Utils.RSS_DEFAULT_CONTENT_TYPE) ||
+        return type.equals(LibUtils.RSS_DEFAULT_CONTENT_TYPE) ||
-                type.equals(Utils.ATOM_CONTENT_TYPE) ||
+                type.equals(LibUtils.ATOM_CONTENT_TYPE) ||
-                type.equals(Utils.JSON_CONTENT_TYPE) ||
+                type.equals(LibUtils.JSON_CONTENT_TYPE) ||
-                type.equals(Utils.RSS_TEXT_CONTENT_TYPE) ||
+                type.equals(LibUtils.RSS_TEXT_CONTENT_TYPE) ||
-                type.equals(Utils.RSS_APPLICATION_CONTENT_TYPE);
+                type.equals(LibUtils.RSS_APPLICATION_CONTENT_TYPE);
     }
 
     /**

readropslibrary/src/main/java/com/readrops/readropslibrary/Utils/LibUtils.java

@@ -3,7 +3,7 @@ package com.readrops.readropslibrary.Utils;
 import java.io.InputStream;
 import java.util.Scanner;
 
-public final class Utils {
+public final class LibUtils {
 
     public static final String RSS_DEFAULT_CONTENT_TYPE = "application/rss+xml";
     public static final String RSS_TEXT_CONTENT_TYPE = "text/xml";
@@ -12,6 +12,12 @@ public final class Utils {
     public static final String JSON_CONTENT_TYPE = "application/json";
     public static final String HTML_CONTENT_TYPE = "text/html";
 
+    public static final String CONTENT_TYPE_HEADER = "content-type";
+    public static final String ETAG_HEADER = "ETag";
+    public static final String IF_NONE_MATCH_HEADER = "If-None-Match";
+    public static final String LAST_MODIFIED_HEADER = "Last-Modified";
+    public static final String IF_MODIFIED_HEADER = "If-Modified-Since";
+
     public static String inputStreamToString(InputStream input) {
         Scanner scanner = new Scanner(input).useDelimiter("\\A");
         return scanner.hasNext() ? scanner.next() : "";

readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/AFeed.java

@@ -4,4 +4,24 @@ package com.readrops.readropslibrary.localfeed;
  A simple class to give an abstract level to rss/atom/json feed classes
  */
 public abstract class AFeed {
+
+    protected String etag;
+
+    protected String lastModified;
+
+    public String getEtag() {
+        return etag;
+    }
+
+    public void setEtag(String etag) {
+        this.etag = etag;
+    }
+
+    public String getLastModified() {
+        return lastModified;
+    }
+
+    public void setLastModified(String lastModified) {
+        this.lastModified = lastModified;
+    }
 }

readropslibrary/src/main/java/com/readrops/readropslibrary/localfeed/RSSNetwork.java

@@ -4,7 +4,7 @@ import android.util.Log;
 
 import com.google.gson.Gson;
 import com.readrops.readropslibrary.QueryCallback;
-import com.readrops.readropslibrary.Utils.Utils;
+import com.readrops.readropslibrary.Utils.LibUtils;
 import com.readrops.readropslibrary.localfeed.atom.ATOMFeed;
 import com.readrops.readropslibrary.localfeed.json.JSONFeed;
 import com.readrops.readropslibrary.localfeed.rss.RSSFeed;
@@ -14,6 +14,7 @@ import org.simpleframework.xml.Serializer;
 import org.simpleframework.xml.core.Persister;
 
 import java.io.InputStream;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -29,37 +30,48 @@ public class RSSNetwork {
 
     private static final String RSS_2_REGEX = "rss.*version=\"2.0\"";
 
+    private QueryCallback callback;
+
     /**
      * Request the url given in parameter.
      * This method is synchronous, <b>it has to be called from another thread than the main one</b>.
      * @param url url to request
-     * @param callback result callback if success or error
      * @throws Exception
      */
-    public void request(String url, final QueryCallback callback) throws Exception {
+    public void request(String url, Map<String, String> headers) throws Exception {
+        if (callback == null)
+            throw new NullPointerException("Callback can't be null");
+
         OkHttpClient okHttpClient = new OkHttpClient();
 
-        Request request = new Request.Builder().url(url).build();
+        Request.Builder builder = new Request.Builder().url(url);
+        for (String header : headers.keySet()) {
+            String value = headers.get(header);
+            builder.addHeader(header, value);
+        }
+
+        Request request = builder.build();
         Response response = okHttpClient.newCall(request).execute();
 
-        Pattern pattern = Pattern.compile(RSS_CONTENT_TYPE_REGEX);
-        Matcher matcher = pattern.matcher(response.header("content-type"));
-
-        String header;
-        if (matcher.find())
-            header = matcher.group(0);
-        else
-            header = response.header("content-type");
-
         if (response.isSuccessful()) {
+            Pattern pattern = Pattern.compile(RSS_CONTENT_TYPE_REGEX);
+            Matcher matcher = pattern.matcher(response.header(LibUtils.CONTENT_TYPE_HEADER));
+
+            String header;
+            if (matcher.find())
+                header = matcher.group(0);
+            else
+                header = response.header(LibUtils.CONTENT_TYPE_HEADER);
+
             RSSType type = getRSSType(header);
             if (type == null) {
                 callback.onSyncFailure(new IllegalArgumentException("bad content type : " + header + "for " + url));
                 return;
             }
-
+            parseFeed(response.body().byteStream(), type, response);
-            parseFeed(response.body().byteStream(), type, callback, url);
+        } else if (response.code() == 304)
-        } else
+            return;
+        else
             callback.onSyncFailure(new Exception("Error " + response.code() + " when requesting url " + url));
     }
 
@@ -67,12 +79,11 @@ public class RSSNetwork {
      * Parse input feed
      * @param stream inputStream to parse
      * @param type rss type, important to know the format
-     * @param callback success callback
+     * @param response query response
-     * @param url feed url
      * @throws Exception
      */
-    private void parseFeed(InputStream stream, RSSType type, QueryCallback callback, String url) throws Exception {
+    private void parseFeed(InputStream stream, RSSType type, Response response) throws Exception {
-        String xml = Utils.inputStreamToString(stream);
+        String xml = LibUtils.inputStreamToString(stream);
         Serializer serializer = new Persister();
 
         if (type == RSSType.RSS_UNKNOWN) {
@@ -86,21 +97,33 @@ public class RSSNetwork {
             }
         }
 
+        String etag = response.header(LibUtils.ETAG_HEADER);
+        String lastModified = response.header(LibUtils.LAST_MODIFIED_HEADER);
+
         switch (type) {
             case RSS_2:
                 RSSFeed rssFeed = serializer.read(RSSFeed.class, xml);
                 if (rssFeed.getChannel().getFeedUrl() == null) // workaround si the channel does not have any atom:link tag
-                    rssFeed.getChannel().getLinks().add(new RSSLink(null, url));
+                    rssFeed.getChannel().getLinks().add(new RSSLink(null, response.request().url().toString()));
+                rssFeed.setEtag(etag);
+                rssFeed.setLastModified(lastModified);
 
                 callback.onSyncSuccess(rssFeed, type);
                 break;
             case RSS_ATOM:
                 ATOMFeed atomFeed = serializer.read(ATOMFeed.class, xml);
+                atomFeed.setEtag(etag);
+                atomFeed.setLastModified(etag);
+
                 callback.onSyncSuccess(atomFeed, type);
                 break;
             case RSS_JSON:
                 Gson gson = new Gson();
                 JSONFeed feed = gson.fromJson(xml, JSONFeed.class);
+
+                feed.setEtag(etag);
+                feed.setLastModified(lastModified);
+
                 callback.onSyncSuccess(feed, type);
                 break;
         }
@@ -113,17 +136,17 @@ public class RSSNetwork {
      */
     private RSSType getRSSType(String contentType) {
         switch (contentType) {
-            case Utils.RSS_DEFAULT_CONTENT_TYPE:
+            case LibUtils.RSS_DEFAULT_CONTENT_TYPE:
                 return  RSSType.RSS_2;
-            case Utils.RSS_TEXT_CONTENT_TYPE:
+            case LibUtils.RSS_TEXT_CONTENT_TYPE:
                 return RSSType.RSS_UNKNOWN;
-            case Utils.RSS_APPLICATION_CONTENT_TYPE:
+            case LibUtils.RSS_APPLICATION_CONTENT_TYPE:
                 return RSSType.RSS_UNKNOWN;
-            case Utils.ATOM_CONTENT_TYPE:
+            case LibUtils.ATOM_CONTENT_TYPE:
                 return RSSType.RSS_ATOM;
-            case Utils.JSON_CONTENT_TYPE:
+            case LibUtils.JSON_CONTENT_TYPE:
                 return RSSType.RSS_JSON;
-            case Utils.HTML_CONTENT_TYPE:
+            case LibUtils.HTML_CONTENT_TYPE:
                 return RSSType.RSS_UNKNOWN;
             default:
                 Log.d(TAG, "bad content type : " + contentType);
@@ -132,6 +155,10 @@ public class RSSNetwork {
         }
     }
 
+    public void setCallback(QueryCallback callback) {
+        this.callback = callback;
+    }
+
     public enum RSSType {
         RSS_2("rss"),
         RSS_ATOM("atom"),