From 51cea7c4c28de0a9d57633041a3f0a50b7ac8721 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 25 Aug 2020 23:35:19 +0200 Subject: [PATCH] Add data source for local rss, with tests --- api/build.gradle | 7 +- .../androidTest/assets/localfeed/rss_feed.xml | 57 +++++++++ .../assets/{ => opml}/lite_subscriptions.opml | 0 .../assets/{ => opml}/subscriptions.opml | 0 .../assets/{ => opml}/wrong_version.opml | 0 .../api/localfeed/LocalRSSDataSourceTest.kt | 116 ++++++++++++++++++ api/src/debug/AndroidManifest.xml | 5 +- .../api/localfeed/LocalRSSDataSource.kt | 88 +++++++++++++ .../java/com/readrops/api/utils/LibUtils.java | 17 +++ .../api/{ => localfeed}/LocalRSSHelperTest.kt | 0 10 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 api/src/androidTest/assets/localfeed/rss_feed.xml rename api/src/androidTest/assets/{ => opml}/lite_subscriptions.opml (100%) rename api/src/androidTest/assets/{ => opml}/subscriptions.opml (100%) rename api/src/androidTest/assets/{ => opml}/wrong_version.opml (100%) create mode 100644 api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt create mode 100644 api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt rename api/src/test/java/com/readrops/api/{ => localfeed}/LocalRSSHelperTest.kt (100%) diff --git a/api/build.gradle b/api/build.gradle index 05f25476..ab07a8eb 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -56,8 +56,13 @@ dependencies { androidTestImplementation 'androidx.test:runner:1.2.0' androidTestImplementation 'androidx.test:rules:1.2.0' androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' + androidTestImplementation 'com.squareup.okhttp3:mockwebserver:4.8.1' - implementation 'com.squareup.retrofit2:retrofit:2.7.1' + implementation 'com.squareup.okhttp3:okhttp:4.8.1' + + implementation('com.squareup.retrofit2:retrofit:2.7.1') { + exclude group: 'okhttp3', module: 'okhttp3' + } implementation('com.squareup.retrofit2:converter-moshi:2.7.1') { exclude group: 'moshi', module: 'moshi' // moshi converter uses moshi 1.8.0 which breaks codegen 1.9.2 } diff --git a/api/src/androidTest/assets/localfeed/rss_feed.xml b/api/src/androidTest/assets/localfeed/rss_feed.xml new file mode 100644 index 00000000..8c012917 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss_feed.xml @@ -0,0 +1,57 @@ + + + + Hacker News + https://news.ycombinator.com/ + Links for the intellectually curious, ranked by readers. + + Africa declared free of wild polio + https://www.bbc.com/news/world-africa-53887947 + Tue, 25 Aug 2020 17:15:49 +0000 + https://news.ycombinator.com/item?id=24273602 + Comments]]> + + + Palantir S-1 + https://www.sec.gov/Archives/edgar/data/1321655/000119312520230013/d904406ds1.htm + Tue, 25 Aug 2020 21:03:42 +0000 + https://news.ycombinator.com/item?id=24276086 + Comments]]> + + + Openwifi: Linux mac80211 compatible full-stack 802.11/Wi-Fi design based on SDR + https://github.com/open-sdr/openwifi + Tue, 25 Aug 2020 17:45:19 +0000 + https://news.ycombinator.com/item?id=24273919 + Comments]]> + + + Syllabus for Eric's PhD Students + https://docs.google.com/document/d/11D3kHElzS2HQxTwPqcaTnU5HCJ8WGE5brTXI4KLf4dM/edit + Tue, 25 Aug 2020 18:55:12 +0000 + https://news.ycombinator.com/item?id=24274699 + Comments]]> + + + WebBundles harmful to content blocking, security tools, and the open web + https://brave.com/webbundles-harmful-to-content-blocking-security-tools-and-the-open-web/ + Tue, 25 Aug 2020 19:18:50 +0000 + https://news.ycombinator.com/item?id=24274968 + Comments]]> + + + Zappos CEO Tony Hsieh is stepping down after 21 years + https://footwearnews.com/2020/business/executive-moves/zappos-ceo-tony-hsieh-steps-down-1203045974/ + Tue, 25 Aug 2020 06:11:42 +0000 + https://news.ycombinator.com/item?id=24268522 + Comments]]> + + + Evgeny Kuznetsov practices with Bauer stick that has hole in the blade + https://russianmachineneverbreaks.com/2020/07/17/evgeny-kuznetsov-practices-with-bauer-stick-that-has-hole-in-the-blade/ + Tue, 25 Aug 2020 19:38:09 +0000 + https://news.ycombinator.com/item?id=24275159 + Comments]]> + + + \ No newline at end of file diff --git a/api/src/androidTest/assets/lite_subscriptions.opml b/api/src/androidTest/assets/opml/lite_subscriptions.opml similarity index 100% rename from api/src/androidTest/assets/lite_subscriptions.opml rename to api/src/androidTest/assets/opml/lite_subscriptions.opml diff --git a/api/src/androidTest/assets/subscriptions.opml b/api/src/androidTest/assets/opml/subscriptions.opml similarity index 100% rename from api/src/androidTest/assets/subscriptions.opml rename to api/src/androidTest/assets/opml/subscriptions.opml diff --git a/api/src/androidTest/assets/wrong_version.opml b/api/src/androidTest/assets/opml/wrong_version.opml similarity index 100% rename from api/src/androidTest/assets/wrong_version.opml rename to api/src/androidTest/assets/opml/wrong_version.opml diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt new file mode 100644 index 00000000..2d0b3db7 --- /dev/null +++ b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt @@ -0,0 +1,116 @@ +package com.readrops.api.localfeed + +import android.accounts.NetworkErrorException +import android.content.Context +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.readrops.api.utils.HttpManager +import com.readrops.api.utils.ParseException +import junit.framework.TestCase.* +import okhttp3.HttpUrl +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.junit.After +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import java.net.HttpURLConnection + + +@RunWith(AndroidJUnit4::class) +class LocalRSSDataSourceTest { + + private val context: Context = InstrumentationRegistry.getInstrumentation().context + private lateinit var url: HttpUrl + + private val mockServer: MockWebServer = MockWebServer() + private val localRSSDataSource = LocalRSSDataSource(HttpManager.getInstance().okHttpClient) + + @Before + fun before() { + mockServer.start() + url = mockServer.url("/rss") + } + + @After + fun tearDown() { + mockServer.close() + } + + @Test + fun successfulQueryTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader("Content-Type", "application/rss+xml; charset=UTF-8") + .setBody(context.resources.assets.open("localfeed/rss_feed.xml").toString())) + + + val pair = localRSSDataSource.queryRSSResource(url.toString(), null, false) + + assertNotNull(pair?.first) + assertNotNull(pair?.second) + } + + @Test + fun response304Test() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_NOT_MODIFIED)) + + val pair = localRSSDataSource.queryRSSResource(url.toString(), null, false) + + assertNull(pair) + } + + @Test(expected = NetworkErrorException::class) + fun response404Test() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_NOT_FOUND)) + + localRSSDataSource.queryRSSResource(url.toString(), null, false) + } + + @Test(expected = ParseException::class) + fun noContentTypeTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK)) + + localRSSDataSource.queryRSSResource(url.toString(), null, false) + } + + @Test(expected = ParseException::class) + fun badContentTypeTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader("Content-Type", "")) + + localRSSDataSource.queryRSSResource(url.toString(), null, false) + } + + @Test(expected = ParseException::class) + fun badContentTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader("Content-Type", "application/xml") + .setBody(" ")) + + localRSSDataSource.queryRSSResource(url.toString(), null, false) + } + + @Test + fun isUrlResourceSuccessfulTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader("Content-Type", "application/atom+xml")) + + assertTrue(localRSSDataSource.isUrlRSSResource(url.toString())) + } + + @Test + fun isUrlRSSResourceFailureTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_NOT_FOUND)) + + assertFalse(localRSSDataSource.isUrlRSSResource(url.toString())) + } + + @Test + fun isUrlRSSResourceBadContentTypeTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader("Content-Type", "application/xml") + .setBody(" ")) + + assertFalse(localRSSDataSource.isUrlRSSResource(url.toString())) + } +} \ No newline at end of file diff --git a/api/src/debug/AndroidManifest.xml b/api/src/debug/AndroidManifest.xml index 7c249c25..bc80eafd 100644 --- a/api/src/debug/AndroidManifest.xml +++ b/api/src/debug/AndroidManifest.xml @@ -3,7 +3,10 @@ + - + \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt new file mode 100644 index 00000000..a6f2a007 --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -0,0 +1,88 @@ +package com.readrops.api.localfeed + +import android.accounts.NetworkErrorException +import androidx.annotation.WorkerThread +import com.readrops.api.utils.LibUtils +import com.readrops.api.utils.ParseException +import com.readrops.db.entities.Feed +import com.readrops.db.entities.Item +import okhttp3.Headers +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import java.io.IOException +import java.io.InputStream +import java.net.HttpURLConnection + +class LocalRSSDataSource(private val httpClient: OkHttpClient) { + + /** + * Query RSS url + * @param url url to query + * @param headers request headers + * @param withItems parse items with their feed + * @return a Feed object with its items if specified by [withItems] + */ + @WorkerThread + fun queryRSSResource(url: String, headers: Headers?, withItems: Boolean): Pair>? { + val response = queryUrl(url, headers) + + return when { + response.isSuccessful -> { + val header = response.header(LibUtils.CONTENT_TYPE_HEADER) + ?: throw ParseException("Unable to get $url content-type") + + val contentType = LibUtils.parseContentType(header) + ?: throw ParseException("Unable to get $url content-type") + + var type = LocalRSSHelper.getRSSType(contentType) + + // if we can't guess type based on content-type header, we use the content + if (type == LocalRSSHelper.RSSType.UNKNOWN) type = LocalRSSHelper.getRSSContentType(response.body?.byteStream()!!) + // if we can't guess type even with the content, we are unable to go further + if (type == LocalRSSHelper.RSSType.UNKNOWN) throw ParseException("Unable to guess $url RSS type") + + val feed = parseFeed(response, type) + val items = if (withItems) parseItems(response.body?.byteStream()!!, type) else listOf() + + return Pair(feed, items) + } + response.code == HttpURLConnection.HTTP_NOT_MODIFIED -> null + else -> throw NetworkErrorException("$url returned ${response.code} code : ${response.message}") + } + } + + @WorkerThread + fun isUrlRSSResource(url: String): Boolean { + val response = queryUrl(url, null) + + return if (response.isSuccessful) { + val contentType = response.header(LibUtils.CONTENT_TYPE_HEADER) + ?: return false + + var type = LocalRSSHelper.getRSSType(contentType) + + if (type == LocalRSSHelper.RSSType.UNKNOWN) + type = LocalRSSHelper.getRSSContentType(response.body?.byteStream()!!) // stream is closed in helper method + + type != LocalRSSHelper.RSSType.UNKNOWN + } else false + } + + @Throws(IOException::class) + private fun queryUrl(url: String, headers: Headers?): Response { + val requestBuilder = Request.Builder().url(url) + headers?.let { requestBuilder.headers(it) } + + return httpClient.newCall(requestBuilder.build()).execute() + } + + private fun parseFeed(response: Response, type: LocalRSSHelper.RSSType): Feed { + response.body?.close() + return Feed() + } + + private fun parseItems(inputStream: InputStream, type: LocalRSSHelper.RSSType): List { + return listOf() + } +} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/utils/LibUtils.java b/api/src/main/java/com/readrops/api/utils/LibUtils.java index bbc4c426..88c238d2 100644 --- a/api/src/main/java/com/readrops/api/utils/LibUtils.java +++ b/api/src/main/java/com/readrops/api/utils/LibUtils.java @@ -4,10 +4,13 @@ import android.content.Context; import android.net.Uri; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import java.io.FileNotFoundException; import java.io.InputStream; import java.util.Scanner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public final class LibUtils { @@ -28,6 +31,8 @@ public final class LibUtils { public static final int HTTP_NOT_FOUND = 404; public static final int HTTP_CONFLICT = 409; + private static final String RSS_CONTENT_TYPE_REGEX = "([^;]+)"; + public static String inputStreamToString(InputStream input) { Scanner scanner = new Scanner(input).useDelimiter("\\A"); @@ -44,4 +49,16 @@ public final class LibUtils { return type.equals("image") || type.equals("image/jpeg") || type.equals("image/jpg") || type.equals("image/png"); } + + @Nullable + public static String parseContentType(String header) { + Matcher matcher = Pattern.compile(RSS_CONTENT_TYPE_REGEX) + .matcher(header); + + if (matcher.find()) { + return matcher.group(0); + } else { + return null; + } + } } diff --git a/api/src/test/java/com/readrops/api/LocalRSSHelperTest.kt b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt similarity index 100% rename from api/src/test/java/com/readrops/api/LocalRSSHelperTest.kt rename to api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt