From 355b5a4375a11591f98725df67c225e762590df8 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Mon, 24 Aug 2020 21:38:46 +0200 Subject: [PATCH 01/58] Add new helper class to get RSS feed type, with tests --- .../readrops/api/localfeed/LocalRSSHelper.kt | 68 +++++++++++++++++++ .../com/readrops/api/LocalRSSHelperTest.kt | 60 ++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt create mode 100644 api/src/test/java/com/readrops/api/LocalRSSHelperTest.kt diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt new file mode 100644 index 00000000..f3cc57f3 --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt @@ -0,0 +1,68 @@ +package com.readrops.api.localfeed + +import com.readrops.api.utils.ParseException +import java.io.InputStream +import java.util.regex.Pattern + +object LocalRSSHelper { + + private const val RSS_DEFAULT_CONTENT_TYPE = "application/rss+xml" + private const val RSS_TEXT_CONTENT_TYPE = "text/xml" + private const val RSS_APPLICATION_CONTENT_TYPE = "application/xml" + private const val ATOM_CONTENT_TYPE = "application/atom+xml" + private const val JSON_CONTENT_TYPE = "application/json" + private const val HTML_CONTENT_TYPE = "text/html" + + private const val RSS_2_REGEX = "rss.*version=\"2.0\"" + + private const val ATOM_REGEX = " RSSType.RSS_2 + ATOM_CONTENT_TYPE -> RSSType.ATOM + JSON_CONTENT_TYPE -> RSSType.JSONFEED + RSS_TEXT_CONTENT_TYPE, RSS_APPLICATION_CONTENT_TYPE, HTML_CONTENT_TYPE -> RSSType.UNKNOWN + else -> throw ParseException("Unknown content type") + } + } + + /** + * Guess RSS type based on xml content + */ + fun getRSSContentType(content: InputStream): RSSType { + val stringBuffer = StringBuffer() + val reader = content.bufferedReader() + + var currentLine = reader.readLine() + while (currentLine != null) { + stringBuffer.append(currentLine) + + if (Pattern.compile(RSS_2_REGEX).matcher(stringBuffer.toString()).find()) { + reader.close() + content.close() + + return RSSType.RSS_2 + } else if (Pattern.compile(ATOM_REGEX).matcher(stringBuffer.toString()).find()) { + reader.close() + content.close() + + return RSSType.ATOM + } + + currentLine = reader.readLine() + } + + return RSSType.UNKNOWN + } + + enum class RSSType { + RSS_2, + ATOM, + JSONFEED, + UNKNOWN + } +} \ No newline at end of file diff --git a/api/src/test/java/com/readrops/api/LocalRSSHelperTest.kt b/api/src/test/java/com/readrops/api/LocalRSSHelperTest.kt new file mode 100644 index 00000000..681352d7 --- /dev/null +++ b/api/src/test/java/com/readrops/api/LocalRSSHelperTest.kt @@ -0,0 +1,60 @@ +package com.readrops.api + +import com.readrops.api.localfeed.LocalRSSHelper +import com.readrops.api.utils.ParseException +import junit.framework.TestCase.assertEquals +import org.junit.Test +import java.io.ByteArrayInputStream + +class LocalRSSHelperTest { + + @Test + fun standardContentTypesTest() { + assertEquals(LocalRSSHelper.getRSSType("application/rss+xml"), + LocalRSSHelper.RSSType.RSS_2) + assertEquals(LocalRSSHelper.getRSSType("application/atom+xml"), + LocalRSSHelper.RSSType.ATOM) + assertEquals(LocalRSSHelper.getRSSType("application/json"), + LocalRSSHelper.RSSType.JSONFEED) + } + + @Test + fun nonStandardContentTypesTest() { + assertEquals(LocalRSSHelper.getRSSType("application/xml"), + LocalRSSHelper.RSSType.UNKNOWN) + assertEquals(LocalRSSHelper.getRSSType("text/xml"), + LocalRSSHelper.RSSType.UNKNOWN) + assertEquals(LocalRSSHelper.getRSSType("text/html"), + LocalRSSHelper.RSSType.UNKNOWN) + } + + @Test(expected = ParseException::class) + fun nonSupportedContentTypeTest() { + LocalRSSHelper.getRSSType("image/jpeg") + } + + @Test + fun rssContentTest() { + assertEquals(LocalRSSHelper.getRSSContentType(ByteArrayInputStream( + """ + """.toByteArray() + )), LocalRSSHelper.RSSType.RSS_2) + } + + @Test + fun atomContentTest() { + assertEquals(LocalRSSHelper.getRSSContentType(ByteArrayInputStream( + """ + + """.toByteArray() + )), LocalRSSHelper.RSSType.ATOM) + + } +} \ No newline at end of file From 51cea7c4c28de0a9d57633041a3f0a50b7ac8721 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 25 Aug 2020 23:35:19 +0200 Subject: [PATCH 02/58] Add data source for local rss, with tests --- api/build.gradle | 7 +- .../androidTest/assets/localfeed/rss_feed.xml | 57 +++++++++ .../assets/{ => opml}/lite_subscriptions.opml | 0 .../assets/{ => opml}/subscriptions.opml | 0 .../assets/{ => opml}/wrong_version.opml | 0 .../api/localfeed/LocalRSSDataSourceTest.kt | 116 ++++++++++++++++++ api/src/debug/AndroidManifest.xml | 5 +- .../api/localfeed/LocalRSSDataSource.kt | 88 +++++++++++++ .../java/com/readrops/api/utils/LibUtils.java | 17 +++ .../api/{ => localfeed}/LocalRSSHelperTest.kt | 0 10 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 api/src/androidTest/assets/localfeed/rss_feed.xml rename api/src/androidTest/assets/{ => opml}/lite_subscriptions.opml (100%) rename api/src/androidTest/assets/{ => opml}/subscriptions.opml (100%) rename api/src/androidTest/assets/{ => opml}/wrong_version.opml (100%) create mode 100644 api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt create mode 100644 api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt rename api/src/test/java/com/readrops/api/{ => localfeed}/LocalRSSHelperTest.kt (100%) diff --git a/api/build.gradle b/api/build.gradle index 05f25476..ab07a8eb 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -56,8 +56,13 @@ dependencies { androidTestImplementation 'androidx.test:runner:1.2.0' androidTestImplementation 'androidx.test:rules:1.2.0' androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' + androidTestImplementation 'com.squareup.okhttp3:mockwebserver:4.8.1' - implementation 'com.squareup.retrofit2:retrofit:2.7.1' + implementation 'com.squareup.okhttp3:okhttp:4.8.1' + + implementation('com.squareup.retrofit2:retrofit:2.7.1') { + exclude group: 'okhttp3', module: 'okhttp3' + } implementation('com.squareup.retrofit2:converter-moshi:2.7.1') { exclude group: 'moshi', module: 'moshi' // moshi converter uses moshi 1.8.0 which breaks codegen 1.9.2 } diff --git a/api/src/androidTest/assets/localfeed/rss_feed.xml b/api/src/androidTest/assets/localfeed/rss_feed.xml new file mode 100644 index 00000000..8c012917 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss_feed.xml @@ -0,0 +1,57 @@ + + + + Hacker News + https://news.ycombinator.com/ + Links for the intellectually curious, ranked by readers. + + Africa declared free of wild polio + https://www.bbc.com/news/world-africa-53887947 + Tue, 25 Aug 2020 17:15:49 +0000 + https://news.ycombinator.com/item?id=24273602 + Comments]]> + + + Palantir S-1 + https://www.sec.gov/Archives/edgar/data/1321655/000119312520230013/d904406ds1.htm + Tue, 25 Aug 2020 21:03:42 +0000 + https://news.ycombinator.com/item?id=24276086 + Comments]]> + + + Openwifi: Linux mac80211 compatible full-stack 802.11/Wi-Fi design based on SDR + https://github.com/open-sdr/openwifi + Tue, 25 Aug 2020 17:45:19 +0000 + https://news.ycombinator.com/item?id=24273919 + Comments]]> + + + Syllabus for Eric's PhD Students + https://docs.google.com/document/d/11D3kHElzS2HQxTwPqcaTnU5HCJ8WGE5brTXI4KLf4dM/edit + Tue, 25 Aug 2020 18:55:12 +0000 + https://news.ycombinator.com/item?id=24274699 + Comments]]> + + + WebBundles harmful to content blocking, security tools, and the open web + https://brave.com/webbundles-harmful-to-content-blocking-security-tools-and-the-open-web/ + Tue, 25 Aug 2020 19:18:50 +0000 + https://news.ycombinator.com/item?id=24274968 + Comments]]> + + + Zappos CEO Tony Hsieh is stepping down after 21 years + https://footwearnews.com/2020/business/executive-moves/zappos-ceo-tony-hsieh-steps-down-1203045974/ + Tue, 25 Aug 2020 06:11:42 +0000 + https://news.ycombinator.com/item?id=24268522 + Comments]]> + + + Evgeny Kuznetsov practices with Bauer stick that has hole in the blade + https://russianmachineneverbreaks.com/2020/07/17/evgeny-kuznetsov-practices-with-bauer-stick-that-has-hole-in-the-blade/ + Tue, 25 Aug 2020 19:38:09 +0000 + https://news.ycombinator.com/item?id=24275159 + Comments]]> + + + \ No newline at end of file diff --git a/api/src/androidTest/assets/lite_subscriptions.opml b/api/src/androidTest/assets/opml/lite_subscriptions.opml similarity index 100% rename from api/src/androidTest/assets/lite_subscriptions.opml rename to api/src/androidTest/assets/opml/lite_subscriptions.opml diff --git a/api/src/androidTest/assets/subscriptions.opml b/api/src/androidTest/assets/opml/subscriptions.opml similarity index 100% rename from api/src/androidTest/assets/subscriptions.opml rename to api/src/androidTest/assets/opml/subscriptions.opml diff --git a/api/src/androidTest/assets/wrong_version.opml b/api/src/androidTest/assets/opml/wrong_version.opml similarity index 100% rename from api/src/androidTest/assets/wrong_version.opml rename to api/src/androidTest/assets/opml/wrong_version.opml diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt new file mode 100644 index 00000000..2d0b3db7 --- /dev/null +++ b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt @@ -0,0 +1,116 @@ +package com.readrops.api.localfeed + +import android.accounts.NetworkErrorException +import android.content.Context +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.readrops.api.utils.HttpManager +import com.readrops.api.utils.ParseException +import junit.framework.TestCase.* +import okhttp3.HttpUrl +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.junit.After +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import java.net.HttpURLConnection + + +@RunWith(AndroidJUnit4::class) +class LocalRSSDataSourceTest { + + private val context: Context = InstrumentationRegistry.getInstrumentation().context + private lateinit var url: HttpUrl + + private val mockServer: MockWebServer = MockWebServer() + private val localRSSDataSource = LocalRSSDataSource(HttpManager.getInstance().okHttpClient) + + @Before + fun before() { + mockServer.start() + url = mockServer.url("/rss") + } + + @After + fun tearDown() { + mockServer.close() + } + + @Test + fun successfulQueryTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader("Content-Type", "application/rss+xml; charset=UTF-8") + .setBody(context.resources.assets.open("localfeed/rss_feed.xml").toString())) + + + val pair = localRSSDataSource.queryRSSResource(url.toString(), null, false) + + assertNotNull(pair?.first) + assertNotNull(pair?.second) + } + + @Test + fun response304Test() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_NOT_MODIFIED)) + + val pair = localRSSDataSource.queryRSSResource(url.toString(), null, false) + + assertNull(pair) + } + + @Test(expected = NetworkErrorException::class) + fun response404Test() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_NOT_FOUND)) + + localRSSDataSource.queryRSSResource(url.toString(), null, false) + } + + @Test(expected = ParseException::class) + fun noContentTypeTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK)) + + localRSSDataSource.queryRSSResource(url.toString(), null, false) + } + + @Test(expected = ParseException::class) + fun badContentTypeTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader("Content-Type", "")) + + localRSSDataSource.queryRSSResource(url.toString(), null, false) + } + + @Test(expected = ParseException::class) + fun badContentTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader("Content-Type", "application/xml") + .setBody(" ")) + + localRSSDataSource.queryRSSResource(url.toString(), null, false) + } + + @Test + fun isUrlResourceSuccessfulTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader("Content-Type", "application/atom+xml")) + + assertTrue(localRSSDataSource.isUrlRSSResource(url.toString())) + } + + @Test + fun isUrlRSSResourceFailureTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_NOT_FOUND)) + + assertFalse(localRSSDataSource.isUrlRSSResource(url.toString())) + } + + @Test + fun isUrlRSSResourceBadContentTypeTest() { + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader("Content-Type", "application/xml") + .setBody(" ")) + + assertFalse(localRSSDataSource.isUrlRSSResource(url.toString())) + } +} \ No newline at end of file diff --git a/api/src/debug/AndroidManifest.xml b/api/src/debug/AndroidManifest.xml index 7c249c25..bc80eafd 100644 --- a/api/src/debug/AndroidManifest.xml +++ b/api/src/debug/AndroidManifest.xml @@ -3,7 +3,10 @@ + - + \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt new file mode 100644 index 00000000..a6f2a007 --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -0,0 +1,88 @@ +package com.readrops.api.localfeed + +import android.accounts.NetworkErrorException +import androidx.annotation.WorkerThread +import com.readrops.api.utils.LibUtils +import com.readrops.api.utils.ParseException +import com.readrops.db.entities.Feed +import com.readrops.db.entities.Item +import okhttp3.Headers +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import java.io.IOException +import java.io.InputStream +import java.net.HttpURLConnection + +class LocalRSSDataSource(private val httpClient: OkHttpClient) { + + /** + * Query RSS url + * @param url url to query + * @param headers request headers + * @param withItems parse items with their feed + * @return a Feed object with its items if specified by [withItems] + */ + @WorkerThread + fun queryRSSResource(url: String, headers: Headers?, withItems: Boolean): Pair>? { + val response = queryUrl(url, headers) + + return when { + response.isSuccessful -> { + val header = response.header(LibUtils.CONTENT_TYPE_HEADER) + ?: throw ParseException("Unable to get $url content-type") + + val contentType = LibUtils.parseContentType(header) + ?: throw ParseException("Unable to get $url content-type") + + var type = LocalRSSHelper.getRSSType(contentType) + + // if we can't guess type based on content-type header, we use the content + if (type == LocalRSSHelper.RSSType.UNKNOWN) type = LocalRSSHelper.getRSSContentType(response.body?.byteStream()!!) + // if we can't guess type even with the content, we are unable to go further + if (type == LocalRSSHelper.RSSType.UNKNOWN) throw ParseException("Unable to guess $url RSS type") + + val feed = parseFeed(response, type) + val items = if (withItems) parseItems(response.body?.byteStream()!!, type) else listOf() + + return Pair(feed, items) + } + response.code == HttpURLConnection.HTTP_NOT_MODIFIED -> null + else -> throw NetworkErrorException("$url returned ${response.code} code : ${response.message}") + } + } + + @WorkerThread + fun isUrlRSSResource(url: String): Boolean { + val response = queryUrl(url, null) + + return if (response.isSuccessful) { + val contentType = response.header(LibUtils.CONTENT_TYPE_HEADER) + ?: return false + + var type = LocalRSSHelper.getRSSType(contentType) + + if (type == LocalRSSHelper.RSSType.UNKNOWN) + type = LocalRSSHelper.getRSSContentType(response.body?.byteStream()!!) // stream is closed in helper method + + type != LocalRSSHelper.RSSType.UNKNOWN + } else false + } + + @Throws(IOException::class) + private fun queryUrl(url: String, headers: Headers?): Response { + val requestBuilder = Request.Builder().url(url) + headers?.let { requestBuilder.headers(it) } + + return httpClient.newCall(requestBuilder.build()).execute() + } + + private fun parseFeed(response: Response, type: LocalRSSHelper.RSSType): Feed { + response.body?.close() + return Feed() + } + + private fun parseItems(inputStream: InputStream, type: LocalRSSHelper.RSSType): List { + return listOf() + } +} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/utils/LibUtils.java b/api/src/main/java/com/readrops/api/utils/LibUtils.java index bbc4c426..88c238d2 100644 --- a/api/src/main/java/com/readrops/api/utils/LibUtils.java +++ b/api/src/main/java/com/readrops/api/utils/LibUtils.java @@ -4,10 +4,13 @@ import android.content.Context; import android.net.Uri; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import java.io.FileNotFoundException; import java.io.InputStream; import java.util.Scanner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public final class LibUtils { @@ -28,6 +31,8 @@ public final class LibUtils { public static final int HTTP_NOT_FOUND = 404; public static final int HTTP_CONFLICT = 409; + private static final String RSS_CONTENT_TYPE_REGEX = "([^;]+)"; + public static String inputStreamToString(InputStream input) { Scanner scanner = new Scanner(input).useDelimiter("\\A"); @@ -44,4 +49,16 @@ public final class LibUtils { return type.equals("image") || type.equals("image/jpeg") || type.equals("image/jpg") || type.equals("image/png"); } + + @Nullable + public static String parseContentType(String header) { + Matcher matcher = Pattern.compile(RSS_CONTENT_TYPE_REGEX) + .matcher(header); + + if (matcher.find()) { + return matcher.group(0); + } else { + return null; + } + } } diff --git a/api/src/test/java/com/readrops/api/LocalRSSHelperTest.kt b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt similarity index 100% rename from api/src/test/java/com/readrops/api/LocalRSSHelperTest.kt rename to api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt From 4e27a0b86f3a64be4f3657bc3e0c9c2f462352c2 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Fri, 28 Aug 2020 22:50:53 +0200 Subject: [PATCH 03/58] Add some unit tests for RSS type and content guess methods --- .../api/localfeed/LocalRSSHelperTest.kt | 13 +++++++++++-- .../com/readrops/api/utils/LibUtilsTest.kt | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 api/src/test/java/com/readrops/api/utils/LibUtilsTest.kt diff --git a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt index 681352d7..a5678bbc 100644 --- a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt +++ b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt @@ -1,6 +1,5 @@ -package com.readrops.api +package com.readrops.api.localfeed -import com.readrops.api.localfeed.LocalRSSHelper import com.readrops.api.utils.ParseException import junit.framework.TestCase.assertEquals import org.junit.Test @@ -55,6 +54,16 @@ class LocalRSSHelperTest { """.toByteArray() )), LocalRSSHelper.RSSType.ATOM) + } + + @Test + fun unknownContentTest() { + assertEquals(LocalRSSHelper.getRSSContentType(ByteArrayInputStream( + """ + + + """.trimMargin().toByteArray() + )), LocalRSSHelper.RSSType.UNKNOWN) } } \ No newline at end of file diff --git a/api/src/test/java/com/readrops/api/utils/LibUtilsTest.kt b/api/src/test/java/com/readrops/api/utils/LibUtilsTest.kt new file mode 100644 index 00000000..a6801cae --- /dev/null +++ b/api/src/test/java/com/readrops/api/utils/LibUtilsTest.kt @@ -0,0 +1,19 @@ +package com.readrops.api.utils + +import junit.framework.TestCase.assertEquals +import org.junit.Test + +class LibUtilsTest { + + @Test + fun contentTypeWithCharsetTest() { + assertEquals(LibUtils.parseContentType("application/rss+xml; charset=UTF-8"), + "application/rss+xml") + } + + @Test + fun contentTypeWithoutCharsetText() { + assertEquals(LibUtils.parseContentType("text/xml"), + "text/xml") + } +} \ No newline at end of file From 752135621d771ac968a57f0aef976335f6ab9088 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 30 Aug 2020 16:39:08 +0200 Subject: [PATCH 04/58] Move DateUtils in api package --- .../src/main/java/com/readrops/api}/utils/DateUtils.java | 2 +- .../src/test/java/com/readrops/api/utils}/DateUtilsTest.java | 4 +--- .../main/java/com/readrops/app/activities/ItemActivity.java | 2 +- .../java/com/readrops/app/adapters/MainItemListAdapter.java | 2 +- .../java/com/readrops/app/utils/matchers/ItemMatcher.java | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) rename {app/src/main/java/com/readrops/app => api/src/main/java/com/readrops/api}/utils/DateUtils.java (98%) rename {app/src/test/java/com/readrops/app => api/src/test/java/com/readrops/api/utils}/DateUtilsTest.java (96%) diff --git a/app/src/main/java/com/readrops/app/utils/DateUtils.java b/api/src/main/java/com/readrops/api/utils/DateUtils.java similarity index 98% rename from app/src/main/java/com/readrops/app/utils/DateUtils.java rename to api/src/main/java/com/readrops/api/utils/DateUtils.java index f7ea1e36..68133f3a 100644 --- a/app/src/main/java/com/readrops/app/utils/DateUtils.java +++ b/api/src/main/java/com/readrops/api/utils/DateUtils.java @@ -1,4 +1,4 @@ -package com.readrops.app.utils; +package com.readrops.api.utils; import org.joda.time.LocalDateTime; import org.joda.time.format.DateTimeFormat; diff --git a/app/src/test/java/com/readrops/app/DateUtilsTest.java b/api/src/test/java/com/readrops/api/utils/DateUtilsTest.java similarity index 96% rename from app/src/test/java/com/readrops/app/DateUtilsTest.java rename to api/src/test/java/com/readrops/api/utils/DateUtilsTest.java index 95b9058f..7e36f383 100644 --- a/app/src/test/java/com/readrops/app/DateUtilsTest.java +++ b/api/src/test/java/com/readrops/api/utils/DateUtilsTest.java @@ -1,6 +1,4 @@ -package com.readrops.app; - -import com.readrops.app.utils.DateUtils; +package com.readrops.api.utils; import org.joda.time.LocalDateTime; import org.junit.Test; diff --git a/app/src/main/java/com/readrops/app/activities/ItemActivity.java b/app/src/main/java/com/readrops/app/activities/ItemActivity.java index 86462056..a1493f25 100644 --- a/app/src/main/java/com/readrops/app/activities/ItemActivity.java +++ b/app/src/main/java/com/readrops/app/activities/ItemActivity.java @@ -32,7 +32,7 @@ import com.bumptech.glide.request.target.CustomTarget; import com.bumptech.glide.request.transition.Transition; import com.readrops.app.R; import com.readrops.app.databinding.ActivityItemBinding; -import com.readrops.app.utils.DateUtils; +import com.readrops.api.utils.DateUtils; import com.readrops.app.utils.GlideApp; import com.readrops.app.utils.PermissionManager; import com.readrops.app.utils.SharedPreferencesManager; diff --git a/app/src/main/java/com/readrops/app/adapters/MainItemListAdapter.java b/app/src/main/java/com/readrops/app/adapters/MainItemListAdapter.java index a818387c..30520174 100644 --- a/app/src/main/java/com/readrops/app/adapters/MainItemListAdapter.java +++ b/app/src/main/java/com/readrops/app/adapters/MainItemListAdapter.java @@ -29,7 +29,7 @@ import com.readrops.app.R; import com.readrops.db.entities.Item; import com.readrops.db.pojo.ItemWithFeed; import com.readrops.app.databinding.ListItemBinding; -import com.readrops.app.utils.DateUtils; +import com.readrops.api.utils.DateUtils; import com.readrops.app.utils.GlideRequests; import com.readrops.app.utils.Utils; diff --git a/app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java b/app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java index 096134c6..651fa211 100644 --- a/app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java +++ b/app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java @@ -1,6 +1,6 @@ package com.readrops.app.utils.matchers; -import com.readrops.app.utils.DateUtils; +import com.readrops.api.utils.DateUtils; import com.readrops.app.utils.Utils; import com.readrops.db.entities.Feed; import com.readrops.db.entities.Item; From 8089d9ae6c54257943eca9ec3bf3995a0b8c4f4a Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Fri, 11 Sep 2020 22:53:50 +0200 Subject: [PATCH 05/58] Start replacing simplexml with konsume-xml by parsing RSS feed channel with it --- api/build.gradle | 14 +++--- .../localfeed/rss/rss_feed_special_cases.xml | 9 ++++ .../assets/localfeed/rss/rss_full_feed.xml | 9 ++++ .../api/localfeed/rss/RSSFeedAdapterTest.kt | 36 +++++++++++++++ .../api/localfeed/LocalRSSDataSource.kt | 24 ++++++++-- .../com/readrops/api/localfeed/XmlAdapter.kt | 20 ++++++++ .../api/localfeed/rss/RSSFeedAdapter.kt | 46 +++++++++++++++++++ .../readrops/api/utils/KonsumerExtensions.kt | 13 ++++++ 8 files changed, 160 insertions(+), 11 deletions(-) create mode 100644 api/src/androidTest/assets/localfeed/rss/rss_feed_special_cases.xml create mode 100644 api/src/androidTest/assets/localfeed/rss/rss_full_feed.xml create mode 100644 api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSFeedAdapterTest.kt create mode 100644 api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt create mode 100644 api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt create mode 100644 api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt diff --git a/api/build.gradle b/api/build.gradle index ab07a8eb..6d0d0bca 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -43,11 +43,6 @@ dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) implementation project(':db') - // xpp3 has a conflict with kxml when running connectedCheck task - configurations { - all*.exclude group: 'xpp3', module: 'xpp3' - } - implementation "androidx.core:core-ktx:1.2.0" implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" @@ -58,6 +53,8 @@ dependencies { androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' androidTestImplementation 'com.squareup.okhttp3:mockwebserver:4.8.1' + implementation 'com.gitlab.mvysny.konsume-xml:konsume-xml:0.11' + implementation 'com.squareup.okhttp3:okhttp:4.8.1' implementation('com.squareup.retrofit2:retrofit:2.7.1') { @@ -67,7 +64,12 @@ dependencies { exclude group: 'moshi', module: 'moshi' // moshi converter uses moshi 1.8.0 which breaks codegen 1.9.2 } - implementation 'com.squareup.retrofit2:converter-simplexml:2.7.1' + implementation ('com.squareup.retrofit2:converter-simplexml:2.7.1') { + exclude module: 'stax' + exclude module: 'stax-api' + exclude module: 'xpp3' + } + implementation 'com.squareup.retrofit2:adapter-rxjava2:2.7.1' implementation 'com.squareup.moshi:moshi:1.9.2' diff --git a/api/src/androidTest/assets/localfeed/rss/rss_feed_special_cases.xml b/api/src/androidTest/assets/localfeed/rss/rss_feed_special_cases.xml new file mode 100644 index 00000000..4ec78959 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss/rss_feed_special_cases.xml @@ -0,0 +1,9 @@ + + + + + + https://news.ycombinator.com/ + Links for the intellectually curious, ranked by readers. + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/rss/rss_full_feed.xml b/api/src/androidTest/assets/localfeed/rss/rss_full_feed.xml new file mode 100644 index 00000000..671e4066 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss/rss_full_feed.xml @@ -0,0 +1,9 @@ + + + + Hacker News + + https://news.ycombinator.com/ + Links for the intellectually curious, ranked by readers. + + \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSFeedAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSFeedAdapterTest.kt new file mode 100644 index 00000000..79fbcafb --- /dev/null +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSFeedAdapterTest.kt @@ -0,0 +1,36 @@ +package com.readrops.api.localfeed.rss + +import android.content.Context +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.readrops.api.utils.ParseException +import junit.framework.TestCase.assertEquals +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class RSSFeedAdapterTest { + + private val context: Context = InstrumentationRegistry.getInstrumentation().context + + private val adapter = RSSFeedAdapter() + + @Test + fun normalCasesTest() { + val stream = context.resources.assets.open("localfeed/rss/rss_full_feed.xml") + + val feed = adapter.fromXml(stream) + + assertEquals(feed.name, "Hacker News") + assertEquals(feed.url, "https://news.ycombinator.com/feed/") + assertEquals(feed.siteUrl, "https://news.ycombinator.com/") + assertEquals(feed.description, "Links for the intellectually curious, ranked by readers.") + } + + + @Test(expected = ParseException::class) + fun nullTitleTest() { + val stream = context.resources.assets.open("localfeed/rss/rss_feed_special_cases.xml") + adapter.fromXml(stream) + } +} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index a6f2a007..896213fe 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -4,6 +4,7 @@ import android.accounts.NetworkErrorException import androidx.annotation.WorkerThread import com.readrops.api.utils.LibUtils import com.readrops.api.utils.ParseException +import com.readrops.api.utils.UnknownFormatException import com.readrops.db.entities.Feed import com.readrops.db.entities.Item import okhttp3.Headers @@ -33,19 +34,21 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { ?: throw ParseException("Unable to get $url content-type") val contentType = LibUtils.parseContentType(header) - ?: throw ParseException("Unable to get $url content-type") + ?: throw ParseException("Unable to parse $url content-type") var type = LocalRSSHelper.getRSSType(contentType) // if we can't guess type based on content-type header, we use the content - if (type == LocalRSSHelper.RSSType.UNKNOWN) type = LocalRSSHelper.getRSSContentType(response.body?.byteStream()!!) + if (type == LocalRSSHelper.RSSType.UNKNOWN) + type = LocalRSSHelper.getRSSContentType(response.body?.byteStream()!!) // if we can't guess type even with the content, we are unable to go further if (type == LocalRSSHelper.RSSType.UNKNOWN) throw ParseException("Unable to guess $url RSS type") val feed = parseFeed(response, type) val items = if (withItems) parseItems(response.body?.byteStream()!!, type) else listOf() - return Pair(feed, items) + response.body?.close() + Pair(feed, items) } response.code == HttpURLConnection.HTTP_NOT_MODIFIED -> null else -> throw NetworkErrorException("$url returned ${response.code} code : ${response.message}") @@ -78,8 +81,19 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { } private fun parseFeed(response: Response, type: LocalRSSHelper.RSSType): Feed { - response.body?.close() - return Feed() + val feed = if (type != LocalRSSHelper.RSSType.JSONFEED) { + val adapter = XmlAdapter.xmlFeedAdapterFactory(type) + + //adapter.fromXml(response.body?.byteStream()!!) + Feed() + } else { + Feed() + } + + feed.etag = response.header(LibUtils.ETAG_HEADER) + feed.lastModified = response.header(LibUtils.IF_MODIFIED_HEADER) + + return feed } private fun parseItems(inputStream: InputStream, type: LocalRSSHelper.RSSType): List { diff --git a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt new file mode 100644 index 00000000..824d8005 --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt @@ -0,0 +1,20 @@ +package com.readrops.api.localfeed + +import com.readrops.api.localfeed.rss.RSSFeedAdapter +import com.readrops.db.entities.Feed +import java.io.InputStream + +interface XmlAdapter { + + fun fromXml(inputStream: InputStream): T + + companion object { + fun xmlFeedAdapterFactory(type: LocalRSSHelper.RSSType): XmlAdapter { + return when (type) { + LocalRSSHelper.RSSType.RSS_2 -> RSSFeedAdapter() + else -> throw Exception("Unknown RSS type : $type") + } + } + } +} + diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt new file mode 100644 index 00000000..52c6fcff --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt @@ -0,0 +1,46 @@ +package com.readrops.api.localfeed.rss + +import com.gitlab.mvysny.konsumexml.Names +import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore +import com.gitlab.mvysny.konsumexml.konsumeXml +import com.readrops.api.localfeed.XmlAdapter +import com.readrops.api.utils.ParseException +import com.readrops.api.utils.nonNullText +import com.readrops.api.utils.nullableText +import com.readrops.db.entities.Feed +import org.jsoup.Jsoup +import java.io.InputStream + +class RSSFeedAdapter : XmlAdapter { + + override fun fromXml(inputStream: InputStream): Feed { + val konsume = inputStream.konsumeXml() + val feed = Feed() + + return try { + konsume.child("rss") { + child("channel") { + allChildrenAutoIgnore(names) { + with(feed) { + when (tagName) { + "title" -> name = Jsoup.parse(nonNullText()).text() + "description" -> description = nullableText(failOnElement = false) + "link" -> siteUrl = nullableText() + "atom:link" -> url = attributes.getValueOpt("href") + } + } + } + } + } + + konsume.close() + feed + } catch (e: Exception) { + throw ParseException(e.message) + } + } + + companion object { + val names = Names.of("title", "description", "link") + } +} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt b/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt new file mode 100644 index 00000000..adef66de --- /dev/null +++ b/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt @@ -0,0 +1,13 @@ +package com.readrops.api.utils + +import com.gitlab.mvysny.konsumexml.Konsumer + +fun Konsumer.nonNullText(failOnElement: Boolean = true): String { + val text = text(failOnElement = failOnElement) + return if (text.isNotEmpty()) text else throw ParseException("Xml field $name can't be null") +} + +fun Konsumer.nullableText(failOnElement: Boolean = true): String? { + val text = text(failOnElement = failOnElement) + return if (text.isNotEmpty()) text else null +} \ No newline at end of file From 4bf56b8f7f1c2502a8525cd964f27c7bc0fbac46 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sat, 12 Sep 2020 12:02:03 +0200 Subject: [PATCH 06/58] Use UnknownFormatException instead of ParseException for some cases --- .../com/readrops/api/localfeed/LocalRSSDataSourceTest.kt | 3 ++- .../java/com/readrops/api/localfeed/LocalRSSDataSource.kt | 2 +- .../main/java/com/readrops/api/localfeed/LocalRSSHelper.kt | 6 +++--- .../java/com/readrops/api/localfeed/LocalRSSHelperTest.kt | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt index 2d0b3db7..aa6b7ee2 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt @@ -6,6 +6,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry import com.readrops.api.utils.HttpManager import com.readrops.api.utils.ParseException +import com.readrops.api.utils.UnknownFormatException import junit.framework.TestCase.* import okhttp3.HttpUrl import okhttp3.mockwebserver.MockResponse @@ -81,7 +82,7 @@ class LocalRSSDataSourceTest { localRSSDataSource.queryRSSResource(url.toString(), null, false) } - @Test(expected = ParseException::class) + @Test(expected = UnknownFormatException::class) fun badContentTest() { mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) .addHeader("Content-Type", "application/xml") diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index 896213fe..af72392d 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -42,7 +42,7 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { if (type == LocalRSSHelper.RSSType.UNKNOWN) type = LocalRSSHelper.getRSSContentType(response.body?.byteStream()!!) // if we can't guess type even with the content, we are unable to go further - if (type == LocalRSSHelper.RSSType.UNKNOWN) throw ParseException("Unable to guess $url RSS type") + if (type == LocalRSSHelper.RSSType.UNKNOWN) throw UnknownFormatException("Unable to guess $url RSS type") val feed = parseFeed(response, type) val items = if (withItems) parseItems(response.body?.byteStream()!!, type) else listOf() diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt index f3cc57f3..3d4a42b5 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt @@ -1,6 +1,6 @@ package com.readrops.api.localfeed -import com.readrops.api.utils.ParseException +import com.readrops.api.utils.UnknownFormatException import java.io.InputStream import java.util.regex.Pattern @@ -12,7 +12,7 @@ object LocalRSSHelper { private const val ATOM_CONTENT_TYPE = "application/atom+xml" private const val JSON_CONTENT_TYPE = "application/json" private const val HTML_CONTENT_TYPE = "text/html" - + private const val RSS_2_REGEX = "rss.*version=\"2.0\"" private const val ATOM_REGEX = " RSSType.ATOM JSON_CONTENT_TYPE -> RSSType.JSONFEED RSS_TEXT_CONTENT_TYPE, RSS_APPLICATION_CONTENT_TYPE, HTML_CONTENT_TYPE -> RSSType.UNKNOWN - else -> throw ParseException("Unknown content type") + else -> throw UnknownFormatException("Unknown content type : $contentType") } } diff --git a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt index a5678bbc..c35c9e5e 100644 --- a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt +++ b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt @@ -1,6 +1,6 @@ package com.readrops.api.localfeed -import com.readrops.api.utils.ParseException +import com.readrops.api.utils.UnknownFormatException import junit.framework.TestCase.assertEquals import org.junit.Test import java.io.ByteArrayInputStream @@ -27,7 +27,7 @@ class LocalRSSHelperTest { LocalRSSHelper.RSSType.UNKNOWN) } - @Test(expected = ParseException::class) + @Test(expected = UnknownFormatException::class) fun nonSupportedContentTypeTest() { LocalRSSHelper.getRSSType("image/jpeg") } From 20e814f36d764d7b3ebe090e23baad38e6521dc0 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sat, 12 Sep 2020 19:34:03 +0200 Subject: [PATCH 07/58] Add adapter for RSS item parsing --- .../localfeed/rss/rss_items_enclosure.xml | 25 ++++++ .../localfeed/rss/rss_items_media_content.xml | 24 ++++++ .../localfeed/rss/rss_items_no_date.xml | 21 +++++ .../localfeed/rss/rss_items_no_link.xml | 21 +++++ .../localfeed/rss/rss_items_no_title.xml | 21 +++++ .../rss/rss_items_other_namespaces.xml | 23 ++++++ .../androidTest/assets/localfeed/rss_feed.xml | 1 + .../api/localfeed/rss/RSSItemsAdapterTest.kt | 82 +++++++++++++++++++ .../api/localfeed/LocalRSSDataSource.kt | 9 +- .../com/readrops/api/localfeed/XmlAdapter.kt | 9 ++ .../api/localfeed/rss/RSSItemsAdapter.kt | 82 +++++++++++++++++++ 11 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 api/src/androidTest/assets/localfeed/rss/rss_items_enclosure.xml create mode 100644 api/src/androidTest/assets/localfeed/rss/rss_items_media_content.xml create mode 100644 api/src/androidTest/assets/localfeed/rss/rss_items_no_date.xml create mode 100644 api/src/androidTest/assets/localfeed/rss/rss_items_no_link.xml create mode 100644 api/src/androidTest/assets/localfeed/rss/rss_items_no_title.xml create mode 100644 api/src/androidTest/assets/localfeed/rss/rss_items_other_namespaces.xml create mode 100644 api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt create mode 100644 api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_enclosure.xml b/api/src/androidTest/assets/localfeed/rss/rss_items_enclosure.xml new file mode 100644 index 00000000..58601828 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss/rss_items_enclosure.xml @@ -0,0 +1,25 @@ + + + + + title + link + + 2020-08-05T14:03:48Z + + + + + + + guid + + + + + + + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_media_content.xml b/api/src/androidTest/assets/localfeed/rss/rss_items_media_content.xml new file mode 100644 index 00000000..323246c7 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss/rss_items_media_content.xml @@ -0,0 +1,24 @@ + + + + + title + link + + 2020-08-05T14:03:48Z + + + + + + + guid + + + + + + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_no_date.xml b/api/src/androidTest/assets/localfeed/rss/rss_items_no_date.xml new file mode 100644 index 00000000..3ee2e090 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss/rss_items_no_date.xml @@ -0,0 +1,21 @@ + + + + + title + link + + + + + + + + guid + + + + + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_no_link.xml b/api/src/androidTest/assets/localfeed/rss/rss_items_no_link.xml new file mode 100644 index 00000000..69c76fcd --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss/rss_items_no_link.xml @@ -0,0 +1,21 @@ + + + + + title + + 2020-08-05T14:03:48Z + + + + + + + guid + + + + + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_no_title.xml b/api/src/androidTest/assets/localfeed/rss/rss_items_no_title.xml new file mode 100644 index 00000000..a602c8d5 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss/rss_items_no_title.xml @@ -0,0 +1,21 @@ + + + + + link + + 2020-08-05T14:03:48Z + + + + + + + guid + + + + + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_other_namespaces.xml b/api/src/androidTest/assets/localfeed/rss/rss_items_other_namespaces.xml new file mode 100644 index 00000000..cde19568 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss/rss_items_other_namespaces.xml @@ -0,0 +1,23 @@ + + + + + title + link + guid + + 2020-08-05T14:03:48Z + + + + + + + guid + + + + + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/rss_feed.xml b/api/src/androidTest/assets/localfeed/rss_feed.xml index 8c012917..fbd76101 100644 --- a/api/src/androidTest/assets/localfeed/rss_feed.xml +++ b/api/src/androidTest/assets/localfeed/rss_feed.xml @@ -9,6 +9,7 @@ https://www.bbc.com/news/world-africa-53887947 Tue, 25 Aug 2020 17:15:49 +0000 https://news.ycombinator.com/item?id=24273602 + Author 1 Comments]]> diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt new file mode 100644 index 00000000..2358d872 --- /dev/null +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt @@ -0,0 +1,82 @@ +package com.readrops.api.localfeed.rss + +import android.content.Context +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.readrops.api.utils.DateUtils +import com.readrops.api.utils.ParseException +import junit.framework.TestCase.assertEquals +import junit.framework.TestCase.assertNotNull +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class RSSItemsAdapterTest { + + private val context: Context = InstrumentationRegistry.getInstrumentation().context + + private val adapter = RSSItemsAdapter() + + @Test + fun normalCasesTest() { + val stream = context.resources.assets.open("localfeed/rss_feed.xml") + + val items = adapter.fromXml(stream) + assertEquals(items.size, 7) + + val item = items[0] + + assertEquals(item.title, "Africa declared free of wild polio") + assertEquals(item.link, "https://www.bbc.com/news/world-africa-53887947") + assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("Tue, 25 Aug 2020 17:15:49 +0000")) + assertEquals(item.author, "Author 1") + assertNotNull(item.description) + assertEquals(item.guid, "https://www.bbc.com/news/world-africa-53887947") + } + + @Test + fun otherNamespacesTest() { + val stream = context.resources.assets.open("localfeed/rss/rss_items_other_namespaces.xml") + val item = adapter.fromXml(stream)[0] + + assertEquals(item.guid, "guid") + assertEquals(item.author, "creator") + assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("2020-08-05T14:03:48Z")) + assertEquals(item.content, "content:encoded") + } + + @Test + fun noTitleTest() { + val stream = context.resources.assets.open("localfeed/rss/rss_items_no_title.xml") + Assert.assertThrows("Item title can't be null", ParseException::class.java) { adapter.fromXml(stream) } + } + + @Test + fun noLinkTest() { + val stream = context.resources.assets.open("localfeed/rss/rss_items_no_link.xml") + Assert.assertThrows("Item link can't be null", ParseException::class.java) { adapter.fromXml(stream) } + } + + @Test + fun noDateTest() { + val stream = context.resources.assets.open("localfeed/rss/rss_items_no_date.xml") + Assert.assertThrows("Item date can't be null", ParseException::class.java) { adapter.fromXml(stream) } + } + + @Test + fun enclosureTest() { + val stream = context.resources.assets.open("localfeed/rss/rss_items_enclosure.xml") + val item = adapter.fromXml(stream)[0] + + assertEquals(item.imageLink, "https://image1.jpg") + } + + @Test + fun mediaContentTest() { + val stream = context.resources.assets.open("localfeed/rss/rss_items_media_content.xml") + val item = adapter.fromXml(stream)[0] + + assertEquals(item.imageLink, "https://image2.jpg") + } +} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index af72392d..32ce19c4 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -97,6 +97,13 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { } private fun parseItems(inputStream: InputStream, type: LocalRSSHelper.RSSType): List { - return listOf() + return if (type != LocalRSSHelper.RSSType.JSONFEED) { + val adapter = XmlAdapter.xmlItemsAdapterFactory(type) + + //adapter.fromXml(inputStream) + listOf() + } else { + listOf() + } } } \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt index 824d8005..874d2944 100644 --- a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt @@ -1,7 +1,9 @@ package com.readrops.api.localfeed import com.readrops.api.localfeed.rss.RSSFeedAdapter +import com.readrops.api.localfeed.rss.RSSItemsAdapter import com.readrops.db.entities.Feed +import com.readrops.db.entities.Item import java.io.InputStream interface XmlAdapter { @@ -15,6 +17,13 @@ interface XmlAdapter { else -> throw Exception("Unknown RSS type : $type") } } + + fun xmlItemsAdapterFactory(type: LocalRSSHelper.RSSType): XmlAdapter> { + return when (type) { + LocalRSSHelper.RSSType.RSS_2 -> RSSItemsAdapter() + else -> throw Exception("Unknown RSS type : $type") + } + } } } diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt new file mode 100644 index 00000000..3a069b19 --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt @@ -0,0 +1,82 @@ +package com.readrops.api.localfeed.rss + +import com.gitlab.mvysny.konsumexml.* +import com.readrops.api.localfeed.XmlAdapter +import com.readrops.api.utils.* +import com.readrops.db.entities.Item +import java.io.InputStream + +class RSSItemsAdapter : XmlAdapter> { + + override fun fromXml(inputStream: InputStream): List { + val konsume = inputStream.konsumeXml() + val items = mutableListOf() + + return try { + konsume.child("rss") { + child("channel") { + allChildrenAutoIgnore("item") { + val enclosures = arrayListOf() + val mediaContents = arrayListOf() + + val item = Item().apply { + allChildrenAutoIgnore(names) { + when (tagName) { + "title" -> title = nonNullText() + "link" -> link = nonNullText() + "author" -> author = nullableText() + "dc:creator" -> author = nullableText() + "pubDate" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) + "dc:date" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) + "guid" -> guid = nullableText() + "description" -> description = nullableText() + "content:encoded" -> content = nullableText() + "enclosure" -> parseEnclosure(this, enclosures) + "media:content" -> parseMediaContent(this, mediaContents) + } + } + } + + validateItem(item) + if (item.guid == null) item.guid = item.link + + if (enclosures.isNotEmpty()) item.imageLink = enclosures.first() + else if (mediaContents.isNotEmpty()) item.imageLink = mediaContents.first() + + items += item + } + } + } + + konsume.close() + items + } catch (e: KonsumerException) { + throw ParseException(e.message) + } + } + + private fun parseEnclosure(konsume: Konsumer, enclosures: MutableList) { + if (konsume.attributes.getValueOpt("type") != null + && LibUtils.isMimeImage(konsume.attributes["type"])) + enclosures += konsume.attributes["url"] + } + + private fun parseMediaContent(konsume: Konsumer, mediaContents: MutableList) { + if (konsume.attributes.getValueOpt("medium") != null + && LibUtils.isMimeImage(konsume.attributes["medium"])) + mediaContents += konsume.attributes["url"] + } + + private fun validateItem(item: Item) { + when { + item.title == null -> throw ParseException("Item title can't be null") + item.link == null -> throw ParseException("Item link can't be null") + item.pubDate == null -> throw ParseException("Item date can't be null") + } + } + + companion object { + val names = Names.of("title", "link", "author", "creator", "pubDate", "date", + "guid", "description", "encoded", "enclosure", "content") + } +} \ No newline at end of file From 8e5900833bfa75a6d61cde9eef53ca2fc637deb0 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 13 Sep 2020 12:28:33 +0200 Subject: [PATCH 08/58] Support multiple creator elements in RSS items --- .../assets/localfeed/rss/rss_items_other_namespaces.xml | 7 ++++++- .../com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt | 2 +- .../java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt | 5 ++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_other_namespaces.xml b/api/src/androidTest/assets/localfeed/rss/rss_items_other_namespaces.xml index cde19568..05bb2187 100644 --- a/api/src/androidTest/assets/localfeed/rss/rss_items_other_namespaces.xml +++ b/api/src/androidTest/assets/localfeed/rss/rss_items_other_namespaces.xml @@ -6,7 +6,12 @@ title link guid - + + + + + + 2020-08-05T14:03:48Z diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt index 2358d872..7644cc24 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt @@ -41,7 +41,7 @@ class RSSItemsAdapterTest { val item = adapter.fromXml(stream)[0] assertEquals(item.guid, "guid") - assertEquals(item.author, "creator") + assertEquals(item.author, "creator 1") assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("2020-08-05T14:03:48Z")) assertEquals(item.content, "content:encoded") } diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt index 3a069b19..ea1736dd 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt @@ -18,6 +18,7 @@ class RSSItemsAdapter : XmlAdapter> { allChildrenAutoIgnore("item") { val enclosures = arrayListOf() val mediaContents = arrayListOf() + val creators = arrayListOf() val item = Item().apply { allChildrenAutoIgnore(names) { @@ -25,7 +26,7 @@ class RSSItemsAdapter : XmlAdapter> { "title" -> title = nonNullText() "link" -> link = nonNullText() "author" -> author = nullableText() - "dc:creator" -> author = nullableText() + "dc:creator" -> creators += nullableText() "pubDate" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) "dc:date" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) "guid" -> guid = nullableText() @@ -39,6 +40,8 @@ class RSSItemsAdapter : XmlAdapter> { validateItem(item) if (item.guid == null) item.guid = item.link + if (item.author == null && creators.filterNotNull().isNotEmpty()) + item.author = creators.filterNotNull().first() if (enclosures.isNotEmpty()) item.imageLink = enclosures.first() else if (mediaContents.isNotEmpty()) item.imageLink = mediaContents.first() From 15e1723893ef8ccf016f0166c5eea756877e3f6b Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 13 Sep 2020 12:38:51 +0200 Subject: [PATCH 09/58] Move cleanText method to api package --- .../readrops/api/localfeed/rss/RSSItemsAdapter.kt | 2 +- .../java/com/readrops/api/utils/LibUtils.java | 12 ++++++++++++ .../java/com/readrops/api/utils/LibUtilsTest.kt | 6 ++++++ .../main/java/com/readrops/app/utils/Utils.java | 12 ------------ .../readrops/app/utils/matchers/ItemMatcher.java | 15 ++++++++------- app/src/test/java/com/readrops/app/UtilsTest.java | 8 -------- 6 files changed, 27 insertions(+), 28 deletions(-) diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt index ea1736dd..e2344061 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt @@ -23,7 +23,7 @@ class RSSItemsAdapter : XmlAdapter> { val item = Item().apply { allChildrenAutoIgnore(names) { when (tagName) { - "title" -> title = nonNullText() + "title" -> title = LibUtils.cleanText(nonNullText()) "link" -> link = nonNullText() "author" -> author = nullableText() "dc:creator" -> creators += nullableText() diff --git a/api/src/main/java/com/readrops/api/utils/LibUtils.java b/api/src/main/java/com/readrops/api/utils/LibUtils.java index 88c238d2..cf7a6813 100644 --- a/api/src/main/java/com/readrops/api/utils/LibUtils.java +++ b/api/src/main/java/com/readrops/api/utils/LibUtils.java @@ -6,6 +6,8 @@ import android.net.Uri; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import org.jsoup.Jsoup; + import java.io.FileNotFoundException; import java.io.InputStream; import java.util.Scanner; @@ -61,4 +63,14 @@ public final class LibUtils { return null; } } + + /** + * Remove html tags and trim the text + * + * @param text string to clean + * @return cleaned text + */ + public static String cleanText(String text) { + return Jsoup.parse(text).text().trim(); + } } diff --git a/api/src/test/java/com/readrops/api/utils/LibUtilsTest.kt b/api/src/test/java/com/readrops/api/utils/LibUtilsTest.kt index a6801cae..ba6b8bc4 100644 --- a/api/src/test/java/com/readrops/api/utils/LibUtilsTest.kt +++ b/api/src/test/java/com/readrops/api/utils/LibUtilsTest.kt @@ -16,4 +16,10 @@ class LibUtilsTest { assertEquals(LibUtils.parseContentType("text/xml"), "text/xml") } + + @Test + fun cleanTextTest() { + val text = "

This is a text
to

clean " + assertEquals("This is a text to clean", LibUtils.cleanText(text)) + } } \ No newline at end of file diff --git a/app/src/main/java/com/readrops/app/utils/Utils.java b/app/src/main/java/com/readrops/app/utils/Utils.java index ea220f4e..7da6514a 100644 --- a/app/src/main/java/com/readrops/app/utils/Utils.java +++ b/app/src/main/java/com/readrops/app/utils/Utils.java @@ -17,8 +17,6 @@ import androidx.annotation.NonNull; import com.google.android.material.snackbar.Snackbar; import com.readrops.api.utils.HttpManager; -import org.jsoup.Jsoup; - import java.io.InputStream; import java.util.Locale; @@ -97,16 +95,6 @@ public final class Utils { snackbar.show(); } - /** - * Remove html tags and trim the text - * - * @param text string to clean - * @return cleaned text - */ - public static String cleanText(String text) { - return Jsoup.parse(text).text().trim(); - } - public static Bitmap getBitmapFromDrawable(Drawable drawable) { Bitmap bitmap = Bitmap.createBitmap(drawable.getIntrinsicWidth(), drawable.getIntrinsicHeight(), Bitmap.Config.ARGB_8888); diff --git a/app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java b/app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java index 651fa211..58803743 100644 --- a/app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java +++ b/app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java @@ -1,15 +1,16 @@ package com.readrops.app.utils.matchers; -import com.readrops.api.utils.DateUtils; -import com.readrops.app.utils.Utils; -import com.readrops.db.entities.Feed; -import com.readrops.db.entities.Item; import com.readrops.api.localfeed.atom.ATOMEntry; import com.readrops.api.localfeed.json.JSONItem; import com.readrops.api.localfeed.rss.RSSEnclosure; import com.readrops.api.localfeed.rss.RSSItem; import com.readrops.api.localfeed.rss.RSSMediaContent; +import com.readrops.api.utils.DateUtils; +import com.readrops.api.utils.LibUtils; import com.readrops.api.utils.ParseException; +import com.readrops.app.utils.Utils; +import com.readrops.db.entities.Feed; +import com.readrops.db.entities.Item; import java.util.ArrayList; import java.util.List; @@ -26,7 +27,7 @@ public final class ItemMatcher { newItem.setContent(item.getContent()); // Jsoup.clean(item.getContent(), Whitelist.relaxed()) newItem.setDescription(item.getDescription()); newItem.setGuid(item.getGuid() != null ? item.getGuid() : item.getLink()); - newItem.setTitle(Utils.cleanText(item.getTitle())); + newItem.setTitle(LibUtils.cleanText(item.getTitle())); try { newItem.setPubDate(DateUtils.stringToLocalDateTime(item.getDate())); @@ -72,7 +73,7 @@ public final class ItemMatcher { dbItem.setContent(item.getContent()); // Jsoup.clean(item.getContent(), Whitelist.relaxed()) dbItem.setDescription(item.getSummary()); dbItem.setGuid(item.getId()); - dbItem.setTitle(Utils.cleanText(item.getTitle())); + dbItem.setTitle(LibUtils.cleanText(item.getTitle())); try { dbItem.setPubDate(DateUtils.stringToLocalDateTime(item.getUpdated())); @@ -102,7 +103,7 @@ public final class ItemMatcher { dbItem.setContent(item.getContent()); // Jsoup.clean(item.getContent(), Whitelist.relaxed()) dbItem.setDescription(item.getSummary()); dbItem.setGuid(item.getId()); - dbItem.setTitle(Utils.cleanText(item.getTitle())); + dbItem.setTitle(LibUtils.cleanText(item.getTitle())); try { dbItem.setPubDate(DateUtils.stringToLocalDateTime(item.getPubDate())); diff --git a/app/src/test/java/com/readrops/app/UtilsTest.java b/app/src/test/java/com/readrops/app/UtilsTest.java index b15e86a8..f9c4eb22 100644 --- a/app/src/test/java/com/readrops/app/UtilsTest.java +++ b/app/src/test/java/com/readrops/app/UtilsTest.java @@ -7,17 +7,9 @@ import com.readrops.app.utils.Utils; import org.junit.Test; import static junit.framework.Assert.assertTrue; -import static junit.framework.TestCase.assertEquals; public class UtilsTest { - @Test - public void cleanTextTest() { - String text = "

This is a text
to

clean "; - - assertEquals("This is a text to clean", Utils.cleanText(text)); - } - @Test public void colorTooBrightTest() { assertTrue(Utils.isColorTooBright(-986896)); From 156601e0c5b1ababdf87b013ce5b5867f485e768 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 13 Sep 2020 15:15:42 +0200 Subject: [PATCH 10/58] Use multiple streams based on a byteArray for parsing the body response --- .../api/localfeed/LocalRSSDataSourceTest.kt | 38 +++++++++++++++++-- .../api/localfeed/LocalRSSDataSource.kt | 19 +++++----- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt index aa6b7ee2..60c76b80 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt @@ -5,12 +5,15 @@ import android.content.Context import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry import com.readrops.api.utils.HttpManager +import com.readrops.api.utils.LibUtils import com.readrops.api.utils.ParseException import com.readrops.api.utils.UnknownFormatException import junit.framework.TestCase.* +import okhttp3.Headers import okhttp3.HttpUrl import okhttp3.mockwebserver.MockResponse import okhttp3.mockwebserver.MockWebServer +import okio.Buffer import org.junit.After import org.junit.Before import org.junit.Test @@ -40,15 +43,42 @@ class LocalRSSDataSourceTest { @Test fun successfulQueryTest() { + val stream = context.resources.assets.open("localfeed/rss_feed.xml") + + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader(LibUtils.CONTENT_TYPE_HEADER, "application/xml; charset=UTF-8") + .addHeader(LibUtils.ETAG_HEADER, "ETag-value") + .addHeader(LibUtils.LAST_MODIFIED_HEADER, "Last-Modified") + .setBody(Buffer().readFrom(stream))) + + val pair = localRSSDataSource.queryRSSResource(url.toString(), null, true) + val feed = pair?.first!! + + assertEquals(feed.name, "Hacker News") + assertEquals(feed.siteUrl, "https://news.ycombinator.com/") + assertEquals(feed.description, "Links for the intellectually curious, ranked by readers.") + + assertEquals(feed.etag, "ETag-value") + assertEquals(feed.lastModified, "Last-Modified") + + assertEquals(pair.second.size, 7) + } + + @Test + fun headersTest() { + val stream = context.resources.assets.open("localfeed/rss_feed.xml") + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) .addHeader("Content-Type", "application/rss+xml; charset=UTF-8") - .setBody(context.resources.assets.open("localfeed/rss_feed.xml").toString())) + .setBody(Buffer().readFrom(stream))) + val headers = Headers.headersOf(LibUtils.ETAG_HEADER, "ETag", LibUtils.LAST_MODIFIED_HEADER, "Last-Modified") + localRSSDataSource.queryRSSResource(url.toString(), headers, false) - val pair = localRSSDataSource.queryRSSResource(url.toString(), null, false) + val request = mockServer.takeRequest() - assertNotNull(pair?.first) - assertNotNull(pair?.second) + assertEquals(request.headers[LibUtils.ETAG_HEADER], "ETag") + assertEquals(request.headers[LibUtils.LAST_MODIFIED_HEADER], "Last-Modified") } @Test diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index 32ce19c4..817bc126 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -11,6 +11,7 @@ import okhttp3.Headers import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response +import java.io.ByteArrayInputStream import java.io.IOException import java.io.InputStream import java.net.HttpURLConnection @@ -38,14 +39,16 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { var type = LocalRSSHelper.getRSSType(contentType) + val bodyArray = response.peekBody(Long.MAX_VALUE).bytes() + // if we can't guess type based on content-type header, we use the content if (type == LocalRSSHelper.RSSType.UNKNOWN) - type = LocalRSSHelper.getRSSContentType(response.body?.byteStream()!!) + type = LocalRSSHelper.getRSSContentType(ByteArrayInputStream(bodyArray)) // if we can't guess type even with the content, we are unable to go further if (type == LocalRSSHelper.RSSType.UNKNOWN) throw UnknownFormatException("Unable to guess $url RSS type") - val feed = parseFeed(response, type) - val items = if (withItems) parseItems(response.body?.byteStream()!!, type) else listOf() + val feed = parseFeed(ByteArrayInputStream(bodyArray), type, response) + val items = if (withItems) parseItems(ByteArrayInputStream(bodyArray), type) else listOf() response.body?.close() Pair(feed, items) @@ -80,18 +83,17 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { return httpClient.newCall(requestBuilder.build()).execute() } - private fun parseFeed(response: Response, type: LocalRSSHelper.RSSType): Feed { + private fun parseFeed(stream: InputStream, type: LocalRSSHelper.RSSType, response: Response): Feed { val feed = if (type != LocalRSSHelper.RSSType.JSONFEED) { val adapter = XmlAdapter.xmlFeedAdapterFactory(type) - //adapter.fromXml(response.body?.byteStream()!!) - Feed() + adapter.fromXml(stream) } else { Feed() } feed.etag = response.header(LibUtils.ETAG_HEADER) - feed.lastModified = response.header(LibUtils.IF_MODIFIED_HEADER) + feed.lastModified = response.header(LibUtils.LAST_MODIFIED_HEADER) return feed } @@ -100,8 +102,7 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { return if (type != LocalRSSHelper.RSSType.JSONFEED) { val adapter = XmlAdapter.xmlItemsAdapterFactory(type) - //adapter.fromXml(inputStream) - listOf() + adapter.fromXml(inputStream) } else { listOf() } From f1bf65d629d1209939f045e48f72a6d4c04fbf88 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 13 Sep 2020 15:19:41 +0200 Subject: [PATCH 11/58] Trigger workflow for every branch --- .github/workflows/android.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index a00b8662..a1070e53 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -3,12 +3,10 @@ name: Android CI on: push: branches: - - master - - develop + - '**' pull_request: branches: - - master - - develop + - '**' jobs: build: From 8b4b7b0f1c4464bb30b3a5310fc94cd0927b90b3 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 13 Sep 2020 15:33:06 +0200 Subject: [PATCH 12/58] Fix opml parser tests --- api/src/androidTest/java/com/readrops/api/OPMLParserTest.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/OPMLParserTest.kt b/api/src/androidTest/java/com/readrops/api/OPMLParserTest.kt index b3e4e091..33da88f0 100644 --- a/api/src/androidTest/java/com/readrops/api/OPMLParserTest.kt +++ b/api/src/androidTest/java/com/readrops/api/OPMLParserTest.kt @@ -29,7 +29,7 @@ class OPMLParserTest { @Test fun readOpmlTest() { - val stream = context.resources.assets.open("subscriptions.opml") + val stream = context.resources.assets.open("opml/subscriptions.opml") var foldersAndFeeds: Map>? = null @@ -52,7 +52,7 @@ class OPMLParserTest { @Test fun readLiteSubscriptionsTest() { - val stream = context.resources.assets.open("lite_subscriptions.opml") + val stream = context.resources.assets.open("opml/lite_subscriptions.opml") var foldersAndFeeds: Map>? = null @@ -68,7 +68,7 @@ class OPMLParserTest { @Test fun opmlVersionTest() { - val stream = context.resources.assets.open("wrong_version.opml") + val stream = context.resources.assets.open("opml/wrong_version.opml") OPMLParser.read(stream) .test() From 5c342a45b6ba25975c34c101740363594a400884 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Mon, 14 Sep 2020 17:56:25 +0200 Subject: [PATCH 13/58] Add adapter for parsing atom feed --- .../assets/localfeed/atom/atom_feed.xml | 9 ++++ .../api/localfeed/atom/ATOMFeedAdapterTest.kt | 28 ++++++++++ .../com/readrops/api/localfeed/XmlAdapter.kt | 2 + .../api/localfeed/atom/ATOMFeedAdapter.kt | 52 +++++++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 api/src/androidTest/assets/localfeed/atom/atom_feed.xml create mode 100644 api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMFeedAdapterTest.kt create mode 100644 api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeedAdapter.kt diff --git a/api/src/androidTest/assets/localfeed/atom/atom_feed.xml b/api/src/androidTest/assets/localfeed/atom/atom_feed.xml new file mode 100644 index 00000000..797a7917 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/atom/atom_feed.xml @@ -0,0 +1,9 @@ + + + tag:github.com,2008:/readrops/Readrops/commits/develop + + + Recent Commits to Readrops:develop + 2020-09-06T21:09:59Z + Here is a subtitle + \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMFeedAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMFeedAdapterTest.kt new file mode 100644 index 00000000..57711b22 --- /dev/null +++ b/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMFeedAdapterTest.kt @@ -0,0 +1,28 @@ +package com.readrops.api.localfeed.atom + +import android.content.Context +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import junit.framework.TestCase.assertEquals +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class ATOMFeedAdapterTest { + + private val context: Context = InstrumentationRegistry.getInstrumentation().context + + private val adapter = ATOMFeedAdapter() + + @Test + fun normalCasesTest() { + val stream = context.assets.open("localfeed/atom/atom_feed.xml") + + val feed = adapter.fromXml(stream) + + assertEquals(feed.name, "Recent Commits to Readrops:develop") + assertEquals(feed.url, "https://github.com/readrops/Readrops/commits/develop.atom") + assertEquals(feed.siteUrl, "https://github.com/readrops/Readrops/commits/develop") + assertEquals(feed.description, "Here is a subtitle") + } +} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt index 874d2944..c41a4478 100644 --- a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt @@ -1,5 +1,6 @@ package com.readrops.api.localfeed +import com.readrops.api.localfeed.atom.ATOMFeedAdapter import com.readrops.api.localfeed.rss.RSSFeedAdapter import com.readrops.api.localfeed.rss.RSSItemsAdapter import com.readrops.db.entities.Feed @@ -14,6 +15,7 @@ interface XmlAdapter { fun xmlFeedAdapterFactory(type: LocalRSSHelper.RSSType): XmlAdapter { return when (type) { LocalRSSHelper.RSSType.RSS_2 -> RSSFeedAdapter() + LocalRSSHelper.RSSType.ATOM -> ATOMFeedAdapter() else -> throw Exception("Unknown RSS type : $type") } } diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeedAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeedAdapter.kt new file mode 100644 index 00000000..e53a4bf2 --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeedAdapter.kt @@ -0,0 +1,52 @@ +package com.readrops.api.localfeed.atom + +import com.gitlab.mvysny.konsumexml.Konsumer +import com.gitlab.mvysny.konsumexml.Names +import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore +import com.gitlab.mvysny.konsumexml.konsumeXml +import com.readrops.api.localfeed.XmlAdapter +import com.readrops.api.utils.ParseException +import com.readrops.api.utils.nonNullText +import com.readrops.api.utils.nullableText +import com.readrops.db.entities.Feed +import java.io.InputStream + +class ATOMFeedAdapter : XmlAdapter { + + override fun fromXml(inputStream: InputStream): Feed { + val konsume = inputStream.konsumeXml() + val feed = Feed() + + return try { + konsume.child("feed") { + allChildrenAutoIgnore(names) { + with(feed) { + when (tagName) { + "title" -> name = nonNullText() + "link" -> parseLink(this@allChildrenAutoIgnore, feed) + "subtitle" -> description = nullableText() + } + } + } + } + + konsume.close() + feed + } catch (e: Exception) { + throw ParseException(e.message) + } + } + + private fun parseLink(konsume: Konsumer, feed: Feed) { + val rel = konsume.attributes["rel"] + + if (rel == "self") + feed.url = konsume.attributes["href"] + else if (rel == "alternate") + feed.siteUrl = konsume.attributes["href"] + } + + companion object { + val names = Names.of("title", "link", "subtitle") + } +} \ No newline at end of file From 0c48bd2adb1f0587a1112ac1cae2c70ecb7c56e1 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Mon, 14 Sep 2020 19:41:10 +0200 Subject: [PATCH 14/58] Add adapter for parsing atom entries --- .../assets/localfeed/atom/atom_items.xml | 71 +++++++++++++++++++ .../localfeed/atom/atom_items_no_date.xml | 22 ++++++ .../localfeed/atom/atom_items_no_link.xml | 22 ++++++ .../localfeed/atom/atom_items_no_title.xml | 22 ++++++ .../localfeed/atom/ATOMItemsAdapterTest.kt | 58 +++++++++++++++ .../com/readrops/api/localfeed/XmlAdapter.kt | 2 + .../api/localfeed/atom/ATOMItemsAdapter.kt | 62 ++++++++++++++++ 7 files changed, 259 insertions(+) create mode 100644 api/src/androidTest/assets/localfeed/atom/atom_items.xml create mode 100644 api/src/androidTest/assets/localfeed/atom/atom_items_no_date.xml create mode 100644 api/src/androidTest/assets/localfeed/atom/atom_items_no_link.xml create mode 100644 api/src/androidTest/assets/localfeed/atom/atom_items_no_title.xml create mode 100644 api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt create mode 100644 api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt diff --git a/api/src/androidTest/assets/localfeed/atom/atom_items.xml b/api/src/androidTest/assets/localfeed/atom/atom_items.xml new file mode 100644 index 00000000..107a0950 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/atom/atom_items.xml @@ -0,0 +1,71 @@ + + + tag:github.com,2008:/readrops/Readrops/commits/develop + + + Recent Commits to Readrops:develop + 2020-09-06T21:09:59Z + + tag:github.com,2008:Grit::Commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac + + Add an option to open item url in custom tab + 2020-09-06T21:09:59Z + + + Shinokuni + https://github.com/Shinokuni + + Summary + + <pre style='white-space:pre-wrap;width:81ex'>Add an option to open item url in custom tab</pre> + + + + tag:github.com,2008:Grit::Commit/e0945823eecf269e5beea646ac5d7e630e08afbf + + + Use gradle parallel builds + + 2020-09-05T13:28:23Z + + + Shinokuni + https://github.com/Shinokuni + + + <pre style='white-space:pre-wrap;width:81ex'>Use gradle parallel builds</pre> + + + + tag:github.com,2008:Grit::Commit/85fcf03e64d8b482e4d2af8c2bcd1509d946944f + + + Use clear text mode for the feed url text input in AddFeedActivity + + 2020-09-05T12:23:48Z + + + Shinokuni + https://github.com/Shinokuni + + + <pre style='white-space:pre-wrap;width:81ex'>Use clear text mode for the feed url text input in AddFeedActivity</pre> + + + + tag:github.com,2008:Grit::Commit/d59e38ee9d11da186131b602425231eff0896956 + + + Use project level okhttp client with glide + + 2020-09-05T12:05:16Z + + + Shinokuni + https://github.com/Shinokuni + + + <pre style='white-space:pre-wrap;width:81ex'>Use project level okhttp client with glide</pre> + + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/atom/atom_items_no_date.xml b/api/src/androidTest/assets/localfeed/atom/atom_items_no_date.xml new file mode 100644 index 00000000..08ae5692 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/atom/atom_items_no_date.xml @@ -0,0 +1,22 @@ + + + tag:github.com,2008:/readrops/Readrops/commits/develop + + + Recent Commits to Readrops:develop + 2020-09-06T21:09:59Z + + tag:github.com,2008:Grit::Commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac + Add an option to open item url in custom tab + + + + Shinokuni + https://github.com/Shinokuni + + Summary + + <pre style='white-space:pre-wrap;width:81ex'>Add an option to open item url in custom tab</pre> + + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/atom/atom_items_no_link.xml b/api/src/androidTest/assets/localfeed/atom/atom_items_no_link.xml new file mode 100644 index 00000000..c0d817d8 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/atom/atom_items_no_link.xml @@ -0,0 +1,22 @@ + + + tag:github.com,2008:/readrops/Readrops/commits/develop + + + Recent Commits to Readrops:develop + 2020-09-06T21:09:59Z + + tag:github.com,2008:Grit::Commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac + Add an option to open item url in custom tab + 2020-09-06T21:09:59Z + + + Shinokuni + https://github.com/Shinokuni + + Summary + + <pre style='white-space:pre-wrap;width:81ex'>Add an option to open item url in custom tab</pre> + + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/atom/atom_items_no_title.xml b/api/src/androidTest/assets/localfeed/atom/atom_items_no_title.xml new file mode 100644 index 00000000..8bf4adbd --- /dev/null +++ b/api/src/androidTest/assets/localfeed/atom/atom_items_no_title.xml @@ -0,0 +1,22 @@ + + + tag:github.com,2008:/readrops/Readrops/commits/develop + + + Recent Commits to Readrops:develop + 2020-09-06T21:09:59Z + + tag:github.com,2008:Grit::Commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac + + 2020-09-06T21:09:59Z + + + Shinokuni + https://github.com/Shinokuni + + Summary + + <pre style='white-space:pre-wrap;width:81ex'>Add an option to open item url in custom tab</pre> + + + \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt new file mode 100644 index 00000000..f6095a97 --- /dev/null +++ b/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt @@ -0,0 +1,58 @@ +package com.readrops.api.localfeed.atom + +import android.content.Context +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.readrops.api.utils.DateUtils +import com.readrops.api.utils.ParseException +import junit.framework.TestCase.assertEquals +import junit.framework.TestCase.assertNotNull +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class ATOMItemsAdapterTest { + + private val context: Context = InstrumentationRegistry.getInstrumentation().context + + private val adapter = ATOMItemsAdapter() + + @Test + fun normalCasesTest() { + val stream = context.resources.assets.open("localfeed/atom/atom_items.xml") + + val items = adapter.fromXml(stream) + val item = items[0] + + assertEquals(items.size, 4) + assertEquals(item.title, "Add an option to open item url in custom tab") + assertEquals(item.link, "https://github.com/readrops/Readrops/commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac") + assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("2020-09-06T21:09:59Z")) + assertEquals(item.author, "Shinokuni") + assertEquals(item.description, "Summary") + assertEquals(item.guid, "tag:github.com,2008:Grit::Commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac") + assertNotNull(item.content) + } + + @Test + fun noTitleTest() { + val stream = context.resources.assets.open("localfeed/atom/atom_items_no_title.xml") + + Assert.assertThrows("Item title is required", ParseException::class.java) { adapter.fromXml(stream) } + } + + @Test + fun noLinkTest() { + val stream = context.resources.assets.open("localfeed/atom/atom_items_no_link.xml") + + Assert.assertThrows("Item link is required", ParseException::class.java) { adapter.fromXml(stream) } + } + + @Test + fun noDateTest() { + val stream = context.resources.assets.open("localfeed/atom/atom_items_no_date.xml") + + Assert.assertThrows("Item date is required", ParseException::class.java) { adapter.fromXml(stream) } + } +} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt index c41a4478..b7f39d64 100644 --- a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt @@ -1,6 +1,7 @@ package com.readrops.api.localfeed import com.readrops.api.localfeed.atom.ATOMFeedAdapter +import com.readrops.api.localfeed.atom.ATOMItemsAdapter import com.readrops.api.localfeed.rss.RSSFeedAdapter import com.readrops.api.localfeed.rss.RSSItemsAdapter import com.readrops.db.entities.Feed @@ -23,6 +24,7 @@ interface XmlAdapter { fun xmlItemsAdapterFactory(type: LocalRSSHelper.RSSType): XmlAdapter> { return when (type) { LocalRSSHelper.RSSType.RSS_2 -> RSSItemsAdapter() + LocalRSSHelper.RSSType.ATOM -> ATOMItemsAdapter() else -> throw Exception("Unknown RSS type : $type") } } diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt new file mode 100644 index 00000000..db7fe178 --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt @@ -0,0 +1,62 @@ +package com.readrops.api.localfeed.atom + +import com.gitlab.mvysny.konsumexml.Names +import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore +import com.gitlab.mvysny.konsumexml.konsumeXml +import com.readrops.api.localfeed.XmlAdapter +import com.readrops.api.utils.DateUtils +import com.readrops.api.utils.ParseException +import com.readrops.api.utils.nonNullText +import com.readrops.api.utils.nullableText +import com.readrops.db.entities.Item +import java.io.InputStream + +class ATOMItemsAdapter : XmlAdapter> { + + override fun fromXml(inputStream: InputStream): List { + val konsume = inputStream.konsumeXml() + val items = arrayListOf() + + return try { + konsume.child("feed") { + allChildrenAutoIgnore("entry") { + val item = Item().apply { + allChildrenAutoIgnore(names) { + when (tagName) { + "title" -> title = nonNullText() + "id" -> guid = nullableText() + "updated" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) + "link" -> if (attributes["rel"] == "alternate") link = attributes["href"] + "author" -> allChildrenAutoIgnore("name") { author = text() } + "summary" -> description = nullableText() + "content" -> content = nullableText() + } + } + } + + validateItem(item) + if (item.guid == null) item.guid = item.link + + items += item + } + } + + konsume.close() + items + } catch (e: Exception) { + throw ParseException(e.message) + } + } + + private fun validateItem(item: Item) { + when { + item.title == null -> throw ParseException("Item title is required") + item.link == null -> throw ParseException("Item link is required") + item.pubDate == null -> throw ParseException("Item date id required") + } + } + + companion object { + val names = Names.of("title", "id", "updated", "link", "author", "summary", "content") + } +} \ No newline at end of file From c9c316b5795b01a66558b13431408766365a10e6 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 15 Sep 2020 13:48:52 +0200 Subject: [PATCH 15/58] Add adapter for parsing jsonfeed feed --- .../assets/localfeed/json/json_feed.json | 82 +++++++++++++++++++ .../api/localfeed/json/JSONFeedAdapterTest.kt | 35 ++++++++ .../api/localfeed/json/JSONFeedAdapter.kt | 39 +++++++++ 3 files changed, 156 insertions(+) create mode 100644 api/src/androidTest/assets/localfeed/json/json_feed.json create mode 100644 api/src/androidTest/java/com/readrops/api/localfeed/json/JSONFeedAdapterTest.kt create mode 100644 api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt diff --git a/api/src/androidTest/assets/localfeed/json/json_feed.json b/api/src/androidTest/assets/localfeed/json/json_feed.json new file mode 100644 index 00000000..35ca0e3b --- /dev/null +++ b/api/src/androidTest/assets/localfeed/json/json_feed.json @@ -0,0 +1,82 @@ +{ + "version": "https://jsonfeed.org/version/1", + "title": "News from Flying Meat", + "home_page_url": "http://flyingmeat.com/blog/", + "feed_url": "http://flyingmeat.com/blog/feed.json", + "description": "News from your friends at Flying Meat.", + "author": { + "name": "Gus Mueller" + }, + "items": [ + { + "id": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html", + "title": "Acorn and 10.13", + "content_html": "

Happy Mac OS High Sierra release day everyone.

\n

I'm happy to say that there are no known issues with Acorn 6.0.3 or Acorn 5.6.6 when running on Mac OS 10.13 High Sierra. In fact, you might even notice that some things are actually faster and it can now open HEIF images. How awesome is that?

\n

I'm also working on some 10.13 goodies for Acorn 6 folks later this year. I can't wait to share that with you, but you'll have to wait just a little bit.

\n", + "date_published": "2017-09-25T14:27:27-07:00", + "url": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html" + }, + { + "id": "http://flyingmeat.com/blog/archives/2018/2/acorn_6.1_is_out.html", + "title": "Acorn 6.1 Is Out", + "content_html": "

Acorn 6.1 has been released.

\n

You can read a longer post about it over on Gus's blog, but the short of it is: Better, faster, smoother, stronger. And now with Metal 2 support.

\n", + "date_published": "2018-02-16T09:59:11-08:00", + "url": "http://flyingmeat.com/blog/archives/2018/2/acorn_6.1_is_out.html" + }, + { + "id": "http://flyingmeat.com/blog/archives/2018/6/a_pair_of_updates.html", + "title": "A Pair of Updates", + "content_html": "

Happy summer solstice everybody! (at least for folks in the northern hemisphere, and for folks in the south… sorry. It's going to start getting brighter for you though).

\n

Today I've got a pair of minor app updates to annouce for you.

\n

First up is Acorn 6.1.3, which fixes a number of bugs including one that stemmed from trying to use QuickLook on a file that was created with Acorn 1.0. For the one or two of you that this was affecting, hurray!

\n

Next up is Retrobatch, which also includes some bug fixes, the beginnings of Voice Over support, performance improvements, and more.

\n

What's next for these apps? Work on Acorn 6.2 will begin shortly, as will Retrobatch 1.1. WWDC introduced some great new APIs that I want to take advantage of (cool new machine learning things), so that'll be a focus- as well as Dark Mode for Acorn and one other major thing I've got planned. Retrobatch will probably also get the Dark Mode treatment, but not until I've done it for Acorn first.

\n

So it's going to be a busy summer, but I'm looking forward to it.

\n", + "date_published": "2018-06-21T10:18:46-07:00", + "url": "http://flyingmeat.com/blog/archives/2018/6/a_pair_of_updates.html" + }, + { + "id": "http://flyingmeat.com/blog/archives/2018/9/retrobatch_1.1_is_out.html", + "title": "Retrobatch 1.1 Is Out", + "content_html": "

Here's something new for your lazy August September* morning: Retrobatch 1.1 is out.

\n

What's new and awesome? Well, Retrobatch now has some great scripting goodness in the form of a new Automator action which will run a workflow for you (and create Automator droplets), a new JavaScript node*, and the ability to run Retrobatch workflows from the terminal.

\n

We've added a handful of new nodes such as Dither, Auto Enhance, Instant Alpha, and Color Posterize. New options to existing nodes have also shown up, such as "Only scale smaller" for the Scale node.

\n

And an interesting idea that I've had folks ask about a number of times- it's now possible to run an image through a machine learning classifier, and then have the classification written to metadata such as the image title, or keywords. This was done by adding token support to the Set Specific Metadata node. This also means you can use other tokens such as the Current Year in metadata fields. Awesome? We think so.

\n

The full release notes are available, and if you have ideas or questions- make sure to poke around on the forums or write us: support@flyingmeat.com. We've got lots of ideas for future releases, but if you'd like something specific in there make sure to let us know.

\n
\n\n
\n* Whoa, it's September already?

\n\n

**I'm calling the JavaScript node a "preview". It works very well, but I'm not 100% sold on the API that I've provided to folks. So this is a disclaimer that it might change a little bit in the future.

\n", + "date_published": "2018-09-07T09:43:10-07:00", + "url": "http://flyingmeat.com/blog/archives/2018/9/retrobatch_1.1_is_out.html" + }, + { + "id": "http://flyingmeat.com/blog/archives/2018/9/acorn_6.2_with_mojave_dark_mode_is_out.html", + "title": "Acorn 6.2 With Mojave Dark Mode Is Out", + "content_html": "

On Monday I flipped some switches on the FM servers and Acorn 6.2 was released to the universe. You might also remember that Monday a little known operating system from Apple was updated, which includes a neat new feature known as Dark Mode.

\n
\n\n

I think Acorn looks pretty good in Dark Aqua, especially the icon refresh from Matthew Skiles.

\n

To celebrate the new release, we've put Acorn on sale for 50% off. So go grab it at the insanely low price of $14.99. If you haven't already upgraded from previous versions of Acorn, now is a good time to do so.

\n

We've also packed a bunch of little changes, bug fixes, and compatibility with Mojave in there. And of course, there's more to come in the future as always.

\n", + "date_published": "2018-09-27T14:37:21-07:00", + "url": "http://flyingmeat.com/blog/archives/2018/9/acorn_6.2_with_mojave_dark_mode_is_out.html" + }, + { + "id": "http://flyingmeat.com/blog/archives/2019/1/acorn_6.3_is_out.html", + "title": "Acorn 6.3 Is Out", + "content_html": "

Acorn 6.3 is available, and the full release notes are up as well.

\n

Here's what I think is awesome in this release:

\n

Portrait Mask Support. If you have an iPhone running iOS 12 (and can take Portrait photos), Acorn will now detect the Portrait Matte from those images and turn it into a layer mask. The Portrait Matte is the image data which enables blurring in the background, or other fancy camera tricks. This means you can use this matte to erase and add fancy backgrounds or custom blurs for your image, all within Acorn.

\n

Other Mask Features. You can now drag and drop masks from the layers list into another layer, or copy it out as a new layer. When exporting layers you now have an option to apply the mask on export, or just write it as an additional image along with everything else. There are a number of new shortcuts when dealing with layer masks as well.

\n

Brush Stuff. If you're running MacOS 10.13 or later, you get a performance boost when brushing (painting, smudging, cloning, etc…). This is especially noticeable when brusing on deep color images.

\n

I've also added options to the brush palette for adjusting flow, softness and blending. In addition to all this, there's a bunch of new brushes under the "Basic Round" category which are designed for the new brush engine.

\n

Other Stuff. There's other good things including improved PDF export, various MacOS Mojave UI fixes, additional speed improvements with with deep images, and more. And as always, it's a free upgrade for anyone who has already purchased Acorn 6.

\n", + "date_published": "2019-01-09T13:00:07-08:00", + "url": "http://flyingmeat.com/blog/archives/2019/1/acorn_6.3_is_out.html" + }, + { + "id": "http://flyingmeat.com/blog/archives/2019/4/retrobatch_1.2_released.html", + "title": "Retrobatch 1.2 Released", + "content_html": "

We're happy to announce that Retrobatch 1.2 has now been released, which is a free update for all owners of Retrobatch. Highlights of this release include:

\n
    \n

  • Create animated GIF and PNG images with the Animated Image node. When using Retrobatch you can load in a folder of images and produce an optimized animated image with options for setting the frame rate, format, as well as letting the image loop or not.

    \n
    • \n

  • New nodes including "Round Corner", "Image Grid", and "Limit". We've also added improvements to the Write node allowing you to write back to the original processed image.

    \n
    • \n

  • Droplet support (Retrobatch Pro). Turn your workflow into an an application which you can drag and drop images onto. The droplet can work anywhere an application normally would, even in the Dock.

    \n
    • \n

  • Write Plug-Ins using JavaScript (Retrobatch Pro). Using the combined power of JavaScript and the native to MacOS Cocoa APIs, you can make and distribute new plugins for Retrobatch. Got an idea for a plug-in and you want to use Core Image to make it? Or maybe you want to use Core Graphics to add some funky text to your images? Now you can do this with JavaScript and Cocoa.

    \n
    • \n
\n

The full release notes are available, as well as information on bug fixes we delivered in this update.

\n

As always, we're always listening for feedback and feature requests. And don't forget to head over to the Retrobatch community formus to chat with us and other Retrobatch users.

\n", + "date_published": "2019-04-01T13:38:21-07:00", + "url": "http://flyingmeat.com/blog/archives/2019/4/retrobatch_1.2_released.html" + }, + { + "id": "http://flyingmeat.com/blog/archives/2019/10/catalina_ready.html", + "title": "Catalina Ready", + "content_html": "

MacOS 10.15 Catalina was just released, and we're happy to let you know that both Acorn 6.5.1 and Retrobatch 1.2 are compatible with it.

\n

And to celebrate the release of Catalina, we're discounting Acorn by 50% for a limited time. So if you haven't upgraded yet, now is a good time.

\n", + "date_published": "2019-10-07T10:48:03-07:00", + "url": "http://flyingmeat.com/blog/archives/2019/10/catalina_ready.html" + }, + { + "id": "http://flyingmeat.com/blog/archives/2020/3/retrobatch_1.4_is_out.html", + "title": "Retrobatch 1.4 Is Out", + "content_html": "

I've just typed the magic commands* and let the servers do their thing and now Retrobatch 1.4 is loose on the world.

\n

There's a couple of interesting new features in this update I'd like to call out. First up is JavaScript expressions in Retrobatch Pro. Various nodes in Retrobatch which allow you to set the size or length of a value (such as the Crop, Border, Gradient, Adjust Margin nodes) now have an option of running a little snippet of JavaScript code to figure out the value. This is a super powerful feature, which you can read about in our JavaScript Expressions documentation

\n

Let's say you have some images of varying sizes, which are all at 480 x 380 or smaller, and you want them to expand to meet that size. But- you only want it to grow evenly on either side of the image, but you want to keep a baseline so only transparent area is added to the top of the image, and the bottom stays in the same spot. This little picture of the new Adjust Margins node shows how this can be done:

\n
\n\n

Yes, this is an oddball (and very real) case- but there's a billion of these little oddball cases out there. With the new JavaScript expressions support, these small but hard to do scenarios are now super easy.

\n

And yes, all of the JavaScript support in Retrobatch now sits atop FMJS, which any developer can use to build similar support into their apps.

\n

What else is new?

\n

File numbers with leading zeros for the Write node. You can add (and it's case sensitive) $FileNumber04$ in the File name: field of the Write node to have the file number of your image written out as part of the name, with a padding of up to 4 zeros. If you'd like to pad that number to 6, you would enter $FileNumber06$, and so on.

\n

The Mask to Alpha node got a new "invert colors" option. Normally Mask to Alpha will convert the black areas of your image to transparent, and the white to opaque (with gray somewhere inbetween). With the new Invert Colors option, Mask to Alpha will now convert the white areas of your image to transparent, and keep the black opaque. This is great if you are scanning in line drawings from your own artwork, and want to make the backgrounds transparent.

\n

This request comes up a lot in Acorn as well. Previously you'd have to add an Invert Colors node (or filter for Acorn), then the Mask to Alpha, and then Invert Colors again. Now it's just a checkbox in Mask to Alpha, which is super easy. I've also added an update to the same filter in Acorn for the next release. You can grab a preview of it from here.

\n

And finally for my short list, you can now make a droplet which doesn't take any files. Why is this useful? Well, imagine you have a workflow that reads an image from the clipboard, resizes it to a specific width, and then writes it back to the clipboard. Now you can make a little droplet to do just this. Just a double click from the Finder (or a single click from the Dock) and your workflow is run.

\n

The full release notes for Retrobatch 1.4 are available in the usual place.

\n

* ./bin/otbuild.sh -e 1.4

\n", + "date_published": "2020-03-31T14:37:15-07:00", + "url": "http://flyingmeat.com/blog/archives/2020/3/retrobatch_1.4_is_out.html" + }, + { + "id": "http://flyingmeat.com/blog/archives/2020/5/acorn_6.6_released.html", + "title": "Acorn 6.6 is out with new Shape Processors and more", + "content_html": "

Acorn 6.6 is out. You can update to this release via the App Store as or the Acorn ▸ Check for Updates… menu if you bought it directly from us.

\n

Originally this was going to be a bug fix release but I kept on adding useful things and it snowballed into a feature release. As usual, the full release notes have all the details about what was updated.

\n

The main new features are with the Shape Processor. If you're not already familiar with the shape processor, it's a neat ability Acorn has to take shapes on vector layers and pipe them through a series of actions, similar to how Automator or Acorn's bitmap filters work. Only instead of working on pixels, the processors will alter the shapes by scaling them or moving them around, or changing colors or blend modes. There's even a processor which will generate shapes for you- so if you want your canvas to fill up with hundreds of stars, you can do that.

\n

Acorn 6.6 adds new processors which let you set the stroke, fill, and blend mode of your processed shapes. You can now also flip your shapes and even shift colors.

\n\n\n

Chaining these processors together can get you some neat looking images. You can make interesting desktop backgrounds, as well as textures for your photos. Or if you just need a bunch of hexagons arranged in a circle, that's just two processors stacked together.

\n

Have you made something interesting with the Shape Processor? I'd love to see it either via Twitter (I'm @ccgus) or via email.

\n

There are of course the usual bug fixes and other minor details. And if you don't already have Acorn, a no-strings attached free trial is available on our website. Try it out, and we're always looking to hear from you about feature requests, thoughts, and anything else.

\n", + "date_published": "2020-05-28T12:17:57-07:00", + "url": "http://flyingmeat.com/blog/archives/2020/5/acorn_6.6_released.html" + } + ] +} \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONFeedAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONFeedAdapterTest.kt new file mode 100644 index 00000000..def7cd0c --- /dev/null +++ b/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONFeedAdapterTest.kt @@ -0,0 +1,35 @@ +package com.readrops.api.localfeed.json + +import android.content.Context +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.readrops.db.entities.Feed +import com.squareup.moshi.Moshi +import junit.framework.TestCase.assertEquals +import okio.Buffer +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class JSONFeedAdapterTest { + + private val context: Context = InstrumentationRegistry.getInstrumentation().context + + private val adapter = Moshi.Builder() + .add(JSONFeedAdapter()) + .build() + .adapter(Feed::class.java) + + @Test + fun normalCasesTest() { + val stream = context.assets.open("localfeed/json/json_feed.json") + + val feed = adapter.fromJson(Buffer().readFrom(stream))!! + + assertEquals(feed.name, "News from Flying Meat") + assertEquals(feed.url, "http://flyingmeat.com/blog/feed.json") + assertEquals(feed.siteUrl, "http://flyingmeat.com/blog/") + assertEquals(feed.description, "News from your friends at Flying Meat.") + } + +} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt new file mode 100644 index 00000000..4358f315 --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt @@ -0,0 +1,39 @@ +package com.readrops.api.localfeed.json + +import com.readrops.api.utils.nextNullableString +import com.readrops.db.entities.Feed +import com.squareup.moshi.FromJson +import com.squareup.moshi.JsonReader +import com.squareup.moshi.ToJson + +class JSONFeedAdapter { + + @ToJson + fun toJson(feed: Feed) = "" + + @FromJson + fun fromJson(reader: JsonReader): Feed { + val feed = Feed() + reader.beginObject() + + while (reader.hasNext()) { + with(feed) { + when (reader.selectName(names)) { + 0 -> name = reader.nextString() + 1 -> siteUrl = reader.nextNullableString() + 2 -> url = reader.nextNullableString() + 3 -> description = reader.nextNullableString() + else -> reader.skipValue() + } + } + } + + reader.endObject() + return feed + } + + companion object { + val names: JsonReader.Options = JsonReader.Options.of("title", "home_page_url", + "feed_url", "description") + } +} \ No newline at end of file From 592fa4603aa79bd75bf9013545bc273ade11dc77 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 15 Sep 2020 13:51:17 +0200 Subject: [PATCH 16/58] Add support for jsonfeed new mime-type : application/feed+json --- api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt | 3 ++- .../test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt index 3d4a42b5..13894e6a 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt @@ -10,6 +10,7 @@ object LocalRSSHelper { private const val RSS_TEXT_CONTENT_TYPE = "text/xml" private const val RSS_APPLICATION_CONTENT_TYPE = "application/xml" private const val ATOM_CONTENT_TYPE = "application/atom+xml" + private const val JSONFEED_CONTENT_TYPE = "application/feed+json" private const val JSON_CONTENT_TYPE = "application/json" private const val HTML_CONTENT_TYPE = "text/html" @@ -24,7 +25,7 @@ object LocalRSSHelper { return when (contentType) { RSS_DEFAULT_CONTENT_TYPE -> RSSType.RSS_2 ATOM_CONTENT_TYPE -> RSSType.ATOM - JSON_CONTENT_TYPE -> RSSType.JSONFEED + JSON_CONTENT_TYPE, JSONFEED_CONTENT_TYPE -> RSSType.JSONFEED RSS_TEXT_CONTENT_TYPE, RSS_APPLICATION_CONTENT_TYPE, HTML_CONTENT_TYPE -> RSSType.UNKNOWN else -> throw UnknownFormatException("Unknown content type : $contentType") } diff --git a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt index c35c9e5e..bec3cbf1 100644 --- a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt +++ b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt @@ -15,6 +15,8 @@ class LocalRSSHelperTest { LocalRSSHelper.RSSType.ATOM) assertEquals(LocalRSSHelper.getRSSType("application/json"), LocalRSSHelper.RSSType.JSONFEED) + assertEquals(LocalRSSHelper.getRSSType("application/feed+json"), + LocalRSSHelper.RSSType.JSONFEED) } @Test From ca29c4acb06feb91caa7c4b9abbbdfedad3cbf57 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 15 Sep 2020 23:30:56 +0200 Subject: [PATCH 17/58] Add adapter for jsonfeed items --- .../assets/localfeed/json/json_feed.json | 6 +- .../json/json_items_other_cases.json | 28 +++++ .../json/json_items_required_elements.json | 22 ++++ .../localfeed/json/JSONItemsAdapterTest.kt | 77 ++++++++++++ .../api/localfeed/json/JSONItemsAdapter.kt | 116 ++++++++++++++++++ 5 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 api/src/androidTest/assets/localfeed/json/json_items_other_cases.json create mode 100644 api/src/androidTest/assets/localfeed/json/json_items_required_elements.json create mode 100644 api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt create mode 100644 api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt diff --git a/api/src/androidTest/assets/localfeed/json/json_feed.json b/api/src/androidTest/assets/localfeed/json/json_feed.json index 35ca0e3b..4e5b032f 100644 --- a/api/src/androidTest/assets/localfeed/json/json_feed.json +++ b/api/src/androidTest/assets/localfeed/json/json_feed.json @@ -13,7 +13,11 @@ "title": "Acorn and 10.13", "content_html": "

Happy Mac OS High Sierra release day everyone.

\n

I'm happy to say that there are no known issues with Acorn 6.0.3 or Acorn 5.6.6 when running on Mac OS 10.13 High Sierra. In fact, you might even notice that some things are actually faster and it can now open HEIF images. How awesome is that?

\n

I'm also working on some 10.13 goodies for Acorn 6 folks later this year. I can't wait to share that with you, but you'll have to wait just a little bit.

\n", "date_published": "2017-09-25T14:27:27-07:00", - "url": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html" + "url": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html", + "author": { + "url": "this is an url", + "name": "Author 1" + } }, { "id": "http://flyingmeat.com/blog/archives/2018/2/acorn_6.1_is_out.html", diff --git a/api/src/androidTest/assets/localfeed/json/json_items_other_cases.json b/api/src/androidTest/assets/localfeed/json/json_items_other_cases.json new file mode 100644 index 00000000..e91ee941 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/json/json_items_other_cases.json @@ -0,0 +1,28 @@ +{ + "items": [ + { + "id": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html", + "title": "Acorn and 10.13", + "summary": "This is a summary", + "content_html": "content_html", + "content_text": "content_text", + "date_published": "2017-09-25T14:27:27-07:00", + "url": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html", + "image": "https://image.com", + "authors": [ + { + "url": "url 1", + "name": "Author 1" + }, + { + "url": "url 2", + "name": "Author 2" + }, + { + "url": "url 3", + "name": "Author 3" + } + ] + } + ] +} \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/json/json_items_required_elements.json b/api/src/androidTest/assets/localfeed/json/json_items_required_elements.json new file mode 100644 index 00000000..f46aa9a1 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/json/json_items_required_elements.json @@ -0,0 +1,22 @@ +{ + "items": [ + { + "id": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html", + "content_html": "

Happy Mac OS High Sierra release day everyone.

\n

I'm happy to say that there are no known issues with Acorn 6.0.3 or Acorn 5.6.6 when running on Mac OS 10.13 High Sierra. In fact, you might even notice that some things are actually faster and it can now open HEIF images. How awesome is that?

\n

I'm also working on some 10.13 goodies for Acorn 6 folks later this year. I can't wait to share that with you, but you'll have to wait just a little bit.

\n", + "date_published": "2017-09-25T14:27:27-07:00", + "url": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html" + }, + { + "id": "http://flyingmeat.com/blog/archives/2018/2/acorn_6.1_is_out.html", + "title": "Acorn 6.1 Is Out", + "content_html": "

Acorn 6.1 has been released.

\n

You can read a longer post about it over on Gus's blog, but the short of it is: Better, faster, smoother, stronger. And now with Metal 2 support.

\n", + "date_published": "2018-02-16T09:59:11-08:00", + }, + { + "id": "http://flyingmeat.com/blog/archives/2018/6/a_pair_of_updates.html", + "title": "A Pair of Updates", + "content_html": "

Happy summer solstice everybody! (at least for folks in the northern hemisphere, and for folks in the south… sorry. It's going to start getting brighter for you though).

\n

Today I've got a pair of minor app updates to annouce for you.

\n

First up is Acorn 6.1.3, which fixes a number of bugs including one that stemmed from trying to use QuickLook on a file that was created with Acorn 1.0. For the one or two of you that this was affecting, hurray!

\n

Next up is Retrobatch, which also includes some bug fixes, the beginnings of Voice Over support, performance improvements, and more.

\n

What's next for these apps? Work on Acorn 6.2 will begin shortly, as will Retrobatch 1.1. WWDC introduced some great new APIs that I want to take advantage of (cool new machine learning things), so that'll be a focus- as well as Dark Mode for Acorn and one other major thing I've got planned. Retrobatch will probably also get the Dark Mode treatment, but not until I've done it for Acorn first.

\n

So it's going to be a busy summer, but I'm looking forward to it.

\n", + "url": "http://flyingmeat.com/blog/archives/2018/6/a_pair_of_updates.html" + } + ] +} \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt new file mode 100644 index 00000000..c45e3e34 --- /dev/null +++ b/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt @@ -0,0 +1,77 @@ +package com.readrops.api.localfeed.json + +import android.content.Context +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.readrops.api.utils.DateUtils +import com.readrops.api.utils.ParseException +import com.readrops.db.entities.Item +import com.squareup.moshi.Moshi +import com.squareup.moshi.Types +import junit.framework.TestCase.assertEquals +import junit.framework.TestCase.assertNotNull +import okio.Buffer +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith + + +@RunWith(AndroidJUnit4::class) +class JSONItemsAdapterTest { + + private val context: Context = InstrumentationRegistry.getInstrumentation().context + + private val adapter = Moshi.Builder() + .add(Types.newParameterizedType(List::class.java, Item::class.java), JSONItemsAdapter()) + .build() + .adapter>(Types.newParameterizedType(List::class.java, Item::class.java)) + + @Test + fun normalCasesTest() { + val stream = context.resources.assets.open("localfeed/json/json_feed.json") + + val items = adapter.fromJson(Buffer().readFrom(stream))!! + val item = items[0] + + assertEquals(items.size, 10) + assertEquals(item.guid, "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html") + assertEquals(item.title, "Acorn and 10.13") + assertEquals(item.link, "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html") + assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("2017-09-25T14:27:27-07:00")) + assertEquals(item.author, "Author 1") + assertNotNull(item.content) + } + + @Test + fun otherCasesTest() { + val stream = context.resources.assets.open("localfeed/json/json_items_other_cases.json") + + val item = adapter.fromJson(Buffer().readFrom(stream))!![0] + + assertEquals(item.description, "This is a summary") + assertEquals(item.content, "content_html") + assertEquals(item.imageLink, "https://image.com") + assertEquals(item.author, "Author 1") + } + + @Test + fun nullTitleTest() { + val stream = context.resources.assets.open("localfeed/json/json_items_required_elements.json") + + Assert.assertThrows("Item title is required", ParseException::class.java) { adapter.fromJson(Buffer().readFrom(stream))!![0] } + } + + @Test + fun nullLinkTest() { + val stream = context.resources.assets.open("localfeed/json/json_items_required_elements.json") + + Assert.assertThrows("Item link is required", ParseException::class.java) { adapter.fromJson(Buffer().readFrom(stream))!![1] } + } + + @Test + fun nullDateTest() { + val stream = context.resources.assets.open("localfeed/json/json_items_required_elements.json") + + Assert.assertThrows("Item date is required", ParseException::class.java) { adapter.fromJson(Buffer().readFrom(stream))!![2] } + } +} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt new file mode 100644 index 00000000..b2fbb4f8 --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt @@ -0,0 +1,116 @@ +package com.readrops.api.localfeed.json + +import com.readrops.api.utils.DateUtils +import com.readrops.api.utils.ParseException +import com.readrops.api.utils.nextNullableString +import com.readrops.db.entities.Item +import com.squareup.moshi.FromJson +import com.squareup.moshi.JsonAdapter +import com.squareup.moshi.JsonReader +import com.squareup.moshi.JsonWriter + +class JSONItemsAdapter : JsonAdapter>() { + + override fun toJson(writer: JsonWriter, value: List?) { + // not useful + } + + @FromJson + override fun fromJson(reader: JsonReader): List { + try { + val items = arrayListOf() + reader.beginObject() + + while (reader.hasNext()) { + when (reader.nextName()) { + "items" -> parseItems(reader, items) + else -> reader.skipValue() + } + } + + return items + } catch (e: Exception) { + throw ParseException(e.message) + } + } + + private fun parseItems(reader: JsonReader, items: MutableList) { + reader.beginArray() + + while (reader.hasNext()) { + reader.beginObject() + val item = Item() + + var contentText: String? = null + var contentHtml: String? = null + + while (reader.hasNext()) { + with(item) { + when (reader.selectName(names)) { + 0 -> guid = reader.nextString() + 1 -> link = reader.nextString() + 2 -> title = reader.nextString() + 3 -> contentHtml = reader.nextNullableString() + 4 -> contentText = reader.nextNullableString() + 5 -> description = reader.nextNullableString() + 6 -> imageLink = reader.nextNullableString() + 7 -> pubDate = DateUtils.stringToLocalDateTime(reader.nextString()) + 8 -> author = parseAuthor(reader) // jsonfeed 1.0 + 9 -> author = parseAuthors(reader) // jsonfeed 1.1 + } + } + } + + validateItem(item) + item.content = if (contentHtml != null) contentHtml else contentText + + reader.endObject() + items += item + } + + reader.endArray() + } + + private fun parseAuthor(reader: JsonReader): String? { + var author: String? = null + reader.beginObject() + + while (reader.hasNext()) { + when (reader.nextName()) { + "name" -> author = reader.nextNullableString() + else -> reader.skipValue() + } + } + + reader.endObject() + return author + } + + /** + * Returns the first author of the array + */ + private fun parseAuthors(reader: JsonReader): String? { + val authors = arrayListOf() + reader.beginArray() + + while (reader.hasNext()) { + authors.add(parseAuthor(reader)) + } + + reader.endArray() + return if (authors.filterNotNull().isNotEmpty()) authors.filterNotNull().first() else null + } + + private fun validateItem(item: Item) { + when { + item.title == null -> throw ParseException("Item title is required") + item.link == null -> throw ParseException("Item link is required") + item.pubDate == null -> throw ParseException("Item date id required") + } + } + + companion object { + val names: JsonReader.Options = JsonReader.Options.of("id", "url", "title", "content_html", "content_text", + "summary", "image", "date_published", "author", "authors") + } +} \ No newline at end of file From 4b999e9fd65aa4bf39059af1663dcfa068791f33 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Wed, 16 Sep 2020 13:57:45 +0200 Subject: [PATCH 18/58] Add jsonfeed parsing in LocalRSSDataSource --- .../api/localfeed/LocalRSSDataSourceTest.kt | 14 +++++++++++ .../api/localfeed/LocalRSSDataSource.kt | 23 +++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt index 60c76b80..6960c9f5 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt @@ -81,6 +81,20 @@ class LocalRSSDataSourceTest { assertEquals(request.headers[LibUtils.LAST_MODIFIED_HEADER], "Last-Modified") } + @Test + fun jsonFeedTest() { + val stream = context.resources.assets.open("localfeed/json/json_feed.json") + + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader(LibUtils.CONTENT_TYPE_HEADER, "application/feed+json") + .setBody(Buffer().readFrom(stream))) + + val pair = localRSSDataSource.queryRSSResource(url.toString(), null, true)!! + + assertEquals(pair.first.name, "News from Flying Meat") + assertEquals(pair.second.size, 10) + } + @Test fun response304Test() { mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_NOT_MODIFIED)) diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index 817bc126..3541ff1d 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -2,15 +2,20 @@ package com.readrops.api.localfeed import android.accounts.NetworkErrorException import androidx.annotation.WorkerThread +import com.readrops.api.localfeed.json.JSONFeedAdapter +import com.readrops.api.localfeed.json.JSONItemsAdapter import com.readrops.api.utils.LibUtils import com.readrops.api.utils.ParseException import com.readrops.api.utils.UnknownFormatException import com.readrops.db.entities.Feed import com.readrops.db.entities.Item +import com.squareup.moshi.Moshi +import com.squareup.moshi.Types import okhttp3.Headers import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response +import okio.Buffer import java.io.ByteArrayInputStream import java.io.IOException import java.io.InputStream @@ -89,7 +94,12 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { adapter.fromXml(stream) } else { - Feed() + val adapter = Moshi.Builder() + .add(JSONFeedAdapter()) + .build() + .adapter(Feed::class.java) + + adapter.fromJson(Buffer().readFrom(stream))!! } feed.etag = response.header(LibUtils.ETAG_HEADER) @@ -98,13 +108,18 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { return feed } - private fun parseItems(inputStream: InputStream, type: LocalRSSHelper.RSSType): List { + private fun parseItems(stream: InputStream, type: LocalRSSHelper.RSSType): List { return if (type != LocalRSSHelper.RSSType.JSONFEED) { val adapter = XmlAdapter.xmlItemsAdapterFactory(type) - adapter.fromXml(inputStream) + adapter.fromXml(stream) } else { - listOf() + val adapter = Moshi.Builder() + .add(Types.newParameterizedType(MutableList::class.java, Item::class.java), JSONItemsAdapter()) + .build() + .adapter>(Types.newParameterizedType(MutableList::class.java, Item::class.java)) + + adapter.fromJson(Buffer().readFrom(stream))!! } } } \ No newline at end of file From 838768800a641aa6c549b6da9636c01b47acea71 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Thu, 17 Sep 2020 22:11:05 +0200 Subject: [PATCH 19/58] Handle the absence of some tags in RSS2 and ATOM feeds --- .../readrops/api/localfeed/LocalRSSDataSource.kt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index 3541ff1d..5b2726c5 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -102,6 +102,8 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { adapter.fromJson(Buffer().readFrom(stream))!! } + handleSpecialCases(feed, type, response) + feed.etag = response.header(LibUtils.ETAG_HEADER) feed.lastModified = response.header(LibUtils.LAST_MODIFIED_HEADER) @@ -122,4 +124,15 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { adapter.fromJson(Buffer().readFrom(stream))!! } } + + private fun handleSpecialCases(feed: Feed, type: LocalRSSHelper.RSSType, response: Response) { + with(feed) { + if (type == LocalRSSHelper.RSSType.RSS_2) { + if (url == null) url = response.request.url.toString() + } else if (type == LocalRSSHelper.RSSType.ATOM) { + if (url == null) url = response.request.url.toString() + if (siteUrl == null) siteUrl = response.request.url.scheme + "://" + response.request.url.host + } + } + } } \ No newline at end of file From 82c29d073215e3158a1783b230d4620a96bb4ce3 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Thu, 17 Sep 2020 22:15:33 +0200 Subject: [PATCH 20/58] Use new LocalRSSDataSource in LocalFeedRepository --- .../api/localfeed/LocalRSSDataSource.kt | 1 + .../app/repositories/LocalFeedRepository.java | 181 ++++++------------ .../java/com/readrops/db/dao/FeedDao.java | 3 + 3 files changed, 64 insertions(+), 121 deletions(-) diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index 5b2726c5..15074d79 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -30,6 +30,7 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { * @param withItems parse items with their feed * @return a Feed object with its items if specified by [withItems] */ + @Throws(ParseException::class, UnknownFormatException::class, NetworkErrorException::class, IOException::class) @WorkerThread fun queryRSSResource(url: String, headers: Headers?, withItems: Boolean): Pair>? { val response = queryUrl(url, headers) diff --git a/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java b/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java index e3d9651d..dc21cb85 100644 --- a/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java +++ b/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java @@ -2,30 +2,25 @@ package com.readrops.app.repositories; import android.accounts.NetworkErrorException; import android.content.Context; +import android.util.Log; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import com.readrops.api.localfeed.LocalRSSDataSource; +import com.readrops.api.services.SyncResult; +import com.readrops.api.utils.HttpManager; +import com.readrops.api.utils.LibUtils; +import com.readrops.api.utils.ParseException; +import com.readrops.api.utils.UnknownFormatException; import com.readrops.app.utils.FeedInsertionResult; import com.readrops.app.utils.HtmlParser; import com.readrops.app.utils.ParsingResult; import com.readrops.app.utils.SharedPreferencesManager; import com.readrops.app.utils.Utils; -import com.readrops.app.utils.matchers.FeedMatcher; -import com.readrops.app.utils.matchers.ItemMatcher; import com.readrops.db.entities.Feed; import com.readrops.db.entities.Item; import com.readrops.db.entities.account.Account; -import com.readrops.api.localfeed.AFeed; -import com.readrops.api.localfeed.RSSQuery; -import com.readrops.api.localfeed.RSSQueryResult; -import com.readrops.api.localfeed.atom.ATOMFeed; -import com.readrops.api.localfeed.json.JSONFeed; -import com.readrops.api.localfeed.rss.RSSFeed; -import com.readrops.api.services.SyncResult; -import com.readrops.api.utils.LibUtils; -import com.readrops.api.utils.ParseException; -import com.readrops.api.utils.UnknownFormatException; import org.jsoup.Jsoup; @@ -33,20 +28,24 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import io.reactivex.Observable; import io.reactivex.Single; +import kotlin.Pair; +import okhttp3.Headers; public class LocalFeedRepository extends ARepository { private static final String TAG = LocalFeedRepository.class.getSimpleName(); + private LocalRSSDataSource dataSource; + public LocalFeedRepository(@NonNull Context context, @Nullable Account account) { super(context, account); syncResult = new SyncResult(); + dataSource = new LocalRSSDataSource(HttpManager.getInstance().getOkHttpClient()); } @Override @@ -64,47 +63,31 @@ public class LocalFeedRepository extends ARepository { return Observable.create(emitter -> { List feedList; - if (feeds == null || feeds.size() == 0) + if (feeds == null || feeds.isEmpty()) { feedList = database.feedDao().getFeeds(account.getId()); - else - feedList = new ArrayList<>(feeds); - - RSSQuery rssQuery = new RSSQuery(); - List syncErrors = new ArrayList<>(); + } else { + feedList = feeds; + } for (Feed feed : feedList) { emitter.onNext(feed); - FeedInsertionResult syncError = new FeedInsertionResult(); try { - HashMap headers = new HashMap<>(); - if (feed.getEtag() != null) - headers.put(LibUtils.IF_NONE_MATCH_HEADER, feed.getEtag()); - if (feed.getLastModified() != null) - headers.put(LibUtils.IF_MODIFIED_HEADER, feed.getLastModified()); + Headers.Builder headers = new Headers.Builder(); + if (feed.getEtag() != null) { + headers.add(LibUtils.IF_NONE_MATCH_HEADER, feed.getEtag()); + } + if (feed.getLastModified() != null) { + headers.add(LibUtils.IF_MODIFIED_HEADER, feed.getLastModified()); + } - RSSQueryResult queryResult = rssQuery.queryUrl(feed.getUrl(), headers); - if (queryResult != null && queryResult.getException() == null) - insertNewItems(queryResult.getFeed(), queryResult.getRssType()); - else if (queryResult != null && queryResult.getException() != null) { - Exception e = queryResult.getException(); + Pair> pair = dataSource.queryRSSResource(feed.getUrl(), headers.build(), true); - if (e instanceof UnknownFormatException) - syncError.setInsertionError(FeedInsertionResult.FeedInsertionError.FORMAT_ERROR); - else if (e instanceof NetworkErrorException) - syncError.setInsertionError(FeedInsertionResult.FeedInsertionError.NETWORK_ERROR); - - syncError.setFeed(feed); - syncErrors.add(syncError); + if (pair != null) { + insertNewItems(feed, pair.getSecond()); } } catch (Exception e) { - if (e instanceof IOException) - syncError.setInsertionError(FeedInsertionResult.FeedInsertionError.NETWORK_ERROR); - else - syncError.setInsertionError(FeedInsertionResult.FeedInsertionError.PARSE_ERROR); - - syncError.setFeed(feed); - syncErrors.add(syncError); + Log.d(TAG, "sync: " + e.getMessage()); } } @@ -121,28 +104,22 @@ public class LocalFeedRepository extends ARepository { FeedInsertionResult insertionResult = new FeedInsertionResult(); try { - RSSQuery rssNet = new RSSQuery(); - RSSQueryResult queryResult = rssNet.queryUrl(parsingResult.getUrl(), new HashMap<>()); + Pair> pair = dataSource.queryRSSResource(parsingResult.getUrl(), + null, false); + Feed feed = insertFeed(pair.getFirst(), parsingResult); - if (queryResult != null && queryResult.getException() == null) { - Feed feed = insertFeed(queryResult.getFeed(), queryResult.getRssType(), parsingResult); - if (feed != null) { - insertionResult.setFeed(feed); - insertionResult.setParsingResult(parsingResult); - insertionResults.add(insertionResult); - } - } else if (queryResult != null && queryResult.getException() != null) { - insertionResult.setParsingResult(parsingResult); - insertionResult.setInsertionError(getErrorFromException(queryResult.getException())); - - insertionResults.add(insertionResult); + if (feed != null) { + insertionResult.setFeed(feed); } + } catch (ParseException e) { + insertionResult.setInsertionError(FeedInsertionResult.FeedInsertionError.PARSE_ERROR); + } catch (UnknownFormatException e) { + insertionResult.setInsertionError(FeedInsertionResult.FeedInsertionError.FORMAT_ERROR); + } catch (NetworkErrorException | IOException e) { + insertionResult.setInsertionError(FeedInsertionResult.FeedInsertionError.NETWORK_ERROR); } catch (Exception e) { - if (e instanceof IOException) - insertionResult.setInsertionError(FeedInsertionResult.FeedInsertionError.NETWORK_ERROR); - else - insertionResult.setInsertionError(FeedInsertionResult.FeedInsertionError.PARSE_ERROR); - + insertionResult.setInsertionError(FeedInsertionResult.FeedInsertionError.UNKNOWN_ERROR); + } finally { insertionResult.setParsingResult(parsingResult); insertionResults.add(insertionResult); } @@ -152,67 +129,38 @@ public class LocalFeedRepository extends ARepository { }); } - private void insertNewItems(AFeed feed, RSSQuery.RSSType type) throws ParseException { - Feed dbFeed; - List items; + @SuppressWarnings("SimplifyStreamApiCallChains") + private void insertNewItems(Feed feed, List items) { + database.feedDao().updateHeaders(feed.getEtag(), feed.getLastModified(), feed.getId()); - switch (type) { - case RSS_2: - dbFeed = database.feedDao().getFeedByUrl(((RSSFeed) feed).getChannel().getFeedUrl(), account.getId()); - items = ItemMatcher.itemsFromRSS(((RSSFeed) feed).getChannel().getItems(), dbFeed); - break; - case RSS_ATOM: - dbFeed = database.feedDao().getFeedByUrl(((ATOMFeed) feed).getUrl(), account.getId()); - items = ItemMatcher.itemsFromATOM(((ATOMFeed) feed).getEntries(), dbFeed); - break; - case RSS_JSON: - dbFeed = database.feedDao().getFeedByUrl(((JSONFeed) feed).getFeedUrl(), account.getId()); - items = ItemMatcher.itemsFromJSON(((JSONFeed) feed).getItems(), dbFeed); - break; - default: - throw new IllegalArgumentException("Unknown RSS type"); - } - - database.feedDao().updateHeaders(dbFeed.getEtag(), dbFeed.getLastModified(), dbFeed.getId()); Collections.sort(items, Item::compareTo); - int maxItems = Integer.parseInt(SharedPreferencesManager.readString(context, SharedPreferencesManager.SharedPrefKey.ITEMS_TO_PARSE_MAX_NB)); - if (maxItems > 0 && items.size() > maxItems) + int maxItems = Integer.parseInt(SharedPreferencesManager.readString(context, + SharedPreferencesManager.SharedPrefKey.ITEMS_TO_PARSE_MAX_NB)); + if (maxItems > 0 && items.size() > maxItems) { items = items.subList(items.size() - maxItems, items.size()); - - insertItems(items, dbFeed); - } - - private Feed insertFeed(AFeed feed, RSSQuery.RSSType type, ParsingResult parsingResult) { - Feed dbFeed; - switch (type) { - case RSS_2: - dbFeed = FeedMatcher.feedFromRSS((RSSFeed) feed); - break; - case RSS_ATOM: - dbFeed = FeedMatcher.feedFromATOM((ATOMFeed) feed); - break; - case RSS_JSON: - dbFeed = FeedMatcher.feedFromJSON((JSONFeed) feed); - break; - default: - throw new IllegalArgumentException("Unknown RSS type"); } - dbFeed.setFolderId(parsingResult.getFolderId()); + items.stream().forEach(item -> item.setFeedId(feed.getId())); + insertItems(items, feed); + } - if (database.feedDao().feedExists(dbFeed.getUrl(), account.getId())) + private Feed insertFeed(Feed feed, ParsingResult parsingResult) { + feed.setFolderId(parsingResult.getFolderId()); + + if (database.feedDao().feedExists(feed.getUrl(), account.getId())) { return null; // feed already inserted + } - setFeedColors(dbFeed); - dbFeed.setAccountId(account.getId()); + setFeedColors(feed); + feed.setAccountId(account.getId()); // we need empty headers to query the feed just after, without any 304 result - dbFeed.setEtag(null); - dbFeed.setLastModified(null); + feed.setEtag(null); + feed.setLastModified(null); - dbFeed.setId((int) (database.feedDao().compatInsert(dbFeed))); - return dbFeed; + feed.setId((int) (database.feedDao().compatInsert(feed))); + return feed; } private void insertItems(Collection items, Feed feed) { @@ -253,13 +201,4 @@ public class LocalFeedRepository extends ARepository { syncResult.getItems().addAll(itemsToInsert); database.itemDao().insert(itemsToInsert); } - - private FeedInsertionResult.FeedInsertionError getErrorFromException(Exception e) { - if (e instanceof UnknownFormatException) - return FeedInsertionResult.FeedInsertionError.FORMAT_ERROR; - else if (e instanceof NetworkErrorException) - return FeedInsertionResult.FeedInsertionError.NETWORK_ERROR; - else - return FeedInsertionResult.FeedInsertionError.UNKNOWN_ERROR; - } } diff --git a/db/src/main/java/com/readrops/db/dao/FeedDao.java b/db/src/main/java/com/readrops/db/dao/FeedDao.java index 9ff6a5b7..f1a70691 100644 --- a/db/src/main/java/com/readrops/db/dao/FeedDao.java +++ b/db/src/main/java/com/readrops/db/dao/FeedDao.java @@ -29,6 +29,9 @@ public abstract class FeedDao implements BaseDao { @Query("Select * from Feed Where id = :feedId") public abstract Feed getFeedById(int feedId); + @Query("Select id From Feed Where url = :url and account_id = :accountId") + public abstract int getFeedIdByUrl(String url, int accountId); + @Query("Select case When :feedUrl In (Select url from Feed Where account_id = :accountId) Then 1 else 0 end") public abstract boolean feedExists(String feedUrl, int accountId); From f6f8807b3a3562ed1b690e7c642ca7dd9c40a14e Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Thu, 17 Sep 2020 22:17:58 +0200 Subject: [PATCH 21/58] Preserve feeds elements whitespaces --- .../main/java/com/readrops/api/utils/KonsumerExtensions.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt b/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt index adef66de..f1760e85 100644 --- a/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt +++ b/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt @@ -1,13 +1,14 @@ package com.readrops.api.utils import com.gitlab.mvysny.konsumexml.Konsumer +import com.gitlab.mvysny.konsumexml.Whitespace fun Konsumer.nonNullText(failOnElement: Boolean = true): String { - val text = text(failOnElement = failOnElement) + val text = text(failOnElement = failOnElement, whitespace = Whitespace.preserve) return if (text.isNotEmpty()) text else throw ParseException("Xml field $name can't be null") } fun Konsumer.nullableText(failOnElement: Boolean = true): String? { - val text = text(failOnElement = failOnElement) + val text = text(failOnElement = failOnElement, whitespace = Whitespace.preserve) return if (text.isNotEmpty()) text else null } \ No newline at end of file From ccce0a810da8b492ac1dee65a52093988c97c1cc Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Fri, 18 Sep 2020 14:12:07 +0200 Subject: [PATCH 22/58] Fix some RSS2 parsing issues --- .../assets/localfeed/rss/rss_feed_special_cases.xml | 2 +- .../androidTest/assets/localfeed/rss/rss_full_feed.xml | 3 ++- api/src/androidTest/assets/localfeed/rss_feed.xml | 3 ++- .../com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt | 2 +- .../java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt | 8 ++++++-- .../com/readrops/api/localfeed/rss/RSSItemsAdapter.kt | 1 + 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/api/src/androidTest/assets/localfeed/rss/rss_feed_special_cases.xml b/api/src/androidTest/assets/localfeed/rss/rss_feed_special_cases.xml index 4ec78959..55f74777 100644 --- a/api/src/androidTest/assets/localfeed/rss/rss_feed_special_cases.xml +++ b/api/src/androidTest/assets/localfeed/rss/rss_feed_special_cases.xml @@ -2,7 +2,7 @@ - + https://news.ycombinator.com/ Links for the intellectually curious, ranked by readers. diff --git a/api/src/androidTest/assets/localfeed/rss/rss_full_feed.xml b/api/src/androidTest/assets/localfeed/rss/rss_full_feed.xml index 671e4066..e56ad91b 100644 --- a/api/src/androidTest/assets/localfeed/rss/rss_full_feed.xml +++ b/api/src/androidTest/assets/localfeed/rss/rss_full_feed.xml @@ -2,7 +2,8 @@ Hacker News - + + https://news.ycombinator.com/ Links for the intellectually curious, ranked by readers. diff --git a/api/src/androidTest/assets/localfeed/rss_feed.xml b/api/src/androidTest/assets/localfeed/rss_feed.xml index fbd76101..79d3b7a1 100644 --- a/api/src/androidTest/assets/localfeed/rss_feed.xml +++ b/api/src/androidTest/assets/localfeed/rss_feed.xml @@ -1,5 +1,5 @@ - + Hacker News https://news.ycombinator.com/ @@ -11,6 +11,7 @@ https://news.ycombinator.com/item?id=24273602 Author 1 Comments]]> + media description Palantir S-1 diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt index 7644cc24..992a2cdb 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt @@ -31,7 +31,7 @@ class RSSItemsAdapterTest { assertEquals(item.link, "https://www.bbc.com/news/world-africa-53887947") assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("Tue, 25 Aug 2020 17:15:49 +0000")) assertEquals(item.author, "Author 1") - assertNotNull(item.description) + assertEquals(item.description, "Comments") assertEquals(item.guid, "https://www.bbc.com/news/world-africa-53887947") } diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt index 52c6fcff..35f805a0 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt @@ -24,9 +24,13 @@ class RSSFeedAdapter : XmlAdapter { with(feed) { when (tagName) { "title" -> name = Jsoup.parse(nonNullText()).text() - "description" -> description = nullableText(failOnElement = false) + "description" -> description = nullableText() "link" -> siteUrl = nullableText() - "atom:link" -> url = attributes.getValueOpt("href") + "atom:link" -> { + if (attributes.getValueOpt("rel") == "self") + url = attributes.getValueOpt("href") + } + else -> skipContents() } } } diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt index e2344061..cae98764 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt @@ -34,6 +34,7 @@ class RSSItemsAdapter : XmlAdapter> { "content:encoded" -> content = nullableText() "enclosure" -> parseEnclosure(this, enclosures) "media:content" -> parseMediaContent(this, mediaContents) + else -> skipContents() // for example media:description } } } From 0eb518d69227456029bc9fb89064f3c36b3c1d36 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sat, 19 Sep 2020 17:25:36 +0200 Subject: [PATCH 23/58] Ignore media content sub elements --- .../localfeed/rss/rss_items_media_content.xml | 2 +- .../readrops/api/localfeed/rss/RSSItemsAdapter.kt | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_media_content.xml b/api/src/androidTest/assets/localfeed/rss/rss_items_media_content.xml index 323246c7..74355f7b 100644 --- a/api/src/androidTest/assets/localfeed/rss/rss_items_media_content.xml +++ b/api/src/androidTest/assets/localfeed/rss/rss_items_media_content.xml @@ -18,7 +18,7 @@ - + image2 title \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt index cae98764..5f074af3 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt @@ -30,10 +30,15 @@ class RSSItemsAdapter : XmlAdapter> { "pubDate" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) "dc:date" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) "guid" -> guid = nullableText() - "description" -> description = nullableText() - "content:encoded" -> content = nullableText() + "description" -> description = text(failOnElement = false) + "content:encoded" -> content = nullableText(failOnElement = false) "enclosure" -> parseEnclosure(this, enclosures) "media:content" -> parseMediaContent(this, mediaContents) + "media:group" -> allChildrenAutoIgnore("content") { + when (tagName) { + "media:content" -> parseMediaContent(this, mediaContents) + } + } else -> skipContents() // for example media:description } } @@ -69,6 +74,8 @@ class RSSItemsAdapter : XmlAdapter> { if (konsume.attributes.getValueOpt("medium") != null && LibUtils.isMimeImage(konsume.attributes["medium"])) mediaContents += konsume.attributes["url"] + + konsume.skipContents() // ignore media content sub elements } private fun validateItem(item: Item) { @@ -81,6 +88,6 @@ class RSSItemsAdapter : XmlAdapter> { companion object { val names = Names.of("title", "link", "author", "creator", "pubDate", "date", - "guid", "description", "encoded", "enclosure", "content") + "guid", "description", "encoded", "enclosure", "content", "group") } } \ No newline at end of file From 581de2e1dd09baa39585088d1180a45d32581a2b Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sat, 19 Sep 2020 18:50:23 +0200 Subject: [PATCH 24/58] Use nullable attributes in ATOM adapters --- .../java/com/readrops/api/localfeed/atom/ATOMFeedAdapter.kt | 2 +- .../com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeedAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeedAdapter.kt index e53a4bf2..43446073 100644 --- a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeedAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeedAdapter.kt @@ -38,7 +38,7 @@ class ATOMFeedAdapter : XmlAdapter { } private fun parseLink(konsume: Konsumer, feed: Feed) { - val rel = konsume.attributes["rel"] + val rel = konsume.attributes.getValueOpt("rel") if (rel == "self") feed.url = konsume.attributes["href"] diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt index db7fe178..3d2ccf8f 100644 --- a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt @@ -26,7 +26,11 @@ class ATOMItemsAdapter : XmlAdapter> { "title" -> title = nonNullText() "id" -> guid = nullableText() "updated" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) - "link" -> if (attributes["rel"] == "alternate") link = attributes["href"] + "link" -> { + if (attributes.getValueOpt("rel") == null || + attributes["rel"] == "alternate") + link = attributes["href"] + } "author" -> allChildrenAutoIgnore("name") { author = text() } "summary" -> description = nullableText() "content" -> content = nullableText() From 6a1ddaeabb63ac5325037ef2a8bfe1dcc9a7fa40 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 20 Sep 2020 18:12:05 +0200 Subject: [PATCH 25/58] Use LocalRSSDataSource isUrlRSSResource --- .../readrops/api/localfeed/LocalRSSHelper.kt | 35 +++++++------------ .../app/viewmodels/AddFeedsViewModel.java | 15 ++++---- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt index 13894e6a..313d564a 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt @@ -1,18 +1,14 @@ package com.readrops.api.localfeed -import com.readrops.api.utils.UnknownFormatException import java.io.InputStream import java.util.regex.Pattern object LocalRSSHelper { private const val RSS_DEFAULT_CONTENT_TYPE = "application/rss+xml" - private const val RSS_TEXT_CONTENT_TYPE = "text/xml" - private const val RSS_APPLICATION_CONTENT_TYPE = "application/xml" private const val ATOM_CONTENT_TYPE = "application/atom+xml" private const val JSONFEED_CONTENT_TYPE = "application/feed+json" private const val JSON_CONTENT_TYPE = "application/json" - private const val HTML_CONTENT_TYPE = "text/html" private const val RSS_2_REGEX = "rss.*version=\"2.0\"" @@ -26,8 +22,7 @@ object LocalRSSHelper { RSS_DEFAULT_CONTENT_TYPE -> RSSType.RSS_2 ATOM_CONTENT_TYPE -> RSSType.ATOM JSON_CONTENT_TYPE, JSONFEED_CONTENT_TYPE -> RSSType.JSONFEED - RSS_TEXT_CONTENT_TYPE, RSS_APPLICATION_CONTENT_TYPE, HTML_CONTENT_TYPE -> RSSType.UNKNOWN - else -> throw UnknownFormatException("Unknown content type : $contentType") + else -> RSSType.UNKNOWN } } @@ -37,27 +32,21 @@ object LocalRSSHelper { fun getRSSContentType(content: InputStream): RSSType { val stringBuffer = StringBuffer() val reader = content.bufferedReader() + var type = RSSType.UNKNOWN - var currentLine = reader.readLine() - while (currentLine != null) { - stringBuffer.append(currentLine) + // we get the first 10 lines which should be sufficient to get the type, + // otherwise iterating over the whole file could be too slow + for (i in 0..9) stringBuffer.append(reader.readLine()) - if (Pattern.compile(RSS_2_REGEX).matcher(stringBuffer.toString()).find()) { - reader.close() - content.close() - - return RSSType.RSS_2 - } else if (Pattern.compile(ATOM_REGEX).matcher(stringBuffer.toString()).find()) { - reader.close() - content.close() - - return RSSType.ATOM - } - - currentLine = reader.readLine() + if (Pattern.compile(RSS_2_REGEX).matcher(stringBuffer.toString()).find()) { + type = RSSType.RSS_2 + } else if (Pattern.compile(ATOM_REGEX).matcher(stringBuffer.toString()).find()) { + type = RSSType.ATOM } - return RSSType.UNKNOWN + reader.close() + content.close() + return type } enum class RSSType { diff --git a/app/src/main/java/com/readrops/app/viewmodels/AddFeedsViewModel.java b/app/src/main/java/com/readrops/app/viewmodels/AddFeedsViewModel.java index 804ae2eb..303fdb0e 100644 --- a/app/src/main/java/com/readrops/app/viewmodels/AddFeedsViewModel.java +++ b/app/src/main/java/com/readrops/app/viewmodels/AddFeedsViewModel.java @@ -7,13 +7,14 @@ import androidx.annotation.NonNull; import androidx.lifecycle.AndroidViewModel; import androidx.lifecycle.LiveData; -import com.readrops.db.Database; -import com.readrops.db.entities.account.Account; +import com.readrops.api.localfeed.LocalRSSDataSource; +import com.readrops.api.utils.HttpManager; import com.readrops.app.repositories.ARepository; import com.readrops.app.utils.FeedInsertionResult; import com.readrops.app.utils.HtmlParser; import com.readrops.app.utils.ParsingResult; -import com.readrops.api.localfeed.RSSQuery; +import com.readrops.db.Database; +import com.readrops.db.entities.account.Account; import java.util.ArrayList; import java.util.List; @@ -47,15 +48,15 @@ public class AddFeedsViewModel extends AndroidViewModel { public Single> parseUrl(String url) { return Single.create(emitter -> { - RSSQuery rssApi = new RSSQuery(); + LocalRSSDataSource dataSource = new LocalRSSDataSource(HttpManager.getInstance().getOkHttpClient()); List results = new ArrayList<>(); - if (rssApi.isUrlFeedLink(url)) { + if (dataSource.isUrlRSSResource(url)) { ParsingResult parsingResult = new ParsingResult(url, null); results.add(parsingResult); - - } else + } else { results.addAll(HtmlParser.getFeedLink(url)); + } emitter.onSuccess(results); }); From 5998fa9126f975b21c2b82144c75eb529d293430 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 20 Sep 2020 18:17:43 +0200 Subject: [PATCH 26/58] Delete unused code --- .../java/com/readrops/api/localfeed/AFeed.kt | 9 - .../com/readrops/api/localfeed/RSSQuery.java | 198 ------------------ .../api/localfeed/RSSQueryResult.java | 42 ---- .../api/localfeed/atom/ATOMAuthor.java | 22 -- .../api/localfeed/atom/ATOMEntry.java | 98 --------- .../readrops/api/localfeed/atom/ATOMFeed.java | 128 ----------- .../readrops/api/localfeed/atom/ATOMLink.java | 30 --- .../readrops/api/localfeed/json/JSONAuthor.kt | 9 - .../readrops/api/localfeed/json/JSONFeed.kt | 15 -- .../readrops/api/localfeed/json/JSONItem.kt | 21 -- .../api/localfeed/rss/RSSChannel.java | 87 -------- .../api/localfeed/rss/RSSEnclosure.java | 30 --- .../readrops/api/localfeed/rss/RSSFeed.java | 21 -- .../readrops/api/localfeed/rss/RSSItem.java | 154 -------------- .../readrops/api/localfeed/rss/RSSLink.java | 40 ---- .../api/localfeed/rss/RSSMediaContent.java | 30 --- .../app/utils/matchers/FeedMatcher.java | 65 ------ .../app/utils/matchers/ItemMatcher.java | 123 ----------- 18 files changed, 1122 deletions(-) delete mode 100644 api/src/main/java/com/readrops/api/localfeed/AFeed.kt delete mode 100644 api/src/main/java/com/readrops/api/localfeed/RSSQuery.java delete mode 100644 api/src/main/java/com/readrops/api/localfeed/RSSQueryResult.java delete mode 100644 api/src/main/java/com/readrops/api/localfeed/atom/ATOMAuthor.java delete mode 100644 api/src/main/java/com/readrops/api/localfeed/atom/ATOMEntry.java delete mode 100644 api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeed.java delete mode 100644 api/src/main/java/com/readrops/api/localfeed/atom/ATOMLink.java delete mode 100644 api/src/main/java/com/readrops/api/localfeed/json/JSONAuthor.kt delete mode 100644 api/src/main/java/com/readrops/api/localfeed/json/JSONFeed.kt delete mode 100644 api/src/main/java/com/readrops/api/localfeed/json/JSONItem.kt delete mode 100644 api/src/main/java/com/readrops/api/localfeed/rss/RSSChannel.java delete mode 100644 api/src/main/java/com/readrops/api/localfeed/rss/RSSEnclosure.java delete mode 100644 api/src/main/java/com/readrops/api/localfeed/rss/RSSFeed.java delete mode 100644 api/src/main/java/com/readrops/api/localfeed/rss/RSSItem.java delete mode 100644 api/src/main/java/com/readrops/api/localfeed/rss/RSSLink.java delete mode 100644 api/src/main/java/com/readrops/api/localfeed/rss/RSSMediaContent.java delete mode 100644 app/src/main/java/com/readrops/app/utils/matchers/FeedMatcher.java delete mode 100644 app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java diff --git a/api/src/main/java/com/readrops/api/localfeed/AFeed.kt b/api/src/main/java/com/readrops/api/localfeed/AFeed.kt deleted file mode 100644 index 3cbd7653..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/AFeed.kt +++ /dev/null @@ -1,9 +0,0 @@ -package com.readrops.api.localfeed - -/* - A simple class to give an abstract level to rss/atom/json feed classes - */ -abstract class AFeed { - var etag: String? = null - var lastModified: String? = null -} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/RSSQuery.java b/api/src/main/java/com/readrops/api/localfeed/RSSQuery.java deleted file mode 100644 index 75b8c8fc..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/RSSQuery.java +++ /dev/null @@ -1,198 +0,0 @@ -package com.readrops.api.localfeed; - -import android.accounts.NetworkErrorException; -import android.util.Log; - -import com.readrops.api.localfeed.atom.ATOMFeed; -import com.readrops.api.localfeed.json.JSONFeed; -import com.readrops.api.localfeed.rss.RSSFeed; -import com.readrops.api.localfeed.rss.RSSLink; -import com.readrops.api.utils.HttpManager; -import com.readrops.api.utils.LibUtils; -import com.readrops.api.utils.UnknownFormatException; -import com.squareup.moshi.JsonAdapter; -import com.squareup.moshi.Moshi; - -import org.simpleframework.xml.Serializer; -import org.simpleframework.xml.core.Persister; - -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; - -public class RSSQuery { - - private static final String TAG = RSSQuery.class.getSimpleName(); - - private static final String RSS_CONTENT_TYPE_REGEX = "([^;]+)"; - - private static final String RSS_2_REGEX = "rss.*version=\"2.0\""; - - private static final String ATOM_REGEX = "has to be called from another thread than the main one. - * - * @param url url to request - * @throws Exception - */ - public RSSQueryResult queryUrl(String url, Map headers) throws Exception { - Response response = query(url, headers); - - if (response.isSuccessful()) { - String header = response.header(LibUtils.CONTENT_TYPE_HEADER); - RSSType type = getRSSType(header); - - if (type == null) - return new RSSQueryResult(new UnknownFormatException("bad content type : " + header + "for " + url)); - - return parseFeed(response.body().byteStream(), type, response); - } else if (response.code() == 304) - return null; - else - return new RSSQueryResult(new NetworkErrorException("Error " + response.code() + " when requesting url " + url)); - } - - public boolean isUrlFeedLink(String url) throws IOException { - Response response = query(url, new HashMap<>()); - - if (response.isSuccessful()) { - String header = response.header(LibUtils.CONTENT_TYPE_HEADER); - RSSType type = getRSSType(header); - - if (type == RSSType.RSS_UNKNOWN) { - RSSType contentType = getContentRSSType(response.body().string()); - return contentType != RSSType.RSS_UNKNOWN; - } else return type != null; - } else - return false; - } - - private Response query(String url, Map headers) throws IOException { - OkHttpClient okHttpClient = HttpManager.getInstance().getOkHttpClient(); - HttpManager.getInstance().setCredentials(null); - - Request.Builder builder = new Request.Builder().url(url); - for (String header : headers.keySet()) { - String value = headers.get(header); - builder.addHeader(header, value); - } - - Request request = builder.build(); - return okHttpClient.newCall(request).execute(); - } - - private RSSType getRSSType(String contentType) { - Pattern pattern = Pattern.compile(RSS_CONTENT_TYPE_REGEX); - Matcher matcher = pattern.matcher(contentType); - - String header; - if (matcher.find()) - header = matcher.group(0); - else - header = contentType; - - switch (header) { - case LibUtils.RSS_DEFAULT_CONTENT_TYPE: - return RSSType.RSS_2; - case LibUtils.RSS_TEXT_CONTENT_TYPE: - case LibUtils.HTML_CONTENT_TYPE: - case LibUtils.RSS_APPLICATION_CONTENT_TYPE: - return RSSType.RSS_UNKNOWN; - case LibUtils.ATOM_CONTENT_TYPE: - return RSSType.RSS_ATOM; - case LibUtils.JSON_CONTENT_TYPE: - return RSSType.RSS_JSON; - default: - Log.d(TAG, "bad content type : " + contentType); - return null; - } - } - - /** - * Parse input feed - * - * @param stream source to parse - * @param type rss type, important to know the feed format - * @param response query response - * @throws Exception - */ - private RSSQueryResult parseFeed(InputStream stream, RSSType type, Response response) throws Exception { - String xml = LibUtils.inputStreamToString(stream); - Serializer serializer = new Persister(); - - if (type == RSSType.RSS_UNKNOWN) { - RSSType contentType = getContentRSSType(xml); - if (contentType == RSSType.RSS_UNKNOWN) { - return new RSSQueryResult(new UnknownFormatException("Unknown content format")); - } else - type = contentType; - } - - String etag = response.header(LibUtils.ETAG_HEADER); - String lastModified = response.header(LibUtils.LAST_MODIFIED_HEADER); - AFeed feed = null; - RSSQueryResult queryResult = new RSSQueryResult(); - - switch (type) { - case RSS_2: - feed = serializer.read(RSSFeed.class, xml); - - // workaround if the channel does not have any atom:link tag - if (((RSSFeed) feed).getChannel().getFeedUrl() == null) { - ((RSSFeed) feed).getChannel().getLinks().add(new RSSLink(null, response.request().url().toString())); - } - break; - case RSS_ATOM: - feed = serializer.read(ATOMFeed.class, xml); - ((ATOMFeed) feed).setWebsiteUrl(response.request().url().scheme() + "://" + response.request().url().host()); - ((ATOMFeed) feed).setUrl(response.request().url().toString()); - break; - case RSS_JSON: - Moshi moshi = new Moshi.Builder() - .build(); - - JsonAdapter jsonFeedAdapter = moshi.adapter(JSONFeed.class); - feed = jsonFeedAdapter.fromJson(xml); - break; - } - - queryResult.setFeed(feed); - queryResult.setRssType(type); - - feed.setEtag(etag); - feed.setLastModified(lastModified); - - return queryResult; - } - - private RSSType getContentRSSType(String content) { - RSSType type; - - if (Pattern.compile(RSS_2_REGEX).matcher(content).find()) - type = RSSType.RSS_2; - else if (Pattern.compile(ATOM_REGEX).matcher(content).find()) - type = RSSType.RSS_ATOM; - else - type = RSSType.RSS_UNKNOWN; - - return type; - } - - public enum RSSType { - RSS_2, - RSS_ATOM, - RSS_JSON, - RSS_UNKNOWN - } - - -} diff --git a/api/src/main/java/com/readrops/api/localfeed/RSSQueryResult.java b/api/src/main/java/com/readrops/api/localfeed/RSSQueryResult.java deleted file mode 100644 index 449e162a..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/RSSQueryResult.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.readrops.api.localfeed; - -public class RSSQueryResult { - - private AFeed feed; - - private RSSQuery.RSSType rssType; - - private Exception exception; - - public RSSQueryResult(Exception exception) { - this.exception = exception; - } - - public RSSQueryResult() { - - } - - public AFeed getFeed() { - return feed; - } - - public void setFeed(AFeed feed) { - this.feed = feed; - } - - public RSSQuery.RSSType getRssType() { - return rssType; - } - - public void setRssType(RSSQuery.RSSType rssType) { - this.rssType = rssType; - } - - public void setException(Exception exception) { - this.exception = exception; - } - - public Exception getException() { - return exception; - } -} diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMAuthor.java b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMAuthor.java deleted file mode 100644 index 302d06c3..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMAuthor.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.readrops.api.localfeed.atom; - -import org.simpleframework.xml.Element; -import org.simpleframework.xml.Root; - -@Root(name = "author", strict = false) -public class ATOMAuthor { - - @Element(required = false) - private String name; - - @Element(required = false) - private String email; - - public String getName() { - return name; - } - - public String getEmail() { - return email; - } -} diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMEntry.java b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMEntry.java deleted file mode 100644 index 08671f1e..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMEntry.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.readrops.api.localfeed.atom; - -import org.simpleframework.xml.Attribute; -import org.simpleframework.xml.Element; -import org.simpleframework.xml.ElementList; -import org.simpleframework.xml.Root; - -import java.util.List; - -@Root(name = "entry", strict = false) -public class ATOMEntry { - - @Element(required = false) - private String title; - - @ElementList(name = "link", inline = true, required = false) - private List links; - - @Element(required = false) - private String updated; - - @Element(required = false) - private String summary; - - @Element(required = false) - private String id; - - @Element(required = false) - private String content; - - @Attribute(name = "type", required = false) - private String contentType; - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public List getLinks() { - return links; - } - - public void setLinks(List links) { - this.links = links; - } - - public String getUpdated() { - return updated; - } - - public void setUpdated(String updated) { - this.updated = updated; - } - - public String getSummary() { - return summary; - } - - public void setSummary(String summary) { - this.summary = summary; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getContent() { - return content; - } - - public void setContent(String content) { - this.content = content; - } - - public String getContentType() { - return contentType; - } - - public void setContentType(String contentType) { - this.contentType = contentType; - } - - public String getUrl() { - for (ATOMLink link : links) { - if (link.getRel() == null || link.getRel().equals("self") || link.getRel().equals("alternate")) - return link.getHref(); - } - - return null; - } -} diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeed.java b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeed.java deleted file mode 100644 index e83e3c1f..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMFeed.java +++ /dev/null @@ -1,128 +0,0 @@ -package com.readrops.api.localfeed.atom; - -import com.readrops.api.localfeed.AFeed; - -import org.simpleframework.xml.Element; -import org.simpleframework.xml.ElementList; -import org.simpleframework.xml.Root; - -import java.util.List; - -@Root(name = "feed", strict = false) -public class ATOMFeed extends AFeed { - - @Element(required = false) - private String title; - - @ElementList(name = "link", inline = true, required = false) - private List links; - - private String url; - - private String websiteUrl; - - @Element(required = false) - private String id; - - @Element(required = false) - private String subtitle; - - @Element(required = false) - private String updated; - - @Element(required = false) - private ATOMAuthor author; - - @ElementList(inline = true, required = false) - private List entries; - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public List getLinks() { - return links; - } - - public void setLinks(List links) { - this.links = links; - } - - public String getSubtitle() { - return subtitle; - } - - public void setSubtitle(String subtitle) { - this.subtitle = subtitle; - } - - public String getUpdated() { - return updated; - } - - public void setUpdated(String updated) { - this.updated = updated; - } - - public ATOMAuthor getAuthor() { - return author; - } - - public void setAuthor(ATOMAuthor author) { - this.author = author; - } - - public List getEntries() { - return entries; - } - - public void setEntries(List entries) { - this.entries = entries; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public String getWebsiteUrl() { - return websiteUrl; - } - - public void setWebsiteUrl(String websiteUrl) { - this.websiteUrl = websiteUrl; - } - - /*public String getWebSiteUrl() { - return id; - } - - public String getUrl() { - if (links.size() > 0) { - if (links.get(0).getRel() != null) - return links.get(0).getHref(); - else { - if (links.size() > 1) - return links.get(1).getHref(); - else - return null; - } - } else - return null; - }*/ -} diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMLink.java b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMLink.java deleted file mode 100644 index 024a137f..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMLink.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.readrops.api.localfeed.atom; - -import org.simpleframework.xml.Attribute; -import org.simpleframework.xml.Root; - -@Root(name = "link", strict = false) -public class ATOMLink { - - @Attribute(name = "href", required = false) - private String href; - - @Attribute(name = "rel", required = false) - private String rel; - - public String getHref() { - return href; - } - - public void setHref(String href) { - this.href = href; - } - - public String getRel() { - return rel; - } - - public void setRel(String rel) { - this.rel = rel; - } -} diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONAuthor.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONAuthor.kt deleted file mode 100644 index 125e69a1..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/json/JSONAuthor.kt +++ /dev/null @@ -1,9 +0,0 @@ -package com.readrops.api.localfeed.json - -import com.squareup.moshi.Json -import com.squareup.moshi.JsonClass - -@JsonClass(generateAdapter = true) -data class JSONAuthor(val name: String, - val url: String, - @Json(name = "avatar") val avatarUrl: String?) \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONFeed.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONFeed.kt deleted file mode 100644 index 7d83d834..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/json/JSONFeed.kt +++ /dev/null @@ -1,15 +0,0 @@ -package com.readrops.api.localfeed.json - -import com.readrops.api.localfeed.AFeed -import com.squareup.moshi.Json -import com.squareup.moshi.JsonClass - -@JsonClass(generateAdapter = true) -data class JSONFeed(val version: String, - val title: String, - @Json(name = "home_page_url") val homePageUrl: String?, - @Json(name = "feed_url") val feedUrl: String?, - val description: String?, - @Json(name = "icon") val iconUrl: String?, - @Json(name = "favicon") val faviconUrl: String?, - val items: List) : AFeed() \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONItem.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONItem.kt deleted file mode 100644 index 94e50b51..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/json/JSONItem.kt +++ /dev/null @@ -1,21 +0,0 @@ -package com.readrops.api.localfeed.json - -import com.squareup.moshi.Json -import com.squareup.moshi.JsonClass - -@JsonClass(generateAdapter = true) -data class JSONItem(val id: String, - val title: String?, - val summary: String?, - @Json(name = "content_text") val contentText: String?, - @Json(name = "content_html") val contentHtml: String?, - val url: String?, - @Json(name = "image") val imageUrl: String?, - @Json(name = "date_published") val pubDate: String, - @Json(name = "date_modified") val modDate: String?, - val author: JSONAuthor?) { - - fun getContent(): String? { - return contentHtml ?: contentText - } -} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSChannel.java b/api/src/main/java/com/readrops/api/localfeed/rss/RSSChannel.java deleted file mode 100644 index 32c88be3..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSChannel.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.readrops.api.localfeed.rss; - -import org.simpleframework.xml.Element; -import org.simpleframework.xml.ElementList; -import org.simpleframework.xml.Root; - -import java.util.List; - -@Root(name = "channel", strict = false) -public class RSSChannel { - - @Element(name = "title", required = false) - private String title; - - @Element(name = "description", required = false) - private String description; - - // workaround to get the two links (feed and regular) - @ElementList(name = "link", inline = true, required = false) - private List links; - - @Element(name = "lastBuildDate", required = false) - private String lastUpdated; - - @ElementList(inline = true, required = false) - private List items; - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public List getLinks() { - return links; - } - - public void setLinks(List links) { - this.links = links; - } - - public List getItems() { - return items; - } - - public void setItems(List items) { - this.items = items; - } - - public String getLastUpdated() { - return lastUpdated; - } - - public void setLastUpdated(String lastUpdated) { - this.lastUpdated = lastUpdated; - } - - public String getFeedUrl() { - if (links.size() > 1) { - if (links.get(0).getHref() != null) - return links.get(0).getHref(); - else - return links.get(1).getHref(); - } else - return null; - } - - public String getUrl() { - if (links.size() > 1) { - if (links.get(1).getText() != null) - return links.get(1).getText(); - else - return links.get(0).getText(); - } else - return null; - } -} diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSEnclosure.java b/api/src/main/java/com/readrops/api/localfeed/rss/RSSEnclosure.java deleted file mode 100644 index 45a25dbd..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSEnclosure.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.readrops.api.localfeed.rss; - -import org.simpleframework.xml.Attribute; -import org.simpleframework.xml.Root; - -@Root(name = "enclosure", strict = false) -public class RSSEnclosure { - - @Attribute(required = false) - private String type; - - @Attribute(required = false) - private String url; - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } -} diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeed.java b/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeed.java deleted file mode 100644 index e52ab253..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeed.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.readrops.api.localfeed.rss; - -import com.readrops.api.localfeed.AFeed; - -import org.simpleframework.xml.Element; -import org.simpleframework.xml.Root; - -@Root(name = "rss", strict = false) -public class RSSFeed extends AFeed { - - @Element(name = "channel", required = false) - private RSSChannel channel; - - public RSSChannel getChannel() { - return channel; - } - - public void setChannel(RSSChannel channel) { - this.channel = channel; - } -} diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItem.java b/api/src/main/java/com/readrops/api/localfeed/rss/RSSItem.java deleted file mode 100644 index 7cd15406..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItem.java +++ /dev/null @@ -1,154 +0,0 @@ -package com.readrops.api.localfeed.rss; - -import org.simpleframework.xml.Element; -import org.simpleframework.xml.ElementList; -import org.simpleframework.xml.Namespace; -import org.simpleframework.xml.Root; - -import java.util.List; - -@Root(name = "item", strict = false) -public class RSSItem { - - @Element - private String title; - - @Element(name = "link", required = false) - private String link; - - @Element(name = "imageLink", required = false) - private String imageLink; - - @ElementList(name = "content", inline = true, required = false) - @Namespace(prefix = "media") - private List mediaContents; - - @ElementList(name = "enclosure", inline = true, required = false) - private List enclosures; - - @ElementList(name = "creator", inline = true, required = false) - @Namespace(prefix = "dc", reference = "http://purl.org/dc/elements/1.1/") - private List creator; - - @Element(required = false) - private String author; - - @Element(name = "pubDate", required = false) - private String pubDate; - - @Element(name = "date", required = false) - @Namespace(prefix = "dc") - private String date; - - @Element(name = "description", required = false) - private String description; - - @Element(name = "encoded", required = false) - @Namespace(prefix = "content") - private String content; - - @Element(required = false) - private String guid; - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getLink() { - return link; - } - - public void setLink(String link) { - this.link = link; - } - - public String getImageLink() { - return imageLink; - } - - public void setImageLink(String imageLink) { - this.imageLink = imageLink; - } - - public List getCreator() { - return creator; - } - - public void setCreator(List creator) { - this.creator = creator; - } - - public String getPubDate() { - return pubDate; - } - - public void setPubDate(String pubDate) { - this.pubDate = pubDate; - } - - public String getContent() { - return content; - } - - public void setContent(String content) { - this.content = content; - } - - public String getGuid() { - return guid; - } - - public void setGuid(String guid) { - this.guid = guid; - } - - public List getMediaContents() { - return mediaContents; - } - - public void setMediaContents(List mediaContents) { - this.mediaContents = mediaContents; - } - - public List getEnclosures() { - return enclosures; - } - - public void setEnclosures(List enclosures) { - this.enclosures = enclosures; - } - - public String getDate() { - if (pubDate != null) - return pubDate; - else - return date; - } - - public void setDate(String date) { - this.date = date; - } - - public String getAuthor() { - if (creator != null && !creator.isEmpty()) - return creator.get(0); - else - return author; - } - - public void setAuthor(String author) { - this.author = author; - } -} diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSLink.java b/api/src/main/java/com/readrops/api/localfeed/rss/RSSLink.java deleted file mode 100644 index e6511c33..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSLink.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.readrops.api.localfeed.rss; - -import org.simpleframework.xml.Attribute; -import org.simpleframework.xml.Root; -import org.simpleframework.xml.Text; - -@Root(name = "link", strict = false) -public class RSSLink { - - @Text(required = false) - private String text; - - @Attribute(name = "href", required = false) - private String href; - - public RSSLink() { - - } - - public RSSLink(String text, String href) { - this.text = text; - this.href = href; - } - - public String getHref() { - return href; - } - - public void setHref(String href) { - this.href = href; - } - - public String getText() { - return text; - } - - public void setText(String text) { - this.text = text; - } -} diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSMediaContent.java b/api/src/main/java/com/readrops/api/localfeed/rss/RSSMediaContent.java deleted file mode 100644 index 4e7c38e1..00000000 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSMediaContent.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.readrops.api.localfeed.rss; - -import org.simpleframework.xml.Attribute; -import org.simpleframework.xml.Root; - -@Root(name = "content", strict = false) -public class RSSMediaContent { - - @Attribute(required = false) - private String url; - - @Attribute(required = false) - private String medium; - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public String getMedium() { - return medium; - } - - public void setMedium(String medium) { - this.medium = medium; - } -} diff --git a/app/src/main/java/com/readrops/app/utils/matchers/FeedMatcher.java b/app/src/main/java/com/readrops/app/utils/matchers/FeedMatcher.java deleted file mode 100644 index bb07ae91..00000000 --- a/app/src/main/java/com/readrops/app/utils/matchers/FeedMatcher.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.readrops.app.utils.matchers; - -import com.readrops.db.entities.Feed; -import com.readrops.api.localfeed.atom.ATOMFeed; -import com.readrops.api.localfeed.json.JSONFeed; -import com.readrops.api.localfeed.rss.RSSChannel; -import com.readrops.api.localfeed.rss.RSSFeed; - -import org.jsoup.Jsoup; - -public final class FeedMatcher { - - public static Feed feedFromRSS(RSSFeed rssFeed) { - Feed feed = new Feed(); - RSSChannel channel = rssFeed.getChannel(); - - feed.setName(Jsoup.parse(channel.getTitle()).text()); - feed.setUrl(channel.getFeedUrl()); - feed.setSiteUrl(channel.getUrl()); - feed.setDescription(channel.getDescription()); - feed.setLastUpdated(channel.getLastUpdated()); - - feed.setEtag(rssFeed.getEtag()); - feed.setLastModified(rssFeed.getLastModified()); - - feed.setFolderId(null); - - return feed; - } - - public static Feed feedFromATOM(ATOMFeed atomFeed) { - Feed feed = new Feed(); - - feed.setName(atomFeed.getTitle()); - feed.setDescription(atomFeed.getSubtitle()); - feed.setUrl(atomFeed.getUrl()); - feed.setSiteUrl(atomFeed.getWebsiteUrl()); - feed.setDescription(atomFeed.getSubtitle()); - feed.setLastUpdated(atomFeed.getUpdated()); - - feed.setEtag(atomFeed.getEtag()); - feed.setLastModified(atomFeed.getLastModified()); - - feed.setFolderId(null); - - return feed; - } - - public static Feed feedFromJSON(JSONFeed jsonFeed) { - Feed feed = new Feed(); - - feed.setName(jsonFeed.getTitle()); - feed.setUrl(jsonFeed.getFeedUrl()); - feed.setSiteUrl(jsonFeed.getHomePageUrl()); - feed.setDescription(jsonFeed.getDescription()); - - feed.setEtag(jsonFeed.getEtag()); - feed.setLastModified(jsonFeed.getLastModified()); - feed.setIconUrl(jsonFeed.getFaviconUrl()); - - feed.setFolderId(null); - - return feed; - } -} diff --git a/app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java b/app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java deleted file mode 100644 index 58803743..00000000 --- a/app/src/main/java/com/readrops/app/utils/matchers/ItemMatcher.java +++ /dev/null @@ -1,123 +0,0 @@ -package com.readrops.app.utils.matchers; - -import com.readrops.api.localfeed.atom.ATOMEntry; -import com.readrops.api.localfeed.json.JSONItem; -import com.readrops.api.localfeed.rss.RSSEnclosure; -import com.readrops.api.localfeed.rss.RSSItem; -import com.readrops.api.localfeed.rss.RSSMediaContent; -import com.readrops.api.utils.DateUtils; -import com.readrops.api.utils.LibUtils; -import com.readrops.api.utils.ParseException; -import com.readrops.app.utils.Utils; -import com.readrops.db.entities.Feed; -import com.readrops.db.entities.Item; - -import java.util.ArrayList; -import java.util.List; - -public final class ItemMatcher { - - public static List itemsFromRSS(List items, Feed feed) throws ParseException { - List dbItems = new ArrayList<>(); - - for (RSSItem item : items) { - Item newItem = new Item(); - - newItem.setAuthor(item.getAuthor()); - newItem.setContent(item.getContent()); // Jsoup.clean(item.getContent(), Whitelist.relaxed()) - newItem.setDescription(item.getDescription()); - newItem.setGuid(item.getGuid() != null ? item.getGuid() : item.getLink()); - newItem.setTitle(LibUtils.cleanText(item.getTitle())); - - try { - newItem.setPubDate(DateUtils.stringToLocalDateTime(item.getDate())); - } catch (Exception e) { - throw new ParseException(); - } - - newItem.setLink(item.getLink()); - newItem.setFeedId(feed.getId()); - - if (item.getMediaContents() != null && !item.getMediaContents().isEmpty()) { - for (RSSMediaContent mediaContent : item.getMediaContents()) { - if (mediaContent.getMedium() != null && Utils.isTypeImage(mediaContent.getMedium())) { - newItem.setImageLink(mediaContent.getUrl()); - break; - } - } - } else { - if (item.getEnclosures() != null) { - for (RSSEnclosure enclosure : item.getEnclosures()) { - if (enclosure.getType() != null && Utils.isTypeImage(enclosure.getType()) - && enclosure.getUrl() != null) { - newItem.setImageLink(enclosure.getUrl()); - break; - } - } - - } - } - - dbItems.add(newItem); - } - - return dbItems; - } - - public static List itemsFromATOM(List items, Feed feed) throws ParseException { - List dbItems = new ArrayList<>(); - - for (ATOMEntry item : items) { - Item dbItem = new Item(); - - dbItem.setContent(item.getContent()); // Jsoup.clean(item.getContent(), Whitelist.relaxed()) - dbItem.setDescription(item.getSummary()); - dbItem.setGuid(item.getId()); - dbItem.setTitle(LibUtils.cleanText(item.getTitle())); - - try { - dbItem.setPubDate(DateUtils.stringToLocalDateTime(item.getUpdated())); - } catch (Exception e) { - throw new ParseException(); - } - - dbItem.setLink(item.getUrl()); - - dbItem.setFeedId(feed.getId()); - - dbItems.add(dbItem); - } - - return dbItems; - } - - public static List itemsFromJSON(List items, Feed feed) throws ParseException { - List dbItems = new ArrayList<>(); - - for (JSONItem item : items) { - Item dbItem = new Item(); - - if (item.getAuthor() != null) - dbItem.setAuthor(item.getAuthor().getName()); - - dbItem.setContent(item.getContent()); // Jsoup.clean(item.getContent(), Whitelist.relaxed()) - dbItem.setDescription(item.getSummary()); - dbItem.setGuid(item.getId()); - dbItem.setTitle(LibUtils.cleanText(item.getTitle())); - - try { - dbItem.setPubDate(DateUtils.stringToLocalDateTime(item.getPubDate())); - } catch (Exception e) { - throw new ParseException(); - } - - dbItem.setLink(item.getUrl()); - - dbItem.setFeedId(feed.getId()); - - dbItems.add(dbItem); - } - - return dbItems; - } -} From de21a308b6dae64f661d6d8d3bfa8e1373eccf7b Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 20 Sep 2020 18:22:53 +0200 Subject: [PATCH 27/58] Fix LocalRSSHelper unit tests --- .../java/com/readrops/api/localfeed/LocalRSSHelperTest.kt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt index bec3cbf1..c039d38d 100644 --- a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt +++ b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt @@ -1,6 +1,5 @@ package com.readrops.api.localfeed -import com.readrops.api.utils.UnknownFormatException import junit.framework.TestCase.assertEquals import org.junit.Test import java.io.ByteArrayInputStream @@ -20,7 +19,7 @@ class LocalRSSHelperTest { } @Test - fun nonStandardContentTypesTest() { + fun nonSupportedContentTypesTest() { assertEquals(LocalRSSHelper.getRSSType("application/xml"), LocalRSSHelper.RSSType.UNKNOWN) assertEquals(LocalRSSHelper.getRSSType("text/xml"), @@ -29,10 +28,6 @@ class LocalRSSHelperTest { LocalRSSHelper.RSSType.UNKNOWN) } - @Test(expected = UnknownFormatException::class) - fun nonSupportedContentTypeTest() { - LocalRSSHelper.getRSSType("image/jpeg") - } @Test fun rssContentTest() { From 694ff6331e4abc28979834a8a66b551c2c482c8b Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 20 Sep 2020 18:44:55 +0200 Subject: [PATCH 28/58] Add tests for XmlAdapter --- api/build.gradle | 10 +++---- .../com/readrops/api/localfeed/XmlAdapter.kt | 5 ++-- .../readrops/api/localfeed/XmlAdapterTest.kt | 28 +++++++++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt diff --git a/api/build.gradle b/api/build.gradle index 6d0d0bca..c7fa49aa 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -46,11 +46,11 @@ dependencies { implementation "androidx.core:core-ktx:1.2.0" implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" - testImplementation 'junit:junit:4.12' - androidTestImplementation 'androidx.test.ext:junit:1.1.1' - androidTestImplementation 'androidx.test:runner:1.2.0' - androidTestImplementation 'androidx.test:rules:1.2.0' - androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' + testImplementation 'junit:junit:4.13' + androidTestImplementation 'androidx.test.ext:junit:1.1.2' + androidTestImplementation 'androidx.test:runner:1.3.0' + androidTestImplementation 'androidx.test:rules:1.3.0' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' androidTestImplementation 'com.squareup.okhttp3:mockwebserver:4.8.1' implementation 'com.gitlab.mvysny.konsume-xml:konsume-xml:0.11' diff --git a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt index b7f39d64..bbc05bda 100644 --- a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt @@ -7,6 +7,7 @@ import com.readrops.api.localfeed.rss.RSSItemsAdapter import com.readrops.db.entities.Feed import com.readrops.db.entities.Item import java.io.InputStream +import java.lang.IllegalArgumentException interface XmlAdapter { @@ -17,7 +18,7 @@ interface XmlAdapter { return when (type) { LocalRSSHelper.RSSType.RSS_2 -> RSSFeedAdapter() LocalRSSHelper.RSSType.ATOM -> ATOMFeedAdapter() - else -> throw Exception("Unknown RSS type : $type") + else -> throw IllegalArgumentException("Unknown RSS type : $type") } } @@ -25,7 +26,7 @@ interface XmlAdapter { return when (type) { LocalRSSHelper.RSSType.RSS_2 -> RSSItemsAdapter() LocalRSSHelper.RSSType.ATOM -> ATOMItemsAdapter() - else -> throw Exception("Unknown RSS type : $type") + else -> throw IllegalArgumentException("Unknown RSS type : $type") } } } diff --git a/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt b/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt new file mode 100644 index 00000000..5da0b20c --- /dev/null +++ b/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt @@ -0,0 +1,28 @@ +package com.readrops.api.localfeed + +import com.readrops.api.localfeed.atom.ATOMFeedAdapter +import com.readrops.api.localfeed.atom.ATOMItemsAdapter +import com.readrops.api.localfeed.rss.RSSFeedAdapter +import com.readrops.api.localfeed.rss.RSSItemsAdapter +import junit.framework.TestCase.assertTrue +import org.junit.Assert +import org.junit.Test + +class XmlAdapterTest { + + @Test + fun xmlFeedAdapterFactoryTest() { + assertTrue(XmlAdapter.xmlFeedAdapterFactory(LocalRSSHelper.RSSType.RSS_2) is RSSFeedAdapter) + assertTrue(XmlAdapter.xmlFeedAdapterFactory(LocalRSSHelper.RSSType.ATOM) is ATOMFeedAdapter) + + Assert.assertThrows(IllegalArgumentException::class.java) { XmlAdapter.xmlFeedAdapterFactory(LocalRSSHelper.RSSType.UNKNOWN) } + } + + @Test + fun xmlItemsAdapterFactoryTest() { + assertTrue(XmlAdapter.xmlItemsAdapterFactory(LocalRSSHelper.RSSType.RSS_2) is RSSItemsAdapter) + assertTrue(XmlAdapter.xmlItemsAdapterFactory(LocalRSSHelper.RSSType.ATOM) is ATOMItemsAdapter) + + Assert.assertThrows(IllegalArgumentException::class.java) { XmlAdapter.xmlItemsAdapterFactory(LocalRSSHelper.RSSType.UNKNOWN) } + } +} \ No newline at end of file From cb41f3c7ac030fbe2b17adb29ddd1e2c6c625e03 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 20 Sep 2020 18:48:43 +0200 Subject: [PATCH 29/58] Catch JSONFeedAdapter exceptions --- .../api/localfeed/json/JSONFeedAdapter.kt | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt index 4358f315..a67a6a16 100644 --- a/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt @@ -1,5 +1,6 @@ package com.readrops.api.localfeed.json +import com.readrops.api.utils.ParseException import com.readrops.api.utils.nextNullableString import com.readrops.db.entities.Feed import com.squareup.moshi.FromJson @@ -13,23 +14,27 @@ class JSONFeedAdapter { @FromJson fun fromJson(reader: JsonReader): Feed { - val feed = Feed() - reader.beginObject() + try { + val feed = Feed() + reader.beginObject() - while (reader.hasNext()) { - with(feed) { - when (reader.selectName(names)) { - 0 -> name = reader.nextString() - 1 -> siteUrl = reader.nextNullableString() - 2 -> url = reader.nextNullableString() - 3 -> description = reader.nextNullableString() - else -> reader.skipValue() + while (reader.hasNext()) { + with(feed) { + when (reader.selectName(names)) { + 0 -> name = reader.nextString() + 1 -> siteUrl = reader.nextNullableString() + 2 -> url = reader.nextNullableString() + 3 -> description = reader.nextNullableString() + else -> reader.skipValue() + } } } - } - reader.endObject() - return feed + reader.endObject() + return feed + } catch (e: Exception) { + throw ParseException(e.message) + } } companion object { From 963350d1dd686ba2730f443075a15be050a19050 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 20 Sep 2020 19:00:49 +0200 Subject: [PATCH 30/58] Prepare files for the addition of RSS1 adapters --- .../RSSFeedAdapterTest.kt => rss2/RSS2FeedAdapterTest.kt} | 6 +++--- .../RSS2ItemsAdapterTest.kt} | 7 +++---- .../main/java/com/readrops/api/localfeed/XmlAdapter.kt | 8 ++++---- .../{rss/RSSFeedAdapter.kt => rss2/RSS2FeedAdapter.kt} | 4 ++-- .../{rss/RSSItemsAdapter.kt => rss2/RSS2ItemsAdapter.kt} | 4 ++-- .../java/com/readrops/api/localfeed/XmlAdapterTest.kt | 8 ++++---- 6 files changed, 18 insertions(+), 19 deletions(-) rename api/src/androidTest/java/com/readrops/api/localfeed/{rss/RSSFeedAdapterTest.kt => rss2/RSS2FeedAdapterTest.kt} (90%) rename api/src/androidTest/java/com/readrops/api/localfeed/{rss/RSSItemsAdapterTest.kt => rss2/RSS2ItemsAdapterTest.kt} (94%) rename api/src/main/java/com/readrops/api/localfeed/{rss/RSSFeedAdapter.kt => rss2/RSS2FeedAdapter.kt} (95%) rename api/src/main/java/com/readrops/api/localfeed/{rss/RSSItemsAdapter.kt => rss2/RSS2ItemsAdapter.kt} (97%) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSFeedAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2FeedAdapterTest.kt similarity index 90% rename from api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSFeedAdapterTest.kt rename to api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2FeedAdapterTest.kt index 79fbcafb..343cdae7 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSFeedAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2FeedAdapterTest.kt @@ -1,4 +1,4 @@ -package com.readrops.api.localfeed.rss +package com.readrops.api.localfeed.rss2 import android.content.Context import androidx.test.ext.junit.runners.AndroidJUnit4 @@ -9,11 +9,11 @@ import org.junit.Test import org.junit.runner.RunWith @RunWith(AndroidJUnit4::class) -class RSSFeedAdapterTest { +class RSS2FeedAdapterTest { private val context: Context = InstrumentationRegistry.getInstrumentation().context - private val adapter = RSSFeedAdapter() + private val adapter = RSS2FeedAdapter() @Test fun normalCasesTest() { diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt similarity index 94% rename from api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt rename to api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt index 992a2cdb..b5d75e08 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt @@ -1,4 +1,4 @@ -package com.readrops.api.localfeed.rss +package com.readrops.api.localfeed.rss2 import android.content.Context import androidx.test.ext.junit.runners.AndroidJUnit4 @@ -6,17 +6,16 @@ import androidx.test.platform.app.InstrumentationRegistry import com.readrops.api.utils.DateUtils import com.readrops.api.utils.ParseException import junit.framework.TestCase.assertEquals -import junit.framework.TestCase.assertNotNull import org.junit.Assert import org.junit.Test import org.junit.runner.RunWith @RunWith(AndroidJUnit4::class) -class RSSItemsAdapterTest { +class RSS2ItemsAdapterTest { private val context: Context = InstrumentationRegistry.getInstrumentation().context - private val adapter = RSSItemsAdapter() + private val adapter = RSS2ItemsAdapter() @Test fun normalCasesTest() { diff --git a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt index bbc05bda..8f21a9e5 100644 --- a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt @@ -2,8 +2,8 @@ package com.readrops.api.localfeed import com.readrops.api.localfeed.atom.ATOMFeedAdapter import com.readrops.api.localfeed.atom.ATOMItemsAdapter -import com.readrops.api.localfeed.rss.RSSFeedAdapter -import com.readrops.api.localfeed.rss.RSSItemsAdapter +import com.readrops.api.localfeed.rss2.RSS2FeedAdapter +import com.readrops.api.localfeed.rss2.RSS2ItemsAdapter import com.readrops.db.entities.Feed import com.readrops.db.entities.Item import java.io.InputStream @@ -16,7 +16,7 @@ interface XmlAdapter { companion object { fun xmlFeedAdapterFactory(type: LocalRSSHelper.RSSType): XmlAdapter { return when (type) { - LocalRSSHelper.RSSType.RSS_2 -> RSSFeedAdapter() + LocalRSSHelper.RSSType.RSS_2 -> RSS2FeedAdapter() LocalRSSHelper.RSSType.ATOM -> ATOMFeedAdapter() else -> throw IllegalArgumentException("Unknown RSS type : $type") } @@ -24,7 +24,7 @@ interface XmlAdapter { fun xmlItemsAdapterFactory(type: LocalRSSHelper.RSSType): XmlAdapter> { return when (type) { - LocalRSSHelper.RSSType.RSS_2 -> RSSItemsAdapter() + LocalRSSHelper.RSSType.RSS_2 -> RSS2ItemsAdapter() LocalRSSHelper.RSSType.ATOM -> ATOMItemsAdapter() else -> throw IllegalArgumentException("Unknown RSS type : $type") } diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2FeedAdapter.kt similarity index 95% rename from api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt rename to api/src/main/java/com/readrops/api/localfeed/rss2/RSS2FeedAdapter.kt index 35f805a0..2751a428 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2FeedAdapter.kt @@ -1,4 +1,4 @@ -package com.readrops.api.localfeed.rss +package com.readrops.api.localfeed.rss2 import com.gitlab.mvysny.konsumexml.Names import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore @@ -11,7 +11,7 @@ import com.readrops.db.entities.Feed import org.jsoup.Jsoup import java.io.InputStream -class RSSFeedAdapter : XmlAdapter { +class RSS2FeedAdapter : XmlAdapter { override fun fromXml(inputStream: InputStream): Feed { val konsume = inputStream.konsumeXml() diff --git a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt similarity index 97% rename from api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt rename to api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt index 5f074af3..fde85f49 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss/RSSItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt @@ -1,4 +1,4 @@ -package com.readrops.api.localfeed.rss +package com.readrops.api.localfeed.rss2 import com.gitlab.mvysny.konsumexml.* import com.readrops.api.localfeed.XmlAdapter @@ -6,7 +6,7 @@ import com.readrops.api.utils.* import com.readrops.db.entities.Item import java.io.InputStream -class RSSItemsAdapter : XmlAdapter> { +class RSS2ItemsAdapter : XmlAdapter> { override fun fromXml(inputStream: InputStream): List { val konsume = inputStream.konsumeXml() diff --git a/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt b/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt index 5da0b20c..23d74683 100644 --- a/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt +++ b/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt @@ -2,8 +2,8 @@ package com.readrops.api.localfeed import com.readrops.api.localfeed.atom.ATOMFeedAdapter import com.readrops.api.localfeed.atom.ATOMItemsAdapter -import com.readrops.api.localfeed.rss.RSSFeedAdapter -import com.readrops.api.localfeed.rss.RSSItemsAdapter +import com.readrops.api.localfeed.rss2.RSS2FeedAdapter +import com.readrops.api.localfeed.rss2.RSS2ItemsAdapter import junit.framework.TestCase.assertTrue import org.junit.Assert import org.junit.Test @@ -12,7 +12,7 @@ class XmlAdapterTest { @Test fun xmlFeedAdapterFactoryTest() { - assertTrue(XmlAdapter.xmlFeedAdapterFactory(LocalRSSHelper.RSSType.RSS_2) is RSSFeedAdapter) + assertTrue(XmlAdapter.xmlFeedAdapterFactory(LocalRSSHelper.RSSType.RSS_2) is RSS2FeedAdapter) assertTrue(XmlAdapter.xmlFeedAdapterFactory(LocalRSSHelper.RSSType.ATOM) is ATOMFeedAdapter) Assert.assertThrows(IllegalArgumentException::class.java) { XmlAdapter.xmlFeedAdapterFactory(LocalRSSHelper.RSSType.UNKNOWN) } @@ -20,7 +20,7 @@ class XmlAdapterTest { @Test fun xmlItemsAdapterFactoryTest() { - assertTrue(XmlAdapter.xmlItemsAdapterFactory(LocalRSSHelper.RSSType.RSS_2) is RSSItemsAdapter) + assertTrue(XmlAdapter.xmlItemsAdapterFactory(LocalRSSHelper.RSSType.RSS_2) is RSS2ItemsAdapter) assertTrue(XmlAdapter.xmlItemsAdapterFactory(LocalRSSHelper.RSSType.ATOM) is ATOMItemsAdapter) Assert.assertThrows(IllegalArgumentException::class.java) { XmlAdapter.xmlItemsAdapterFactory(LocalRSSHelper.RSSType.UNKNOWN) } From 694d6842232924dec2aceca7b1b476e45570d065 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Wed, 23 Sep 2020 18:57:53 +0200 Subject: [PATCH 31/58] Add adapter for RSS1 feed --- .../assets/localfeed/rss1/rss1_feed.xml | 268 ++++++++++++++++++ .../api/localfeed/rss1/RSS1FeedAdapterTest.kt | 28 ++ .../readrops/api/localfeed/LocalRSSHelper.kt | 7 +- .../com/readrops/api/localfeed/XmlAdapter.kt | 3 +- .../api/localfeed/rss1/RSS1FeedAdapter.kt | 48 ++++ .../api/localfeed/LocalRSSHelperTest.kt | 2 + .../readrops/api/localfeed/XmlAdapterTest.kt | 2 + 7 files changed, 355 insertions(+), 3 deletions(-) create mode 100644 api/src/androidTest/assets/localfeed/rss1/rss1_feed.xml create mode 100644 api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapterTest.kt create mode 100644 api/src/main/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapter.kt diff --git a/api/src/androidTest/assets/localfeed/rss1/rss1_feed.xml b/api/src/androidTest/assets/localfeed/rss1/rss1_feed.xml new file mode 100644 index 00000000..2dae7a51 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss1/rss1_feed.xml @@ -0,0 +1,268 @@ + + + + Slashdot + https://slashdot.org/ + News for nerds, stuff that matters + en-us + Copyright 1997-2016, SlashdotMedia. All Rights Reserved. + 2020-09-23T16:20:20+00:00 + Dice + help@slashdot.org + Technology + 1970-01-01T00:00+00:00 + 1 + hourly + + + + + + + + + + + + + + + + + + + + + + + + + + + Slashdot + https://a.fsdn.com/sd/topics/topicslashdot.gif + https://slashdot.org/ + + + Google Expands its Flutter Development Kit To Windows Apps + + https://developers.slashdot.org/story/20/09/23/1616231/google-expands-its-flutter-development-kit-to-windows-apps?utm_source=rss1.0mainlinkanon&utm_medium=feed + + Google has announced that Flutter, its open source UI development kit for + building cross-platform software from the same codebase, is finally available for + Windows apps in alpha. From a report:For the world's leading desktop operating system + with some 1 billion installations of Windows 10 alone, this has been a long time coming. + Flutter's alpha incarnation was initially launched at Google's I/O developer conference + back in 2017, before arriving in beta less than a year later. In its original guise, + Flutter was designed for Android and iOS app development, but it has since expanded to + cover the web, MacOS, and Linux, which are currently available in various alpha or beta + iterations. Developers have had to consider unique platform-specific factors when + designing for the desktop or mobile phones, such as different screen sizes and how + people interact with their devices. On smartphones, people typically use touch and + swipe-based gestures, while keyboards and mice are commonly used on PCs and laptops. + This means Flutter has had to expand its support to cover the additional inputs.<p><div + class="share_submission" style="position:relative;"> <a class="slashpop" + href="http://twitter.com/home?status=Google+Expands+its+Flutter+Development+Kit+To+Windows+Apps%3A+https%3A%2F%2Fbit.ly%2F32X36MW"><img + src="https://a.fsdn.com/sd/twitter_icon_large.png"></a> <a class="slashpop" + href="http://www.facebook.com/sharer.php?u=https%3A%2F%2Fdevelopers.slashdot.org%2Fstory%2F20%2F09%2F23%2F1616231%2Fgoogle-expands-its-flutter-development-kit-to-windows-apps%3Futm_source%3Dslashdot%26utm_medium%3Dfacebook"><img + src="https://a.fsdn.com/sd/facebook_icon_large.png"></a> + + + </div></p><p><a + href="https://developers.slashdot.org/story/20/09/23/1616231/google-expands-its-flutter-development-kit-to-windows-apps?utm_source=rss1.0moreanon&amp;utm_medium=feed">Read + more of this story</a> at Slashdot.</p><iframe + src="https://slashdot.org/slashdot-it.pl?op=discuss&amp;id=17251868&amp;smallembed=1" + style="height: 300px; width: 100%; border: none;"></iframe> + + msmash + 2020-09-23T16:15:00+00:00 + programming + how-about-that + developers + 1 + 1,1,1,1,0,0,0 + + + Firefox Usage is Down 85% Despite Mozilla's Top Exec Pay Going Up 400% + + https://news.slashdot.org/story/20/09/23/1528219/firefox-usage-is-down-85-despite-mozillas-top-exec-pay-going-up-400?utm_source=rss1.0mainlinkanon&utm_medium=feed + + Software engineer Cal Paterson writes: Mozilla recently announced that they + would be dismissing 250 people. That's a quarter of their workforce so there are some + deep cuts to their work too. The victims include: the MDN docs (those are the web + standards docs everyone likes better than w3schools), the Rust compiler and even some + cuts to Firefox development. Like most people I want to see Mozilla do well but those + three projects comprise pretty much what I think of as the whole point of Mozilla, so + this news is a a big let down. The stated reason for the cuts is falling income. Mozilla + largely relies on "royalties" for funding. In return for payment, Mozilla allows big + technology companies to choose the default search engine in Firefox - the technology + companies are ultimately paying to increase the number of searches Firefox users make + with them. Mozilla haven't been particularly transparent about why these royalties are + being reduced, except to blame the coronavirus. I'm sure the coronavirus is not a great + help but I suspect the bigger problem is that Firefox's market share is now a tiny + fraction of its previous size and so the royalties will be smaller too - fewer users, so + fewer searches and therefore less money for Mozilla. + + The real problem is not the royalty cuts, though. Mozilla has already received more than + enough money to set themselves up for financial independence. Mozilla received up to + half a billion dollars a year (each year!) for many years. The real problem is that + Mozilla didn't use that money to achieve financial independence and instead just spent + it each year, doing the organisational equivalent of living hand-to-mouth. Despite their + slightly contrived legal structure as a non-profit that owns a for-profit, Mozilla are + an NGO just like any other. In this article I want to apply the traditional measures + that are applied to other NGOs to Mozilla in order to show what's wrong. These three + measures are: overheads, ethics and results.<p><div class="share_submission" + style="position:relative;"> <a class="slashpop" + href="http://twitter.com/home?status=Firefox+Usage+is+Down+85%25+Despite+Mozilla's+Top+Exec+Pay+Going+Up+400%25%3A+https%3A%2F%2Fbit.ly%2F33M9FB2"><img + src="https://a.fsdn.com/sd/twitter_icon_large.png"></a> <a class="slashpop" + href="http://www.facebook.com/sharer.php?u=https%3A%2F%2Fnews.slashdot.org%2Fstory%2F20%2F09%2F23%2F1528219%2Ffirefox-usage-is-down-85-despite-mozillas-top-exec-pay-going-up-400%3Futm_source%3Dslashdot%26utm_medium%3Dfacebook"><img + src="https://a.fsdn.com/sd/facebook_icon_large.png"></a> + + + </div></p><p><a + href="https://news.slashdot.org/story/20/09/23/1528219/firefox-usage-is-down-85-despite-mozillas-top-exec-pay-going-up-400?utm_source=rss1.0moreanon&amp;utm_medium=feed">Read + more of this story</a> at Slashdot.</p><iframe + src="https://slashdot.org/slashdot-it.pl?op=discuss&amp;id=17251650&amp;smallembed=1" + style="height: 300px; width: 100%; border: none;"></iframe> + + msmash + 2020-09-23T15:27:00+00:00 + firefox + closer-look + news + 31 + 31,29,24,21,3,1,0 + + + Climate Disruption Is Now Locked In. The Next Moves Will Be Crucial. + + https://news.slashdot.org/story/20/09/23/1451213/climate-disruption-is-now-locked-in-the-next-moves-will-be-crucial?utm_source=rss1.0mainlinkanon&utm_medium=feed + + America is now under siege by climate change in ways that scientists have + warned about for years. But there is a second part to their admonition: Decades of + growing crisis are already locked into the global ecosystem and cannot be reversed. From + a report: This means the kinds of cascading disasters occurring today -- drought in the + West fueling historic wildfires that send smoke all the way to the East Coast, or + parades of tropical storms lining up across the Atlantic to march destructively toward + North America -- are no longer features of some dystopian future. They are the here and + now, worsening for the next generation and perhaps longer, depending on humanity's + willingness to take action. "I've been labeled an alarmist," said Peter Kalmus, a + climate scientist in Los Angeles, where he and millions of others have inhaled + dangerously high levels of smoke for weeks. "And I think it's a lot harder for people to + say that I'm being alarmist now." Last month, before the skies over San Francisco turned + a surreal orange, Death Valley reached 130 degrees Fahrenheit, the highest temperature + ever measured on the planet. Dozens of people have perished from the heat in Phoenix, + which in July suffered its hottest month on record, only to surpass that milestone in + August. + + Conversations about climate change have broken into everyday life, to the top of the + headlines and to center stage in the presidential campaign. The questions are profound + and urgent. Can this be reversed? What can be done to minimize the looming dangers for + the decades ahead? Will the destruction of recent weeks become a moment of reckoning, or + just a blip in the news cycle? The Times spoke with two dozen climate experts, including + scientists, economists, sociologists and policymakers, and their answers were by turns + alarming, cynical and hopeful. "It's as if we've been smoking a pack of cigarettes a day + for decades" and the world is now feeling the effects, said Katharine Hayhoe, a climate + scientist at Texas Tech University. But, she said, "we're not dead yet." Their most + sobering message was that the world still hasn't seen the worst of it. Gone is the + climate of yesteryear, and there's no going back. The effects of climate change evident + today are the results of choices that countries made decades ago to keep pumping + heat-trapping greenhouse gases into the atmosphere at ever-increasing rates despite + warnings from scientists about the price to be paid.<p><div + class="share_submission" style="position:relative;"> <a class="slashpop" + href="http://twitter.com/home?status=Climate+Disruption+Is+Now+Locked+In.+The+Next+Moves+Will+Be+Crucial.%3A+https%3A%2F%2Fbit.ly%2F32TsNxO"><img + src="https://a.fsdn.com/sd/twitter_icon_large.png"></a> <a class="slashpop" + href="http://www.facebook.com/sharer.php?u=https%3A%2F%2Fnews.slashdot.org%2Fstory%2F20%2F09%2F23%2F1451213%2Fclimate-disruption-is-now-locked-in-the-next-moves-will-be-crucial%3Futm_source%3Dslashdot%26utm_medium%3Dfacebook"><img + src="https://a.fsdn.com/sd/facebook_icon_large.png"></a> + + + </div></p><p><a + href="https://news.slashdot.org/story/20/09/23/1451213/climate-disruption-is-now-locked-in-the-next-moves-will-be-crucial?utm_source=rss1.0moreanon&amp;utm_medium=feed">Read + more of this story</a> at Slashdot.</p><iframe + src="https://slashdot.org/slashdot-it.pl?op=discuss&amp;id=17251462&amp;smallembed=1" + style="height: 300px; width: 100%; border: none;"></iframe> + + msmash + 2020-09-23T14:51:00+00:00 + earth + closer-look + news + 67 + 67,61,50,33,12,2,1 + + + A New York Clock That Told Time Now Tells the Time Remaining + + https://news.slashdot.org/story/20/09/23/1420240/a-new-york-clock-that-told-time-now-tells-the-time-remaining?utm_source=rss1.0mainlinkanon&utm_medium=feed + + For more than 20 years, Metronome, which includes a 62-foot-wide 15-digit + electronic clock that faces Union Square in Manhattan, has been one of the city's most + prominent and baffling public art projects. Its digital display once told the time in + its own unique way, counting the hours, minutes and seconds (and fractions thereof) to + and from midnight. But for years observers who did not understand how it worked + suggested that it was measuring the acres of rainforest destroyed each year, tracking + the world population or even that it had something to do with pi. On Saturday Metronome + adopted a new ecologically sensitive mission. From a report: Now, instead of measuring + 24-hour cycles, it is measuring what two artists, Gan Golan and Andrew Boyd, present as + a critical window for action to prevent the effects of global warming from becoming + irreversible. On Saturday at 3:20 p.m., messages including "The Earth has a deadline" + began to appear on the display. Then numbers -- 7:103:15:40:07 -- showed up, + representing the years, days, hours, minutes and seconds until that deadline. As a + handful of supporters watched, the number -- which the artists said was based on + calculations by the Mercator Research Institute on Global Commons and Climate Change in + Berlin -- began ticking down, second by second. + + "This is our way to shout that number from the rooftops." Mr. Golan said just before the + countdown began. "The world is literally counting on us." The Climate Clock, as the two + artists call their project, will be displayed on the 14th Street building, One Union + Square South, through Sept. 27, the end of Climate Week. The creators say their aim is + to arrange for the clock to be permanently displayed, there or elsewhere. Mr. Golan said + he came up with the idea to publicly illustrate the urgency of combating climate change + about two years ago, shortly after his daughter was born. He asked Mr. Boyd, an activist + from the Lower East Side, to work with him on the project.<p><div + class="share_submission" style="position:relative;"> <a class="slashpop" + href="http://twitter.com/home?status=A+New+York+Clock+That+Told+Time+Now+Tells+the+Time+Remaining%3A+https%3A%2F%2Fbit.ly%2F2HrAt2b"><img + src="https://a.fsdn.com/sd/twitter_icon_large.png"></a> <a class="slashpop" + href="http://www.facebook.com/sharer.php?u=https%3A%2F%2Fnews.slashdot.org%2Fstory%2F20%2F09%2F23%2F1420240%2Fa-new-york-clock-that-told-time-now-tells-the-time-remaining%3Futm_source%3Dslashdot%26utm_medium%3Dfacebook"><img + src="https://a.fsdn.com/sd/facebook_icon_large.png"></a> + + + </div></p><p><a + href="https://news.slashdot.org/story/20/09/23/1420240/a-new-york-clock-that-told-time-now-tells-the-time-remaining?utm_source=rss1.0moreanon&amp;utm_medium=feed">Read + more of this story</a> at Slashdot.</p><iframe + src="https://slashdot.org/slashdot-it.pl?op=discuss&amp;id=17251272&amp;smallembed=1" + style="height: 300px; width: 100%; border: none;"></iframe> + + msmash + 2020-09-23T14:10:00+00:00 + news + how-about-that + news + 43 + 43,38,33,27,6,1,0 + + \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapterTest.kt new file mode 100644 index 00000000..47d023b0 --- /dev/null +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapterTest.kt @@ -0,0 +1,28 @@ +package com.readrops.api.localfeed.rss1 + +import android.content.Context +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import junit.framework.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class RSS1FeedAdapterTest { + + private val context: Context = InstrumentationRegistry.getInstrumentation().context + + private val adapter = RSS1FeedAdapter() + + @Test + fun normalCaseTest() { + val stream = context.resources.assets.open("localfeed/rss1/rss1_feed.xml") + + val feed = adapter.fromXml(stream) + + assertEquals(feed.name, "Slashdot") + assertEquals(feed.url, "https://slashdot.org/") + assertEquals(feed.siteUrl, "https://slashdot.org/") + assertEquals(feed.description, "News for nerds, stuff that matters") + } +} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt index 313d564a..b8b020e3 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt @@ -5,7 +5,8 @@ import java.util.regex.Pattern object LocalRSSHelper { - private const val RSS_DEFAULT_CONTENT_TYPE = "application/rss+xml" + private const val RSS_1_CONTENT_TYPE = "application/rdf+xml" + private const val RSS_2_CONTENT_TYPE = "application/rss+xml" private const val ATOM_CONTENT_TYPE = "application/atom+xml" private const val JSONFEED_CONTENT_TYPE = "application/feed+json" private const val JSON_CONTENT_TYPE = "application/json" @@ -19,7 +20,8 @@ object LocalRSSHelper { */ fun getRSSType(contentType: String): RSSType { return when (contentType) { - RSS_DEFAULT_CONTENT_TYPE -> RSSType.RSS_2 + RSS_1_CONTENT_TYPE -> RSSType.RSS_1 + RSS_2_CONTENT_TYPE -> RSSType.RSS_2 ATOM_CONTENT_TYPE -> RSSType.ATOM JSON_CONTENT_TYPE, JSONFEED_CONTENT_TYPE -> RSSType.JSONFEED else -> RSSType.UNKNOWN @@ -50,6 +52,7 @@ object LocalRSSHelper { } enum class RSSType { + RSS_1, RSS_2, ATOM, JSONFEED, diff --git a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt index 8f21a9e5..1641c00f 100644 --- a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt @@ -2,12 +2,12 @@ package com.readrops.api.localfeed import com.readrops.api.localfeed.atom.ATOMFeedAdapter import com.readrops.api.localfeed.atom.ATOMItemsAdapter +import com.readrops.api.localfeed.rss1.RSS1FeedAdapter import com.readrops.api.localfeed.rss2.RSS2FeedAdapter import com.readrops.api.localfeed.rss2.RSS2ItemsAdapter import com.readrops.db.entities.Feed import com.readrops.db.entities.Item import java.io.InputStream -import java.lang.IllegalArgumentException interface XmlAdapter { @@ -16,6 +16,7 @@ interface XmlAdapter { companion object { fun xmlFeedAdapterFactory(type: LocalRSSHelper.RSSType): XmlAdapter { return when (type) { + LocalRSSHelper.RSSType.RSS_1 -> RSS1FeedAdapter() LocalRSSHelper.RSSType.RSS_2 -> RSS2FeedAdapter() LocalRSSHelper.RSSType.ATOM -> ATOMFeedAdapter() else -> throw IllegalArgumentException("Unknown RSS type : $type") diff --git a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapter.kt new file mode 100644 index 00000000..697258c1 --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapter.kt @@ -0,0 +1,48 @@ +package com.readrops.api.localfeed.rss1 + +import com.gitlab.mvysny.konsumexml.Names +import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore +import com.gitlab.mvysny.konsumexml.konsumeXml +import com.readrops.api.localfeed.XmlAdapter +import com.readrops.api.utils.ParseException +import com.readrops.api.utils.nonNullText +import com.readrops.api.utils.nullableText +import com.readrops.db.entities.Feed +import java.io.InputStream + +class RSS1FeedAdapter : XmlAdapter { + + override fun fromXml(inputStream: InputStream): Feed { + val konsume = inputStream.konsumeXml() + val feed = Feed() + + return try { + konsume.child("RDF") { + allChildrenAutoIgnore("channel") { + feed.url = attributes.getValue("about", + namespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#") + + allChildrenAutoIgnore(names) { + with(feed) { + when (tagName) { + "title" -> name = nonNullText() + "link" -> siteUrl = nonNullText() + "description" -> description = nullableText() + } + } + } + } + } + + konsume.close() + feed + } catch (e: Exception) { + throw ParseException(e.message) + } + + } + + companion object { + val names = Names.of("title", "link", "description") + } +} \ No newline at end of file diff --git a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt index c039d38d..21a225dc 100644 --- a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt +++ b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt @@ -8,6 +8,8 @@ class LocalRSSHelperTest { @Test fun standardContentTypesTest() { + assertEquals(LocalRSSHelper.getRSSType("application/rdf+xml"), + LocalRSSHelper.RSSType.RSS_1) assertEquals(LocalRSSHelper.getRSSType("application/rss+xml"), LocalRSSHelper.RSSType.RSS_2) assertEquals(LocalRSSHelper.getRSSType("application/atom+xml"), diff --git a/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt b/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt index 23d74683..47df7eb9 100644 --- a/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt +++ b/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt @@ -2,6 +2,7 @@ package com.readrops.api.localfeed import com.readrops.api.localfeed.atom.ATOMFeedAdapter import com.readrops.api.localfeed.atom.ATOMItemsAdapter +import com.readrops.api.localfeed.rss1.RSS1FeedAdapter import com.readrops.api.localfeed.rss2.RSS2FeedAdapter import com.readrops.api.localfeed.rss2.RSS2ItemsAdapter import junit.framework.TestCase.assertTrue @@ -12,6 +13,7 @@ class XmlAdapterTest { @Test fun xmlFeedAdapterFactoryTest() { + assertTrue(XmlAdapter.xmlFeedAdapterFactory(LocalRSSHelper.RSSType.RSS_1) is RSS1FeedAdapter) assertTrue(XmlAdapter.xmlFeedAdapterFactory(LocalRSSHelper.RSSType.RSS_2) is RSS2FeedAdapter) assertTrue(XmlAdapter.xmlFeedAdapterFactory(LocalRSSHelper.RSSType.ATOM) is ATOMFeedAdapter) From c742c4fbf2625d2c067e2ea6a1de0c9f47f6599f Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Wed, 23 Sep 2020 22:02:41 +0200 Subject: [PATCH 32/58] Add adapter for RSS1 items --- .../localfeed/rss1/rss1_items_no_date.xml | 43 +++++++ .../localfeed/rss1/rss1_items_no_link.xml | 40 ++++++ .../localfeed/rss1/rss1_items_no_title.xml | 43 +++++++ .../rss1/rss1_items_special_cases.xml | 120 ++++++++++++++++++ .../localfeed/rss1/RSS1ItemsAdapterTest.kt | 70 ++++++++++ .../com/readrops/api/localfeed/XmlAdapter.kt | 2 + .../api/localfeed/rss1/RSS1ItemsAdapter.kt | 68 ++++++++++ .../readrops/api/localfeed/XmlAdapterTest.kt | 2 + 8 files changed, 388 insertions(+) create mode 100644 api/src/androidTest/assets/localfeed/rss1/rss1_items_no_date.xml create mode 100644 api/src/androidTest/assets/localfeed/rss1/rss1_items_no_link.xml create mode 100644 api/src/androidTest/assets/localfeed/rss1/rss1_items_no_title.xml create mode 100644 api/src/androidTest/assets/localfeed/rss1/rss1_items_special_cases.xml create mode 100644 api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt create mode 100644 api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt diff --git a/api/src/androidTest/assets/localfeed/rss1/rss1_items_no_date.xml b/api/src/androidTest/assets/localfeed/rss1/rss1_items_no_date.xml new file mode 100644 index 00000000..b4aaab5c --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss1/rss1_items_no_date.xml @@ -0,0 +1,43 @@ + + + Google Expands its Flutter Development Kit To Windows Apps + + https://developers.slashdot.org/story/20/09/23/1616231/google-expands-its-flutter-development-kit-to-windows-apps?utm_source=rss1.0mainlinkanon&utm_medium=feed + + Google has announced that Flutter, its open source UI development kit for + building cross-platform software from the same codebase, is finally available for + Windows apps in alpha. From a report:For the world's leading desktop operating system + with some 1 billion installations of Windows 10 alone, this has been a long time coming. + Flutter's alpha incarnation was initially launched at Google's I/O developer conference + back in 2017, before arriving in beta less than a year later. In its original guise, + Flutter was designed for Android and iOS app development, but it has since expanded to + cover the web, MacOS, and Linux, which are currently available in various alpha or beta + iterations. Developers have had to consider unique platform-specific factors when + designing for the desktop or mobile phones, such as different screen sizes and how + people interact with their devices. On smartphones, people typically use touch and + swipe-based gestures, while keyboards and mice are commonly used on PCs and laptops. + This means Flutter has had to expand its support to cover the additional inputs.<p><div + class="share_submission" style="position:relative;"> <a class="slashpop" + href="http://twitter.com/home?status=Google+Expands+its+Flutter+Development+Kit+To+Windows+Apps%3A+https%3A%2F%2Fbit.ly%2F32X36MW"><img + src="https://a.fsdn.com/sd/twitter_icon_large.png"></a> <a class="slashpop" + href="http://www.facebook.com/sharer.php?u=https%3A%2F%2Fdevelopers.slashdot.org%2Fstory%2F20%2F09%2F23%2F1616231%2Fgoogle-expands-its-flutter-development-kit-to-windows-apps%3Futm_source%3Dslashdot%26utm_medium%3Dfacebook"><img + src="https://a.fsdn.com/sd/facebook_icon_large.png"></a> + + + </div></p><p><a + href="https://developers.slashdot.org/story/20/09/23/1616231/google-expands-its-flutter-development-kit-to-windows-apps?utm_source=rss1.0moreanon&amp;utm_medium=feed">Read + more of this story</a> at Slashdot.</p><iframe + src="https://slashdot.org/slashdot-it.pl?op=discuss&amp;id=17251868&amp;smallembed=1" + style="height: 300px; width: 100%; border: none;"></iframe> + + msmash + programming + how-about-that + developers + 1 + 1,1,1,1,0,0,0 + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/rss1/rss1_items_no_link.xml b/api/src/androidTest/assets/localfeed/rss1/rss1_items_no_link.xml new file mode 100644 index 00000000..b927c248 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss1/rss1_items_no_link.xml @@ -0,0 +1,40 @@ + + + Google Expands its Flutter Development Kit To Windows Apps + Google has announced that Flutter, its open source UI development kit for + building cross-platform software from the same codebase, is finally available for + Windows apps in alpha. From a report:For the world's leading desktop operating system + with some 1 billion installations of Windows 10 alone, this has been a long time coming. + Flutter's alpha incarnation was initially launched at Google's I/O developer conference + back in 2017, before arriving in beta less than a year later. In its original guise, + Flutter was designed for Android and iOS app development, but it has since expanded to + cover the web, MacOS, and Linux, which are currently available in various alpha or beta + iterations. Developers have had to consider unique platform-specific factors when + designing for the desktop or mobile phones, such as different screen sizes and how + people interact with their devices. On smartphones, people typically use touch and + swipe-based gestures, while keyboards and mice are commonly used on PCs and laptops. + This means Flutter has had to expand its support to cover the additional inputs.<p><div + class="share_submission" style="position:relative;"> <a class="slashpop" + href="http://twitter.com/home?status=Google+Expands+its+Flutter+Development+Kit+To+Windows+Apps%3A+https%3A%2F%2Fbit.ly%2F32X36MW"><img + src="https://a.fsdn.com/sd/twitter_icon_large.png"></a> <a class="slashpop" + href="http://www.facebook.com/sharer.php?u=https%3A%2F%2Fdevelopers.slashdot.org%2Fstory%2F20%2F09%2F23%2F1616231%2Fgoogle-expands-its-flutter-development-kit-to-windows-apps%3Futm_source%3Dslashdot%26utm_medium%3Dfacebook"><img + src="https://a.fsdn.com/sd/facebook_icon_large.png"></a> + + + </div></p><p><a + href="https://developers.slashdot.org/story/20/09/23/1616231/google-expands-its-flutter-development-kit-to-windows-apps?utm_source=rss1.0moreanon&amp;utm_medium=feed">Read + more of this story</a> at Slashdot.</p><iframe + src="https://slashdot.org/slashdot-it.pl?op=discuss&amp;id=17251868&amp;smallembed=1" + style="height: 300px; width: 100%; border: none;"></iframe> + + msmash + 2020-09-23T16:15:00+00:00 + programming + how-about-that + developers + 1 + 1,1,1,1,0,0,0 + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/rss1/rss1_items_no_title.xml b/api/src/androidTest/assets/localfeed/rss1/rss1_items_no_title.xml new file mode 100644 index 00000000..96e0c123 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss1/rss1_items_no_title.xml @@ -0,0 +1,43 @@ + + + + https://developers.slashdot.org/story/20/09/23/1616231/google-expands-its-flutter-development-kit-to-windows-apps?utm_source=rss1.0mainlinkanon&utm_medium=feed + + Google has announced that Flutter, its open source UI development kit for + building cross-platform software from the same codebase, is finally available for + Windows apps in alpha. From a report:For the world's leading desktop operating system + with some 1 billion installations of Windows 10 alone, this has been a long time coming. + Flutter's alpha incarnation was initially launched at Google's I/O developer conference + back in 2017, before arriving in beta less than a year later. In its original guise, + Flutter was designed for Android and iOS app development, but it has since expanded to + cover the web, MacOS, and Linux, which are currently available in various alpha or beta + iterations. Developers have had to consider unique platform-specific factors when + designing for the desktop or mobile phones, such as different screen sizes and how + people interact with their devices. On smartphones, people typically use touch and + swipe-based gestures, while keyboards and mice are commonly used on PCs and laptops. + This means Flutter has had to expand its support to cover the additional inputs.<p><div + class="share_submission" style="position:relative;"> <a class="slashpop" + href="http://twitter.com/home?status=Google+Expands+its+Flutter+Development+Kit+To+Windows+Apps%3A+https%3A%2F%2Fbit.ly%2F32X36MW"><img + src="https://a.fsdn.com/sd/twitter_icon_large.png"></a> <a class="slashpop" + href="http://www.facebook.com/sharer.php?u=https%3A%2F%2Fdevelopers.slashdot.org%2Fstory%2F20%2F09%2F23%2F1616231%2Fgoogle-expands-its-flutter-development-kit-to-windows-apps%3Futm_source%3Dslashdot%26utm_medium%3Dfacebook"><img + src="https://a.fsdn.com/sd/facebook_icon_large.png"></a> + + + </div></p><p><a + href="https://developers.slashdot.org/story/20/09/23/1616231/google-expands-its-flutter-development-kit-to-windows-apps?utm_source=rss1.0moreanon&amp;utm_medium=feed">Read + more of this story</a> at Slashdot.</p><iframe + src="https://slashdot.org/slashdot-it.pl?op=discuss&amp;id=17251868&amp;smallembed=1" + style="height: 300px; width: 100%; border: none;"></iframe> + + msmash + 2020-09-23T16:15:00+00:00 + programming + how-about-that + developers + 1 + 1,1,1,1,0,0,0 + + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/rss1/rss1_items_special_cases.xml b/api/src/androidTest/assets/localfeed/rss1/rss1_items_special_cases.xml new file mode 100644 index 00000000..2624009b --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss1/rss1_items_special_cases.xml @@ -0,0 +1,120 @@ + + + + Slashdot + https://slashdot.org/ + News for nerds, stuff that matters + en-us + Copyright 1997-2016, SlashdotMedia. All Rights Reserved. + 2020-09-23T16:20:20+00:00 + Dice + help@slashdot.org + Technology + 1970-01-01T00:00+00:00 + 1 + hourly + + + + + + + + + + + + + + + + + + + + + + + + + + + Slashdot + https://a.fsdn.com/sd/topics/topicslashdot.gif + https://slashdot.org/ + + + A New York Clock That Told Time Now Tells the Time Remaining + For more than 20 years, Metronome, which includes a 62-foot-wide 15-digit + electronic clock that faces Union Square in Manhattan, has been one of the city's most + prominent and baffling public art projects. Its digital display once told the time in + its own unique way, counting the hours, minutes and seconds (and fractions thereof) to + and from midnight. But for years observers who did not understand how it worked + suggested that it was measuring the acres of rainforest destroyed each year, tracking + the world population or even that it had something to do with pi. On Saturday Metronome + adopted a new ecologically sensitive mission. From a report: Now, instead of measuring + 24-hour cycles, it is measuring what two artists, Gan Golan and Andrew Boyd, present as + a critical window for action to prevent the effects of global warming from becoming + irreversible. On Saturday at 3:20 p.m., messages including "The Earth has a deadline" + began to appear on the display. Then numbers -- 7:103:15:40:07 -- showed up, + representing the years, days, hours, minutes and seconds until that deadline. As a + handful of supporters watched, the number -- which the artists said was based on + calculations by the Mercator Research Institute on Global Commons and Climate Change in + Berlin -- began ticking down, second by second. + + "This is our way to shout that number from the rooftops." Mr. Golan said just before the + countdown began. "The world is literally counting on us." The Climate Clock, as the two + artists call their project, will be displayed on the 14th Street building, One Union + Square South, through Sept. 27, the end of Climate Week. The creators say their aim is + to arrange for the clock to be permanently displayed, there or elsewhere. Mr. Golan said + he came up with the idea to publicly illustrate the urgency of combating climate change + about two years ago, shortly after his daughter was born. He asked Mr. Boyd, an activist + from the Lower East Side, to work with him on the project.<p><div + class="share_submission" style="position:relative;"> <a class="slashpop" + href="http://twitter.com/home?status=A+New+York+Clock+That+Told+Time+Now+Tells+the+Time+Remaining%3A+https%3A%2F%2Fbit.ly%2F2HrAt2b"><img + src="https://a.fsdn.com/sd/twitter_icon_large.png"></a> <a class="slashpop" + href="http://www.facebook.com/sharer.php?u=https%3A%2F%2Fnews.slashdot.org%2Fstory%2F20%2F09%2F23%2F1420240%2Fa-new-york-clock-that-told-time-now-tells-the-time-remaining%3Futm_source%3Dslashdot%26utm_medium%3Dfacebook"><img + src="https://a.fsdn.com/sd/facebook_icon_large.png"></a> + + + </div></p><p><a + href="https://news.slashdot.org/story/20/09/23/1420240/a-new-york-clock-that-told-time-now-tells-the-time-remaining?utm_source=rss1.0moreanon&amp;utm_medium=feed">Read + more of this story</a> at Slashdot.</p><iframe + src="https://slashdot.org/slashdot-it.pl?op=discuss&amp;id=17251272&amp;smallembed=1" + style="height: 300px; width: 100%; border: none;"></iframe> + + msmash + + creator 2 + creator 3 + creator 4 + creator 5 + 2020-09-23T14:10:00+00:00 + news + how-about-that + news + 43 + 43,38,33,27,6,1,0 + + \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt new file mode 100644 index 00000000..178220da --- /dev/null +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt @@ -0,0 +1,70 @@ +package com.readrops.api.localfeed.rss1 + +import android.content.Context +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.readrops.api.utils.DateUtils +import com.readrops.api.utils.ParseException +import junit.framework.TestCase.assertEquals +import junit.framework.TestCase.assertNotNull +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class RSS1ItemsAdapterTest { + + private val context: Context = InstrumentationRegistry.getInstrumentation().context + + private val adapter = RSS1ItemsAdapter() + + @Test + fun normalCasesTest() { + val stream = context.resources.assets.open("localfeed/rss1/rss1_feed.xml") + + val items = adapter.fromXml(stream) + val item = items.first() + + assertEquals(items.size, 4) + assertEquals(item.title, "Google Expands its Flutter Development Kit To Windows Apps") + assertEquals(item.link.trim(), "https://developers.slashdot.org/story/20/09/23/1616231/google-expands-" + + "its-flutter-development-kit-to-windows-apps?utm_source=rss1.0mainlinkanon&utm_medium=feed") + assertEquals(item.guid.trim(), "https://developers.slashdot.org/story/20/09/23/1616231/google-expands-" + + "its-flutter-development-kit-to-windows-apps?utm_source=rss1.0mainlinkanon&utm_medium=feed") + assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("2020-09-23T16:15:00+00:00")) + assertEquals(item.author, "msmash") + assertNotNull(item.description) + } + + @Test + fun specialCasesTest() { + val stream = context.resources.assets.open("localfeed/rss1/rss1_items_special_cases.xml") + + val item = adapter.fromXml(stream).first() + + assertEquals(item.author, "msmash, creator 2, creator 3, creator 4, ...") + assertEquals(item.link, "https://news.slashdot.org/story/20/09/23/1420240/a-new-york-clock-" + + "that-told-time-now-tells-the-time-remaining?utm_source=rss1.0mainlinkanon&utm_medium=feed") + } + + @Test + fun nullTitleTest() { + val stream = context.resources.assets.open("localfeed/rss1/rss1_items_no_title.xml") + + Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + } + + @Test + fun nullLinkTest() { + val stream = context.resources.assets.open("localfeed/rss1/rss1_items_no_link.xml") + + Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + } + + @Test + fun nullDateTest() { + val stream = context.resources.assets.open("localfeed/rss1/rss1_items_no_date.xml") + + Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + } +} \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt index 1641c00f..91807c54 100644 --- a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt @@ -3,6 +3,7 @@ package com.readrops.api.localfeed import com.readrops.api.localfeed.atom.ATOMFeedAdapter import com.readrops.api.localfeed.atom.ATOMItemsAdapter import com.readrops.api.localfeed.rss1.RSS1FeedAdapter +import com.readrops.api.localfeed.rss1.RSS1ItemsAdapter import com.readrops.api.localfeed.rss2.RSS2FeedAdapter import com.readrops.api.localfeed.rss2.RSS2ItemsAdapter import com.readrops.db.entities.Feed @@ -25,6 +26,7 @@ interface XmlAdapter { fun xmlItemsAdapterFactory(type: LocalRSSHelper.RSSType): XmlAdapter> { return when (type) { + LocalRSSHelper.RSSType.RSS_1 -> RSS1ItemsAdapter() LocalRSSHelper.RSSType.RSS_2 -> RSS2ItemsAdapter() LocalRSSHelper.RSSType.ATOM -> ATOMItemsAdapter() else -> throw IllegalArgumentException("Unknown RSS type : $type") diff --git a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt new file mode 100644 index 00000000..26b6e239 --- /dev/null +++ b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt @@ -0,0 +1,68 @@ +package com.readrops.api.localfeed.rss1 + +import com.gitlab.mvysny.konsumexml.Names +import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore +import com.gitlab.mvysny.konsumexml.konsumeXml +import com.readrops.api.localfeed.XmlAdapter +import com.readrops.api.utils.DateUtils +import com.readrops.api.utils.ParseException +import com.readrops.api.utils.nonNullText +import com.readrops.api.utils.nullableText +import com.readrops.db.entities.Item +import java.io.InputStream + +class RSS1ItemsAdapter : XmlAdapter> { + + override fun fromXml(inputStream: InputStream): List { + val konsume = inputStream.konsumeXml() + val items = arrayListOf() + + return try { + konsume.child("RDF") { + allChildrenAutoIgnore("item") { + val authors = arrayListOf() + val about = attributes.getValueOpt("about", + namespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#") + + val item = Item().apply { + allChildrenAutoIgnore(names) { + when (tagName) { + "title" -> title = nonNullText() + "link" -> link = nullableText() + "dc:date" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) + "dc:creator" -> authors += nullableText() + "description" -> description = nullableText(failOnElement = false) + else -> skipContents() + } + } + } + + item.guid = item.link + if (authors.filterNotNull().isNotEmpty()) item.author = authors.filterNotNull().joinToString(limit = 4) + if (item.link == null) item.link = about + + validateItem(item) + + items += item + } + } + + konsume.close() + items + } catch (e: Exception) { + throw ParseException(e.message) + } + } + + private fun validateItem(item: Item) { + when { + item.title == null -> throw ParseException("Item title is required") + item.link == null -> throw ParseException("Item link is required") + item.pubDate == null -> throw ParseException("Item date is required") + } + } + + companion object { + val names = Names.of("title", "description", "date", "link", "creator") + } +} \ No newline at end of file diff --git a/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt b/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt index 47df7eb9..64e73791 100644 --- a/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt +++ b/api/src/test/java/com/readrops/api/localfeed/XmlAdapterTest.kt @@ -3,6 +3,7 @@ package com.readrops.api.localfeed import com.readrops.api.localfeed.atom.ATOMFeedAdapter import com.readrops.api.localfeed.atom.ATOMItemsAdapter import com.readrops.api.localfeed.rss1.RSS1FeedAdapter +import com.readrops.api.localfeed.rss1.RSS1ItemsAdapter import com.readrops.api.localfeed.rss2.RSS2FeedAdapter import com.readrops.api.localfeed.rss2.RSS2ItemsAdapter import junit.framework.TestCase.assertTrue @@ -22,6 +23,7 @@ class XmlAdapterTest { @Test fun xmlItemsAdapterFactoryTest() { + assertTrue(XmlAdapter.xmlItemsAdapterFactory(LocalRSSHelper.RSSType.RSS_1) is RSS1ItemsAdapter) assertTrue(XmlAdapter.xmlItemsAdapterFactory(LocalRSSHelper.RSSType.RSS_2) is RSS2ItemsAdapter) assertTrue(XmlAdapter.xmlItemsAdapterFactory(LocalRSSHelper.RSSType.ATOM) is ATOMItemsAdapter) From 22fe77d5cf950953fbf5dbf108fd107f74951335 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Wed, 23 Sep 2020 22:29:47 +0200 Subject: [PATCH 33/58] Concatenate jsonfeed authors when they are several --- .../localfeed/json/json_items_other_cases.json | 14 +++++++++++++- .../api/localfeed/json/JSONItemsAdapterTest.kt | 2 +- .../api/localfeed/json/JSONItemsAdapter.kt | 13 +++++++------ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/api/src/androidTest/assets/localfeed/json/json_items_other_cases.json b/api/src/androidTest/assets/localfeed/json/json_items_other_cases.json index e91ee941..8e54b640 100644 --- a/api/src/androidTest/assets/localfeed/json/json_items_other_cases.json +++ b/api/src/androidTest/assets/localfeed/json/json_items_other_cases.json @@ -16,11 +16,23 @@ }, { "url": "url 2", - "name": "Author 2" + "name": "" }, { "url": "url 3", "name": "Author 3" + }, + { + "url": "url 4", + "name": "Author 4" + }, + { + "url": "url 5", + "name": "Author 5" + }, + { + "url": "url 6", + "name": "Author 6" } ] } diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt index c45e3e34..97411d8b 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt @@ -51,7 +51,7 @@ class JSONItemsAdapterTest { assertEquals(item.description, "This is a summary") assertEquals(item.content, "content_html") assertEquals(item.imageLink, "https://image.com") - assertEquals(item.author, "Author 1") + assertEquals(item.author, "Author 1, Author 3, Author 4, Author 5, ...") } @Test diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt index b2fbb4f8..4a942b93 100644 --- a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt @@ -57,6 +57,7 @@ class JSONItemsAdapter : JsonAdapter>() { 7 -> pubDate = DateUtils.stringToLocalDateTime(reader.nextString()) 8 -> author = parseAuthor(reader) // jsonfeed 1.0 9 -> author = parseAuthors(reader) // jsonfeed 1.1 + else -> reader.skipValue() } } } @@ -86,9 +87,6 @@ class JSONItemsAdapter : JsonAdapter>() { return author } - /** - * Returns the first author of the array - */ private fun parseAuthors(reader: JsonReader): String? { val authors = arrayListOf() reader.beginArray() @@ -98,7 +96,10 @@ class JSONItemsAdapter : JsonAdapter>() { } reader.endArray() - return if (authors.filterNotNull().isNotEmpty()) authors.filterNotNull().first() else null + + // here, nextNullableString doesn't check if authors values are empty + return if (authors.filterNot { author -> author.isNullOrEmpty() }.isNotEmpty()) + authors.filterNot { author -> author.isNullOrEmpty() }.joinToString(limit = 4) else null } private fun validateItem(item: Item) { @@ -110,7 +111,7 @@ class JSONItemsAdapter : JsonAdapter>() { } companion object { - val names: JsonReader.Options = JsonReader.Options.of("id", "url", "title", "content_html", "content_text", - "summary", "image", "date_published", "author", "authors") + val names: JsonReader.Options = JsonReader.Options.of("id", "url", "title", + "content_html", "content_text", "summary", "image", "date_published", "author", "authors") } } \ No newline at end of file From 847739c55985946d3573eaba2e86d106b270e370 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Wed, 23 Sep 2020 22:39:12 +0200 Subject: [PATCH 34/58] Rename rss assets folder to rss2 --- .../{rss => rss2}/rss_feed_special_cases.xml | 0 .../assets/localfeed/{rss => rss2}/rss_full_feed.xml | 0 .../localfeed/{rss => rss2}/rss_items_enclosure.xml | 0 .../{rss => rss2}/rss_items_media_content.xml | 0 .../localfeed/{rss => rss2}/rss_items_no_date.xml | 0 .../localfeed/{rss => rss2}/rss_items_no_link.xml | 0 .../localfeed/{rss => rss2}/rss_items_no_title.xml | 0 .../{rss => rss2}/rss_items_other_namespaces.xml | 0 .../api/localfeed/rss2/RSS2FeedAdapterTest.kt | 4 ++-- .../api/localfeed/rss2/RSS2ItemsAdapterTest.kt | 12 ++++++------ 10 files changed, 8 insertions(+), 8 deletions(-) rename api/src/androidTest/assets/localfeed/{rss => rss2}/rss_feed_special_cases.xml (100%) rename api/src/androidTest/assets/localfeed/{rss => rss2}/rss_full_feed.xml (100%) rename api/src/androidTest/assets/localfeed/{rss => rss2}/rss_items_enclosure.xml (100%) rename api/src/androidTest/assets/localfeed/{rss => rss2}/rss_items_media_content.xml (100%) rename api/src/androidTest/assets/localfeed/{rss => rss2}/rss_items_no_date.xml (100%) rename api/src/androidTest/assets/localfeed/{rss => rss2}/rss_items_no_link.xml (100%) rename api/src/androidTest/assets/localfeed/{rss => rss2}/rss_items_no_title.xml (100%) rename api/src/androidTest/assets/localfeed/{rss => rss2}/rss_items_other_namespaces.xml (100%) diff --git a/api/src/androidTest/assets/localfeed/rss/rss_feed_special_cases.xml b/api/src/androidTest/assets/localfeed/rss2/rss_feed_special_cases.xml similarity index 100% rename from api/src/androidTest/assets/localfeed/rss/rss_feed_special_cases.xml rename to api/src/androidTest/assets/localfeed/rss2/rss_feed_special_cases.xml diff --git a/api/src/androidTest/assets/localfeed/rss/rss_full_feed.xml b/api/src/androidTest/assets/localfeed/rss2/rss_full_feed.xml similarity index 100% rename from api/src/androidTest/assets/localfeed/rss/rss_full_feed.xml rename to api/src/androidTest/assets/localfeed/rss2/rss_full_feed.xml diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_enclosure.xml b/api/src/androidTest/assets/localfeed/rss2/rss_items_enclosure.xml similarity index 100% rename from api/src/androidTest/assets/localfeed/rss/rss_items_enclosure.xml rename to api/src/androidTest/assets/localfeed/rss2/rss_items_enclosure.xml diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_media_content.xml b/api/src/androidTest/assets/localfeed/rss2/rss_items_media_content.xml similarity index 100% rename from api/src/androidTest/assets/localfeed/rss/rss_items_media_content.xml rename to api/src/androidTest/assets/localfeed/rss2/rss_items_media_content.xml diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_no_date.xml b/api/src/androidTest/assets/localfeed/rss2/rss_items_no_date.xml similarity index 100% rename from api/src/androidTest/assets/localfeed/rss/rss_items_no_date.xml rename to api/src/androidTest/assets/localfeed/rss2/rss_items_no_date.xml diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_no_link.xml b/api/src/androidTest/assets/localfeed/rss2/rss_items_no_link.xml similarity index 100% rename from api/src/androidTest/assets/localfeed/rss/rss_items_no_link.xml rename to api/src/androidTest/assets/localfeed/rss2/rss_items_no_link.xml diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_no_title.xml b/api/src/androidTest/assets/localfeed/rss2/rss_items_no_title.xml similarity index 100% rename from api/src/androidTest/assets/localfeed/rss/rss_items_no_title.xml rename to api/src/androidTest/assets/localfeed/rss2/rss_items_no_title.xml diff --git a/api/src/androidTest/assets/localfeed/rss/rss_items_other_namespaces.xml b/api/src/androidTest/assets/localfeed/rss2/rss_items_other_namespaces.xml similarity index 100% rename from api/src/androidTest/assets/localfeed/rss/rss_items_other_namespaces.xml rename to api/src/androidTest/assets/localfeed/rss2/rss_items_other_namespaces.xml diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2FeedAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2FeedAdapterTest.kt index 343cdae7..4ac3c43d 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2FeedAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2FeedAdapterTest.kt @@ -17,7 +17,7 @@ class RSS2FeedAdapterTest { @Test fun normalCasesTest() { - val stream = context.resources.assets.open("localfeed/rss/rss_full_feed.xml") + val stream = context.resources.assets.open("localfeed/rss2/rss_full_feed.xml") val feed = adapter.fromXml(stream) @@ -30,7 +30,7 @@ class RSS2FeedAdapterTest { @Test(expected = ParseException::class) fun nullTitleTest() { - val stream = context.resources.assets.open("localfeed/rss/rss_feed_special_cases.xml") + val stream = context.resources.assets.open("localfeed/rss2/rss_feed_special_cases.xml") adapter.fromXml(stream) } } \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt index b5d75e08..5a5f86eb 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt @@ -36,7 +36,7 @@ class RSS2ItemsAdapterTest { @Test fun otherNamespacesTest() { - val stream = context.resources.assets.open("localfeed/rss/rss_items_other_namespaces.xml") + val stream = context.resources.assets.open("localfeed/rss2/rss_items_other_namespaces.xml") val item = adapter.fromXml(stream)[0] assertEquals(item.guid, "guid") @@ -47,25 +47,25 @@ class RSS2ItemsAdapterTest { @Test fun noTitleTest() { - val stream = context.resources.assets.open("localfeed/rss/rss_items_no_title.xml") + val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_title.xml") Assert.assertThrows("Item title can't be null", ParseException::class.java) { adapter.fromXml(stream) } } @Test fun noLinkTest() { - val stream = context.resources.assets.open("localfeed/rss/rss_items_no_link.xml") + val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_link.xml") Assert.assertThrows("Item link can't be null", ParseException::class.java) { adapter.fromXml(stream) } } @Test fun noDateTest() { - val stream = context.resources.assets.open("localfeed/rss/rss_items_no_date.xml") + val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_date.xml") Assert.assertThrows("Item date can't be null", ParseException::class.java) { adapter.fromXml(stream) } } @Test fun enclosureTest() { - val stream = context.resources.assets.open("localfeed/rss/rss_items_enclosure.xml") + val stream = context.resources.assets.open("localfeed/rss2/rss_items_enclosure.xml") val item = adapter.fromXml(stream)[0] assertEquals(item.imageLink, "https://image1.jpg") @@ -73,7 +73,7 @@ class RSS2ItemsAdapterTest { @Test fun mediaContentTest() { - val stream = context.resources.assets.open("localfeed/rss/rss_items_media_content.xml") + val stream = context.resources.assets.open("localfeed/rss2/rss_items_media_content.xml") val item = adapter.fromXml(stream)[0] assertEquals(item.imageLink, "https://image2.jpg") From de383d5e7d80715feeddaafffa02fe4f629d82a6 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Fri, 25 Sep 2020 13:50:00 +0200 Subject: [PATCH 35/58] Parse recursively items content/description --- api/build.gradle | 2 +- .../api/localfeed/rss2/RSS2ItemsAdapterTest.kt | 6 +++--- .../readrops/api/localfeed/atom/ATOMItemsAdapter.kt | 9 +++------ .../readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt | 7 ++----- .../readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt | 10 +++++----- .../java/com/readrops/api/utils/KonsumerExtensions.kt | 8 +++++++- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/api/build.gradle b/api/build.gradle index c7fa49aa..bf8dcf78 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -53,7 +53,7 @@ dependencies { androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' androidTestImplementation 'com.squareup.okhttp3:mockwebserver:4.8.1' - implementation 'com.gitlab.mvysny.konsume-xml:konsume-xml:0.11' + implementation 'com.gitlab.mvysny.konsume-xml:konsume-xml:0.12' implementation 'com.squareup.okhttp3:okhttp:4.8.1' diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt index 5a5f86eb..646dbfeb 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt @@ -48,19 +48,19 @@ class RSS2ItemsAdapterTest { @Test fun noTitleTest() { val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_title.xml") - Assert.assertThrows("Item title can't be null", ParseException::class.java) { adapter.fromXml(stream) } + Assert.assertThrows("Item title is required", ParseException::class.java) { adapter.fromXml(stream) } } @Test fun noLinkTest() { val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_link.xml") - Assert.assertThrows("Item link can't be null", ParseException::class.java) { adapter.fromXml(stream) } + Assert.assertThrows("Item link is required", ParseException::class.java) { adapter.fromXml(stream) } } @Test fun noDateTest() { val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_date.xml") - Assert.assertThrows("Item date can't be null", ParseException::class.java) { adapter.fromXml(stream) } + Assert.assertThrows("Item date is required", ParseException::class.java) { adapter.fromXml(stream) } } @Test diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt index 3d2ccf8f..b4480e0e 100644 --- a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt @@ -4,10 +4,7 @@ import com.gitlab.mvysny.konsumexml.Names import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore import com.gitlab.mvysny.konsumexml.konsumeXml import com.readrops.api.localfeed.XmlAdapter -import com.readrops.api.utils.DateUtils -import com.readrops.api.utils.ParseException -import com.readrops.api.utils.nonNullText -import com.readrops.api.utils.nullableText +import com.readrops.api.utils.* import com.readrops.db.entities.Item import java.io.InputStream @@ -32,8 +29,8 @@ class ATOMItemsAdapter : XmlAdapter> { link = attributes["href"] } "author" -> allChildrenAutoIgnore("name") { author = text() } - "summary" -> description = nullableText() - "content" -> content = nullableText() + "summary" -> description = nullableTextRecursively() + "content" -> content = nullableTextRecursively() } } } diff --git a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt index 26b6e239..9b040cfb 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt @@ -4,10 +4,7 @@ import com.gitlab.mvysny.konsumexml.Names import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore import com.gitlab.mvysny.konsumexml.konsumeXml import com.readrops.api.localfeed.XmlAdapter -import com.readrops.api.utils.DateUtils -import com.readrops.api.utils.ParseException -import com.readrops.api.utils.nonNullText -import com.readrops.api.utils.nullableText +import com.readrops.api.utils.* import com.readrops.db.entities.Item import java.io.InputStream @@ -31,7 +28,7 @@ class RSS1ItemsAdapter : XmlAdapter> { "link" -> link = nullableText() "dc:date" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) "dc:creator" -> authors += nullableText() - "description" -> description = nullableText(failOnElement = false) + "description" -> description = nullableTextRecursively() else -> skipContents() } } diff --git a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt index fde85f49..7a5c6890 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt @@ -30,8 +30,8 @@ class RSS2ItemsAdapter : XmlAdapter> { "pubDate" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) "dc:date" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) "guid" -> guid = nullableText() - "description" -> description = text(failOnElement = false) - "content:encoded" -> content = nullableText(failOnElement = false) + "description" -> description = nullableTextRecursively() + "content:encoded" -> content = nullableTextRecursively() "enclosure" -> parseEnclosure(this, enclosures) "media:content" -> parseMediaContent(this, mediaContents) "media:group" -> allChildrenAutoIgnore("content") { @@ -80,9 +80,9 @@ class RSS2ItemsAdapter : XmlAdapter> { private fun validateItem(item: Item) { when { - item.title == null -> throw ParseException("Item title can't be null") - item.link == null -> throw ParseException("Item link can't be null") - item.pubDate == null -> throw ParseException("Item date can't be null") + item.title == null -> throw ParseException("Item title is required") + item.link == null -> throw ParseException("Item link is required") + item.pubDate == null -> throw ParseException("Item date is required") } } diff --git a/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt b/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt index f1760e85..8da218e6 100644 --- a/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt +++ b/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt @@ -2,13 +2,19 @@ package com.readrops.api.utils import com.gitlab.mvysny.konsumexml.Konsumer import com.gitlab.mvysny.konsumexml.Whitespace +import com.gitlab.mvysny.konsumexml.textRecursively fun Konsumer.nonNullText(failOnElement: Boolean = true): String { val text = text(failOnElement = failOnElement, whitespace = Whitespace.preserve) - return if (text.isNotEmpty()) text else throw ParseException("Xml field $name can't be null") + return if (text.isNotEmpty()) text else throw ParseException("$name text can't be null") } fun Konsumer.nullableText(failOnElement: Boolean = true): String? { val text = text(failOnElement = failOnElement, whitespace = Whitespace.preserve) return if (text.isNotEmpty()) text else null +} + +fun Konsumer.nullableTextRecursively(): String? { + val text = textRecursively() + return if (text.isNotEmpty()) text else null } \ No newline at end of file From a4d6139848302434a90496238310656eebe8be91 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Fri, 25 Sep 2020 14:34:19 +0200 Subject: [PATCH 36/58] Parse content:encoded element in RSS1 items --- api/src/androidTest/assets/localfeed/rss1/rss1_feed.xml | 3 ++- .../com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt | 1 + .../java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/api/src/androidTest/assets/localfeed/rss1/rss1_feed.xml b/api/src/androidTest/assets/localfeed/rss1/rss1_feed.xml index 2dae7a51..3c60e50e 100644 --- a/api/src/androidTest/assets/localfeed/rss1/rss1_feed.xml +++ b/api/src/androidTest/assets/localfeed/rss1/rss1_feed.xml @@ -1,6 +1,6 @@ + xmlns="http://purl.org/rss/1.0/" xmlns:content="http://purl.org/rss/1.0/modules/content/"> Slashdot @@ -95,6 +95,7 @@ src="https://slashdot.org/slashdot-it.pl?op=discuss&amp;id=17251868&amp;smallembed=1" style="height: 300px; width: 100%; border: none;"></iframe> + content:encoded msmash 2020-09-23T16:15:00+00:00 programming diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt index 178220da..c5dcb452 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt @@ -34,6 +34,7 @@ class RSS1ItemsAdapterTest { assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("2020-09-23T16:15:00+00:00")) assertEquals(item.author, "msmash") assertNotNull(item.description) + assertEquals(item.content, "content:encoded") } @Test diff --git a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt index 9b040cfb..d5134784 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt @@ -29,6 +29,7 @@ class RSS1ItemsAdapter : XmlAdapter> { "dc:date" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) "dc:creator" -> authors += nullableText() "description" -> description = nullableTextRecursively() + "content:encoded" -> content = nullableTextRecursively() else -> skipContents() } } @@ -60,6 +61,6 @@ class RSS1ItemsAdapter : XmlAdapter> { } companion object { - val names = Names.of("title", "description", "date", "link", "creator") + val names = Names.of("title", "description", "date", "link", "creator", "encoded") } } \ No newline at end of file From bcedd025bac544e0f55dd51cba1b486a49933385 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sat, 26 Sep 2020 13:32:30 +0200 Subject: [PATCH 37/58] Add xml content type detection for RSS1 --- .../com/readrops/api/localfeed/LocalRSSHelper.kt | 14 +++++++------- .../readrops/api/localfeed/LocalRSSHelperTest.kt | 13 ++++++++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt index b8b020e3..b0cb5431 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSHelper.kt @@ -1,7 +1,6 @@ package com.readrops.api.localfeed import java.io.InputStream -import java.util.regex.Pattern object LocalRSSHelper { @@ -11,8 +10,8 @@ object LocalRSSHelper { private const val JSONFEED_CONTENT_TYPE = "application/feed+json" private const val JSON_CONTENT_TYPE = "application/json" + private const val RSS_1_REGEX = " RSSType.RSS_1 + RSS_2_REGEX.toRegex().containsMatchIn(string) -> RSSType.RSS_2 + ATOM_REGEX.toRegex().containsMatchIn(string) -> RSSType.ATOM + else -> RSSType.UNKNOWN } reader.close() diff --git a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt index 21a225dc..dd6537ff 100644 --- a/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt +++ b/api/src/test/java/com/readrops/api/localfeed/LocalRSSHelperTest.kt @@ -30,9 +30,20 @@ class LocalRSSHelperTest { LocalRSSHelper.RSSType.UNKNOWN) } + @Test + fun rss1ContentTest() { + assertEquals(LocalRSSHelper.getRSSContentType(ByteArrayInputStream( + """ + + Date: Mon, 28 Sep 2020 14:34:06 +0200 Subject: [PATCH 38/58] Set a global limit for the authors concatenation --- api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt | 4 +++- .../java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt | 3 ++- .../java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt | 3 ++- .../java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt index 91807c54..798e476d 100644 --- a/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt @@ -11,7 +11,7 @@ import com.readrops.db.entities.Item import java.io.InputStream interface XmlAdapter { - + fun fromXml(inputStream: InputStream): T companion object { @@ -32,6 +32,8 @@ interface XmlAdapter { else -> throw IllegalArgumentException("Unknown RSS type : $type") } } + + const val AUTHORS_MAX = 4 } } diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt index 4a942b93..8ff9a80b 100644 --- a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt @@ -1,5 +1,6 @@ package com.readrops.api.localfeed.json +import com.readrops.api.localfeed.XmlAdapter.Companion.AUTHORS_MAX import com.readrops.api.utils.DateUtils import com.readrops.api.utils.ParseException import com.readrops.api.utils.nextNullableString @@ -99,7 +100,7 @@ class JSONItemsAdapter : JsonAdapter>() { // here, nextNullableString doesn't check if authors values are empty return if (authors.filterNot { author -> author.isNullOrEmpty() }.isNotEmpty()) - authors.filterNot { author -> author.isNullOrEmpty() }.joinToString(limit = 4) else null + authors.filterNot { author -> author.isNullOrEmpty() }.joinToString(limit = AUTHORS_MAX) else null } private fun validateItem(item: Item) { diff --git a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt index d5134784..ddc93353 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt @@ -4,6 +4,7 @@ import com.gitlab.mvysny.konsumexml.Names import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore import com.gitlab.mvysny.konsumexml.konsumeXml import com.readrops.api.localfeed.XmlAdapter +import com.readrops.api.localfeed.XmlAdapter.Companion.AUTHORS_MAX import com.readrops.api.utils.* import com.readrops.db.entities.Item import java.io.InputStream @@ -36,7 +37,7 @@ class RSS1ItemsAdapter : XmlAdapter> { } item.guid = item.link - if (authors.filterNotNull().isNotEmpty()) item.author = authors.filterNotNull().joinToString(limit = 4) + if (authors.filterNotNull().isNotEmpty()) item.author = authors.filterNotNull().joinToString(limit = AUTHORS_MAX) if (item.link == null) item.link = about validateItem(item) diff --git a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt index 7a5c6890..9ecc01fe 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt @@ -2,6 +2,7 @@ package com.readrops.api.localfeed.rss2 import com.gitlab.mvysny.konsumexml.* import com.readrops.api.localfeed.XmlAdapter +import com.readrops.api.localfeed.XmlAdapter.Companion.AUTHORS_MAX import com.readrops.api.utils.* import com.readrops.db.entities.Item import java.io.InputStream @@ -47,7 +48,7 @@ class RSS2ItemsAdapter : XmlAdapter> { validateItem(item) if (item.guid == null) item.guid = item.link if (item.author == null && creators.filterNotNull().isNotEmpty()) - item.author = creators.filterNotNull().first() + item.author = creators.filterNotNull().joinToString(limit = AUTHORS_MAX) if (enclosures.isNotEmpty()) item.imageLink = enclosures.first() else if (mediaContents.isNotEmpty()) item.imageLink = mediaContents.first() From f6f5f27dd47c8d56fb23702beed7da0ddbd261e2 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 29 Sep 2020 11:29:22 +0200 Subject: [PATCH 39/58] Fix RSS2 items tests --- .../com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt index 646dbfeb..0c06a30e 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt @@ -40,7 +40,7 @@ class RSS2ItemsAdapterTest { val item = adapter.fromXml(stream)[0] assertEquals(item.guid, "guid") - assertEquals(item.author, "creator 1") + assertEquals(item.author, "creator 1, creator 2, creator 3, creator 4") assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("2020-08-05T14:03:48Z")) assertEquals(item.content, "content:encoded") } From 2bbc8c3e1092e978fefeb1736d9cb24bf6a860ae Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 29 Sep 2020 22:18:56 +0200 Subject: [PATCH 40/58] Add test for RSS2 media:group element --- .../localfeed/rss2/rss_items_media_group.xml | 30 +++++++++++++++++++ .../localfeed/rss2/RSS2ItemsAdapterTest.kt | 8 +++++ .../api/localfeed/rss2/RSS2ItemsAdapter.kt | 15 ++++++---- 3 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 api/src/androidTest/assets/localfeed/rss2/rss_items_media_group.xml diff --git a/api/src/androidTest/assets/localfeed/rss2/rss_items_media_group.xml b/api/src/androidTest/assets/localfeed/rss2/rss_items_media_group.xml new file mode 100644 index 00000000..39f73f50 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss2/rss_items_media_group.xml @@ -0,0 +1,30 @@ + + + + + title + link + + 2020-08-05T14:03:48Z + + + + + + + guid + + + + + + + image2 title + + + + + + \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt index 0c06a30e..3894db35 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt @@ -78,4 +78,12 @@ class RSS2ItemsAdapterTest { assertEquals(item.imageLink, "https://image2.jpg") } + + @Test + fun mediaGroupTest() { + val stream = context.resources.assets.open("localfeed/rss2/rss_items_media_group.xml") + val item = adapter.fromXml(stream).first() + + assertEquals(item.imageLink, "https://image1.jpg") + } } \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt index 9ecc01fe..5e8d2219 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt @@ -35,11 +35,7 @@ class RSS2ItemsAdapter : XmlAdapter> { "content:encoded" -> content = nullableTextRecursively() "enclosure" -> parseEnclosure(this, enclosures) "media:content" -> parseMediaContent(this, mediaContents) - "media:group" -> allChildrenAutoIgnore("content") { - when (tagName) { - "media:content" -> parseMediaContent(this, mediaContents) - } - } + "media:group" -> parseMediaGroup(this, mediaContents) else -> skipContents() // for example media:description } } @@ -79,6 +75,15 @@ class RSS2ItemsAdapter : XmlAdapter> { konsume.skipContents() // ignore media content sub elements } + private fun parseMediaGroup(konsume: Konsumer, mediaContents: MutableList) { + konsume.allChildrenAutoIgnore("content") { + when (tagName) { + "media:content" -> parseMediaContent(this, mediaContents) + else -> skipContents() + } + } + } + private fun validateItem(item: Item) { when { item.title == null -> throw ParseException("Item title is required") From d958dcbf0442eec3748402010ec5bac833ab3caf Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 29 Sep 2020 22:23:54 +0200 Subject: [PATCH 41/58] Simplify ATOMItemsAdapter fromXml method --- .../api/localfeed/atom/ATOMItemsAdapter.kt | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt index b4480e0e..9cb00c15 100644 --- a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt @@ -1,5 +1,6 @@ package com.readrops.api.localfeed.atom +import com.gitlab.mvysny.konsumexml.Konsumer import com.gitlab.mvysny.konsumexml.Names import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore import com.gitlab.mvysny.konsumexml.konsumeXml @@ -23,12 +24,8 @@ class ATOMItemsAdapter : XmlAdapter> { "title" -> title = nonNullText() "id" -> guid = nullableText() "updated" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) - "link" -> { - if (attributes.getValueOpt("rel") == null || - attributes["rel"] == "alternate") - link = attributes["href"] - } - "author" -> allChildrenAutoIgnore("name") { author = text() } + "link" -> parseLink(this, this@apply) + "author" -> allChildrenAutoIgnore("name") { author = nullableText() } "summary" -> description = nullableTextRecursively() "content" -> content = nullableTextRecursively() } @@ -49,6 +46,15 @@ class ATOMItemsAdapter : XmlAdapter> { } } + private fun parseLink(konsume: Konsumer, item: Item) { + konsume.apply { + if (attributes.getValueOpt("rel") == null || + attributes["rel"] == "alternate") + item.link = attributes["href"] + } + + } + private fun validateItem(item: Item) { when { item.title == null -> throw ParseException("Item title is required") From 2bfb061f484a0c4c4da1a563d409f4444c9d0e42 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 29 Sep 2020 22:37:16 +0200 Subject: [PATCH 42/58] Treat empty strings as null values in jsonfeed adapters --- .../api/localfeed/json/JSONFeedAdapter.kt | 3 ++- .../api/localfeed/json/JSONItemsAdapter.kt | 19 +++++++++---------- .../api/utils/JsonReaderExtensions.kt | 7 ++++++- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt index a67a6a16..772caeec 100644 --- a/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt @@ -1,6 +1,7 @@ package com.readrops.api.localfeed.json import com.readrops.api.utils.ParseException +import com.readrops.api.utils.nextNonEmptyString import com.readrops.api.utils.nextNullableString import com.readrops.db.entities.Feed import com.squareup.moshi.FromJson @@ -21,7 +22,7 @@ class JSONFeedAdapter { while (reader.hasNext()) { with(feed) { when (reader.selectName(names)) { - 0 -> name = reader.nextString() + 0 -> name = reader.nextNonEmptyString() 1 -> siteUrl = reader.nextNullableString() 2 -> url = reader.nextNullableString() 3 -> description = reader.nextNullableString() diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt index 8ff9a80b..c239fa17 100644 --- a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt @@ -3,6 +3,7 @@ package com.readrops.api.localfeed.json import com.readrops.api.localfeed.XmlAdapter.Companion.AUTHORS_MAX import com.readrops.api.utils.DateUtils import com.readrops.api.utils.ParseException +import com.readrops.api.utils.nextNonEmptyString import com.readrops.api.utils.nextNullableString import com.readrops.db.entities.Item import com.squareup.moshi.FromJson @@ -16,9 +17,8 @@ class JSONItemsAdapter : JsonAdapter>() { // not useful } - @FromJson override fun fromJson(reader: JsonReader): List { - try { + return try { val items = arrayListOf() reader.beginObject() @@ -29,7 +29,7 @@ class JSONItemsAdapter : JsonAdapter>() { } } - return items + items } catch (e: Exception) { throw ParseException(e.message) } @@ -48,14 +48,14 @@ class JSONItemsAdapter : JsonAdapter>() { while (reader.hasNext()) { with(item) { when (reader.selectName(names)) { - 0 -> guid = reader.nextString() - 1 -> link = reader.nextString() - 2 -> title = reader.nextString() + 0 -> guid = reader.nextNonEmptyString() + 1 -> link = reader.nextNonEmptyString() + 2 -> title = reader.nextNonEmptyString() 3 -> contentHtml = reader.nextNullableString() 4 -> contentText = reader.nextNullableString() 5 -> description = reader.nextNullableString() 6 -> imageLink = reader.nextNullableString() - 7 -> pubDate = DateUtils.stringToLocalDateTime(reader.nextString()) + 7 -> pubDate = DateUtils.stringToLocalDateTime(reader.nextNonEmptyString()) 8 -> author = parseAuthor(reader) // jsonfeed 1.0 9 -> author = parseAuthors(reader) // jsonfeed 1.1 else -> reader.skipValue() @@ -98,9 +98,8 @@ class JSONItemsAdapter : JsonAdapter>() { reader.endArray() - // here, nextNullableString doesn't check if authors values are empty - return if (authors.filterNot { author -> author.isNullOrEmpty() }.isNotEmpty()) - authors.filterNot { author -> author.isNullOrEmpty() }.joinToString(limit = AUTHORS_MAX) else null + return if (authors.filterNotNull().isNotEmpty()) + authors.filterNotNull().joinToString(limit = AUTHORS_MAX) else null } private fun validateItem(item: Item) { diff --git a/api/src/main/java/com/readrops/api/utils/JsonReaderExtensions.kt b/api/src/main/java/com/readrops/api/utils/JsonReaderExtensions.kt index ae8e4bde..09ecf36a 100644 --- a/api/src/main/java/com/readrops/api/utils/JsonReaderExtensions.kt +++ b/api/src/main/java/com/readrops/api/utils/JsonReaderExtensions.kt @@ -3,4 +3,9 @@ package com.readrops.api.utils import com.squareup.moshi.JsonReader fun JsonReader.nextNullableString(): String? = - if (peek() != JsonReader.Token.NULL) nextString() else nextNull() \ No newline at end of file + if (peek() != JsonReader.Token.NULL) nextString().ifEmpty { null }?.trim() else nextNull() + +fun JsonReader.nextNonEmptyString(): String { + val text = nextString() + return if (text.isNotEmpty()) text.trim() else throw ParseException("Json value can't be null") +} \ No newline at end of file From 9027a1c140feea53a0fda2d43c530d977c1014e4 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 29 Sep 2020 23:08:22 +0200 Subject: [PATCH 43/58] Add tests for Konsume-xml extensions --- .../readrops/api/utils/KonsumerExtensions.kt | 14 ++--- .../api/utils/KonsumerExtensionsTest.kt | 61 +++++++++++++++++++ 2 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 api/src/test/java/com/readrops/api/utils/KonsumerExtensionsTest.kt diff --git a/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt b/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt index 8da218e6..46de8a39 100644 --- a/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt +++ b/api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt @@ -4,17 +4,17 @@ import com.gitlab.mvysny.konsumexml.Konsumer import com.gitlab.mvysny.konsumexml.Whitespace import com.gitlab.mvysny.konsumexml.textRecursively -fun Konsumer.nonNullText(failOnElement: Boolean = true): String { - val text = text(failOnElement = failOnElement, whitespace = Whitespace.preserve) - return if (text.isNotEmpty()) text else throw ParseException("$name text can't be null") +fun Konsumer.nonNullText(): String { + val text = text(whitespace = Whitespace.preserve) + return if (text.isNotEmpty()) text.trim() else throw ParseException("$name text can't be null") } -fun Konsumer.nullableText(failOnElement: Boolean = true): String? { - val text = text(failOnElement = failOnElement, whitespace = Whitespace.preserve) - return if (text.isNotEmpty()) text else null +fun Konsumer.nullableText(): String? { + val text = text(whitespace = Whitespace.preserve) + return if (text.isNotEmpty()) text.trim() else null } fun Konsumer.nullableTextRecursively(): String? { val text = textRecursively() - return if (text.isNotEmpty()) text else null + return if (text.isNotEmpty()) text.trim() else null } \ No newline at end of file diff --git a/api/src/test/java/com/readrops/api/utils/KonsumerExtensionsTest.kt b/api/src/test/java/com/readrops/api/utils/KonsumerExtensionsTest.kt new file mode 100644 index 00000000..4cff9f41 --- /dev/null +++ b/api/src/test/java/com/readrops/api/utils/KonsumerExtensionsTest.kt @@ -0,0 +1,61 @@ +package com.readrops.api.utils + +import com.gitlab.mvysny.konsumexml.KonsumerException +import com.gitlab.mvysny.konsumexml.konsumeXml +import junit.framework.TestCase.assertEquals +import junit.framework.TestCase.assertNull +import org.junit.Test + +class KonsumerExtensionsTest { + + @Test(expected = KonsumerException::class) + fun nonNullTextNullCaseTest() { + val xml = """ + + """.trimIndent() + + xml.konsumeXml().apply { + child("description") { nonNullText() } + } + } + + @Test + fun nonNullTextNonNullCaseTest() { + val xml = """ + +description + + """.trimIndent() + + xml.konsumeXml().apply { + val description = child("description") { nonNullText() } + assertEquals(description, "description") + } + } + + @Test + fun nullableTextNullCaseTest() { + val xml = """ + + """.trimIndent() + + xml.konsumeXml().apply { + val description = child("description") { nullableText() } + assertNull(description) + } + } + + @Test + fun nullableTextNonNullCaseTest() { + val xml = """ + +description + + """.trimIndent() + + xml.konsumeXml().apply { + val description = child("description") { nullableText() } + assertEquals(description, "description") + } + } +} \ No newline at end of file From 7eecbc8e8b439feae8b27abf673e8d104996a329 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 29 Sep 2020 23:16:26 +0200 Subject: [PATCH 44/58] Check item link nullabilty before validateItem() in RSS1ItemsAdapter as item link is used as guid --- .../com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt index ddc93353..c03a2c3e 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt @@ -36,9 +36,12 @@ class RSS1ItemsAdapter : XmlAdapter> { } } - item.guid = item.link - if (authors.filterNotNull().isNotEmpty()) item.author = authors.filterNotNull().joinToString(limit = AUTHORS_MAX) if (item.link == null) item.link = about + ?: throw ParseException("RSS1 link or about element is required") + item.guid = item.link + + if (authors.filterNotNull().isNotEmpty()) item.author = authors.filterNotNull() + .joinToString(limit = AUTHORS_MAX) validateItem(item) @@ -56,7 +59,6 @@ class RSS1ItemsAdapter : XmlAdapter> { private fun validateItem(item: Item) { when { item.title == null -> throw ParseException("Item title is required") - item.link == null -> throw ParseException("Item link is required") item.pubDate == null -> throw ParseException("Item date is required") } } From 872a894db068660d95269df46dbec68f904c8090 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Tue, 29 Sep 2020 23:30:51 +0200 Subject: [PATCH 45/58] Replace RSS2 atom:link element value by response url, as atom:link element is unreliable --- .../com/readrops/api/localfeed/LocalRSSDataSourceTest.kt | 3 ++- .../java/com/readrops/api/localfeed/LocalRSSDataSource.kt | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt index 6960c9f5..880471b5 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt @@ -32,7 +32,7 @@ class LocalRSSDataSourceTest { @Before fun before() { - mockServer.start() + mockServer.start(8080) url = mockServer.url("/rss") } @@ -55,6 +55,7 @@ class LocalRSSDataSourceTest { val feed = pair?.first!! assertEquals(feed.name, "Hacker News") + assertEquals(feed.url, "http://localhost:8080/rss") assertEquals(feed.siteUrl, "https://news.ycombinator.com/") assertEquals(feed.description, "Links for the intellectually curious, ranked by readers.") diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index 15074d79..7104b40e 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -98,7 +98,7 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { val adapter = Moshi.Builder() .add(JSONFeedAdapter()) .build() - .adapter(Feed::class.java) + .adapter(Feed::class.java) adapter.fromJson(Buffer().readFrom(stream))!! } @@ -129,7 +129,9 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { private fun handleSpecialCases(feed: Feed, type: LocalRSSHelper.RSSType, response: Response) { with(feed) { if (type == LocalRSSHelper.RSSType.RSS_2) { - if (url == null) url = response.request.url.toString() + // if an atom:link element was parsed, we still replace its value as it is unreliable, + // otherwise we just add the rss url + url = response.request.url.toString() } else if (type == LocalRSSHelper.RSSType.ATOM) { if (url == null) url = response.request.url.toString() if (siteUrl == null) siteUrl = response.request.url.scheme + "://" + response.request.url.host From 6f2ff36ecebbf7e947a1891fb290cfc205607f32 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Wed, 30 Sep 2020 18:10:18 +0200 Subject: [PATCH 46/58] Add tests for Konsumer.nullableTextRecursively() --- .../api/utils/KonsumerExtensionsTest.kt | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/api/src/test/java/com/readrops/api/utils/KonsumerExtensionsTest.kt b/api/src/test/java/com/readrops/api/utils/KonsumerExtensionsTest.kt index 4cff9f41..7698dfb2 100644 --- a/api/src/test/java/com/readrops/api/utils/KonsumerExtensionsTest.kt +++ b/api/src/test/java/com/readrops/api/utils/KonsumerExtensionsTest.kt @@ -58,4 +58,30 @@ description assertEquals(description, "description") } } + + @Test + fun nullableTextRecursivelyNullCaseTest() { + val xml = """ + + """.trimIndent() + + xml.konsumeXml().apply { + val description = child("description") { nullableTextRecursively() } + assertNull(description) + } + } + + @Test + fun nullableTextRecursivelyNonNullCaseTest() { + val xml = """ + +description + + """.trimIndent() + + xml.konsumeXml().apply { + val description = child("description") { nullableTextRecursively() } + assertEquals(description, "description") + } + } } \ No newline at end of file From e41ab29264a5b19224b0fbaebf8a21af2dbb838b Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Thu, 1 Oct 2020 19:38:48 +0200 Subject: [PATCH 47/58] Rename DateUtils.stringToLocalDateTime() to parse() --- .../api/localfeed/atom/ATOMItemsAdapterTest.kt | 2 +- .../api/localfeed/json/JSONItemsAdapterTest.kt | 2 +- .../api/localfeed/rss1/RSS1ItemsAdapterTest.kt | 2 +- .../api/localfeed/rss2/RSS2ItemsAdapterTest.kt | 4 ++-- .../api/localfeed/atom/ATOMItemsAdapter.kt | 2 +- .../api/localfeed/json/JSONItemsAdapter.kt | 3 +-- .../api/localfeed/rss1/RSS1ItemsAdapter.kt | 2 +- .../api/localfeed/rss2/RSS2ItemsAdapter.kt | 4 ++-- .../java/com/readrops/api/utils/DateUtils.java | 2 +- .../java/com/readrops/api/utils/DateUtilsTest.java | 14 +++++++------- 10 files changed, 18 insertions(+), 19 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt index f6095a97..0c8085ea 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt @@ -28,7 +28,7 @@ class ATOMItemsAdapterTest { assertEquals(items.size, 4) assertEquals(item.title, "Add an option to open item url in custom tab") assertEquals(item.link, "https://github.com/readrops/Readrops/commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac") - assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("2020-09-06T21:09:59Z")) + assertEquals(item.pubDate, DateUtils.parse("2020-09-06T21:09:59Z")) assertEquals(item.author, "Shinokuni") assertEquals(item.description, "Summary") assertEquals(item.guid, "tag:github.com,2008:Grit::Commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac") diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt index 97411d8b..474ecbff 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt @@ -37,7 +37,7 @@ class JSONItemsAdapterTest { assertEquals(item.guid, "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html") assertEquals(item.title, "Acorn and 10.13") assertEquals(item.link, "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html") - assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("2017-09-25T14:27:27-07:00")) + assertEquals(item.pubDate, DateUtils.parse("2017-09-25T14:27:27-07:00")) assertEquals(item.author, "Author 1") assertNotNull(item.content) } diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt index c5dcb452..4860877d 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt @@ -31,7 +31,7 @@ class RSS1ItemsAdapterTest { "its-flutter-development-kit-to-windows-apps?utm_source=rss1.0mainlinkanon&utm_medium=feed") assertEquals(item.guid.trim(), "https://developers.slashdot.org/story/20/09/23/1616231/google-expands-" + "its-flutter-development-kit-to-windows-apps?utm_source=rss1.0mainlinkanon&utm_medium=feed") - assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("2020-09-23T16:15:00+00:00")) + assertEquals(item.pubDate, DateUtils.parse("2020-09-23T16:15:00+00:00")) assertEquals(item.author, "msmash") assertNotNull(item.description) assertEquals(item.content, "content:encoded") diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt index 3894db35..e7640d2a 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt @@ -28,7 +28,7 @@ class RSS2ItemsAdapterTest { assertEquals(item.title, "Africa declared free of wild polio") assertEquals(item.link, "https://www.bbc.com/news/world-africa-53887947") - assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("Tue, 25 Aug 2020 17:15:49 +0000")) + assertEquals(item.pubDate, DateUtils.parse("Tue, 25 Aug 2020 17:15:49 +0000")) assertEquals(item.author, "Author 1") assertEquals(item.description, "Comments") assertEquals(item.guid, "https://www.bbc.com/news/world-africa-53887947") @@ -41,7 +41,7 @@ class RSS2ItemsAdapterTest { assertEquals(item.guid, "guid") assertEquals(item.author, "creator 1, creator 2, creator 3, creator 4") - assertEquals(item.pubDate, DateUtils.stringToLocalDateTime("2020-08-05T14:03:48Z")) + assertEquals(item.pubDate, DateUtils.parse("2020-08-05T14:03:48Z")) assertEquals(item.content, "content:encoded") } diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt index 9cb00c15..d7a09e13 100644 --- a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt @@ -23,7 +23,7 @@ class ATOMItemsAdapter : XmlAdapter> { when (tagName) { "title" -> title = nonNullText() "id" -> guid = nullableText() - "updated" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) + "updated" -> pubDate = DateUtils.parse(nonNullText()) "link" -> parseLink(this, this@apply) "author" -> allChildrenAutoIgnore("name") { author = nullableText() } "summary" -> description = nullableTextRecursively() diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt index c239fa17..6dcdc239 100644 --- a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt @@ -6,7 +6,6 @@ import com.readrops.api.utils.ParseException import com.readrops.api.utils.nextNonEmptyString import com.readrops.api.utils.nextNullableString import com.readrops.db.entities.Item -import com.squareup.moshi.FromJson import com.squareup.moshi.JsonAdapter import com.squareup.moshi.JsonReader import com.squareup.moshi.JsonWriter @@ -55,7 +54,7 @@ class JSONItemsAdapter : JsonAdapter>() { 4 -> contentText = reader.nextNullableString() 5 -> description = reader.nextNullableString() 6 -> imageLink = reader.nextNullableString() - 7 -> pubDate = DateUtils.stringToLocalDateTime(reader.nextNonEmptyString()) + 7 -> pubDate = DateUtils.parse(reader.nextNonEmptyString()) 8 -> author = parseAuthor(reader) // jsonfeed 1.0 9 -> author = parseAuthors(reader) // jsonfeed 1.1 else -> reader.skipValue() diff --git a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt index c03a2c3e..83eaf08a 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt @@ -27,7 +27,7 @@ class RSS1ItemsAdapter : XmlAdapter> { when (tagName) { "title" -> title = nonNullText() "link" -> link = nullableText() - "dc:date" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) + "dc:date" -> pubDate = DateUtils.parse(nonNullText()) "dc:creator" -> authors += nullableText() "description" -> description = nullableTextRecursively() "content:encoded" -> content = nullableTextRecursively() diff --git a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt index 5e8d2219..3bc13b51 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt @@ -28,8 +28,8 @@ class RSS2ItemsAdapter : XmlAdapter> { "link" -> link = nonNullText() "author" -> author = nullableText() "dc:creator" -> creators += nullableText() - "pubDate" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) - "dc:date" -> pubDate = DateUtils.stringToLocalDateTime(nonNullText()) + "pubDate" -> pubDate = DateUtils.parse(nonNullText()) + "dc:date" -> pubDate = DateUtils.parse(nonNullText()) "guid" -> guid = nullableText() "description" -> description = nullableTextRecursively() "content:encoded" -> content = nullableTextRecursively() diff --git a/api/src/main/java/com/readrops/api/utils/DateUtils.java b/api/src/main/java/com/readrops/api/utils/DateUtils.java index 68133f3a..bfa3dad5 100644 --- a/api/src/main/java/com/readrops/api/utils/DateUtils.java +++ b/api/src/main/java/com/readrops/api/utils/DateUtils.java @@ -30,7 +30,7 @@ public final class DateUtils { */ private static final String ATOM_JSON_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"; - public static LocalDateTime stringToLocalDateTime(String value) { + public static LocalDateTime parse(String value) { DateTimeFormatter formatter = new DateTimeFormatterBuilder() .appendOptional(DateTimeFormat.forPattern(RSS_2_BASE_PATTERN + " ").getParser()) // with timezone .appendOptional(DateTimeFormat.forPattern(RSS_2_BASE_PATTERN).getParser()) // no timezone, important order here diff --git a/api/src/test/java/com/readrops/api/utils/DateUtilsTest.java b/api/src/test/java/com/readrops/api/utils/DateUtilsTest.java index 7e36f383..af3d98c7 100644 --- a/api/src/test/java/com/readrops/api/utils/DateUtilsTest.java +++ b/api/src/test/java/com/readrops/api/utils/DateUtilsTest.java @@ -11,48 +11,48 @@ public class DateUtilsTest { public void rssDateTest() { LocalDateTime dateTime = new LocalDateTime(2019, 1, 4, 22, 21, 46); - assertEquals(0, dateTime.compareTo(DateUtils.stringToLocalDateTime("Fri, 04 Jan 2019 22:21:46 GMT"))); + assertEquals(0, dateTime.compareTo(DateUtils.parse("Fri, 04 Jan 2019 22:21:46 GMT"))); } @Test public void rssDate2Test() { LocalDateTime dateTime = new LocalDateTime(2019, 1, 4, 22, 21, 46); - assertEquals(0, dateTime.compareTo(DateUtils.stringToLocalDateTime("Fri, 04 Jan 2019 22:21:46 +0000"))); + assertEquals(0, dateTime.compareTo(DateUtils.parse("Fri, 04 Jan 2019 22:21:46 +0000"))); } @Test public void rssDate3Test() { LocalDateTime dateTime = new LocalDateTime(2019, 1, 4, 22, 21, 46); - assertEquals(0, dateTime.compareTo(DateUtils.stringToLocalDateTime("Fri, 04 Jan 2019 22:21:46"))); + assertEquals(0, dateTime.compareTo(DateUtils.parse("Fri, 04 Jan 2019 22:21:46"))); } @Test public void atomJsonDateTest() { LocalDateTime dateTime = new LocalDateTime(2019, 1, 4, 22, 21, 46); - assertEquals(0, dateTime.compareTo(DateUtils.stringToLocalDateTime("2019-01-04T22:21:46+00:00"))); + assertEquals(0, dateTime.compareTo(DateUtils.parse("2019-01-04T22:21:46+00:00"))); } @Test public void atomJsonDate2Test() { LocalDateTime dateTime = new LocalDateTime(2019, 1, 4, 22, 21, 46); - assertEquals(0, dateTime.compareTo(DateUtils.stringToLocalDateTime("2019-01-04T22:21:46-0000"))); + assertEquals(0, dateTime.compareTo(DateUtils.parse("2019-01-04T22:21:46-0000"))); } @Test public void isoPatternTest() { LocalDateTime dateTime = new LocalDateTime(2020, 6, 30, 11, 39, 37, 206); - assertEquals(0, dateTime.compareTo(DateUtils.stringToLocalDateTime("2020-06-30T11:39:37.206-07:00"))); + assertEquals(0, dateTime.compareTo(DateUtils.parse("2020-06-30T11:39:37.206-07:00"))); } @Test public void edtPatternTest() { LocalDateTime dateTime = new LocalDateTime(2020, 7, 17, 16, 30, 0); - assertEquals(0, dateTime.compareTo(DateUtils.stringToLocalDateTime("Fri, 17 Jul 2020 16:30:00 EDT"))); + assertEquals(0, dateTime.compareTo(DateUtils.parse("Fri, 17 Jul 2020 16:30:00 EDT"))); } } \ No newline at end of file From 10a7b99e597804782cdc250f63c47417579d37e2 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Thu, 1 Oct 2020 21:17:51 +0200 Subject: [PATCH 48/58] Fallback to current date time if a RSS2 item doesn't have any date --- .../localfeed/rss2/RSS2ItemsAdapterTest.kt | 25 ++++---- .../api/localfeed/rss2/RSS2ItemsAdapter.kt | 63 ++++++++++--------- .../com/readrops/api/utils/DateUtils.java | 40 ++++++++---- 3 files changed, 75 insertions(+), 53 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt index e7640d2a..3911c0a5 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt @@ -6,6 +6,7 @@ import androidx.test.platform.app.InstrumentationRegistry import com.readrops.api.utils.DateUtils import com.readrops.api.utils.ParseException import junit.framework.TestCase.assertEquals +import junit.framework.TestCase.assertNotNull import org.junit.Assert import org.junit.Test import org.junit.runner.RunWith @@ -37,7 +38,7 @@ class RSS2ItemsAdapterTest { @Test fun otherNamespacesTest() { val stream = context.resources.assets.open("localfeed/rss2/rss_items_other_namespaces.xml") - val item = adapter.fromXml(stream)[0] + val item = adapter.fromXml(stream).first() assertEquals(item.guid, "guid") assertEquals(item.author, "creator 1, creator 2, creator 3, creator 4") @@ -45,28 +46,30 @@ class RSS2ItemsAdapterTest { assertEquals(item.content, "content:encoded") } + @Test + fun noDateTest() { + val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_date.xml") + val item = adapter.fromXml(stream).first() + + assertNotNull(item.pubDate) + } + @Test fun noTitleTest() { val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_title.xml") - Assert.assertThrows("Item title is required", ParseException::class.java) { adapter.fromXml(stream) } + Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } } @Test fun noLinkTest() { val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_link.xml") - Assert.assertThrows("Item link is required", ParseException::class.java) { adapter.fromXml(stream) } - } - - @Test - fun noDateTest() { - val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_date.xml") - Assert.assertThrows("Item date is required", ParseException::class.java) { adapter.fromXml(stream) } + Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } } @Test fun enclosureTest() { val stream = context.resources.assets.open("localfeed/rss2/rss_items_enclosure.xml") - val item = adapter.fromXml(stream)[0] + val item = adapter.fromXml(stream).first() assertEquals(item.imageLink, "https://image1.jpg") } @@ -74,7 +77,7 @@ class RSS2ItemsAdapterTest { @Test fun mediaContentTest() { val stream = context.resources.assets.open("localfeed/rss2/rss_items_media_content.xml") - val item = adapter.fromXml(stream)[0] + val item = adapter.fromXml(stream).first() assertEquals(item.imageLink, "https://image2.jpg") } diff --git a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt index 3bc13b51..c2c76fa7 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt @@ -5,20 +5,19 @@ import com.readrops.api.localfeed.XmlAdapter import com.readrops.api.localfeed.XmlAdapter.Companion.AUTHORS_MAX import com.readrops.api.utils.* import com.readrops.db.entities.Item +import org.joda.time.LocalDateTime import java.io.InputStream class RSS2ItemsAdapter : XmlAdapter> { override fun fromXml(inputStream: InputStream): List { - val konsume = inputStream.konsumeXml() + val konsumer = inputStream.konsumeXml() val items = mutableListOf() return try { - konsume.child("rss") { + konsumer.child("rss") { child("channel") { allChildrenAutoIgnore("item") { - val enclosures = arrayListOf() - val mediaContents = arrayListOf() val creators = arrayListOf() val item = Item().apply { @@ -28,67 +27,71 @@ class RSS2ItemsAdapter : XmlAdapter> { "link" -> link = nonNullText() "author" -> author = nullableText() "dc:creator" -> creators += nullableText() - "pubDate" -> pubDate = DateUtils.parse(nonNullText()) - "dc:date" -> pubDate = DateUtils.parse(nonNullText()) + "pubDate" -> pubDate = DateUtils.parse(nullableText()) + "dc:date" -> pubDate = DateUtils.parse(nullableText()) "guid" -> guid = nullableText() "description" -> description = nullableTextRecursively() "content:encoded" -> content = nullableTextRecursively() - "enclosure" -> parseEnclosure(this, enclosures) - "media:content" -> parseMediaContent(this, mediaContents) - "media:group" -> parseMediaGroup(this, mediaContents) + "enclosure" -> parseEnclosure(this, item = this@apply) + "media:content" -> parseMediaContent(this, item = this@apply) + "media:group" -> parseMediaGroup(this, item = this@apply) else -> skipContents() // for example media:description } } } - validateItem(item) - if (item.guid == null) item.guid = item.link - if (item.author == null && creators.filterNotNull().isNotEmpty()) - item.author = creators.filterNotNull().joinToString(limit = AUTHORS_MAX) - - if (enclosures.isNotEmpty()) item.imageLink = enclosures.first() - else if (mediaContents.isNotEmpty()) item.imageLink = mediaContents.first() + finalizeItem(item, creators) items += item } } } - konsume.close() + konsumer.close() items } catch (e: KonsumerException) { throw ParseException(e.message) } } - private fun parseEnclosure(konsume: Konsumer, enclosures: MutableList) { - if (konsume.attributes.getValueOpt("type") != null - && LibUtils.isMimeImage(konsume.attributes["type"])) - enclosures += konsume.attributes["url"] + private fun parseEnclosure(konsumer: Konsumer, item: Item) { + if (konsumer.attributes.getValueOpt("type") != null + && LibUtils.isMimeImage(konsumer.attributes["type"]) && item.imageLink == null) + item.imageLink = konsumer.attributes.getValueOpt("url") } - private fun parseMediaContent(konsume: Konsumer, mediaContents: MutableList) { - if (konsume.attributes.getValueOpt("medium") != null - && LibUtils.isMimeImage(konsume.attributes["medium"])) - mediaContents += konsume.attributes["url"] + private fun parseMediaContent(konsumer: Konsumer, item: Item) { + if (konsumer.attributes.getValueOpt("medium") != null + && LibUtils.isMimeImage(konsumer.attributes["medium"]) && item.imageLink == null) + item.imageLink = konsumer.attributes.getValueOpt("url") - konsume.skipContents() // ignore media content sub elements + konsumer.skipContents() // ignore media content sub elements } - private fun parseMediaGroup(konsume: Konsumer, mediaContents: MutableList) { - konsume.allChildrenAutoIgnore("content") { + private fun parseMediaGroup(konsumer: Konsumer, item: Item) { + konsumer.allChildrenAutoIgnore("content") { when (tagName) { - "media:content" -> parseMediaContent(this, mediaContents) + "media:content" -> parseMediaContent(this, item) else -> skipContents() } } } + private fun finalizeItem(item: Item, creators: List) { + item.apply { + validateItem(this) + + if (pubDate == null) pubDate = LocalDateTime.now() + if (guid == null) guid = link + if (author == null && creators.filterNotNull().isNotEmpty()) + author = creators.filterNotNull().joinToString(limit = AUTHORS_MAX) + } + } + private fun validateItem(item: Item) { when { item.title == null -> throw ParseException("Item title is required") item.link == null -> throw ParseException("Item link is required") - item.pubDate == null -> throw ParseException("Item date is required") } } diff --git a/api/src/main/java/com/readrops/api/utils/DateUtils.java b/api/src/main/java/com/readrops/api/utils/DateUtils.java index bfa3dad5..f681d2ad 100644 --- a/api/src/main/java/com/readrops/api/utils/DateUtils.java +++ b/api/src/main/java/com/readrops/api/utils/DateUtils.java @@ -1,5 +1,9 @@ package com.readrops.api.utils; +import android.util.Log; + +import androidx.annotation.Nullable; + import org.joda.time.LocalDateTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -9,6 +13,8 @@ import java.util.Locale; public final class DateUtils { + private static final String TAG = DateUtils.class.getSimpleName(); + /** * Base of common RSS 2 date formats. * Examples : @@ -30,20 +36,30 @@ public final class DateUtils { */ private static final String ATOM_JSON_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"; + @Nullable public static LocalDateTime parse(String value) { - DateTimeFormatter formatter = new DateTimeFormatterBuilder() - .appendOptional(DateTimeFormat.forPattern(RSS_2_BASE_PATTERN + " ").getParser()) // with timezone - .appendOptional(DateTimeFormat.forPattern(RSS_2_BASE_PATTERN).getParser()) // no timezone, important order here - .appendOptional(DateTimeFormat.forPattern(ATOM_JSON_DATE_FORMAT).getParser()) - .appendOptional(DateTimeFormat.forPattern(GMT_PATTERN).getParser()) - .appendOptional(DateTimeFormat.forPattern(OFFSET_PATTERN).getParser()) - .appendOptional(DateTimeFormat.forPattern(ISO_PATTERN).getParser()) - .appendOptional(DateTimeFormat.forPattern(EDT_PATTERN).getParser()) - .toFormatter() - .withLocale(Locale.ENGLISH) - .withOffsetParsed(); + if (value == null) { + return null; + } - return formatter.parseLocalDateTime(value); + try { + DateTimeFormatter formatter = new DateTimeFormatterBuilder() + .appendOptional(DateTimeFormat.forPattern(RSS_2_BASE_PATTERN + " ").getParser()) // with timezone + .appendOptional(DateTimeFormat.forPattern(RSS_2_BASE_PATTERN).getParser()) // no timezone, important order here + .appendOptional(DateTimeFormat.forPattern(ATOM_JSON_DATE_FORMAT).getParser()) + .appendOptional(DateTimeFormat.forPattern(GMT_PATTERN).getParser()) + .appendOptional(DateTimeFormat.forPattern(OFFSET_PATTERN).getParser()) + .appendOptional(DateTimeFormat.forPattern(ISO_PATTERN).getParser()) + .appendOptional(DateTimeFormat.forPattern(EDT_PATTERN).getParser()) + .toFormatter() + .withLocale(Locale.ENGLISH) + .withOffsetParsed(); + + return formatter.parseLocalDateTime(value); + } catch (Exception e) { + Log.d(TAG, e.getMessage()); + return null; + } } public static String formattedDateByLocal(LocalDateTime dateTime) { From 5c4cc8162843e831d12c8396ea5e71f575fc4b7d Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Fri, 2 Oct 2020 18:53:52 +0200 Subject: [PATCH 49/58] Fallback to current date time if it is missing for RSS1 and ATOM items --- .../localfeed/atom/ATOMItemsAdapterTest.kt | 23 ++++++++++-------- .../localfeed/rss1/RSS1ItemsAdapterTest.kt | 24 ++++++++++--------- .../api/localfeed/atom/ATOMItemsAdapter.kt | 18 +++++++------- .../api/localfeed/rss1/RSS1ItemsAdapter.kt | 15 ++++++------ 4 files changed, 43 insertions(+), 37 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt index 0c8085ea..3d8b850f 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/atom/ATOMItemsAdapterTest.kt @@ -5,8 +5,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry import com.readrops.api.utils.DateUtils import com.readrops.api.utils.ParseException -import junit.framework.TestCase.assertEquals -import junit.framework.TestCase.assertNotNull +import junit.framework.TestCase.* import org.junit.Assert import org.junit.Test import org.junit.runner.RunWith @@ -35,24 +34,28 @@ class ATOMItemsAdapterTest { assertNotNull(item.content) } + @Test + fun noDateTest() { + val stream = context.resources.assets.open("localfeed/atom/atom_items_no_date.xml") + + val item = adapter.fromXml(stream).first() + assertNotNull(item.pubDate) + } + @Test fun noTitleTest() { val stream = context.resources.assets.open("localfeed/atom/atom_items_no_title.xml") - Assert.assertThrows("Item title is required", ParseException::class.java) { adapter.fromXml(stream) } + val exception = Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + assertTrue(exception.message!!.contains("Item title is required")) } @Test fun noLinkTest() { val stream = context.resources.assets.open("localfeed/atom/atom_items_no_link.xml") - Assert.assertThrows("Item link is required", ParseException::class.java) { adapter.fromXml(stream) } + val exception = Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + assertTrue(exception.message!!.contains("Item link is required")) } - @Test - fun noDateTest() { - val stream = context.resources.assets.open("localfeed/atom/atom_items_no_date.xml") - - Assert.assertThrows("Item date is required", ParseException::class.java) { adapter.fromXml(stream) } - } } \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt index 4860877d..40939c27 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapterTest.kt @@ -5,8 +5,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry import com.readrops.api.utils.DateUtils import com.readrops.api.utils.ParseException -import junit.framework.TestCase.assertEquals -import junit.framework.TestCase.assertNotNull +import junit.framework.TestCase.* import org.junit.Assert import org.junit.Test import org.junit.runner.RunWith @@ -48,24 +47,27 @@ class RSS1ItemsAdapterTest { "that-told-time-now-tells-the-time-remaining?utm_source=rss1.0mainlinkanon&utm_medium=feed") } + @Test + fun nullDateTest() { + val stream = context.resources.assets.open("localfeed/rss1/rss1_items_no_date.xml") + + val item = adapter.fromXml(stream).first() + assertNotNull(item.pubDate) + } + @Test fun nullTitleTest() { val stream = context.resources.assets.open("localfeed/rss1/rss1_items_no_title.xml") - Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + val exception = Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + assertTrue(exception.message!!.contains("Item title is required")) } @Test fun nullLinkTest() { val stream = context.resources.assets.open("localfeed/rss1/rss1_items_no_link.xml") - Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } - } - - @Test - fun nullDateTest() { - val stream = context.resources.assets.open("localfeed/rss1/rss1_items_no_date.xml") - - Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + val exception = Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + assertTrue(exception.message!!.contains("RSS1 link or about element is required")) } } \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt index d7a09e13..be61a987 100644 --- a/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/atom/ATOMItemsAdapter.kt @@ -7,50 +7,53 @@ import com.gitlab.mvysny.konsumexml.konsumeXml import com.readrops.api.localfeed.XmlAdapter import com.readrops.api.utils.* import com.readrops.db.entities.Item +import org.joda.time.LocalDateTime import java.io.InputStream class ATOMItemsAdapter : XmlAdapter> { override fun fromXml(inputStream: InputStream): List { - val konsume = inputStream.konsumeXml() + val konsumer = inputStream.konsumeXml() val items = arrayListOf() return try { - konsume.child("feed") { + konsumer.child("feed") { allChildrenAutoIgnore("entry") { val item = Item().apply { allChildrenAutoIgnore(names) { when (tagName) { "title" -> title = nonNullText() "id" -> guid = nullableText() - "updated" -> pubDate = DateUtils.parse(nonNullText()) + "updated" -> pubDate = DateUtils.parse(nullableText()) "link" -> parseLink(this, this@apply) "author" -> allChildrenAutoIgnore("name") { author = nullableText() } "summary" -> description = nullableTextRecursively() "content" -> content = nullableTextRecursively() + else -> skipContents() } } } validateItem(item) + if (item.pubDate == null) item.pubDate = LocalDateTime.now() if (item.guid == null) item.guid = item.link items += item } } - konsume.close() + konsumer.close() items } catch (e: Exception) { throw ParseException(e.message) } } - private fun parseLink(konsume: Konsumer, item: Item) { - konsume.apply { + private fun parseLink(konsumer: Konsumer, item: Item) { + konsumer.apply { if (attributes.getValueOpt("rel") == null || attributes["rel"] == "alternate") - item.link = attributes["href"] + item.link = attributes.getValueOpt("href") } } @@ -59,7 +62,6 @@ class ATOMItemsAdapter : XmlAdapter> { when { item.title == null -> throw ParseException("Item title is required") item.link == null -> throw ParseException("Item link is required") - item.pubDate == null -> throw ParseException("Item date id required") } } diff --git a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt index 83eaf08a..0796fed9 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1ItemsAdapter.kt @@ -7,16 +7,17 @@ import com.readrops.api.localfeed.XmlAdapter import com.readrops.api.localfeed.XmlAdapter.Companion.AUTHORS_MAX import com.readrops.api.utils.* import com.readrops.db.entities.Item +import org.joda.time.LocalDateTime import java.io.InputStream class RSS1ItemsAdapter : XmlAdapter> { override fun fromXml(inputStream: InputStream): List { - val konsume = inputStream.konsumeXml() + val konsumer = inputStream.konsumeXml() val items = arrayListOf() return try { - konsume.child("RDF") { + konsumer.child("RDF") { allChildrenAutoIgnore("item") { val authors = arrayListOf() val about = attributes.getValueOpt("about", @@ -27,7 +28,7 @@ class RSS1ItemsAdapter : XmlAdapter> { when (tagName) { "title" -> title = nonNullText() "link" -> link = nullableText() - "dc:date" -> pubDate = DateUtils.parse(nonNullText()) + "dc:date" -> pubDate = DateUtils.parse(nullableText()) "dc:creator" -> authors += nullableText() "description" -> description = nullableTextRecursively() "content:encoded" -> content = nullableTextRecursively() @@ -36,6 +37,7 @@ class RSS1ItemsAdapter : XmlAdapter> { } } + if (item.pubDate == null) item.pubDate = LocalDateTime.now() if (item.link == null) item.link = about ?: throw ParseException("RSS1 link or about element is required") item.guid = item.link @@ -49,7 +51,7 @@ class RSS1ItemsAdapter : XmlAdapter> { } } - konsume.close() + konsumer.close() items } catch (e: Exception) { throw ParseException(e.message) @@ -57,10 +59,7 @@ class RSS1ItemsAdapter : XmlAdapter> { } private fun validateItem(item: Item) { - when { - item.title == null -> throw ParseException("Item title is required") - item.pubDate == null -> throw ParseException("Item date is required") - } + if (item.title == null) throw ParseException("Item title is required") } companion object { From 6733890c16c08b516749edeb4535a1e9717202d6 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Fri, 2 Oct 2020 19:08:12 +0200 Subject: [PATCH 50/58] Improve jsonfeed items tests and fallback to current date time if it is missing --- .../localfeed/json/json_items_no_date.json | 10 ++++++ .../localfeed/json/json_items_no_link.json | 10 ++++++ .../localfeed/json/json_items_no_title.json | 10 ++++++ .../json/json_items_required_elements.json | 22 ------------- .../localfeed/json/JSONItemsAdapterTest.kt | 33 ++++++++++--------- .../api/localfeed/json/JSONFeedAdapter.kt | 4 +-- .../api/localfeed/json/JSONItemsAdapter.kt | 5 +-- 7 files changed, 53 insertions(+), 41 deletions(-) create mode 100644 api/src/androidTest/assets/localfeed/json/json_items_no_date.json create mode 100644 api/src/androidTest/assets/localfeed/json/json_items_no_link.json create mode 100644 api/src/androidTest/assets/localfeed/json/json_items_no_title.json delete mode 100644 api/src/androidTest/assets/localfeed/json/json_items_required_elements.json diff --git a/api/src/androidTest/assets/localfeed/json/json_items_no_date.json b/api/src/androidTest/assets/localfeed/json/json_items_no_date.json new file mode 100644 index 00000000..ce9aa1fc --- /dev/null +++ b/api/src/androidTest/assets/localfeed/json/json_items_no_date.json @@ -0,0 +1,10 @@ +{ + "items": [ + { + "id": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html", + "title": "Acorn and 10.13", + "content_html": "

Happy Mac OS High Sierra release day everyone.

\n

I'm happy to say that there are no known issues with Acorn 6.0.3 or Acorn 5.6.6 when running on Mac OS 10.13 High Sierra. In fact, you might even notice that some things are actually faster and it can now open HEIF images. How awesome is that?

\n

I'm also working on some 10.13 goodies for Acorn 6 folks later this year. I can't wait to share that with you, but you'll have to wait just a little bit.

\n", + "url": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html" + } + ] +} \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/json/json_items_no_link.json b/api/src/androidTest/assets/localfeed/json/json_items_no_link.json new file mode 100644 index 00000000..e23377e6 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/json/json_items_no_link.json @@ -0,0 +1,10 @@ +{ + "items": [ + { + "id": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html", + "title": "Acorn and 10.13", + "content_html": "

Happy Mac OS High Sierra release day everyone.

\n

I'm happy to say that there are no known issues with Acorn 6.0.3 or Acorn 5.6.6 when running on Mac OS 10.13 High Sierra. In fact, you might even notice that some things are actually faster and it can now open HEIF images. How awesome is that?

\n

I'm also working on some 10.13 goodies for Acorn 6 folks later this year. I can't wait to share that with you, but you'll have to wait just a little bit.

\n", + "date_published": "2017-09-25T14:27:27-07:00" + } + ] +} \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/json/json_items_no_title.json b/api/src/androidTest/assets/localfeed/json/json_items_no_title.json new file mode 100644 index 00000000..e63cae47 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/json/json_items_no_title.json @@ -0,0 +1,10 @@ +{ + "items": [ + { + "id": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html", + "content_html": "

Happy Mac OS High Sierra release day everyone.

\n

I'm happy to say that there are no known issues with Acorn 6.0.3 or Acorn 5.6.6 when running on Mac OS 10.13 High Sierra. In fact, you might even notice that some things are actually faster and it can now open HEIF images. How awesome is that?

\n

I'm also working on some 10.13 goodies for Acorn 6 folks later this year. I can't wait to share that with you, but you'll have to wait just a little bit.

\n", + "date_published": "2017-09-25T14:27:27-07:00", + "url": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html" + } + ] +} \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/json/json_items_required_elements.json b/api/src/androidTest/assets/localfeed/json/json_items_required_elements.json deleted file mode 100644 index f46aa9a1..00000000 --- a/api/src/androidTest/assets/localfeed/json/json_items_required_elements.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "items": [ - { - "id": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html", - "content_html": "

Happy Mac OS High Sierra release day everyone.

\n

I'm happy to say that there are no known issues with Acorn 6.0.3 or Acorn 5.6.6 when running on Mac OS 10.13 High Sierra. In fact, you might even notice that some things are actually faster and it can now open HEIF images. How awesome is that?

\n

I'm also working on some 10.13 goodies for Acorn 6 folks later this year. I can't wait to share that with you, but you'll have to wait just a little bit.

\n", - "date_published": "2017-09-25T14:27:27-07:00", - "url": "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html" - }, - { - "id": "http://flyingmeat.com/blog/archives/2018/2/acorn_6.1_is_out.html", - "title": "Acorn 6.1 Is Out", - "content_html": "

Acorn 6.1 has been released.

\n

You can read a longer post about it over on Gus's blog, but the short of it is: Better, faster, smoother, stronger. And now with Metal 2 support.

\n", - "date_published": "2018-02-16T09:59:11-08:00", - }, - { - "id": "http://flyingmeat.com/blog/archives/2018/6/a_pair_of_updates.html", - "title": "A Pair of Updates", - "content_html": "

Happy summer solstice everybody! (at least for folks in the northern hemisphere, and for folks in the south… sorry. It's going to start getting brighter for you though).

\n

Today I've got a pair of minor app updates to annouce for you.

\n

First up is Acorn 6.1.3, which fixes a number of bugs including one that stemmed from trying to use QuickLook on a file that was created with Acorn 1.0. For the one or two of you that this was affecting, hurray!

\n

Next up is Retrobatch, which also includes some bug fixes, the beginnings of Voice Over support, performance improvements, and more.

\n

What's next for these apps? Work on Acorn 6.2 will begin shortly, as will Retrobatch 1.1. WWDC introduced some great new APIs that I want to take advantage of (cool new machine learning things), so that'll be a focus- as well as Dark Mode for Acorn and one other major thing I've got planned. Retrobatch will probably also get the Dark Mode treatment, but not until I've done it for Acorn first.

\n

So it's going to be a busy summer, but I'm looking forward to it.

\n", - "url": "http://flyingmeat.com/blog/archives/2018/6/a_pair_of_updates.html" - } - ] -} \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt index 474ecbff..3d62a836 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/json/JSONItemsAdapterTest.kt @@ -8,8 +8,7 @@ import com.readrops.api.utils.ParseException import com.readrops.db.entities.Item import com.squareup.moshi.Moshi import com.squareup.moshi.Types -import junit.framework.TestCase.assertEquals -import junit.framework.TestCase.assertNotNull +import junit.framework.TestCase.* import okio.Buffer import org.junit.Assert import org.junit.Test @@ -31,7 +30,7 @@ class JSONItemsAdapterTest { val stream = context.resources.assets.open("localfeed/json/json_feed.json") val items = adapter.fromJson(Buffer().readFrom(stream))!! - val item = items[0] + val item = items.first() assertEquals(items.size, 10) assertEquals(item.guid, "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html") @@ -46,7 +45,7 @@ class JSONItemsAdapterTest { fun otherCasesTest() { val stream = context.resources.assets.open("localfeed/json/json_items_other_cases.json") - val item = adapter.fromJson(Buffer().readFrom(stream))!![0] + val item = adapter.fromJson(Buffer().readFrom(stream))!!.first() assertEquals(item.description, "This is a summary") assertEquals(item.content, "content_html") @@ -55,23 +54,27 @@ class JSONItemsAdapterTest { } @Test - fun nullTitleTest() { - val stream = context.resources.assets.open("localfeed/json/json_items_required_elements.json") + fun nullDateTest() { + val stream = context.resources.assets.open("localfeed/json/json_items_no_date.json") - Assert.assertThrows("Item title is required", ParseException::class.java) { adapter.fromJson(Buffer().readFrom(stream))!![0] } + val item = adapter.fromJson(Buffer().readFrom(stream))!!.first() + assertNotNull(item.pubDate) + } + + @Test + fun nullTitleTest() { + val stream = context.resources.assets.open("localfeed/json/json_items_no_title.json") + + val exception = Assert.assertThrows(ParseException::class.java) { adapter.fromJson(Buffer().readFrom(stream)) } + assertTrue(exception.message!!.contains("Item title is required")) } @Test fun nullLinkTest() { - val stream = context.resources.assets.open("localfeed/json/json_items_required_elements.json") + val stream = context.resources.assets.open("localfeed/json/json_items_no_link.json") - Assert.assertThrows("Item link is required", ParseException::class.java) { adapter.fromJson(Buffer().readFrom(stream))!![1] } + val exception = Assert.assertThrows(ParseException::class.java) { adapter.fromJson(Buffer().readFrom(stream)) } + assertTrue(exception.message!!.contains("Item link is required")) } - @Test - fun nullDateTest() { - val stream = context.resources.assets.open("localfeed/json/json_items_required_elements.json") - - Assert.assertThrows("Item date is required", ParseException::class.java) { adapter.fromJson(Buffer().readFrom(stream))!![2] } - } } \ No newline at end of file diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt index 772caeec..81ff0828 100644 --- a/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/json/JSONFeedAdapter.kt @@ -15,7 +15,7 @@ class JSONFeedAdapter { @FromJson fun fromJson(reader: JsonReader): Feed { - try { + return try { val feed = Feed() reader.beginObject() @@ -32,7 +32,7 @@ class JSONFeedAdapter { } reader.endObject() - return feed + feed } catch (e: Exception) { throw ParseException(e.message) } diff --git a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt index 6dcdc239..e209ff3f 100644 --- a/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/json/JSONItemsAdapter.kt @@ -9,6 +9,7 @@ import com.readrops.db.entities.Item import com.squareup.moshi.JsonAdapter import com.squareup.moshi.JsonReader import com.squareup.moshi.JsonWriter +import org.joda.time.LocalDateTime class JSONItemsAdapter : JsonAdapter>() { @@ -54,7 +55,7 @@ class JSONItemsAdapter : JsonAdapter>() { 4 -> contentText = reader.nextNullableString() 5 -> description = reader.nextNullableString() 6 -> imageLink = reader.nextNullableString() - 7 -> pubDate = DateUtils.parse(reader.nextNonEmptyString()) + 7 -> pubDate = DateUtils.parse(reader.nextNullableString()) 8 -> author = parseAuthor(reader) // jsonfeed 1.0 9 -> author = parseAuthors(reader) // jsonfeed 1.1 else -> reader.skipValue() @@ -64,6 +65,7 @@ class JSONItemsAdapter : JsonAdapter>() { validateItem(item) item.content = if (contentHtml != null) contentHtml else contentText + if (item.pubDate == null) item.pubDate = LocalDateTime.now() reader.endObject() items += item @@ -105,7 +107,6 @@ class JSONItemsAdapter : JsonAdapter>() { when { item.title == null -> throw ParseException("Item title is required") item.link == null -> throw ParseException("Item link is required") - item.pubDate == null -> throw ParseException("Item date id required") } } From ca3efd45092862f829a0f5656d91c902f1310a70 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Fri, 2 Oct 2020 19:46:40 +0200 Subject: [PATCH 51/58] Add support for media:content element type attribute --- .../rss2/rss_items_media_content.xml | 19 ++++++++++++++++++- .../localfeed/rss2/RSS2ItemsAdapterTest.kt | 5 +++-- .../api/localfeed/rss2/RSS2ItemsAdapter.kt | 11 +++++++++-- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/api/src/androidTest/assets/localfeed/rss2/rss_items_media_content.xml b/api/src/androidTest/assets/localfeed/rss2/rss_items_media_content.xml index 74355f7b..fcefa3c4 100644 --- a/api/src/androidTest/assets/localfeed/rss2/rss_items_media_content.xml +++ b/api/src/androidTest/assets/localfeed/rss2/rss_items_media_content.xml @@ -18,7 +18,24 @@ - image2 title + image1 title + + + title + link + + 2020-08-05T14:03:48Z + + + + + + + guid + + + + image2 title \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt index 3911c0a5..d3844363 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt @@ -77,9 +77,10 @@ class RSS2ItemsAdapterTest { @Test fun mediaContentTest() { val stream = context.resources.assets.open("localfeed/rss2/rss_items_media_content.xml") - val item = adapter.fromXml(stream).first() + val items = adapter.fromXml(stream) - assertEquals(item.imageLink, "https://image2.jpg") + assertEquals(items.first().imageLink, "https://image1.jpg") + assertEquals(items[1].imageLink, "https://image2.jpg") } @Test diff --git a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt index c2c76fa7..ef2dfcc2 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapter.kt @@ -60,9 +60,16 @@ class RSS2ItemsAdapter : XmlAdapter> { item.imageLink = konsumer.attributes.getValueOpt("url") } + private fun isMediumImage(konsumer: Konsumer) = with(konsumer) { + attributes.getValueOpt("medium") != null && LibUtils.isMimeImage(attributes["medium"]) + } + + private fun isTypeImage(konsumer: Konsumer) = with(konsumer) { + attributes.getValueOpt("type") != null && LibUtils.isMimeImage(attributes["type"]) + } + private fun parseMediaContent(konsumer: Konsumer, item: Item) { - if (konsumer.attributes.getValueOpt("medium") != null - && LibUtils.isMimeImage(konsumer.attributes["medium"]) && item.imageLink == null) + if ((isMediumImage(konsumer) || isTypeImage(konsumer)) && item.imageLink == null) item.imageLink = konsumer.attributes.getValueOpt("url") konsumer.skipContents() // ignore media content sub elements From 101753a1a73fe547cdb126fd8cc9b2f3c194798a Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sat, 3 Oct 2020 16:29:15 +0200 Subject: [PATCH 52/58] Improve some RSS2 items tests --- .../api/localfeed/rss2/RSS2ItemsAdapterTest.kt | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt index d3844363..1d3aa690 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/rss2/RSS2ItemsAdapterTest.kt @@ -5,8 +5,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry import com.readrops.api.utils.DateUtils import com.readrops.api.utils.ParseException -import junit.framework.TestCase.assertEquals -import junit.framework.TestCase.assertNotNull +import junit.framework.TestCase.* import org.junit.Assert import org.junit.Test import org.junit.runner.RunWith @@ -23,10 +22,9 @@ class RSS2ItemsAdapterTest { val stream = context.resources.assets.open("localfeed/rss_feed.xml") val items = adapter.fromXml(stream) + val item = items.first() + assertEquals(items.size, 7) - - val item = items[0] - assertEquals(item.title, "Africa declared free of wild polio") assertEquals(item.link, "https://www.bbc.com/news/world-africa-53887947") assertEquals(item.pubDate, DateUtils.parse("Tue, 25 Aug 2020 17:15:49 +0000")) @@ -57,13 +55,17 @@ class RSS2ItemsAdapterTest { @Test fun noTitleTest() { val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_title.xml") - Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + + val exception = Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + assertTrue(exception.message!!.contains("Item title is required")) } @Test fun noLinkTest() { val stream = context.resources.assets.open("localfeed/rss2/rss_items_no_link.xml") - Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + + val exception = Assert.assertThrows(ParseException::class.java) { adapter.fromXml(stream) } + assertTrue(exception.message!!.contains("Item link is required")) } @Test From 63497bd0493fc3db1763e81786b1d2dd240894d9 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 4 Oct 2020 22:17:52 +0200 Subject: [PATCH 53/58] Always parse feed and items in the same time to detect malformed feeds --- .../api/localfeed/LocalRSSDataSourceTest.kt | 16 ++++++++-------- .../readrops/api/localfeed/LocalRSSDataSource.kt | 7 +++---- .../app/repositories/LocalFeedRepository.java | 4 ++-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt index 880471b5..9e90506d 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt @@ -51,7 +51,7 @@ class LocalRSSDataSourceTest { .addHeader(LibUtils.LAST_MODIFIED_HEADER, "Last-Modified") .setBody(Buffer().readFrom(stream))) - val pair = localRSSDataSource.queryRSSResource(url.toString(), null, true) + val pair = localRSSDataSource.queryRSSResource(url.toString(), null) val feed = pair?.first!! assertEquals(feed.name, "Hacker News") @@ -74,7 +74,7 @@ class LocalRSSDataSourceTest { .setBody(Buffer().readFrom(stream))) val headers = Headers.headersOf(LibUtils.ETAG_HEADER, "ETag", LibUtils.LAST_MODIFIED_HEADER, "Last-Modified") - localRSSDataSource.queryRSSResource(url.toString(), headers, false) + localRSSDataSource.queryRSSResource(url.toString(), headers) val request = mockServer.takeRequest() @@ -90,7 +90,7 @@ class LocalRSSDataSourceTest { .addHeader(LibUtils.CONTENT_TYPE_HEADER, "application/feed+json") .setBody(Buffer().readFrom(stream))) - val pair = localRSSDataSource.queryRSSResource(url.toString(), null, true)!! + val pair = localRSSDataSource.queryRSSResource(url.toString(), null)!! assertEquals(pair.first.name, "News from Flying Meat") assertEquals(pair.second.size, 10) @@ -100,7 +100,7 @@ class LocalRSSDataSourceTest { fun response304Test() { mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_NOT_MODIFIED)) - val pair = localRSSDataSource.queryRSSResource(url.toString(), null, false) + val pair = localRSSDataSource.queryRSSResource(url.toString(), null) assertNull(pair) } @@ -109,14 +109,14 @@ class LocalRSSDataSourceTest { fun response404Test() { mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_NOT_FOUND)) - localRSSDataSource.queryRSSResource(url.toString(), null, false) + localRSSDataSource.queryRSSResource(url.toString(), null) } @Test(expected = ParseException::class) fun noContentTypeTest() { mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK)) - localRSSDataSource.queryRSSResource(url.toString(), null, false) + localRSSDataSource.queryRSSResource(url.toString(), null) } @Test(expected = ParseException::class) @@ -124,7 +124,7 @@ class LocalRSSDataSourceTest { mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) .addHeader("Content-Type", "")) - localRSSDataSource.queryRSSResource(url.toString(), null, false) + localRSSDataSource.queryRSSResource(url.toString(), null) } @Test(expected = UnknownFormatException::class) @@ -133,7 +133,7 @@ class LocalRSSDataSourceTest { .addHeader("Content-Type", "application/xml") .setBody(" ")) - localRSSDataSource.queryRSSResource(url.toString(), null, false) + localRSSDataSource.queryRSSResource(url.toString(), null) } @Test diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index 7104b40e..2a6b5af7 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -27,12 +27,11 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { * Query RSS url * @param url url to query * @param headers request headers - * @param withItems parse items with their feed - * @return a Feed object with its items if specified by [withItems] + * @return a Feed object with its items */ @Throws(ParseException::class, UnknownFormatException::class, NetworkErrorException::class, IOException::class) @WorkerThread - fun queryRSSResource(url: String, headers: Headers?, withItems: Boolean): Pair>? { + fun queryRSSResource(url: String, headers: Headers?): Pair>? { val response = queryUrl(url, headers) return when { @@ -54,7 +53,7 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { if (type == LocalRSSHelper.RSSType.UNKNOWN) throw UnknownFormatException("Unable to guess $url RSS type") val feed = parseFeed(ByteArrayInputStream(bodyArray), type, response) - val items = if (withItems) parseItems(ByteArrayInputStream(bodyArray), type) else listOf() + val items = parseItems(ByteArrayInputStream(bodyArray), type) response.body?.close() Pair(feed, items) diff --git a/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java b/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java index dc21cb85..2167dd40 100644 --- a/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java +++ b/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java @@ -81,7 +81,7 @@ public class LocalFeedRepository extends ARepository { headers.add(LibUtils.IF_MODIFIED_HEADER, feed.getLastModified()); } - Pair> pair = dataSource.queryRSSResource(feed.getUrl(), headers.build(), true); + Pair> pair = dataSource.queryRSSResource(feed.getUrl(), headers.build()); if (pair != null) { insertNewItems(feed, pair.getSecond()); @@ -105,7 +105,7 @@ public class LocalFeedRepository extends ARepository { try { Pair> pair = dataSource.queryRSSResource(parsingResult.getUrl(), - null, false); + null); Feed feed = insertFeed(pair.getFirst(), parsingResult); if (feed != null) { From 8304e7709fc990bc428526329cd5de32727dc210 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 4 Oct 2020 22:53:00 +0200 Subject: [PATCH 54/58] Add tests for method LocalRSSDataSource.handleSpecialCases() --- .../atom/atom_feed_no_url_siteurl.xml | 7 +++++ .../rss1/rss1_feed_no_url_siteurl.xml | 31 +++++++++++++++++++ .../api/localfeed/LocalRSSDataSourceTest.kt | 28 +++++++++++++++++ .../api/localfeed/LocalRSSDataSource.kt | 7 ++++- .../api/localfeed/rss1/RSS1FeedAdapter.kt | 2 +- 5 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 api/src/androidTest/assets/localfeed/atom/atom_feed_no_url_siteurl.xml create mode 100644 api/src/androidTest/assets/localfeed/rss1/rss1_feed_no_url_siteurl.xml diff --git a/api/src/androidTest/assets/localfeed/atom/atom_feed_no_url_siteurl.xml b/api/src/androidTest/assets/localfeed/atom/atom_feed_no_url_siteurl.xml new file mode 100644 index 00000000..6b2d1530 --- /dev/null +++ b/api/src/androidTest/assets/localfeed/atom/atom_feed_no_url_siteurl.xml @@ -0,0 +1,7 @@ + + + tag:github.com,2008:/readrops/Readrops/commits/develop + Recent Commits to Readrops:develop + 2020-09-06T21:09:59Z + Here is a subtitle + \ No newline at end of file diff --git a/api/src/androidTest/assets/localfeed/rss1/rss1_feed_no_url_siteurl.xml b/api/src/androidTest/assets/localfeed/rss1/rss1_feed_no_url_siteurl.xml new file mode 100644 index 00000000..dd49bb0c --- /dev/null +++ b/api/src/androidTest/assets/localfeed/rss1/rss1_feed_no_url_siteurl.xml @@ -0,0 +1,31 @@ + + + + Slashdot + News for nerds, stuff that matters + en-us + Copyright 1997-2016, SlashdotMedia. All Rights Reserved. + 2020-09-23T16:20:20+00:00 + Dice + help@slashdot.org + Technology + 1970-01-01T00:00+00:00 + 1 + hourly + + + + + + + + Slashdot + https://a.fsdn.com/sd/topics/topicslashdot.gif + https://slashdot.org/ + + \ No newline at end of file diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt index 9e90506d..f51ce144 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt @@ -96,6 +96,34 @@ class LocalRSSDataSourceTest { assertEquals(pair.second.size, 10) } + @Test + fun specialCasesAtomTest() { + val stream = context.resources.assets.open("localfeed/atom/atom_feed_no_url_siteurl.xml") + + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader(LibUtils.CONTENT_TYPE_HEADER, "application/atom+xml") + .setBody(Buffer().readFrom(stream))) + + val pair = localRSSDataSource.queryRSSResource(url.toString(), null)!! + + assertEquals(pair.first.url, "http://localhost:8080/rss") + assertEquals(pair.first.siteUrl, "http://localhost") + } + + @Test + fun specialCasesRSS1Test() { + val stream = context.resources.assets.open("localfeed/rss1/rss1_feed_no_url_siteurl.xml") + + mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) + .addHeader(LibUtils.CONTENT_TYPE_HEADER, "application/rdf+xml") + .setBody(Buffer().readFrom(stream))) + + val pair = localRSSDataSource.queryRSSResource(url.toString(), null)!! + + assertEquals(pair.first.url, "http://localhost:8080/rss") + assertEquals(pair.first.siteUrl, "http://localhost") + } + @Test fun response304Test() { mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_NOT_MODIFIED)) diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index 2a6b5af7..594857bb 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -63,6 +63,11 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { } } + /** + * Checks if the provided url is a RSS resource + * @param url url to check + * @return true if [url] is a RSS resource, false otherwise + */ @WorkerThread fun isUrlRSSResource(url: String): Boolean { val response = queryUrl(url, null) @@ -131,7 +136,7 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { // if an atom:link element was parsed, we still replace its value as it is unreliable, // otherwise we just add the rss url url = response.request.url.toString() - } else if (type == LocalRSSHelper.RSSType.ATOM) { + } else if (type == LocalRSSHelper.RSSType.ATOM || type == LocalRSSHelper.RSSType.RSS_1) { if (url == null) url = response.request.url.toString() if (siteUrl == null) siteUrl = response.request.url.scheme + "://" + response.request.url.host } diff --git a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapter.kt b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapter.kt index 697258c1..0b7e9cb0 100644 --- a/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapter.kt +++ b/api/src/main/java/com/readrops/api/localfeed/rss1/RSS1FeedAdapter.kt @@ -19,7 +19,7 @@ class RSS1FeedAdapter : XmlAdapter { return try { konsume.child("RDF") { allChildrenAutoIgnore("channel") { - feed.url = attributes.getValue("about", + feed.url = attributes.getValueOpt("about", namespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#") allChildrenAutoIgnore(names) { From cc17c8884b1bc689daea44033050b68bdbeb7015 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Sun, 4 Oct 2020 23:13:37 +0200 Subject: [PATCH 55/58] Add tests for JsonReaderExtensions --- .../api/utils/JsonReaderExtensionsTest.kt | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 api/src/test/java/com/readrops/api/utils/JsonReaderExtensionsTest.kt diff --git a/api/src/test/java/com/readrops/api/utils/JsonReaderExtensionsTest.kt b/api/src/test/java/com/readrops/api/utils/JsonReaderExtensionsTest.kt new file mode 100644 index 00000000..0bba568b --- /dev/null +++ b/api/src/test/java/com/readrops/api/utils/JsonReaderExtensionsTest.kt @@ -0,0 +1,85 @@ +package com.readrops.api.utils + +import com.squareup.moshi.JsonReader +import junit.framework.TestCase.assertEquals +import junit.framework.TestCase.assertNull +import okio.Buffer +import org.junit.Test + +class JsonReaderExtensionsTest { + + @Test + fun nextNullableStringNullCaseTest() { + val reader = JsonReader.of(Buffer().readFrom(""" + { + "field": null + } + """.trimIndent().byteInputStream())) + + reader.beginObject() + reader.nextName() + + assertNull(reader.nextNullableString()) + reader.endObject() + } + + @Test + fun nextNullableStringEmptyCaseTest() { + val reader = JsonReader.of(Buffer().readFrom(""" + { + "field": "" + } + """.trimIndent().byteInputStream())) + + reader.beginObject() + reader.nextName() + + assertNull(reader.nextNullableString()) + reader.endObject() + } + + @Test + fun nextNullableValueNormalCaseTest() { + val reader = JsonReader.of(Buffer().readFrom(""" + { + "field": "value" + } + """.trimIndent().byteInputStream())) + + reader.beginObject() + reader.nextName() + + assertEquals(reader.nextNullableString(), "value") + reader.endObject() + } + + @Test + fun nextNonEmptyStringTest() { + val reader = JsonReader.of(Buffer().readFrom(""" + { + "field": "value" + } + """.trimIndent().byteInputStream())) + + reader.beginObject() + reader.nextName() + + assertEquals(reader.nextNullableString(), "value") + reader.endObject() + } + + @Test(expected = ParseException::class) + fun nextNonEmptyStringEmptyCaseTest() { + val reader = JsonReader.of(Buffer().readFrom(""" + { + "field": "" + } + """.trimIndent().byteInputStream())) + + reader.beginObject() + reader.nextName() + + reader.nextNonEmptyString() + } + +} \ No newline at end of file From eb5831104067eccdcb130271854975b6b56ffccc Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Mon, 5 Oct 2020 22:11:48 +0200 Subject: [PATCH 56/58] Log LocalFeedRepository.addFeeds() exceptions --- .../com/readrops/app/repositories/LocalFeedRepository.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java b/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java index 2167dd40..f5159027 100644 --- a/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java +++ b/app/src/main/java/com/readrops/app/repositories/LocalFeedRepository.java @@ -112,12 +112,16 @@ public class LocalFeedRepository extends ARepository { insertionResult.setFeed(feed); } } catch (ParseException e) { + Log.d(TAG, "addFeeds: " + e.getMessage()); insertionResult.setInsertionError(FeedInsertionResult.FeedInsertionError.PARSE_ERROR); } catch (UnknownFormatException e) { + Log.d(TAG, "addFeeds: " + e.getMessage()); insertionResult.setInsertionError(FeedInsertionResult.FeedInsertionError.FORMAT_ERROR); } catch (NetworkErrorException | IOException e) { + Log.d(TAG, "addFeeds: " + e.getMessage()); insertionResult.setInsertionError(FeedInsertionResult.FeedInsertionError.NETWORK_ERROR); } catch (Exception e) { + Log.d(TAG, "addFeeds: " + e.getMessage()); insertionResult.setInsertionError(FeedInsertionResult.FeedInsertionError.UNKNOWN_ERROR); } finally { insertionResult.setParsingResult(parsingResult); From ec53cbd6833977d5f6ffaf6982fe20f105ccedb2 Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Mon, 5 Oct 2020 22:18:02 +0200 Subject: [PATCH 57/58] Fix content-type not being parsed when getting url rss type --- .../com/readrops/api/localfeed/LocalRSSDataSourceTest.kt | 4 ++-- .../java/com/readrops/api/localfeed/LocalRSSDataSource.kt | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt index f51ce144..017fd9fa 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt @@ -167,7 +167,7 @@ class LocalRSSDataSourceTest { @Test fun isUrlResourceSuccessfulTest() { mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) - .addHeader("Content-Type", "application/atom+xml")) + .addHeader("Content-Type", "application/atom+xml; charset=UTF-8")) assertTrue(localRSSDataSource.isUrlRSSResource(url.toString())) } @@ -182,7 +182,7 @@ class LocalRSSDataSourceTest { @Test fun isUrlRSSResourceBadContentTypeTest() { mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK) - .addHeader("Content-Type", "application/xml") + .addHeader("Content-Type", "application/xml; charset=UTF-8") .setBody(" ")) assertFalse(localRSSDataSource.isUrlRSSResource(url.toString())) diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index 594857bb..20a38da9 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -73,7 +73,10 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { val response = queryUrl(url, null) return if (response.isSuccessful) { - val contentType = response.header(LibUtils.CONTENT_TYPE_HEADER) + val header = response.header(LibUtils.CONTENT_TYPE_HEADER) + ?: return false + + val contentType = LibUtils.parseContentType(header) ?: return false var type = LocalRSSHelper.getRSSType(contentType) From 6874ef2452adc1a21ad52a8711089adb11bec9ae Mon Sep 17 00:00:00 2001 From: Shinokuni Date: Mon, 5 Oct 2020 22:24:47 +0200 Subject: [PATCH 58/58] If a response doesn't have a content-type header, throw UnknownFormatException instead of ParseException --- .../java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt | 2 +- .../main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt index 017fd9fa..a03f6032 100644 --- a/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt +++ b/api/src/androidTest/java/com/readrops/api/localfeed/LocalRSSDataSourceTest.kt @@ -140,7 +140,7 @@ class LocalRSSDataSourceTest { localRSSDataSource.queryRSSResource(url.toString(), null) } - @Test(expected = ParseException::class) + @Test(expected = UnknownFormatException::class) fun noContentTypeTest() { mockServer.enqueue(MockResponse().setResponseCode(HttpURLConnection.HTTP_OK)) diff --git a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt index 20a38da9..6069b2b0 100644 --- a/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt +++ b/api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt @@ -37,7 +37,7 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) { return when { response.isSuccessful -> { val header = response.header(LibUtils.CONTENT_TYPE_HEADER) - ?: throw ParseException("Unable to get $url content-type") + ?: throw UnknownFormatException("Unable to get $url content-type") val contentType = LibUtils.parseContentType(header) ?: throw ParseException("Unable to parse $url content-type")