FreshRSS/Readrops
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Start replacing simplexml with konsume-xml by parsing RSS feed channel with it

Browse Source
This commit is contained in:
Shinokuni 2020-09-11 22:53:50 +02:00
parent 752135621d
commit 8089d9ae6c
8 changed files with 160 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
api/build.gradle
View File

@ -43,11 +43,6 @@ dependencies {
implementation fileTree(dir: 'libs', include: ['*.jar'])
implementation project(':db')
// xpp3 has a conflict with kxml when running connectedCheck task
configurations {
all*.exclude group: 'xpp3', module: 'xpp3'
}
implementation "androidx.core:core-ktx:1.2.0"
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
@ -58,6 +53,8 @@ dependencies {
androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0'
androidTestImplementation 'com.squareup.okhttp3:mockwebserver:4.8.1'
implementation 'com.gitlab.mvysny.konsume-xml:konsume-xml:0.11'
implementation 'com.squareup.okhttp3:okhttp:4.8.1'
implementation('com.squareup.retrofit2:retrofit:2.7.1') {
@ -67,7 +64,12 @@ dependencies {
exclude group: 'moshi', module: 'moshi' // moshi converter uses moshi 1.8.0 which breaks codegen 1.9.2
}
implementation 'com.squareup.retrofit2:converter-simplexml:2.7.1'
implementation ('com.squareup.retrofit2:converter-simplexml:2.7.1') {
exclude module: 'stax'
exclude module: 'stax-api'
exclude module: 'xpp3'
}
implementation 'com.squareup.retrofit2:adapter-rxjava2:2.7.1'
implementation 'com.squareup.moshi:moshi:1.9.2'

9
api/src/androidTest/assets/localfeed/rss/rss_feed_special_cases.xml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
<channel>
<title></title>
<atom:link href="https://news.ycombinator.com/feed/" />
<link>https://news.ycombinator.com/</link>
<description>Links for the intellectually curious, ranked by readers.</description>
</channel>
</rss>

9
api/src/androidTest/assets/localfeed/rss/rss_full_feed.xml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
<channel>
<title>Hacker News</title>
<atom:link href="https://news.ycombinator.com/feed/" />
<link>https://news.ycombinator.com/</link>
<description>Links for the intellectually curious, ranked by readers.</description>
</channel>
</rss>

36
api/src/androidTest/java/com/readrops/api/localfeed/rss/RSSFeedAdapterTest.kt Normal file
View File

@ -0,0 +1,36 @@
package com.readrops.api.localfeed.rss
import android.content.Context
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import com.readrops.api.utils.ParseException
import junit.framework.TestCase.assertEquals
import org.junit.Test
import org.junit.runner.RunWith
@RunWith(AndroidJUnit4::class)
class RSSFeedAdapterTest {
private val context: Context = InstrumentationRegistry.getInstrumentation().context
private val adapter = RSSFeedAdapter()
@Test
fun normalCasesTest() {
val stream = context.resources.assets.open("localfeed/rss/rss_full_feed.xml")
val feed = adapter.fromXml(stream)
assertEquals(feed.name, "Hacker News")
assertEquals(feed.url, "https://news.ycombinator.com/feed/")
assertEquals(feed.siteUrl, "https://news.ycombinator.com/")
assertEquals(feed.description, "Links for the intellectually curious, ranked by readers.")
}
@Test(expected = ParseException::class)
fun nullTitleTest() {
val stream = context.resources.assets.open("localfeed/rss/rss_feed_special_cases.xml")
adapter.fromXml(stream)
}
}

24
api/src/main/java/com/readrops/api/localfeed/LocalRSSDataSource.kt
View File

@ -4,6 +4,7 @@ import android.accounts.NetworkErrorException
import androidx.annotation.WorkerThread
import com.readrops.api.utils.LibUtils
import com.readrops.api.utils.ParseException
import com.readrops.api.utils.UnknownFormatException
import com.readrops.db.entities.Feed
import com.readrops.db.entities.Item
import okhttp3.Headers
@ -33,19 +34,21 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) {
?: throw ParseException("Unable to get $url content-type")
val contentType = LibUtils.parseContentType(header)
?: throw ParseException("Unable to get $url content-type")
?: throw ParseException("Unable to parse $url content-type")
var type = LocalRSSHelper.getRSSType(contentType)
// if we can't guess type based on content-type header, we use the content
if (type == LocalRSSHelper.RSSType.UNKNOWN) type = LocalRSSHelper.getRSSContentType(response.body?.byteStream()!!)
if (type == LocalRSSHelper.RSSType.UNKNOWN)
type = LocalRSSHelper.getRSSContentType(response.body?.byteStream()!!)
// if we can't guess type even with the content, we are unable to go further
if (type == LocalRSSHelper.RSSType.UNKNOWN) throw ParseException("Unable to guess $url RSS type")
val feed = parseFeed(response, type)
val items = if (withItems) parseItems(response.body?.byteStream()!!, type) else listOf()
return Pair(feed, items)
response.body?.close()
Pair(feed, items)
}
response.code == HttpURLConnection.HTTP_NOT_MODIFIED -> null
else -> throw NetworkErrorException("$url returned ${response.code} code : ${response.message}")
@ -78,8 +81,19 @@ class LocalRSSDataSource(private val httpClient: OkHttpClient) {
}
private fun parseFeed(response: Response, type: LocalRSSHelper.RSSType): Feed {
response.body?.close()
return Feed()
val feed = if (type != LocalRSSHelper.RSSType.JSONFEED) {
val adapter = XmlAdapter.xmlFeedAdapterFactory(type)
//adapter.fromXml(response.body?.byteStream()!!)
Feed()
} else {
Feed()
}
feed.etag = response.header(LibUtils.ETAG_HEADER)
feed.lastModified = response.header(LibUtils.IF_MODIFIED_HEADER)
return feed
}
private fun parseItems(inputStream: InputStream, type: LocalRSSHelper.RSSType): List<Item> {

20
api/src/main/java/com/readrops/api/localfeed/XmlAdapter.kt Normal file
View File

@ -0,0 +1,20 @@
package com.readrops.api.localfeed
import com.readrops.api.localfeed.rss.RSSFeedAdapter
import com.readrops.db.entities.Feed
import java.io.InputStream
interface XmlAdapter<T> {
fun fromXml(inputStream: InputStream): T
companion object {
fun xmlFeedAdapterFactory(type: LocalRSSHelper.RSSType): XmlAdapter<Feed> {
return when (type) {
LocalRSSHelper.RSSType.RSS_2 -> RSSFeedAdapter()
else -> throw Exception("Unknown RSS type : $type")
}
}
}
}

46
api/src/main/java/com/readrops/api/localfeed/rss/RSSFeedAdapter.kt Normal file
View File

@ -0,0 +1,46 @@
package com.readrops.api.localfeed.rss
import com.gitlab.mvysny.konsumexml.Names
import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore
import com.gitlab.mvysny.konsumexml.konsumeXml
import com.readrops.api.localfeed.XmlAdapter
import com.readrops.api.utils.ParseException
import com.readrops.api.utils.nonNullText
import com.readrops.api.utils.nullableText
import com.readrops.db.entities.Feed
import org.jsoup.Jsoup
import java.io.InputStream
class RSSFeedAdapter : XmlAdapter<Feed> {
override fun fromXml(inputStream: InputStream): Feed {
val konsume = inputStream.konsumeXml()
val feed = Feed()
return try {
konsume.child("rss") {
child("channel") {
allChildrenAutoIgnore(names) {
with(feed) {
when (tagName) {
"title" -> name = Jsoup.parse(nonNullText()).text()
"description" -> description = nullableText(failOnElement = false)
"link" -> siteUrl = nullableText()
"atom:link" -> url = attributes.getValueOpt("href")
}
}
}
}
}
konsume.close()
feed
} catch (e: Exception) {
throw ParseException(e.message)
}
}
companion object {
val names = Names.of("title", "description", "link")
}
}

13
api/src/main/java/com/readrops/api/utils/KonsumerExtensions.kt Normal file
View File

@ -0,0 +1,13 @@
package com.readrops.api.utils
import com.gitlab.mvysny.konsumexml.Konsumer
fun Konsumer.nonNullText(failOnElement: Boolean = true): String {
val text = text(failOnElement = failOnElement)
return if (text.isNotEmpty()) text else throw ParseException("Xml field $name can't be null")
}
fun Konsumer.nullableText(failOnElement: Boolean = true): String? {
val text = text(failOnElement = failOnElement)
return if (text.isNotEmpty()) text else null
}