fix: Improve parsing of Friendica (and other server) version formats (#376)
Previous code could return an error on Friendica version strings like `2024.03-dev-1547`. Fix this: - Extend the list of explicitly supported servers to include Fedibird, Friendica, Glitch, Hometown, Iceshrimp, Pixelfed, and Sharkey. - Add version parsing routines for these servers. - Test the version parsing routines fetching every server and version seen by Fediverse Observer (~ 2,000 servers) and ensuring that the server and version information can be parsed. Improve the error message: - Show the hostname with a `ServerRepository` error Clean up the code: - Remove the custom `resultOf` and `mapResult` functions, they have equivalents in newer versions of the library (like `runSuspendCatching`) Fixes #372
This commit is contained in:
Nik Clayton
GitHub
parent
7e3cde4c65
commit
5cfe6d055b
|
|
@ -25,3 +25,8 @@ max_line_length = off
|
|||
|
||||
[*.{yml,yaml}]
|
||||
indent_size = 2
|
||||
|
||||
# Disable ktlint on generated source code, see
|
||||
# https://github.com/JLLeitschuh/ktlint-gradle/issues/746
|
||||
[**/build/generated/source/**]
|
||||
ktlint = disabled
|
||||
|
|
|
|||
|
|
@ -3534,7 +3534,7 @@
|
|||
errorLine2=" ~~~~~~~~~~~~~~~~~~~">
|
||||
<location
|
||||
file="src/main/java/app/pachli/fragment/SFragment.kt"
|
||||
line="239"
|
||||
line="240"
|
||||
column="48"/>
|
||||
</issue>
|
||||
|
||||
|
|
|
|||
|
|
@ -127,6 +127,7 @@ abstract class SFragment : Fragment() {
|
|||
result.onFailure {
|
||||
val msg = getString(
|
||||
R.string.server_repository_error,
|
||||
accountManager.activeAccount!!.domain,
|
||||
it.msg(requireContext()),
|
||||
)
|
||||
Timber.e(msg)
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ import app.pachli.core.network.model.nodeinfo.NodeInfo
|
|||
import app.pachli.core.network.retrofit.MastodonApi
|
||||
import app.pachli.core.network.retrofit.NodeInfoApi
|
||||
import app.pachli.network.ServerRepository.Error.Capabilities
|
||||
import app.pachli.network.ServerRepository.Error.GetInstanceInfo
|
||||
import app.pachli.network.ServerRepository.Error.GetInstanceInfoV1
|
||||
import app.pachli.network.ServerRepository.Error.GetNodeInfo
|
||||
import app.pachli.network.ServerRepository.Error.GetWellKnownNodeInfo
|
||||
import app.pachli.network.ServerRepository.Error.UnsupportedSchema
|
||||
|
|
@ -75,8 +75,8 @@ class ServerRepository @Inject constructor(
|
|||
}
|
||||
|
||||
/**
|
||||
* @return the current server or a [Server.Error] subclass error if the
|
||||
* server can not be determined.
|
||||
* @return the server info or a [Server.Error] if the server info can not
|
||||
* be determined.
|
||||
*/
|
||||
private suspend fun getServer(): Result<Server, Error> = binding {
|
||||
// Fetch the /.well-known/nodeinfo document
|
||||
|
|
@ -107,7 +107,7 @@ class ServerRepository @Inject constructor(
|
|||
{
|
||||
mastodonApi.getInstanceV1().fold(
|
||||
{ Server.from(nodeInfo.software, it).mapError(::Capabilities) },
|
||||
{ Err(GetInstanceInfo(it)) },
|
||||
{ Err(GetInstanceInfoV1(it)) },
|
||||
)
|
||||
},
|
||||
).bind()
|
||||
|
|
@ -139,7 +139,7 @@ class ServerRepository @Inject constructor(
|
|||
source = error,
|
||||
)
|
||||
|
||||
data class GetInstanceInfo(val throwable: Throwable) : Error(
|
||||
data class GetInstanceInfoV1(val throwable: Throwable) : Error(
|
||||
R.string.server_repository_error_get_instance_info,
|
||||
throwable.localizedMessage,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -766,7 +766,7 @@ Your description here:\n\n
|
|||
----\n
|
||||
</string>
|
||||
|
||||
<string name="server_repository_error">Could not fetch server info: %1$s</string>
|
||||
<string name="server_repository_error">Could not fetch server info for %1$s: %2$s</string>
|
||||
<string name="server_repository_error_get_well_known_node_info">fetching /.well-known/nodeinfo failed: %1$s</string>
|
||||
<string name="server_repository_error_unsupported_schema">/.well-known/nodeinfo did not contain understandable schemas</string>
|
||||
<string name="server_repository_error_get_node_info">fetching nodeinfo %1$s failed: %2$s</string>
|
||||
|
|
|
|||
|
|
@ -19,12 +19,6 @@ package app.pachli.core.common
|
|||
|
||||
import android.content.Context
|
||||
import androidx.annotation.StringRes
|
||||
import com.github.michaelbull.result.Err
|
||||
import com.github.michaelbull.result.Ok
|
||||
import com.github.michaelbull.result.Result
|
||||
import com.github.michaelbull.result.getError
|
||||
import com.github.michaelbull.result.getOr
|
||||
import kotlin.coroutines.cancellation.CancellationException
|
||||
|
||||
/**
|
||||
* Base class for errors throughout the app.
|
||||
|
|
@ -83,46 +77,3 @@ open class PachliError(
|
|||
return context.getString(resourceId, *args.toTypedArray())
|
||||
}
|
||||
}
|
||||
|
||||
// See https://www.jacobras.nl/2022/04/resilient-use-cases-with-kotlin-result-coroutines-and-annotations/
|
||||
|
||||
/**
|
||||
* Like [runCatching], but with proper coroutines cancellation handling. Also only catches [Exception] instead of [Throwable].
|
||||
*
|
||||
* Cancellation exceptions need to be rethrown. See https://github.com/Kotlin/kotlinx.coroutines/issues/1814.
|
||||
*/
|
||||
inline fun <R> resultOf(block: () -> R): Result<R, Exception> {
|
||||
return try {
|
||||
Ok(block())
|
||||
} catch (e: CancellationException) {
|
||||
throw e
|
||||
} catch (e: Exception) {
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Like [runCatching], but with proper coroutines cancellation handling. Also only catches [Exception] instead of [Throwable].
|
||||
*
|
||||
* Cancellation exceptions need to be rethrown. See https://github.com/Kotlin/kotlinx.coroutines/issues/1814.
|
||||
*/
|
||||
inline fun <T, R> T.resultOf(block: T.() -> R): Result<R, Exception> {
|
||||
return try {
|
||||
Ok(block())
|
||||
} catch (e: CancellationException) {
|
||||
throw e
|
||||
} catch (e: Exception) {
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Like [mapCatching], but uses [resultOf] instead of [runCatching].
|
||||
*/
|
||||
inline fun <R, T> Result<T, Exception>.mapResult(transform: (value: T) -> R): Result<R, Exception> {
|
||||
val successResult = getOr { null } // getOrNull()
|
||||
return when {
|
||||
successResult != null -> resultOf { transform(successResult) }
|
||||
else -> Err(getError() ?: error("Unreachable state"))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,13 +18,21 @@
|
|||
package app.pachli.core.network
|
||||
|
||||
import androidx.annotation.StringRes
|
||||
import androidx.annotation.VisibleForTesting
|
||||
import androidx.annotation.VisibleForTesting.Companion.PRIVATE
|
||||
import app.pachli.core.common.PachliError
|
||||
import app.pachli.core.common.resultOf
|
||||
import app.pachli.core.network.Server.Error.UnparseableVersion
|
||||
import app.pachli.core.network.ServerKind.AKKOMA
|
||||
import app.pachli.core.network.ServerKind.FEDIBIRD
|
||||
import app.pachli.core.network.ServerKind.FRIENDICA
|
||||
import app.pachli.core.network.ServerKind.GLITCH
|
||||
import app.pachli.core.network.ServerKind.GOTOSOCIAL
|
||||
import app.pachli.core.network.ServerKind.HOMETOWN
|
||||
import app.pachli.core.network.ServerKind.ICESHRIMP
|
||||
import app.pachli.core.network.ServerKind.MASTODON
|
||||
import app.pachli.core.network.ServerKind.PIXELFED
|
||||
import app.pachli.core.network.ServerKind.PLEROMA
|
||||
import app.pachli.core.network.ServerKind.SHARKEY
|
||||
import app.pachli.core.network.ServerKind.UNKNOWN
|
||||
import app.pachli.core.network.ServerOperation.ORG_JOINMASTODON_FILTERS_CLIENT
|
||||
import app.pachli.core.network.ServerOperation.ORG_JOINMASTODON_FILTERS_SERVER
|
||||
|
|
@ -32,14 +40,19 @@ import app.pachli.core.network.ServerOperation.ORG_JOINMASTODON_STATUSES_TRANSLA
|
|||
import app.pachli.core.network.model.InstanceV1
|
||||
import app.pachli.core.network.model.InstanceV2
|
||||
import app.pachli.core.network.model.nodeinfo.NodeInfo
|
||||
import com.github.michaelbull.result.Ok
|
||||
import com.github.michaelbull.result.Result
|
||||
import com.github.michaelbull.result.andThen
|
||||
import com.github.michaelbull.result.binding
|
||||
import com.github.michaelbull.result.getOrElse
|
||||
import com.github.michaelbull.result.coroutines.runSuspendCatching
|
||||
import com.github.michaelbull.result.mapError
|
||||
import com.github.michaelbull.result.recover
|
||||
import com.github.michaelbull.result.toResultOr
|
||||
import io.github.z4kn4fein.semver.Version
|
||||
import io.github.z4kn4fein.semver.constraints.Constraint
|
||||
import io.github.z4kn4fein.semver.satisfies
|
||||
import io.github.z4kn4fein.semver.toVersion
|
||||
import java.text.ParseException
|
||||
import kotlin.collections.set
|
||||
|
||||
/**
|
||||
|
|
@ -97,7 +110,7 @@ data class Server(
|
|||
* Constructs a server from its [NodeInfo] and [InstanceV2] details.
|
||||
*/
|
||||
fun from(software: NodeInfo.Software, instanceV2: InstanceV2): Result<Server, Error> = binding {
|
||||
val serverKind = ServerKind.from(software.name)
|
||||
val serverKind = ServerKind.from(software)
|
||||
val version = parseVersionString(serverKind, software.version).bind()
|
||||
val capabilities = capabilitiesFromServerVersion(serverKind, version)
|
||||
|
||||
|
|
@ -120,7 +133,7 @@ data class Server(
|
|||
* Constructs a server from its [NodeInfo] and [InstanceV1] details.
|
||||
*/
|
||||
fun from(software: NodeInfo.Software, instanceV1: InstanceV1): Result<Server, Error> = binding {
|
||||
val serverKind = ServerKind.from(software.name)
|
||||
val serverKind = ServerKind.from(software)
|
||||
val version = parseVersionString(serverKind, software.version).bind()
|
||||
val capabilities = capabilitiesFromServerVersion(serverKind, version)
|
||||
|
||||
|
|
@ -130,37 +143,108 @@ data class Server(
|
|||
/**
|
||||
* Parse a [version] string from the given [serverKind] in to a [Version].
|
||||
*/
|
||||
private fun parseVersionString(serverKind: ServerKind, version: String): Result<Version, UnparseableVersion> = binding {
|
||||
// Real world examples of version strings from nodeinfo
|
||||
// pleroma - 2.6.50-875-g2eb5c453.service-origin+soapbox
|
||||
// akkoma - 3.9.3-0-gd83f5f66f-blob
|
||||
// firefish - 1.1.0-dev29-hf1
|
||||
// hometown - 4.0.10+hometown-1.1.1
|
||||
// cherrypick - 4.6.0+cs-8f0ba0f
|
||||
// gotosocial - 0.13.1-SNAPSHOT git-dfc7656
|
||||
@VisibleForTesting(otherwise = PRIVATE)
|
||||
fun parseVersionString(serverKind: ServerKind, version: String): Result<Version, Error> {
|
||||
val result = runSuspendCatching {
|
||||
Version.parse(version, strict = false)
|
||||
}.mapError { UnparseableVersion(version, it) }
|
||||
|
||||
val semver = when (serverKind) {
|
||||
// These servers have semver compatible versions
|
||||
AKKOMA, MASTODON, PLEROMA, UNKNOWN -> {
|
||||
resultOf { Version.parse(version, strict = false) }
|
||||
.mapError { UnparseableVersion(version, it) }.bind()
|
||||
if (result is Ok) return result
|
||||
|
||||
return when (serverKind) {
|
||||
// These servers should have semver compatible versions, but perhaps
|
||||
// the server operator has changed them. Try looking for a matching
|
||||
// <major>.<minor>.<patch> somewhere in the version string and hope
|
||||
// it's correct
|
||||
AKKOMA, FEDIBIRD, GLITCH, HOMETOWN, MASTODON, PIXELFED, UNKNOWN -> {
|
||||
val rx = """(?<major>\d+)\.(?<minor>\d+).(?<patch>\d+)""".toRegex()
|
||||
rx.find(version)
|
||||
.toResultOr { UnparseableVersion(version, ParseException("unexpected null", 0)) }
|
||||
.andThen {
|
||||
val adjusted = "${it.groups["major"]?.value}.${it.groups["minor"]?.value}.${it.groups["patch"]?.value}"
|
||||
runSuspendCatching { Version.parse(adjusted, strict = false) }
|
||||
.mapError { UnparseableVersion(version, it) }
|
||||
}
|
||||
}
|
||||
// GoToSocial does not report a semver compatible version, expect something
|
||||
// where the possible version number is space-separated, like "0.13.1 git-ccecf5a"
|
||||
|
||||
// Friendica does not report a semver compatible version, expect something
|
||||
// where the version looks like "yyyy.mm", with an optional suffix that
|
||||
// starts with a "-". The date-like parts of the string may have leading
|
||||
// zeros.
|
||||
//
|
||||
// Try to extract the "yyyy.mm", without any leading zeros, append ".0".
|
||||
// https://github.com/friendica/friendica/issues/11264
|
||||
FRIENDICA -> {
|
||||
val rx = """^0*(?<major>\d+)\.0*(?<minor>\d+)""".toRegex()
|
||||
rx.find(version)
|
||||
.toResultOr { UnparseableVersion(version, ParseException("unexpected null", 0)) }
|
||||
.andThen {
|
||||
val adjusted = "${it.groups["major"]?.value}.${it.groups["minor"]?.value}.0"
|
||||
runSuspendCatching { Version.parse(adjusted, strict = false) }
|
||||
.mapError { UnparseableVersion(version, it) }
|
||||
}
|
||||
}
|
||||
|
||||
// GoToSocial does not always report a semver compatible version, and is all
|
||||
// over the place, including:
|
||||
//
|
||||
// - "" (empty)
|
||||
// - "git-8ab30d0"
|
||||
// - "kalaclista git-212fecf"
|
||||
// - "f4fcffc8b56ef73c184ae17892b69181961c15c7"
|
||||
//
|
||||
// as well as instances where the version number is semver compatible, but is
|
||||
// separated by whitespace or a "_".
|
||||
//
|
||||
// https://github.com/superseriousbusiness/gotosocial/issues/1953
|
||||
//
|
||||
// Since GoToSocial has comparatively few features at the moment just fall
|
||||
// back to "0.0.0" if there are problems.
|
||||
GOTOSOCIAL -> {
|
||||
// Try and parse as semver, just in case
|
||||
resultOf { Version.parse(version, strict = false) }
|
||||
.getOrElse {
|
||||
// Didn't parse, use the first component, fall back to 0.0.0
|
||||
val components = version.split(" ")
|
||||
resultOf { Version.parse(components[0], strict = false) }
|
||||
.getOrElse { "0.0.0".toVersion() }
|
||||
// Failed, split on spaces and parse the first component
|
||||
val components = version.split(" ", "_")
|
||||
runSuspendCatching { Version.parse(components[0], strict = false) }
|
||||
.recover { "0.0.0".toVersion() }
|
||||
}
|
||||
|
||||
// IceShrimp uses "yyyy.mm.dd" with leading zeros in the month and day
|
||||
// components, similar to Friendica.
|
||||
// https://iceshrimp.dev/iceshrimp/iceshrimp/issues/502 and
|
||||
// https://iceshrimp.dev/iceshrimp/iceshrimp-rewrite/issues/1
|
||||
ICESHRIMP -> {
|
||||
val rx = """^0*(?<major>\d+)\.0*(?<minor>\d+)\.0*(?<patch>\d+)""".toRegex()
|
||||
rx.find(version).toResultOr { UnparseableVersion(version, ParseException("unexpected null", 0)) }
|
||||
.andThen {
|
||||
val adjusted = "${it.groups["major"]?.value}.${it.groups["minor"]?.value ?: 0}.${it.groups["patch"]?.value ?: 0}"
|
||||
runSuspendCatching { Version.parse(adjusted, strict = false) }
|
||||
.mapError { UnparseableVersion(adjusted, it) }
|
||||
}
|
||||
}
|
||||
|
||||
// Seen "Pleroma 0.9.0 d93789dfde3c44c76a56732088a897ddddfe9716" in
|
||||
// the wild
|
||||
PLEROMA -> {
|
||||
val rx = """Pleroma (?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)""".toRegex()
|
||||
rx.find(version).toResultOr { UnparseableVersion(version, ParseException("unexpected null", 0)) }
|
||||
.andThen {
|
||||
val adjusted = "${it.groups["major"]?.value}.${it.groups["minor"]?.value}.${it.groups["patch"]?.value}"
|
||||
runSuspendCatching { Version.parse(adjusted, strict = false) }
|
||||
.mapError { UnparseableVersion(adjusted, it) }
|
||||
}
|
||||
}
|
||||
|
||||
// Uses format "yyyy.mm.dd" with an optional ".beta..." suffix.
|
||||
// https://git.joinsharkey.org/Sharkey/Sharkey/issues/371
|
||||
SHARKEY -> {
|
||||
val rx = """^(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)""".toRegex()
|
||||
rx.find(version).toResultOr { UnparseableVersion(version, ParseException("unexpected null", 0)) }
|
||||
.andThen {
|
||||
val adjusted = "${it.groups["major"]?.value}.${it.groups["minor"]?.value}.${it.groups["patch"]?.value}"
|
||||
runSuspendCatching { Version.parse(adjusted, strict = false) }
|
||||
.mapError { UnparseableVersion(adjusted, it) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
semver
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -188,7 +272,7 @@ data class Server(
|
|||
|
||||
// Everything else. Assume server side filtering and no translation. This may be an
|
||||
// incorrect assumption.
|
||||
AKKOMA, PLEROMA, UNKNOWN -> {
|
||||
AKKOMA, FEDIBIRD, FRIENDICA, GLITCH, HOMETOWN, ICESHRIMP, PIXELFED, PLEROMA, SHARKEY, UNKNOWN -> {
|
||||
c[ORG_JOINMASTODON_FILTERS_SERVER] = "1.0.0".toVersion()
|
||||
}
|
||||
}
|
||||
|
|
@ -210,20 +294,45 @@ data class Server(
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Servers that are known to implement the Mastodon client API
|
||||
*/
|
||||
enum class ServerKind {
|
||||
AKKOMA,
|
||||
FEDIBIRD,
|
||||
FRIENDICA,
|
||||
GLITCH,
|
||||
GOTOSOCIAL,
|
||||
HOMETOWN,
|
||||
ICESHRIMP,
|
||||
MASTODON,
|
||||
PLEROMA,
|
||||
PIXELFED,
|
||||
SHARKEY,
|
||||
|
||||
/**
|
||||
* Catch-all for servers we don't recognise but that responded to either
|
||||
* /api/v1/instance or /api/v2/instance
|
||||
*/
|
||||
UNKNOWN,
|
||||
;
|
||||
|
||||
companion object {
|
||||
fun from(s: String) = when (s.lowercase()) {
|
||||
fun from(s: NodeInfo.Software) = when (s.name.lowercase()) {
|
||||
"akkoma" -> AKKOMA
|
||||
"fedibird" -> FEDIBIRD
|
||||
"friendica" -> FRIENDICA
|
||||
"gotosocial" -> GOTOSOCIAL
|
||||
"mastodon" -> MASTODON
|
||||
"hometown" -> HOMETOWN
|
||||
"iceshrimp" -> ICESHRIMP
|
||||
"mastodon" -> {
|
||||
// Glitch doesn't report a different software name it stuffs it
|
||||
// in the version (https://github.com/glitch-soc/mastodon/issues/2582).
|
||||
if (s.version.contains("+glitch")) GLITCH else MASTODON
|
||||
}
|
||||
"pixelfed" -> PIXELFED
|
||||
"pleroma" -> PLEROMA
|
||||
"sharkey" -> SHARKEY
|
||||
else -> UNKNOWN
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
package app.pachli.core.network
|
||||
|
||||
import app.pachli.core.network.ServerKind.AKKOMA
|
||||
import app.pachli.core.network.ServerKind.FRIENDICA
|
||||
import app.pachli.core.network.ServerKind.GOTOSOCIAL
|
||||
import app.pachli.core.network.ServerKind.MASTODON
|
||||
import app.pachli.core.network.ServerKind.PLEROMA
|
||||
|
|
@ -43,7 +44,10 @@ import app.pachli.core.network.model.nodeinfo.NodeInfo
|
|||
import com.github.michaelbull.result.Ok
|
||||
import com.github.michaelbull.result.Result
|
||||
import com.google.common.truth.Truth.assertWithMessage
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.reflect.TypeToken
|
||||
import io.github.z4kn4fein.semver.toVersion
|
||||
import java.io.BufferedReader
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.junit.runners.Parameterized
|
||||
|
|
@ -238,6 +242,22 @@ class ServerTest(
|
|||
),
|
||||
),
|
||||
),
|
||||
arrayOf(
|
||||
Triple(
|
||||
"Friendica can filter",
|
||||
NodeInfo.Software("friendica", "2023.05-1542"),
|
||||
defaultInstance,
|
||||
),
|
||||
Ok(
|
||||
Server(
|
||||
kind = FRIENDICA,
|
||||
version = "2023.5.0".toVersion(),
|
||||
capabilities = mapOf(
|
||||
ORG_JOINMASTODON_FILTERS_SERVER to "1.0.0".toVersion(),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -252,3 +272,60 @@ class ServerTest(
|
|||
.isEqualTo(want)
|
||||
}
|
||||
}
|
||||
|
||||
class ServerVersionTest() {
|
||||
private val gson = Gson()
|
||||
|
||||
private fun loadJsonAsString(fileName: String): String {
|
||||
return javaClass.getResourceAsStream("/$fileName")!!
|
||||
.bufferedReader().use(BufferedReader::readText)
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that parsing all possible versions succeeds.
|
||||
*
|
||||
* To do this tools/mkserverversions generates a JSON file that
|
||||
* contains a map of server names to a list of server version strings
|
||||
* that have been seen by Fediverse Observer. These version strings
|
||||
* are then parsed and are all expected to parse correctly.
|
||||
*/
|
||||
@Test
|
||||
fun parseVersionString() {
|
||||
val mapType: TypeToken<Map<String, Set<String>>> =
|
||||
object : TypeToken<Map<String, Set<String>>>() {}
|
||||
|
||||
val serverVersions = gson.fromJson(
|
||||
loadJsonAsString("server-versions.json5"),
|
||||
mapType,
|
||||
)
|
||||
|
||||
// Sanity check that data was loaded correctly. Expect at least 5 distinct keys
|
||||
assertWithMessage("number of server types in server-versions.json5 is too low")
|
||||
.that(serverVersions.size)
|
||||
.isGreaterThan(5)
|
||||
|
||||
for (entry in serverVersions.entries) {
|
||||
for (version in entry.value) {
|
||||
val serverKind = ServerKind.from(
|
||||
NodeInfo.Software(
|
||||
name = entry.key,
|
||||
version = version,
|
||||
),
|
||||
)
|
||||
|
||||
// Skip unknown/unsupported servers, as their version strings
|
||||
// could be anything.
|
||||
if (serverKind == UNKNOWN) continue
|
||||
|
||||
val result = Server.parseVersionString(
|
||||
serverKind,
|
||||
version,
|
||||
)
|
||||
|
||||
assertWithMessage("${entry.key} : $version")
|
||||
.that(result)
|
||||
.isInstanceOf(Ok::class.java)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -47,4 +47,5 @@ include(":core:navigation")
|
|||
include(":core:network")
|
||||
include(":core:testing")
|
||||
include(":tools:mklanguages")
|
||||
include(":tools:mkserverversions")
|
||||
include(":checks")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
# mkserverversions
|
||||
|
||||
## Synopsis
|
||||
|
||||
`mkserverversions` creates a JSON5 file that maps different Fediverse server
|
||||
names to a list of the different version strings seen for those servers, as
|
||||
recorded by [Fediverse Observer](https://fediverse.observer).
|
||||
|
||||
This is used as input data for `core/network/ServerTest`, to ensure that the
|
||||
version parsing code can handle real-world version strings.
|
||||
|
||||
Run `mkserverversions` every month to update the test data, and update the
|
||||
parsing code if any of the tests fail.
|
||||
|
||||
## Usage
|
||||
|
||||
From the parent directory, run:
|
||||
|
||||
```shell
|
||||
./runtools mkserverversions
|
||||
```
|
||||
|
||||
Verify the modifications made to `server-versions.json5`, re-run the tests,
|
||||
and commit the result.
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
/*
|
||||
* Copyright 2024 Pachli Association
|
||||
*
|
||||
* This file is a part of Pachli.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify it under the terms of the
|
||||
* GNU General Public License as published by the Free Software Foundation; either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
|
||||
* the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
|
||||
* Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with Pachli; if not,
|
||||
* see <http://www.gnu.org/licenses>.
|
||||
*/
|
||||
|
||||
plugins {
|
||||
id("com.apollographql.apollo3") version "3.8.2"
|
||||
}
|
||||
|
||||
application {
|
||||
mainClass.set("app.pachli.mkserverversions.MainKt")
|
||||
}
|
||||
|
||||
dependencies {
|
||||
// GraphQL client
|
||||
implementation("com.apollographql.apollo3:apollo-runtime:3.8.2")
|
||||
|
||||
// Logging
|
||||
implementation("io.github.oshai:kotlin-logging-jvm:5.1.0")
|
||||
implementation("ch.qos.logback:logback-classic:1.4.11")
|
||||
|
||||
implementation(libs.gson)
|
||||
|
||||
// Testing
|
||||
testImplementation(kotlin("test"))
|
||||
testImplementation("org.junit.jupiter:junit-jupiter-params:5.10.1") // for parameterized tests
|
||||
}
|
||||
|
||||
tasks.test {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
|
||||
apollo {
|
||||
service("service") {
|
||||
packageName.set("app.pachli.mkserverversions.fediverseobserver")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
/*
|
||||
* Copyright 2024 Pachli Association
|
||||
*
|
||||
* This file is a part of Pachli.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify it under the terms of the
|
||||
* GNU General Public License as published by the Free Software Foundation; either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
|
||||
* the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
|
||||
* Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with Pachli; if not,
|
||||
* see <http://www.gnu.org/licenses>.
|
||||
*/
|
||||
|
||||
dependencyResolutionManagement {
|
||||
repositories {
|
||||
google()
|
||||
mavenCentral()
|
||||
}
|
||||
versionCatalogs {
|
||||
create("libs") {
|
||||
from(files("../../gradle/libs.versions.toml"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rootProject.name = "mkserverversions"
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
query ServerVersions {
|
||||
nodes(status: "UP") {
|
||||
softwarename
|
||||
fullversion
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,353 @@
|
|||
type Node {
|
||||
"A unique idenifier the database creates"
|
||||
id: Int!
|
||||
"The name in the nodeinfo"
|
||||
name: String
|
||||
"The HTML Title tag of the server homepage"
|
||||
metatitle: String
|
||||
"The HTML Description tag of the server homepage"
|
||||
metadescription: String
|
||||
"The full json of the nodeinfo"
|
||||
metanodeinfo: String
|
||||
"The full json of the IP data from the IP"
|
||||
metalocation: String
|
||||
"The owner defined in your nodeinfo"
|
||||
owner: String
|
||||
"The HTML Image tag of the server homepage"
|
||||
metaimage: String
|
||||
"The Onion address from your web server header Onion-Location"
|
||||
onion: String
|
||||
"The i2p address from your web server header X-I2P-Location"
|
||||
i2p: String
|
||||
"The link to the terms of service based on best known for your software (owner editable)"
|
||||
terms: String
|
||||
"The link to the privacy policy based on best known for your software (owner editable)"
|
||||
pp: String
|
||||
"The link to the support based on best known for your software (owner editable)"
|
||||
support: String
|
||||
"The camo json data string from the nodeinfo"
|
||||
camo: String
|
||||
"The zipcode found from the IP location data"
|
||||
zipcode: String
|
||||
"The domain of server"
|
||||
domain: String
|
||||
"A open text field for the public comments on the server"
|
||||
podmin_statement: String
|
||||
"The main code version from the git repository for the software"
|
||||
masterversion: String
|
||||
"The short human readable version from the nodeinfo"
|
||||
shortversion: String
|
||||
"The software name from the nodeinfo"
|
||||
softwarename: String
|
||||
"Days we have been checking the server"
|
||||
daysmonitored: Int
|
||||
"Months we have been checking your server"
|
||||
monthsmonitored: Int
|
||||
"The full version found on the server from the nodeinfo"
|
||||
fullversion: String
|
||||
"Score based on the checks of the server being online each time checked"
|
||||
score: Int
|
||||
"The IP from looking at the DNS records"
|
||||
ip: String
|
||||
"Detected language from your html title and description tags on the site"
|
||||
detectedlanguage: String
|
||||
"Detected country code from the IP data"
|
||||
country: String
|
||||
"Detected country name from the IP data"
|
||||
countryname: String
|
||||
"Detected city name from the IP data"
|
||||
city: String
|
||||
"Detected state name from the IP data"
|
||||
state: String
|
||||
"Detected lattitude from the IP data"
|
||||
lat: String
|
||||
"Detected longitude name from the IP data"
|
||||
long: String
|
||||
"The IPv6 Address from DNS records"
|
||||
ipv6: String
|
||||
"True or False that the SSL certificate valid"
|
||||
sslvalid: String
|
||||
"If server allow new users to register or not from the nodeinfo"
|
||||
signup: Boolean
|
||||
"All total users on the server from the nodeinfo"
|
||||
total_users: Int
|
||||
"All total users last 6 months on the server from the nodeinfo"
|
||||
active_users_halfyear: Int
|
||||
"All total users last 1 month on the server from the nodeinfo"
|
||||
active_users_monthly: Int
|
||||
"All posts from the server from the nodeinfo"
|
||||
local_posts: Int
|
||||
"Percent server up from our checks"
|
||||
uptime_alltime: String
|
||||
"Current status of the server. 1 = UP. More at https://gitlab.com/diasporg/poduptime/-/blob/master/lib/PodStatus.php"
|
||||
status: Int
|
||||
"Latency connecting to the server"
|
||||
latency: Int
|
||||
"If you support XMPP from the nodeinfo"
|
||||
service_xmpp: Boolean
|
||||
"Json string of the services the server supports from the nodeinfo"
|
||||
services: String
|
||||
"Json string of the protocols the server supports from the nodeinfo"
|
||||
protocols: String
|
||||
"When the SSL certificate expires"
|
||||
sslexpire: String
|
||||
"DNSSEC Signature True or False"
|
||||
dnssec: Boolean
|
||||
"Type of Webserver"
|
||||
servertype: String
|
||||
"All comments on the server from the nodeinfo"
|
||||
comment_counts: Int
|
||||
"How the server is weighed in the table and search results"
|
||||
weight: Int
|
||||
"Date this data was updated"
|
||||
date_updated: String
|
||||
"Date last checked your server"
|
||||
date_laststats: String
|
||||
"Date added your server"
|
||||
date_created: String
|
||||
"Date server score hit 0"
|
||||
date_diedoff: String
|
||||
"Date server language was checked"
|
||||
date_lastlanguage: String
|
||||
}
|
||||
|
||||
type Nodes {
|
||||
"A unique idenifier the database creates"
|
||||
id: Int!
|
||||
"The name in the nodeinfo"
|
||||
name: String
|
||||
"The HTML Title tag of the server homepage"
|
||||
metatitle: String
|
||||
"The HTML Description tag of the server homepage"
|
||||
metadescription: String
|
||||
"The full json of the nodeinfo"
|
||||
metanodeinfo: String
|
||||
"The full json of the IP data from the IP"
|
||||
metalocation: String
|
||||
"The owner defined in your nodeinfo"
|
||||
owner: String
|
||||
"The HTML Image tag of the server homepage"
|
||||
metaimage: String
|
||||
"The Onion address from your web server header Onion-Location"
|
||||
onion: String
|
||||
"The i2p address from your web server header X-I2P-Location"
|
||||
i2p: String
|
||||
"The link to the terms of service based on best known for your software (owner editable)"
|
||||
terms: String
|
||||
"The link to the privacy policy based on best known for your software (owner editable)"
|
||||
pp: String
|
||||
"The link to the support based on best known for your software (owner editable)"
|
||||
support: String
|
||||
"The camo json data string from the nodeinfo"
|
||||
camo: String
|
||||
"The zipcode found from the IP location data"
|
||||
zipcode: String
|
||||
"The domain of server"
|
||||
domain: String
|
||||
"A open text field for the public comments on the server"
|
||||
podmin_statement: String
|
||||
"The main code version from the git repository for the software"
|
||||
masterversion: String
|
||||
"The short human readable version from the nodeinfo"
|
||||
shortversion: String
|
||||
"The software name from the nodeinfo"
|
||||
softwarename: String
|
||||
"Days we have been checking the server"
|
||||
daysmonitored: Int
|
||||
"Months we have been checking your server"
|
||||
monthsmonitored: Int
|
||||
"The full version found on the server from the nodeinfo"
|
||||
fullversion: String
|
||||
"Score based on the checks of the server being online each time checked"
|
||||
score: Int
|
||||
"The IP from looking at the DNS records"
|
||||
ip: String
|
||||
"Detected language from your html title and description tags on the site"
|
||||
detectedlanguage: String
|
||||
"Detected country code from the IP data"
|
||||
country: String
|
||||
"Detected country name from the IP data"
|
||||
countryname: String
|
||||
"Detected city name from the IP data"
|
||||
city: String
|
||||
"Detected state name from the IP data"
|
||||
state: String
|
||||
"Detected lattitude from the IP data"
|
||||
lat: String
|
||||
"Detected longitude name from the IP data"
|
||||
long: String
|
||||
"The IPv6 Address from DNS records"
|
||||
ipv6: String
|
||||
"True or False that the SSL certificate valid"
|
||||
sslvalid: String
|
||||
"If server allow new users to register or not from the nodeinfo"
|
||||
signup: Boolean
|
||||
"All total users on the server from the nodeinfo"
|
||||
total_users: Int
|
||||
"All total users last 6 months on the server from the nodeinfo"
|
||||
active_users_halfyear: Int
|
||||
"All total users last 1 month on the server from the nodeinfo"
|
||||
active_users_monthly: Int
|
||||
"All posts from the server from the nodeinfo"
|
||||
local_posts: Int
|
||||
"Percent server up from our checks"
|
||||
uptime_alltime: String
|
||||
"Current status of the server. 1 = UP. More at https://gitlab.com/diasporg/poduptime/-/blob/master/lib/PodStatus.php"
|
||||
status: Int
|
||||
"Latency connecting to the server"
|
||||
latency: Int
|
||||
"If you support XMPP from the nodeinfo"
|
||||
service_xmpp: Boolean
|
||||
"Json string of the services the server supports from the nodeinfo"
|
||||
services: String
|
||||
"Json string of the protocols the server supports from the nodeinfo"
|
||||
protocols: String
|
||||
"When the SSL certificate expires"
|
||||
sslexpire: String
|
||||
"Type of Webserver"
|
||||
servertype: String
|
||||
"DNSSEC Signature True or False"
|
||||
dnssec: Boolean
|
||||
"All comments on the server from the nodeinfo"
|
||||
comment_counts: Int
|
||||
"How the server is weighed in the table and search results"
|
||||
weight: Int
|
||||
"Date this data was updated"
|
||||
date_updated: String
|
||||
"Date last checked your server"
|
||||
date_laststats: String
|
||||
"Date added your server"
|
||||
date_created: String
|
||||
"Date server score hit 0"
|
||||
date_diedoff: String
|
||||
"Date server language was checked"
|
||||
date_lastlanguage: String
|
||||
}
|
||||
|
||||
type Click {
|
||||
"A unique idenifier the database creates"
|
||||
id: Int!
|
||||
"The domain of server"
|
||||
domain: String
|
||||
"Clicks by a user to this domain"
|
||||
manualclick: Int
|
||||
"Auto sends from the go.php auto pick method"
|
||||
autoclick: Int
|
||||
"Datestamp of the click"
|
||||
date_clicked: String
|
||||
}
|
||||
|
||||
type Check {
|
||||
"A unique idenifier the database creates"
|
||||
id: Int!
|
||||
"The domain of server"
|
||||
domain: String
|
||||
"Was server online"
|
||||
online: Boolean
|
||||
"If error what was it"
|
||||
error: String
|
||||
"Latency from checking server"
|
||||
latency: Int
|
||||
"Total users in nodeinfo"
|
||||
total_users: Int
|
||||
"Active users in last 6 months from nodeinfo"
|
||||
active_users_halfyear: Int
|
||||
"Active users in last month from nodeinfo"
|
||||
active_users_monthly: Int
|
||||
"Total posts in nodeinfo"
|
||||
local_posts: Int
|
||||
"Total comments in nodeinfo"
|
||||
comment_counts: Int
|
||||
"Version code in human format"
|
||||
shortversion: String
|
||||
"Full version code"
|
||||
version: String
|
||||
"Date Checked"
|
||||
date_checked: String
|
||||
}
|
||||
|
||||
type MonthlyStat {
|
||||
"A unique idenifier the database creates"
|
||||
id: Int!
|
||||
"Software Name"
|
||||
softwarename: String
|
||||
"Total users for the Software"
|
||||
total_users: Int
|
||||
"Total active users for the Software Halfyear"
|
||||
total_active_users_halfyear: Int
|
||||
"Total active users for the Software Monthly"
|
||||
total_active_users_monthly: Int
|
||||
"Total posts for the Software"
|
||||
total_posts: Int
|
||||
"Total comments for the Software"
|
||||
total_comments: Int
|
||||
"Total servers for the Software"
|
||||
total_pods: Int
|
||||
"Total uptime for Software"
|
||||
total_uptime: Int
|
||||
"Date last updated the monthly stats, this is always represented as the first day of the month for these stats"
|
||||
date_checked: String
|
||||
}
|
||||
|
||||
type DailyStat {
|
||||
"A unique idenifier the database creates"
|
||||
id: Int!
|
||||
"Software Name"
|
||||
softwarename: String
|
||||
"Total users for the Software"
|
||||
total_users: Int
|
||||
"Total active users for the Software Halfyear"
|
||||
total_active_users_halfyear: Int
|
||||
"Total active users for the Software Monthly"
|
||||
total_active_users_monthly: Int
|
||||
"Total posts for the Software"
|
||||
total_posts: Int
|
||||
"Total comments for the Software"
|
||||
total_comments: Int
|
||||
"Total servers for the Software"
|
||||
total_pods: Int
|
||||
"Total uptime for Software"
|
||||
total_uptime: Int
|
||||
"Date last updated the monthly stats, this is always represented as the first day of the month for these stats"
|
||||
date_checked: String
|
||||
}
|
||||
|
||||
type MasterVersion {
|
||||
"A unique idenifier the database creates"
|
||||
id: Int!
|
||||
"Software name from the git repo"
|
||||
software: String
|
||||
"Version label from git repo"
|
||||
version: String
|
||||
"Last development branch update"
|
||||
devlastcommit: String
|
||||
"The date of the git release"
|
||||
releasedate: String
|
||||
"Date masterversions updated from git"
|
||||
date_checked: String
|
||||
}
|
||||
|
||||
type Software {
|
||||
"Name of software"
|
||||
softwarename: String
|
||||
count: Int
|
||||
}
|
||||
|
||||
type Query {
|
||||
"A full list of all servers in the database."
|
||||
nodes ("UP, DOWN, RECHECK, PAUSED, SYSTEM_DELETED, USER_DELETED" status: String, "Name of a federated software type, like diaspora or mastodon" softwarename: String): [Node]
|
||||
"A single server by domain name"
|
||||
node (domain: String): [Node]
|
||||
"All auto and manual clicks to a server"
|
||||
clicks (limit: Int, domain: String): [Click]
|
||||
"All checks ran to servers"
|
||||
checks (limit: Int, domain: String, "true, false" online: String): [Check]
|
||||
"All monthlystats data"
|
||||
monthlystats ("Name of a federated software type, like diaspora or mastodon or all" softwarename: String, "true, false - only the latest result" mostcurrent: String): [MonthlyStat]
|
||||
"All dailystats data"
|
||||
dailystats ("Name of a federated software type, like diaspora or mastodon or all" softwarename: String, "true, false - only the latest result" mostcurrent: String): [DailyStat]
|
||||
"All git master code versions"
|
||||
masterversions: [MasterVersion]
|
||||
"All software types data"
|
||||
softwares: [Software]
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
/*
|
||||
* Copyright 2024 Pachli Association
|
||||
*
|
||||
* This file is a part of Pachli.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify it under the terms of the
|
||||
* GNU General Public License as published by the Free Software Foundation; either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
|
||||
* the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
|
||||
* Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with Pachli; if not,
|
||||
* see <http://www.gnu.org/licenses>.
|
||||
*/
|
||||
|
||||
package app.pachli.mkserverversions
|
||||
|
||||
import app.pachli.mkserverversions.fediverseobserver.ServerVersionsQuery
|
||||
import ch.qos.logback.classic.Level
|
||||
import ch.qos.logback.classic.Logger
|
||||
import com.apollographql.apollo3.ApolloClient
|
||||
import com.github.ajalt.clikt.core.CliktCommand
|
||||
import com.github.ajalt.clikt.core.UsageError
|
||||
import com.github.ajalt.clikt.parameters.options.flag
|
||||
import com.github.ajalt.clikt.parameters.options.option
|
||||
import com.google.gson.GsonBuilder
|
||||
import io.github.oshai.kotlinlogging.DelegatingKLogger
|
||||
import io.github.oshai.kotlinlogging.KotlinLogging
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import java.util.TreeMap
|
||||
import java.util.TreeSet
|
||||
import kotlin.io.path.Path
|
||||
import kotlin.io.path.div
|
||||
import kotlin.io.path.exists
|
||||
import kotlinx.coroutines.runBlocking
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
const val DEST_DIR = "core/network/src/test/resources"
|
||||
|
||||
class App : CliktCommand(help = """Update server-versions.json5""") {
|
||||
private val verbose by option("-n", "--verbose", help = "show additional information").flag()
|
||||
|
||||
/**
|
||||
* Returns the full path to the Pachli `.../core/network/src/test/resources` directory,
|
||||
* starting from the given [start] directory, walking up the tree if it can't be found
|
||||
* there.
|
||||
*
|
||||
* @return the path, or null if it's not a subtree of [start] or any of its parents.
|
||||
*/
|
||||
private fun findResourcePath(start: Path): Path? {
|
||||
val suffix = Path(DEST_DIR)
|
||||
|
||||
var prefix = start
|
||||
var resourcePath: Path
|
||||
do {
|
||||
resourcePath = prefix / suffix
|
||||
if (resourcePath.exists()) return resourcePath
|
||||
prefix = prefix.parent
|
||||
} while (prefix != prefix.root)
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
override fun run() = runBlocking {
|
||||
System.setProperty("file.encoding", "UTF8")
|
||||
((log as? DelegatingKLogger<*>)?.underlyingLogger as Logger).level = if (verbose) Level.INFO else Level.WARN
|
||||
|
||||
val cwd = Paths.get("").toAbsolutePath()
|
||||
log.info { "working directory: $cwd" }
|
||||
|
||||
val resourcePath = findResourcePath(cwd) ?: throw UsageError("could not find $DEST_DIR in tree")
|
||||
|
||||
val apolloClient = ApolloClient.Builder()
|
||||
.serverUrl("https://api.fediverse.observer/")
|
||||
.build()
|
||||
|
||||
val response = apolloClient.query(ServerVersionsQuery()).execute()
|
||||
|
||||
if (response.hasErrors()) {
|
||||
response.errors?.forEach {
|
||||
log.error { it }
|
||||
}
|
||||
return@runBlocking
|
||||
}
|
||||
|
||||
val serverVersionsPath = resourcePath / "server-versions.json5"
|
||||
val w = serverVersionsPath.toFile().printWriter()
|
||||
w.println("// GENERATED BY \"runtools mkserverversions\", DO NOT HAND EDIT")
|
||||
|
||||
// Use TreeMap and TreeSet for a consistent sort order in the output to
|
||||
// minimise git diffs when the output is regenerated
|
||||
val m = TreeMap<String, MutableSet<String>>()
|
||||
|
||||
response.data?.nodes.orEmpty().filterNotNull().forEach {
|
||||
if (it.fullversion != null && it.softwarename != null) {
|
||||
val versions = m.getOrPut(it.softwarename) { TreeSet() }
|
||||
versions.add(it.fullversion)
|
||||
}
|
||||
}
|
||||
|
||||
val gson = GsonBuilder().setPrettyPrinting().create()
|
||||
w.print(gson.toJson(m))
|
||||
w.close()
|
||||
}
|
||||
}
|
||||
|
||||
fun main(args: Array<String>) = App().main(args)
|
||||
Loading…
Reference in New Issue