change: Add tools/mklanguages

The existing language list is incomplete, sorted incorrectly and does
not use the correct language names.

Add a small tool that parses the resource directories that contain
string translations, determines the correct language name and sort
order, and updates the correct application resources so language
lists are displayed correctly.

runtools

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Run one of the tools.
+# The first argument must be the name of the tool task (e.g. mklanguages).
+# Any remaining arguments are forwarded to the tool's argv.
+
+task=$1
+shift 1
+
+if [ -z "${task}" ] || [ ! -d "tools/${task}" ]
+then
+    echo "Unknown tool: '${task}'"
+    exit 1
+fi
+
+./gradlew --quiet ":tools:${task}:installDist" && "./tools/${task}/build/install/${task}/bin/${task}" "$@"

runtools.bat

@@ -0,0 +1,24 @@
+@if "%DEBUG%"=="" @echo off
+:: Run one of the tools.
+:: The first argument must be the name of the tools (e.g. mklanguages).
+:: Any remaining arguments are forwarded to the tool's argv.
+
+if "%OS%"=="Windows_NT" setlocal EnableDelayedExpansion
+
+set TASK=%~1
+
+set TOOL=false
+if defined TASK if not "!TASK: =!"=="" if exist "tools\%TASK%\*" set TOOL=true
+
+if "%TOOL%"=="false" (
+    echo Unknown tool: '%TASK%'
+    exit /b 1
+)
+
+set ARGS=%*
+set ARGS=!ARGS:*%1=!
+if "!ARGS:~0,1!"==" " set ARGS=!ARGS:~1!
+
+call gradlew --quiet ":tools:%TASK%:installDist" && call "tools\%TASK%\build\install\%TASK%\bin\%TASK%" %ARGS%
+
+if "%OS%"=="Windows_NT" endlocal

settings.gradle

@@ -17,3 +17,4 @@ dependencyResolutionManagement {
 enableFeaturePreview("STABLE_CONFIGURATION_CACHE")
 
 include ':app'
+include ':tools:mklanguages'

tools/build.gradle.kts

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2023 Tusky Contributors
+ *
+ * This file is a part of Tusky.
+ *
+ * This program is free software; you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
+ * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with Tusky; if not,
+ * see <http://www.gnu.org/licenses>.
+ */
+
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
+subprojects {
+    apply(plugin = "kotlin")
+    apply(plugin = "application")
+
+    dependencies {
+        "implementation"("com.github.ajalt.clikt:clikt:3.5.2")
+    }
+
+    tasks.withType<KotlinCompile>().configureEach {
+        kotlinOptions {
+            freeCompilerArgs = freeCompilerArgs + "-opt-in=kotlin.RequiresOptIn"
+        }
+    }
+}

tools/mklanguages/README.md

@@ -0,0 +1,21 @@
+# mklanguages
+
+## Synopsis
+
+`mklanguages` ensures that the language list in Pachli is:
+
+- Up to date
+- Sorted according to ICU guidelines
+- Uses language names according to ICU guidelines
+
+Use `mklanguages` whenever a new language is added to Pachli.
+
+## Usage
+
+From the parent directory, run:
+
+```shell
+./runtools mklanguages
+```
+
+Verify the modifications made to the Pachli resource files, and commit the result.

tools/mklanguages/build.gradle.kts

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2023 Pachli Association
+ *
+ * This file is a part of Pachli.
+ *
+ * This program is free software; you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
+ * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with Pachli; if not,
+ * see <http://www.gnu.org/licenses>.
+ */
+
+application {
+    mainClass.set("app.pachli.mklanguages.MainKt")
+}
+
+dependencies {
+    // ICU
+    implementation("com.ibm.icu:icu4j:73.1")
+
+    // Parsing
+    implementation("com.github.h0tk3y.betterParse:better-parse:0.4.4")
+
+    // Logging
+    implementation("io.github.oshai:kotlin-logging-jvm:4.0.0-beta-28")
+    implementation("ch.qos.logback:logback-classic:1.3.0")
+
+    // Testing
+    testImplementation(kotlin("test"))
+    testImplementation("org.junit.jupiter:junit-jupiter-params:5.9.2") // for parameterized tests
+}
+
+tasks.test {
+    useJUnitPlatform()
+}

tools/mklanguages/src/main/kotlin/app/pachli/mklanguages/Main.kt

@@ -0,0 +1,206 @@
+/*
+ * Copyright 2023 Pachli Association
+ *
+ * This file is a part of Pachli.
+ *
+ * This program is free software; you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
+ * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with Pachli; if not,
+ * see <http://www.gnu.org/licenses>.
+ */
+
+package app.pachli.mklanguages
+
+import ch.qos.logback.classic.Level
+import ch.qos.logback.classic.Logger
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.core.UsageError
+import com.github.ajalt.clikt.parameters.options.flag
+import com.github.ajalt.clikt.parameters.options.option
+import com.github.h0tk3y.betterParse.grammar.parseToEnd
+import com.ibm.icu.text.CaseMap
+import com.ibm.icu.text.Collator
+import com.ibm.icu.util.ULocale
+import io.github.oshai.KotlinLogging
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
+import kotlin.io.path.Path
+import kotlin.io.path.createTempFile
+import kotlin.io.path.div
+import kotlin.io.path.exists
+import kotlin.io.path.isDirectory
+import kotlin.io.path.isRegularFile
+import kotlin.io.path.listDirectoryEntries
+
+private val log = KotlinLogging.logger {}
+
+/** The information needed to encode a language in the XML resources */
+data class Language(
+    /** Language code */
+    val code: String,
+
+    /**
+     * Name of the language, in that language. E.g., the display name for English is "English",
+     * the display name for Icelandic is "Íslenska".
+     */
+    val displayName: String,
+
+    /** Name of the language in English */
+    val displayNameEnglish: String
+) {
+    companion object {
+        private val toTitle = CaseMap.toTitle()
+
+        /** Create a [Language] from a [ULocale] */
+        fun from(locale: ULocale) = Language(
+            locale.name.replace("_", "-"),
+            toTitle.apply(locale.toLocale(), null, locale.getDisplayName(locale)),
+            locale.getDisplayName(ULocale.ENGLISH)
+        )
+    }
+}
+
+/**
+ * Constructs the `language_entries` and `language_values` string arrays in donottranslate.xml.
+ *
+ * - Finds all the `values-*` directories that contain `strings.xml`
+ * - Parses out the language code from the directory name
+ * - Uses the ICU libraries to determine the correct name for the language
+ * - Sorts the list of languages using ICU collation rules
+ * - Updates donottranslate.xml with the new data
+ *
+ * Run this after creating a new translation.
+ *
+ * Run with `gradlew :tools:mklanguages:run` or `runtools mklanguages`.
+ */
+class App : CliktCommand(help = """Update languages in donottranslate.xml""") {
+    private val verbose by option("-n", "--verbose", help = "show additional information").flag()
+
+    /**
+     * Returns the full path to the Pachli `.../app/src/main/res` directory, starting from the
+     * given [start] directory, walking up the tree if it can't be found there.
+     *
+     * @return the path, or null if it's not a subtree of [start] or any of its parents.
+     */
+    private fun findResourcePath(start: Path): Path? {
+        val suffix = Path("app/src/main/res")
+
+        var prefix = start
+        var resourcePath: Path
+        do {
+            resourcePath = prefix / suffix
+            if (resourcePath.exists()) return resourcePath
+            prefix = prefix.parent
+        } while (prefix != prefix.root)
+
+        return null
+    }
+
+    override fun run() {
+        System.setProperty("file.encoding", "UTF8")
+        (log.underlyingLogger as Logger).level = if (verbose) Level.INFO else Level.WARN
+
+        val cwd = Paths.get("").toAbsolutePath()
+        log.info("working directory: $cwd")
+
+        val resourcePath = findResourcePath(cwd) ?: throw UsageError("could not find app/src/main/res in tree")
+
+        // Enumerate all the values-* directories that contain a strings.xml file
+        val resourceDirs = resourcePath.listDirectoryEntries("values-*")
+            .filter { entry -> entry.isDirectory() }
+            .filter { dir -> (dir / "strings.xml").isRegularFile() }
+
+        if (resourceDirs.isEmpty()) throw UsageError("no strings.xml files found in $resourcePath/values-*")
+
+        // Convert the `values-...` directory names to instances of ULocale.
+        val valuesParser = ValuesParser()
+        val locales = resourceDirs
+            .asSequence()
+            .map { it.fileName.toString() }
+            .onEach { log.info("parsing directory name: $it") }
+            // Special-case ber, see https://github.com/tuskyapp/Tusky/issues/3637
+            .map { if (it == "values-ber") "values-b+tzm+Tfng" else it }
+            .mapNotNull { valuesParser.parseToEnd(it).locale }
+            .onEach { log.info("  --> $it") }
+            .toMutableList()
+            .apply { add(Locale(lang = "en")) }
+            .map { ULocale(it.lang, it.region, it.script) }
+
+        // Construct the languages. Sort each locale by its display name, as rendered in that
+        // locale, and fold case.
+        val collator = Collator.getInstance(ULocale.ENGLISH)
+        val casemapFold = CaseMap.fold()
+
+        val languages = locales.sortedBy { collator.getCollationKey(casemapFold.apply(it.getDisplayName(it))) }
+            .map { Language.from(it) }
+            .toMutableList()
+
+        // The first language in the list is the system default
+        languages.add(0, Language("default", "@string/system_default", "System default"))
+
+        // Copy donottranslate.xml line by line to a new file, replacing the contents of the
+        // `language_entries` and `language_values` arrays with fresh data.
+        val tmpFile = createTempFile().toFile()
+        val w = tmpFile.printWriter()
+        val donottranslate_xml = resourcePath / "values" / "donottranslate.xml"
+        donottranslate_xml.toFile().useLines { lines ->
+            var inLanguageEntries = false
+            var inLanguageValues = false
+
+            for (line in lines) {
+                // Default behaviour, copy the line unless inside one of the arrays
+                if (!inLanguageEntries && !inLanguageValues) {
+                    w.println(line)
+                }
+
+                // Started the `language_entries` array
+                if (line.contains("<string-array name=\"language_entries\">")) {
+                    inLanguageEntries = true
+                    continue
+                }
+
+                // Started the `language_values` array
+                if (line.contains("<string-array name=\"language_values\">")) {
+                    inLanguageValues = true
+                    continue
+                }
+
+                // At the end of `language_entries`? Emit each language, one per line. The
+                // item is the language's name, then a comment with the code and English name.
+                // Then close the array.
+                if (inLanguageEntries && line.contains("</string-array>")) {
+                    languages.forEach {
+                        w.println("        <item>${it.displayName}</item> <!-- ${it.code}: ${it.displayNameEnglish} -->")
+                    }
+                    w.println(line)
+                    inLanguageEntries = false
+                    continue
+                }
+
+                // At the end of `language_values`? Emit each language code, one per line.
+                // Then close the array.
+                if (inLanguageValues && line.contains("</string-array>")) {
+                    languages.forEach { w.println("        <item>${it.code}</item>") }
+                    w.println(line)
+                    inLanguageValues = false
+                    continue
+                }
+            }
+        }
+
+        // Close, then replace donotranslate.xml
+        w.close()
+        Files.move(tmpFile.toPath(), donottranslate_xml, StandardCopyOption.REPLACE_EXISTING)
+        log.info("replaced ${donottranslate_xml.toAbsolutePath()}")
+    }
+}
+
+fun main(args: Array<String>) = App().main(args)

tools/mklanguages/src/main/kotlin/app/pachli/mklanguages/ValuesParser.kt

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2023 Pachli Association
+ *
+ * This file is a part of Pachli.
+ *
+ * This program is free software; you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
+ * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with Pachli; if not,
+ * see <http://www.gnu.org/licenses>.
+ */
+
+package app.pachli.mklanguages
+
+import com.github.h0tk3y.betterParse.combinators.and
+import com.github.h0tk3y.betterParse.combinators.optional
+import com.github.h0tk3y.betterParse.combinators.or
+import com.github.h0tk3y.betterParse.combinators.skip
+import com.github.h0tk3y.betterParse.combinators.times
+import com.github.h0tk3y.betterParse.combinators.use
+import com.github.h0tk3y.betterParse.grammar.Grammar
+import com.github.h0tk3y.betterParse.lexer.literalToken
+import com.github.h0tk3y.betterParse.lexer.regexToken
+
+data class MobileCodes(val mcc: String, val mnc: String? = null)
+
+data class Locale(val lang: String, val region: String? = null, val script: String? = null)
+
+data class ConfigurationQualifier(val mobileCodes: MobileCodes? = null, val locale: Locale? = null)
+
+/**
+ * Parse an Android `values-*` resource directory name and extract the configuration qualifiers
+ *
+ * Directory name components and their ordering is described in
+ * https://developer.android.com/guide/topics/resources/providing-resources#table2).
+ */
+// TODO: At the moment this only deals with Locale, as that's what mklanguages needs. This could
+// be expanded in to a general parser for `values-*` directories if we needed that.
+class ValuesParser : Grammar<ConfigurationQualifier>() {
+    // Tokenizers
+    private val values by literalToken("values")
+    private val sep by literalToken("-")
+    private val mobileCodes by regexToken("(?i:mcc\\d+)(?i:mnc\\d+)?")
+    private val locale by regexToken("(?i:[a-z]{2,3})(?i:-r([a-z]{2,3}))?(?=-|$)")
+    private val bcpStartTag by regexToken("(?i:b\\+[a-z]{2,3})")
+    private val bcpSubtag by regexToken("(?i:\\+[a-z]+)")
+    private val layoutDirection by regexToken("(?i:ldrtl|ldltr)")
+    private val smallestWidth by regexToken("(?i:sw\\d+dp)")
+    private val availableDimen by regexToken("(?i:[wh]\\d+dp)")
+    private val screenSize by regexToken("(?i:small|normal|large|xlarge)")
+    private val screenAspect by regexToken("(?i:long|notlong)")
+    private val roundScreen by regexToken("(?i:round|notround)")
+    private val wideColorGamut by regexToken("(?i:widecg|nowidecg)")
+    private val highDynamicRange by regexToken("(?i:highdr|lowdr)")
+    private val screenOrientation by regexToken("(?i:port|land)")
+    private val uiMode by regexToken("(?i:car|desk|television|appliance|watch|vrheadset)")
+    private val nightMode by regexToken("(?i:night|notNight)")
+    private val screenDpi by regexToken("(?i:(?:l|m|h|x|xx|xxx|no|tv|any|\\d+)dpi)")
+    private val touchScreen by regexToken("(?i:notouch|finger)")
+    private val keyboardAvailability by regexToken("(?i:keysexposed|keyshidden|keyssoft)")
+    private val inputMethod by regexToken("(?i:nokeys|qwerty|12key)")
+    private val navKeyAvailability by regexToken("(?i:naxexposed|navhidden)")
+    private val navMethod by regexToken("(?i:nonav|dpad|trackball|wheel)")
+    private val platformVersion by regexToken("(?i:v\\d+)")
+
+    // Parsers
+    private val mobileCodesParser by mobileCodes use {
+        val parts = this.text.split("-")
+        MobileCodes(mcc = parts[0], mnc = parts.getOrNull(1))
+    }
+
+    private val localeParser by locale use {
+        val parts = this.text.split("-r".toRegex(), 2)
+        Locale(lang = parts[0], region = parts.getOrNull(1))
+    }
+
+    private val bcpLocaleParser = bcpStartTag and (0..2 times bcpSubtag) use {
+        Locale(
+            lang = this.t1.text.split("+")[1],
+            script = this.t2.getOrNull(0)?.text?.split("+")?.get(1),
+            region = this.t2.getOrNull(1)?.text?.split("+")?.get(1)
+        )
+    }
+
+    private val qualifier = skip(values) and
+        optional(skip(sep) and mobileCodesParser) and
+        optional(skip(sep) and (localeParser or bcpLocaleParser)) and
+        optional(skip(sep) and layoutDirection) and
+        optional(skip(sep) and smallestWidth) and
+        optional(skip(sep) and availableDimen) and
+        optional(skip(sep) and screenSize) and
+        optional(skip(sep) and screenAspect) and
+        optional(skip(sep) and roundScreen) and
+        optional(skip(sep) and wideColorGamut) and
+        optional(skip(sep) and highDynamicRange) and
+        optional(skip(sep) and screenOrientation) and
+        optional(skip(sep) and uiMode) and
+        optional(skip(sep) and nightMode) and
+        optional(skip(sep) and screenDpi) and
+        optional(skip(sep) and touchScreen) and
+        // Commented out to work around https://github.com/h0tk3y/better-parse/issues/64
+//        optional(skip(sep) and keyboardAvailability) and
+//        optional(skip(sep) and inputMethod) and
+//        optional(skip(sep) and navKeyAvailability) and
+//        optional(skip(sep) and navMethod) and
+        optional(skip(sep) and platformVersion)
+
+    private val qualifierParser by qualifier use {
+        ConfigurationQualifier(
+            mobileCodes = this.t1,
+            locale = this.t2
+        )
+    }
+
+    override val rootParser by qualifierParser
+}

tools/mklanguages/src/test/kotlin/app/pachli/mklanguages/ValuesParserTest.kt

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2023 Pachli Association
+ *
+ * This file is a part of Pachli.
+ *
+ * This program is free software; you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
+ * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with Pachli; if not,
+ * see <http://www.gnu.org/licenses>.
+ */
+
+package app.pachli.mklanguages
+
+import com.github.h0tk3y.betterParse.grammar.parseToEnd
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Nested
+import org.junit.jupiter.api.parallel.Execution
+import org.junit.jupiter.api.parallel.ExecutionMode
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.MethodSource
+import java.util.stream.Stream
+
+@Execution(ExecutionMode.CONCURRENT)
+internal class ValuesParserTest {
+    @Nested
+    @Execution(ExecutionMode.CONCURRENT)
+    inner class ParseLocale {
+        inner class Params(val filename: String, val expected: Locale?)
+
+        private val parser = ValuesParser()
+
+        private fun getParams(): Stream<Params> {
+            return Stream.of(
+                Params("values", null),
+                Params("values-en", Locale(lang = "en")),
+                Params("values-en-rGB", Locale(lang = "en", region = "GB")),
+                Params("values-b+tzm+Tfng", Locale(lang = "tzm", script = "Tfng")),
+                Params("values-mcc001", null),
+                Params("values-land", null)
+            )
+        }
+
+        @ParameterizedTest
+        @MethodSource("getParams")
+        fun `returns the expected locale`(params: Params) {
+            assertEquals(params.expected, parser.parseToEnd(params.filename).locale)
+        }
+    }
+}

tools/mklanguages/src/test/resources/junit-platform.properties

@@ -0,0 +1,19 @@
+#
+# Copyright 2023 Pachli Association
+#
+# This file is a part of Pachli.
+#
+# This program is free software; you can redistribute it and/or modify it under the terms of the
+# GNU General Public License as published by the Free Software Foundation; either version 3 of the
+# License, or (at your option) any later version.
+#
+# Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
+# the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with Pachli; if not,
+# see <http://www.gnu.org/licenses>.
+#
+
+junit.jupiter.testinstance.lifecycle.default = per_class
+junit.jupiter.execution.parallel.enabled = true