change: Add tools/mklanguages

The existing language list is incomplete, sorted incorrectly and does
not use the correct language names.

Add a small tool that parses the resource directories that contain
string translations, determines the correct language name and sort
order, and updates the correct application resources so language
lists are displayed correctly.
This commit is contained in:
Nik Clayton 2023-09-01 20:00:49 +02:00
parent 369979a60e
commit d7b504f31e
No known key found for this signature in database
GPG Key ID: F95268159C2EC897
10 changed files with 535 additions and 0 deletions

15
runtools Normal file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env bash
# Run one of the tools.
# The first argument must be the name of the tool task (e.g. mklanguages).
# Any remaining arguments are forwarded to the tool's argv.
task=$1
shift 1
if [ -z "${task}" ] || [ ! -d "tools/${task}" ]
then
echo "Unknown tool: '${task}'"
exit 1
fi
./gradlew --quiet ":tools:${task}:installDist" && "./tools/${task}/build/install/${task}/bin/${task}" "$@"

24
runtools.bat Normal file
View File

@ -0,0 +1,24 @@
@if "%DEBUG%"=="" @echo off
:: Run one of the tools.
:: The first argument must be the name of the tools (e.g. mklanguages).
:: Any remaining arguments are forwarded to the tool's argv.
if "%OS%"=="Windows_NT" setlocal EnableDelayedExpansion
set TASK=%~1
set TOOL=false
if defined TASK if not "!TASK: =!"=="" if exist "tools\%TASK%\*" set TOOL=true
if "%TOOL%"=="false" (
echo Unknown tool: '%TASK%'
exit /b 1
)
set ARGS=%*
set ARGS=!ARGS:*%1=!
if "!ARGS:~0,1!"==" " set ARGS=!ARGS:~1!
call gradlew --quiet ":tools:%TASK%:installDist" && call "tools\%TASK%\build\install\%TASK%\bin\%TASK%" %ARGS%
if "%OS%"=="Windows_NT" endlocal

View File

@ -17,3 +17,4 @@ dependencyResolutionManagement {
enableFeaturePreview("STABLE_CONFIGURATION_CACHE") enableFeaturePreview("STABLE_CONFIGURATION_CACHE")
include ':app' include ':app'
include ':tools:mklanguages'

33
tools/build.gradle.kts Normal file
View File

@ -0,0 +1,33 @@
/*
* Copyright 2023 Tusky Contributors
*
* This file is a part of Tusky.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU General Public License as published by the Free Software Foundation; either version 3 of the
* License, or (at your option) any later version.
*
* Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
* the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with Tusky; if not,
* see <http://www.gnu.org/licenses>.
*/
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
subprojects {
apply(plugin = "kotlin")
apply(plugin = "application")
dependencies {
"implementation"("com.github.ajalt.clikt:clikt:3.5.2")
}
tasks.withType<KotlinCompile>().configureEach {
kotlinOptions {
freeCompilerArgs = freeCompilerArgs + "-opt-in=kotlin.RequiresOptIn"
}
}
}

View File

@ -0,0 +1,21 @@
# mklanguages
## Synopsis
`mklanguages` ensures that the language list in Pachli is:
- Up to date
- Sorted according to ICU guidelines
- Uses language names according to ICU guidelines
Use `mklanguages` whenever a new language is added to Pachli.
## Usage
From the parent directory, run:
```shell
./runtools mklanguages
```
Verify the modifications made to the Pachli resource files, and commit the result.

View File

@ -0,0 +1,40 @@
/*
* Copyright 2023 Pachli Association
*
* This file is a part of Pachli.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU General Public License as published by the Free Software Foundation; either version 3 of the
* License, or (at your option) any later version.
*
* Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
* the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with Pachli; if not,
* see <http://www.gnu.org/licenses>.
*/
application {
mainClass.set("app.pachli.mklanguages.MainKt")
}
dependencies {
// ICU
implementation("com.ibm.icu:icu4j:73.1")
// Parsing
implementation("com.github.h0tk3y.betterParse:better-parse:0.4.4")
// Logging
implementation("io.github.oshai:kotlin-logging-jvm:4.0.0-beta-28")
implementation("ch.qos.logback:logback-classic:1.3.0")
// Testing
testImplementation(kotlin("test"))
testImplementation("org.junit.jupiter:junit-jupiter-params:5.9.2") // for parameterized tests
}
tasks.test {
useJUnitPlatform()
}

View File

@ -0,0 +1,206 @@
/*
* Copyright 2023 Pachli Association
*
* This file is a part of Pachli.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU General Public License as published by the Free Software Foundation; either version 3 of the
* License, or (at your option) any later version.
*
* Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
* the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with Pachli; if not,
* see <http://www.gnu.org/licenses>.
*/
package app.pachli.mklanguages
import ch.qos.logback.classic.Level
import ch.qos.logback.classic.Logger
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.core.UsageError
import com.github.ajalt.clikt.parameters.options.flag
import com.github.ajalt.clikt.parameters.options.option
import com.github.h0tk3y.betterParse.grammar.parseToEnd
import com.ibm.icu.text.CaseMap
import com.ibm.icu.text.Collator
import com.ibm.icu.util.ULocale
import io.github.oshai.KotlinLogging
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.StandardCopyOption
import kotlin.io.path.Path
import kotlin.io.path.createTempFile
import kotlin.io.path.div
import kotlin.io.path.exists
import kotlin.io.path.isDirectory
import kotlin.io.path.isRegularFile
import kotlin.io.path.listDirectoryEntries
private val log = KotlinLogging.logger {}
/** The information needed to encode a language in the XML resources */
data class Language(
/** Language code */
val code: String,
/**
* Name of the language, in that language. E.g., the display name for English is "English",
* the display name for Icelandic is "Íslenska".
*/
val displayName: String,
/** Name of the language in English */
val displayNameEnglish: String
) {
companion object {
private val toTitle = CaseMap.toTitle()
/** Create a [Language] from a [ULocale] */
fun from(locale: ULocale) = Language(
locale.name.replace("_", "-"),
toTitle.apply(locale.toLocale(), null, locale.getDisplayName(locale)),
locale.getDisplayName(ULocale.ENGLISH)
)
}
}
/**
* Constructs the `language_entries` and `language_values` string arrays in donottranslate.xml.
*
* - Finds all the `values-*` directories that contain `strings.xml`
* - Parses out the language code from the directory name
* - Uses the ICU libraries to determine the correct name for the language
* - Sorts the list of languages using ICU collation rules
* - Updates donottranslate.xml with the new data
*
* Run this after creating a new translation.
*
* Run with `gradlew :tools:mklanguages:run` or `runtools mklanguages`.
*/
class App : CliktCommand(help = """Update languages in donottranslate.xml""") {
private val verbose by option("-n", "--verbose", help = "show additional information").flag()
/**
* Returns the full path to the Pachli `.../app/src/main/res` directory, starting from the
* given [start] directory, walking up the tree if it can't be found there.
*
* @return the path, or null if it's not a subtree of [start] or any of its parents.
*/
private fun findResourcePath(start: Path): Path? {
val suffix = Path("app/src/main/res")
var prefix = start
var resourcePath: Path
do {
resourcePath = prefix / suffix
if (resourcePath.exists()) return resourcePath
prefix = prefix.parent
} while (prefix != prefix.root)
return null
}
override fun run() {
System.setProperty("file.encoding", "UTF8")
(log.underlyingLogger as Logger).level = if (verbose) Level.INFO else Level.WARN
val cwd = Paths.get("").toAbsolutePath()
log.info("working directory: $cwd")
val resourcePath = findResourcePath(cwd) ?: throw UsageError("could not find app/src/main/res in tree")
// Enumerate all the values-* directories that contain a strings.xml file
val resourceDirs = resourcePath.listDirectoryEntries("values-*")
.filter { entry -> entry.isDirectory() }
.filter { dir -> (dir / "strings.xml").isRegularFile() }
if (resourceDirs.isEmpty()) throw UsageError("no strings.xml files found in $resourcePath/values-*")
// Convert the `values-...` directory names to instances of ULocale.
val valuesParser = ValuesParser()
val locales = resourceDirs
.asSequence()
.map { it.fileName.toString() }
.onEach { log.info("parsing directory name: $it") }
// Special-case ber, see https://github.com/tuskyapp/Tusky/issues/3637
.map { if (it == "values-ber") "values-b+tzm+Tfng" else it }
.mapNotNull { valuesParser.parseToEnd(it).locale }
.onEach { log.info(" --> $it") }
.toMutableList()
.apply { add(Locale(lang = "en")) }
.map { ULocale(it.lang, it.region, it.script) }
// Construct the languages. Sort each locale by its display name, as rendered in that
// locale, and fold case.
val collator = Collator.getInstance(ULocale.ENGLISH)
val casemapFold = CaseMap.fold()
val languages = locales.sortedBy { collator.getCollationKey(casemapFold.apply(it.getDisplayName(it))) }
.map { Language.from(it) }
.toMutableList()
// The first language in the list is the system default
languages.add(0, Language("default", "@string/system_default", "System default"))
// Copy donottranslate.xml line by line to a new file, replacing the contents of the
// `language_entries` and `language_values` arrays with fresh data.
val tmpFile = createTempFile().toFile()
val w = tmpFile.printWriter()
val donottranslate_xml = resourcePath / "values" / "donottranslate.xml"
donottranslate_xml.toFile().useLines { lines ->
var inLanguageEntries = false
var inLanguageValues = false
for (line in lines) {
// Default behaviour, copy the line unless inside one of the arrays
if (!inLanguageEntries && !inLanguageValues) {
w.println(line)
}
// Started the `language_entries` array
if (line.contains("<string-array name=\"language_entries\">")) {
inLanguageEntries = true
continue
}
// Started the `language_values` array
if (line.contains("<string-array name=\"language_values\">")) {
inLanguageValues = true
continue
}
// At the end of `language_entries`? Emit each language, one per line. The
// item is the language's name, then a comment with the code and English name.
// Then close the array.
if (inLanguageEntries && line.contains("</string-array>")) {
languages.forEach {
w.println(" <item>${it.displayName}</item> <!-- ${it.code}: ${it.displayNameEnglish} -->")
}
w.println(line)
inLanguageEntries = false
continue
}
// At the end of `language_values`? Emit each language code, one per line.
// Then close the array.
if (inLanguageValues && line.contains("</string-array>")) {
languages.forEach { w.println(" <item>${it.code}</item>") }
w.println(line)
inLanguageValues = false
continue
}
}
}
// Close, then replace donotranslate.xml
w.close()
Files.move(tmpFile.toPath(), donottranslate_xml, StandardCopyOption.REPLACE_EXISTING)
log.info("replaced ${donottranslate_xml.toAbsolutePath()}")
}
}
fun main(args: Array<String>) = App().main(args)

View File

@ -0,0 +1,121 @@
/*
* Copyright 2023 Pachli Association
*
* This file is a part of Pachli.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU General Public License as published by the Free Software Foundation; either version 3 of the
* License, or (at your option) any later version.
*
* Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
* the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with Pachli; if not,
* see <http://www.gnu.org/licenses>.
*/
package app.pachli.mklanguages
import com.github.h0tk3y.betterParse.combinators.and
import com.github.h0tk3y.betterParse.combinators.optional
import com.github.h0tk3y.betterParse.combinators.or
import com.github.h0tk3y.betterParse.combinators.skip
import com.github.h0tk3y.betterParse.combinators.times
import com.github.h0tk3y.betterParse.combinators.use
import com.github.h0tk3y.betterParse.grammar.Grammar
import com.github.h0tk3y.betterParse.lexer.literalToken
import com.github.h0tk3y.betterParse.lexer.regexToken
data class MobileCodes(val mcc: String, val mnc: String? = null)
data class Locale(val lang: String, val region: String? = null, val script: String? = null)
data class ConfigurationQualifier(val mobileCodes: MobileCodes? = null, val locale: Locale? = null)
/**
* Parse an Android `values-*` resource directory name and extract the configuration qualifiers
*
* Directory name components and their ordering is described in
* https://developer.android.com/guide/topics/resources/providing-resources#table2).
*/
// TODO: At the moment this only deals with Locale, as that's what mklanguages needs. This could
// be expanded in to a general parser for `values-*` directories if we needed that.
class ValuesParser : Grammar<ConfigurationQualifier>() {
// Tokenizers
private val values by literalToken("values")
private val sep by literalToken("-")
private val mobileCodes by regexToken("(?i:mcc\\d+)(?i:mnc\\d+)?")
private val locale by regexToken("(?i:[a-z]{2,3})(?i:-r([a-z]{2,3}))?(?=-|$)")
private val bcpStartTag by regexToken("(?i:b\\+[a-z]{2,3})")
private val bcpSubtag by regexToken("(?i:\\+[a-z]+)")
private val layoutDirection by regexToken("(?i:ldrtl|ldltr)")
private val smallestWidth by regexToken("(?i:sw\\d+dp)")
private val availableDimen by regexToken("(?i:[wh]\\d+dp)")
private val screenSize by regexToken("(?i:small|normal|large|xlarge)")
private val screenAspect by regexToken("(?i:long|notlong)")
private val roundScreen by regexToken("(?i:round|notround)")
private val wideColorGamut by regexToken("(?i:widecg|nowidecg)")
private val highDynamicRange by regexToken("(?i:highdr|lowdr)")
private val screenOrientation by regexToken("(?i:port|land)")
private val uiMode by regexToken("(?i:car|desk|television|appliance|watch|vrheadset)")
private val nightMode by regexToken("(?i:night|notNight)")
private val screenDpi by regexToken("(?i:(?:l|m|h|x|xx|xxx|no|tv|any|\\d+)dpi)")
private val touchScreen by regexToken("(?i:notouch|finger)")
private val keyboardAvailability by regexToken("(?i:keysexposed|keyshidden|keyssoft)")
private val inputMethod by regexToken("(?i:nokeys|qwerty|12key)")
private val navKeyAvailability by regexToken("(?i:naxexposed|navhidden)")
private val navMethod by regexToken("(?i:nonav|dpad|trackball|wheel)")
private val platformVersion by regexToken("(?i:v\\d+)")
// Parsers
private val mobileCodesParser by mobileCodes use {
val parts = this.text.split("-")
MobileCodes(mcc = parts[0], mnc = parts.getOrNull(1))
}
private val localeParser by locale use {
val parts = this.text.split("-r".toRegex(), 2)
Locale(lang = parts[0], region = parts.getOrNull(1))
}
private val bcpLocaleParser = bcpStartTag and (0..2 times bcpSubtag) use {
Locale(
lang = this.t1.text.split("+")[1],
script = this.t2.getOrNull(0)?.text?.split("+")?.get(1),
region = this.t2.getOrNull(1)?.text?.split("+")?.get(1)
)
}
private val qualifier = skip(values) and
optional(skip(sep) and mobileCodesParser) and
optional(skip(sep) and (localeParser or bcpLocaleParser)) and
optional(skip(sep) and layoutDirection) and
optional(skip(sep) and smallestWidth) and
optional(skip(sep) and availableDimen) and
optional(skip(sep) and screenSize) and
optional(skip(sep) and screenAspect) and
optional(skip(sep) and roundScreen) and
optional(skip(sep) and wideColorGamut) and
optional(skip(sep) and highDynamicRange) and
optional(skip(sep) and screenOrientation) and
optional(skip(sep) and uiMode) and
optional(skip(sep) and nightMode) and
optional(skip(sep) and screenDpi) and
optional(skip(sep) and touchScreen) and
// Commented out to work around https://github.com/h0tk3y/better-parse/issues/64
// optional(skip(sep) and keyboardAvailability) and
// optional(skip(sep) and inputMethod) and
// optional(skip(sep) and navKeyAvailability) and
// optional(skip(sep) and navMethod) and
optional(skip(sep) and platformVersion)
private val qualifierParser by qualifier use {
ConfigurationQualifier(
mobileCodes = this.t1,
locale = this.t2
)
}
override val rootParser by qualifierParser
}

View File

@ -0,0 +1,55 @@
/*
* Copyright 2023 Pachli Association
*
* This file is a part of Pachli.
*
* This program is free software; you can redistribute it and/or modify it under the terms of the
* GNU General Public License as published by the Free Software Foundation; either version 3 of the
* License, or (at your option) any later version.
*
* Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
* the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with Pachli; if not,
* see <http://www.gnu.org/licenses>.
*/
package app.pachli.mklanguages
import com.github.h0tk3y.betterParse.grammar.parseToEnd
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Nested
import org.junit.jupiter.api.parallel.Execution
import org.junit.jupiter.api.parallel.ExecutionMode
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import java.util.stream.Stream
@Execution(ExecutionMode.CONCURRENT)
internal class ValuesParserTest {
@Nested
@Execution(ExecutionMode.CONCURRENT)
inner class ParseLocale {
inner class Params(val filename: String, val expected: Locale?)
private val parser = ValuesParser()
private fun getParams(): Stream<Params> {
return Stream.of(
Params("values", null),
Params("values-en", Locale(lang = "en")),
Params("values-en-rGB", Locale(lang = "en", region = "GB")),
Params("values-b+tzm+Tfng", Locale(lang = "tzm", script = "Tfng")),
Params("values-mcc001", null),
Params("values-land", null)
)
}
@ParameterizedTest
@MethodSource("getParams")
fun `returns the expected locale`(params: Params) {
assertEquals(params.expected, parser.parseToEnd(params.filename).locale)
}
}
}

View File

@ -0,0 +1,19 @@
#
# Copyright 2023 Pachli Association
#
# This file is a part of Pachli.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the
# GNU General Public License as published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
# the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along with Pachli; if not,
# see <http://www.gnu.org/licenses>.
#
junit.jupiter.testinstance.lifecycle.default = per_class
junit.jupiter.execution.parallel.enabled = true