diff --git a/runtools b/runtools new file mode 100644 index 000000000..5b9768002 --- /dev/null +++ b/runtools @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Run one of the tools. +# The first argument must be the name of the tool task (e.g. mklanguages). +# Any remaining arguments are forwarded to the tool's argv. + +task=$1 +shift 1 + +if [ -z "${task}" ] || [ ! -d "tools/${task}" ] +then + echo "Unknown tool: '${task}'" + exit 1 +fi + +./gradlew --quiet ":tools:${task}:installDist" && "./tools/${task}/build/install/${task}/bin/${task}" "$@" diff --git a/runtools.bat b/runtools.bat new file mode 100644 index 000000000..9801e1b6a --- /dev/null +++ b/runtools.bat @@ -0,0 +1,24 @@ +@if "%DEBUG%"=="" @echo off +:: Run one of the tools. +:: The first argument must be the name of the tools (e.g. mklanguages). +:: Any remaining arguments are forwarded to the tool's argv. + +if "%OS%"=="Windows_NT" setlocal EnableDelayedExpansion + +set TASK=%~1 + +set TOOL=false +if defined TASK if not "!TASK: =!"=="" if exist "tools\%TASK%\*" set TOOL=true + +if "%TOOL%"=="false" ( + echo Unknown tool: '%TASK%' + exit /b 1 +) + +set ARGS=%* +set ARGS=!ARGS:*%1=! +if "!ARGS:~0,1!"==" " set ARGS=!ARGS:~1! + +call gradlew --quiet ":tools:%TASK%:installDist" && call "tools\%TASK%\build\install\%TASK%\bin\%TASK%" %ARGS% + +if "%OS%"=="Windows_NT" endlocal diff --git a/settings.gradle b/settings.gradle index 1b76791c7..59ea9f405 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,3 +17,4 @@ dependencyResolutionManagement { enableFeaturePreview("STABLE_CONFIGURATION_CACHE") include ':app' +include ':tools:mklanguages' diff --git a/tools/build.gradle.kts b/tools/build.gradle.kts new file mode 100644 index 000000000..b70a7dc8a --- /dev/null +++ b/tools/build.gradle.kts @@ -0,0 +1,33 @@ +/* + * Copyright 2023 Tusky Contributors + * + * This file is a part of Tusky. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Tusky; if not, + * see . + */ + +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +subprojects { + apply(plugin = "kotlin") + apply(plugin = "application") + + dependencies { + "implementation"("com.github.ajalt.clikt:clikt:3.5.2") + } + + tasks.withType().configureEach { + kotlinOptions { + freeCompilerArgs = freeCompilerArgs + "-opt-in=kotlin.RequiresOptIn" + } + } +} diff --git a/tools/mklanguages/README.md b/tools/mklanguages/README.md new file mode 100644 index 000000000..7841676a5 --- /dev/null +++ b/tools/mklanguages/README.md @@ -0,0 +1,21 @@ +# mklanguages + +## Synopsis + +`mklanguages` ensures that the language list in Pachli is: + +- Up to date +- Sorted according to ICU guidelines +- Uses language names according to ICU guidelines + +Use `mklanguages` whenever a new language is added to Pachli. + +## Usage + +From the parent directory, run: + +```shell +./runtools mklanguages +``` + +Verify the modifications made to the Pachli resource files, and commit the result. diff --git a/tools/mklanguages/build.gradle.kts b/tools/mklanguages/build.gradle.kts new file mode 100644 index 000000000..9f00c27ca --- /dev/null +++ b/tools/mklanguages/build.gradle.kts @@ -0,0 +1,40 @@ +/* + * Copyright 2023 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +application { + mainClass.set("app.pachli.mklanguages.MainKt") +} + +dependencies { + // ICU + implementation("com.ibm.icu:icu4j:73.1") + + // Parsing + implementation("com.github.h0tk3y.betterParse:better-parse:0.4.4") + + // Logging + implementation("io.github.oshai:kotlin-logging-jvm:4.0.0-beta-28") + implementation("ch.qos.logback:logback-classic:1.3.0") + + // Testing + testImplementation(kotlin("test")) + testImplementation("org.junit.jupiter:junit-jupiter-params:5.9.2") // for parameterized tests +} + +tasks.test { + useJUnitPlatform() +} diff --git a/tools/mklanguages/src/main/kotlin/app/pachli/mklanguages/Main.kt b/tools/mklanguages/src/main/kotlin/app/pachli/mklanguages/Main.kt new file mode 100644 index 000000000..0950df4af --- /dev/null +++ b/tools/mklanguages/src/main/kotlin/app/pachli/mklanguages/Main.kt @@ -0,0 +1,206 @@ +/* + * Copyright 2023 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +package app.pachli.mklanguages + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.core.UsageError +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.option +import com.github.h0tk3y.betterParse.grammar.parseToEnd +import com.ibm.icu.text.CaseMap +import com.ibm.icu.text.Collator +import com.ibm.icu.util.ULocale +import io.github.oshai.KotlinLogging +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.StandardCopyOption +import kotlin.io.path.Path +import kotlin.io.path.createTempFile +import kotlin.io.path.div +import kotlin.io.path.exists +import kotlin.io.path.isDirectory +import kotlin.io.path.isRegularFile +import kotlin.io.path.listDirectoryEntries + +private val log = KotlinLogging.logger {} + +/** The information needed to encode a language in the XML resources */ +data class Language( + /** Language code */ + val code: String, + + /** + * Name of the language, in that language. E.g., the display name for English is "English", + * the display name for Icelandic is "Íslenska". + */ + val displayName: String, + + /** Name of the language in English */ + val displayNameEnglish: String +) { + companion object { + private val toTitle = CaseMap.toTitle() + + /** Create a [Language] from a [ULocale] */ + fun from(locale: ULocale) = Language( + locale.name.replace("_", "-"), + toTitle.apply(locale.toLocale(), null, locale.getDisplayName(locale)), + locale.getDisplayName(ULocale.ENGLISH) + ) + } +} + +/** + * Constructs the `language_entries` and `language_values` string arrays in donottranslate.xml. + * + * - Finds all the `values-*` directories that contain `strings.xml` + * - Parses out the language code from the directory name + * - Uses the ICU libraries to determine the correct name for the language + * - Sorts the list of languages using ICU collation rules + * - Updates donottranslate.xml with the new data + * + * Run this after creating a new translation. + * + * Run with `gradlew :tools:mklanguages:run` or `runtools mklanguages`. + */ +class App : CliktCommand(help = """Update languages in donottranslate.xml""") { + private val verbose by option("-n", "--verbose", help = "show additional information").flag() + + /** + * Returns the full path to the Pachli `.../app/src/main/res` directory, starting from the + * given [start] directory, walking up the tree if it can't be found there. + * + * @return the path, or null if it's not a subtree of [start] or any of its parents. + */ + private fun findResourcePath(start: Path): Path? { + val suffix = Path("app/src/main/res") + + var prefix = start + var resourcePath: Path + do { + resourcePath = prefix / suffix + if (resourcePath.exists()) return resourcePath + prefix = prefix.parent + } while (prefix != prefix.root) + + return null + } + + override fun run() { + System.setProperty("file.encoding", "UTF8") + (log.underlyingLogger as Logger).level = if (verbose) Level.INFO else Level.WARN + + val cwd = Paths.get("").toAbsolutePath() + log.info("working directory: $cwd") + + val resourcePath = findResourcePath(cwd) ?: throw UsageError("could not find app/src/main/res in tree") + + // Enumerate all the values-* directories that contain a strings.xml file + val resourceDirs = resourcePath.listDirectoryEntries("values-*") + .filter { entry -> entry.isDirectory() } + .filter { dir -> (dir / "strings.xml").isRegularFile() } + + if (resourceDirs.isEmpty()) throw UsageError("no strings.xml files found in $resourcePath/values-*") + + // Convert the `values-...` directory names to instances of ULocale. + val valuesParser = ValuesParser() + val locales = resourceDirs + .asSequence() + .map { it.fileName.toString() } + .onEach { log.info("parsing directory name: $it") } + // Special-case ber, see https://github.com/tuskyapp/Tusky/issues/3637 + .map { if (it == "values-ber") "values-b+tzm+Tfng" else it } + .mapNotNull { valuesParser.parseToEnd(it).locale } + .onEach { log.info(" --> $it") } + .toMutableList() + .apply { add(Locale(lang = "en")) } + .map { ULocale(it.lang, it.region, it.script) } + + // Construct the languages. Sort each locale by its display name, as rendered in that + // locale, and fold case. + val collator = Collator.getInstance(ULocale.ENGLISH) + val casemapFold = CaseMap.fold() + + val languages = locales.sortedBy { collator.getCollationKey(casemapFold.apply(it.getDisplayName(it))) } + .map { Language.from(it) } + .toMutableList() + + // The first language in the list is the system default + languages.add(0, Language("default", "@string/system_default", "System default")) + + // Copy donottranslate.xml line by line to a new file, replacing the contents of the + // `language_entries` and `language_values` arrays with fresh data. + val tmpFile = createTempFile().toFile() + val w = tmpFile.printWriter() + val donottranslate_xml = resourcePath / "values" / "donottranslate.xml" + donottranslate_xml.toFile().useLines { lines -> + var inLanguageEntries = false + var inLanguageValues = false + + for (line in lines) { + // Default behaviour, copy the line unless inside one of the arrays + if (!inLanguageEntries && !inLanguageValues) { + w.println(line) + } + + // Started the `language_entries` array + if (line.contains("")) { + inLanguageEntries = true + continue + } + + // Started the `language_values` array + if (line.contains("")) { + inLanguageValues = true + continue + } + + // At the end of `language_entries`? Emit each language, one per line. The + // item is the language's name, then a comment with the code and English name. + // Then close the array. + if (inLanguageEntries && line.contains("")) { + languages.forEach { + w.println(" ${it.displayName} ") + } + w.println(line) + inLanguageEntries = false + continue + } + + // At the end of `language_values`? Emit each language code, one per line. + // Then close the array. + if (inLanguageValues && line.contains("")) { + languages.forEach { w.println(" ${it.code}") } + w.println(line) + inLanguageValues = false + continue + } + } + } + + // Close, then replace donotranslate.xml + w.close() + Files.move(tmpFile.toPath(), donottranslate_xml, StandardCopyOption.REPLACE_EXISTING) + log.info("replaced ${donottranslate_xml.toAbsolutePath()}") + } +} + +fun main(args: Array) = App().main(args) diff --git a/tools/mklanguages/src/main/kotlin/app/pachli/mklanguages/ValuesParser.kt b/tools/mklanguages/src/main/kotlin/app/pachli/mklanguages/ValuesParser.kt new file mode 100644 index 000000000..a3e53d3c5 --- /dev/null +++ b/tools/mklanguages/src/main/kotlin/app/pachli/mklanguages/ValuesParser.kt @@ -0,0 +1,121 @@ +/* + * Copyright 2023 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +package app.pachli.mklanguages + +import com.github.h0tk3y.betterParse.combinators.and +import com.github.h0tk3y.betterParse.combinators.optional +import com.github.h0tk3y.betterParse.combinators.or +import com.github.h0tk3y.betterParse.combinators.skip +import com.github.h0tk3y.betterParse.combinators.times +import com.github.h0tk3y.betterParse.combinators.use +import com.github.h0tk3y.betterParse.grammar.Grammar +import com.github.h0tk3y.betterParse.lexer.literalToken +import com.github.h0tk3y.betterParse.lexer.regexToken + +data class MobileCodes(val mcc: String, val mnc: String? = null) + +data class Locale(val lang: String, val region: String? = null, val script: String? = null) + +data class ConfigurationQualifier(val mobileCodes: MobileCodes? = null, val locale: Locale? = null) + +/** + * Parse an Android `values-*` resource directory name and extract the configuration qualifiers + * + * Directory name components and their ordering is described in + * https://developer.android.com/guide/topics/resources/providing-resources#table2). + */ +// TODO: At the moment this only deals with Locale, as that's what mklanguages needs. This could +// be expanded in to a general parser for `values-*` directories if we needed that. +class ValuesParser : Grammar() { + // Tokenizers + private val values by literalToken("values") + private val sep by literalToken("-") + private val mobileCodes by regexToken("(?i:mcc\\d+)(?i:mnc\\d+)?") + private val locale by regexToken("(?i:[a-z]{2,3})(?i:-r([a-z]{2,3}))?(?=-|$)") + private val bcpStartTag by regexToken("(?i:b\\+[a-z]{2,3})") + private val bcpSubtag by regexToken("(?i:\\+[a-z]+)") + private val layoutDirection by regexToken("(?i:ldrtl|ldltr)") + private val smallestWidth by regexToken("(?i:sw\\d+dp)") + private val availableDimen by regexToken("(?i:[wh]\\d+dp)") + private val screenSize by regexToken("(?i:small|normal|large|xlarge)") + private val screenAspect by regexToken("(?i:long|notlong)") + private val roundScreen by regexToken("(?i:round|notround)") + private val wideColorGamut by regexToken("(?i:widecg|nowidecg)") + private val highDynamicRange by regexToken("(?i:highdr|lowdr)") + private val screenOrientation by regexToken("(?i:port|land)") + private val uiMode by regexToken("(?i:car|desk|television|appliance|watch|vrheadset)") + private val nightMode by regexToken("(?i:night|notNight)") + private val screenDpi by regexToken("(?i:(?:l|m|h|x|xx|xxx|no|tv|any|\\d+)dpi)") + private val touchScreen by regexToken("(?i:notouch|finger)") + private val keyboardAvailability by regexToken("(?i:keysexposed|keyshidden|keyssoft)") + private val inputMethod by regexToken("(?i:nokeys|qwerty|12key)") + private val navKeyAvailability by regexToken("(?i:naxexposed|navhidden)") + private val navMethod by regexToken("(?i:nonav|dpad|trackball|wheel)") + private val platformVersion by regexToken("(?i:v\\d+)") + + // Parsers + private val mobileCodesParser by mobileCodes use { + val parts = this.text.split("-") + MobileCodes(mcc = parts[0], mnc = parts.getOrNull(1)) + } + + private val localeParser by locale use { + val parts = this.text.split("-r".toRegex(), 2) + Locale(lang = parts[0], region = parts.getOrNull(1)) + } + + private val bcpLocaleParser = bcpStartTag and (0..2 times bcpSubtag) use { + Locale( + lang = this.t1.text.split("+")[1], + script = this.t2.getOrNull(0)?.text?.split("+")?.get(1), + region = this.t2.getOrNull(1)?.text?.split("+")?.get(1) + ) + } + + private val qualifier = skip(values) and + optional(skip(sep) and mobileCodesParser) and + optional(skip(sep) and (localeParser or bcpLocaleParser)) and + optional(skip(sep) and layoutDirection) and + optional(skip(sep) and smallestWidth) and + optional(skip(sep) and availableDimen) and + optional(skip(sep) and screenSize) and + optional(skip(sep) and screenAspect) and + optional(skip(sep) and roundScreen) and + optional(skip(sep) and wideColorGamut) and + optional(skip(sep) and highDynamicRange) and + optional(skip(sep) and screenOrientation) and + optional(skip(sep) and uiMode) and + optional(skip(sep) and nightMode) and + optional(skip(sep) and screenDpi) and + optional(skip(sep) and touchScreen) and + // Commented out to work around https://github.com/h0tk3y/better-parse/issues/64 +// optional(skip(sep) and keyboardAvailability) and +// optional(skip(sep) and inputMethod) and +// optional(skip(sep) and navKeyAvailability) and +// optional(skip(sep) and navMethod) and + optional(skip(sep) and platformVersion) + + private val qualifierParser by qualifier use { + ConfigurationQualifier( + mobileCodes = this.t1, + locale = this.t2 + ) + } + + override val rootParser by qualifierParser +} diff --git a/tools/mklanguages/src/test/kotlin/app/pachli/mklanguages/ValuesParserTest.kt b/tools/mklanguages/src/test/kotlin/app/pachli/mklanguages/ValuesParserTest.kt new file mode 100644 index 000000000..0958eb291 --- /dev/null +++ b/tools/mklanguages/src/test/kotlin/app/pachli/mklanguages/ValuesParserTest.kt @@ -0,0 +1,55 @@ +/* + * Copyright 2023 Pachli Association + * + * This file is a part of Pachli. + * + * This program is free software; you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation; either version 3 of the + * License, or (at your option) any later version. + * + * Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even + * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along with Pachli; if not, + * see . + */ + +package app.pachli.mklanguages + +import com.github.h0tk3y.betterParse.grammar.parseToEnd +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.parallel.Execution +import org.junit.jupiter.api.parallel.ExecutionMode +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import java.util.stream.Stream + +@Execution(ExecutionMode.CONCURRENT) +internal class ValuesParserTest { + @Nested + @Execution(ExecutionMode.CONCURRENT) + inner class ParseLocale { + inner class Params(val filename: String, val expected: Locale?) + + private val parser = ValuesParser() + + private fun getParams(): Stream { + return Stream.of( + Params("values", null), + Params("values-en", Locale(lang = "en")), + Params("values-en-rGB", Locale(lang = "en", region = "GB")), + Params("values-b+tzm+Tfng", Locale(lang = "tzm", script = "Tfng")), + Params("values-mcc001", null), + Params("values-land", null) + ) + } + + @ParameterizedTest + @MethodSource("getParams") + fun `returns the expected locale`(params: Params) { + assertEquals(params.expected, parser.parseToEnd(params.filename).locale) + } + } +} diff --git a/tools/mklanguages/src/test/resources/junit-platform.properties b/tools/mklanguages/src/test/resources/junit-platform.properties new file mode 100644 index 000000000..a0bb564b3 --- /dev/null +++ b/tools/mklanguages/src/test/resources/junit-platform.properties @@ -0,0 +1,19 @@ +# +# Copyright 2023 Pachli Association +# +# This file is a part of Pachli. +# +# This program is free software; you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# Pachli is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even +# the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along with Pachli; if not, +# see . +# + +junit.jupiter.testinstance.lifecycle.default = per_class +junit.jupiter.execution.parallel.enabled = true