diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..0542767eff --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +**/snapshots/**/*.png filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fb8e3080ae..5208a38683 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,12 +24,26 @@ jobs: steps: - uses: actions/checkout@v3 with: + lfs: true fetch-depth: 0 - uses: actions/setup-java@v3 with: distribution: 'adopt' java-version: '11' - uses: gradle/gradle-build-action@v2 + + - name: Run screenshot tests + run: ./gradlew verifyScreenshots + + - name: Archive Screenshot Results on Error + if: failure() + uses: actions/upload-artifact@v1 + with: + name: screenshot-results + path: | + **/out/failures/ + **/build/reports/tests/*UnitTest/ + - uses: actions/setup-python@v4 with: python-version: 3.8 diff --git a/.github/workflows/validate-lfs.yml b/.github/workflows/validate-lfs.yml new file mode 100644 index 0000000000..203ecb0481 --- /dev/null +++ b/.github/workflows/validate-lfs.yml @@ -0,0 +1,15 @@ +name: Validate Git LFS + +on: [pull_request] + +jobs: + build: + runs-on: ubuntu-latest + name: Validate + steps: + - uses: actions/checkout@v3 + with: + lfs: 'true' + + - run: | + ./tools/validate_lfs.sh diff --git a/.gitignore b/.gitignore index f1c0b99b58..aae906afc2 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ /package.json /yarn.lock /node_modules +**/out/failures diff --git a/build.gradle b/build.gradle index 78cc80c291..7b67c89502 100644 --- a/build.gradle +++ b/build.gradle @@ -33,6 +33,7 @@ buildscript { classpath "org.jetbrains.dokka:dokka-gradle-plugin:1.7.10" classpath "org.jetbrains.kotlinx:kotlinx-knit:0.4.0" classpath 'com.jakewharton:butterknife-gradle-plugin:10.2.3' + classpath 'app.cash.paparazzi:paparazzi-gradle-plugin:1.0.0' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files } @@ -295,3 +296,28 @@ dependencyAnalysis { } } } + +tasks.register("recordScreenshots", GradleBuild) { + startParameter.projectProperties.screenshot = "" + tasks = [':vector:recordPaparazziDebug'] +} + +tasks.register("verifyScreenshots", GradleBuild) { + startParameter.projectProperties.screenshot = "" + tasks = [':vector:verifyPaparazziDebug'] +} + +ext.initScreenshotTests = { project -> + def hasScreenshots = project.hasProperty("screenshot") + if (hasScreenshots) { + project.apply plugin: 'app.cash.paparazzi' + } + project.android.testOptions.unitTests.all { + def screenshotTestCapture = "**/*ScreenshotTest*" + if (hasScreenshots) { + include screenshotTestCapture + } else { + exclude screenshotTestCapture + } + } +} diff --git a/changelog.d/5798.misc b/changelog.d/5798.misc new file mode 100644 index 0000000000..40185eac0d --- /dev/null +++ b/changelog.d/5798.misc @@ -0,0 +1 @@ +Adds screenshot testing tooling diff --git a/dependencies_groups.gradle b/dependencies_groups.gradle index e44e0bc5c2..a97d80bc7f 100644 --- a/dependencies_groups.gradle +++ b/dependencies_groups.gradle @@ -49,6 +49,7 @@ ext.groups = [ regex: [ ], group: [ + 'app.cash.paparazzi', 'ch.qos.logback', 'com.adevinta.android', 'com.airbnb.android', @@ -150,11 +151,14 @@ ext.groups = [ 'it.unimi.dsi', 'jakarta.activation', 'jakarta.xml.bind', + 'javax.activation', 'javax.annotation', 'javax.inject', + 'javax.xml.bind', 'jline', 'jp.wasabeef', 'junit', + 'kxml2', 'me.saket', 'net.bytebuddy', 'net.java', @@ -183,11 +187,13 @@ ext.groups = [ 'org.hamcrest', 'org.jacoco', 'org.java-websocket', + 'org.jcodec', 'org.jetbrains', 'org.jetbrains.dokka', 'org.jetbrains.intellij.deps', 'org.jetbrains.kotlin', 'org.jetbrains.kotlinx', + 'org.jetbrains.trove4j', 'org.json', 'org.jsoup', 'org.junit', diff --git a/docs/screenshot_testing.md b/docs/screenshot_testing.md new file mode 100644 index 0000000000..93b91cdf67 --- /dev/null +++ b/docs/screenshot_testing.md @@ -0,0 +1,72 @@ +# Screenshot testing + + + +* [Overview](#overview) +* [Setup](#setup) +* [Recording](#recording) +* [Verifying](#verifying) +* [Contributing](#contributing) +* [Example](#example) + + + +## Overview + +- Screenshot tests are tests which record the content of a rendered screen and verify subsequent runs to check if the screen renders differently. +- Element uses [Paparazzi](https://github.com/cashapp/paparazzi) to render, record and verify android layouts. +- The screenshot verification occurs on every pull request as part of the `tests.yml` workflow. + +## Setup + +- Install Git LFS through your package manager of choice (`brew install git-lfs` | `yay -S git-lfs`). +- Install the Git LFS hooks into the project. + +```bash +# with element-android as the current working directory +git lfs install --local +``` + +- If installed correctly, `git push` and `git pull` will now include LFS content. + +## Recording + +- `./gradlew recordScreenshots` +- Paparazzi will generate images in `${module}/src/test/snapshots`, which will need to be committed to the repository using Git LFS. + +## Verifying + +- `./gradlew verifyScreenshots` +- In the case of failure, Paparazzi will generate images in `${module}/out/failure`. The images will show the expected and actual screenshots along with a delta of the two images. + +## Contributing + +- When creating a test, the file (and class) name names must include `ScreenshotTest`, eg `ItemScreenshotTest`. +- After creating the new test, record and commit the newly rendered screens. +- `./tools/validate_lfs` can be ran to ensure everything is working correctly with Git LFS, the CI also runs this check. + +## Example + +```kotlin +class PaparazziExampleScreenshotTest { + + @get:Rule + val paparazzi = Paparazzi( + deviceConfig = PIXEL_3, + theme = "Theme.Vector.Light", + ) + + @Test + fun `example paparazzi test`() { + // Inflate the layout + val view = paparazzi.inflate(R.layout.item_radio) + + // Bind data to the view + view.findViewById(R.id.actionTitle).text = paparazzi.resources.getString(R.string.room_settings_all_messages) + view.findViewById(R.id.radioIcon).setImageResource(R.drawable.ic_radio_on) + + // Record the bound view + paparazzi.snapshot(view) + } +} +``` diff --git a/gradle.properties b/gradle.properties index 0e561faa8d..ded5a43e28 100644 --- a/gradle.properties +++ b/gradle.properties @@ -15,7 +15,7 @@ org.gradle.vfs.watch=true org.gradle.caching=true # Android Settings -android.enableJetifier=true +android.enableJetifier=false android.useAndroidX=true #Project Settings diff --git a/tools/validate_lfs.sh b/tools/validate_lfs.sh new file mode 100755 index 0000000000..ce121057b6 --- /dev/null +++ b/tools/validate_lfs.sh @@ -0,0 +1,29 @@ +#! /bin/bash + +# +# Copyright (c) 2022 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Based on https://cashapp.github.io/paparazzi/#git-lfs + +# Compare the output of `git ls-files ':(attr:filter=lfs)'` against `git lfs ls-files` +# If there's no diff we assume the files have been committed using git lfs +diff <(git ls-files ':(attr:filter=lfs)' | sort) <(git lfs ls-files -n | sort) >/dev/null + +ret=$? +if [[ $ret -ne 0 ]]; then + echo >&2 "Detected files committed without using Git LFS." + echo >&2 "Install git lfs (eg brew install git-lfs) and run 'git lfs install --local' within the root repository directory and re-commit your files." + exit 1 +fi diff --git a/vector/build.gradle b/vector/build.gradle index 3a923f1b5c..ff0d907212 100644 --- a/vector/build.gradle +++ b/vector/build.gradle @@ -24,6 +24,8 @@ project.android.buildTypes.all { buildType -> ] } +initScreenshotTests(project) + android { // Due to a bug introduced in Android gradle plugin 3.6.0, we have to specify the ndk version to use // Ref: https://issuetracker.google.com/issues/144111441