337 lines
14 KiB
YAML
337 lines
14 KiB
YAML
name: Nightly Tests
|
|
|
|
on:
|
|
push:
|
|
branches: [ release/* ]
|
|
schedule:
|
|
# At 20:00 every day UTC
|
|
- cron: '0 20 * * *'
|
|
workflow_dispatch:
|
|
|
|
# Enrich gradle.properties for CI/CD
|
|
env:
|
|
CI_GRADLE_ARG_PROPERTIES: >
|
|
-Porg.gradle.jvmargs=-Xmx4g
|
|
-Porg.gradle.parallel=false
|
|
jobs:
|
|
# Run Android Tests
|
|
integration-tests:
|
|
name: Matrix SDK - Running Integration Tests
|
|
runs-on: macos-latest
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
api-level: [ 28 ]
|
|
# No concurrency required, runs every time on a schedule.
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- uses: gradle/wrapper-validation-action@v1
|
|
- uses: actions/setup-java@v2
|
|
with:
|
|
distribution: 'adopt'
|
|
java-version: 11
|
|
- name: Set up Python 3.8
|
|
uses: actions/setup-python@v3
|
|
with:
|
|
python-version: 3.8
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: |
|
|
~/.gradle/caches
|
|
~/.gradle/wrapper
|
|
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-gradle-
|
|
- name: Start synapse server
|
|
uses: michaelkaye/setup-matrix-synapse@v0.3.0
|
|
with:
|
|
uploadLogs: true
|
|
httpPort: 8080
|
|
disableRateLimiting: true
|
|
# package: org.matrix.android.sdk.session
|
|
- name: Run integration tests for Matrix SDK [org.matrix.android.sdk.session] API[${{ matrix.api-level }}]
|
|
uses: reactivecircus/android-emulator-runner@v2
|
|
with:
|
|
api-level: ${{ matrix.api-level }}
|
|
arch: x86
|
|
profile: Nexus 5X
|
|
force-avd-creation: false
|
|
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
|
|
emulator-build: 7425822
|
|
script: |
|
|
adb root
|
|
adb logcat -c
|
|
touch emulator-session.log
|
|
chmod 777 emulator-session.log
|
|
adb logcat >> emulator-session.log &
|
|
./gradlew $CI_GRADLE_ARG_PROPERTIES -Pandroid.testInstrumentationRunnerArguments.package='org.matrix.android.sdk.session' matrix-sdk-android:connectedDebugAndroidTest
|
|
- name: Read Results [org.matrix.android.sdk.session]
|
|
if: always()
|
|
id: get-comment-body-session
|
|
run: python3 ./tools/ci/render_test_output.py session ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml
|
|
- name: Remove adb logcat
|
|
if: always()
|
|
run: pkill -9 adb
|
|
- name: Run integration tests for Matrix SDK [org.matrix.android.sdk.account] API[${{ matrix.api-level }}]
|
|
if: always()
|
|
uses: reactivecircus/android-emulator-runner@v2
|
|
with:
|
|
api-level: ${{ matrix.api-level }}
|
|
arch: x86
|
|
profile: Nexus 5X
|
|
force-avd-creation: false
|
|
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
|
|
emulator-build: 7425822
|
|
script: |
|
|
adb root
|
|
adb logcat -c
|
|
touch emulator-account.log
|
|
chmod 777 emulator-account.log
|
|
adb logcat >> emulator-account.log &
|
|
./gradlew $CI_GRADLE_ARG_PROPERTIES -Pandroid.testInstrumentationRunnerArguments.package='org.matrix.android.sdk.account' matrix-sdk-android:connectedDebugAndroidTest
|
|
- name: Read Results [org.matrix.android.sdk.account]
|
|
if: always()
|
|
id: get-comment-body-account
|
|
run: python3 ./tools/ci/render_test_output.py account ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml
|
|
- name: Remove adb logcat
|
|
if: always()
|
|
run: pkill -9 adb
|
|
# package: org.matrix.android.sdk.internal
|
|
- name: Run integration tests for Matrix SDK [org.matrix.android.sdk.internal] API[${{ matrix.api-level }}]
|
|
if: always()
|
|
uses: reactivecircus/android-emulator-runner@v2
|
|
with:
|
|
api-level: ${{ matrix.api-level }}
|
|
arch: x86
|
|
profile: Nexus 5X
|
|
force-avd-creation: false
|
|
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
|
|
emulator-build: 7425822
|
|
script: |
|
|
adb root
|
|
adb logcat -c
|
|
touch emulator-internal.log
|
|
chmod 777 emulator-internal.log
|
|
adb logcat >> emulator-internal.log &
|
|
./gradlew $CI_GRADLE_ARG_PROPERTIES -Pandroid.testInstrumentationRunnerArguments.package='org.matrix.android.sdk.internal' matrix-sdk-android:connectedDebugAndroidTest
|
|
- name: Read Results [org.matrix.android.sdk.internal]
|
|
if: always()
|
|
id: get-comment-body-internal
|
|
run: python3 ./tools/ci/render_test_output.py internal ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml
|
|
- name: Remove adb logcat
|
|
if: always()
|
|
run: pkill -9 adb
|
|
# package: org.matrix.android.sdk.ordering
|
|
- name: Run integration tests for Matrix SDK [org.matrix.android.sdk.ordering] API[${{ matrix.api-level }}]
|
|
if: always()
|
|
uses: reactivecircus/android-emulator-runner@v2
|
|
with:
|
|
api-level: ${{ matrix.api-level }}
|
|
arch: x86
|
|
profile: Nexus 5X
|
|
force-avd-creation: false
|
|
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
|
|
emulator-build: 7425822
|
|
script: |
|
|
adb root
|
|
adb logcat -c
|
|
touch emulator-ordering.log
|
|
chmod 777 emulator-ordering.log
|
|
adb logcat >> emulator-ordering.log &
|
|
./gradlew $CI_GRADLE_ARG_PROPERTIES -Pandroid.testInstrumentationRunnerArguments.package='org.matrix.android.sdk.ordering' matrix-sdk-android:connectedDebugAndroidTest
|
|
- name: Read Results [org.matrix.android.sdk.ordering]
|
|
if: always()
|
|
id: get-comment-body-ordering
|
|
run: python3 ./tools/ci/render_test_output.py ordering ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml
|
|
- name: Remove adb logcat
|
|
if: always()
|
|
run: pkill -9 adb
|
|
# package: class PermalinkParserTest
|
|
- name: Run integration tests for Matrix SDK class [org.matrix.android.sdk.PermalinkParserTest] API[${{ matrix.api-level }}]
|
|
if: always()
|
|
uses: reactivecircus/android-emulator-runner@v2
|
|
with:
|
|
api-level: ${{ matrix.api-level }}
|
|
arch: x86
|
|
profile: Nexus 5X
|
|
force-avd-creation: false
|
|
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
|
|
emulator-build: 7425822
|
|
script: |
|
|
adb root
|
|
adb logcat -c
|
|
touch emulator-permalink.log
|
|
chmod 777 emulator-permalink.log
|
|
adb logcat >> emulator-permalink.log &
|
|
./gradlew $CI_GRADLE_ARG_PROPERTIES -Pandroid.testInstrumentationRunnerArguments.class='org.matrix.android.sdk.PermalinkParserTest' matrix-sdk-android:connectedDebugAndroidTest
|
|
- name: Read Results [org.matrix.android.sdk.PermalinkParserTest]
|
|
if: always()
|
|
id: get-comment-body-permalink
|
|
run: python3 ./tools/ci/render_test_output.py permalink ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml
|
|
- name: Remove adb logcat
|
|
if: always()
|
|
run: pkill -9 adb
|
|
# package: class PermalinkParserTest
|
|
- name: Find Comment
|
|
if: always() && github.event_name == 'pull_request'
|
|
uses: peter-evans/find-comment@v1
|
|
id: fc
|
|
with:
|
|
issue-number: ${{ github.event.pull_request.number }}
|
|
comment-author: 'github-actions[bot]'
|
|
body-includes: Integration Tests Results
|
|
- name: Publish results to PR
|
|
if: always() && github.event_name == 'pull_request'
|
|
uses: peter-evans/create-or-update-comment@v1
|
|
with:
|
|
comment-id: ${{ steps.fc.outputs.comment-id }}
|
|
issue-number: ${{ github.event.pull_request.number }}
|
|
body: |
|
|
### Matrix SDK
|
|
## Integration Tests Results:
|
|
- `[org.matrix.android.sdk.session]`<br>${{ steps.get-comment-body-session.outputs.session }}
|
|
- `[org.matrix.android.sdk.account]`<br>${{ steps.get-comment-body-account.outputs.account }}
|
|
- `[org.matrix.android.sdk.internal]`<br>${{ steps.get-comment-body-internal.outputs.internal }}
|
|
- `[org.matrix.android.sdk.ordering]`<br>${{ steps.get-comment-body-ordering.outputs.ordering }}
|
|
- `[org.matrix.android.sdk.PermalinkParserTest]`<br>${{ steps.get-comment-body-permalink.outputs.permalink }}
|
|
edit-mode: replace
|
|
- name: Upload Test Report Log
|
|
uses: actions/upload-artifact@v2
|
|
if: always()
|
|
with:
|
|
name: integrationtest-error-results
|
|
path: |
|
|
emulator-permalink.log
|
|
emulator-internal.log
|
|
emulator-ordering.log
|
|
emulator-account.log
|
|
emulator-session.log
|
|
|
|
ui-tests:
|
|
name: UI Tests (Synapse)
|
|
runs-on: macos-latest
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
api-level: [ 28 ]
|
|
# No concurrency required, runs every time on a schedule.
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- name: Set up Python 3.8
|
|
uses: actions/setup-python@v3
|
|
with:
|
|
python-version: 3.8
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: |
|
|
~/.gradle/caches
|
|
~/.gradle/wrapper
|
|
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-gradle-
|
|
- name: Start synapse server
|
|
uses: michaelkaye/setup-matrix-synapse@v0.3.0
|
|
with:
|
|
uploadLogs: true
|
|
httpPort: 8080
|
|
disableRateLimiting: true
|
|
- uses: actions/setup-java@v2
|
|
with:
|
|
distribution: 'adopt'
|
|
java-version: '11'
|
|
- name: Run sanity tests on API ${{ matrix.api-level }}
|
|
uses: reactivecircus/android-emulator-runner@v2
|
|
with:
|
|
api-level: ${{ matrix.api-level }}
|
|
arch: x86
|
|
profile: Nexus 5X
|
|
force-avd-creation: false
|
|
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
|
|
emulator-build: 7425822 # workaround to emulator bug: https://github.com/ReactiveCircus/android-emulator-runner/issues/160
|
|
script: |
|
|
adb root
|
|
adb logcat -c
|
|
touch emulator.log
|
|
chmod 777 emulator.log
|
|
adb logcat >> emulator.log &
|
|
./gradlew $CI_GRADLE_ARG_PROPERTIES connectedGplayDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.class=im.vector.app.ui.UiAllScreensSanityTest || (adb pull storage/emulated/0/Pictures/failure_screenshots && exit 1 )
|
|
- name: Upload Test Report Log
|
|
uses: actions/upload-artifact@v2
|
|
if: always()
|
|
with:
|
|
name: uitest-error-results
|
|
path: |
|
|
emulator.log
|
|
failure_screenshots/
|
|
|
|
codecov-units:
|
|
runs-on: macos-latest
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- uses: actions/setup-java@v2
|
|
with:
|
|
distribution: 'adopt'
|
|
java-version: '11'
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: |
|
|
~/.gradle/caches
|
|
~/.gradle/wrapper
|
|
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-gradle-
|
|
- run: ./gradlew allCodeCoverageReport $CI_GRADLE_ARG_PROPERTIES
|
|
- name: Upload Codecov data
|
|
uses: actions/upload-artifact@v2
|
|
if: always()
|
|
with:
|
|
name: codecov-xml
|
|
path: |
|
|
build/reports/jacoco/allCodeCoverageReport/allCodeCoverageReport.xml
|
|
|
|
sonarqube:
|
|
runs-on: macos-latest
|
|
if: always()
|
|
needs:
|
|
- codecov-units
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- uses: actions/setup-java@v2
|
|
with:
|
|
distribution: 'adopt'
|
|
java-version: '11'
|
|
- uses: actions/cache@v2
|
|
with:
|
|
path: |
|
|
~/.gradle/caches
|
|
~/.gradle/wrapper
|
|
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-gradle-
|
|
- uses: actions/download-artifact@v3
|
|
with:
|
|
name: codecov-xml # will restore to allCodeCoverageReport.xml by default; we restore to the same location in following tasks
|
|
- run: mkdir -p build/reports/jacoco/allCodeCoverageReport/
|
|
- run: mv allCodeCoverageReport.xml build/reports/jacoco/allCodeCoverageReport/
|
|
- run: ./gradlew sonarqube $CI_GRADLE_ARG_PROPERTIES
|
|
env:
|
|
ORG_GRADLE_PROJECT_SONAR_LOGIN: ${{ secrets.SONAR_TOKEN }}
|
|
|
|
# Notify the channel about scheduled runs, do not notify for manually triggered runs
|
|
notify:
|
|
runs-on: ubuntu-latest
|
|
needs:
|
|
- integration-tests
|
|
- ui-tests
|
|
- sonarqube
|
|
if: always() && github.event_name != 'workflow_dispatch'
|
|
# No concurrency required, runs every time on a schedule.
|
|
steps:
|
|
- uses: michaelkaye/matrix-hookshot-action@v0.3.0
|
|
with:
|
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
matrix_access_token: ${{ secrets.ELEMENT_ANDROID_NOTIFICATION_ACCESS_TOKEN }}
|
|
matrix_room_id: ${{ secrets.ELEMENT_ANDROID_INTERNAL_ROOM_ID }}
|
|
text_template: "Nightly test run: {{#each job_statuses }}{{#with this }}{{#if completed }} {{name}} {{conclusion}} at {{completed_at}}, {{/if}}{{/with}}{{/each}}"
|
|
html_template: "Nightly test run results: {{#each job_statuses }}{{#with this }}{{#if completed }}<br />{{name}} {{conclusion}} at {{completed_at}} <a href=\"{{html_url}}\">[details]</a>{{/if}}{{/with}}{{/each}}"
|