Merge pull request #5339 from vector-im/michaelk/integration_test_render_results

Render integration test results inline.

.github/workflows/integration_tests.yml

@@ -93,10 +93,7 @@ jobs:
       - name: Read Results [org.matrix.android.sdk.session]
         if: always()
         id: get-comment-body-session
-        run: |
+        run: python3 ./tools/ci/render_test_output.py session ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml
-          body="$(cat ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml | grep "<testsuite" | sed "s@.*tests=\(.*\)time=.*@\1@")"
-          echo "::set-output name=session::passed=$body"
-# package: org.matrix.android.sdk.account
       - name: Run integration tests for  Matrix SDK [org.matrix.android.sdk.account] API[${{ matrix.api-level }}]
         if: always()
         uses: reactivecircus/android-emulator-runner@v2
@@ -111,9 +108,7 @@ jobs:
       - name: Read Results [org.matrix.android.sdk.account]
         if: always()
         id: get-comment-body-account
-        run: |
+        run: python3 ./tools/ci/render_test_output.py account ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml
-          body="$(cat ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml | grep "<testsuite" | sed "s@.*tests=\(.*\)time=.*@\1@")"
-          echo "::set-output name=account::passed=$body"
 # package: org.matrix.android.sdk.internal
       - name: Run integration tests for Matrix SDK [org.matrix.android.sdk.internal] API[${{ matrix.api-level }}]
         if: always()
@@ -129,9 +124,7 @@ jobs:
       - name: Read Results [org.matrix.android.sdk.internal]
         if: always()
         id: get-comment-body-internal
-        run: |
+        run: python3 ./tools/ci/render_test_output.py internal ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml
-          body="$(cat ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml | grep "<testsuite" | sed "s@.*tests=\(.*\)time=.*@\1@")"
-          echo "::set-output name=internal::passed=$body"
 # package: org.matrix.android.sdk.ordering
       - name: Run integration tests for Matrix SDK [org.matrix.android.sdk.ordering] API[${{ matrix.api-level }}]
         if: always()
@@ -147,9 +140,7 @@ jobs:
       - name: Read Results [org.matrix.android.sdk.ordering]
         if: always()
         id: get-comment-body-ordering
-        run: |
+        run: python3 ./tools/ci/render_test_output.py ordering ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml
-          body="$(cat ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml | grep "<testsuite" | sed "s@.*tests=\(.*\)time=.*@\1@")"
-          echo "::set-output name=ordering::passed=$body"
 # package: class PermalinkParserTest
       - name: Run integration tests for Matrix SDK class [org.matrix.android.sdk.PermalinkParserTest] API[${{ matrix.api-level }}]
         if: always()
@@ -162,12 +153,11 @@ jobs:
           emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
           emulator-build: 7425822
           script: ./gradlew $CI_GRADLE_ARG_PROPERTIES  -Pandroid.testInstrumentationRunnerArguments.class='org.matrix.android.sdk.PermalinkParserTest' matrix-sdk-android:connectedDebugAndroidTest
-      - name: Read Results [org.matrix.android.sd.PermalinkParserTest]
+      - name: Read Results [org.matrix.android.sdk.PermalinkParserTest]
         if: always()
         id: get-comment-body-permalink
-        run: |
+        run: python3 ./tools/ci/render_test_output.py permalink ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml
-          body="$(cat ./matrix-sdk-android/build/outputs/androidTest-results/connected/*.xml | grep "<testsuite" | sed "s@.*tests=\(.*\)time=.*@\1@")"
+# package: class PermalinkParserTest
-          echo "::set-output name=permalink::passed=$body"
       - name: Find Comment
         if: always() && github.event_name == 'pull_request'
         uses: peter-evans/find-comment@v1

tools/ci/render_test_output.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+#
+# Renders a list of xml files taken as arguments into GHA-styled messages in groups.
+# Explicitly aims not to have any dependencies, to reduce installation load.
+# Should just be able to run in the context of your standard github runner.
+
+# Potentially rewrite as an independent action, use handlebars to template result
+import sys
+import xml.etree.ElementTree as ET
+suitename = sys.argv[1]
+xmlfiles = sys.argv[2:]
+
+print(f"Arguments: {sys.argv}") 
+
+for xmlfile in xmlfiles:
+    print(f"Handling: {xmlfile}") 
+    tree = ET.parse(xmlfile)
+    
+    root = tree.getroot()
+    name = root.attrib['name']
+    time = root.attrib['time']
+    tests = int(root.attrib['tests'])
+    skipped = int(root.attrib['skipped'])
+    errors = int(root.attrib['errors'])
+    failures = int(root.attrib['failures'])
+    success = tests - failures - errors - skipped
+    total = tests - skipped
+    print(f"::group::{name} {success}/{total} ({skipped} skipped) in {time}")
+    
+    for testcase in root:
+        if testcase.tag != "testcase":
+            continue
+        testname = testcase.attrib['classname']
+        message = testcase.attrib['name']
+        time = testcase.attrib['time']
+        child = testcase.find("failure")
+        if child is None:
+            print(f"{message} in {time}s")
+        else:
+            print(f"::error file={testname}::{message} in {time}s")
+            print(child.text)
+    body = f"passed={success} failures={failures} errors={errors} skipped={skipped}"
+    print(f"::set-output name={suitename}::={body}")
+    print("::endgroup::")
+