GitHub Actions: Re-run only failed tests; warn about slow tests

.github/workflows/main.yml

@@ -51,10 +51,13 @@ jobs:
         name: uhabits-android
         path: uhabits-android/build/outputs/
 
-    - name: Install flock
+    - name: Install Linux utils
       run: |
         brew install util-linux
+        brew unlink parallel
+        brew install moreutils
         echo "/usr/local/opt/util-linux/bin" >> $GITHUB_PATH
+        echo "/usr/local/opt/moreutils/bin" >> $GITHUB_PATH
 
     - name: Run Android Tests
       run: ./build.sh android-tests ${{ matrix.api }}

build.sh

@@ -122,14 +122,25 @@ android_test() {
     $ADB install -r ${ANDROID_OUTPUTS_DIR}/apk/androidTest/debug/uhabits-android-debug-androidTest.apk || return 1
 
     for size in medium large; do
-        log_info "Running $size instrumented tests..."
         OUT_INSTRUMENT=${ANDROID_OUTPUTS_DIR}/instrument-${API}.txt
         OUT_LOGCAT=${ANDROID_OUTPUTS_DIR}/logcat-${API}.txt
+        FAILED_TESTS=""
+        for i in {1..5}; do
+            log_info "Running $size instrumented tests (attempt $i)..."
             $ADB shell am instrument \
-            -r -e coverage true -e size $size \
+                -r -e coverage true -e size "$size" $FAILED_TESTS \
                 -w ${PACKAGE_NAME}.test/androidx.test.runner.AndroidJUnitRunner \
-            | tee $OUT_INSTRUMENT
+                | ts "%.s" | tee "$OUT_INSTRUMENT"
-        if grep "\(INSTRUMENTATION_STATUS_CODE.*-1\|FAILURES\|ABORTED\|onError\|Error type\|crashed\)" $OUT_INSTRUMENT; then
+
+            FAILED_TESTS=$(tools/parseInstrument.py "$OUT_INSTRUMENT")
+            SUCCESS=$?
+            if [ $SUCCESS -eq 0 ]; then
+                log_info "$size tests passed."
+                break
+            fi
+        done
+
+        if [ $SUCCESS -ne 0 ]; then
             log_error "Some $size instrumented tests failed."
             log_error "Saving logcat: $OUT_LOGCAT..."
             $ADB logcat -d > $OUT_LOGCAT
@@ -138,7 +149,6 @@ android_test() {
             $ADB shell rm -r /sdcard/Android/data/${PACKAGE_NAME}/files/test-screenshots/
             return 1
         fi
-        log_info "$size tests passed."
     done
 
     return 0
@@ -276,12 +286,7 @@ main() {
                 _print_usage
                 exit 1
             fi
-            for attempt in {1..5}; do
+            android_test $1
-                log_info "Running Android tests (attempt $attempt)..."
-                android_test $1 && return 0
-            done
-            log_error "Maximum number of attempts reached. Failing."
-            return 1
             ;;
         android-tests-parallel)
             shift; _parse_opts "$@"

tools/parseInstrument.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""
+Android Instrumentation Test Parser
+
+Given a raw Android Instrumentation log (produced by "adb shell am instrument -r ...") this script
+return zero if all tests pass and non-zero if some tests fail. In case of failure, this script
+also prints arguments that, if passed to "am instrument", will cause it to re-run just the tests
+that failed. This script additionally prints warning about the tests on the STDERR; e.g. slow tests.
+"""
+import sys
+import re
+
+STATUS_START = 1
+STATUS_DISABLED = -3
+SLOW_TEST_THRESHOLD = 5.0
+
+COLOR_YELLOW = '\033[93m'
+COLOR_END = '\033[0m'
+
+def warn(msg):
+    sys.stderr.write("%s%s%s\n" % (COLOR_YELLOW, msg, COLOR_END))
+
+log_filename = sys.argv[1]
+current_class, current_method = None, None
+failed_tests = ""
+exit_code = 1
+
+for line in open(log_filename).readlines():
+    matches = re.findall('^([0-9.]*)', line)
+    current_time = float(matches[0])
+
+    matches = re.findall('INSTRUMENTATION_STATUS: class=(.*)', line)
+    if len(matches) > 0:
+        current_class = matches[0]
+
+    matches = re.findall('INSTRUMENTATION_STATUS: test=(.*)', line)
+    if len(matches) > 0:
+        current_method = matches[0]
+
+    matches = re.findall('OK \([0-9]* tests\)', line)
+    if len(matches) > 0:
+        exit_code = 0
+
+    matches = re.findall('INSTRUMENTATION_STATUS_CODE: ([-0-9]*)', line)
+    if len(matches) > 0:
+        status_code = int(matches[0])
+        if (status_code < 0) and (status_code != STATUS_DISABLED):
+            failed_tests += f"-e class {current_class}#{current_method} "
+        if status_code == STATUS_START:
+            initial_time = current_time
+        else:
+            elapsed_time = current_time - initial_time
+            if(elapsed_time > SLOW_TEST_THRESHOLD):
+                warn("SLOW_TEST %s#%s (%.2f seconds)" % (current_class, current_method, elapsed_time))
+
+if len(failed_tests) > 0:
+    print(failed_tests)
+sys.exit(exit_code)