diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f3858d848..929a216ed 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,10 +51,13 @@ jobs: name: uhabits-android path: uhabits-android/build/outputs/ - - name: Install flock + - name: Install Linux utils run: | brew install util-linux + brew unlink parallel + brew install moreutils echo "/usr/local/opt/util-linux/bin" >> $GITHUB_PATH + echo "/usr/local/opt/moreutils/bin" >> $GITHUB_PATH - name: Run Android Tests run: ./build.sh android-tests ${{ matrix.api }} diff --git a/build.sh b/build.sh index b865429de..4f17ec653 100755 --- a/build.sh +++ b/build.sh @@ -122,14 +122,25 @@ android_test() { $ADB install -r ${ANDROID_OUTPUTS_DIR}/apk/androidTest/debug/uhabits-android-debug-androidTest.apk || return 1 for size in medium large; do - log_info "Running $size instrumented tests..." OUT_INSTRUMENT=${ANDROID_OUTPUTS_DIR}/instrument-${API}.txt OUT_LOGCAT=${ANDROID_OUTPUTS_DIR}/logcat-${API}.txt - $ADB shell am instrument \ - -r -e coverage true -e size $size \ - -w ${PACKAGE_NAME}.test/androidx.test.runner.AndroidJUnitRunner \ - | tee $OUT_INSTRUMENT - if grep "\(INSTRUMENTATION_STATUS_CODE.*-1\|FAILURES\|ABORTED\|onError\|Error type\|crashed\)" $OUT_INSTRUMENT; then + FAILED_TESTS="" + for i in {1..5}; do + log_info "Running $size instrumented tests (attempt $i)..." + $ADB shell am instrument \ + -r -e coverage true -e size "$size" $FAILED_TESTS \ + -w ${PACKAGE_NAME}.test/androidx.test.runner.AndroidJUnitRunner \ + | ts "%.s" | tee "$OUT_INSTRUMENT" + + FAILED_TESTS=$(tools/parseInstrument.py "$OUT_INSTRUMENT") + SUCCESS=$? + if [ $SUCCESS -eq 0 ]; then + log_info "$size tests passed." + break + fi + done + + if [ $SUCCESS -ne 0 ]; then log_error "Some $size instrumented tests failed." log_error "Saving logcat: $OUT_LOGCAT..." $ADB logcat -d > $OUT_LOGCAT @@ -138,7 +149,6 @@ android_test() { $ADB shell rm -r /sdcard/Android/data/${PACKAGE_NAME}/files/test-screenshots/ return 1 fi - log_info "$size tests passed." done return 0 @@ -276,12 +286,7 @@ main() { _print_usage exit 1 fi - for attempt in {1..5}; do - log_info "Running Android tests (attempt $attempt)..." - android_test $1 && return 0 - done - log_error "Maximum number of attempts reached. Failing." - return 1 + android_test $1 ;; android-tests-parallel) shift; _parse_opts "$@" diff --git a/tools/parseInstrument.py b/tools/parseInstrument.py new file mode 100755 index 000000000..da424d1e4 --- /dev/null +++ b/tools/parseInstrument.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +""" +Android Instrumentation Test Parser + +Given a raw Android Instrumentation log (produced by "adb shell am instrument -r ...") this script +return zero if all tests pass and non-zero if some tests fail. In case of failure, this script +also prints arguments that, if passed to "am instrument", will cause it to re-run just the tests +that failed. This script additionally prints warning about the tests on the STDERR; e.g. slow tests. +""" +import sys +import re + +STATUS_START = 1 +STATUS_DISABLED = -3 +SLOW_TEST_THRESHOLD = 5.0 + +COLOR_YELLOW = '\033[93m' +COLOR_END = '\033[0m' + +def warn(msg): + sys.stderr.write("%s%s%s\n" % (COLOR_YELLOW, msg, COLOR_END)) + +log_filename = sys.argv[1] +current_class, current_method = None, None +failed_tests = "" +exit_code = 1 + +for line in open(log_filename).readlines(): + matches = re.findall('^([0-9.]*)', line) + current_time = float(matches[0]) + + matches = re.findall('INSTRUMENTATION_STATUS: class=(.*)', line) + if len(matches) > 0: + current_class = matches[0] + + matches = re.findall('INSTRUMENTATION_STATUS: test=(.*)', line) + if len(matches) > 0: + current_method = matches[0] + + matches = re.findall('OK \([0-9]* tests\)', line) + if len(matches) > 0: + exit_code = 0 + + matches = re.findall('INSTRUMENTATION_STATUS_CODE: ([-0-9]*)', line) + if len(matches) > 0: + status_code = int(matches[0]) + if (status_code < 0) and (status_code != STATUS_DISABLED): + failed_tests += f"-e class {current_class}#{current_method} " + if status_code == STATUS_START: + initial_time = current_time + else: + elapsed_time = current_time - initial_time + if(elapsed_time > SLOW_TEST_THRESHOLD): + warn("SLOW_TEST %s#%s (%.2f seconds)" % (current_class, current_method, elapsed_time)) + +if len(failed_tests) > 0: + print(failed_tests) +sys.exit(exit_code) \ No newline at end of file