diff options
| author | Paul Dagnelie <[email protected]> | 2021-12-01 09:38:53 -0800 |
|---|---|---|
| committer | Tony Hutter <[email protected]> | 2021-12-06 12:22:43 -0800 |
| commit | d34636151514829ed9bbd330bbe7dad79a1c817f (patch) | |
| tree | 863038fcf092d6183aa4ab6e0a960e7b9b233a90 /scripts | |
| parent | 12d27e71349bbbfe97686fa1e153e29d3355c926 (diff) | |
Add zfs-test facility to automatically rerun failing tests
This was a project proposed as part of the Quality theme for the
hackthon for the 2021 OpenZFS Developer Summit. The idea is to improve
the usability of the automated tests that get run when a PR is created
by having failing tests automatically rerun in order to make flaky
tests less impactful.
Reviewed-by: John Kennedy <[email protected]>
Reviewed-by: Tony Nguyen <[email protected]>
Signed-off-by: Paul Dagnelie <[email protected]>
Closes #12740
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/zfs-tests.sh | 38 |
1 files changed, 35 insertions, 3 deletions
diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh index ac2878858..60499e09e 100755 --- a/scripts/zfs-tests.sh +++ b/scripts/zfs-tests.sh @@ -21,6 +21,10 @@ # CDDL HEADER END # +# +# Copyright 2020 OmniOS Community Edition (OmniOSce) Association. +# + BASE_DIR=$(dirname "$0") SCRIPT_COMMON=common.sh if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then @@ -48,6 +52,7 @@ ITERATIONS=1 ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh" ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh" UNAME=$(uname -s) +RERUN="" # Override some defaults if on FreeBSD if [ "$UNAME" = "FreeBSD" ] ; then @@ -322,6 +327,7 @@ OPTIONS: -f Use files only, disables block device tests -S Enable stack tracer (negative performance impact) -c Only create and populate constrained path + -R Automatically rerun failing tests -n NFSFILE Use the nfsfile to determine the NFS configuration -I NUM Number of iterations -d DIR Use DIR for files and loopback devices @@ -348,7 +354,7 @@ $0 -x EOF } -while getopts 'hvqxkfScn:d:s:r:?t:T:u:I:' OPTION; do +while getopts 'hvqxkfScRn:d:s:r:?t:T:u:I:' OPTION; do case $OPTION in h) usage @@ -376,6 +382,9 @@ while getopts 'hvqxkfScn:d:s:r:?t:T:u:I:' OPTION; do constrain_path exit ;; + R) + RERUN="yes" + ;; n) nfsfile=$OPTARG [ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile" @@ -694,12 +703,35 @@ ${TEST_RUNNER} ${QUIET:+-q} \ -i "${STF_SUITE}" \ -I "${ITERATIONS}" \ 2>&1 | tee "$RESULTS_FILE" - # # Analyze the results. # -${ZTS_REPORT} "$RESULTS_FILE" >"$REPORT_FILE" +${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE" RESULT=$? + +if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then + MAYBES="$($ZTS_REPORT --list-maybes)" + TEMP_RESULTS_FILE=$(mktemp -u -t zts-results-tmp.XXXXX -p "$FILEDIR") + TEST_LIST=$(mktemp -u -t test-list.XXXXX -p "$FILEDIR") + grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE" + for test_name in $MAYBES; do + grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST" + done + ${TEST_RUNNER} ${QUIET:+-q} \ + -c "${RUNFILES}" \ + -T "${TAGS}" \ + -i "${STF_SUITE}" \ + -I "${ITERATIONS}" \ + -l "${TEST_LIST}" \ + 2>&1 | tee "$RESULTS_FILE" + # + # Analyze the results. + # + ${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE" + RESULT=$? +fi + + cat "$REPORT_FILE" RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE") |