[OSSTEST PATCH v2 13/17] cr-daily-branch: Immediately retry failing tests

[Ian Jackson <iwj@xxxxxxxxxxxxxx>]

From: Ian Jackson <ian.jackson@xxxxxxxxxxxxx>

We exclude the self-tests because we don't want to miss breakage, and
the Xen smoke tests because they will be run again RSN anyway.

Signed-off-by: Ian Jackson <iwj@xxxxxxxxxxxxxx>
---
 cr-daily-branch | 48 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/cr-daily-branch b/cr-daily-branch
index 285ea361..bea8734e 100755
--- a/cr-daily-branch
+++ b/cr-daily-branch
@@ -472,12 +472,58 @@ esac
 sgr_args+=" $EXTRA_SGR_ARGS"
 
 date >&2
+original_start=`date +%s`
+
 : $flight $branch $OSSTEST_BLESSING $sgr_args
 eval "$DAILY_BRANCH_PREEXEC_HOOK"
 execute_flight $flight $OSSTEST_BLESSING
 date >&2
 
-start_email $flight $branch "$sgr_args" "$subject_prefix"
+default_immediate_retry=$wantpush
+
+case "$branch" in
+*smoke*)       default_immediate_retry=false ;;
+osstest)       default_immediate_retry=false ;;
+*)             ;;
+esac
+
+: ${OSSTEST_IMMEDIATE_RETRY:-$default_immediate_retry}
+
+while true; do
+       start_email $flight $branch "$sgr_args" "$subject_prefix"
+       if grep '^tolerable$' $mrof >/dev/null 2>&1; then break; fi
+       if ! $OSSTEST_IMMEDIATE_RETRY; then break; fi
+       OSSTEST_IMMEDIATE_RETRY=false
+       retry_jobs=$(
+               perl <$mrof -wne '
+                       next unless m/^regression (\S+) /;
+                       my $j = $1;
+                       next if $j =~ m/^build/;
+                       $r{$j}++;
+                       END {
+                               print "copy-jobs '$flight' $_ "
+                                       foreach sort keys %r;
+                       }'
+       )
+       if [ "x$retry_jobs" = x ]; then break; fi
+
+       rflight=$(
+               ./cs-adjust-flight new:$OSSTEST_BLESSING \
+                       branch-set $branch \
+                       $retry_jobs
+       )
+
+       ./mg-adjust-flight-makexrefs -v $rflight \
+               --branch=$branch --revision-osstest=$narness_rev \
+               '^build-*' --debug --blessings=real
+
+       export OSSTEST_RESOURCE_WAITSTART=$original_start
+       execute_flight $rflight $OSSTEST_BLESSING-retest
+       report_flight $rflight
+       publish_logs $rflight
+
+       sgr_args+=" --refer-to-flight=$rflight"
+done
 
 push=false
 if grep '^tolerable$' $mrof >/dev/null 2>&1; then push=$wantpush; fi
-- 
2.20.1