[xen staging] automation: Rework "build-each-commit-gcc" test

[patchbot@xxxxxxx]

commit 21051291ad8851cac973c7ee70f5928967022c7e
Author:     Anthony PERARD <anthony.perard@xxxxxxxxxx>
AuthorDate: Tue Feb 20 14:07:33 2024 +0000
Commit:     Jan Beulich <jbeulich@xxxxxxxx>
CommitDate: Mon Feb 26 16:05:59 2024 +0100

    automation: Rework "build-each-commit-gcc" test
    
    Current issues with this test are:
    - when the job timeout, the log file is lost as there is no chance to
      run the `mv` command.
    - GitLab job log is limited in size, so one usually have to download
      the artifacts, which may be missing.
    
    Use $GITLAB_CI to detect when the script is run as part of a GitLab
    pipeline. GitLab will add "GITLAB_CI=true" in the environment
    variables.
    
    When run as part of $GITLAB_CI, ignore "dirty" worktree to allow to
    write "build-each-commit-gcc.log", which can then be grabbed as
    artifacts, even when the job timeout. The `git clean` command is
    changed to keep those build logs.
    
    When run as part of $GITLAB_CI, we will also store the build output in
    a log file instead of writing it to stdout, because GitLab's job log
    is limited. But we will write the log to stdout in case of error, so
    we can find out more quickly why there's been an error.
    
    This patch also make use of a GitLab feature, "log sections", which we
    will collapse by default. One section per commit been built.
    
    There's a bit of colour added to the logs.
    
    Signed-off-by: Anthony PERARD <anthony.perard@xxxxxxxxxx>
    Acked-by: Stefano Stabellini <sstabellini@xxxxxxxxxx>
---
 automation/gitlab-ci/build-each-commit.sh |  2 +-
 automation/gitlab-ci/test.yaml            |  4 +--
 automation/scripts/build-test.sh          | 55 +++++++++++++++++++++++++++----
 3 files changed, 50 insertions(+), 11 deletions(-)

diff --git a/automation/gitlab-ci/build-each-commit.sh 
b/automation/gitlab-ci/build-each-commit.sh
index 19e337b468..08fddecbbe 100755
--- a/automation/gitlab-ci/build-each-commit.sh
+++ b/automation/gitlab-ci/build-each-commit.sh
@@ -15,4 +15,4 @@ fi
 echo "Building ${BASE}..${TIP}"
 
 NON_SYMBOLIC_REF=1 ./automation/scripts/build-test.sh ${BASE} ${TIP} \
-    bash -c "git clean -ffdx && ./automation/scripts/build"
+    bash -c "git clean -ffdx -e '/build-*.log' && ./automation/scripts/build"
diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
index 6aabdb9d15..8b7b2e4da9 100644
--- a/automation/gitlab-ci/test.yaml
+++ b/automation/gitlab-ci/test.yaml
@@ -134,9 +134,7 @@ build-each-commit-gcc:
     XEN_TARGET_ARCH: x86_64
     CC: gcc
   script:
-    - BASE=${BASE_SHA:-${CI_COMMIT_BEFORE_SHA}} 
TIP=${TIP_SHA:-${CI_COMMIT_SHA}} ./automation/gitlab-ci/build-each-commit.sh 
2>&1 | tee ../build-each-commit-gcc.log
-  after_script:
-    - mv ../build-each-commit-gcc.log .
+    - BASE=${BASE_SHA:-${CI_COMMIT_BEFORE_SHA}} 
TIP=${TIP_SHA:-${CI_COMMIT_SHA}} ./automation/gitlab-ci/build-each-commit.sh 
2>&1 | tee build-each-commit-gcc.log
   artifacts:
     paths:
       - '*.log'
diff --git a/automation/scripts/build-test.sh b/automation/scripts/build-test.sh
index da643adc01..355d4bf7f6 100755
--- a/automation/scripts/build-test.sh
+++ b/automation/scripts/build-test.sh
@@ -9,6 +9,37 @@
 # Set NON_SYMBOLIC_REF=1 if you want to use this script in detached HEAD state.
 # This is currently used by automated test system.
 
+# Colors with ANSI escape sequences
+txt_info=''
+txt_err=''
+txt_clr=''
+
+# $GITLAB_CI should be "true" or "false".
+if [ "$GITLAB_CI" != true ]; then
+    GITLAB_CI=false
+fi
+
+gitlab_log_section() {
+    if $GITLAB_CI; then
+        echo -n "section_$1:$(date +%s):$2
"
+    fi
+    if [ $# -ge 3 ]; then
+        echo "$3"
+    fi
+}
+log_section_last=
+log_section_start() {
+    log_section_last="${1%\[collapsed=true\]}"
+    gitlab_log_section 'start' "$1" "${txt_info}$2${txt_clr}"
+}
+log_section_end() {
+    if [ "$log_section_last" ]; then
+        gitlab_log_section 'end' "$log_section_last"
+        log_section_last=
+    fi
+}
+
+
 if test $# -lt 2 ; then
     echo "Usage:"
     echo " $0 <BASE> <TIP> [CMD]"
@@ -19,10 +50,12 @@ fi
 
 pushd `git rev-parse --show-toplevel`
 
-status=`git status -s`
-if test -n "$status"; then
-    echo "Tree is dirty, aborted"
-    exit 1
+if ! $GITLAB_CI; then
+    status=`git status -s`
+    if test -n "$status"; then
+        echo "Tree is dirty, aborted"
+        exit 1
+    fi
 fi
 
 BASE=$1; shift
@@ -40,26 +73,34 @@ fi
 
 ret=1
 while read num rev; do
-    echo "Testing $num $rev"
+    log_section_start "commit_$rev[collapsed=true]" "Testing #$num $(git log 
-1 --abbrev=12 --format=tformat:'%h ("%s")' $rev)"
 
     git checkout $rev
     ret=$?
     if test $ret -ne 0; then
-        echo "Failed to checkout $num $rev with $ret"
+        log_section_end
+        echo "${txt_err}Failed to checkout $num $rev with $ret${txt_clr}"
         break
     fi
 
     if test $# -eq 0 ; then
         git clean -fdx && ./configure && make -j4
+    elif $GITLAB_CI; then
+        "$@" > "build-$num.log" 2>&1
     else
         "$@"
     fi
     ret=$?
     if test $ret -ne 0; then
-        echo "Failed at $num $rev with $ret"
+        if $GITLAB_CI; then
+            cat "build-$num.log"
+        fi
+        log_section_end
+        echo "${txt_err}Failed at $num $rev with $ret${txt_clr}"
         break
     fi
     echo
+    log_section_end
 done < <(git rev-list $BASE..$TIP | nl -ba | tac)
 
 echo "Restoring original HEAD"
--
generated by git-patchbot for /home/xen/git/xen.git#staging