[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [OSSTEST PATCH 9/9] mg-repro-flight: Provide --rebuild to make variant build jobs too



This allows a single command to repro a particular job with a variety
of different source code.

The implementation technique is:
  - run the build job in a separate flight, so that it can run
    with a separate task which gives its host up after the build
  - do much of the heavy lifting of runvar fiddling etc. in
    a new helper routine in cs-adjust-flight

Signed-off-by: Ian Jackson <ian.jackson@xxxxxxxxxxxxx>
---
 cs-adjust-flight | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 mg-repro-setup   |  84 +++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 192 insertions(+), 1 deletion(-)

diff --git a/cs-adjust-flight b/cs-adjust-flight
index cc1684b4..5be5af38 100755
--- a/cs-adjust-flight
+++ b/cs-adjust-flight
@@ -469,6 +469,115 @@ END
        $runvarq->execute($newjob, $dstflight, $oldjob);
     })
 }
+
+sub change__repro_buildjobs {
+    # helper for mg-repro-setup
+    # arguments to this change are
+    #    EXAMPLE-FLIGHT EXAMPLE-JOB THING...
+    # where THING is
+    #    .BUILDJOB =BUILDJOBRUNVAR +TREE=URL#REVISION
+    # (we eat all until it doesn't start with . = +)
+    # stdout output is
+    #    list of runvar settings for repro job
+    die unless @changes >= 2;
+    my $eflight = shift @changes;
+    my $ejob = shift @changes;
+    my @refspecs;
+    my @varspecs;
+    my %treespecs;
+    while (@changes && $changes[0] =~ m/^[.=+]/) {
+       local $_ = shift @changes;
+       if (m/^[.=]/) {
+           push @refspecs, $_;
+       } elsif (m/^\+(.*?)\=(.*)\#(.*)$/) {
+           die $1 if $treespecs{$1};
+           $treespecs{$1} = [$2,$3];
+       } elsif (m/^-r/) {
+           push @varspecs, $_;
+       } else {
+           die "bad .BUILDJOB / =BUILDJOBRUNVAR / +TREE=URL#REVISION): $_\n";
+       }
+    }
+    my $testq = db_prepare(<<END);
+SELECT name, val FROM runvars WHERE flight=? AND job=? AND name like '%job';
+END
+    my $buildq_txt = <<END;
+SELECT name FROM runvars WHERE flight=? AND job=? AND ('f'
+END
+    my @buildq_xvars;
+    foreach my $t (sort keys %treespecs) {
+       $buildq_txt .= " OR name=?";
+       push @buildq_xvars, "tree_$t";
+    }
+    $buildq_txt .= ")";
+    my $buildq = db_prepare($buildq_txt);
+
+    my %bjobs;
+
+    $testq->execute($eflight, $ejob);
+    while (my ($refvar, $bjobref) = $testq->fetchrow_array()) {
+       my ($bflight, $bjob) = flight_otherjob($eflight,$bjobref);
+       if (@refspecs) {
+           my $y = 0;
+           foreach my $rs (@refspecs) {
+               if ($rs eq ".$bjob" || $rs eq "=$refvar") {
+                   $y = 1;
+                   last;
+               }
+           }
+           next unless $y;
+       }
+       $buildq->execute($bflight,$bjob,@buildq_xvars);
+       my %got;
+       while (my ($treevar) = $buildq->fetchrow_array()) {
+           $treevar =~ m/^tree_/ or die "$treevar ?";
+           $got{$'} = 1;
+           next unless $treespecs{$'};
+           $treespecs{$'}[2]++;
+       }
+       next unless %got || @refspecs;
+
+       $bjobs{$bjob} //= { Template => $bflight };
+       $bjobs{$bjob}{Template} eq $bflight or
+           die "inconsistent $bjob: $bjobs{$bjob}{Template} != $bflight";
+       push @{ $bjobs{$bjob}{Refs} }, $refvar;
+       $bjobs{$bjob}{Trees}{$_} = 1 foreach keys %got;
+    }
+
+    foreach my $tree (sort keys %treespecs) {
+       die "unused tree/revision adjustment $tree"
+           .(@refspecs
+              ? " (no tree_$tree var in any of "
+              .(join ' ', map { "$bjobs{$_}{Template}.$_" }
+                sort keys %bjobs).")"
+              : '')
+            unless $treespecs{$tree}[2];
+    }
+
+    my @copy_jobs_qs = copy_jobs_qs();
+    foreach my $bjob (sort keys %bjobs) {
+       copy_jobs_do(\@copy_jobs_qs, $bjobs{$bjob}{Template}, $bjob);
+       foreach my $tree (sort keys %{ $bjobs{$bjob}{Trees} }) {
+           runvar_set($bjob, "tree_$tree",     $treespecs{$tree}[0]);
+           runvar_set($bjob, "revision_$tree", $treespecs{$tree}[1]);
+       }
+       foreach (@varspecs) {
+           if (m/^-r!|^-r^/) {
+               $runvar_rm_q->execute($dstflight, $bjob, $');
+           } elsif (m/^-r(.*?)=/) {
+               runvar_set($bjob, $1, $');
+           } else {
+               die "bad -r $_ ?";
+           }
+       }
+       foreach my $refvar (@{ $bjobs{$bjob}{Refs} }) {
+           die "$bjob $refvar $& ?" if
+                "$bjob.$refvar" =~ m{[^+=_./,:0-9a-z-]}i;
+           print "runvar-set . $refvar $dstflight.$bjob\n" or die $!;
+       }
+    }
+}
+
 sub changes () {
     debug("CHANGES...\n");
 
diff --git a/mg-repro-setup b/mg-repro-setup
index 2e1d3b88..abff2ef7 100755
--- a/mg-repro-setup
+++ b/mg-repro-setup
@@ -19,7 +19,7 @@
 
 usage () { cat <<END
 
-./mg-repro-setup [OPTION...] EXAMPLE-FLIGHT JOB TESTID [HOSTSPEC...]
+./mg-repro-setup [OPTION...] EXAMPLE-FLIGHT JOB TESTID [REBUILD|HOSTSPEC...]
 
  Creates a new flight containg a copy of JOB from EXAMPLE-FLIGHT
  adjusted to use the same builds as JOB (ie, no rebuilds),
@@ -42,6 +42,49 @@ usage () { cat <<END
    -E... -f... -P       as for mg-execute-flight
    --autoalloc-nofree   allocate hosts as for production, but keep them
                          (specify no HOSTSPECS; remember to deallocate later)
+
+ REBUILD is
+   --rebuild [-B<blessing>] [-r...]
+             [.BUILDJOB | =BUILDJOBRUNVAR...]
+             +TREE=URL#REVISION...
+       Also use a different build.  Specifically, use URL and REVISION
+       for certain builds which mention TREE and which are referenced 
+       (directly) in the job JOB in EXAMPLE-FLIGHT.  The affected builds
+       are those referenced by any of the runvars BUILDJOBRUNVAR (in
+       which case only that job reference is edited) or any whose job
+       name is any of the BUILDJOB (in which case all references to that
+       job are adjusted).  If no .BUILDJOB and no =BUILDJOBRUNVAR are
+       specified, then all jobs referred to from the example JOB
+       which mention TREE are affected.
+
+       For example,
+           --rebuild =xenbuildjob \
+                 +xen=git://xenbits.xen.org/people/alice/xen.git#fixes
+       would look for xenbuild and (say) find that it referred to
+       build-amd64; it would then copy and use the build-amd64 job
+       that EXAMPLE-FLIGHT.JOB used, adjusting \`tree_xen' and
+       \`revision_xen' as specified, and use that for \`xenbuildjob' in
+       the repro (but not for \`buildjob')
+
+       Whereas
+           --rebuild .build-amd64 \
+                 +linux=git://xenbits.xen.org/people/alice/linux.git#fixes
+       would replace all references to any job named build-amd64
+       with a new build-amd64 job.
+
+       And
+           --rebuild \
+                 +xen=git://xenbits.xen.org/people/alice/linux.git#fixes
+       would replace all builds mentioning xen, including for example
+       build-amd64 (for xenbuildjob) and build-i386 (for buildjob)
+       in an x86 32-on-64 flight.
+
+       Host allocation for the build is done "normally" (ie, the host
+       is thrown away after the build is complete), with a default
+       blessing of \`adhoc'.
+
+       --rebuild is affected by -E or -P but not general -B or -f.
+
 END
 
 }
@@ -122,10 +165,37 @@ progress "logging to $logfile"
 savelog "$logfile"
 exec 3>"$logfile"
 
+rebuilds_flight=''
+
 while [ $# -ne 0 ]; do
        arg=$1; shift
 
        case "$arg" in
+
+       --rebuild)
+               rebuilds_blessing=adhoc
+               rebuild_specs=()
+               while true; do
+                       case "$1" in
+                       [.=+]*|-r) rebuild_specs+=("$1");       shift ;;
+                       -B?*)   rebuilds_blessing=${1#-B};      shift ;;
+                       -*)     badusage ': bad --rebuild option'     ;;
+                        *)      break                                 ;;
+                       esac
+               done
+               if [ x$rebuilds_flight = x ]; then
+                       rebuilds_flight=$(
+                               ./cs-adjust-flight new:$rebuilds_blessing
+                       )
+               fi
+
+               adjusts+=($(
+                       ./cs-adjust-flight $rebuilds_flight \
+                               repro-buildjobs $example_flight $job \
+                                "${rebuild_specs[@]}"
+               ))
+               ;;
+
        none:)
                # provided so we can repro a job with no hosts
                ;;
@@ -170,6 +240,18 @@ done
 flight=$(./cs-adjust-flight new:$blessing)
 progress "new flight is $flight"
 
+if [ "x$rebuilds_flight" != x ]; then
+       progress "running build(s) $rebuilds_flight"
+       ./mg-execute-flight -B$rebuilds_blessing --progress-fd=2 \
+               "${mgexecflags[@]}" $rebuilds_flight
+
+       mro=tmp/$rebuilds_flight.mro
+       if ! egrep '^perfect' >/dev/null $mro; then
+               echo >&2 "build(s) failed (no 'perfect' in $mro)"
+               exit 1
+       fi
+fi
+
 OSSTEST_TASK=$(perl -e '
        use Osstest;
        use Osstest::Executive;
-- 
2.11.0


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel

 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.