|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [OSSTEST PATCH 09/11] mg-repro-flight: Provide --rebuild to make variant build jobs too
This allows a single command to repro a particular job with a variety
of different source code.
The implementation technique is:
- run the build job in a separate flight, so that it can run
with a separate task which gives its host up after the build
- do much of the heavy lifting of runvar fiddling etc. in
a new helper routine in cs-adjust-flight
Signed-off-by: Ian Jackson <ian.jackson@xxxxxxxxxxxxx>
---
v2: Add a missing `continue' (without which everything goes quite wrong)
---
cs-adjust-flight | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
mg-repro-setup | 85 ++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 193 insertions(+), 1 deletion(-)
diff --git a/cs-adjust-flight b/cs-adjust-flight
index cc1684b4..5be5af38 100755
--- a/cs-adjust-flight
+++ b/cs-adjust-flight
@@ -469,6 +469,115 @@ END
$runvarq->execute($newjob, $dstflight, $oldjob);
})
}
+
+sub change__repro_buildjobs {
+ # helper for mg-repro-setup
+ # arguments to this change are
+ # EXAMPLE-FLIGHT EXAMPLE-JOB THING...
+ # where THING is
+ # .BUILDJOB =BUILDJOBRUNVAR +TREE=URL#REVISION
+ # (we eat all until it doesn't start with . = +)
+ # stdout output is
+ # list of runvar settings for repro job
+ die unless @changes >= 2;
+ my $eflight = shift @changes;
+ my $ejob = shift @changes;
+ my @refspecs;
+ my @varspecs;
+ my %treespecs;
+ while (@changes && $changes[0] =~ m/^[.=+]/) {
+ local $_ = shift @changes;
+ if (m/^[.=]/) {
+ push @refspecs, $_;
+ } elsif (m/^\+(.*?)\=(.*)\#(.*)$/) {
+ die $1 if $treespecs{$1};
+ $treespecs{$1} = [$2,$3];
+ } elsif (m/^-r/) {
+ push @varspecs, $_;
+ } else {
+ die "bad .BUILDJOB / =BUILDJOBRUNVAR / +TREE=URL#REVISION): $_\n";
+ }
+ }
+ my $testq = db_prepare(<<END);
+SELECT name, val FROM runvars WHERE flight=? AND job=? AND name like '%job';
+END
+ my $buildq_txt = <<END;
+SELECT name FROM runvars WHERE flight=? AND job=? AND ('f'
+END
+ my @buildq_xvars;
+ foreach my $t (sort keys %treespecs) {
+ $buildq_txt .= " OR name=?";
+ push @buildq_xvars, "tree_$t";
+ }
+ $buildq_txt .= ")";
+ my $buildq = db_prepare($buildq_txt);
+
+ my %bjobs;
+
+ $testq->execute($eflight, $ejob);
+ while (my ($refvar, $bjobref) = $testq->fetchrow_array()) {
+ my ($bflight, $bjob) = flight_otherjob($eflight,$bjobref);
+ if (@refspecs) {
+ my $y = 0;
+ foreach my $rs (@refspecs) {
+ if ($rs eq ".$bjob" || $rs eq "=$refvar") {
+ $y = 1;
+ last;
+ }
+ }
+ next unless $y;
+ }
+ $buildq->execute($bflight,$bjob,@buildq_xvars);
+ my %got;
+ while (my ($treevar) = $buildq->fetchrow_array()) {
+ $treevar =~ m/^tree_/ or die "$treevar ?";
+ $got{$'} = 1;
+ next unless $treespecs{$'};
+ $treespecs{$'}[2]++;
+ }
+ next unless %got || @refspecs;
+
+ $bjobs{$bjob} //= { Template => $bflight };
+ $bjobs{$bjob}{Template} eq $bflight or
+ die "inconsistent $bjob: $bjobs{$bjob}{Template} != $bflight";
+ push @{ $bjobs{$bjob}{Refs} }, $refvar;
+ $bjobs{$bjob}{Trees}{$_} = 1 foreach keys %got;
+ }
+
+ foreach my $tree (sort keys %treespecs) {
+ die "unused tree/revision adjustment $tree"
+ .(@refspecs
+ ? " (no tree_$tree var in any of "
+ .(join ' ', map { "$bjobs{$_}{Template}.$_" }
+ sort keys %bjobs).")"
+ : '')
+ unless $treespecs{$tree}[2];
+ }
+
+ my @copy_jobs_qs = copy_jobs_qs();
+ foreach my $bjob (sort keys %bjobs) {
+ copy_jobs_do(\@copy_jobs_qs, $bjobs{$bjob}{Template}, $bjob);
+ foreach my $tree (sort keys %{ $bjobs{$bjob}{Trees} }) {
+ runvar_set($bjob, "tree_$tree", $treespecs{$tree}[0]);
+ runvar_set($bjob, "revision_$tree", $treespecs{$tree}[1]);
+ }
+ foreach (@varspecs) {
+ if (m/^-r!|^-r^/) {
+ $runvar_rm_q->execute($dstflight, $bjob, $');
+ } elsif (m/^-r(.*?)=/) {
+ runvar_set($bjob, $1, $');
+ } else {
+ die "bad -r $_ ?";
+ }
+ }
+ foreach my $refvar (@{ $bjobs{$bjob}{Refs} }) {
+ die "$bjob $refvar $& ?" if
+ "$bjob.$refvar" =~ m{[^+=_./,:0-9a-z-]}i;
+ print "runvar-set . $refvar $dstflight.$bjob\n" or die $!;
+ }
+ }
+}
+
sub changes () {
debug("CHANGES...\n");
diff --git a/mg-repro-setup b/mg-repro-setup
index 2e1d3b88..7f075f4e 100755
--- a/mg-repro-setup
+++ b/mg-repro-setup
@@ -19,7 +19,7 @@
usage () { cat <<END
-./mg-repro-setup [OPTION...] EXAMPLE-FLIGHT JOB TESTID [HOSTSPEC...]
+./mg-repro-setup [OPTION...] EXAMPLE-FLIGHT JOB TESTID [REBUILD|HOSTSPEC...]
Creates a new flight containg a copy of JOB from EXAMPLE-FLIGHT
adjusted to use the same builds as JOB (ie, no rebuilds),
@@ -42,6 +42,49 @@ usage () { cat <<END
-E... -f... -P as for mg-execute-flight
--autoalloc-nofree allocate hosts as for production, but keep them
(specify no HOSTSPECS; remember to deallocate later)
+
+ REBUILD is
+ --rebuild [-B<blessing>] [-r...]
+ [.BUILDJOB | =BUILDJOBRUNVAR...]
+ +TREE=URL#REVISION...
+ Also use a different build. Specifically, use URL and REVISION
+ for certain builds which mention TREE and which are referenced
+ (directly) in the job JOB in EXAMPLE-FLIGHT. The affected builds
+ are those referenced by any of the runvars BUILDJOBRUNVAR (in
+ which case only that job reference is edited) or any whose job
+ name is any of the BUILDJOB (in which case all references to that
+ job are adjusted). If no .BUILDJOB and no =BUILDJOBRUNVAR are
+ specified, then all jobs referred to from the example JOB
+ which mention TREE are affected.
+
+ For example,
+ --rebuild =xenbuildjob \
+ +xen=git://xenbits.xen.org/people/alice/xen.git#fixes
+ would look for xenbuild and (say) find that it referred to
+ build-amd64; it would then copy and use the build-amd64 job
+ that EXAMPLE-FLIGHT.JOB used, adjusting \`tree_xen' and
+ \`revision_xen' as specified, and use that for \`xenbuildjob' in
+ the repro (but not for \`buildjob')
+
+ Whereas
+ --rebuild .build-amd64 \
+ +linux=git://xenbits.xen.org/people/alice/linux.git#fixes
+ would replace all references to any job named build-amd64
+ with a new build-amd64 job.
+
+ And
+ --rebuild \
+ +xen=git://xenbits.xen.org/people/alice/linux.git#fixes
+ would replace all builds mentioning xen, including for example
+ build-amd64 (for xenbuildjob) and build-i386 (for buildjob)
+ in an x86 32-on-64 flight.
+
+ Host allocation for the build is done "normally" (ie, the host
+ is thrown away after the build is complete), with a default
+ blessing of \`adhoc'.
+
+ --rebuild is affected by -E or -P but not general -B or -f.
+
END
}
@@ -122,10 +165,38 @@ progress "logging to $logfile"
savelog "$logfile"
exec 3>"$logfile"
+rebuilds_flight=''
+
while [ $# -ne 0 ]; do
arg=$1; shift
case "$arg" in
+
+ --rebuild)
+ rebuilds_blessing=adhoc
+ rebuild_specs=()
+ while true; do
+ case "$1" in
+ [.=+]*|-r) rebuild_specs+=("$1"); shift ;;
+ -B?*) rebuilds_blessing=${1#-B}; shift ;;
+ -*) badusage ': bad --rebuild option' ;;
+ *) break ;;
+ esac
+ done
+ if [ x$rebuilds_flight = x ]; then
+ rebuilds_flight=$(
+ ./cs-adjust-flight new:$rebuilds_blessing
+ )
+ fi
+
+ adjusts+=($(
+ ./cs-adjust-flight $rebuilds_flight \
+ repro-buildjobs $example_flight $job \
+ "${rebuild_specs[@]}"
+ ))
+ continue
+ ;;
+
none:)
# provided so we can repro a job with no hosts
;;
@@ -170,6 +241,18 @@ done
flight=$(./cs-adjust-flight new:$blessing)
progress "new flight is $flight"
+if [ "x$rebuilds_flight" != x ]; then
+ progress "running build(s) $rebuilds_flight"
+ ./mg-execute-flight -B$rebuilds_blessing --progress-fd=2 \
+ "${mgexecflags[@]}" $rebuilds_flight
+
+ mro=tmp/$rebuilds_flight.mro
+ if ! egrep '^perfect' >/dev/null $mro; then
+ echo >&2 "build(s) failed (no 'perfect' in $mro)"
+ exit 1
+ fi
+fi
+
OSSTEST_TASK=$(perl -e '
use Osstest;
use Osstest::Executive;
--
2.11.0
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |