|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH OSSTEST] Add a weekly coverity flight
This primarily consists of ts-coverity-scan and make-coverity-flight
which constructs the sole job.
The most recently scanned revision is pushed to a new coverity-scanned
branch in the usual xen.git, tests are run on the master branch.
Move collectversions into Osstest::BuildSupport rather than
duplicating with ts-xen-build (nothing else is really duplicated)
For the cr-* integration we treat branch=coverity as a special case of
tree=xen. I didn't think tree=coverity made much sense, and would
probably reach tendrils into lots of other places (such as the
invocations of check_tested).
TODO: How to pick $c{CoverityEmail}, needs to be a real email which is
in coverity project (which security@xxxxxxx used here is not)
At the moment actually uploading is not implemented and is gated via a
runvar which is currently set to false. Therfore for now
ts-coverity-scan justs prints the curl arguments while we sort the
rest out.
Signed-off-by: Ian Campbell <ian.campbell@xxxxxxxxxx>
---
Could consider running on smoked? staging is probably a step too far.
Deployment notes:
- Put cov-analysis-linux64-7.7.0.4.tar.gz in the Images
directory.
- Populate $HOME/.xen-osstest/coverity-secret with the token
- Populate xen.git#coverity-scanned with an initial baseline, update
ap-fetch-version-old to refer to it instead of master.
---
Osstest/BuildSupport.pm | 12 ++++++
ap-fetch-version | 3 ++
ap-fetch-version-old | 4 ++
ap-print-url | 2 +-
ap-push | 4 ++
cr-daily-branch | 18 +++++++-
cri-common | 1 +
crontab | 1 +
make-coverity-flight | 47 +++++++++++++++++++++
production-config | 4 ++
sg-run-job | 5 +++
ts-coverity-scan | 108 ++++++++++++++++++++++++++++++++++++++++++++++++
ts-xen-build | 11 +----
ts-xen-build-prep | 2 +-
14 files changed, 209 insertions(+), 13 deletions(-)
create mode 100755 make-coverity-flight
create mode 100755 ts-coverity-scan
diff --git a/Osstest/BuildSupport.pm b/Osstest/BuildSupport.pm
index 933f6e1..a183546 100644
--- a/Osstest/BuildSupport.pm
+++ b/Osstest/BuildSupport.pm
@@ -42,6 +42,7 @@ BEGIN {
xendist
$xendist
+ collect_xen_built_versions
submodulefixup submodule_have submodule_find
@@ -84,6 +85,17 @@ sub xendist () {
($ho, 'xendist', '', $r{"buildjob"});
}
+sub collect_xen_built_versions () {
+ my $tools="$builddir/xen/tools";
+ my $extras="$builddir/xen/extras";
+ store_revision($ho, 'qemu', "$tools/ioemu-dir", 1);
+ store_revision($ho, 'qemu', "$tools/qemu-xen-traditional-dir", 1);
+ store_revision($ho, 'qemuu', "$tools/qemu-xen-dir", 1);
+ store_revision($ho, 'seabios', "$tools/firmware/seabios-dir", 1);
+ store_revision($ho, 'ovmf', "$tools/firmware/ovmf-dir", 1);
+ store_revision($ho, 'minios', "$extras/mini-os", 1);
+}
+
#----- submodules -----
sub submodulefixup ($$$$) {
diff --git a/ap-fetch-version b/ap-fetch-version
index a7b658b..1e48b45 100755
--- a/ap-fetch-version
+++ b/ap-fetch-version
@@ -53,6 +53,9 @@ xen-4.*-testing)
repo_tree_rev_fetch_git xen \
$TREE_XEN staging-$branchcore $LOCALREV_XEN
;;
+coverity)
+ repo_tree_rev_fetch_git xen $TREE_XEN master $LOCALREV_XEN
+ ;;
qemu-mainline)
repo_tree_rev_fetch_git $branch \
$TREE_QEMU_MAINLINE master $LOCALREV_QEMU_UPSTREAM
diff --git a/ap-fetch-version-old b/ap-fetch-version-old
index e2c6b3b..9d6190f 100755
--- a/ap-fetch-version-old
+++ b/ap-fetch-version-old
@@ -59,6 +59,10 @@ xen-4.*-testing)
repo_tree_rev_fetch_git xen \
$TREE_XEN stable-$branchcore $LOCALREV_XEN
;;
+coverity)
+ #XXX doesn't exist yet, use master for now repo_tree_rev_fetch_git xen
$TREE_XEN coverity-scanned $LOCALREV_XEN
+ repo_tree_rev_fetch_git xen $TREE_XEN master $LOCALREV_XEN
+ ;;
qemu-mainline)
repo_tree_rev_fetch_git $branch \
$BASE_TREE_QEMU_UPSTREAM upstream-tested $LOCALREV_QEMU_UPSTREAM
diff --git a/ap-print-url b/ap-print-url
index 4088852..6ca000d 100755
--- a/ap-print-url
+++ b/ap-print-url
@@ -31,7 +31,7 @@ if info_linux_tree "$branch"; then
fi
case "$branch" in
-xen-*)
+xen-*|coverity)
echo $TREE_XEN
;;
qemu-mainline)
diff --git a/ap-push b/ap-push
index 8def652..97510c3 100755
--- a/ap-push
+++ b/ap-push
@@ -68,6 +68,10 @@ xen-*-testing)
xenversion=${xenversion#xen-}
git push $TREE_XEN $revision:refs/heads/stable-$xenversion
;;
+coverity)
+ cd $repos/xen
+ git push $TREE_XEN $revision:refs/heads/coverity-scanned
+ ;;
qemu-mainline)
cd $repos/qemu-mainline
git push $TREE_QEMU_UPSTREAM $revision:refs/heads/upstream-tested
diff --git a/cr-daily-branch b/cr-daily-branch
index 364238c..9594e18 100755
--- a/cr-daily-branch
+++ b/cr-daily-branch
@@ -205,7 +205,22 @@ fi
case "$tree" in
xen)
realtree=$xenbranch
- NEW_REVISION=$REVISION_XEN
+
+ case $branch in
+ coverity)
+ if [ "x$TREE_COVERITY" = x ]; then
+ export TREE_COVERITY=$TREE_XEN
+ fi
+ if [ "x$REVISION_COVERITY" = x ]; then
+ determine_version REVISION_COVERITY coverity COVERITY
+ export REVISION_COVERITY
+ fi
+ NEW_REVISION=$REVISION_COVERITY
+ ;;
+ *)
+ NEW_REVISION=$REVISION_XEN
+ ;;
+ esac
;;
linux)
realtree=linux
@@ -259,6 +274,7 @@ fi
case $branch in
distros-*) makeflight=./make-distros-flight ;;
+coverity) makeflight=./make-coverity-flight ;;
*) makeflight=./make-flight ;;
esac
diff --git a/cri-common b/cri-common
index 6dfe8df..9f8bb0b 100644
--- a/cri-common
+++ b/cri-common
@@ -68,6 +68,7 @@ select_xenbranch () {
case "$branch" in
xen-unstable-smoke) tree=xen; xenbranch=$branch;
qemuubranch=qemu-upstream-unstable;;
xen-*) tree=xen; xenbranch=$branch ;;
+ coverity) tree=xen; xenbranch=xen-unstable ;;
qemu-mainline) tree=qemuu; xenbranch=xen-unstable;
qemuubranch=qemu-mainline;;
qemu-upstream-*) tree=qemuu;
xenbranch=xen-${branch#qemu-upstream-};;
linux) tree=linux; xenbranch=xen-unstable ;;
diff --git a/crontab b/crontab
index 09b8d14..b6ced25 100755
--- a/crontab
+++ b/crontab
@@ -8,6 +8,7 @@ MAILTO=ian.jackson@xxxxxxxxxx,ian.campbell@xxxxxxxxxxxxx
0 * * * * cd testing.git &&
BRANCHES=xen-unstable-smoke ./cr-for-branches branches -q "./cr-daily-branch
--real"
4-59/30 * * * * cd testing.git &&
./cr-for-branches branches -q "./cr-daily-branch --real"
18 9 * * 1,3,5 cd testing.git && BRANCHES=linux-next
./cr-for-branches branches -w "./cr-daily-branch --real"
+18 9 * * 7 cd testing.git && BRANCHES=coverity
./cr-for-branches branches -w "./cr-daily-branch --real"
18 4 * * * cd testing.git && BRANCHES='linux-linus
linux-mingo-tip-master linux-3.0 libvirt rumpuserxen' ./cr-for-branches
branches -w "./cr-daily-branch --real"
6-59/15 * * * * cd testing.git &&
EXTRA_BRANCHES='linux-linus linux-3.0 rumpuserxen libvirt' ./cr-for-branches
bisects -w "./cr-try-bisect --real"
#8-59/5 * * * * cd bisects/adhoc.git &&
with-lock-ex -q data-tree-lock bash -c "./cr-try-bisect-adhoc; exit $?"
diff --git a/make-coverity-flight b/make-coverity-flight
new file mode 100755
index 0000000..53d26b0
--- /dev/null
+++ b/make-coverity-flight
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2015 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+set -e -o posix
+
+branch=$1
+xenbranch=$2
+blessing=$3
+buildflight=$4
+
+flight=`./cs-flight-create $blessing $branch`
+
+. ./cri-common
+. ./ap-common
+. ./mfi-common
+
+defsuite=`getconfig DebianSuite`
+defguestsuite=`getconfig GuestDebianSuite`
+
+./cs-job-create $flight coverity-amd64 coverity \
+ tree_xen=$TREE_COVERITY \
+ revision_xen=$REVISION_COVERITY \
+ coverity_upload=false
+
+echo $flight
+
+# Local variables:
+# mode: sh
+# sh-basic-offset: 2
+# indent-tabs-mode: nil
+# End:
diff --git a/production-config b/production-config
index bb8d224..874cbba 100644
--- a/production-config
+++ b/production-config
@@ -100,6 +100,10 @@ TftpGrubVersion XXXX-XX-XX
XenUsePath /usr/groups/xencore/systems/bin/xenuse
XenUseUser osstest
+# Results might include potential vulnerabilities.
+CoverityEmail security@xxxxxxx
+CoverityTools cov-analysis-linux64-7.7.0.4.tar.gz
+
# We use the IP address because Citrix can't manage reliable nameservice
#DebianMirrorHost debian.uk.xensource.com
#DebianMirrorHost 10.80.16.196
diff --git a/sg-run-job b/sg-run-job
index 20ebb64..7e592dd 100755
--- a/sg-run-job
+++ b/sg-run-job
@@ -445,6 +445,11 @@ proc prepare-build-host {} {
run-ts . host-build-prep ts-xen-build-prep
}
+proc need-hosts/coverity {} { return BUILD }
+proc run-job/coverity {} {
+ run-ts . = ts-coverity-scan + host
+}
+
#---------- main program ----------
jobdb::set-flight
diff --git a/ts-coverity-scan b/ts-coverity-scan
new file mode 100755
index 0000000..f8c3a81
--- /dev/null
+++ b/ts-coverity-scan
@@ -0,0 +1,108 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2015 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+use strict qw(vars);
+use DBI;
+use Osstest;
+use File::Path;
+use POSIX;
+use Osstest::TestSupport;
+use Osstest::BuildSupport;
+
+# Require explicit opt in from flight construction
+my $coverity_upload = ($r{coverity_upload}//'false') =~ m/true/ ? 1 : 0;
+
+my $tokenfile = "$ENV{HOME}/.xen-osstest/coverity-secret";
+my $submit_url = "https://scan.coverity.com/builds?project=XenProject";;
+
+tsreadconfig();
+selectbuildhost(\@ARGV);
+# remaining arguments are passed as targets to "make"
+builddirsprops();
+
+sub checkout () {
+ prepbuilddirs();
+
+ build_clone($ho, 'xen', $builddir, 'xen');
+}
+
+sub covtools () {
+ target_putfile($ho, 100, "$c{Images}/$c{CoverityTools}",
"$builddir/covtools.tar.gz");
+ target_cmd($ho, <<END, 100);
+set -xe
+c=$builddir/covtools
+mkdir -p \$c
+cd \$c
+tar --strip-components=1 -xaf $builddir/covtools.tar.gz
+END
+}
+
+sub build () {
+ my $make = "make $makeflags";
+
+ # Pre build things we don't want coverity to scan, but which are
+ # normally built by some other command.
+ target_cmd_build($ho, 1000, $builddir, <<END);
+cd $builddir/xen
+./configure
+$make -C tools/firmware/etherboot all
+$make mini-os-dir
+END
+
+ # Now the stuff we want coverity to look at
+ target_cmd_build($ho, 9000, $builddir, <<END);
+cd $builddir/xen
+export PATH=$builddir/covtools/bin:\$PATH
+cov-build --dir cov-int $make -C extras/mini-os/
+cov-build --dir cov-int $make xen tools
+
+tar czvf xen-coverity.tgz cov-int
+END
+
+ built_stash_file($ho, $builddir,
+ "xen-coverity.tgz", "xen/xen-coverity.tgz", 0);
+}
+
+sub upload() {
+ my $xen_version = target_cmd_output($ho, <<END, 30);
+ cd $builddir/xen
+ make xenversion
+END
+
+ my @form_args;
+ push @form_args, "token=\@$tokenfile";
+ push @form_args, "email=$c{CoverityEmail}";
+ push @form_args, "file=\@$stash/build/xen-coverity.tgz";
+ push @form_args, "version=$xen_version";
+ push @form_args, "description=$r{tree_xen} $r{built_revision_xen}";
+
+ my @args = map { ("--form", $_) } @form_args;
+ push @args, $submit_url;
+
+ if ($coverity_upload) {
+ # TODO: spawn curl here
+ die "Cannot upload for real yet"
+ } else {
+ logm("Not uploading: curl args: ".(join " ", map { qq("$_") } @args));
+ }
+}
+
+checkout();
+covtools();
+build();
+collect_xen_built_versions();
+upload();
diff --git a/ts-xen-build b/ts-xen-build
index b02e737..5c7863d 100755
--- a/ts-xen-build
+++ b/ts-xen-build
@@ -138,15 +138,6 @@ END
}
}
-sub collectversions () {
- my $tools="$builddir/xen/tools";
- store_revision($ho, 'qemu', "$tools/ioemu-dir", 1);
- store_revision($ho, 'qemu', "$tools/qemu-xen-traditional-dir", 1);
- store_revision($ho, 'qemuu', "$tools/qemu-xen-dir", 1);
- store_revision($ho, 'seabios', "$tools/firmware/seabios-dir", 1);
- store_revision($ho, 'ovmf', "$tools/firmware/ovmf-dir", 1);
-}
-
sub divide () {
# Only move hv to xeninstall, so that we can have
# xenpolicy in tools tarball.
@@ -232,7 +223,7 @@ sub trapping ($) {
checkout();
trapping(\&build);
-trapping(\&collectversions);
+trapping(\&collect_xen_built_versions);
die "*** something failed:\n\n".(join "\n\n",@probs)."\n** something failed"
if @probs;
diff --git a/ts-xen-build-prep b/ts-xen-build-prep
index b35e91b..c2383db 100755
--- a/ts-xen-build-prep
+++ b/ts-xen-build-prep
@@ -206,7 +206,7 @@ sub prep () {
autoconf automake libtool xsltproc
libxml2-utils libxml2-dev
libdevmapper-dev w3c-dtd-xhtml libxml-xpath-perl
- ccache nasm checkpolicy ebtables);
+ ccache nasm checkpolicy ebtables curl);
if ($ho->{Suite} =~ m/wheezy|squeeze|lenny/) {
push(@packages, "libnl-dev");
--
2.6.1
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |