|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH 5/6] osstest: introduce a script to build a FreeBSD package repository
The building of the binary packages itself is done with the poudriere
tool, that given a set of port names generates a binary package
repository with the requested packages and all it's dependencies.
Add a packages build job in the FreeBSD flight. Note that the binary
packages generated are tied to the base system used to build them when
not using a FreeBSD stable branch, stable branches guarantee ABI
stability.
Fetching the ports tree that contain the Makefiles to build the ports
is slightly more complicated than what would expected, since the
FreeBSD base system doesn't contain a git client the fetching is done
form the svn repository using the svnlite tool from the base system.
This is required in order to assure that bootstrapping the binary
repository doesn't depend on any external tools not found in the base
system.
Signed-off-by: Roger Pau Monné <roger.pau@xxxxxxxxxx>
---
ap-common | 6 ++
ap-fetch-version | 19 +++--
cr-daily-branch | 57 +++++++++------
make-freebsd-flight | 8 ++-
sg-run-job | 9 ++-
ts-freebsd-build-packages | 145 ++++++++++++++++++++++++++++++++++++++
6 files changed, 215 insertions(+), 29 deletions(-)
create mode 100755 ts-freebsd-build-packages
diff --git a/ap-common b/ap-common
index 87df7948..8083b6c2 100644
--- a/ap-common
+++ b/ap-common
@@ -41,6 +41,11 @@
: ${PUSH_TREE_FREEBSD:=$XENBITS:/home/xen/git/freebsd.git}
: ${BASE_TREE_FREEBSD:=git://xenbits.xen.org/freebsd.git}
+: ${TREE_FREEBSD_PORTS:=git://github.com/freebsd/freebsd-ports.git}
+: ${TREE_FREEBSD_PORTS_SVN:=https://svn.FreeBSD.org/ports/head}
+: ${PUSH_TREE_FREEBSD_PORTS:=$XENBITS:/home/xen/git/freebsd-ports.git}
+: ${BASE_TREE_FREEBSD_PORTS:=git://xenbits.xen.org/freebsd-ports.git}
+
: ${TREE_LIBVIRT:=git://libvirt.org/libvirt.git}
: ${PUSH_TREE_LIBVIRT:=$XENBITS:/home/xen/git/libvirt.git}
: ${BASE_TREE_LIBVIRT:=git://xenbits.xen.org/libvirt.git}
@@ -88,6 +93,7 @@ fi
: ${LOCALREV_OVMF:=daily-cron.$branch}
: ${LOCALREV_XTF:=daily-cron.$branch}
: ${LOCALREV_FREEBSD:=daily-cron.$branch}
+: ${LOCALREV_FREEBSD_PORTS:=daily-cron.$branch}
: ${TREEBASE_LINUX_XCP:=http://hg.uk.xensource.com/carbon/trunk/linux-2.6.27}
diff --git a/ap-fetch-version b/ap-fetch-version
index 87725bf0..a0963cc5 100755
--- a/ap-fetch-version
+++ b/ap-fetch-version
@@ -108,11 +108,22 @@ ovmf)
;;
freebsd-*)
branchcore=${branch#freebsd-}
- if [ "x$branchcore" != "xmaster" ]; then
- branchcore="stable/$branchcore"
+ subbranch=${branchcore#*--}
+ branchcore=${branchcore%--"$subbranch"}
+
+ if [ "$subbranch" = "$branchcore" ]; then
+ if [ "$branchcore" != master ]; then
+ branchcore="stable/$branchcore"
+ fi
+ repo_tree_rev_fetch_git freebsd \
+ $TREE_FREEBSD $branchcore $LOCALREV_FREEBSD
+ else
+ if [ "$subbranch" != ports ]; then
+ exit 1
+ fi
+ repo_tree_rev_fetch_git freebsd-ports \
+ $TREE_FREEBSD_PORTS master $LOCALREV_FREEBSD_PORTS
fi
- repo_tree_rev_fetch_git freebsd \
- $TREE_FREEBSD $branchcore $LOCALREV_FREEBSD
;;
osstest)
if [ "x$OSSTEST_USE_HEAD" = "xy" ] ; then
diff --git a/cr-daily-branch b/cr-daily-branch
index 49b8ad8e..971f4c01 100755
--- a/cr-daily-branch
+++ b/cr-daily-branch
@@ -47,19 +47,28 @@ determine_version () {
local tversionvar=$1
local tbranch=$2
local treevarwhich=$3
- if [ "x$tbranch" = "x$branch" ] && [ "x$force_baseline" = x ]; then
- if [ "x$FORCE_REVISION" != x ]; then
- tversion="$FORCE_REVISION"
- else
- tversion=`$AP_FETCH_PFX ./ap-fetch-version "$tbranch"`
- fi
- determine_tree "$treevarwhich" "" _${treevarwhich}
- determine_tree "$treevarwhich" "" _${treevarwhich}_THIS
- else
+
+ case "$tbranch" in
+ "$branch"|"$branch"--*)
+ if [ "x$force_baseline" = x ]; then
+ if [ "x$FORCE_REVISION" != x ]; then
+ tversion="$FORCE_REVISION"
+ else
+ tversion=`$AP_FETCH_PFX ./ap-fetch-version
"$tbranch"`
+ fi
+ determine_tree "$treevarwhich" "" _${treevarwhich}
+ determine_tree "$treevarwhich" "" _${treevarwhich}_THIS
+ eval "$tversionvar=$tversion"
+ return
+ fi
+ ;& # fallthrough
+ *)
tversion=`$AP_FETCH_PFX ./ap-fetch-version-baseline "$tbranch"`
determine_tree "$treevarwhich" BASE_ _${treevarwhich}
determine_tree "$treevarwhich" BASE_ _${treevarwhich}_THIS
- fi
+ ;;
+ esac
+
eval "$tversionvar=$tversion"
}
@@ -235,18 +244,24 @@ if [ "x$REVISION_LINUXFIRMWARE" = x ]; then
determine_version REVISION_LINUXFIRMWARE linuxfirmware LINUXFIRMWARE
export REVISION_LINUXFIRMWARE
fi
-if [ "x$REVISION_FREEBSD" = x ]; then
- case "$branch" in
- freebsd-*)
- determine_version REVISION_FREEBSD "$branch" FREEBSD
- ;;
- *)
- determine_version REVISION_FREEBSD freebsd-master FREEBSD
- ;;
- esac
-
+case "$branch" in
+freebsd-*)
+ [ "x$REVISION_FREEBSD" = x ] && \
+ determine_version REVISION_FREEBSD "$branch" FREEBSD && \
export REVISION_FREEBSD
-fi
+ [ "x$REVISION_FREEBSD_PORTS" = x ] && \
+ determine_version REVISION_FREEBSD_PORTS "$branch"--ports FREEBSD_PORTS
&& \
+ export REVISION_FREEBSD_PORTS
+ ;;
+*)
+ [ "x$REVISION_FREEBSD" = x ] && \
+ determine_version REVISION_FREEBSD freebsd-master FREEBSD && \
+ export REVISION_FREEBSD
+ [ "x$REVISION_FREEBSD_PORTS" = x ] && \
+ determine_version REVISION_FREEBSD_PORTS freebsd-master--ports
FREEBSD_PORTS && \
+ export REVISION_FREEBSD_PORTS
+ ;;
+esac
case "$tree" in
xen)
diff --git a/make-freebsd-flight b/make-freebsd-flight
index d3c413b5..fc3d2d83 100755
--- a/make-freebsd-flight
+++ b/make-freebsd-flight
@@ -38,13 +38,15 @@ job_create_build_filter_callback () {
for arch in "$arches"; do
set_freebsd_runvars
-
create_freebsd_build_job build-$arch-freebsd
- # Create an identical job that's going to use the build output from
- # the previous one.
+ # Create a job to build the packages against the new world.
freebsd_runvars="$freebsd_runvars freebsdbuildjob=build-$arch-freebsd \
recipe_testinstall=true"
+ create_freebsd_pkg_build_job build-$arch-freebsd-packages
+
+ # Create an identical job that's going to use the build output from
+ # the previous one.
create_freebsd_build_job build-$arch-freebsd-again
# Create a Xen build job that's going to use the output from the first
diff --git a/sg-run-job b/sg-run-job
index 7d27f415..37123980 100755
--- a/sg-run-job
+++ b/sg-run-job
@@ -716,6 +716,7 @@ proc need-hosts/build-rumprun {} { return
BUILD_LINUX }
proc need-hosts/build-xtf {} { return BUILD_LINUX }
proc need-hosts/build-freebsd {} { return BUILD_FREEBSD }
proc need-hosts/build-xen-freebsd {} { return BUILD_FREEBSD }
+proc need-hosts/build-pkg-freebsd {} { return BUILD_FREEBSD }
proc run-job/build {} {
run-ts . = ts-xen-build
@@ -746,6 +747,10 @@ proc run-job/build-freebsd {} {
run-ts . = ts-freebsd-build
}
+proc run-job/build-pkg-freebsd {} {
+ run-ts . = ts-freebsd-build-packages
+}
+
proc run-job/build-xen-freebsd {} {
run-ts . = ts-xen-build-freebsd + host
}
@@ -768,7 +773,9 @@ proc prepare-build-host-freebsd {} {
global jobinfo
if {[recipe-flag testinstall]} { set broken fail } { set broken broken }
run-ts $broken host-install(*) ts-freebsd-host-install
- run-ts . host-build-prep ts-build-prep-freebsd
+ if {![recipe-flag skipbuildprep]} {
+ run-ts . host-build-prep ts-build-prep-freebsd
+ }
}
proc need-hosts/coverity {} { return BUILD_LINUX }
diff --git a/ts-freebsd-build-packages b/ts-freebsd-build-packages
new file mode 100755
index 00000000..9202dd9f
--- /dev/null
+++ b/ts-freebsd-build-packages
@@ -0,0 +1,145 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2019 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# This script will generate a package repository for FreeBSD, as
+# described in the handbook:
+# https://www.freebsd.org/doc/handbook/ports-poudriere.html
+
+# Consumes the following input runvars:
+#
+# svnrevision_freebsdports: ports svn revision id to use.
+# svntree_freebsdports ports svn tree to fetch the source code from.
+#
+# Produces: a binary package repository to consume.
+#
+# Sets the following runvar:
+#
+# path_freebsdpackagesdist: points to the folder where the package tree is
+# stored.
+
+use strict qw(vars);
+use DBI;
+use POSIX;
+use URI;
+
+unshift @INC, qw(.);
+use Osstest;
+use Osstest::TestSupport;
+use Osstest::BuildSupport;
+
+tsreadconfig();
+
+selectbuildhost(\@ARGV);
+builddirsprops();
+
+sub checkout () {
+ my $u = URI->new($c{HttpProxy});
+ my $host = $u->host;
+ my $port = $u->port;
+ prepbuilddirs();
+
+ logm("Checkout ports tree from svn");
+ target_cmd_build($ho, 4000, $builddir, <<END);
+cd $builddir
+rm -rf ports
+# svn ignores HTTP_PROXY envvar
+svnlite checkout --config-option servers:global:http-proxy-host=$host \\
+ --config-option servers:global:http-proxy-port=$port \\
+ --trust-server-cert \\
+ $r{"svntree_freebsdports"} \\
+ -r $r{"svnrevision_freebsdports"} ports
+END
+}
+
+sub install_port ($$) {
+ my ($port, $time) = @_;
+ my $name = $port;
+
+ $name =~ s/\//-/;
+ buildcmd_stamped_logged_root($time, "ports/$port", "$name-install",
+ '', 'make install', '');
+}
+
+sub config_poudriere() {
+ target_cmd_root($ho, <<END, 30);
+set -ex
+mkdir -p $builddir/poudriere
+# Create the directory for the distfiles
+mkdir -p /usr/ports/distfiles
+cat << ENDP >> /usr/local/etc/poudriere.conf
+# Use the ZFS pool created by the installer
+ZPOOL=zroot
+# Setup the http proxy for the builder jail
+export HTTP_PROXY=$c{HttpProxy}
+# Use parallel make jobs
+ALLOW_MAKE_JOBS=yes
+# Store build results in builddir
+# NB: use realpath to workaround a bug in poudriere versions < 3.3.0:
+# https://github.com/freebsd/poudriere/issues/654
+POUDRIERE_DATA=`realpath $builddir/poudriere`
+ENDP
+# Register the local ports tree
+poudriere ports -c -m none -M $builddir/ports -p local
+END
+}
+
+sub create_jail() {
+ my $src_prefix = $r{"freebsd_distpath"} ||
+ get_stashed("path_freebsddist", $r{"freebsdbuildjob"});
+ my $dst_prefix = "/root/sets";
+
+ target_cmd_root($ho, "mkdir -p $dst_prefix", 30);
+ target_putfile_root($ho, 600, "$src_prefix/base.txz",
+ "$dst_prefix/base.txz");
+
+ # NB: create the jail using the same build as the host
+ target_cmd_root($ho, <<END, 1200);
+poudriere jail -c -j builder -v `uname -r` -m tar=$dst_prefix/base.txz
+END
+}
+
+sub build() {
+ my @packages = qw(devel/git devel/glib20 devel/pkgconf devel/yajl
+ devel/gmake x11/pixman textproc/markdown
+ devel/gettext lang/python devel/argp-standalone
+ archivers/lzo2 lang/gcc devel/binutils);
+
+ target_putfilecontents_root_stash($ho, 30, join("\n", @packages),
+ '~/package.list');
+
+ target_cmd_root($ho, <<END, 14400);
+poudriere bulk -j builder -p local -f ~/package.list
+END
+}
+
+checkout();
+
+logm("Build and install poudriere");
+install_port("ports-mgmt/poudriere", 3600);
+config_poudriere();
+
+logm("Create build jail");
+create_jail();
+
+logm("Build packages");
+build();
+
+logm("Stashing build output");
+built_stash_dir($ho, $builddir, "freebsdpackagesdist",
+ "poudriere/packages/builder-local");
+
+logm("FreeBSD packages built successful");
--
2.17.2 (Apple Git-113)
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |