Re: [Xen-devel] [PATCH RFC 1/2] scripts: Add script to do the repetitive bits of the release process

[George Dunlap <george.dunlap@xxxxxxxxxx>]

Ping?

On 4/5/19 6:13 PM, George Dunlap wrote:
> With this script, once the main checks are out of the way, doing a
> release (either an RC or the final release) should mostly be a matter
> of executing a sequence of 4 commands given by the `help` function in
> this script.
> 
> Signed-off-by: George Dunlap <george.dunlap@xxxxxxxxxx>
> ---
> There's one hard-coded "default" path in here that refers to my own
> directory structure.  If Ian finds these scripts useful, we should
> probably move that to a copy on mail.xenproject.org somewhere instead.
> 
> There are also lots of opportunities for this script to be improved,
> by (for instance) implementing programmatic checks for the various
> checks listed as 'manual' at the moment.
> 
> I plan to implement containerize-able tests for the first three steps
> (tag, make tarball, push tag), using "dummy" paths and gpg keys.  I've
> made revisions to tarball-cvs-checkin-and-post which I haven't had the
> opportunity to test yet; ideas for how to keep this "fresh" are
> welcome.
> 
> CC: Ian Jackson <ian.jackson@xxxxxxxxxx>
> ---
>  scripts/release | 450 ++++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 450 insertions(+)
>  create mode 100755 scripts/release
> 
> diff --git a/scripts/release b/scripts/release
> new file mode 100755
> index 0000000000..0442cd4ef9
> --- /dev/null
> +++ b/scripts/release
> @@ -0,0 +1,450 @@
> +#!/bin/bash
> +
> +###
> +# George's bash library core
> +###
> +
> +# arg-parse debug
> +_apd=false
> +
> +arg_parse_cmd=\
> +"local -a args;
> +local _a;
> +local _vn;
> +local _m;
> +local CLVL=\$((\$CLVL+1))
> +
> +_m=true;
> +
> +for _a in \"\$@\" ; do
> +    $_apd && echo \"Evaluating \${_a} [[ \"\${_a/=}\" = \"\${_a}\" ]]\";
> +    if \$_m && [[ \"\${_a/=}\" != \"\${_a}\" ]] ; then
> +        $_apd && echo Parameter;
> +        _vn=\${_a%%=*};
> +        eval \"local \$_vn\";
> +        eval \"\$_a\";
> +    elif \$_m && [[ \"\${_a}\" == \"--\" ]] ; then
> +        $_apd && echo Separator;
> +        _m=false;
> +    else
> +        $_apd && echo Argument;
> +        _m=false;
> +        args+=(\"\$_a\");
> +    fi;
> +done"
> +
> +arg_parse="eval $arg_parse_cmd"
> +
> +# Pass in either the current function name, or the name of the script
> +requireargs="eval _func=\"\$FUNCNAME\" ; eval [[ -n \\\"\$_func\\\" ]] || 
> _func=\$0 ; eval _require-args \$_func"
> +
> +function _require-args()
> +{
> +    local _arg
> +    local _args
> +
> +    _args=($@)
> +
> +    for _arg in ${_args[@]:1} ; do
> +     eval "[[ -n \"\${$_arg}\" ]] || fail \"${_args[0]}: Missing $_arg\""
> +    done
> +}
> +
> +function default()
> +{
> +    # l0: eval i="5"
> +    # l1: default_post="eval $1=\"$2\""
> +    # l3: eval "if [[ -z \"\$$1\" ]] ; then default_post=\"eval 
> \$1=\\\"$2\\\"\" ; fi"
> +    eval "if [[ -z \"\$$1\" ]] ; then default_post=\"eval local 
> \$1=\\\"$2\\\"\" ; else unset default_post ; fi"
> +}
> +
> +function fail()
> +{
> +   echo FATAL $@
> +   [[ -n "$fail_cleanup" ]] && $fail_cleanup
> +   exit 1
> +}
> +
> +function info()
> +{
> +   echo INFO $CLVL $@ 1>&2
> +}
> +
> +function error()
> +{
> +   echo ERROR $@ 1>&2
> +}
> +
> +function status()
> +{
> +   echo STATUS $CLVL $@ 1>&2
> +   return 0
> +}
> +
> +function report-result()
> +{
> +    if [[ -n "$var" ]] ; then
> +     eval "${var}=\"$1\""
> +    else
> +     if [[ -n "$1" ]] ; then
> +         echo "$1"
> +     else
> +         echo "(empty)"
> +     fi
> +    fi
> +}
> +
> +function cmdline()
> +{
> +    local cmd;
> +
> +    if [[ "$#" -eq "0" ]] ; then
> +     help
> +     exit 1
> +    fi
> +
> +    $arg_parse
> +    info Running "${args[0]}"
> +    "${args[0]}" "${args[@]:1}" || exit 1
> +
> +    if ! [[ -z "$RET" ]] ; then
> +     echo $RET
> +    fi
> +}
> +
> +###
> +# release-specific code
> +###
> +
> +# Global / meta variables:
> +#
> +# tdir: "root" directory to do tarball work.
> +# rdir: Directory where tarball & sig will be put (==$tdir/$v)
> +# rtgz: Base filename for tarball ($rdir/xen-$v.tar.gz)
> +#
> +# v: Full release version (e.g., 4.12.0-rc5, 4.10.3)
> +# x: Major+minor xen release version (e.g., 4.12, 4.10)
> +# p: point release (e.g., 0 in 4.12.0; 3 in 4.10.3)
> +# #r: Numbers-only release (e.g., 4.12.0, 4.10.3) # PROBABLY NOT NEEDED
> +# rc: -rcN
> +#
> +# s: branch name (e.g., master, stable-4.12, stable-4.10)
> +# t: Tag from a given release (e.g,. 4.12.0-rc5, RELEASE-4.10.3)
> +# isrc: Boolean indicating whether the version is an rc (e.g., true for 
> 4.12.0-rc5, false for 4.10.3)
> +
> +
> +
> +function xen-make-prefix-config() {
> +    $arg_parse
> +
> +    # TODO: Ping drall.uk.xensource.com to see if we can reach it?
> +    
> +    default cache_prefix "git://drall.uk.xensource.com:9419/" ; $default_post
> +
> +    perl -ne "if(/^([A-Z_]+_URL) \?= (git.*)/) { print \"\$1 ?= 
> ${cache_prefix}\$2\n\"; }" Config.mk >> .config || fail "Generating .config"
> +    cat .config
> +}
> +
> +function set-tdir() {
> +    if [[ -z "$tdir" || ! -e "$tdir" ]] ; then
> +     info "$tdir doesn't exist, using /tmp"
> +     tdir="/tmp"
> +    fi
> +}
> +
> +# Take `v` and generate the appropriate metavariables variables.
> +function parse-version() {
> +    $arg_parse
> +
> +    $requireargs v
> +
> +    if [[ -n "$x" && -n "$p" ]] ; then
> +     echo "Version already parsed"
> +     return
> +    fi
> +
> +    if [[ $v =~ ([0-9]+\.[0-9]+)\.([0-9])(-rc[0-9]) ]] ; then
> +     x=${BASH_REMATCH[1]}
> +     p=${BASH_REMATCH[2]}
> +     rc=${BASH_REMATCH[3]}
> +     isrc=true
> +    elif [[ $v =~ ([0-9]+\.[0-9]+)\.([0-9]) ]] ; then
> +     x=${BASH_REMATCH[1]}
> +     p=${BASH_REMATCH[2]}
> +     isrc=false
> +    else
> +     fail "Bad version"
> +    fi
> +
> +    if $isrc ; then
> +     t=$v
> +    else
> +     t=RELEASE-$v
> +    fi
> +}
> +
> +function check() {
> +    # TODO: Automate some of these
> +    info "Please perform manually: All XSAs  have been applied"
> +    info "Please perform manually: Check 
> http://logs.test-lab.xenproject.org/osstest/results/all-branch-statuses.txt";
> +    info "Please perform manually: Check version in README"
> +    info "Please perform manually: Check version in SUPPORT.md"
> +    info "Please perform manually: Tags for appropriate *_REVISION's in 
> Config.mk"
> +    info "Please perform manually: xen/Makefile:XEN_EXTRAVERSION set to 0"
> +    info "Please perform manually: tools/Rules.mk: debug ?= n"
> +    info "Please perform manually: xen/Kconfig.debug:config DEBUG should 
> default to `n`"
> +}
> +
> +# Usage:
> +#   tag v=[version you want to release] [c=commithash]
> +# eg.
> +#   tag v=4.12.0-rc6
> +# Other arguments:
> +#  key:  Name of key to sign the commit with
> +#  tdir: Name of top-level tarball directory
> +function tag() {
> +    $arg_parse
> +
> +    default key "23E3222C145F4475FA8060A783FE14C957E82BD9"; $default_post
> +
> +    $requireargs v
> +
> +    set-tdir
> +
> +    $requireargs tdir
> +
> +    parse-version
> +
> +    $requireargs t
> +
> +    git fetch origin
> +
> +    if [[ -n "$c" ]] ; then
> +     info "Checking out commit $c"
> +     git checkout $c || fail
> +    else
> +     local q
> +     git checkout stable-$x || fail "Couldn't check out stable branch"
> +     git merge || fail "Merge"
> +     git log -n 10
> +     read -p "Enter to continue, anything else to quit: " q
> +     [[ -z "$q" ]] || return
> +    fi
> +
> +    # FIXME: Add checks:
> +    # - Make sure Config.mk has tags, not hashes
> +    # - sonames?
> +    # - Appropriate version numbers in SUPPORT.md, xen/Makefile, &c
> +
> +    echo git tag -u "$key" -s -m "Xen $v" $t ; sleep 1
> +    git tag -u "$key" -s -m "Xen $v" $t || fail "Creating signed tag"
> +
> +    info "Release tagged.  Now run release make-tarball v=$v"
> +}
> +
> +function push-tag() {
> +    $arg_parse
> +
> +    $requireargs v
> +
> +    parse-version
> +
> +    git push origin $t || fail "Pushing tag"
> +    # FIXME: This is in the release checklist, but I'm not sure why
> +    # git push origin staging-$x || fail "Pushing tag commit"
> +
> +    info "Tag pushed.  Now run release tarball-cvs-checkin-and-post v=$v"
> +}
> +
> +function make-tarball-only()
> +{
> +    $arg_parse
> +
> +    $requireargs v tdir
> +
> +    parse-version
> +
> +    git fetch || fail "git fetch"
> +    
> +    git checkout $t || fail "Checking out tag $t"
> +
> +    git clean -ffdx
> +
> +    xen-make-prefix-config
> +
> +    ./configure || fail "Configuring"
> +    
> +    if $isrc ; then
> +     make src-tarball || fail "Making src-tarball"
> +    else
> +     make src-tarball-release || fail "Making src-tarball"
> +    fi
> +
> +    rm -rf $tdir/$v
> +
> +    mkdir -p $tdir/$v || fail "Couldn't make target directory"
> +
> +    cp dist/xen-$v.tar.gz $tdir/$v || fail "Couldn't copy tarball"
> +}
> +
> +function buildtest-tarball() {
> +    $arg_parse
> +
> +    default bdir "/tmp" ; $default_post
> +
> +    $requireargs tdir v
> +    
> +    cd $bdir || fail "cd $bdir"
> +
> +    rm -rf build-$v
> +    mkdir build-$v || fail "mkdir"
> +
> +    cd build-$v
> +
> +    tar xfz $tdir/$v/xen-$v.tar.gz || fail "Untar"
> +
> +    cd xen-$v || fail "cd"
> +
> +    xen-make-prefix-config
> +    info "Testing build (tail -f $bdir/build-$v/log.$v)..."
> +    (./configure && make -j4 && touch $tdir/$v/build-tested && echo OK) 2>&1 
> > ../log.$v
> +
> +    [[ -e $tdir/$v/build-tested ]] || fail "Build failed; log at 
> $bdir/build-$v/log.$v"
> +}
> +
> +function sign-tarball() {
> +    $arg_parse
> +
> +    $requireargs v
> +
> +    if [[ -z "$rtgz" ]] ; then
> +     set-tdir
> +     rtgz=$tdir/$v/xen-$v.tar.gz
> +    fi
> +
> +    default key "23E3222C145F4475FA8060A783FE14C957E82BD9" ; $default_post
> +
> +    if ! gpg --list-secret-keys | grep $key ; then
> +     info "Signature required; please run the following command with the 
> public key available"
> +     info " gpg --detach-sign -u 'xen tree' $rtgz"
> +     exit 0
> +    fi
> +
> +    gpg --detach-sign -u $key $rtgz || fail "Signing $rtgz"
> +}
> +
> +function tarball-checksig() {
> +    gpg --verify $rtgz.sig || fail "Signature failed"
> +}
> +
> +function make-tarball() {
> +    local rdir
> +    local rtgz
> +    
> +    $arg_parse
> +
> +    $requireargs v
> +
> +    set-tdir
> +
> +    $requireargs tdir
> +
> +    parse-version
> +    
> +    info "Using tag $t"
> +
> +    rdir=$tdir/$v
> +
> +    rtgz=$rdir/xen-$v.tar.gz
> +
> +    if [[ ! -e $rtgz ]] ; then
> +     info "$rtgz not present, generating"
> +     make-tarball-only
> +    fi
> +
> +    info "Tarball created"
> +
> +    if [[ ! -e $rdir/build-tested ]] ; then
> +     buildtest-tarball
> +    fi
> +
> +    info "Build tested"
> +
> +    if [[ ! -e $rtgz.sig ]] ; then
> +     sign-tarball
> +    else
> +     tarball-checksig
> +    fi
> +
> +    info "Tarball made, signed, and build-tested.  Now run release push-tag 
> v=$v"
> +}
> +
> +function tarball-cvs-checkin-and-post() {
> +    $arg_parse
> +
> +    $requireargs v
> +
> +    # TODO: This tree probably wants to be put somewhere on
> +    # mail.xenproject.org
> +    
> +    default cvsdir "/build/hg/push/xen.org/" ; $default_post
> +
> +    if [[ ! -e $cvsdir ]] ; then
> +     fail "$cvsdir does not exist"
> +    fi
> +
> +    if [[ -z "$rtgz" ]] ; then
> +     set-tdir
> +     rtgz=$tdir/$v/xen-$v.tar.gz
> +    fi
> +
> +    cd $cvsdir || fail "cd"
> +
> +    mkdir -p oss-xen/release/$v || fail "Creating directory in CVS"
> +
> +    cvs add -kb oss-xen/release/$v/ || fail "cvs add release directory"
> +
> +    cd oss-xen/release/$v || fail "cd"
> +
> +    cp $tdir/$v/xen-$v.tar.gz . || fail "Copying tarball"
> +    cp $tdir/$v/xen-$v.tar.gz.sig . || fail "Copying sig"
> +    
> +    cvs add -kb xen-$v.tar.gz || fail "cvs add tarball"
> +    cvs add -kb xen-$v.tar.gz.sig || fail "cvs add sig"
> +    
> +    cd ../../..
> +
> +    cvs ci -m $v || fail "cvs checkin"
> +
> +    ssh mail.xenproject.org "cd /data/downloads.xenproject.org/xen.org && 
> cvs -q up -d" || fail "Deploying tarball"
> +
> +    info "Tarball Uploaded.  Xen version $v released."
> +}
> +
> +function help() {
> +    cat <<EOF
> +General workflow:
> +
> +* Do a number of pre-release sanity checks
> +  release check v=4.12.0-rc5
> +
> +* Tag and sign a Xen commit
> +  release tag v=4.12.0-rc5
> +   or
> +  release tag v=4.12.0-rc5 c=07c181c
> +
> +* Create, test, and sign a release tarball
> +  release make-tarball v=4.12.0-rc5
> +
> +* Push tags
> +  release push-tag v=4.12.0-rc5
> +
> +* Publish tarball
> +  release tarball-cvs-checkin-and-post v=4.12.0-rc5
> +EOF
> +}
> +
> +###
> +# The actual command-line
> +###
> +cmdline "$@"
> 


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel