[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH][XM-TEST][1/4] Improvements to the test driver


  • To: Xen Developers <xen-devel@xxxxxxxxxxxxxxxxxxx>
  • From: Dan Smith <danms@xxxxxxxxxx>
  • Date: Wed, 09 Nov 2005 12:29:12 -0800
  • Delivery-date: Wed, 09 Nov 2005 20:29:22 +0000
  • List-id: Xen developer discussion <xen-devel.lists.xensource.com>

Modularize runtest.sh and add a quick option.
I broke up runtest.sh into functions which makes it easier to alter its
behavior.  Now, we can easily re-submit a previously-run test report.

Signed-off-by: Dan Smith <danms@xxxxxxxxxx>
# HG changeset patch
# User danms@xxxxxxxxxx
# Node ID e7aaeeb8c28da6dc07a5907e9557e104024fb00e
# Parent  136b2d20dc81db51924aee54c8ec4ce6232defa2
Modularize runtest.sh and add a quick option.
I broke up runtest.sh into functions which makes it easier to alter its
behavior.  Now, we can easily re-submit a previously-run test report.

diff -r 136b2d20dc81 -r e7aaeeb8c28d tools/xm-test/runtest.sh
--- a/tools/xm-test/runtest.sh  Wed Nov  9 15:08:37 2005
+++ b/tools/xm-test/runtest.sh  Wed Nov  9 20:03:03 2005
@@ -1,215 +1,259 @@
 #!/bin/sh
 
-usage() {
-    echo "Usage: $0 [opts] <logfile>"
-    echo "  Where opts are:"
-    echo "  -d         : do not submit a report for this run"
-    echo "  -b         : do not ask any questions (batch mode)"
-    echo "  -e <email> : set email address for report"
-}
 ##
 ## Test driver script
 ##
 
-#
-# Defaults
-#
-MAXFAIL=10
-report=yes
-batch=no
-
-#
-# Resolve options
-#
-while [ $# -gt 0 ]
-do
-    case "$1" in
-    -d)
-        echo "(Skipping report submission)"
-        report=no
-        ;;
-    -b)
-        echo "(Batch mode)"
-        batch=yes
-        ;;
-    -e)
-        shift
-        echo $1 > contact_info
-        echo "(Email set to $1)"
-        ;;
-    *)
-        LOGFILE=$1
-        break
-        ;;
-    esac
-    shift
-done
-
-#
-# Usage
-#
-if [ -z $LOGFILE ]; then
-       usage
-       exit 1
-fi
-
-#
-# Output files
-#
-OSREPORTTEMP=${LOGFILE}.os.xml
-PROGREPORTTEMP=${LOGFILE}.prog.xml
-RESULTREPORTTEMP=${LOGFILE}.result.xml
-OUTPUT=${LOGFILE}.output
-SUMMARY=${LOGFILE}.summary
-PASSFAIL=${LOGFILE}.passfail
-REPORT=${LOGFILE}.report
-FAILURES=${LOGFILE}.failures
-               
-#
-# Make sure we're root
-#
-uid=$(id -u)
-if [ $uid != 0 ]; then 
-    echo "ERROR: I must be run as root!"
-    exit 1
-fi
-
-#
-# See if the ramdisk has been built
-#
-rdsize=$(stat -c %s ramdisk/initrd.img 2>/dev/null)
-if [ -z "$rdsize" ] || [ $rdsize -le 16384 ]; then
-    echo "Cannot find a valid ramdisk.  You need to run \"make\" or"
-    echo "copy in a previously-built ramdisk to the ramdisk/ directory"
-    exit 1
-fi
-
-#
-# See if xend is running
-#
-if ! xm list >/dev/null 2>&1; then
-    echo "'xm list' failed: is xend running?"
-    exit 1
-fi
-
-#
-#  Make sure permissions are correct
-#
-chmod a+x lib/XmTestReport/*
-chmod a+x mkreport mergereport
-
-#
+usage() {
+    echo "Usage: $0 [opts] <report>"
+    echo "  Where report is a name that will be used for report files"
+    echo ""
+    echo "  Where opts are:"
+    echo "  -d          : do not submit a report for this run"
+    echo "  -b          : do not ask any questions (batch mode)"
+    echo "  -q          : run a quick test set"
+    echo "  -e <email>  : set email address for report"
+    echo "  -s <report> : just submit report <report>"
+}
+
+# Just submit the report
+submit_report() {
+
+    reportfile=$1
+
+    ./lib/XmTestReport/Report.py $reportfile
+}
+
+# Generate XML result report from output file
+make_result_report() {
+    output=$1
+    reportfile=$2
+    if ! ./lib/XmTestReport/ResultReport.py $output > $reportfile; then
+       echo "Unable to generate clean ResultReport"
+       echo "Take a look at $report"
+       exit 1
+    fi
+}
+
+# Collect environment information for XML report
+make_environment_report() {
+    os=$1
+    prog=$2
+    if ! ./lib/XmTestReport/OSReport.py > $os; then
+       echo "Unable to generate clean OSReport"
+       echo "Take a look at $os"
+       exit 1
+    fi
+    if ! ./lib/XmTestReport/ProgReport.py > $prog; then
+       echo "Unable to generate clean ProgReport"
+       echo "Take a look at $prog"
+       exit 1
+    fi
+}
+
+# Check conditions needed to actually run the tests
+runnable_tests() {     
+    # Make sure we're root
+    uid=$(id -u)
+    if [ $uid != 0 ]; then 
+       echo "ERROR: I must be run as root!"
+       exit 1
+    fi
+
+    # See if the ramdisk has been built
+    rdsize=$(stat -c %s ramdisk/initrd.img 2>/dev/null)
+    if [ -z "$rdsize" ] || [ $rdsize -le 16384 ]; then
+       echo "Cannot find a valid ramdisk.  You need to run \"make\" or"
+       echo "copy in a previously-built ramdisk to the ramdisk/ directory"
+       exit 1
+    fi
+
+    # See if xend is running
+    if ! xm list >/dev/null 2>&1; then
+       echo "'xm list' failed: is xend running?"
+       exit 1
+    fi
+    
+}
+
 # Get contact info if needed
-#
-if [ ! -f contact_info ]; then
-    if [ "$batch" = "yes" ]; then
-       echo "Unable to read contact_info!"
-       echo "Please run me once interactively before using batch mode!"
-       exit 1
-    else
-       echo "Please provide your email address so that we can "
-       echo "contact you if we need further information concerning"
-       echo "your results.  Any information provided will be"
-       echo "kept private.  If you wish to remain anonymous, please"
-       echo "hit [ENTER] now."
-       
-       while ! echo "$EMAIL" | grep -q '@'; do
-           echo
-           echo -n "Your email address: "
-           read EMAIL
-           if [ -z $EMAIL ]; then
-               EMAIL="anonymous@xxxxxxxxxxxxx"
-           fi
-       done
-       echo $EMAIL > contact_info
-    fi
-fi
-
-#
-# Collect environment information for XML report
-#
-if ! ./lib/XmTestReport/OSReport.py > $OSREPORTTEMP; then
-    echo "Unable to generate clean OSReport"
-    echo "Take a look at $OSREPORTTEMP"
-    exit 1
-fi
-if ! ./lib/XmTestReport/ProgReport.py > $PROGREPORTTEMP; then
-    echo "Unable to generate clean ProgReport"
-    echo "Take a look at $PROGREPORTTEMP"
-    exit 1
-fi
-
-#
+get_contact_info() {
+    
+    if [ ! -f contact_info ]; then
+       if [ "$batch" = "yes" ]; then
+           echo "Unable to read contact_info!"
+           echo "Please run me once interactively before using batch mode!"
+           exit 1
+       else
+           echo "Please provide your email address so that we can "
+           echo "contact you if we need further information concerning"
+           echo "your results.  Any information provided will be"
+           echo "kept private.  If you wish to remain anonymous, please"
+           echo "hit [ENTER] now."
+           
+           while ! echo "$EMAIL" | grep -q '@'; do
+               echo
+               echo -n "Your email address: "
+               read EMAIL
+               if [ -z $EMAIL ]; then
+                   EMAIL="anonymous@xxxxxxxxxxxxx"
+               fi
+           done
+           echo $EMAIL > contact_info
+       fi
+    fi
+}
+
 # Run the tests
-#
-export TEST_VERBOSE=1
-echo Running tests...
-make -k check > $OUTPUT 2>&1
-
-#
+run_tests() {
+    output=$1
+    echo Running tests...
+    TEST_VERBOSE=1 make -k check > $output 2>&1
+}
+
+run_tests_quick() {
+
+    output=$1
+
+    create_tests="01_create_basic_pos.test 07_create_mem64_pos.test 
10_create_fastdestroy.test 14_create_blockroot_pos.test"
+    unpause_tests="01_unpause_basic_pos.test"
+    memset_tests="01_memset_basic_pos.test 03_memset_random_pos.test"
+    help_tests="06_help_allcmds.test"
+    testgroups="create unpause memset help"
+
+    echo "*** Quick test" > $output
+    for group in $testgroups; do
+       eval $(echo list=\$${group}_tests)
+       echo "*** Running tests [$list] from $group"
+       (cd tests/$group && TEST_VERBOSE=1 make -k check TESTS="$list") >> 
$output 2>&1
+    done
+
+}
+
 # Generate some plain-text reports
-#
-echo "Making PASS/FAIL report ($PASSFAIL)..."
-cat $OUTPUT | egrep '(REASON|PASS|FAIL|XPASS|XFAIL|SKIP)' | perl -pe 
's/^(PASS|FAIL|XPASS|XFAIL)(.+)$/$1$2\n/' > $PASSFAIL
-
-echo "Making FAIL report ($FAILURES)..."
-cat $PASSFAIL | egrep '(REASON|FAIL)' > $FAILURES
-
-NUMPASS=`grep -c PASS $OUTPUT`
-NUMFAIL=`grep -c FAIL $OUTPUT`
-NUMXPASS=`grep -c XPASS $OUTPUT`
-NUMXFAIL=`grep -c XFAIL $OUTPUT`
-cat > $SUMMARY << EOF
+make_text_reports() {
+    passfail=$1
+    failures=$2
+    output=$3
+    reportfile=$4
+    summary=summary.tmp
+    echo "Making PASS/FAIL report ($passfail)..."
+    cat $OUTPUT | egrep '(REASON|PASS|FAIL|XPASS|XFAIL|SKIP)' | perl -pe 
's/^(PASS|FAIL|XPASS|XFAIL)(.+)$/$1$2\n/' > $passfail
+    
+    echo "Making FAIL report ($failures)..."
+    cat $passfail | egrep '(REASON|FAIL)' > $failures
+    
+    NUMPASS=`grep -c PASS $output`
+    NUMFAIL=`grep -c FAIL $output`
+    NUMXPASS=`grep -c XPASS $output`
+    NUMXFAIL=`grep -c XFAIL $output`
+    cat > $summary << EOF
 Xm-test execution summary:
   PASS:  $NUMPASS
   FAIL:  $NUMFAIL
   XPASS: $NUMXPASS
   XFAIL: $NUMXFAIL
 EOF
-
-cat $SUMMARY > $REPORT
-
-echo -e '\n\nDetails:\n' >> $REPORT
- 
-./mkreport $PASSFAIL >> $REPORT
-
-#
-# Check to see if it's worth reporting these results
-#
-#if  [ "$batch"  =   "no" ] && 
-#    [ "$report" =   "yes" ] && 
-#    [ $NUMFAIL  -gt $MAXFAIL ]; then
-#    echo "NOTE: $NUMFAIL tests failed, which may be erroneous.  It may"
-#    echo "be a good idea to review the report before sending.  If you"
-#    echo "choose not to submit the report, it will be saved for your review"
-#    echo "and later submission."
-#    echo
-#    echo -n "Submit anyway? [y/n] "
-#    read ANSWER
-#    if [ "$ANSWER" = "n" ]; then
-#      report=no
-#    fi
-#fi
-
-#
-# Generate the XML result report
-#
-if ! ./lib/XmTestReport/ResultReport.py $OUTPUT > $RESULTREPORTTEMP; then
-    echo "Unable to generate clean ResultReport"
-    echo "Take a look at $RESULTREPORTTEMP"
-    exit 1
-fi
-
-#
-# Maybe submit report and save the combined XML file
-#
-if [ "$report" = "yes" ]; then
-    echo "Sending report..."
-    ./lib/XmTestReport/Report.py -D $OSREPORTTEMP $PROGREPORTTEMP \
-                                    $RESULTREPORTTEMP > $1.xml
-    echo "Report also saved in $1.xml"
-else
-    echo "Saving report to $1.xml..."
-    ./lib/XmTestReport/Report.py -d $OSREPORTTEMP $PROGREPORTTEMP \
-                                    $RESULTREPORTTEMP > $1.xml
-fi
+    
+    cat $summary > $reportfile
+    
+    echo -e '\n\nDetails:\n' >> $reportfile
+    
+    ./mkreport $passfail >> $reportfile
+
+    rm $summary
+}
+
+############
+### Main ###
+############
+
+# Defaults
+MAXFAIL=10
+report=yes
+batch=no
+run=yes
+
+# Resolve options
+while [ $# -gt 0 ]
+  do
+  case "$1" in
+      -d)
+         echo "(Skipping report submission)"
+         report=no
+         ;;
+      -b)
+         echo "(Batch mode)"
+         batch=yes
+         ;;
+      -e)
+         shift
+         echo $1 > contact_info
+         echo "(Email set to $1)"
+         ;;
+      -q)
+         run=quick
+         ;;
+      -s)
+         run=no
+         ;;
+      *)
+         REPORT=$1
+         break
+         ;;
+  esac
+  shift
+done
+
+# Usage
+if [ -z $REPORT ]; then
+       usage
+       exit 1
+fi
+
+# Output files
+OSREPORTTEMP=${REPORT}.os.xml
+PROGREPORTTEMP=${REPORT}.prog.xml
+RESULTREPORTTEMP=${REPORT}.result.xml
+XMLREPORT=${REPORT}.xml
+OUTPUT=${REPORT}.output
+SUMMARY=${REPORT}.summary
+PASSFAIL=${REPORT}.passfail
+TXTREPORT=${REPORT}.report
+FAILURES=${REPORT}.failures
+       
+#  Make sure permissions are correct
+chmod a+x lib/XmTestReport/*
+chmod a+x mkreport mergereport
+
+if [ ! -f contact_info ]; then
+    if [ "$batch" = "yes" ]; then
+       echo "Unable to read contact_info"
+       echo "You must run me interactively once!"
+       exit 1
+    else
+       get_contact_info
+    fi
+fi
+
+if [ "$run" != "no" ]; then
+    runnable_tests
+    make_environment_report $OSREPORTTEMP $PROGREPORTTEMP
+    if [ "$run" = "yes" ]; then
+       run_tests $OUTPUT
+    else
+       run_tests_quick $OUTPUT
+    fi
+    make_text_reports $PASSFAIL $FAILURES $OUTPUT $TXTREPORT
+    make_result_report $OUTPUT $RESULTREPORTTEMP
+    cat $OSREPORTTEMP $PROGREPORTTEMP $RESULTREPORTTEMP > $XMLREPORT
+    rm $OSREPORTTEMP $PROGREPORTTEMP $RESULTREPORTTEMP
+fi
+
+if [ "$report" = "yes" ] && [ "$run" = "yes" ]; then
+    if [ ! -f "$XMLREPORT" ]; then
+       echo "No such file: $XMLREPORT"
+       exit 1
+    fi
+    submit_report $XMLREPORT
+fi
-- 
Dan Smith
IBM Linux Technology Center
Open Hypervisor Team
email: danms@xxxxxxxxxx
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel

 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.