[Xen-devel] [PATCH 2/2] xtf: add a launcher script

[Roger Pau Monne <roger.pau@xxxxxxxxxx>]

Add a simple script that can list the tests relevant to the current
environment and run them. In it's current form it's functionality is quite
limited, and consists in one of this two options:

 - list: list tests relevant to the current environment. This information is
   fetched from each test test-info.json and the output of `xl info`.
 - test: launch a test (which can include creating several VMs) and report
   the results back. The results are printed on the terminal, and the error
   code is set appropriately (see backend/return_code.py for the list of
   possible error codes).

The path to the directory where the tests are stored is mandatory, and
should always be passed to the script. As an example, I've used the
following bash runes in order to launch all tests suitable for my
environment:

for test in `dist/bin/xtf-launcher.py -f dist/tests/ -l`
do
dist/bin/xtf-launcher.py -f dist/tests/ -t $test
echo "Test $test return code: $?"
done

Which yields the following (trimmed for convenience):

Parsing config from dist/tests/cpuid/test-pv64-cpuid.cfg
test-pv64-cpuid took 0s
Parsing config from dist/tests/cpuid/test-pv32pae-cpuid.cfg
test-pv32pae-cpuid took 0s
Parsing config from dist/tests/cpuid/test-hvm64-cpuid.cfg
libxl: warning: libxl_dm.c:1486:libxl__build_device_model_args_new: Could not 
find user xen-qemuuser-shared, starting QEMU as root
test-hvm64-cpuid took 0s
Parsing config from dist/tests/cpuid/test-hvm32pae-cpuid.cfg
libxl: warning: libxl_dm.c:1486:libxl__build_device_model_args_new: Could not 
find user xen-qemuuser-shared, starting QEMU as root
test-hvm32pae-cpuid took 0s
Parsing config from dist/tests/cpuid/test-hvm32pse-cpuid.cfg
libxl: warning: libxl_dm.c:1486:libxl__build_device_model_args_new: Could not 
find user xen-qemuuser-shared, starting QEMU as root
test-hvm32pse-cpuid took 0s
Parsing config from dist/tests/cpuid/test-hvm32-cpuid.cfg
libxl: warning: libxl_dm.c:1486:libxl__build_device_model_args_new: Could not 
find user xen-qemuuser-shared, starting QEMU as root
test-hvm32-cpuid took 0s
test-pv64-cpuid:        SUCCESS
test-hvm64-cpuid:       SUCCESS
test-hvm32-cpuid:       SUCCESS
test-hvm32pse-cpuid:    SUCCESS
test-hvm32pae-cpuid:    SUCCESS
test-pv32pae-cpuid:     SUCCESS
Test cpuid return code: 0
[...]

Note that the specific runes to interact with xl have been placed in a
separate file, and that other backends could be easily added provided that
config files suitable for them are also added to XTF tests. A new
--toolstack option should be added then in order to tell the launcher which
toolstack to use.

Signed-off-by: Roger Pau Monné <roger.pau@xxxxxxxxxx>
---
Cc: andrew.cooper3@xxxxxxxxxx
Cc: Ian.Jackson@xxxxxxxxxxxxx
Cc: wei.liu2@xxxxxxxxxx
Cc: anshul.makkar@xxxxxxxxxx
---
 Makefile                               |   2 +-
 tools/launcher/Makefile                |  12 ++++
 tools/launcher/backends/__init__.py    |   0
 tools/launcher/backends/return_code.py |  28 +++++++++
 tools/launcher/backends/xl.py          |  59 +++++++++++++++++++
 tools/launcher/xtf-launcher.py         | 101 +++++++++++++++++++++++++++++++++
 6 files changed, 201 insertions(+), 1 deletion(-)
 create mode 100644 tools/launcher/Makefile
 create mode 100644 tools/launcher/backends/__init__.py
 create mode 100644 tools/launcher/backends/return_code.py
 create mode 100644 tools/launcher/backends/xl.py
 create mode 100644 tools/launcher/xtf-launcher.py

diff --git a/Makefile b/Makefile
index fd8c3e0..eda9a25 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ all:
 
 .PHONY: install
 install:
-       @for D in $(wildcard tests/*); do \
+       @for D in $(wildcard tests/* tools/*); do \
                [ ! -e $$D/Makefile ] && continue; \
                $(MAKE) -C $$D install; \
        done
diff --git a/tools/launcher/Makefile b/tools/launcher/Makefile
new file mode 100644
index 0000000..2c07604
--- /dev/null
+++ b/tools/launcher/Makefile
@@ -0,0 +1,12 @@
+ROOT := $(abspath $(CURDIR)/../..)
+
+include $(ROOT)/build/common.mk
+
+TOOLS := xtf-launcher.py
+BACKENDS := __init__.py xl.py return_code.py
+
+install: $(TOOLS) $(addprefix backends/,$(BACKENDS))
+       @mkdir -p $(DESTDIR)/bin
+       @mkdir -p $(DESTDIR)/bin/backends
+       install -m775 -p $(TOOLS) $(DESTDIR)/bin
+       install -m664 -p $(addprefix backends/,$(BACKENDS)) 
$(DESTDIR)/bin/backends/
diff --git a/tools/launcher/backends/__init__.py 
b/tools/launcher/backends/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/launcher/backends/return_code.py 
b/tools/launcher/backends/return_code.py
new file mode 100644
index 0000000..509460a
--- /dev/null
+++ b/tools/launcher/backends/return_code.py
@@ -0,0 +1,28 @@
+#
+# Error codes for XTF launcher
+#
+
+# All went fine.
+SUCCESS = 0
+
+# Bug in the launcher.
+ERROR_LAUNCHER = 1
+
+# Test cannot be completed.
+ERROR_SKIP = 2
+
+# Bug in the test code or environment.
+ERROR_ERROR = 3
+
+# Bug in the code under test.
+ERROR_FAILURE = 4
+
+def code_to_str(code):
+    err_trans = {
+        0   :   'SUCCESS',
+        1   :   'LAUNCHER ERROR',
+        2   :   'SKIP',
+        3   :   'ERROR',
+        4   :   'FAILURE',
+    }
+    return err_trans[code]
diff --git a/tools/launcher/backends/xl.py b/tools/launcher/backends/xl.py
new file mode 100644
index 0000000..c1ac111
--- /dev/null
+++ b/tools/launcher/backends/xl.py
@@ -0,0 +1,59 @@
+#
+# XL backend for XTF
+#
+
+import subprocess, sys
+import return_code
+
+# XXX: multiprocessing is _not_ available in python < 2.6
+from multiprocessing.pool import Pool
+
+from timeit import default_timer as timer
+
+def _xl_console_helper(domain_name):
+    output = subprocess.check_output(['xl', 'console', domain_name])
+    result_line = output.splitlines()[-1]
+    if "SUCCESS" in result_line:
+        return return_code.SUCCESS
+    elif "SKIP" in result_line:
+        return return_code.ERROR_SKIP
+    elif "ERROR" in result_line:
+        return return_code.ERROR_ERROR
+    elif "FAILURE" in result_line:
+        return return_code.ERROR_FALIURE
+
+    # Something went wrong, no test report line was found (or it's invalid).
+    return return_code.ERROR_LAUNCHER
+
+class xl:
+    """XL driver for XTF"""
+
+    def __init__(self):
+        self._thpool = Pool(processes=1)
+
+    def get_supported_environments(self):
+        output = subprocess.check_output(['xl', 'info'])
+        #print output
+        for line in output.splitlines():
+            #print line
+            if not line.startswith("xen_caps"):
+                continue
+            caps = line.split()[2:]
+            return caps
+        return None
+
+    def run_test(self, test_cfg, domain_name):
+        rc = subprocess.call(['xl', 'create', '-p', test_cfg])
+        if rc:
+            sys.stderr.write("Failed to create VM from config file %s\n" %
+                             test_cfg)
+            return return_code.ERROR_LAUNCHER
+        thread = self._thpool.apply_async(_xl_console_helper, (domain_name, ))
+        start = timer()
+        rc = subprocess.call(['xl', 'unpause', domain_name])
+        if rc:
+            sys.stderr.write("Failed to unpause VM %s\n" % domain_name)
+            return return_code.ERROR_LAUNCHER
+        rc = thread.get()
+        sys.stderr.write("%s took %ds\n" % (domain_name, timer() - start))
+        return rc
diff --git a/tools/launcher/xtf-launcher.py b/tools/launcher/xtf-launcher.py
new file mode 100644
index 0000000..0685c2f
--- /dev/null
+++ b/tools/launcher/xtf-launcher.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+#
+# Launcher script for XTF
+#
+
+# XXX: JSON module is _not_ available on Python < 2.6
+import os, sys, json
+import backends.xl
+import backends.return_code as return_code
+
+from optparse import OptionParser
+
+def exit_error(error_str):
+    sys.stderr.write(error_str)
+    sys.exit(return_code.ERROR_LAUNCHER)
+
+def check_envinronment(req_envs, avail_envs):
+    env_trans = {
+        'pv32pae'   :   'xen-3.0-x86_32p',
+        'pv64'      :   'xen-3.0-x86_64',
+        'hvm32'     :   'hvm-3.0-x86_32',
+        'hvm32pse'  :   'hvm-3.0-x86_32',
+        'hvm32pae'  :   'hvm-3.0-x86_32p',
+        'hvm64'     :   'hvm-3.0-x86_64',
+    }
+    for env in req_envs:
+        if env_trans[env] not in avail_envs:
+            return False
+    return True
+
+def list():
+    for folder in os.listdir(options.folder):
+        if not os.path.isdir(os.path.join(options.folder, folder)):
+            continue
+        test_info = os.path.join(options.folder, folder, 'test-info.json')
+        if not os.path.isfile(test_info):
+            sys.stderr.write("Unable to find %s, ignoring folder\n" % testinfo)
+            continue
+        with open(test_info) as data_file:
+            info = json.load(data_file)
+        envs = backend.get_supported_environments()
+        if not envs:
+            exit_error("Unable to get Xen supported environments\n")
+        avail = check_envinronment(info["environments"], envs)
+        if not avail:
+            sys.stderr.write("Skipping test %s: not supported\n" % folder)
+            continue
+        print folder
+    return return_code.SUCCESS
+
+def test():
+    test_folder = os.path.join(options.folder, options.test)
+    if not os.path.isdir(test_folder):
+        exit_error("%s is not a directory\n" % test_folder)
+    test_info = os.path.join(test_folder, 'test-info.json')
+    if not os.path.isfile(test_info):
+        exit_error("%s doesn't exist" % test_info)
+    with open(test_info) as data_file:
+        info = json.load(data_file)
+    result = {}
+    for env in info["environments"]:
+        test_cfg = os.path.join(test_folder,
+                                "test-%s-%s.cfg" % (env, options.test))
+        test_name = "test-%s-%s" % (env, options.test)
+        if not os.path.isfile(test_cfg):
+            exit_error("Unable to find test config file %s\n" % test_cfg)
+        result[test_name] = backend.run_test(test_cfg, test_name)
+    # Print all the results
+    result_code = 0
+    for key, value in result.iteritems():
+        print "%s:\t%s" % (key, return_code.code_to_str(value))
+        # Return the highest error code in case of error.
+        result_code = max(value, result_code)
+    return result_code
+
+parser = OptionParser()
+parser.add_option("-f", "--folder", dest="folder",
+                  help="FOLDER where XTF tests are located", metavar="FOLDER")
+parser.add_option("-t", "--test", dest="test",
+                  help="NAME of test to run", metavar="NAME")
+parser.add_option("-l", "--list", action="store_true", dest="list",
+                  help="List test that are applicable to current environment")
+(options, args) = parser.parse_args()
+
+if not options.folder:
+    exit_error("folder not given\n")
+
+if options.list and options.test:
+    exit_error("options -l and -t are mutually exclusive\n")
+
+if not os.path.isdir(options.folder):
+    exit_error("%s does not exist or is not a folder\n" % options.folder)
+
+backend = backends.xl.xl()
+
+if options.list:
+    rc = list()
+elif options.test:
+    rc = test()
+
+sys.exit(rc)
-- 
2.7.4 (Apple Git-66)


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel