|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH 2 of 4 V5] tools/libxl: Remus network buffering - hotplug scripts and setup code
# HG changeset patch
# User Shriram Rajagopalan <rshriram@xxxxxxxxx>
# Date 1384801774 28800
# Node ID 5b4e029bb79b8db4210a182911a4ff7ec7957c4f
# Parent 4b471809a5bb016afb2d24521b5f8f548057b29f
tools/libxl: Remus network buffering - hotplug scripts and setup code
This patch introduces remus-netbuf-setup hotplug script responsible for
setting up and tearing down the necessary infrastructure required for
network output buffering in Remus. This script is intended to be invoked
by libxl for each guest interface, when starting or stopping Remus.
Apart from returning success/failure indication via the usual hotplug
entries in xenstore, this script also writes to xenstore, the name of
the IFB device to be used to control the vif's network output.
The script relies on libnl3 command line utilities to perform various
setup/teardown functions. The script is confined to Linux platforms only
since NetBSD does not seem to have libnl3.
Additionally, this patch introduces code to setup/teardown remus network
buffering for a given guest, when libxl_domain_remus_start API is invoked.
The following steps are taken during setup:
a) call the hotplug script for each vif to setup its network buffer
b) establish a dedicated remus context containing libnl related
state (netlink sockets, qdisc caches, etc.,)
c) Obtain handles to plug qdiscs installed on the IFB devices
chosen by the hotplug scripts.
During teardown, the netlink resources are released, followed by
invocation of hotplug scripts to remove the IFB devices.
Signed-off-by: Shriram Rajagopalan <rshriram@xxxxxxxxx>
diff -r 4b471809a5bb -r 5b4e029bb79b docs/misc/xenstore-paths.markdown
--- a/docs/misc/xenstore-paths.markdown Mon Nov 18 10:17:35 2013 -0800
+++ b/docs/misc/xenstore-paths.markdown Mon Nov 18 11:09:34 2013 -0800
@@ -356,6 +356,10 @@ The guest's virtual time offset from UTC
The device model version for a domain.
+#### /libxl/$DOMID/remus/netbuf/$DEVID/ifb = STRING [n,INTERNAL]
+
+IFB device used by Remus to buffer network output from the associated vif.
+
[BLKIF]:
http://xenbits.xen.org/docs/unstable/hypercall/include,public,io,blkif.h.html
[FBIF]:
http://xenbits.xen.org/docs/unstable/hypercall/include,public,io,fbif.h.html
[HVMPARAMS]:
http://xenbits.xen.org/docs/unstable/hypercall/include,public,hvm,params.h.html
diff -r 4b471809a5bb -r 5b4e029bb79b tools/hotplug/Linux/Makefile
--- a/tools/hotplug/Linux/Makefile Mon Nov 18 10:17:35 2013 -0800
+++ b/tools/hotplug/Linux/Makefile Mon Nov 18 11:09:34 2013 -0800
@@ -16,6 +16,7 @@ XEN_SCRIPTS += network-nat vif-nat
XEN_SCRIPTS += vif-openvswitch
XEN_SCRIPTS += vif2
XEN_SCRIPTS += vif-setup
+XEN_SCRIPTS-$(CONFIG_REMUS_NETBUF) += remus-netbuf-setup
XEN_SCRIPTS += block
XEN_SCRIPTS += block-enbd block-nbd
XEN_SCRIPTS-$(CONFIG_BLKTAP1) += blktap
diff -r 4b471809a5bb -r 5b4e029bb79b tools/hotplug/Linux/remus-netbuf-setup
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/hotplug/Linux/remus-netbuf-setup Mon Nov 18 11:09:34 2013 -0800
@@ -0,0 +1,183 @@
+#!/bin/bash
+#============================================================================
+# ${XEN_SCRIPT_DIR}/remus-netbuf-setup
+#
+# Script for attaching a network buffer to the specified vif (in any mode).
+# The hotplugging system will call this script when starting remus via libxl
+# API, libxl_domain_remus_start.
+#
+# Usage:
+# remus-netbuf-setup (setup|teardown)
+#
+# Environment vars:
+# vifname vif interface name (required).
+# XENBUS_PATH path in Xenstore, where the IFB device details will be stored
+# or read from (required).
+# (libxl passes /libxl/<domid>/remus/netbuf/<devid>)
+# IFB ifb interface to be cleaned up (required). [for teardown op only]
+
+# Written to the store: (setup operation)
+# XENBUS_PATH/ifb=<ifbdevName> the IFB device serving
+# as the intermediate buffer through which the interface's network output
+# can be controlled.
+#
+# To install a network buffer on a guest vif (vif1.0) using ifb (ifb0)
+# we need to do the following
+#
+# ip link set dev ifb0 up
+# tc qdisc add dev vif1.0 ingress
+# tc filter add dev vif1.0 parent ffff: proto ip \
+# prio 10 u32 match u32 0 0 action mirred egress redirect dev ifb0
+# nl-qdisc-add --dev=ifb0 --parent root plug
+# nl-qdisc-add --dev=ifb0 --parent root --update plug --limit=10000000
+# (10MB limit on buffer)
+#
+# So order of operations when installing a network buffer on vif1.0
+# 1. find a free ifb and bring up the device
+# 2. redirect traffic from vif1.0 to ifb:
+# 2.1 add ingress qdisc to vif1.0 (to capture outgoing packets from guest)
+# 2.2 use tc filter command with actions mirred egress + redirect
+# 3. install plug_qdisc on ifb device, with which we can buffer/release
+# guest's network output from vif1.0
+#
+#
+
+#============================================================================
+
+# Unlike other vif scripts, vif-common is not needed here as it executes vif
+#specific setup code such as renaming.
+dir=$(dirname "$0")
+. "$dir/xen-hotplug-common.sh"
+
+findCommand "$@"
+
+if [ "$command" != "setup" -a "$command" != "teardown" ]
+then
+ echo "Invalid command: $command"
+ log err "Invalid command: $command"
+ exit 1
+fi
+
+evalVariables "$@"
+
+: ${vifname:?}
+: ${XENBUS_PATH:?}
+
+check_libnl_tools() {
+ if ! command -v nl-qdisc-list > /dev/null 2>&1; then
+ fatal "Unable to find nl-qdisc-list tool"
+ fi
+ if ! command -v nl-qdisc-add > /dev/null 2>&1; then
+ fatal "Unable to find nl-qdisc-add tool"
+ fi
+ if ! command -v nl-qdisc-delete > /dev/null 2>&1; then
+ fatal "Unable to find nl-qdisc-delete tool"
+ fi
+}
+
+# We only check for modules. We don't load them.
+# User/Admin is supposed to load ifb during boot time,
+# ensuring that there are enough free ifbs in the system.
+# Other modules will be loaded automatically by tc commands.
+check_modules() {
+ for m in ifb sch_plug sch_ingress act_mirred cls_u32
+ do
+ if ! modinfo $m > /dev/null 2>&1; then
+ fatal "Unable to find $m kernel module"
+ fi
+ done
+}
+
+setup_ifb() {
+
+ for ifb in `ifconfig -a -s|egrep ^ifb|cut -d ' ' -f1`
+ do
+ local installed=`nl-qdisc-list -d $ifb`
+ [ -n "$installed" ] && continue
+ IFB="$ifb"
+ break
+ done
+
+ if [ -z "$IFB" ]
+ then
+ fatal "Unable to find a free IFB device for $vifname"
+ fi
+
+ do_or_die ip link set dev "$IFB" up
+}
+
+redirect_vif_traffic() {
+ local vif=$1
+ local ifb=$2
+
+ do_or_die tc qdisc add dev "$vif" ingress
+
+ tc filter add dev "$vif" parent ffff: proto ip prio 10 \
+ u32 match u32 0 0 action mirred egress redirect dev "$ifb" >/dev/null
2>&1
+
+ if [ $? -ne 0 ]
+ then
+ do_without_error tc qdisc del dev "$vif" ingress
+ fatal "Failed to redirect traffic from $vif to $ifb"
+ fi
+}
+
+add_plug_qdisc() {
+ local vif=$1
+ local ifb=$2
+
+ nl-qdisc-add --dev="$ifb" --parent root plug >/dev/null 2>&1
+ if [ $? -ne 0 ]
+ then
+ do_without_error tc qdisc del dev "$vif" ingress
+ fatal "Failed to add plug qdisc to $ifb"
+ fi
+
+ #set ifb buffering limit in bytes. Its okay if this command fails
+ nl-qdisc-add --dev="$ifb" --parent root \
+ --update plug --limit=10000000 >/dev/null 2>&1
+}
+
+teardown_netbuf() {
+ local vif=$1
+ local ifb=$2
+
+ if [ "$ifb" ]; then
+ do_without_error ip link set dev "$ifb" down
+ do_without_error nl-qdisc-delete --dev="$ifb" --parent root plug
>/dev/null 2>&1
+ xenstore-rm -t "$XENBUS_PATH/ifb" 2>/dev/null || true
+ fi
+ do_without_error tc qdisc del dev "$vif" ingress
+ xenstore-rm -t "$XENBUS_PATH/hotplug-status" 2>/dev/null || true
+}
+
+xs_write_failed() {
+ local vif=$1
+ local ifb=$2
+ teardown_netbuf "$vifname" "$IFB"
+ fatal "failed to write ifb name to xenstore"
+}
+
+case "$command" in
+ setup)
+ check_libnl_tools
+ check_modules
+
+ claim_lock "pickifb"
+ setup_ifb
+ redirect_vif_traffic "$vifname" "$IFB"
+ add_plug_qdisc "$vifname" "$IFB"
+ release_lock "pickifb"
+
+ #not using xenstore_write that automatically exits on error
+ #because we need to cleanup
+ _xenstore_write "$XENBUS_PATH/ifb" "$IFB" || xs_write_failed "$vifname"
"$IFB"
+ success
+ ;;
+ teardown)
+ : ${IFB:?}
+ teardown_netbuf "$vifname" "$IFB"
+ ;;
+esac
+
+log debug "Successful remus-netbuf-setup $command for $vifname, ifb $IFB."
diff -r 4b471809a5bb -r 5b4e029bb79b tools/libxl/Makefile
--- a/tools/libxl/Makefile Mon Nov 18 10:17:35 2013 -0800
+++ b/tools/libxl/Makefile Mon Nov 18 11:09:34 2013 -0800
@@ -42,6 +42,13 @@ LIBXL_OBJS-y += libxl_blktap2.o
else
LIBXL_OBJS-y += libxl_noblktap2.o
endif
+
+ifeq ($(CONFIG_REMUS_NETBUF),y)
+LIBXL_OBJS-y += libxl_netbuffer.o
+else
+LIBXL_OBJS-y += libxl_nonetbuffer.o
+endif
+
LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o
LIBXL_OBJS-$(CONFIG_IA64) += libxl_nocpuid.o libxl_noarch.o
LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_noarch.o
diff -r 4b471809a5bb -r 5b4e029bb79b tools/libxl/libxl.c
--- a/tools/libxl/libxl.c Mon Nov 18 10:17:35 2013 -0800
+++ b/tools/libxl/libxl.c Mon Nov 18 11:09:34 2013 -0800
@@ -698,8 +698,9 @@ libxl_vminfo * libxl_list_vm(libxl_ctx *
return ptr;
}
-static void remus_failover_cb(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc);
+static void remus_replication_failure_cb(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ int rc);
/* TODO: Explicit Checkpoint acknowledgements via recv_fd. */
int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
@@ -718,40 +719,57 @@ int libxl_domain_remus_start(libxl_ctx *
GCNEW(dss);
dss->ao = ao;
- dss->callback = remus_failover_cb;
+ dss->callback = remus_replication_failure_cb;
dss->domid = domid;
dss->fd = send_fd;
/* TODO do something with recv_fd */
dss->type = type;
dss->live = 1;
dss->debug = 0;
- dss->remus = info;
assert(info);
- /* TBD: Remus setup - i.e. attach qdisc, enable disk buffering, etc */
+ GCNEW(dss->remus_ctx);
+
+ /* convenience shorthand */
+ libxl__remus_ctx *remus_ctx = dss->remus_ctx;
+ remus_ctx->blackhole = info->blackhole;
+ remus_ctx->interval = info->interval;
+ remus_ctx->compression = info->compression;
+ remus_ctx->dss = dss;
+
+ /* TODO: enable disk buffering */
+
+ /* Setup network buffering */
+ if (info->netbuf) {
+ if (!libxl__netbuffer_enabled(gc)) {
+ LOG(ERROR, "Remus: No support for network buffering");
+ goto out;
+ }
+
+ if (info->netbufscript) {
+ remus_ctx->netbufscript =
+ libxl__strdup(gc, info->netbufscript);
+ } else {
+ remus_ctx->netbufscript =
+ GCSPRINTF("%s/remus-netbuf-setup",
+ libxl__xen_script_dir_path());
+ }
+ }
/* Point of no return */
- libxl__domain_suspend(egc, dss);
+ libxl__remus_setup_initiate(egc, dss);
return AO_INPROGRESS;
out:
return AO_ABORT(rc);
}
-static void remus_failover_cb(libxl__egc *egc,
- libxl__domain_suspend_state *dss, int rc)
+static void remus_replication_failure_cb(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ int rc)
{
STATE_AO_GC(dss->ao);
- /*
- * With Remus, if we reach this point, it means either
- * backup died or some network error occurred preventing us
- * from sending checkpoints.
- */
-
- /* TBD: Remus cleanup - i.e. detach qdisc, release other
- * resources.
- */
libxl__ao_complete(egc, ao, rc);
}
diff -r 4b471809a5bb -r 5b4e029bb79b tools/libxl/libxl.h
--- a/tools/libxl/libxl.h Mon Nov 18 10:17:35 2013 -0800
+++ b/tools/libxl/libxl.h Mon Nov 18 11:09:34 2013 -0800
@@ -375,6 +375,19 @@
*/
#define LIBXL_HAVE_CREATEINFO_PVH 1
+/*
+ * LIBXL_HAVE_REMUS_NETBUF 1
+ *
+ * If this is defined, then the libxl_domain_remus_info structure will
+ * have a boolean field (netbuf) and a string field (netbufscript).
+ *
+ * netbuf, if true, indicates that network buffering should be enabled.
+ *
+ * netbufscript, if set, indicates the path to the hotplug script to
+ * setup or teardown network buffers.
+ */
+#define LIBXL_HAVE_REMUS_NETBUF 1
+
/* Functions annotated with LIBXL_EXTERNAL_CALLERS_ONLY may not be
* called from within libxl itself. Callers outside libxl, who
* do not #include libxl_internal.h, are fine. */
diff -r 4b471809a5bb -r 5b4e029bb79b tools/libxl/libxl_dom.c
--- a/tools/libxl/libxl_dom.c Mon Nov 18 10:17:35 2013 -0800
+++ b/tools/libxl/libxl_dom.c Mon Nov 18 11:09:34 2013 -0800
@@ -1228,6 +1228,51 @@ int libxl__toolstack_save(uint32_t domid
return 0;
}
+/*----- remus setup/teardown code -----*/
+void libxl__remus_setup_initiate(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+ libxl__ev_time_init(&dss->remus_ctx->timeout);
+ if (!dss->remus_ctx->netbufscript)
+ libxl__remus_setup_done(egc, dss, 0);
+ else
+ libxl__remus_netbuf_setup(egc, dss);
+}
+
+void libxl__remus_setup_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ int rc)
+{
+ STATE_AO_GC(dss->ao);
+ if (!rc) {
+ libxl__domain_suspend(egc, dss);
+ return;
+ }
+
+ LOG(ERROR, "Remus: failed to setup network buffering"
+ " for guest with domid %u", dss->domid);
+ domain_suspend_done(egc, dss, rc);
+}
+
+void libxl__remus_teardown_initiate(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ int rc)
+{
+ /* stash rc somewhere before invoking teardown ops. */
+ dss->remus_ctx->saved_rc = rc;
+
+ if (!dss->remus_ctx->netbuf_ctx)
+ libxl__remus_teardown_done(egc, dss);
+ else
+ libxl__remus_netbuf_teardown(egc, dss);
+}
+
+void libxl__remus_teardown_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+ dss->callback(egc, dss, dss->remus_ctx->saved_rc);
+}
+
/*----- remus callbacks -----*/
static int libxl__remus_domain_suspend_callback(void *data)
@@ -1276,7 +1321,7 @@ static void remus_checkpoint_dm_saved(li
/* REMUS TODO: Wait for disk and memory ack, release network buffer */
/* REMUS TODO: make this asynchronous */
assert(!rc); /* REMUS TODO handle this error properly */
- usleep(dss->interval * 1000);
+ usleep(dss->remus_ctx->interval * 1000);
libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1);
}
@@ -1294,7 +1339,6 @@ void libxl__domain_suspend(libxl__egc *e
const libxl_domain_type type = dss->type;
const int live = dss->live;
const int debug = dss->debug;
- const libxl_domain_remus_info *const r_info = dss->remus;
libxl__srm_save_autogen_callbacks *const callbacks =
&dss->shs.callbacks.save.a;
@@ -1329,11 +1373,8 @@ void libxl__domain_suspend(libxl__egc *e
dss->guest_responded = 0;
dss->dm_savefile = libxl__device_model_savefile(gc, domid);
- if (r_info != NULL) {
- dss->interval = r_info->interval;
- if (r_info->compression)
- dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
- }
+ if (dss->remus_ctx && dss->remus_ctx->compression)
+ dss->xcflags |= XCFLAGS_CHECKPOINT_COMPRESS;
dss->xce = xc_evtchn_open(NULL, 0);
if (dss->xce == NULL)
@@ -1352,7 +1393,7 @@ void libxl__domain_suspend(libxl__egc *e
}
memset(callbacks, 0, sizeof(*callbacks));
- if (r_info != NULL) {
+ if (dss->remus_ctx != NULL) {
callbacks->suspend = libxl__remus_domain_suspend_callback;
callbacks->postcopy = libxl__remus_domain_resume_callback;
callbacks->checkpoint = libxl__remus_domain_checkpoint_callback;
@@ -1512,6 +1553,17 @@ static void domain_suspend_done(libxl__e
if (dss->xce != NULL)
xc_evtchn_close(dss->xce);
+ if (dss->remus_ctx) {
+ /*
+ * With Remus, if we reach this point, it means either
+ * backup died or some network error occurred preventing us
+ * from sending checkpoints. Teardown the network buffers and
+ * release netlink resources. This is an async op.
+ */
+ libxl__remus_teardown_initiate(egc, dss, rc);
+ return;
+ }
+
dss->callback(egc, dss, rc);
}
diff -r 4b471809a5bb -r 5b4e029bb79b tools/libxl/libxl_internal.h
--- a/tools/libxl/libxl_internal.h Mon Nov 18 10:17:35 2013 -0800
+++ b/tools/libxl/libxl_internal.h Mon Nov 18 11:09:34 2013 -0800
@@ -2291,6 +2291,50 @@ typedef struct libxl__logdirty_switch {
libxl__ev_time timeout;
} libxl__logdirty_switch;
+typedef struct libxl__remus_ctx {
+ /* checkpoint interval */
+ int interval;
+ int blackhole;
+ int compression;
+ int saved_rc;
+ /* Script to setup/teardown network buffers */
+ const char *netbufscript;
+ /* Opaque context containing network buffer related stuff */
+ void *netbuf_ctx;
+ libxl__domain_suspend_state *dss;
+ libxl__ev_time timeout;
+ libxl__ev_child child;
+ int num_exec;
+} libxl__remus_ctx;
+
+_hidden int libxl__netbuffer_enabled(libxl__gc *gc);
+
+_hidden void libxl__remus_setup_initiate(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+
+_hidden void libxl__remus_setup_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ int rc);
+
+_hidden void libxl__remus_teardown_initiate(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ int rc);
+
+_hidden void libxl__remus_teardown_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+
+_hidden void libxl__remus_netbuf_setup(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+
+_hidden void libxl__remus_netbuf_teardown(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+
+_hidden int libxl__remus_netbuf_start_new_epoch(libxl__gc *gc, uint32_t domid,
+ libxl__remus_ctx *remus_ctx);
+
+_hidden int libxl__remus_netbuf_release_prev_epoch(libxl__gc *gc, uint32_t
domid,
+ libxl__remus_ctx *remus_ctx);
+
struct libxl__domain_suspend_state {
/* set by caller of libxl__domain_suspend */
libxl__ao *ao;
@@ -2301,7 +2345,7 @@ struct libxl__domain_suspend_state {
libxl_domain_type type;
int live;
int debug;
- const libxl_domain_remus_info *remus;
+ libxl__remus_ctx *remus_ctx;
/* private */
xc_evtchn *xce; /* event channel handle */
int suspend_eventchn;
@@ -2309,7 +2353,6 @@ struct libxl__domain_suspend_state {
int xcflags;
int guest_responded;
const char *dm_savefile;
- int interval; /* checkpoint interval (for Remus) */
libxl__save_helper_state shs;
libxl__logdirty_switch logdirty;
/* private for libxl__domain_save_device_model */
diff -r 4b471809a5bb -r 5b4e029bb79b tools/libxl/libxl_netbuffer.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/libxl/libxl_netbuffer.c Mon Nov 18 11:09:34 2013 -0800
@@ -0,0 +1,516 @@
+/*
+ * Copyright (C) 2013
+ * Author Shriram Rajagopalan <rshriram@xxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+#include <netlink/cache.h>
+#include <netlink/socket.h>
+#include <netlink/attr.h>
+#include <netlink/route/link.h>
+#include <netlink/route/route.h>
+#include <netlink/route/qdisc.h>
+#include <netlink/route/qdisc/plug.h>
+
+typedef struct libxl__remus_netbuf_ctx {
+ struct rtnl_qdisc **netbuf_qdisc_list;
+ struct nl_sock *nlsock;
+ struct nl_cache *qdisc_cache;
+ char **vif_list;
+ char **ifb_list;
+ uint32_t num_netbufs;
+ uint32_t unused;
+} libxl__remus_netbuf_ctx;
+
+/* If the device has a vifname, then use that instead of
+ * the vifX.Y format.
+ */
+static char *get_vifname(libxl__gc *gc, uint32_t domid,
+ libxl_device_nic *nic)
+{
+ const char *vifname = NULL;
+ vifname = libxl__xs_read(gc, XBT_NULL,
+ libxl__sprintf(gc,
+ "%s/backend/vif/%d/%d/vifname",
+ libxl__xs_get_dompath(gc, 0),
+ domid, nic->devid));
+ if (!vifname) {
+ vifname = libxl__device_nic_devname(gc, domid,
+ nic->devid,
+ nic->nictype);
+ }
+
+ return libxl__strdup(gc, vifname);
+}
+
+static char **get_guest_vif_list(libxl__gc *gc, uint32_t domid,
+ int *num_vifs)
+{
+ libxl_device_nic *nics = NULL;
+ int nb, i = 0;
+ char **vif_list = NULL;
+
+ *num_vifs = 0;
+ nics = libxl_device_nic_list(CTX, domid, &nb);
+ if (!nics) return NULL;
+
+ /* Ensure that none of the vifs are backed by driver domains */
+ for (i = 0; i < nb; i++) {
+ if (nics[i].backend_domid != LIBXL_TOOLSTACK_DOMID) {
+ LOG(ERROR, "vif %s has driver domain (%u) as its backend.\n"
+ "Network buffering is not supported with driver domains",
+ get_vifname(gc, domid, &nics[i]), nics[i].backend_domid);
+ *num_vifs = -1;
+ goto end;
+ }
+ }
+
+ vif_list = libxl__calloc(gc, nb, sizeof(char *));
+ for (i = 0; i < nb; ++i)
+ vif_list[i] = get_vifname(gc, domid, &nics[i]);
+ *num_vifs = nb;
+
+ end:
+ for (i = 0; i < nb; i++)
+ libxl_device_nic_dispose(&nics[i]);
+ free(nics);
+ return vif_list;
+}
+
+static int init_qdiscs(libxl__gc *gc,
+ libxl__remus_ctx *remus_ctx);
+
+static int exec_netbuf_script(libxl__gc *gc, libxl__remus_ctx *remus_ctx,
+ char *op, libxl__ev_child_callback *death);
+
+static void netbuf_setup_script_cb(libxl__egc *egc,
+ libxl__ev_child *child,
+ pid_t pid, int status);
+
+static void netbuf_teardown_script_cb(libxl__egc *egc,
+ libxl__ev_child *child,
+ pid_t pid, int status);
+
+static void netbuf_setup_timeout_cb(libxl__egc *egc,
+ libxl__ev_time *ev,
+ const struct timeval *requested_abs);
+
+static int init_qdiscs(libxl__gc *gc,
+ libxl__remus_ctx *remus_ctx)
+{
+ int i, ret, ifindex;
+ struct rtnl_link *ifb = NULL;
+ struct rtnl_qdisc *qdisc = NULL;
+
+ /* convenience vars */
+ libxl__remus_netbuf_ctx *netbuf_ctx = remus_ctx->netbuf_ctx;
+ int num_netbufs = netbuf_ctx->num_netbufs;
+ char **ifb_list = netbuf_ctx->ifb_list;
+
+ /* Now that we have brought up IFB devices with plug qdisc for
+ * each vif, lets get a netlink handle on the plug qdisc for use
+ * during checkpointing.
+ */
+ netbuf_ctx->nlsock = nl_socket_alloc();
+ if (!netbuf_ctx->nlsock) {
+ LOG(ERROR, "cannot allocate nl socket");
+ goto end;
+ }
+
+ ret = nl_connect(netbuf_ctx->nlsock, NETLINK_ROUTE);
+ if (ret) {
+ LOG(ERROR, "failed to open netlink socket: %s",
+ nl_geterror(ret));
+ goto end;
+ }
+
+ /* get list of all qdiscs installed on network devs. */
+ ret = rtnl_qdisc_alloc_cache(netbuf_ctx->nlsock,
+ &netbuf_ctx->qdisc_cache);
+ if (ret) {
+ LOG(ERROR, "failed to allocate qdisc cache: %s",
+ nl_geterror(ret));
+ goto end;
+ }
+
+ /* list of handles to plug qdiscs */
+ netbuf_ctx->netbuf_qdisc_list =
+ libxl__calloc(gc, num_netbufs,
+ sizeof(struct rtnl_qdisc *));
+
+ ifb = rtnl_link_alloc();
+
+ for (i = 0; i < num_netbufs; ++i) {
+
+ /* get a handle to the IFB interface */
+ ret = rtnl_link_get_kernel(netbuf_ctx->nlsock, 0,
+ ifb_list[i], &ifb);
+ if (ret) {
+ LOG(ERROR, "cannot obtain handle for %s: %s", ifb_list[i],
+ nl_geterror(ret));
+ goto end;
+ }
+
+ ifindex = rtnl_link_get_ifindex(ifb);
+ if (!ifindex) {
+ LOG(ERROR, "interface %s has no index", ifb_list[i]);
+ goto end;
+ }
+
+ /* Get a reference to the root qdisc installed on the IFB, by
+ * querying the qdisc list we obtained earlier. The netbufscript
+ * sets up the plug qdisc as the root qdisc, so we don't have to
+ * search the entire qdisc tree on the IFB dev.
+
+ * There is no need to explicitly free this qdisc as its just a
+ * reference from the qdisc cache we allocated earlier.
+ */
+ qdisc = rtnl_qdisc_get_by_parent(netbuf_ctx->qdisc_cache, ifindex,
+ TC_H_ROOT);
+
+ if (qdisc) {
+ const char *tc_kind = rtnl_tc_get_kind(TC_CAST(qdisc));
+ /* Sanity check: Ensure that the root qdisc is a plug qdisc. */
+ if (!tc_kind || strcmp(tc_kind, "plug")) {
+ LOG(ERROR, "plug qdisc is not installed on %s", ifb_list[i]);
+ goto end;
+ }
+ netbuf_ctx->netbuf_qdisc_list[i] = qdisc;
+ } else {
+ LOG(ERROR, "Cannot get qdisc handle from ifb %s", ifb_list[i]);
+ goto end;
+ }
+ }
+
+ rtnl_link_put(ifb);
+ return 0;
+
+ end:
+ if (ifb) rtnl_link_put(ifb);
+ return ERROR_FAIL;
+}
+
+/* the script needs the following env & args
+ * $vifname
+ * $XENBUS_PATH (/libxl/<domid>/remus/netbuf/<devid>/)
+ * $IFB (for teardown)
+ * setup/teardown as command line arg.
+ * In return, the script writes the name of IFB device (during setup) to be
+ * used for output buffering into XENBUS_PATH/ifb
+ */
+static int exec_netbuf_script(libxl__gc *gc, libxl__remus_ctx *remus_ctx,
+ char *op, libxl__ev_child_callback *death)
+{
+ int arraysize, nr = 0;
+ char **env = NULL, **args = NULL;
+ pid_t pid;
+ libxl__ev_child *child = &remus_ctx->child;
+ libxl__ev_time *timeout = &remus_ctx->timeout;
+ char *script = libxl__strdup(gc, remus_ctx->netbufscript);
+ uint32_t domid = remus_ctx->dss->domid;
+ int devid = remus_ctx->num_exec;
+ libxl__remus_netbuf_ctx *netbuf_ctx = remus_ctx->netbuf_ctx;
+ char *vif = netbuf_ctx->vif_list[devid];
+ char *ifb = netbuf_ctx->ifb_list[devid];
+
+ if (!strcmp(op, "setup"))
+ arraysize = 5;
+ else
+ arraysize = 7;
+
+ GCNEW_ARRAY(env, arraysize);
+ env[nr++] = "vifname";
+ env[nr++] = vif;
+ env[nr++] = "XENBUS_PATH";
+ env[nr++] = GCSPRINTF("%s/remus/netbuf/%d",
+ libxl__xs_libxl_path(gc, domid), devid);
+ if (!strcmp(op, "teardown")) {
+ env[nr++] = "IFB";
+ env[nr++] = ifb;
+ }
+ env[nr++] = NULL;
+ assert(nr <= arraysize);
+
+ arraysize = 3; nr = 0;
+ GCNEW_ARRAY(args, arraysize);
+ args[nr++] = script;
+ args[nr++] = op;
+ args[nr++] = NULL;
+ assert(nr == arraysize);
+
+ libxl__ev_child_init(child);
+
+ /* Set hotplug timeout */
+ if (libxl__ev_time_register_rel(gc, timeout,
+ netbuf_setup_timeout_cb,
+ LIBXL_HOTPLUG_TIMEOUT * 1000)) {
+ LOG(ERROR, "unable to register timeout for "
+ "netbuf setup script %s on vif %s", script, vif);
+ return ERROR_FAIL;
+ }
+
+ LOG(DEBUG, "Calling netbuf script: %s %s on vif %s",
+ script, op, vif);
+
+ /* Fork and exec netbuf script */
+ pid = libxl__ev_child_fork(gc, child, death);
+ if (pid == -1) {
+ LOG(ERROR, "unable to fork netbuf script %s", script);
+ return ERROR_FAIL;
+ }
+
+ if (!pid) {
+ /* child: Launch netbuf script */
+ libxl__exec(gc, -1, -1, -1, args[0], args, env);
+ /* notreached */
+ abort();
+ }
+
+ return 0;
+}
+
+static void netbuf_teardown_script_cb(libxl__egc *egc,
+ libxl__ev_child *child,
+ pid_t pid, int status)
+{
+ libxl__remus_ctx *remus_ctx = CONTAINER_OF(child, *remus_ctx, child);
+ libxl__remus_netbuf_ctx *netbuf_ctx = remus_ctx->netbuf_ctx;
+
+ STATE_AO_GC(remus_ctx->dss->ao);
+
+ if (status) {
+ libxl_report_child_exitstatus(CTX, LIBXL__LOG_ERROR,
+ remus_ctx->netbufscript,
+ pid, status);
+ }
+
+ remus_ctx->num_exec++;
+ if (remus_ctx->num_exec < netbuf_ctx->num_netbufs) {
+ if (exec_netbuf_script(gc, remus_ctx,
+ "teardown", netbuf_teardown_script_cb))
+ goto end;
+ return;
+ }
+
+ end:
+ libxl__remus_teardown_done(egc, remus_ctx->dss);
+}
+
+static void netbuf_setup_script_cb(libxl__egc *egc,
+ libxl__ev_child *child,
+ pid_t pid, int status)
+{
+ libxl__remus_ctx *remus_ctx = CONTAINER_OF(child, *remus_ctx, child);
+ char *out_path_base, *hotplug_error;
+ uint32_t domid = remus_ctx->dss->domid;
+ int devid = remus_ctx->num_exec;
+ libxl__remus_netbuf_ctx *netbuf_ctx = remus_ctx->netbuf_ctx;
+ char *vif = netbuf_ctx->vif_list[devid];
+ char **ifb = &netbuf_ctx->ifb_list[devid];
+ int rc = ERROR_FAIL;
+
+ STATE_AO_GC(remus_ctx->dss->ao);
+
+ libxl__ev_time_deregister(gc, &remus_ctx->timeout);
+
+ out_path_base = GCSPRINTF("%s/remus/netbuf/%d",
+ libxl__xs_libxl_path(gc, domid), devid);
+
+ hotplug_error = libxl__xs_read(gc, XBT_NULL,
+ GCSPRINTF("%s/hotplug-error",
+ out_path_base));
+ if (hotplug_error) {
+ LOG(ERROR, "netbuf script %s setup failed for vif %s: %s",
+ remus_ctx->netbufscript,
+ netbuf_ctx->vif_list[devid], hotplug_error);
+ goto end;
+ }
+
+ if (status) {
+ libxl_report_child_exitstatus(CTX, LIBXL__LOG_ERROR,
+ remus_ctx->netbufscript,
+ pid, status);
+ goto end;
+ }
+
+ *ifb = libxl__xs_read(gc, XBT_NULL,
+ GCSPRINTF("%s/remus/netbuf/%d/ifb",
+ libxl__xs_libxl_path(gc, domid),
+ devid));
+ if (!(*ifb)) {
+ LOG(ERROR, "Cannot get ifb dev name for domain %u dev %s",
+ domid, vif);
+ goto end;
+ }
+
+ LOG(DEBUG, "%s will buffer packets from vif %s", *ifb, vif);
+ remus_ctx->num_exec++;
+ if (remus_ctx->num_exec < netbuf_ctx->num_netbufs) {
+ if (exec_netbuf_script(gc, remus_ctx,
+ "setup", netbuf_setup_script_cb))
+ goto end;
+ return;
+ }
+
+ rc = init_qdiscs(gc, remus_ctx);
+ end:
+ libxl__remus_setup_done(egc, remus_ctx->dss, rc);
+}
+
+static void netbuf_setup_timeout_cb(libxl__egc *egc,
+ libxl__ev_time *ev,
+ const struct timeval *requested_abs)
+{
+ libxl__remus_ctx *remus_ctx = CONTAINER_OF(ev, *remus_ctx, timeout);
+ int devid = remus_ctx->num_exec;
+ libxl__remus_netbuf_ctx *netbuf_ctx = remus_ctx->netbuf_ctx;
+ char *vif = netbuf_ctx->vif_list[devid];
+
+ STATE_AO_GC(remus_ctx->dss->ao);
+
+ libxl__ev_time_deregister(gc, &remus_ctx->timeout);
+ assert(libxl__ev_child_inuse(&remus_ctx->child));
+
+ LOG(DEBUG, "killing hotplug script %s (on vif %s) because of timeout",
+ remus_ctx->netbufscript, vif);
+
+ if (kill(remus_ctx->child.pid, SIGKILL)) {
+ LOGEV(ERROR, errno, "unable to kill hotplug script %s [%ld]",
+ remus_ctx->netbufscript,
+ (unsigned long)remus_ctx->child.pid);
+ }
+
+ return;
+}
+
+int libxl__netbuffer_enabled(libxl__gc *gc)
+{
+ return 1;
+}
+
+/* Scan through the list of vifs belonging to domid and
+ * invoke the netbufscript to setup the IFB device & plug qdisc
+ * for each vif. Then scan through the list of IFB devices to obtain
+ * a handle on the plug qdisc installed on these IFB devices.
+ * Network output buffering is controlled via these qdiscs.
+ */
+void libxl__remus_netbuf_setup(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+ libxl__remus_netbuf_ctx *netbuf_ctx = NULL;
+ uint32_t domid = dss->domid;
+ libxl__remus_ctx *remus_ctx = dss->remus_ctx;
+ int num_netbufs = 0;
+ int rc = ERROR_FAIL;
+
+ STATE_AO_GC(dss->ao);
+
+ netbuf_ctx = libxl__zalloc(gc, sizeof(libxl__remus_netbuf_ctx));
+ netbuf_ctx->vif_list = get_guest_vif_list(gc, domid, &num_netbufs);
+ if (!num_netbufs) {
+ rc = 0;
+ goto end;
+ }
+
+ if (num_netbufs < 0) goto end;
+
+ netbuf_ctx->ifb_list = libxl__calloc(gc, num_netbufs,
+ sizeof(char *));
+ netbuf_ctx->num_netbufs = num_netbufs;
+ remus_ctx->netbuf_ctx = netbuf_ctx;
+ remus_ctx->num_exec = 0; //start at devid == 0
+ if (exec_netbuf_script(gc, remus_ctx, "setup",
+ netbuf_setup_script_cb))
+ goto end;
+ return;
+
+ end:
+ libxl__remus_setup_done(egc, dss, rc);
+}
+
+/* Note: This function will be called in the same gc context as
+ * libxl__remus_netbuf_setup, created during the libxl_domain_remus_start
+ * API call.
+ */
+void libxl__remus_netbuf_teardown(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+ libxl__remus_ctx *remus_ctx = dss->remus_ctx;
+ libxl__remus_netbuf_ctx *netbuf_ctx = remus_ctx->netbuf_ctx;
+ STATE_AO_GC(dss->ao);
+
+ if (netbuf_ctx->qdisc_cache)
+ nl_cache_free(netbuf_ctx->qdisc_cache);
+
+ if (netbuf_ctx->nlsock)
+ nl_close(netbuf_ctx->nlsock);
+
+ remus_ctx->num_exec = 0; //start at devid == 0
+ if (exec_netbuf_script(gc, remus_ctx, "teardown",
+ netbuf_teardown_script_cb))
+ libxl__remus_teardown_done(egc, dss);
+}
+
+#define TC_BUFFER_START 1
+#define TC_BUFFER_RELEASE 2
+static int remus_netbuf_op(libxl__gc *gc, uint32_t domid,
+ libxl__remus_ctx *remus_ctx,
+ int buffer_op)
+{
+ int i, ret;
+ libxl__remus_netbuf_ctx *netbuf_ctx = remus_ctx->netbuf_ctx;
+
+ for (i = 0; i < netbuf_ctx->num_netbufs; ++i) {
+ if (buffer_op == TC_BUFFER_START)
+ ret = rtnl_qdisc_plug_buffer(netbuf_ctx->netbuf_qdisc_list[i]);
+ else
+ ret =
rtnl_qdisc_plug_release_one(netbuf_ctx->netbuf_qdisc_list[i]);
+
+ if (!ret)
+ ret = rtnl_qdisc_add(netbuf_ctx->nlsock,
+ netbuf_ctx->netbuf_qdisc_list[i],
+ NLM_F_REQUEST);
+ if (ret) {
+ LOG(ERROR, "Remus: cannot do netbuf op %s on %s:%s",
+ ((buffer_op == TC_BUFFER_START) ?
+ "start_new_epoch" : "release_prev_epoch"),
+ netbuf_ctx->ifb_list[i], nl_geterror(ret));
+ return ERROR_FAIL;
+ }
+ }
+
+ return 0;
+}
+
+int libxl__remus_netbuf_start_new_epoch(libxl__gc *gc, uint32_t domid,
+ libxl__remus_ctx *remus_ctx)
+{
+ return remus_netbuf_op(gc, domid, remus_ctx, TC_BUFFER_START);
+}
+
+int libxl__remus_netbuf_release_prev_epoch(libxl__gc *gc, uint32_t domid,
+ libxl__remus_ctx *remus_ctx)
+{
+ return remus_netbuf_op(gc, domid, remus_ctx, TC_BUFFER_RELEASE);
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff -r 4b471809a5bb -r 5b4e029bb79b tools/libxl/libxl_nonetbuffer.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/libxl/libxl_nonetbuffer.c Mon Nov 18 11:09:34 2013 -0800
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2013
+ * Author Shriram Rajagopalan <rshriram@xxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+int libxl__netbuffer_enabled(libxl__gc *gc)
+{
+ return 0;
+}
+
+/* Remus network buffer related stubs */
+void libxl__remus_netbuf_setup(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+}
+
+void libxl__remus_netbuf_teardown(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
+{
+}
+
+int libxl__remus_netbuf_start_new_epoch(libxl__gc *gc, uint32_t domid,
+ libxl__remus_ctx *remus_ctx)
+{
+ LOG(ERROR, "Remus: No support for network buffering");
+ return ERROR_FAIL;
+}
+
+int libxl__remus_netbuf_release_prev_epoch(libxl__gc *gc, uint32_t domid,
+ libxl__remus_ctx *remus_ctx)
+{
+ LOG(ERROR, "Remus: No support for network buffering");
+ return ERROR_FAIL;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff -r 4b471809a5bb -r 5b4e029bb79b tools/libxl/libxl_types.idl
--- a/tools/libxl/libxl_types.idl Mon Nov 18 10:17:35 2013 -0800
+++ b/tools/libxl/libxl_types.idl Mon Nov 18 11:09:34 2013 -0800
@@ -558,6 +558,8 @@ libxl_domain_remus_info = Struct("domain
("interval", integer),
("blackhole", bool),
("compression", bool),
+ ("netbuf", bool),
+ ("netbufscript", string),
])
libxl_event_type = Enumeration("event_type", [
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |