[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH v6 COLO 08/15] Support colo mode for qemu disk



From: Wen Congyang <wency@xxxxxxxxxxxxxx>

Usage: disk = ['...,colo,colo-params=xxx,active-disk=xxx,hidden-disk=xxx...']
The format of colo-params: host:port:exportname=xx

Signed-off-by: Wen Congyang <wency@xxxxxxxxxxxxxx>
Signed-off-by: Yang Hongyang <yanghy@xxxxxxxxxxxxxx>
---
 docs/man/xl.pod.1           |   2 +-
 tools/libxl/libxl.c         |  42 ++++++-
 tools/libxl/libxl_create.c  |  25 ++++-
 tools/libxl/libxl_device.c  |  38 +++++++
 tools/libxl/libxl_dm.c      | 262 ++++++++++++++++++++++++++++++++++++++++++--
 tools/libxl/libxl_types.idl |   5 +
 tools/libxl/libxlu_disk_l.l |   5 +
 7 files changed, 367 insertions(+), 12 deletions(-)

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index f5e97d7..1c2ee24 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -454,7 +454,7 @@ N.B: Remus support in xl is still in experimental 
(proof-of-concept) phase.
      Disk replication support is limited to DRBD disks.
 
      COLO support in xl is still in experimental (proof-of-concept) phase.
-     There is no support for network or disk at the moment.
+     There is no support for network at the moment.
 
 B<OPTIONS>
 
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 7df2466..4a5957c 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -2273,6 +2273,8 @@ int libxl__device_disk_setdefault(libxl__gc *gc, 
libxl_device_disk *disk)
     int rc;
 
     libxl_defbool_setdefault(&disk->discard_enable, !!disk->readwrite);
+    libxl_defbool_setdefault(&disk->colo_enable, false);
+    libxl_defbool_setdefault(&disk->colo_restore_enable, false);
 
     rc = libxl__resolve_domid(gc, disk->backend_domname, &disk->backend_domid);
     if (rc < 0) return rc;
@@ -2473,6 +2475,14 @@ static void device_disk_add(libxl__egc *egc, uint32_t 
domid,
                 flexarray_append(back, "params");
                 flexarray_append(back, libxl__sprintf(gc, "%s:%s",
                               
libxl__device_disk_string_of_format(disk->format), disk->pdev_path));
+                if (libxl_defbool_val(disk->colo_enable)) {
+                    flexarray_append(back, "colo-params");
+                    flexarray_append(back, libxl__sprintf(gc, "%s", 
disk->colo_params));
+                    flexarray_append(back, "active-disk");
+                    flexarray_append(back, libxl__sprintf(gc, "%s", 
disk->active_disk));
+                    flexarray_append(back, "hidden-disk");
+                    flexarray_append(back, libxl__sprintf(gc, "%s", 
disk->hidden_disk));
+                }
                 assert(device->backend_kind == LIBXL__DEVICE_KIND_QDISK);
                 break;
             default:
@@ -2587,7 +2597,10 @@ static int libxl__device_disk_from_xs_be(libxl__gc *gc,
         goto cleanup;
     }
 
-    /* "params" may not be present; but everything else must be. */
+    /*
+     * "params" and "colo-params" may not be present; but everything
+     * else must be.
+     */
     tmp = xs_read(ctx->xsh, XBT_NULL,
                   libxl__sprintf(gc, "%s/params", be_path), &len);
     if (tmp && strchr(tmp, ':')) {
@@ -2597,6 +2610,33 @@ static int libxl__device_disk_from_xs_be(libxl__gc *gc,
         disk->pdev_path = tmp;
     }
 
+    tmp = xs_read(ctx->xsh, XBT_NULL,
+                  libxl__sprintf(gc, "%s/colo-params", be_path), &len);
+    if (tmp) {
+        libxl_defbool_set(&disk->colo_enable, true);
+        disk->colo_params = tmp;
+    } else {
+        libxl_defbool_set(&disk->colo_enable, false);
+    }
+
+    if (libxl_defbool_val(disk->colo_enable)) {
+        tmp = xs_read(ctx->xsh, XBT_NULL,
+                      libxl__sprintf(gc, "%s/active-disk", be_path), &len);
+        if (!tmp) {
+            LOG(ERROR, "Missing xenstore node %s/active-disk", be_path);
+            goto cleanup;
+        }
+        disk->active_disk = tmp;
+
+        tmp = xs_read(ctx->xsh, XBT_NULL,
+                      libxl__sprintf(gc, "%s/hidden-disk", be_path), &len);
+        if (!tmp) {
+            LOG(ERROR, "Missing xenstore node %s/hidden-disk", be_path);
+            goto cleanup;
+        }
+        disk->hidden_disk = tmp;
+    }
+
 
     tmp = libxl__xs_read(gc, XBT_NULL,
                          libxl__sprintf(gc, "%s/type", be_path));
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 6e307f3..17d0d18 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -1727,12 +1727,29 @@ static void domain_create_cb(libxl__egc *egc,
 
     libxl__ao_complete(egc, ao, rc);
 }
-    
+
+static void set_disk_colo_restore(libxl_domain_config *d_config)
+{
+    int i;
+
+    for (i = 0; i < d_config->num_disks; i++)
+        libxl_defbool_set(&d_config->disks[i].colo_restore_enable, true);
+}
+
+static void unset_disk_colo_restore(libxl_domain_config *d_config)
+{
+    int i;
+
+    for (i = 0; i < d_config->num_disks; i++)
+        libxl_defbool_set(&d_config->disks[i].colo_restore_enable, false);
+}
+
 int libxl_domain_create_new(libxl_ctx *ctx, libxl_domain_config *d_config,
                             uint32_t *domid,
                             const libxl_asyncop_how *ao_how,
                             const libxl_asyncprogress_how *aop_console_how)
 {
+    unset_disk_colo_restore(d_config);
     return do_domain_create(ctx, d_config, domid, -1, -1, 0,
                             ao_how, aop_console_how);
 }
@@ -1745,8 +1762,12 @@ int libxl_domain_create_restore(libxl_ctx *ctx, 
libxl_domain_config *d_config,
 {
     int send_fd = -1;
 
-    if (params->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_COLO)
+    if (params->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_COLO) {
         send_fd = params->send_fd;
+        set_disk_colo_restore(d_config);
+    } else {
+        unset_disk_colo_restore(d_config);
+    }
 
     return do_domain_create(ctx, d_config, domid, restore_fd, send_fd,
                             params->checkpointed_stream, ao_how, 
aop_console_how);
diff --git a/tools/libxl/libxl_device.c b/tools/libxl/libxl_device.c
index 93bb41e..df29bc3 100644
--- a/tools/libxl/libxl_device.c
+++ b/tools/libxl/libxl_device.c
@@ -196,6 +196,10 @@ static int disk_try_backend(disk_try_backend_args *a,
             goto bad_format;
         }
 
+        if (libxl_defbool_val(a->disk->colo_enable) ||
+            a->disk->active_disk || a->disk->hidden_disk)
+            goto bad_colo;
+
         if (a->disk->backend_domid != LIBXL_TOOLSTACK_DOMID) {
             LOG(DEBUG, "Disk vdev=%s, is using a storage driver domain, "
                        "skipping physical device check", a->disk->vdev);
@@ -218,6 +222,10 @@ static int disk_try_backend(disk_try_backend_args *a,
     case LIBXL_DISK_BACKEND_TAP:
         if (a->disk->script) goto bad_script;
 
+        if (libxl_defbool_val(a->disk->colo_enable) ||
+            a->disk->active_disk || a->disk->hidden_disk)
+            goto bad_colo;
+
         if (a->disk->is_cdrom) {
             LOG(DEBUG, "Disk vdev=%s, backend tap unsuitable for cdroms",
                        a->disk->vdev);
@@ -236,6 +244,16 @@ static int disk_try_backend(disk_try_backend_args *a,
 
     case LIBXL_DISK_BACKEND_QDISK:
         if (a->disk->script) goto bad_script;
+        if (libxl_defbool_val(a->disk->colo_enable)) {
+            if (!a->disk->colo_params)
+                goto bad_colo_params;
+
+            if (!a->disk->active_disk)
+                goto bad_active_disk;
+
+            if (!a->disk->hidden_disk)
+                goto bad_hidden_disk;
+        }
         return backend;
 
     default:
@@ -256,6 +274,26 @@ static int disk_try_backend(disk_try_backend_args *a,
     LOG(DEBUG, "Disk vdev=%s, backend %s not compatible with script=...",
         a->disk->vdev, libxl_disk_backend_to_string(backend));
     return 0;
+
+ bad_colo:
+    LOG(DEBUG, "Disk vdev=%s, backend %s not compatible with colo",
+        a->disk->vdev, libxl_disk_backend_to_string(backend));
+    return 0;
+
+ bad_colo_params:
+    LOG(DEBUG, "Disk vdev=%s, backend %s needs colo-params=... for colo",
+        a->disk->vdev, libxl_disk_backend_to_string(backend));
+    return 0;
+
+ bad_active_disk:
+    LOG(DEBUG, "Disk vdev=%s, backend %s needs active-disk=... for colo",
+        a->disk->vdev, libxl_disk_backend_to_string(backend));
+    return 0;
+
+ bad_hidden_disk:
+    LOG(DEBUG, "Disk vdev=%s, backend %s needs hidden-disk=... for colo",
+        a->disk->vdev, libxl_disk_backend_to_string(backend));
+    return 0;
 }
 
 int libxl__device_disk_set_backend(libxl__gc *gc, libxl_device_disk *disk) {
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index 33f9ce6..ac97baa 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -427,6 +427,211 @@ static char *dm_spice_options(libxl__gc *gc,
     return opt;
 }
 
+/* colo mode */
+enum {
+    LIBXL__COLO_NONE = 0,
+    LIBXL__COLO_PRIMARY,
+    LIBXL__COLO_SECONDARY,
+};
+
+/* The format of colo-params: host:port:exportname=xx */
+static int parse_colo_params(libxl__gc *gc, const char *colo_params,
+                             const char **host, const char **port,
+                             const char **exportname)
+{
+    const char *delim;
+
+    delim = strstr(colo_params, ":");
+    if (!delim)
+        return 1;
+    if (delim == colo_params)
+        return 1;
+    *host = libxl__strndup(gc, colo_params, delim - colo_params);
+    colo_params = delim + 1;
+
+    delim = strstr(colo_params, ":");
+    if (!delim)
+        return 1;
+    if (delim == colo_params)
+        return 1;
+    *port = libxl__strndup(gc, colo_params, delim - colo_params);
+    colo_params = delim + 1;
+
+    if (strncmp(colo_params, "exportname=", strlen("exportname=")))
+        return 1;
+    *exportname = colo_params + strlen("exportname=");
+    if ((*exportname)[0] == 0)
+        return 1;
+
+    return 0;
+}
+
+static char *qemu_disk_scsi_drive_string(libxl__gc *gc, const char *pdev_path,
+                                         int unit, const char *format,
+                                         const libxl_device_disk *disk,
+                                         const char *nbd_target,
+                                         int colo_mode)
+{
+    char *drive = NULL;
+    const char *host = NULL, *port = NULL, *exportname = NULL;
+    libxl_ctx *ctx = libxl__gc_owner(gc);
+    const char *colo_params = disk->colo_params;
+    const char *active_disk = disk->active_disk;
+    const char *hidden_disk = disk->hidden_disk;
+
+    switch (colo_mode) {
+    case LIBXL__COLO_NONE:
+        drive = libxl__sprintf
+            (gc, "file=%s,if=scsi,bus=0,unit=%d,format=%s,cache=writeback",
+             pdev_path, unit, format);
+        break;
+    case LIBXL__COLO_PRIMARY:
+        /*
+         * primary:
+         *  -dirve if=scsi,bus=0,unit=x,cache=writeback,driver=quorum,\
+         *  children.0.file.filename=pdev_path,\
+         *  children.0.driver=format,\
+         *  children.1.file.host=host,\
+         *  children.1.file.port=port,\
+         *  children.1.file.export=exportname,\
+         *  children.1.file.driver=nbd+colo,\
+         *  children.1.driver=raw,\
+         *  children.1.ignore-errors=on,\
+         *  read-pattern=fifo
+         */
+
+        if (parse_colo_params(gc, colo_params, &host, &port, &exportname))
+            break;
+
+        drive = libxl__sprintf
+            (gc, "if=scsi,bus=0,unit=%d,cache=writeback,driver=quorum,"
+                 "children.0.file.filename=%s,"
+                 "children.0.driver=%s,"
+                 "children.1.file.host=%s,"
+                 "children.1.file.port=%s,"
+                 "children.1.file.export=%s,"
+                 "children.1.file.driver=nbd+colo,"
+                 "children.1.driver=raw,"
+                 "children.1.ignore-errors=on,"
+                 "read-pattern=fifo",
+             unit, pdev_path, format, host, port, exportname);
+        break;
+    case LIBXL__COLO_SECONDARY:
+        /*
+         * secondary:
+         *  -drive if=scsi,bus=0,unit=x,cache=writeback,driver=qcow2+colo,\
+         *  file=active_disk,\
+         *  backing_reference.drive_id=nbd_target,\
+         *  backing_reference.hidden-disk.file.filename=hidden_disk,\
+         *  backing_reference.hidden-disk.allow-write-backing-file=on,\
+         *  export=exportname,
+         */
+
+        if (parse_colo_params(gc, colo_params, &host, &port, &exportname))
+            break;
+
+        drive = libxl__sprintf
+            (gc, "if=scsi,bus=0,unit=%d,cache=writeback,driver=qcow2+colo,"
+                 "file=%s,"
+                 "backing_reference.drive_id=%s,"
+                 "backing_reference.hidden-disk.file.filename=%s,"
+                 "backing_reference.hidden-disk.allow-write-backing-file=on,"
+                 "export=%s",
+             unit, active_disk, nbd_target, hidden_disk, exportname);
+        break;
+    default:
+        abort();
+    }
+
+    if (!drive)
+        LIBXL__LOG(ctx, LIBXL__LOG_WARNING,
+                   "colo-params is invalid for %s", pdev_path);
+    return drive;
+}
+
+static char *qemu_disk_ide_drive_string(libxl__gc *gc, const char *pdev_path,
+                                        int unit, const char *format,
+                                        const libxl_device_disk *disk,
+                                        const char *nbd_target,
+                                        int colo_mode)
+{
+    char *drive = NULL;
+    const char *host = NULL, *port = NULL, *exportname = NULL;
+    libxl_ctx *ctx = libxl__gc_owner(gc);
+    const char *colo_params = disk->colo_params;
+    const char *active_disk = disk->active_disk;
+    const char *hidden_disk = disk->hidden_disk;
+
+    switch (colo_mode) {
+    case LIBXL__COLO_NONE:
+        drive = libxl__sprintf
+            (gc, 
"file=%s,if=ide,index=%d,media=disk,format=%s,cache=writeback",
+             pdev_path, unit, format);
+        break;
+    case LIBXL__COLO_PRIMARY:
+        /*
+         * primary:
+         *  -dirve if=ide,index=x,media=disk,cache=writeback,driver=quorum,\
+         *  children.0.file.filename=pdev_path,\
+         *  children.0.driver=format,\
+         *  children.1.file.host=host,\
+         *  children.1.file.port=port,\
+         *  children.1.file.export=exportname,\
+         *  children.1.file.driver=nbd+colo,\
+         *  children.1.driver=raw,\
+         *  children.1.ignored-errors=on,\
+         *  read-pattern=fifo
+         */
+
+        if (parse_colo_params(gc, colo_params, &host, &port, &exportname))
+            break;
+
+        drive = libxl__sprintf
+            (gc, "if=ide,index=%d,media=disk,cache=writeback,driver=quorum,"
+                 "children.0.file.filename=%s,"
+                 "children.0.driver=%s,"
+                 "children.1.file.host=%s,"
+                 "children.1.file.port=%s,"
+                 "children.1.file.export=%s,"
+                 "children.1.file.driver=nbd+colo,"
+                 "children.1.driver=raw,"
+                 "children.1.ignore-errors=on,"
+                 "read-pattern=fifo",
+             unit, pdev_path, format, host, port, exportname);
+        break;
+    case LIBXL__COLO_SECONDARY:
+        /*
+         * secondary:
+         *  -drive 
if=ide,index=x,media=disk,cache=writeback,driver=qcow2+colo,\
+         *  file=active_disk,\
+         *  backing_reference.drive_id=nbd_target,\
+         *  backing_reference.hidden-disk.file.filename=hidden_disk,\
+         *  backing_reference.hidden-disk.allow-write-backing-file=on,\
+         *  export=exportname,
+         */
+
+        if (parse_colo_params(gc, colo_params, &host, &port, &exportname))
+            break;
+
+        drive = libxl__sprintf
+            (gc, 
"if=ide,index=%d,media=disk,cache=writeback,driver=qcow2+colo,"
+                 "file=%s,"
+                 "backing_reference.drive_id=%s,"
+                 "backing_reference.hidden-disk.file.filename=%s,"
+                 "backing_reference.hidden-disk.allow-write-backing-file=on,"
+                 "export=%s",
+             unit, active_disk, nbd_target, hidden_disk, exportname);
+        break;
+    default:
+        abort();
+    }
+
+    if (!drive)
+        LIBXL__LOG(ctx, LIBXL__LOG_WARNING,
+                   "colo-params is invalid for %s", pdev_path);
+    return drive;
+}
+
 static int libxl__build_device_model_args_new(libxl__gc *gc,
                                         const char *dm, int guest_domid,
                                         const libxl_domain_config 
*guest_config,
@@ -825,6 +1030,8 @@ static int libxl__build_device_model_args_new(libxl__gc 
*gc,
             const char *format = qemu_disk_format_string(disks[i].format);
             char *drive;
             const char *pdev_path;
+            int colo_mode;
+            char *drive_id;
 
             if (dev_number == -1) {
                 LIBXL__LOG(ctx, LIBXL__LOG_WARNING, "unable to determine"
@@ -868,16 +1075,55 @@ static int libxl__build_device_model_args_new(libxl__gc 
*gc,
                  * For other disks we translate devices 0..3 into
                  * hd[a-d] and ignore the rest.
                  */
-                if (strncmp(disks[i].vdev, "sd", 2) == 0)
-                    drive = libxl__sprintf
-                        (gc, 
"file=%s,if=scsi,bus=0,unit=%d,format=%s,cache=writeback",
-                         pdev_path, disk, format);
-                else if (disk < 4)
+                if (libxl_defbool_val(disks[i].colo_enable)) {
+                    if (libxl_defbool_val(disks[i].colo_restore_enable))
+                        colo_mode = LIBXL__COLO_SECONDARY;
+                    else
+                        colo_mode = LIBXL__COLO_PRIMARY;
+                } else {
+                    colo_mode = LIBXL__COLO_NONE;
+                }
+
+                if (colo_mode == LIBXL__COLO_SECONDARY) {
+                    /*
+                     * -drive if=none,driver=format,file=pdev_path,\
+                     * id=nbd_targetx
+                     */
+                    if (strncmp(disks[i].vdev, "sd", 2) == 0) {
+                        drive_id = libxl__sprintf(gc, "nbd_target%d", disk + 
4);
+                    } else if (disk < 4) {
+                        drive_id = libxl__sprintf(gc, "nbd_target%d", disk);
+                    } else {
+                        continue; /* Do not emulate this disk */
+                    }
                     drive = libxl__sprintf
-                        (gc, 
"file=%s,if=ide,index=%d,media=disk,format=%s,cache=writeback",
-                         pdev_path, disk, format);
-                else
+                        (gc, "if=none,driver=%s,file=%s,id=%s",
+                         format, pdev_path, drive_id);
+
+                    flexarray_append(dm_args, "-drive");
+                    flexarray_append(dm_args, drive);
+                } else {
+                    drive_id = NULL;
+                }
+
+                if (strncmp(disks[i].vdev, "sd", 2) == 0) {
+                    drive = qemu_disk_scsi_drive_string(gc, pdev_path, disk,
+                                                        format,
+                                                        &disks[i],
+                                                        drive_id,
+                                                        colo_mode);
+                } else if (disk < 4) {
+                    drive = qemu_disk_ide_drive_string(gc, pdev_path, disk,
+                                                       format,
+                                                       &disks[i],
+                                                       drive_id,
+                                                       colo_mode);
+                } else {
                     continue; /* Do not emulate this disk */
+                }
+
+                if (!drive)
+                    continue;
             }
 
             flexarray_append(dm_args, "-drive");
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 7f07f8b..1e6b5ae 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -516,6 +516,11 @@ libxl_device_disk = Struct("device_disk", [
     ("is_cdrom", integer),
     ("direct_io_safe", bool),
     ("discard_enable", libxl_defbool),
+    ("colo_enable", libxl_defbool),
+    ("colo_restore_enable", libxl_defbool),
+    ("colo_params", string),
+    ("active_disk", string),
+    ("hidden_disk", string)
     ])
 
 libxl_device_nic = Struct("device_nic", [
diff --git a/tools/libxl/libxlu_disk_l.l b/tools/libxl/libxlu_disk_l.l
index 1a5deb5..566aa1e 100644
--- a/tools/libxl/libxlu_disk_l.l
+++ b/tools/libxl/libxlu_disk_l.l
@@ -176,6 +176,11 @@ script=[^,]*,?     { STRIP(','); SAVESTRING("script", 
script, FROMEQUALS); }
 direct-io-safe,? { DPC->disk->direct_io_safe = 1; }
 discard,?      { libxl_defbool_set(&DPC->disk->discard_enable, true); }
 no-discard,?   { libxl_defbool_set(&DPC->disk->discard_enable, false); }
+colo,?         { libxl_defbool_set(&DPC->disk->colo_enable, true); }
+no-colo,?      { libxl_defbool_set(&DPC->disk->colo_enable, false); }
+colo-params=[^,]*,?    { STRIP(','); SAVESTRING("colo-params", colo_params, 
FROMEQUALS); }
+active-disk=[^,]*,?    { STRIP(','); SAVESTRING("active-disk", active_disk, 
FROMEQUALS); }
+hidden-disk=[^,]*,?    { STRIP(','); SAVESTRING("hidden-disk", hidden_disk, 
FROMEQUALS); }
 
  /* the target magic parameter, eats the rest of the string */
 
-- 
1.9.1


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.