|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v7 COLO 11/18] Support colo mode for qemu disk
From: Wen Congyang <wency@xxxxxxxxxxxxxx> Usage: disk = ['...,colo,colo-params=xxx,active-disk=xxx,hidden-disk=xxx...'] The format of colo-params: host:port:exportname=xx For QEMU block replication details: http://wiki.qemu.org/Features/BlockReplication Signed-off-by: Wen Congyang <wency@xxxxxxxxxxxxxx> Signed-off-by: Yang Hongyang <yanghy@xxxxxxxxxxxxxx> --- docs/man/xl.pod.1 | 2 +- docs/misc/xl-disk-configuration.txt | 38 ++++++ tools/libxl/libxl.c | 42 +++++- tools/libxl/libxl_create.c | 25 +++- tools/libxl/libxl_device.c | 38 ++++++ tools/libxl/libxl_dm.c | 262 ++++++++++++++++++++++++++++++++++-- tools/libxl/libxl_types.idl | 5 + tools/libxl/libxlu_disk_l.l | 5 + 8 files changed, 406 insertions(+), 11 deletions(-) diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1 index 4260c60..600facb 100644 --- a/docs/man/xl.pod.1 +++ b/docs/man/xl.pod.1 @@ -454,7 +454,7 @@ N.B: Remus support in xl is still in experimental (proof-of-concept) phase. Disk replication support is limited to DRBD disks. COLO support in xl is still in experimental (proof-of-concept) phase. - There is no support for network or disk at the moment. + There is no support for network at the moment. B<OPTIONS> diff --git a/docs/misc/xl-disk-configuration.txt b/docs/misc/xl-disk-configuration.txt index 6a2118d..e366e8d 100644 --- a/docs/misc/xl-disk-configuration.txt +++ b/docs/misc/xl-disk-configuration.txt @@ -234,6 +234,44 @@ were intentionally created non-sparse to avoid fragmentation of the file. +=============== +COLO PARAMETERS +=============== + + +colo +---- + +Enable COLO HA for disk. For better understanding block replication on +QEMU, please refer to: +http://wiki.qemu.org/Features/BlockReplication + + +colo-params=host:port:exportname=<name> +--------------------------------------- + +Description: Secondary host's address and port information, + We will run a nbd server on secondary host, + exportname is the nbd server's disk export name. +Mandatory: Yes when COLO enabled + + +active-disk +----------- + +Description: This is used by secondary. Secondary guest's write + will be buffered in this disk. +Mandatory: Yes when COLO enabled + + +hidden-disk +----------- + +Description: This is used by secondary. It buffers the original + content that is modified by the primary VM. +Mandatory: Yes when COLO enabled + + ============================================ DEPRECATED PARAMETERS, PREFIXES AND SYNTAXES ============================================ diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index 08ae7a7..db774e4 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -2241,6 +2241,8 @@ int libxl__device_disk_setdefault(libxl__gc *gc, libxl_device_disk *disk) int rc; libxl_defbool_setdefault(&disk->discard_enable, !!disk->readwrite); + libxl_defbool_setdefault(&disk->colo_enable, false); + libxl_defbool_setdefault(&disk->colo_restore_enable, false); rc = libxl__resolve_domid(gc, disk->backend_domname, &disk->backend_domid); if (rc < 0) return rc; @@ -2441,6 +2443,14 @@ static void device_disk_add(libxl__egc *egc, uint32_t domid, flexarray_append(back, "params"); flexarray_append(back, libxl__sprintf(gc, "%s:%s", libxl__device_disk_string_of_format(disk->format), disk->pdev_path)); + if (libxl_defbool_val(disk->colo_enable)) { + flexarray_append(back, "colo-params"); + flexarray_append(back, libxl__sprintf(gc, "%s", disk->colo_params)); + flexarray_append(back, "active-disk"); + flexarray_append(back, libxl__sprintf(gc, "%s", disk->active_disk)); + flexarray_append(back, "hidden-disk"); + flexarray_append(back, libxl__sprintf(gc, "%s", disk->hidden_disk)); + } assert(device->backend_kind == LIBXL__DEVICE_KIND_QDISK); break; default: @@ -2555,7 +2565,10 @@ static int libxl__device_disk_from_xs_be(libxl__gc *gc, goto cleanup; } - /* "params" may not be present; but everything else must be. */ + /* + * "params" and "colo-params" may not be present; but everything + * else must be. + */ tmp = xs_read(ctx->xsh, XBT_NULL, libxl__sprintf(gc, "%s/params", be_path), &len); if (tmp && strchr(tmp, ':')) { @@ -2565,6 +2578,33 @@ static int libxl__device_disk_from_xs_be(libxl__gc *gc, disk->pdev_path = tmp; } + tmp = xs_read(ctx->xsh, XBT_NULL, + libxl__sprintf(gc, "%s/colo-params", be_path), &len); + if (tmp) { + libxl_defbool_set(&disk->colo_enable, true); + disk->colo_params = tmp; + } else { + libxl_defbool_set(&disk->colo_enable, false); + } + + if (libxl_defbool_val(disk->colo_enable)) { + tmp = xs_read(ctx->xsh, XBT_NULL, + libxl__sprintf(gc, "%s/active-disk", be_path), &len); + if (!tmp) { + LOG(ERROR, "Missing xenstore node %s/active-disk", be_path); + goto cleanup; + } + disk->active_disk = tmp; + + tmp = xs_read(ctx->xsh, XBT_NULL, + libxl__sprintf(gc, "%s/hidden-disk", be_path), &len); + if (!tmp) { + LOG(ERROR, "Missing xenstore node %s/hidden-disk", be_path); + goto cleanup; + } + disk->hidden_disk = tmp; + } + tmp = libxl__xs_read(gc, XBT_NULL, libxl__sprintf(gc, "%s/type", be_path)); diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c index aaa14e3..f7bf629 100644 --- a/tools/libxl/libxl_create.c +++ b/tools/libxl/libxl_create.c @@ -1711,12 +1711,29 @@ static void domain_create_cb(libxl__egc *egc, libxl__ao_complete(egc, ao, rc); } - + +static void set_disk_colo_restore(libxl_domain_config *d_config) +{ + int i; + + for (i = 0; i < d_config->num_disks; i++) + libxl_defbool_set(&d_config->disks[i].colo_restore_enable, true); +} + +static void unset_disk_colo_restore(libxl_domain_config *d_config) +{ + int i; + + for (i = 0; i < d_config->num_disks; i++) + libxl_defbool_set(&d_config->disks[i].colo_restore_enable, false); +} + int libxl_domain_create_new(libxl_ctx *ctx, libxl_domain_config *d_config, uint32_t *domid, const libxl_asyncop_how *ao_how, const libxl_asyncprogress_how *aop_console_how) { + unset_disk_colo_restore(d_config); return do_domain_create(ctx, d_config, domid, -1, -1, NULL, ao_how, aop_console_how); } @@ -1727,6 +1744,12 @@ int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config, const libxl_asyncop_how *ao_how, const libxl_asyncprogress_how *aop_console_how) { + if (params->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_COLO) { + set_disk_colo_restore(d_config); + } else { + unset_disk_colo_restore(d_config); + } + return do_domain_create(ctx, d_config, domid, restore_fd, send_fd, params, ao_how, aop_console_how); } diff --git a/tools/libxl/libxl_device.c b/tools/libxl/libxl_device.c index 93bb41e..df29bc3 100644 --- a/tools/libxl/libxl_device.c +++ b/tools/libxl/libxl_device.c @@ -196,6 +196,10 @@ static int disk_try_backend(disk_try_backend_args *a, goto bad_format; } + if (libxl_defbool_val(a->disk->colo_enable) || + a->disk->active_disk || a->disk->hidden_disk) + goto bad_colo; + if (a->disk->backend_domid != LIBXL_TOOLSTACK_DOMID) { LOG(DEBUG, "Disk vdev=%s, is using a storage driver domain, " "skipping physical device check", a->disk->vdev); @@ -218,6 +222,10 @@ static int disk_try_backend(disk_try_backend_args *a, case LIBXL_DISK_BACKEND_TAP: if (a->disk->script) goto bad_script; + if (libxl_defbool_val(a->disk->colo_enable) || + a->disk->active_disk || a->disk->hidden_disk) + goto bad_colo; + if (a->disk->is_cdrom) { LOG(DEBUG, "Disk vdev=%s, backend tap unsuitable for cdroms", a->disk->vdev); @@ -236,6 +244,16 @@ static int disk_try_backend(disk_try_backend_args *a, case LIBXL_DISK_BACKEND_QDISK: if (a->disk->script) goto bad_script; + if (libxl_defbool_val(a->disk->colo_enable)) { + if (!a->disk->colo_params) + goto bad_colo_params; + + if (!a->disk->active_disk) + goto bad_active_disk; + + if (!a->disk->hidden_disk) + goto bad_hidden_disk; + } return backend; default: @@ -256,6 +274,26 @@ static int disk_try_backend(disk_try_backend_args *a, LOG(DEBUG, "Disk vdev=%s, backend %s not compatible with script=...", a->disk->vdev, libxl_disk_backend_to_string(backend)); return 0; + + bad_colo: + LOG(DEBUG, "Disk vdev=%s, backend %s not compatible with colo", + a->disk->vdev, libxl_disk_backend_to_string(backend)); + return 0; + + bad_colo_params: + LOG(DEBUG, "Disk vdev=%s, backend %s needs colo-params=... for colo", + a->disk->vdev, libxl_disk_backend_to_string(backend)); + return 0; + + bad_active_disk: + LOG(DEBUG, "Disk vdev=%s, backend %s needs active-disk=... for colo", + a->disk->vdev, libxl_disk_backend_to_string(backend)); + return 0; + + bad_hidden_disk: + LOG(DEBUG, "Disk vdev=%s, backend %s needs hidden-disk=... for colo", + a->disk->vdev, libxl_disk_backend_to_string(backend)); + return 0; } int libxl__device_disk_set_backend(libxl__gc *gc, libxl_device_disk *disk) { diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c index 33f9ce6..ac97baa 100644 --- a/tools/libxl/libxl_dm.c +++ b/tools/libxl/libxl_dm.c @@ -427,6 +427,211 @@ static char *dm_spice_options(libxl__gc *gc, return opt; } +/* colo mode */ +enum { + LIBXL__COLO_NONE = 0, + LIBXL__COLO_PRIMARY, + LIBXL__COLO_SECONDARY, +}; + +/* The format of colo-params: host:port:exportname=xx */ +static int parse_colo_params(libxl__gc *gc, const char *colo_params, + const char **host, const char **port, + const char **exportname) +{ + const char *delim; + + delim = strstr(colo_params, ":"); + if (!delim) + return 1; + if (delim == colo_params) + return 1; + *host = libxl__strndup(gc, colo_params, delim - colo_params); + colo_params = delim + 1; + + delim = strstr(colo_params, ":"); + if (!delim) + return 1; + if (delim == colo_params) + return 1; + *port = libxl__strndup(gc, colo_params, delim - colo_params); + colo_params = delim + 1; + + if (strncmp(colo_params, "exportname=", strlen("exportname="))) + return 1; + *exportname = colo_params + strlen("exportname="); + if ((*exportname)[0] == 0) + return 1; + + return 0; +} + +static char *qemu_disk_scsi_drive_string(libxl__gc *gc, const char *pdev_path, + int unit, const char *format, + const libxl_device_disk *disk, + const char *nbd_target, + int colo_mode) +{ + char *drive = NULL; + const char *host = NULL, *port = NULL, *exportname = NULL; + libxl_ctx *ctx = libxl__gc_owner(gc); + const char *colo_params = disk->colo_params; + const char *active_disk = disk->active_disk; + const char *hidden_disk = disk->hidden_disk; + + switch (colo_mode) { + case LIBXL__COLO_NONE: + drive = libxl__sprintf + (gc, "file=%s,if=scsi,bus=0,unit=%d,format=%s,cache=writeback", + pdev_path, unit, format); + break; + case LIBXL__COLO_PRIMARY: + /* + * primary: + * -dirve if=scsi,bus=0,unit=x,cache=writeback,driver=quorum,\ + * children.0.file.filename=pdev_path,\ + * children.0.driver=format,\ + * children.1.file.host=host,\ + * children.1.file.port=port,\ + * children.1.file.export=exportname,\ + * children.1.file.driver=nbd+colo,\ + * children.1.driver=raw,\ + * children.1.ignore-errors=on,\ + * read-pattern=fifo + */ + + if (parse_colo_params(gc, colo_params, &host, &port, &exportname)) + break; + + drive = libxl__sprintf + (gc, "if=scsi,bus=0,unit=%d,cache=writeback,driver=quorum," + "children.0.file.filename=%s," + "children.0.driver=%s," + "children.1.file.host=%s," + "children.1.file.port=%s," + "children.1.file.export=%s," + "children.1.file.driver=nbd+colo," + "children.1.driver=raw," + "children.1.ignore-errors=on," + "read-pattern=fifo", + unit, pdev_path, format, host, port, exportname); + break; + case LIBXL__COLO_SECONDARY: + /* + * secondary: + * -drive if=scsi,bus=0,unit=x,cache=writeback,driver=qcow2+colo,\ + * file=active_disk,\ + * backing_reference.drive_id=nbd_target,\ + * backing_reference.hidden-disk.file.filename=hidden_disk,\ + * backing_reference.hidden-disk.allow-write-backing-file=on,\ + * export=exportname, + */ + + if (parse_colo_params(gc, colo_params, &host, &port, &exportname)) + break; + + drive = libxl__sprintf + (gc, "if=scsi,bus=0,unit=%d,cache=writeback,driver=qcow2+colo," + "file=%s," + "backing_reference.drive_id=%s," + "backing_reference.hidden-disk.file.filename=%s," + "backing_reference.hidden-disk.allow-write-backing-file=on," + "export=%s", + unit, active_disk, nbd_target, hidden_disk, exportname); + break; + default: + abort(); + } + + if (!drive) + LIBXL__LOG(ctx, LIBXL__LOG_WARNING, + "colo-params is invalid for %s", pdev_path); + return drive; +} + +static char *qemu_disk_ide_drive_string(libxl__gc *gc, const char *pdev_path, + int unit, const char *format, + const libxl_device_disk *disk, + const char *nbd_target, + int colo_mode) +{ + char *drive = NULL; + const char *host = NULL, *port = NULL, *exportname = NULL; + libxl_ctx *ctx = libxl__gc_owner(gc); + const char *colo_params = disk->colo_params; + const char *active_disk = disk->active_disk; + const char *hidden_disk = disk->hidden_disk; + + switch (colo_mode) { + case LIBXL__COLO_NONE: + drive = libxl__sprintf + (gc, "file=%s,if=ide,index=%d,media=disk,format=%s,cache=writeback", + pdev_path, unit, format); + break; + case LIBXL__COLO_PRIMARY: + /* + * primary: + * -dirve if=ide,index=x,media=disk,cache=writeback,driver=quorum,\ + * children.0.file.filename=pdev_path,\ + * children.0.driver=format,\ + * children.1.file.host=host,\ + * children.1.file.port=port,\ + * children.1.file.export=exportname,\ + * children.1.file.driver=nbd+colo,\ + * children.1.driver=raw,\ + * children.1.ignored-errors=on,\ + * read-pattern=fifo + */ + + if (parse_colo_params(gc, colo_params, &host, &port, &exportname)) + break; + + drive = libxl__sprintf + (gc, "if=ide,index=%d,media=disk,cache=writeback,driver=quorum," + "children.0.file.filename=%s," + "children.0.driver=%s," + "children.1.file.host=%s," + "children.1.file.port=%s," + "children.1.file.export=%s," + "children.1.file.driver=nbd+colo," + "children.1.driver=raw," + "children.1.ignore-errors=on," + "read-pattern=fifo", + unit, pdev_path, format, host, port, exportname); + break; + case LIBXL__COLO_SECONDARY: + /* + * secondary: + * -drive if=ide,index=x,media=disk,cache=writeback,driver=qcow2+colo,\ + * file=active_disk,\ + * backing_reference.drive_id=nbd_target,\ + * backing_reference.hidden-disk.file.filename=hidden_disk,\ + * backing_reference.hidden-disk.allow-write-backing-file=on,\ + * export=exportname, + */ + + if (parse_colo_params(gc, colo_params, &host, &port, &exportname)) + break; + + drive = libxl__sprintf + (gc, "if=ide,index=%d,media=disk,cache=writeback,driver=qcow2+colo," + "file=%s," + "backing_reference.drive_id=%s," + "backing_reference.hidden-disk.file.filename=%s," + "backing_reference.hidden-disk.allow-write-backing-file=on," + "export=%s", + unit, active_disk, nbd_target, hidden_disk, exportname); + break; + default: + abort(); + } + + if (!drive) + LIBXL__LOG(ctx, LIBXL__LOG_WARNING, + "colo-params is invalid for %s", pdev_path); + return drive; +} + static int libxl__build_device_model_args_new(libxl__gc *gc, const char *dm, int guest_domid, const libxl_domain_config *guest_config, @@ -825,6 +1030,8 @@ static int libxl__build_device_model_args_new(libxl__gc *gc, const char *format = qemu_disk_format_string(disks[i].format); char *drive; const char *pdev_path; + int colo_mode; + char *drive_id; if (dev_number == -1) { LIBXL__LOG(ctx, LIBXL__LOG_WARNING, "unable to determine" @@ -868,16 +1075,55 @@ static int libxl__build_device_model_args_new(libxl__gc *gc, * For other disks we translate devices 0..3 into * hd[a-d] and ignore the rest. */ - if (strncmp(disks[i].vdev, "sd", 2) == 0) - drive = libxl__sprintf - (gc, "file=%s,if=scsi,bus=0,unit=%d,format=%s,cache=writeback", - pdev_path, disk, format); - else if (disk < 4) + if (libxl_defbool_val(disks[i].colo_enable)) { + if (libxl_defbool_val(disks[i].colo_restore_enable)) + colo_mode = LIBXL__COLO_SECONDARY; + else + colo_mode = LIBXL__COLO_PRIMARY; + } else { + colo_mode = LIBXL__COLO_NONE; + } + + if (colo_mode == LIBXL__COLO_SECONDARY) { + /* + * -drive if=none,driver=format,file=pdev_path,\ + * id=nbd_targetx + */ + if (strncmp(disks[i].vdev, "sd", 2) == 0) { + drive_id = libxl__sprintf(gc, "nbd_target%d", disk + 4); + } else if (disk < 4) { + drive_id = libxl__sprintf(gc, "nbd_target%d", disk); + } else { + continue; /* Do not emulate this disk */ + } drive = libxl__sprintf - (gc, "file=%s,if=ide,index=%d,media=disk,format=%s,cache=writeback", - pdev_path, disk, format); - else + (gc, "if=none,driver=%s,file=%s,id=%s", + format, pdev_path, drive_id); + + flexarray_append(dm_args, "-drive"); + flexarray_append(dm_args, drive); + } else { + drive_id = NULL; + } + + if (strncmp(disks[i].vdev, "sd", 2) == 0) { + drive = qemu_disk_scsi_drive_string(gc, pdev_path, disk, + format, + &disks[i], + drive_id, + colo_mode); + } else if (disk < 4) { + drive = qemu_disk_ide_drive_string(gc, pdev_path, disk, + format, + &disks[i], + drive_id, + colo_mode); + } else { continue; /* Do not emulate this disk */ + } + + if (!drive) + continue; } flexarray_append(dm_args, "-drive"); diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index cf1eeb2..9adc3ce 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -517,6 +517,11 @@ libxl_device_disk = Struct("device_disk", [ ("is_cdrom", integer), ("direct_io_safe", bool), ("discard_enable", libxl_defbool), + ("colo_enable", libxl_defbool), + ("colo_restore_enable", libxl_defbool), + ("colo_params", string), + ("active_disk", string), + ("hidden_disk", string) ]) libxl_device_nic = Struct("device_nic", [ diff --git a/tools/libxl/libxlu_disk_l.l b/tools/libxl/libxlu_disk_l.l index 1a5deb5..566aa1e 100644 --- a/tools/libxl/libxlu_disk_l.l +++ b/tools/libxl/libxlu_disk_l.l @@ -176,6 +176,11 @@ script=[^,]*,? { STRIP(','); SAVESTRING("script", script, FROMEQUALS); } direct-io-safe,? { DPC->disk->direct_io_safe = 1; } discard,? { libxl_defbool_set(&DPC->disk->discard_enable, true); } no-discard,? { libxl_defbool_set(&DPC->disk->discard_enable, false); } +colo,? { libxl_defbool_set(&DPC->disk->colo_enable, true); } +no-colo,? { libxl_defbool_set(&DPC->disk->colo_enable, false); } +colo-params=[^,]*,? { STRIP(','); SAVESTRING("colo-params", colo_params, FROMEQUALS); } +active-disk=[^,]*,? { STRIP(','); SAVESTRING("active-disk", active_disk, FROMEQUALS); } +hidden-disk=[^,]*,? { STRIP(','); SAVESTRING("hidden-disk", hidden_disk, FROMEQUALS); } /* the target magic parameter, eats the rest of the string */ -- 1.9.1 _______________________________________________ Xen-devel mailing list Xen-devel@xxxxxxxxxxxxx http://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |