|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] Re: [Xen-devel] [PATCH v13.1 20/26] Support colo mode for qemu disk
Changlong Xie writes ("[PATCH v13.1 20/26] Support colo mode for qemu disk"):
> From 468ff9fb2f6699314c28f30a7d7d09eac9aa6756 Mon Sep 17 00:00:00 2001
> From: Wen Congyang <wency@xxxxxxxxxxxxxx>
> Date: Mon, 21 Mar 2016 15:38:30 +0800
> Subject: [PATCH v13.1 20/26] Support colo mode for qemu disk
I found a proper copy of this in:
https://github.com/Pating/xen.git#b226be1b0751
That commit is
Acked-by: Ian Jackson <ian.jackson@xxxxxxxxxxxxx>
Ian.
From b226be1b0751d675dcd604c4a1dbeec5757bb051 Mon Sep 17 00:00:00 2001
From: Wen Congyang <wency@xxxxxxxxxxxxxx>
Date: Mon, 21 Mar 2016 15:38:30 +0800
Subject: [PATCH 21/28] Support colo mode for qemu disk
Usage: disk =
['...,colo,colo-host=xxx,colo-port=xxx,colo-export=xxx,active-disk=xxx,hidden-disk=xxx...']
For QEMU block replication details:
http://wiki.qemu.org/Features/BlockReplication
Note: we just introduce COLO framework, but don't implement COLO
operations in this patch.
Signed-off-by: Wen Congyang <wency@xxxxxxxxxxxxxx>
Signed-off-by: Yang Hongyang <hongyang.yang@xxxxxxxxxxxx>
Signed-off-by: Changlong Xie <xiecl.fnst@xxxxxxxxxxxxxx>
---
docs/man/xl.pod.1 | 41 +++++++++++--
docs/misc/xl-disk-configuration.txt | 55 +++++++++++++++++
tools/libxl/libxl.c | 51 +++++++++++++++-
tools/libxl/libxl_create.c | 26 +++++++-
tools/libxl/libxl_device.c | 11 ++++
tools/libxl/libxl_dm.c | 113 ++++++++++++++++++++++++++++++++++-
tools/libxl/libxl_types.idl | 9 +++
tools/libxl/libxlu_disk_l.l | 19 ++++++
8 files changed, 317 insertions(+), 8 deletions(-)
diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index a992a45..2664402 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -450,12 +450,43 @@ Print huge (!) amount of debug during the migration
process.
Enable Remus HA or COLO HA for domain. By default B<xl> relies on ssh as a
transport mechanism between the two hosts.
-N.B: Remus support in xl is still in experimental (proof-of-concept) phase.
- Disk replication support is limited to DRBD disks.
+B<NOTES>
+
+=over 4
+
+Remus support in xl is still in experimental (proof-of-concept) phase.
+Disk replication support is limited to DRBD disks.
+
+COLO support in xl is still in experimental (proof-of-concept) phase.
+There is no support for network, so the guest will confuse its network
+peers at the moment.
- COLO support in xl is still in experimental (proof-of-concept) phase.
- There is no support for network or disk, so the guest will corrupt its
- disk and confuse its network peers at the moment.
+=back
+
+B<EXAMPLE>
+
+=over 4
+
+(a) An example for COLO replication's configuration: disk =['...,colo,colo-host
+=xxx,colo-port=xxx,colo-export=xxx,active-disk=xxx,hidden-disk=xxx...']
+
+=item B<colo-host> :Secondary host's ip address.
+
+=item B<colo-port> :Secondary host's port, we will run a nbd server on
+secondary host, and the nbd server will listen this port.
+
+=item B<colo-export> :Nbd server's disk export name of secondary host.
+
+=item B<active-disk> :Secondary's guest write will be buffered in this disk,
+and it's used by secondary.
+
+=item B<hidden-disk> :Primary's modified contents will be buffered in this
+disk, and it's used by secondary.
+
+Note that the COLO configuration settings should be considered unstable. They
+may change incompatibly in future versions of Xen.
+
+=back
B<OPTIONS>
diff --git a/docs/misc/xl-disk-configuration.txt
b/docs/misc/xl-disk-configuration.txt
index 29f6ddb..b3402bc 100644
--- a/docs/misc/xl-disk-configuration.txt
+++ b/docs/misc/xl-disk-configuration.txt
@@ -234,6 +234,61 @@ were intentionally created non-sparse to avoid
fragmentation of the
file.
+===============
+COLO PARAMETERS
+===============
+
+
+colo
+----
+
+Enable COLO HA for disk. For better understanding block replication on
+QEMU, please refer to:
+http://wiki.qemu.org/Features/BlockReplication
+Note that the COLO configuration settings should be considered unstable.
+They may change incompatibly in future versions of Xen.
+
+
+colo-host
+---------
+
+Description: Secondary host's address
+Mandatory: Yes when COLO enabled
+
+
+colo-port
+---------
+
+Description: Secondary port
+ We will run a nbd server on secondary host,
+ and the nbd server will listen this port.
+Mandatory: Yes when COLO enabled
+
+
+colo-export
+-----------
+
+Description: We will run a nbd server on secondary host,
+ exportname is the nbd server's disk export name.
+Mandatory: Yes when COLO enabled
+
+
+active-disk
+-----------
+
+Description: This is used by secondary. Secondary guest's write
+ will be buffered in this disk.
+Mandatory: Yes when COLO enabled
+
+
+hidden-disk
+-----------
+
+Description: This is used by secondary. It buffers the original
+ content that is modified by the primary VM.
+Mandatory: Yes when COLO enabled
+
+
============================================
DEPRECATED PARAMETERS, PREFIXES AND SYNTAXES
============================================
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 349a3c6..63fbe16 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -2306,6 +2306,8 @@ int libxl__device_disk_setdefault(libxl__gc *gc,
libxl_device_disk *disk)
int rc;
libxl_defbool_setdefault(&disk->discard_enable, !!disk->readwrite);
+ libxl_defbool_setdefault(&disk->colo_enable, false);
+ libxl_defbool_setdefault(&disk->colo_restore_enable, false);
rc = libxl__resolve_domid(gc, disk->backend_domname, &disk->backend_domid);
if (rc < 0) return rc;
@@ -2504,6 +2506,18 @@ static void device_disk_add(libxl__egc *egc, uint32_t
domid,
flexarray_append(back, "params");
flexarray_append(back, GCSPRINTF("%s:%s",
libxl__device_disk_string_of_format(disk->format), disk->pdev_path));
+ if (libxl_defbool_val(disk->colo_enable)) {
+ flexarray_append(back, "colo-host");
+ flexarray_append(back, libxl__sprintf(gc, "%s",
disk->colo_host));
+ flexarray_append(back, "colo-port");
+ flexarray_append(back, libxl__sprintf(gc, "%d",
disk->colo_port));
+ flexarray_append(back, "colo-export");
+ flexarray_append(back, libxl__sprintf(gc, "%s",
disk->colo_export));
+ flexarray_append(back, "active-disk");
+ flexarray_append(back, libxl__sprintf(gc, "%s",
disk->active_disk));
+ flexarray_append(back, "hidden-disk");
+ flexarray_append(back, libxl__sprintf(gc, "%s",
disk->hidden_disk));
+ }
assert(device->backend_kind == LIBXL__DEVICE_KIND_QDISK);
break;
default:
@@ -2619,7 +2633,12 @@ static int libxl__device_disk_from_xs_be(libxl__gc *gc,
goto cleanup;
}
- /* "params" may not be present; but everything else must be. */
+ /*
+ * "params" may not be present; but everything else must be.
+ * colo releated entries(colo-host, colo-port, colo-export,
+ * active-disk and hidden-disk) are present only if colo is
+ * enabled.
+ */
tmp = xs_read(ctx->xsh, XBT_NULL,
GCSPRINTF("%s/params", be_path), &len);
if (tmp && strchr(tmp, ':')) {
@@ -2629,6 +2648,36 @@ static int libxl__device_disk_from_xs_be(libxl__gc *gc,
disk->pdev_path = tmp;
}
+ tmp = xs_read(ctx->xsh, XBT_NULL,
+ GCSPRINTF("%s/colo-host", be_path), &len);
+ if (tmp) {
+ libxl_defbool_set(&disk->colo_enable, true);
+ disk->colo_host = tmp;
+
+ tmp = xs_read(ctx->xsh, XBT_NULL,
+ GCSPRINTF("%s/colo-port", be_path), &len);
+ if (!tmp) {
+ LOG(ERROR, "Missing xenstore node %s/colo-port", be_path);
+ goto cleanup;
+ }
+ disk->colo_port = atoi(tmp);
+
+#define XS_READ_COLO(param, item) do { \
+ tmp = xs_read(ctx->xsh, XBT_NULL, \
+ GCSPRINTF("%s/"#param"", be_path), &len); \
+ if (!tmp) { \
+ LOG(ERROR, "Missing xenstore node %s/"#param"", be_path); \
+ goto cleanup; \
+ } \
+ disk->item = tmp; \
+} while (0)
+ XS_READ_COLO(colo-export, colo_export);
+ XS_READ_COLO(active-disk, active_disk);
+ XS_READ_COLO(hidden-disk, hidden_disk);
+#undef XS_READ_COLO
+ } else {
+ libxl_defbool_set(&disk->colo_enable, false);
+ }
tmp = libxl__xs_read(gc, XBT_NULL,
GCSPRINTF("%s/type", be_path));
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index be604e5..e2ec25c 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -1876,12 +1876,30 @@ static void domain_create_cb(libxl__egc *egc,
libxl__ao_complete(egc, ao, rc);
}
-
+
+
+static void set_disk_colo_restore(libxl_domain_config *d_config)
+{
+ int i;
+
+ for (i = 0; i < d_config->num_disks; i++)
+ libxl_defbool_set(&d_config->disks[i].colo_restore_enable, true);
+}
+
+static void unset_disk_colo_restore(libxl_domain_config *d_config)
+{
+ int i;
+
+ for (i = 0; i < d_config->num_disks; i++)
+ libxl_defbool_set(&d_config->disks[i].colo_restore_enable, false);
+}
+
int libxl_domain_create_new(libxl_ctx *ctx, libxl_domain_config *d_config,
uint32_t *domid,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
{
+ unset_disk_colo_restore(d_config);
return do_domain_create(ctx, d_config, domid, -1, -1, NULL,
ao_how, aop_console_how);
}
@@ -1893,6 +1911,12 @@ int libxl_domain_create_restore(libxl_ctx *ctx,
libxl_domain_config *d_config,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
{
+ if (params->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_COLO) {
+ set_disk_colo_restore(d_config);
+ } else {
+ unset_disk_colo_restore(d_config);
+ }
+
return do_domain_create(ctx, d_config, domid, restore_fd, send_back_fd,
params, ao_how, aop_console_how);
}
diff --git a/tools/libxl/libxl_device.c b/tools/libxl/libxl_device.c
index 4ced9b6..6a411c6 100644
--- a/tools/libxl/libxl_device.c
+++ b/tools/libxl/libxl_device.c
@@ -196,6 +196,9 @@ static int disk_try_backend(disk_try_backend_args *a,
goto bad_format;
}
+ if (libxl_defbool_val(a->disk->colo_enable))
+ goto bad_colo;
+
if (a->disk->backend_domid != LIBXL_TOOLSTACK_DOMID) {
LOG(DEBUG, "Disk vdev=%s, is using a storage driver domain, "
"skipping physical device check", a->disk->vdev);
@@ -218,6 +221,9 @@ static int disk_try_backend(disk_try_backend_args *a,
case LIBXL_DISK_BACKEND_TAP:
if (a->disk->script) goto bad_script;
+ if (libxl_defbool_val(a->disk->colo_enable))
+ goto bad_colo;
+
if (a->disk->is_cdrom) {
LOG(DEBUG, "Disk vdev=%s, backend tap unsuitable for cdroms",
a->disk->vdev);
@@ -256,6 +262,11 @@ static int disk_try_backend(disk_try_backend_args *a,
LOG(DEBUG, "Disk vdev=%s, backend %s not compatible with script=...",
a->disk->vdev, libxl_disk_backend_to_string(backend));
return 0;
+
+ bad_colo:
+ LOG(DEBUG, "Disk vdev=%s, backend %s not compatible with colo",
+ a->disk->vdev, libxl_disk_backend_to_string(backend));
+ return 0;
}
int libxl__device_disk_set_backend(libxl__gc *gc, libxl_device_disk *disk) {
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index 1d1b25b..4c3dff8 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -754,6 +754,8 @@ static int libxl__dm_runas_helper(libxl__gc *gc, const char
*username)
/* colo mode */
enum {
LIBXL__COLO_NONE = 0,
+ LIBXL__COLO_PRIMARY,
+ LIBXL__COLO_SECONDARY,
};
static char *qemu_disk_scsi_drive_string(libxl__gc *gc, const char *pdev_path,
@@ -762,6 +764,9 @@ static char *qemu_disk_scsi_drive_string(libxl__gc *gc,
const char *pdev_path,
int colo_mode)
{
char *drive = NULL;
+ const char *exportname = disk->colo_export;
+ const char *active_disk = disk->active_disk;
+ const char *hidden_disk = disk->hidden_disk;
switch (colo_mode) {
case LIBXL__COLO_NONE:
@@ -769,6 +774,45 @@ static char *qemu_disk_scsi_drive_string(libxl__gc *gc,
const char *pdev_path,
(gc, "file=%s,if=scsi,bus=0,unit=%d,format=%s,cache=writeback",
pdev_path, unit, format);
break;
+ case LIBXL__COLO_PRIMARY:
+ /*
+ * primary:
+ * -dirve if=scsi,bus=0,unit=x,cache=writeback,driver=quorum,\
+ * id=exportname,\
+ * children.0.file.filename=pdev_path,\
+ * children.0.driver=format,\
+ * read-pattern=fifo,\
+ * vote-threshold=1
+ */
+ drive = GCSPRINTF(
+ "if=scsi,bus=0,unit=%d,cache=writeback,driver=quorum,"
+ "id=%s,"
+ "children.0.file.filename=%s,"
+ "children.0.driver=%s,"
+ "read-pattern=fifo,"
+ "vote-threshold=1",
+ unit, exportname, pdev_path, format);
+ break;
+ case LIBXL__COLO_SECONDARY:
+ /*
+ * secondary:
+ * -drive if=scsi,bus=0,unit=x,cache=writeback,driver=replication,\
+ * mode=secondary,\
+ * file.driver=qcow2,\
+ * file.file.filename=active_disk,\
+ * file.backing.driver=qcow2,\
+ * file.backing.file.filename=hidden_disk,\
+ * file.backing.backing=exportname,
+ */
+ drive = GCSPRINTF(
+ "if=scsi,bus=0,unit=%d,cache=writeback,driver=replication,"
+ "mode=secondary,"
+ "file.driver=qcow2,"
+ "file.file.filename=%s,"
+ "file.backing.driver=qcow2,"
+ "file.backing.file.filename=%s,"
+ "file.backing.backing=%s",
+ unit, active_disk, hidden_disk, exportname);
default:
abort();
}
@@ -782,6 +826,9 @@ static char *qemu_disk_ide_drive_string(libxl__gc *gc,
const char *pdev_path,
int colo_mode)
{
char *drive = NULL;
+ const char *exportname = disk->colo_export;
+ const char *active_disk = disk->active_disk;
+ const char *hidden_disk = disk->hidden_disk;
switch (colo_mode) {
case LIBXL__COLO_NONE:
@@ -789,6 +836,46 @@ static char *qemu_disk_ide_drive_string(libxl__gc *gc,
const char *pdev_path,
("file=%s,if=ide,index=%d,media=disk,format=%s,cache=writeback",
pdev_path, unit, format);
break;
+ case LIBXL__COLO_PRIMARY:
+ /*
+ * primary:
+ * -dirve if=ide,index=x,media=disk,cache=writeback,driver=quorum,\
+ * id=exportname,\
+ * children.0.file.filename=pdev_path,\
+ * children.0.driver=format,\
+ * read-pattern=fifo,\
+ * vote-threshold=1
+ */
+ drive = GCSPRINTF(
+ "if=ide,index=%d,media=disk,cache=writeback,driver=quorum,"
+ "id=%s,"
+ "children.0.file.filename=%s,"
+ "children.0.driver=%s,"
+ "read-pattern=fifo,"
+ "vote-threshold=1",
+ unit, exportname, pdev_path, format);
+ break;
+ case LIBXL__COLO_SECONDARY:
+ /*
+ * secondary:
+ * -drive
if=ide,index=x,media=disk,cache=writeback,driver=replication,\
+ * mode=secondary,\
+ * file.driver=qcow2,\
+ * file.file.filename=active_disk,\
+ * file.backing.driver=qcow2,\
+ * file.backing.file.filename=hidden_disk,\
+ * file.backing.backing=exportname,
+ */
+ drive = GCSPRINTF(
+ "if=ide,index=%d,media=disk,cache=writeback,driver=replication,"
+ "mode=secondary,"
+ "file.driver=qcow2,"
+ "file.file.filename=%s,"
+ "file.backing.driver=qcow2,"
+ "file.backing.file.filename=%s,"
+ "file.backing.backing=%s",
+ unit, active_disk, hidden_disk, exportname);
+ break;
default:
abort();
}
@@ -1261,8 +1348,24 @@ static int libxl__build_device_model_args_new(libxl__gc
*gc,
* hd[a-d] and ignore the rest.
*/
- colo_mode = LIBXL__COLO_NONE;
+ if (libxl_defbool_val(disks[i].colo_enable)) {
+ if (libxl_defbool_val(disks[i].colo_restore_enable))
+ colo_mode = LIBXL__COLO_SECONDARY;
+ else
+ colo_mode = LIBXL__COLO_PRIMARY;
+ } else {
+ colo_mode = LIBXL__COLO_NONE;
+ }
+
if (strncmp(disks[i].vdev, "sd", 2) == 0) {
+ if (colo_mode == LIBXL__COLO_SECONDARY) {
+ drive = libxl__sprintf
+ (gc, "if=none,driver=%s,file=%s,id=%s",
+ format, pdev_path, disks[i].colo_export);
+
+ flexarray_append(dm_args, "-drive");
+ flexarray_append(dm_args, drive);
+ }
drive = qemu_disk_scsi_drive_string(gc, pdev_path, disk,
format,
&disks[i],
@@ -1289,6 +1392,14 @@ static int libxl__build_device_model_args_new(libxl__gc
*gc,
LOG(ERROR, "qemu-xen doesn't support read-only IDE
disk drivers");
return ERROR_INVAL;
}
+ if (colo_mode == LIBXL__COLO_SECONDARY) {
+ drive = libxl__sprintf
+ (gc, "if=none,driver=%s,file=%s,id=%s",
+ format, pdev_path, disks[i].colo_export);
+
+ flexarray_append(dm_args, "-drive");
+ flexarray_append(dm_args, drive);
+ }
drive = qemu_disk_ide_drive_string(gc, pdev_path, disk,
format,
&disks[i],
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 95efd82..8335291 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -576,6 +576,15 @@ libxl_device_disk = Struct("device_disk", [
("is_cdrom", integer),
("direct_io_safe", bool),
("discard_enable", libxl_defbool),
+ # Note that the COLO configuration settings should be considered unstable.
+ # They may change incompatibly in future versions of Xen.
+ ("colo_enable", libxl_defbool),
+ ("colo_restore_enable", libxl_defbool),
+ ("colo_host", string),
+ ("colo_port", integer),
+ ("colo_export", string),
+ ("active_disk", string),
+ ("hidden_disk", string)
])
libxl_device_nic = Struct("device_nic", [
diff --git a/tools/libxl/libxlu_disk_l.l b/tools/libxl/libxlu_disk_l.l
index 1a5deb5..5b6db22 100644
--- a/tools/libxl/libxlu_disk_l.l
+++ b/tools/libxl/libxlu_disk_l.l
@@ -113,6 +113,16 @@ static void setbackendtype(DiskParseContext *dpc, const
char *str) {
else xlu__disk_err(dpc,str,"unknown value for backendtype");
}
+/* Sets ->colo-port from the string. COLO need this. */
+static void setcoloport(DiskParseContext *dpc, const char *str) {
+ int port = atoi(str);
+ if (port) {
+ dpc->disk->colo_port = port;
+ } else {
+ xlu__disk_err(dpc,str,"unknown value for colo_port");
+ }
+}
+
#define DEPRECATE(usewhatinstead) /* not currently reported */
/* Handles a vdev positional parameter which includes a devtype. */
@@ -176,6 +186,15 @@ script=[^,]*,? { STRIP(','); SAVESTRING("script",
script, FROMEQUALS); }
direct-io-safe,? { DPC->disk->direct_io_safe = 1; }
discard,? { libxl_defbool_set(&DPC->disk->discard_enable, true); }
no-discard,? { libxl_defbool_set(&DPC->disk->discard_enable, false); }
+ /* Note that the COLO configuration settings should be considered unstable.
+ * They may change incompatibly in future versions of Xen. */
+colo,? { libxl_defbool_set(&DPC->disk->colo_enable, true); }
+no-colo,? { libxl_defbool_set(&DPC->disk->colo_enable, false); }
+colo-host=[^,]*,? { STRIP(','); SAVESTRING("colo-host", colo_host,
FROMEQUALS); }
+colo-port=[^,]*,? { STRIP(','); setcoloport(DPC, FROMEQUALS); }
+colo-export=[^,]*,? { STRIP(','); SAVESTRING("colo-export", colo_export,
FROMEQUALS); }
+active-disk=[^,]*,? { STRIP(','); SAVESTRING("active-disk", active_disk,
FROMEQUALS); }
+hidden-disk=[^,]*,? { STRIP(','); SAVESTRING("hidden-disk", hidden_disk,
FROMEQUALS); }
/* the target magic parameter, eats the rest of the string */
--
1.7.10.4
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |