|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v13.1 20/26] Support colo mode for qemu disk
From 468ff9fb2f6699314c28f30a7d7d09eac9aa6756 Mon Sep 17 00:00:00 2001 From: Wen Congyang <wency@xxxxxxxxxxxxxx> Date: Mon, 21 Mar 2016 15:38:30 +0800 Subject: [PATCH v13.1 20/26] Support colo mode for qemu diskUsage: disk = ['...,colo,colo-host=xxx,colo-port=xxx,colo-export=xxx,active-disk=xxx,hidden-disk=xxx...'] For QEMU block replication details: http://wiki.qemu.org/Features/BlockReplication Note: we just introduce COLO framework, but don't implement COLO operations in this patch. Signed-off-by: Wen Congyang <wency@xxxxxxxxxxxxxx> Signed-off-by: Yang Hongyang <hongyang.yang@xxxxxxxxxxxx> Signed-off-by: Changlong Xie <xiecl.fnst@xxxxxxxxxxxxxx> --- docs/man/xl.pod.1 | 41 +++++++++++-- docs/misc/xl-disk-configuration.txt | 55 ++++++++++++++++++ tools/libxl/libxl.c | 51 +++++++++++++++- tools/libxl/libxl_create.c | 26 ++++++++- tools/libxl/libxl_device.c | 11 ++++tools/libxl/libxl_dm.c | 113 +++++++++++++++++++++++++++++++++++- tools/libxl/libxl_types.idl | 9 +++ tools/libxl/libxlu_disk_l.l | 19 ++++++ 8 files changed, 317 insertions(+), 8 deletions(-) diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1 index a992a45..2664402 100644 --- a/docs/man/xl.pod.1 +++ b/docs/man/xl.pod.1@@ -450,12 +450,43 @@ Print huge (!) amount of debug during the migration process. Enable Remus HA or COLO HA for domain. By default B<xl> relies on ssh as a transport mechanism between the two hosts. -N.B: Remus support in xl is still in experimental (proof-of-concept) phase. - Disk replication support is limited to DRBD disks. +B<NOTES> + +=over 4 + +Remus support in xl is still in experimental (proof-of-concept) phase. +Disk replication support is limited to DRBD disks. + +COLO support in xl is still in experimental (proof-of-concept) phase. +There is no support for network, so the guest will confuse its network +peers at the moment. - COLO support in xl is still in experimental (proof-of-concept) phase. - There is no support for network or disk, so the guest will corrupt its - disk and confuse its network peers at the moment. +=back + +B<EXAMPLE> + +=over 4 ++(a) An example for COLO replication's configuration: disk =['...,colo,colo-host +=xxx,colo-port=xxx,colo-export=xxx,active-disk=xxx,hidden-disk=xxx...'] + +=item B<colo-host> :Secondary host's ip address. + +=item B<colo-port> :Secondary host's port, we will run a nbd server on +secondary host, and the nbd server will listen this port. + +=item B<colo-export> :Nbd server's disk export name of secondary host. ++=item B<active-disk> :Secondary's guest write will be buffered in this disk, +and it's used by secondary. ++=item B<hidden-disk> :Primary's modified contents will be buffered in this +disk, and it's used by secondary. ++Note that the COLO configuration settings should be considered unstable. They +may change incompatibly in future versions of Xen. + +=back B<OPTIONS>diff --git a/docs/misc/xl-disk-configuration.txt b/docs/misc/xl-disk-configuration.txt index 29f6ddb..b3402bc 100644 --- a/docs/misc/xl-disk-configuration.txt +++ b/docs/misc/xl-disk-configuration.txt@@ -234,6 +234,61 @@ were intentionally created non-sparse to avoid fragmentation of the file. +=============== +COLO PARAMETERS +=============== + + +colo +---- + +Enable COLO HA for disk. For better understanding block replication on +QEMU, please refer to: +http://wiki.qemu.org/Features/BlockReplication +Note that the COLO configuration settings should be considered unstable. +They may change incompatibly in future versions of Xen. + + +colo-host +--------- + +Description: Secondary host's address +Mandatory: Yes when COLO enabled + + +colo-port +--------- + +Description: Secondary port + We will run a nbd server on secondary host, + and the nbd server will listen this port. +Mandatory: Yes when COLO enabled + + +colo-export +----------- + +Description: We will run a nbd server on secondary host, + exportname is the nbd server's disk export name. +Mandatory: Yes when COLO enabled + + +active-disk +----------- + +Description: This is used by secondary. Secondary guest's write + will be buffered in this disk. +Mandatory: Yes when COLO enabled + + +hidden-disk +----------- + +Description: This is used by secondary. It buffers the original + content that is modified by the primary VM. +Mandatory: Yes when COLO enabled + + ============================================ DEPRECATED PARAMETERS, PREFIXES AND SYNTAXES ============================================ diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index 349a3c6..63fbe16 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c@@ -2306,6 +2306,8 @@ int libxl__device_disk_setdefault(libxl__gc *gc, libxl_device_disk *disk)
int rc;
libxl_defbool_setdefault(&disk->discard_enable, !!disk->readwrite);
+ libxl_defbool_setdefault(&disk->colo_enable, false);
+ libxl_defbool_setdefault(&disk->colo_restore_enable, false);
rc = libxl__resolve_domid(gc, disk->backend_domname,
&disk->backend_domid);
if (rc < 0) return rc;
@@ -2504,6 +2506,18 @@ static void device_disk_add(libxl__egc *egc,
uint32_t domid,
flexarray_append(back, "params");
flexarray_append(back, GCSPRINTF("%s:%s",
libxl__device_disk_string_of_format(disk->format), disk->pdev_path));
+ if (libxl_defbool_val(disk->colo_enable)) {
+ flexarray_append(back, "colo-host");
+ flexarray_append(back, libxl__sprintf(gc, "%s",
disk->colo_host));
+ flexarray_append(back, "colo-port");+ flexarray_append(back, libxl__sprintf(gc, "%d", disk->colo_port)); + flexarray_append(back, "colo-export");+ flexarray_append(back, libxl__sprintf(gc, "%s", disk->colo_export)); + flexarray_append(back, "active-disk");+ flexarray_append(back, libxl__sprintf(gc, "%s", disk->active_disk)); + flexarray_append(back, "hidden-disk");+ flexarray_append(back, libxl__sprintf(gc, "%s", disk->hidden_disk));
+ }
assert(device->backend_kind == LIBXL__DEVICE_KIND_QDISK);
break;
default:
@@ -2619,7 +2633,12 @@ static int
libxl__device_disk_from_xs_be(libxl__gc *gc,
goto cleanup;
}
- /* "params" may not be present; but everything else must be. */
+ /*
+ * "params" may not be present; but everything else must be.
+ * colo releated entries(colo-host, colo-port, colo-export,
+ * active-disk and hidden-disk) are present only if colo is
+ * enabled.
+ */
tmp = xs_read(ctx->xsh, XBT_NULL,
GCSPRINTF("%s/params", be_path), &len);
if (tmp && strchr(tmp, ':')) {
@@ -2629,6 +2648,36 @@ static int
libxl__device_disk_from_xs_be(libxl__gc *gc,
disk->pdev_path = tmp;
}
+ tmp = xs_read(ctx->xsh, XBT_NULL,
+ GCSPRINTF("%s/colo-host", be_path), &len);
+ if (tmp) {
+ libxl_defbool_set(&disk->colo_enable, true);
+ disk->colo_host = tmp;
+
+ tmp = xs_read(ctx->xsh, XBT_NULL,
+ GCSPRINTF("%s/colo-port", be_path), &len);
+ if (!tmp) {
+ LOG(ERROR, "Missing xenstore node %s/colo-port", be_path);
+ goto cleanup;
+ }
+ disk->colo_port = atoi(tmp);
+
+#define XS_READ_COLO(param, item) do { \
+ tmp = xs_read(ctx->xsh, XBT_NULL, \
+ GCSPRINTF("%s/"#param"", be_path), &len); \
+ if (!tmp) { \
+ LOG(ERROR, "Missing xenstore node %s/"#param"", be_path); \
+ goto cleanup; \
+ } \
+ disk->item = tmp; \
+} while (0)
+ XS_READ_COLO(colo-export, colo_export);
+ XS_READ_COLO(active-disk, active_disk);
+ XS_READ_COLO(hidden-disk, hidden_disk);
+#undef XS_READ_COLO
+ } else {
+ libxl_defbool_set(&disk->colo_enable, false);
+ }
tmp = libxl__xs_read(gc, XBT_NULL,
GCSPRINTF("%s/type", be_path));
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index be604e5..e2ec25c 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -1876,12 +1876,30 @@ static void domain_create_cb(libxl__egc *egc,
libxl__ao_complete(egc, ao, rc);
}
-
+
+
+static void set_disk_colo_restore(libxl_domain_config *d_config)
+{
+ int i;
+
+ for (i = 0; i < d_config->num_disks; i++)
+ libxl_defbool_set(&d_config->disks[i].colo_restore_enable, true);
+}
+
+static void unset_disk_colo_restore(libxl_domain_config *d_config)
+{
+ int i;
+
+ for (i = 0; i < d_config->num_disks; i++)
+ libxl_defbool_set(&d_config->disks[i].colo_restore_enable, false);
+}
+
int libxl_domain_create_new(libxl_ctx *ctx, libxl_domain_config *d_config,
uint32_t *domid,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how
*aop_console_how)
{
+ unset_disk_colo_restore(d_config);
return do_domain_create(ctx, d_config, domid, -1, -1, NULL,
ao_how, aop_console_how);
}
@@ -1893,6 +1911,12 @@ int libxl_domain_create_restore(libxl_ctx *ctx,
libxl_domain_config *d_config,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how
*aop_console_how)
{
+ if (params->checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_COLO) {
+ set_disk_colo_restore(d_config);
+ } else {
+ unset_disk_colo_restore(d_config);
+ }
+
return do_domain_create(ctx, d_config, domid, restore_fd,
send_back_fd,
params, ao_how, aop_console_how);
}
diff --git a/tools/libxl/libxl_device.c b/tools/libxl/libxl_device.c
index 4ced9b6..6a411c6 100644
--- a/tools/libxl/libxl_device.c
+++ b/tools/libxl/libxl_device.c
@@ -196,6 +196,9 @@ static int disk_try_backend(disk_try_backend_args *a,
goto bad_format;
}
+ if (libxl_defbool_val(a->disk->colo_enable))
+ goto bad_colo;
+
if (a->disk->backend_domid != LIBXL_TOOLSTACK_DOMID) {
LOG(DEBUG, "Disk vdev=%s, is using a storage driver domain, "
"skipping physical device check", a->disk->vdev);
@@ -218,6 +221,9 @@ static int disk_try_backend(disk_try_backend_args *a,
case LIBXL_DISK_BACKEND_TAP:
if (a->disk->script) goto bad_script;
+ if (libxl_defbool_val(a->disk->colo_enable))
+ goto bad_colo;
+
if (a->disk->is_cdrom) {
LOG(DEBUG, "Disk vdev=%s, backend tap unsuitable for cdroms",
a->disk->vdev);
@@ -256,6 +262,11 @@ static int disk_try_backend(disk_try_backend_args *a,
LOG(DEBUG, "Disk vdev=%s, backend %s not compatible with script=...",
a->disk->vdev, libxl_disk_backend_to_string(backend));
return 0;
+
+ bad_colo:
+ LOG(DEBUG, "Disk vdev=%s, backend %s not compatible with colo",
+ a->disk->vdev, libxl_disk_backend_to_string(backend));
+ return 0;
}
int libxl__device_disk_set_backend(libxl__gc *gc, libxl_device_disk
*disk) {
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c index 1d1b25b..4c3dff8 100644 --- a/tools/libxl/libxl_dm.c +++ b/tools/libxl/libxl_dm.c@@ -754,6 +754,8 @@ static int libxl__dm_runas_helper(libxl__gc *gc, const char *username)
/* colo mode */
enum {
LIBXL__COLO_NONE = 0,
+ LIBXL__COLO_PRIMARY,
+ LIBXL__COLO_SECONDARY,
};
static char *qemu_disk_scsi_drive_string(libxl__gc *gc, const char
*pdev_path,
@@ -762,6 +764,9 @@ static char *qemu_disk_scsi_drive_string(libxl__gc
*gc, const char *pdev_path,
int colo_mode)
{
char *drive = NULL;
+ const char *exportname = disk->colo_export;
+ const char *active_disk = disk->active_disk;
+ const char *hidden_disk = disk->hidden_disk;
switch (colo_mode) {
case LIBXL__COLO_NONE:
@@ -769,6 +774,45 @@ static char *qemu_disk_scsi_drive_string(libxl__gc
*gc, const char *pdev_path,
(gc,
"file=%s,if=scsi,bus=0,unit=%d,format=%s,cache=writeback",
pdev_path, unit, format);
break;
+ case LIBXL__COLO_PRIMARY:
+ /*
+ * primary:
+ * -dirve if=scsi,bus=0,unit=x,cache=writeback,driver=quorum,\
+ * id=exportname,\
+ * children.0.file.filename=pdev_path,\
+ * children.0.driver=format,\
+ * read-pattern=fifo,\
+ * vote-threshold=1
+ */
+ drive = GCSPRINTF(
+ "if=scsi,bus=0,unit=%d,cache=writeback,driver=quorum,"
+ "id=%s,"
+ "children.0.file.filename=%s,"
+ "children.0.driver=%s,"
+ "read-pattern=fifo,"
+ "vote-threshold=1",
+ unit, exportname, pdev_path, format);
+ break;
+ case LIBXL__COLO_SECONDARY:
+ /*
+ * secondary:
+ * -drive
if=scsi,bus=0,unit=x,cache=writeback,driver=replication,\
+ * mode=secondary,\
+ * file.driver=qcow2,\
+ * file.file.filename=active_disk,\
+ * file.backing.driver=qcow2,\
+ * file.backing.file.filename=hidden_disk,\
+ * file.backing.backing=exportname,
+ */
+ drive = GCSPRINTF(
+ "if=scsi,bus=0,unit=%d,cache=writeback,driver=replication,"
+ "mode=secondary,"
+ "file.driver=qcow2,"
+ "file.file.filename=%s,"
+ "file.backing.driver=qcow2,"
+ "file.backing.file.filename=%s,"
+ "file.backing.backing=%s",
+ unit, active_disk, hidden_disk, exportname);
default:
abort();
}
@@ -782,6 +826,9 @@ static char *qemu_disk_ide_drive_string(libxl__gc
*gc, const char *pdev_path,
int colo_mode)
{
char *drive = NULL;
+ const char *exportname = disk->colo_export;
+ const char *active_disk = disk->active_disk;
+ const char *hidden_disk = disk->hidden_disk;
switch (colo_mode) {
case LIBXL__COLO_NONE:
@@ -789,6 +836,46 @@ static char *qemu_disk_ide_drive_string(libxl__gc
*gc, const char *pdev_path,
("file=%s,if=ide,index=%d,media=disk,format=%s,cache=writeback",
pdev_path, unit, format);
break;
+ case LIBXL__COLO_PRIMARY:
+ /*
+ * primary:
+ * -dirve
if=ide,index=x,media=disk,cache=writeback,driver=quorum,\
+ * id=exportname,\ + * children.0.file.filename=pdev_path,\ + * children.0.driver=format,\ + * read-pattern=fifo,\ + * vote-threshold=1 + */ + drive = GCSPRINTF( + "if=ide,index=%d,media=disk,cache=writeback,driver=quorum," + "id=%s," + "children.0.file.filename=%s," + "children.0.driver=%s," + "read-pattern=fifo," + "vote-threshold=1", + unit, exportname, pdev_path, format); + break; + case LIBXL__COLO_SECONDARY: + /* + * secondary:+ * -drive if=ide,index=x,media=disk,cache=writeback,driver=replication,\ + * mode=secondary,\ + * file.driver=qcow2,\ + * file.file.filename=active_disk,\ + * file.backing.driver=qcow2,\ + * file.backing.file.filename=hidden_disk,\ + * file.backing.backing=exportname, + */ + drive = GCSPRINTF(+ "if=ide,index=%d,media=disk,cache=writeback,driver=replication,"
+ "mode=secondary,"
+ "file.driver=qcow2,"
+ "file.file.filename=%s,"
+ "file.backing.driver=qcow2,"
+ "file.backing.file.filename=%s,"
+ "file.backing.backing=%s",
+ unit, active_disk, hidden_disk, exportname);
+ break;
default:
abort();
}
@@ -1261,8 +1348,24 @@ static int
libxl__build_device_model_args_new(libxl__gc *gc,
* hd[a-d] and ignore the rest.
*/
- colo_mode = LIBXL__COLO_NONE;
+ if (libxl_defbool_val(disks[i].colo_enable)) {
+ if (libxl_defbool_val(disks[i].colo_restore_enable))
+ colo_mode = LIBXL__COLO_SECONDARY;
+ else
+ colo_mode = LIBXL__COLO_PRIMARY;
+ } else {
+ colo_mode = LIBXL__COLO_NONE;
+ }
+
if (strncmp(disks[i].vdev, "sd", 2) == 0) {
+ if (colo_mode == LIBXL__COLO_SECONDARY) {
+ drive = libxl__sprintf
+ (gc, "if=none,driver=%s,file=%s,id=%s",
+ format, pdev_path, disks[i].colo_export);
+
+ flexarray_append(dm_args, "-drive");
+ flexarray_append(dm_args, drive);
+ }
drive = qemu_disk_scsi_drive_string(gc, pdev_path,
disk,
format,
&disks[i],
@@ -1289,6 +1392,14 @@ static int
libxl__build_device_model_args_new(libxl__gc *gc,
LOG(ERROR, "qemu-xen doesn't support read-only
IDE disk drivers");
return ERROR_INVAL;
}
+ if (colo_mode == LIBXL__COLO_SECONDARY) {
+ drive = libxl__sprintf
+ (gc, "if=none,driver=%s,file=%s,id=%s",
+ format, pdev_path, disks[i].colo_export);
+
+ flexarray_append(dm_args, "-drive");
+ flexarray_append(dm_args, drive);
+ }
drive = qemu_disk_ide_drive_string(gc, pdev_path,
disk,
format,
&disks[i],
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 95efd82..8335291 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -576,6 +576,15 @@ libxl_device_disk = Struct("device_disk", [
("is_cdrom", integer),
("direct_io_safe", bool),
("discard_enable", libxl_defbool),
+ # Note that the COLO configuration settings should be considered
unstable.
+ # They may change incompatibly in future versions of Xen.
+ ("colo_enable", libxl_defbool),
+ ("colo_restore_enable", libxl_defbool),
+ ("colo_host", string),
+ ("colo_port", integer),
+ ("colo_export", string),
+ ("active_disk", string),
+ ("hidden_disk", string)
])
libxl_device_nic = Struct("device_nic", [
diff --git a/tools/libxl/libxlu_disk_l.l b/tools/libxl/libxlu_disk_l.l
index 1a5deb5..5b6db22 100644
--- a/tools/libxl/libxlu_disk_l.l
+++ b/tools/libxl/libxlu_disk_l.l
@@ -113,6 +113,16 @@ static void setbackendtype(DiskParseContext *dpc,
const char *str) {
else xlu__disk_err(dpc,str,"unknown value for backendtype");
}
+/* Sets ->colo-port from the string. COLO need this. */
+static void setcoloport(DiskParseContext *dpc, const char *str) {
+ int port = atoi(str);
+ if (port) {
+ dpc->disk->colo_port = port;
+ } else {
+ xlu__disk_err(dpc,str,"unknown value for colo_port");
+ }
+}
+
#define DEPRECATE(usewhatinstead) /* not currently reported */
/* Handles a vdev positional parameter which includes a devtype. */
@@ -176,6 +186,15 @@ script=[^,]*,? { STRIP(','); SAVESTRING("script",
script, FROMEQUALS); }
direct-io-safe,? { DPC->disk->direct_io_safe = 1; }
discard,? { libxl_defbool_set(&DPC->disk->discard_enable, true); }
no-discard,? { libxl_defbool_set(&DPC->disk->discard_enable, false); }
+ /* Note that the COLO configuration settings should be considered
unstable.
+ * They may change incompatibly in future versions of Xen. */
+colo,? { libxl_defbool_set(&DPC->disk->colo_enable, true); }
+no-colo,? { libxl_defbool_set(&DPC->disk->colo_enable, false); }
+colo-host=[^,]*,? { STRIP(','); SAVESTRING("colo-host", colo_host,
FROMEQUALS); }
+colo-port=[^,]*,? { STRIP(','); setcoloport(DPC, FROMEQUALS); }
+colo-export=[^,]*,? { STRIP(','); SAVESTRING("colo-export",
colo_export, FROMEQUALS); }
+active-disk=[^,]*,? { STRIP(','); SAVESTRING("active-disk",
active_disk, FROMEQUALS); }
+hidden-disk=[^,]*,? { STRIP(','); SAVESTRING("hidden-disk",
hidden_disk, FROMEQUALS); }
/* the target magic parameter, eats the rest of the string */ -- 1.9.3 _______________________________________________ Xen-devel mailing list Xen-devel@xxxxxxxxxxxxx http://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |