[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH RFC v2 3/8] Adapted libxl to handle migration of instance with qemu based disks



---
 tools/libxl/libxl.h              |  11 ++-
 tools/libxl/libxl_create.c       | 191 ++++++++++++++++++++++++++++++++++++---
 tools/libxl/libxl_dm.c           |  26 ++++--
 tools/libxl/libxl_dom_save.c     |  85 ++++++++++++++++-
 tools/libxl/libxl_domain.c       |   4 +-
 tools/libxl/libxl_internal.h     |  60 ++++++++++--
 tools/libxl/libxl_save_callout.c |  38 +++++---
 tools/libxl/libxl_save_helper.c  |   4 +-
 tools/libxl/libxl_stream_read.c  |  17 +++-
 tools/libxl/libxl_stream_write.c |  28 +++++-
 10 files changed, 406 insertions(+), 58 deletions(-)

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 91408b4..4022c37 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -1333,7 +1333,7 @@ int libxl_domain_create_new(libxl_ctx *ctx, 
libxl_domain_config *d_config,
                             LIBXL_EXTERNAL_CALLERS_ONLY;
 int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
                                 uint32_t *domid, int restore_fd,
-                                int send_back_fd,
+                                int send_back_fd, int mirror_qemu_disks,
                                 const libxl_domain_restore_params *params,
                                 const libxl_asyncop_how *ao_how,
                                 const libxl_asyncprogress_how *aop_console_how)
@@ -1373,8 +1373,9 @@ static inline int libxl_domain_create_restore_0x040400(
     const libxl_asyncprogress_how *aop_console_how)
     LIBXL_EXTERNAL_CALLERS_ONLY
 {
+    //TODO: balvisio: Review
     return libxl_domain_create_restore(ctx, d_config, domid, restore_fd,
-                                       -1, params, ao_how, aop_console_how);
+                                       -1, 0, params, ao_how, aop_console_how);
 }
 
 #define libxl_domain_create_restore libxl_domain_create_restore_0x040400
@@ -1408,10 +1409,16 @@ int libxl_retrieve_domain_configuration(libxl_ctx *ctx, 
uint32_t domid,
 
 int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
                          int flags, /* LIBXL_SUSPEND_* */
+                         const char* hostname,
                          const libxl_asyncop_how *ao_how)
                          LIBXL_EXTERNAL_CALLERS_ONLY;
 #define LIBXL_SUSPEND_DEBUG 1
 #define LIBXL_SUSPEND_LIVE 2
+#define LIBXL_SUSPEND_MIRROR_QEMU_DISKS 4
+
+//TODO: balvisio: DO NOT HARD-CODE THIS PARAMS
+#define QEMU_DRIVE_MIRROR_PORT "11000"
+#define QEMU_DRIVE_MIRROR_DEVICE "ide0-hd0"
 
 /* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )]
  *   If this parameter is true, use co-operative resume. The guest
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 9123585..f10f2ce 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -744,6 +744,10 @@ static int store_libxl_entry(libxl__gc *gc, uint32_t domid,
 static void domcreate_devmodel_started(libxl__egc *egc,
                                        libxl__dm_spawn_state *dmss,
                                        int rc);
+
+static void start_nbd_server(libxl__egc *egc, libxl__dm_spawn_state *dmss,
+                             int ret);
+
 static void domcreate_bootloader_console_available(libxl__egc *egc,
                                                    libxl__bootloader_state 
*bl);
 static void domcreate_bootloader_done(libxl__egc *egc,
@@ -760,10 +764,17 @@ static void domcreate_stream_done(libxl__egc *egc,
                                   libxl__stream_read_state *srs,
                                   int ret);
 
+static void domcreate_pre_build(libxl__egc *egc,
+                                libxl__domain_create_state *dcs,
+                                int ret);
+
 static void domcreate_rebuild_done(libxl__egc *egc,
                                    libxl__domain_create_state *dcs,
                                    int ret);
 
+static void domcreate_multidev_begin(libxl__egc *egc,
+                                     libxl__domain_create_state *dcs);
+
 /* Our own function to clean up and call the user's callback.
  * The final call in the sequence. */
 static void domcreate_complete(libxl__egc *egc,
@@ -1016,6 +1027,64 @@ static void libxl__colo_restore_setup_done(libxl__egc 
*egc,
     libxl__stream_read_start(egc, &dcs->srs);
 }
 
+static void start_nbd_server(libxl__egc *egc, libxl__dm_spawn_state *dmss,
+                             int ret){
+
+    libxl__domain_create_state *dcs = CONTAINER_OF(dmss, *dcs, sdss.dm);
+    STATE_AO_GC(dmss->spawn.ao);
+    const uint32_t domid = dcs->guest_domid;
+    dcs->sdss.dm.guest_domid = domid;
+
+    if (ret) {
+        LOGD(ERROR, domid, "device model did not start: %d", ret);
+        goto error_out;
+    }
+
+    if(dcs->restore_fd >= 0 && dcs->mirror_qemu_disks) {
+             /*
+              * Start and add the NBD server
+              * Host is set it to "::" for now
+              * Port we hard code a port for now
+
+              * This code just handles the case when -M pc is used.
+              * (The config xen_platform_pci = 0)
+              *
+              * TODO: Current implementation only works with upstream qemu
+              * TODO: consider the case when qemu-xen-traditional is used.
+              * TODO: Check and copy only those disks which are local
+              * TODO: Assign port dynamically
+              */
+
+            LOGD(DEBUG, "Starting NBD Server\n");
+            ret = libxl__qmp_nbd_server_start(gc, domid, "::", 
QEMU_DRIVE_MIRROR_PORT);
+            if (ret) {
+                ret = ERROR_FAIL;
+                LOGD(ERROR, "Failed to start NBD Server\n");
+                goto skip_nbd;
+            }else{
+                LOGD(INFO, "Started NBD Server Successfully\n");
+            }
+
+            ret = libxl__qmp_nbd_server_add(gc, domid, 
QEMU_DRIVE_MIRROR_DEVICE);
+
+            if (ret) {
+                ret = ERROR_FAIL;
+                LOGD(ERROR, "Failed to add NBD Server\n");
+                goto skip_nbd;
+            } else {
+                LOGD(INFO, "NBD Add Successful\n");
+            }
+        }
+
+skip_nbd:
+    libxl__stream_read_start(egc, &dcs->srs);
+    return;
+
+error_out:
+    assert(ret);
+    domcreate_complete(egc, dcs, ret);
+}
+
 static void domcreate_bootloader_done(libxl__egc *egc,
                                       libxl__bootloader_state *bl,
                                       int rc)
@@ -1033,6 +1102,8 @@ static void domcreate_bootloader_done(libxl__egc *egc,
     libxl_domain_build_info *const info = &d_config->b_info;
     libxl__srm_restore_autogen_callbacks *const callbacks =
         &dcs->srs.shs.callbacks.restore.a;
+    libxl__srm_restore_autogen_callbacks *const callbacks_mirror_qemu_disks =
+        &dcs->srs_local_disks.shs.callbacks.restore.a;
 
     if (rc) {
         domcreate_rebuild_done(egc, dcs, rc);
@@ -1050,8 +1121,41 @@ static void domcreate_bootloader_done(libxl__egc *egc,
     dcs->sdss.dm.spawn.ao = ao;
     dcs->sdss.dm.guest_config = dcs->guest_config;
     dcs->sdss.dm.build_state = &dcs->build_state;
-    dcs->sdss.dm.callback = domcreate_devmodel_started;
-    dcs->sdss.callback = domcreate_devmodel_started;
+    if(!dcs->mirror_qemu_disks) {
+        dcs->sdss.dm.callback = domcreate_devmodel_started;
+        dcs->sdss.callback = domcreate_devmodel_started;
+    }else{
+        /*
+        Original calling sequence:
+        0. domcreate_bootloader_done
+        1. MEMORY STREAM
+        2. domcreate_stream_done
+        3. domcreate_pre_build
+        4. domcreate_rebuild_done
+        5. domcreate_multidev_begin
+        6. domcreate_launch_dm
+        7. domcreate_devmodel_started
+
+        In the case of local_disk domcreate_devmodel_started will be called:
+        through the following sequence:
+        0. domcreate_bootloader_done
+        1. 1st MEMORY STREAM
+        2. domcreate_stream_done
+        3. domcreate_pre_build
+        4. domcreate_rebuild_done
+        5. domcreate_multidev_begin
+        6. domcreate_launch_dm
+        7. start_nbd_server
+        8. 2nd MEMORY STREAM
+        9. domcreate_stream_done
+            9.a. Calls -incoming to start QEMU process
+        10. domcreate_devmodel_started
+        */
+
+        dcs->sdss.dm.callback = start_nbd_server;
+        dcs->sdss.callback = start_nbd_server;
+    }
+
 
     if (restore_fd < 0 && dcs->domid_soft_reset == INVALID_DOMID) {
         rc = libxl__domain_build(gc, d_config, domid, state);
@@ -1061,6 +1165,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
 
     /* Restore */
     callbacks->restore_results = libxl__srm_callout_callback_restore_results;
+    callbacks_mirror_qemu_disks->restore_results = 
libxl__srm_callout_callback_restore_results;
 
     /* COLO only supports HVM now because it does not work very
      * well with pv drivers:
@@ -1087,6 +1192,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
     dcs->srs.legacy = (dcs->restore_params.stream_version == 1);
     dcs->srs.back_channel = false;
     dcs->srs.completion_callback = domcreate_stream_done;
+    dcs->srs.mirror_qemu_disks = 0;
 
     if (restore_fd >= 0) {
         switch (checkpointed_stream) {
@@ -1104,7 +1210,20 @@ static void domcreate_bootloader_done(libxl__egc *egc,
             libxl__remus_restore_setup(egc, dcs);
             /* fall through */
         case LIBXL_CHECKPOINTED_STREAM_NONE:
-            libxl__stream_read_start(egc, &dcs->srs);
+            if (dcs->mirror_qemu_disks) {
+                /* Spawn the local_dm process before reading the memory stream 
*/
+                dcs->srs_mirror_qemu_disks.ao = ao;
+                dcs->srs_mirror_qemu_disks.dcs = dcs;
+                dcs->srs_mirror_qemu_disks.fd = restore_fd;
+                dcs->srs_mirror_qemu_disks.legacy = 
(dcs->restore_params.stream_version == 1);
+                dcs->srs_mirror_qemu_disks.back_channel = false;
+                dcs->srs_mirror_qemu_disks.completion_callback = 
domcreate_stream_done;
+                dcs->srs_mirror_qemu_disks.mirror_qemu_disks = 1;
+
+                libxl__stream_read_start(egc, &dcs->srs_mirror_qemu_disks);
+            } else {
+                libxl__stream_read_start(egc, &dcs->srs);
+            }
         }
         return;
     }
@@ -1132,6 +1251,35 @@ static void domcreate_stream_done(libxl__egc *egc,
 {
     libxl__domain_create_state *dcs = srs->dcs;
     STATE_AO_GC(dcs->ao);
+
+    const uint32_t domid = dcs->guest_domid;
+    const char* uri;
+    const char* state_file = GCSPRINTF(
+                             LIBXL_DEVICE_MODEL_RESTORE_FILE".%d", domid);
+
+    if(!dcs->mirror_qemu_disks || srs->mirror_qemu_disks){
+        domcreate_pre_build(egc, dcs, ret);
+    }else{
+        //Stop NBD server
+        fprintf(stderr, "Stopping NBD server\n");
+        rc = libxl__qmp_nbd_server_stop(gc, domid);
+        if (rc){
+            fprintf(stderr, "Failed to stop NBD server\n");
+        }else{
+            fprintf(stderr, "Stopped NBD server successfully\n");
+        }
+        uri = GCSPRINTF("exec: /bin/cat %s", 
(&dcs->sdss.dm)->build_state->saved_state);
+        libxl__qmp_migrate_incoming(gc, domid, uri);
+        domcreate_devmodel_started(egc, &dcs->sdss.dm, 0);
+    }
+    return;
+}
+
+static void domcreate_pre_build(libxl__egc *egc,
+                                libxl__domain_create_state *dcs,
+                                int ret)
+{
+    STATE_AO_GC(dcs->ao);
     libxl_ctx *ctx = libxl__gc_owner(gc);
     char **vments = NULL, **localents = NULL;
     struct timeval start_time;
@@ -1185,7 +1333,7 @@ static void domcreate_stream_done(libxl__egc *egc,
     if (ret)
         goto out;
 
-    if (info->type == LIBXL_DOMAIN_TYPE_HVM) {
+    if (info->type == LIBXL_DOMAIN_TYPE_HVM && !dcs->mirror_qemu_disks) {
         state->saved_state = GCSPRINTF(
                        LIBXL_DEVICE_MODEL_RESTORE_FILE".%d", domid);
     }
@@ -1223,10 +1371,7 @@ static void domcreate_rebuild_done(libxl__egc *egc,
 
     store_libxl_entry(gc, domid, &d_config->b_info);
 
-    libxl__multidev_begin(ao, &dcs->multidev);
-    dcs->multidev.callback = domcreate_launch_dm;
-    libxl__add_disks(egc, ao, domid, d_config, &dcs->multidev);
-    libxl__multidev_prepared(egc, &dcs->multidev, 0);
+    domcreate_multidev_begin(egc, dcs);
 
     return;
 
@@ -1235,6 +1380,24 @@ static void domcreate_rebuild_done(libxl__egc *egc,
     domcreate_complete(egc, dcs, ret);
 }
 
+static void domcreate_multidev_begin(libxl__egc *egc,
+                                   libxl__domain_create_state *dcs){
+
+    STATE_AO_GC(dcs->ao);
+
+    /* convenience aliases */
+    const uint32_t domid = dcs->guest_domid;
+    libxl_domain_config *const d_config = dcs->guest_config;
+
+    libxl__multidev_begin(ao, &dcs->multidev);
+    dcs->multidev.callback = domcreate_launch_dm;
+
+    libxl__add_disks(egc, ao, domid, d_config, &dcs->multidev);
+    libxl__multidev_prepared(egc, &dcs->multidev, 0);
+
+    return;
+}
+
 static void domcreate_launch_dm(libxl__egc *egc, libxl__multidev *multidev,
                                 int ret)
 {
@@ -1355,7 +1518,7 @@ static void domcreate_launch_dm(libxl__egc *egc, 
libxl__multidev *multidev,
         if (libxl_defbool_val(d_config->b_info.device_model_stubdomain))
             libxl__spawn_stub_dm(egc, &dcs->sdss);
         else
-            libxl__spawn_local_dm(egc, &dcs->sdss.dm);
+            libxl__spawn_local_dm(egc, &dcs->sdss.dm, dcs->mirror_qemu_disks);
 
         /*
          * Handle the domain's (and the related stubdomain's) access to
@@ -1387,7 +1550,7 @@ static void domcreate_launch_dm(libxl__egc *egc, 
libxl__multidev *multidev,
             goto error_out;
         if (ret) {
             dcs->sdss.dm.guest_domid = domid;
-            libxl__spawn_local_dm(egc, &dcs->sdss.dm);
+            libxl__spawn_local_dm(egc, &dcs->sdss.dm, 0);
             return;
         } else {
             assert(!dcs->sdss.dm.guest_domid);
@@ -1604,6 +1767,7 @@ static void domain_create_cb(libxl__egc *egc,
 
 static int do_domain_create(libxl_ctx *ctx, libxl_domain_config *d_config,
                             uint32_t *domid, int restore_fd, int send_back_fd,
+                            int mirror_qemu_disks,
                             const libxl_domain_restore_params *params,
                             const libxl_asyncop_how *ao_how,
                             const libxl_asyncprogress_how *aop_console_how)
@@ -1619,6 +1783,7 @@ static int do_domain_create(libxl_ctx *ctx, 
libxl_domain_config *d_config,
     libxl_domain_config_copy(ctx, &cdcs->dcs.guest_config_saved, d_config);
     cdcs->dcs.restore_fd = cdcs->dcs.libxc_fd = restore_fd;
     cdcs->dcs.send_back_fd = send_back_fd;
+    cdcs->dcs.mirror_qemu_disks = mirror_qemu_disks;
     if (restore_fd > -1) {
         cdcs->dcs.restore_params = *params;
         rc = libxl__fd_flags_modify_save(gc, cdcs->dcs.restore_fd,
@@ -1847,13 +2012,13 @@ int libxl_domain_create_new(libxl_ctx *ctx, 
libxl_domain_config *d_config,
                             const libxl_asyncprogress_how *aop_console_how)
 {
     unset_disk_colo_restore(d_config);
-    return do_domain_create(ctx, d_config, domid, -1, -1, NULL,
+    return do_domain_create(ctx, d_config, domid, -1, -1, 0, NULL,
                             ao_how, aop_console_how);
 }
 
 int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
                                 uint32_t *domid, int restore_fd,
-                                int send_back_fd,
+                                int send_back_fd, int mirror_qemu_disks,
                                 const libxl_domain_restore_params *params,
                                 const libxl_asyncop_how *ao_how,
                                 const libxl_asyncprogress_how *aop_console_how)
@@ -1865,7 +2030,7 @@ int libxl_domain_create_restore(libxl_ctx *ctx, 
libxl_domain_config *d_config,
     }
 
     return do_domain_create(ctx, d_config, domid, restore_fd, send_back_fd,
-                            params, ao_how, aop_console_how);
+                            mirror_qemu_disks, params, ao_how, 
aop_console_how);
 }
 
 int libxl_domain_soft_reset(libxl_ctx *ctx,
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index e0e6a99..ff6721d 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -915,7 +915,7 @@ static int libxl__build_device_model_args_new(libxl__gc *gc,
                                         const libxl_domain_config 
*guest_config,
                                         char ***args, char ***envs,
                                         const libxl__domain_build_state *state,
-                                        int *dm_state_fd)
+                                        int *dm_state_fd, int 
mirror_qemu_disks)
 {
     const libxl_domain_create_info *c_info = &guest_config->c_info;
     const libxl_domain_build_info *b_info = &guest_config->b_info;
@@ -1396,12 +1396,18 @@ static int libxl__build_device_model_args_new(libxl__gc 
*gc,
         }
     }
 
-    if (state->saved_state) {
+    if (state->saved_state && !mirror_qemu_disks) {
         /* This file descriptor is meant to be used by QEMU */
         *dm_state_fd = open(state->saved_state, O_RDONLY);
         flexarray_append(dm_args, "-incoming");
         flexarray_append(dm_args, GCSPRINTF("fd:%d",*dm_state_fd));
     }
+
+    if(mirror_qemu_disks) {
+        flexarray_append(dm_args, "-incoming");
+        flexarray_append(dm_args, "defer");
+    }
+
     for (i = 0; b_info->extra && b_info->extra[i] != NULL; i++)
         flexarray_append(dm_args, b_info->extra[i]);
 
@@ -1664,7 +1670,7 @@ static int libxl__build_device_model_args(libxl__gc *gc,
                                         const libxl_domain_config 
*guest_config,
                                         char ***args, char ***envs,
                                         const libxl__domain_build_state *state,
-                                        int *dm_state_fd)
+                                        int *dm_state_fd, int 
mirror_qemu_disks)
 /* dm_state_fd may be NULL iff caller knows we are using old stubdom
  * and therefore will be passing a filename rather than a fd. */
 {
@@ -1680,7 +1686,8 @@ static int libxl__build_device_model_args(libxl__gc *gc,
         return libxl__build_device_model_args_new(gc, dm,
                                                   guest_domid, guest_config,
                                                   args, envs,
-                                                  state, dm_state_fd);
+                                                  state, dm_state_fd,
+                                                  mirror_qemu_disks);
     default:
         LOGED(ERROR, guest_domid, "unknown device model version %d",
               guest_config->b_info.device_model_version);
@@ -1897,7 +1904,7 @@ void libxl__spawn_stub_dm(libxl__egc *egc, 
libxl__stub_dm_spawn_state *sdss)
 
     ret = libxl__build_device_model_args(gc, "stubdom-dm", guest_domid,
                                          guest_config, &args, NULL,
-                                         d_state, NULL);
+                                         d_state, NULL, 0);
     if (ret) {
         ret = ERROR_FAIL;
         goto out;
@@ -2055,7 +2062,7 @@ static void spawn_stub_launch_dm(libxl__egc *egc,
         /* If dom0 qemu not needed, do not launch it */
         spawn_stubdom_pvqemu_cb(egc, &sdss->pvqemu, 0);
     } else {
-        libxl__spawn_local_dm(egc, &sdss->pvqemu);
+        libxl__spawn_local_dm(egc, &sdss->pvqemu, 0);
     }
 
     return;
@@ -2160,7 +2167,8 @@ static void device_model_spawn_outcome(libxl__egc *egc,
                                        libxl__dm_spawn_state *dmss,
                                        int rc);
 
-void libxl__spawn_local_dm(libxl__egc *egc, libxl__dm_spawn_state *dmss)
+void libxl__spawn_local_dm(libxl__egc *egc, libxl__dm_spawn_state *dmss,
+                           int mirror_qemu_disks)
 {
     /* convenience aliases */
     const int domid = dmss->guest_domid;
@@ -2200,7 +2208,7 @@ void libxl__spawn_local_dm(libxl__egc *egc, 
libxl__dm_spawn_state *dmss)
     }
     rc = libxl__build_device_model_args(gc, dm, domid, guest_config,
                                           &args, &envs, state,
-                                          &dm_state_fd);
+                                          &dm_state_fd, mirror_qemu_disks);
     if (rc)
         goto out;
 
@@ -2330,7 +2338,7 @@ static void device_model_confirm(libxl__egc *egc, 
libxl__spawn_state *spawn,
     if (!xsdata)
         return;
 
-    if (strcmp(xsdata, "running"))
+    if (strcmp(xsdata, "running") && strcmp(xsdata, "inmigrate"))
         return;
 
     libxl__spawn_initiate_detach(gc, spawn);
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 77fe30e..a2730f5 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -408,18 +408,97 @@ void libxl__domain_save(libxl__egc *egc, 
libxl__domain_save_state *dss)
     dss->sws.fd  = dss->fd;
     dss->sws.back_channel = false;
     dss->sws.completion_callback = stream_done;
-
-    libxl__stream_write_start(egc, &dss->sws);
+    dss->sws.mirror_qemu_disks = 0;
+
+    if(!dss->mirror_qemu_disks) {
+        libxl__stream_write_start(egc, &dss->sws);
+    } else {
+        dss->sws_mirror_qemu_disks.ao  = dss->ao;
+        dss->sws_mirror_qemu_disks.dss = dss;
+        dss->sws_mirror_qemu_disks.fd  = dss->fd;
+        dss->sws_mirror_qemu_disks.back_channel = false;
+        dss->sws_mirror_qemu_disks.mirror_qemu_disks = 1;
+        dss->sws_mirror_qemu_disks.completion_callback = stream_done;
+        libxl__stream_write_start(egc, &dss->sws_mirror_qemu_disks);
+    }
     return;
 
  out:
     domain_save_done(egc, dss, rc);
 }
 
+static void mirror_qemu_disks(libxl__egc *egc, libxl__stream_write_state *sws,
+                              int rc)
+{
+    int counter = 20;
+    char* target;
+    bool job_is_ready = false;
+    libxl__domain_save_state *dss = sws->dss;
+    const uint32_t domid = dss->domid;
+    STATE_AO_GC(dss->ao);
+
+    if (dss->mirror_qemu_disks) {
+    /*
+     * If the -q was provided, the drive-mirror job is started.
+     * TODO: Move the following code as part of the domain_suspend
+     * TODO: The port should be sent by the destination.
+    */
+start_mirror:
+        LOGD(DEBUG, domid, "Sleeping for a bit so that source can start 
NBD\n");
+        sleep(30);
+        LOGD(DEBUG, "Starting mirror-drive of device %s\n",
+             QEMU_DRIVE_MIRROR_DEVICE);
+        target = GCSPRINTF("nbd:%s:%s:exportname=%s", dss->hostname,
+                           QEMU_DRIVE_MIRROR_PORT, QEMU_DRIVE_MIRROR_DEVICE);
+        rc = libxl__qmp_drive_mirror(gc, dss->domid, QEMU_DRIVE_MIRROR_DEVICE,
+                                     target, "raw");
+        if (!rc) {
+            LOGD(INFO, domid, "Drive mirror command returned successfully\n");
+        }else{
+            LOGD(ERROR, domid, "Sending drive mirror command failed\n");
+            if(counter > 0){
+                LOGD(INFO, domid, "Counter: %d. Sleeping for 10 sec and 
retry\n", counter);
+                sleep(10);
+                counter--;
+                goto start_mirror;
+            }else{
+                goto cont;
+            }
+        }
+
+        /*
+         * Query job status until it is ready
+         * TODO: This code is just an inefficient busy wait. QMP sends an
+         * TODO: asynchronous message when mirroring job is completed. Consider
+         * TODO: adding the capability to handle asynchronous QMP messages 
(already done?)
+         */
+        while(!job_is_ready) {
+            LOGD(INFO, domid, "Checking for drive-mirror job");
+            rc = libxl__qmp_query_block_jobs(gc, dss->domid, &job_is_ready);
+            if(rc){
+                LOGD(ERROR, domid, "Checking block job failed\n");
+                goto cont;
+            }else{
+                LOGD(INFO, domid, "Checking block job succeeded\n");
+            }
+            if(!job_is_ready){
+                LOGD(INFO, domid, "Sleeping 5 sec\n");
+                sleep(5);
+            }
+        }
+    }
+cont:
+    libxl__stream_write_start(egc, &sws->dss->sws);
+}
+
 static void stream_done(libxl__egc *egc,
                         libxl__stream_write_state *sws, int rc)
 {
-    domain_save_done(egc, sws->dss, rc);
+    if(!sws->dss->mirror_qemu_disks || !sws->mirror_qemu_disks) {
+        domain_save_done(egc, sws->dss, rc);
+    } else {
+        mirror_qemu_disks(egc, sws, rc);
+    }
 }
 
 static void domain_save_done(libxl__egc *egc,
diff --git a/tools/libxl/libxl_domain.c b/tools/libxl/libxl_domain.c
index 08eccd0..76c6d3d 100644
--- a/tools/libxl/libxl_domain.c
+++ b/tools/libxl/libxl_domain.c
@@ -487,7 +487,7 @@ static void domain_suspend_cb(libxl__egc *egc,
 }
 
 int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
-                         const libxl_asyncop_how *ao_how)
+                         const char* hostname, const libxl_asyncop_how *ao_how)
 {
     AO_CREATE(ctx, domid, ao_how);
     int rc;
@@ -509,6 +509,8 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, 
int fd, int flags,
     dss->type = type;
     dss->live = flags & LIBXL_SUSPEND_LIVE;
     dss->debug = flags & LIBXL_SUSPEND_DEBUG;
+    dss->mirror_qemu_disks = flags & LIBXL_SUSPEND_MIRROR_QEMU_DISKS;
+    dss->hostname = hostname;
     dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
 
     rc = libxl__fd_flags_modify_save(gc, dss->fd,
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 1349a8f..16a476f 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -1840,19 +1840,42 @@ _hidden int libxl__qmp_start_replication(libxl__gc *gc, 
int domid,
 _hidden int libxl__qmp_nbd_server_add(libxl__gc *gc, int domid,
                                        const char *disk);
 
+/* Add a disk to NBD server */
+_hidden int libxl__qmp_nbd_server_add(libxl__gc *gc, int domid,
+                                      const char *disk);
+/* Start replication */
+_hidden int libxl__qmp_start_replication(libxl__gc *gc, int domid,
+                                         bool primary);
+/* Add a disk to NBD server */
+ _hidden int libxl__qmp_nbd_server_add(libxl__gc *gc, int domid,
+                                       const char *disk);
 /* Mirror disk drive */
-_hidden int libxl__qmp_drive_mirror(libxl__gc *gc, int domid, const char* 
device,
-                                    const char* target, const char* format);
-
+_hidden int libxl__qmp_drive_mirror(libxl__gc *gc, int domid,
+                                    const char* device, const char* target,
+                                    const char* format);
 /* Query block devices */
-_hidden int libxl__qmp_query_block(libxl__gc *gc, int domid, char 
*device_names);
-
+_hidden int libxl__qmp_query_block(libxl__gc *gc, int domid,
+                                   char *device_names);
 /* Query existing block jobs*/
-_hidden int libxl__qmp_query_block_jobs(libxl__gc *gc, int domid, bool 
*is_ready);
-
+_hidden int libxl__qmp_query_block_jobs(libxl__gc *gc, int domid,
+                                        bool *is_ready);
 /* Resume QEMU process started with -incoming defer */
-_hidden int libxl__qmp_migrate_incoming(libxl__gc *gc, int domid, const char* 
uri);
+_hidden int libxl__qmp_migrate_incoming(libxl__gc *gc, int domid,
+                                        const char* uri);
+/* Mirror disk drive */
+_hidden int libxl__qmp_drive_mirror(libxl__gc *gc, int domid,
+                                    const char* device, const char* target,
+                                    const char* format);
+/* Query block devices */
+_hidden int libxl__qmp_query_block(libxl__gc *gc, int domid,
+                                   char *device_names);
 
+/* Query existing block jobs*/
+_hidden int libxl__qmp_query_block_jobs(libxl__gc *gc, int domid,
+                                        bool *is_ready);
+/* Resume QEMU process started with -incoming defer */
+_hidden int libxl__qmp_migrate_incoming(libxl__gc *gc, int domid,
+                                        const char* uri);
 /* Get replication error that occurs when the vm is running */
 _hidden int libxl__qmp_query_xen_replication_status(libxl__gc *gc, int domid);
 /* Do checkpoint */
@@ -3142,6 +3165,7 @@ struct libxl__stream_read_state {
     int fd;
     bool legacy;
     bool back_channel;
+    int mirror_qemu_disks;
     void (*completion_callback)(libxl__egc *egc,
                                 libxl__stream_read_state *srs,
                                 int rc);
@@ -3220,6 +3244,7 @@ struct libxl__stream_write_state {
     libxl__domain_save_state *dss;
     int fd;
     bool back_channel;
+    int mirror_qemu_disks;
     void (*completion_callback)(libxl__egc *egc,
                                 libxl__stream_write_state *sws,
                                 int rc);
@@ -3319,6 +3344,8 @@ struct libxl__domain_save_state {
     libxl_domain_type type;
     int live;
     int debug;
+    int mirror_qemu_disks;
+    const char* hostname;
     int checkpointed_stream;
     const libxl_domain_remus_info *remus;
     /* private */
@@ -3334,6 +3361,7 @@ struct libxl__domain_save_state {
     };
     libxl__checkpoint_devices_state cds;
     libxl__stream_write_state sws;
+    libxl__stream_write_state sws_mirror_qemu_disks;
     libxl__logdirty_switch logdirty;
 };
 
@@ -3668,6 +3696,7 @@ struct libxl__dm_spawn_state {
     libxl__spawn_state spawn;
     /* filled in by user, must remain valid: */
     uint32_t guest_domid; /* domain being served */
+    int mirror_qemu_disks;
     libxl_domain_config *guest_config;
     libxl__domain_build_state *build_state; /* relates to guest_domid */
     libxl__dm_spawn_cb *callback;
@@ -3713,6 +3742,7 @@ struct libxl__domain_create_state {
     int restore_fd, libxc_fd;
     int restore_fdfl; /* original flags of restore_fd */
     int send_back_fd;
+    int mirror_qemu_disks;
     libxl_domain_restore_params restore_params;
     uint32_t domid_soft_reset;
     libxl__domain_create_cb *callback;
@@ -3729,6 +3759,7 @@ struct libxl__domain_create_state {
         /* If we're not doing stubdom, we use only dmss.dm,
          * for the non-stubdom device model. */
     libxl__stream_read_state srs;
+    libxl__stream_read_state srs_local_disks;
     /* necessary if the domain creation failed and we have to destroy it */
     libxl__domain_destroy_state dds;
     libxl__multidev multidev;
@@ -3748,13 +3779,17 @@ _hidden void libxl__domain_save(libxl__egc *egc,
 /* calls libxl__xc_domain_suspend_done when done */
 _hidden void libxl__xc_domain_save(libxl__egc *egc,
                                    libxl__domain_save_state *dss,
-                                   libxl__save_helper_state *shs);
+                                   libxl__save_helper_state *shs,
+                                   int mirror_qemu_disks);
 /* If rc==0 then retval is the return value from xc_domain_save
  * and errnoval is the errno value it provided.
  * If rc!=0, retval and errnoval are undefined. */
 _hidden void libxl__xc_domain_save_done(libxl__egc*, void *dss_void,
                                         int rc, int retval, int errnoval);
 
+_hidden void libxl__xc_mirror_disks_save_done(libxl__egc *egc, void *dss_void,
+                               int rc, int retval, int errnoval);
+
 /* Used by asynchronous callbacks: ie ones which xc regards as
  * returning a value, but which we want to handle asynchronously.
  * Such functions' actual callback function return void in libxl
@@ -3778,13 +3813,18 @@ _hidden int libxl__restore_emulator_xenstore_data
 _hidden void libxl__xc_domain_restore(libxl__egc *egc,
                                       libxl__domain_create_state *dcs,
                                       libxl__save_helper_state *shs,
-                                      int hvm, int pae);
+                                      int hvm, int pae, int mirror_qemu_disks);
 /* If rc==0 then retval is the return value from xc_domain_save
  * and errnoval is the errno value it provided.
  * If rc!=0, retval and errnoval are undefined. */
 _hidden void libxl__xc_domain_restore_done(libxl__egc *egc, void *dcs_void,
                                            int rc, int retval, int errnoval);
 
+_hidden void libxl__xc_mirror_disks_restore_done(libxl__egc *egc,
+                                                 void *dcs_void,
+                                                 int rc, int retval,
+                                                 int errnoval);
+
 _hidden void libxl__save_helper_init(libxl__save_helper_state *shs);
 _hidden void libxl__save_helper_abort(libxl__egc *egc,
                                       libxl__save_helper_state *shs);
diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
index 891c669..290d91d 100644
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -43,7 +43,7 @@ static void helper_done(libxl__egc *egc, 
libxl__save_helper_state *shs);
 
 void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
                               libxl__save_helper_state *shs,
-                              int hvm, int pae)
+                              int hvm, int pae, int mirror_qemu_disks)
 {
     STATE_AO_GC(dcs->ao);
 
@@ -63,16 +63,22 @@ void libxl__xc_domain_restore(libxl__egc *egc, 
libxl__domain_create_state *dcs,
         state->console_domid,
         hvm, pae,
         cbflags, dcs->restore_params.checkpointed_stream,
+        mirror_qemu_disks,
     };
 
     shs->ao = ao;
     shs->domid = domid;
     shs->recv_callback = libxl__srm_callout_received_restore;
     if (dcs->restore_params.checkpointed_stream ==
-        LIBXL_CHECKPOINTED_STREAM_COLO)
+        LIBXL_CHECKPOINTED_STREAM_COLO) {
         shs->completion_callback = libxl__colo_restore_teardown;
-    else
-        shs->completion_callback = libxl__xc_domain_restore_done;
+    } else {
+        if(!mirror_qemu_disks){
+            shs->completion_callback = libxl__xc_domain_restore_done;
+        } else {
+            shs->completion_callback = libxl__xc_hvm_params_restore_done;
+        }
+    }
     shs->caller_state = dcs;
     shs->need_results = 1;
 
@@ -81,7 +87,7 @@ void libxl__xc_domain_restore(libxl__egc *egc, 
libxl__domain_create_state *dcs,
 }
 
 void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_save_state *dss,
-                           libxl__save_helper_state *shs)
+                           libxl__save_helper_state *shs, int 
mirror_qemu_disks)
 {
     STATE_AO_GC(dss->ao);
 
@@ -90,13 +96,17 @@ void libxl__xc_domain_save(libxl__egc *egc, 
libxl__domain_save_state *dss,
 
     const unsigned long argnums[] = {
         dss->domid, 0, 0, dss->xcflags, dss->hvm,
-        cbflags, dss->checkpointed_stream,
+        cbflags, dss->checkpointed_stream, mirror_qemu_disks,
     };
 
     shs->ao = ao;
     shs->domid = dss->domid;
     shs->recv_callback = libxl__srm_callout_received_save;
-    shs->completion_callback = libxl__xc_domain_save_done;
+    if(!mirror_qemu_disks){
+        shs->completion_callback = libxl__xc_domain_save_done;
+    }else{
+        shs->completion_callback = libxl__xc_hvm_params_save_done;
+    }
     shs->caller_state = dss;
     shs->need_results = 0;
 
@@ -336,13 +346,13 @@ static void helper_exited(libxl__egc *egc, 
libxl__ev_child *ch,
         if (!shs->rc)
             shs->rc = ERROR_FAIL;
     }
-
-    if (shs->need_results) {
-        if (!shs->rc) {
-            LOGD(ERROR,shs->domid,"%s exited without providing results",what);
-            shs->rc = ERROR_FAIL;
-        }
-    }
+//    balvisio: TODO: FIX
+//    if (shs->need_results) {
+//        if (!shs->rc) {
+//            LOGD(ERROR,shs->domid,"%s exited without providing 
results",what);
+//            shs->rc = ERROR_FAIL;
+//        }
+//    }
 
     if (!shs->completed) {
         if (!shs->rc) {
diff --git a/tools/libxl/libxl_save_helper.c b/tools/libxl/libxl_save_helper.c
index 1dece23..cfc9279 100644
--- a/tools/libxl/libxl_save_helper.c
+++ b/tools/libxl/libxl_save_helper.c
@@ -257,6 +257,7 @@ int main(int argc, char **argv)
         int hvm =                           atoi(NEXTARG);
         unsigned cbflags =                  strtoul(NEXTARG,0,10);
         xc_migration_stream_t stream_type = strtoul(NEXTARG,0,10);
+        int mirror_qemu_disks =                   atoi(NEXTARG);
         assert(!*++argv);
 
         helper_setcallbacks_save(&helper_save_callbacks, cbflags);
@@ -266,7 +267,7 @@ int main(int argc, char **argv)
 
         r = xc_domain_save(xch, io_fd, dom, max_iters, max_factor, flags,
                            &helper_save_callbacks, hvm, stream_type,
-                           recv_fd);
+                           recv_fd, mirror_qemu_disks);
         complete(r);
 
     } else if (!strcmp(mode,"--restore-domain")) {
@@ -282,6 +283,7 @@ int main(int argc, char **argv)
         unsigned int pae =                  strtoul(NEXTARG,0,10);
         unsigned cbflags =                  strtoul(NEXTARG,0,10);
         xc_migration_stream_t stream_type = strtoul(NEXTARG,0,10);
+        int mirror_qemu_disks =             atoi(NEXTARG);
         assert(!*++argv);
 
         helper_setcallbacks_restore(&helper_restore_callbacks, cbflags);
diff --git a/tools/libxl/libxl_stream_read.c b/tools/libxl/libxl_stream_read.c
index 4838750..ea5ffd3 100644
--- a/tools/libxl/libxl_stream_read.c
+++ b/tools/libxl/libxl_stream_read.c
@@ -561,6 +561,7 @@ static bool process_record(libxl__egc *egc,
 {
     STATE_AO_GC(stream->ao);
     libxl__domain_create_state *dcs = stream->dcs;
+    int create_mirror_qemu_disks = stream->dcs->mirror_qemu_disks;
     libxl__sr_record_buf *rec;
     libxl_sr_checkpoint_state *srcs;
     bool further_action_needed = false;
@@ -580,7 +581,9 @@ static bool process_record(libxl__egc *egc,
         break;
 
     case REC_TYPE_LIBXC_CONTEXT:
-        libxl__xc_domain_restore(egc, dcs, &stream->shs, 0, 0);
+        libxl__xc_domain_restore(egc, dcs, &stream->shs, 0, 0,
+                                 stream->mirror_qemu_disks +
+                                 create_mirror_qemu_disks);
         break;
 
     case REC_TYPE_EMULATOR_XENSTORE_DATA:
@@ -835,6 +838,18 @@ static void stream_done(libxl__egc *egc,
     }
 }
 
+void libxl__xc_mirror_disks_restore_done(libxl__egc *egc, void *dcs_void,
+                                       int rc, int retval, int errnoval)
+{
+    libxl__domain_create_state *dcs = dcs_void;
+    libxl__stream_read_state *stream = &dcs->srs_mirror_qemu_disks;
+    STATE_AO_GC(dcs->ao);
+
+    check_all_finished(egc, stream, rc);
+
+    stream_continue(egc, stream);
+}
+
 void libxl__xc_domain_restore_done(libxl__egc *egc, void *dcs_void,
                                    int rc, int retval, int errnoval)
 {
diff --git a/tools/libxl/libxl_stream_write.c b/tools/libxl/libxl_stream_write.c
index c96a6a2..ad312ed 100644
--- a/tools/libxl/libxl_stream_write.c
+++ b/tools/libxl/libxl_stream_write.c
@@ -335,7 +335,24 @@ static void stream_header_done(libxl__egc *egc,
 static void libxc_header_done(libxl__egc *egc,
                               libxl__stream_write_state *stream)
 {
-    libxl__xc_domain_save(egc, stream->dss, &stream->shs);
+    int save_mirror_qemu_disks = stream->dss->local_disks;
+    libxl__xc_domain_save(egc, stream->dss, &stream->shs,
+                          save_mirror_qemu_disks + stream->mirror_qemu_disks);
+}
+
+void libxl__xc_mirror_disks_save_done(libxl__egc *egc, void *dss_void,
+                                    int rc, int retval, int errnoval)
+{
+    libxl__domain_save_state *dss = dss_void;
+    libxl__stream_write_state *stream = &dss->sws_mirror_qemu_disks;
+    STATE_AO_GC(dss->ao);
+
+    check_all_finished(egc, stream, rc);
+
+    if (stream->in_checkpoint)
+        write_checkpoint_end_record(egc, stream);
+    else
+        write_emulator_xenstore_record(egc, stream);
 }
 
 void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
@@ -429,9 +446,12 @@ static void emulator_xenstore_record_done(libxl__egc *egc,
 {
     libxl__domain_save_state *dss = stream->dss;
 
-    if (dss->type == LIBXL_DOMAIN_TYPE_HVM)
-        write_emulator_context_record(egc, stream);
-    else {
+    if (dss->type == LIBXL_DOMAIN_TYPE_HVM) {
+        if(!stream->mirror_qemu_disks)
+            write_emulator_context_record(egc, stream);
+        else
+            write_end_record(egc, stream);
+    } else {
         if (stream->in_checkpoint)
             write_checkpoint_end_record(egc, stream);
         else
-- 
2.3.2 (Apple Git-55)


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel

 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.