[Xen-devel] [PATCH V4 7/8] COLO-Proxy: Use socket to get checkpoint event.

[Zhang Chen <zhangchen.fnst@xxxxxxxxxxxxxx>]

We use kernel colo proxy's way to get the checkpoint event
from qemu colo-compare.
Qemu colo-compare need add a API to support this(I will add this in qemu).
Qemu side patch:
 https://lists.nongnu.org/archive/html/qemu-devel/2017-02/msg07265.html

Signed-off-by: Zhang Chen <zhangchen.fnst@xxxxxxxxxxxxxx>
---
 tools/libxl/libxl_colo.h         |   2 +
 tools/libxl/libxl_colo_proxy.c   | 126 +++++++++++++++++++++++++++++++++++++--
 tools/libxl/libxl_colo_restore.c |  11 ++--
 tools/libxl/libxl_colo_save.c    |  21 ++++---
 tools/libxl/libxl_nic.c          |   4 ++
 tools/libxl/libxl_types.idl      |   4 +-
 tools/xl/xl_parse.c              |   4 ++
 7 files changed, 156 insertions(+), 16 deletions(-)

diff --git a/tools/libxl/libxl_colo.h b/tools/libxl/libxl_colo.h
index 4746d8c..6c01b55 100644
--- a/tools/libxl/libxl_colo.h
+++ b/tools/libxl/libxl_colo.h
@@ -69,6 +69,8 @@ struct libxl__colo_proxy_state {
      *          False means use kernel colo proxy.
      */
     bool is_userspace_proxy;
+    const char *checkpoint_host;
+    const char *checkpoint_port;
 };
 
 struct libxl__colo_save_state {
diff --git a/tools/libxl/libxl_colo_proxy.c b/tools/libxl/libxl_colo_proxy.c
index ec76e37..c3d5510 100644
--- a/tools/libxl/libxl_colo_proxy.c
+++ b/tools/libxl/libxl_colo_proxy.c
@@ -18,9 +18,13 @@
 #include "libxl_internal.h"
 
 #include <netlink/netlink.h>
+#include <arpa/inet.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
 
 /* Consistent with the new COLO netlink channel in kernel side */
 #define NETLINK_COLO 28
+#define COLO_DEFAULT_WAIT_TIME 500000
 
 enum colo_netlink_op {
     COLO_QUERY_CHECKPOINT = (NLMSG_MIN_TYPE + 1),
@@ -76,6 +80,63 @@ static int colo_proxy_send(libxl__colo_proxy_state *cps, 
uint8_t *buff,
     return ret;
 }
 
+static int colo_userspace_proxy_send(libxl__colo_proxy_state *cps,
+                                     uint8_t *buff,
+                                     uint32_t size)
+{
+    int ret = 0;
+    uint32_t len = 0;
+
+    len = htonl(size);
+    ret = send(cps->sock_fd, (uint8_t *)&len, sizeof(len), 0);
+    if (ret != sizeof(len)) {
+        goto err;
+    }
+
+    ret = send(cps->sock_fd, (uint8_t *)buff, size, 0);
+    if (ret != size) {
+        goto err;
+    }
+
+err:
+    return ret;
+}
+
+static int colo_userspace_proxy_recv(libxl__colo_proxy_state *cps,
+                                     char *buff,
+                                     unsigned int timeout_us)
+{
+    struct timeval tv;
+    int ret;
+    uint32_t len = 0;
+    uint32_t size = 0;
+
+    STATE_AO_GC(cps->ao);
+
+    if (timeout_us) {
+        tv.tv_sec = timeout_us / 1000000;
+        tv.tv_usec = timeout_us % 1000000;
+        ret = setsockopt(cps->sock_fd, SOL_SOCKET, SO_RCVTIMEO, &tv,
+                         sizeof(tv));
+        if (ret < 0) {
+            LOGD(ERROR, ao->domid,
+                 "colo_userspace_proxy_recv setsockopt error: %s",
+                 strerror(errno));
+        }
+    }
+
+    ret = recv(cps->sock_fd, (uint8_t *)&len, sizeof(len), 0);
+    if (ret < 0) {
+        goto err;
+    }
+
+    size = ntohl(len);
+    ret = recv(cps->sock_fd, buff, size, 0);
+
+err:
+    return ret;
+}
+
 /* error: return -1, otherwise return 0 */
 static int64_t colo_proxy_recv(libxl__colo_proxy_state *cps, uint8_t **buff,
                                unsigned int timeout_us)
@@ -153,8 +214,45 @@ int colo_proxy_setup(libxl__colo_proxy_state *cps)
     STATE_AO_GC(cps->ao);
 
     /* If enable userspace proxy mode, we don't need setup kernel proxy */
-    if (cps->is_userspace_proxy)
+    if (cps->is_userspace_proxy) {
+        struct sockaddr_in addr;
+        int port;
+        char recvbuff[1024];
+        const char sendbuf[] = "COLO_USERSPACE_PROXY_INIT";
+
+        memset(&addr, 0, sizeof(addr));
+        port = atoi(cps->checkpoint_port);
+        addr.sin_family = AF_INET;
+        addr.sin_port = htons(port);
+        addr.sin_addr.s_addr = inet_addr(cps->checkpoint_host);
+
+        skfd = socket(AF_INET, SOCK_STREAM, 0);
+        if (skfd < 0) {
+            LOGD(ERROR, ao->domid, "can not create a TCP socket: %s",
+                 strerror(errno));
+            goto out;
+        }
+
+        cps->sock_fd = skfd;
+
+        if (connect(skfd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
+            LOGD(ERROR, ao->domid, "connect error");
+            goto out;
+        }
+
+        ret = colo_userspace_proxy_send(cps, (uint8_t *)sendbuf, 
strlen(sendbuf));
+        if (ret < 0)
+            goto out;
+
+        ret = colo_userspace_proxy_recv(cps, recvbuff, COLO_DEFAULT_WAIT_TIME);
+        if (ret < 0) {
+            LOGD(ERROR, ao->domid, "Can't recv msg from qemu colo-compare: %s",
+                 strerror(errno));
+            goto out;
+        }
+
         return 0;
+    }
 
     skfd = socket(PF_NETLINK, SOCK_RAW, NETLINK_COLO);
     if (skfd < 0) {
@@ -247,8 +345,13 @@ void colo_proxy_preresume(libxl__colo_proxy_state *cps)
      * If enable userspace proxy mode,
      * we don't need preresume kernel proxy
      */
-    if (cps->is_userspace_proxy)
+    if (cps->is_userspace_proxy) {
+        const char sendbuf[] = "COLO_CHECKPOINT";
+        colo_userspace_proxy_send(cps,
+                                  (uint8_t *)sendbuf,
+                                  strlen(sendbuf));
         return;
+    }
 
     colo_proxy_send(cps, NULL, 0, COLO_CHECKPOINT);
     /* TODO: need to handle if the call fails... */
@@ -277,6 +380,7 @@ int colo_proxy_checkpoint(libxl__colo_proxy_state *cps,
     struct nlmsghdr *h;
     struct colo_msg *m;
     int ret = -1;
+    char recvbuff[1024];
 
     STATE_AO_GC(cps->ao);
 
@@ -289,8 +393,19 @@ int colo_proxy_checkpoint(libxl__colo_proxy_state *cps,
      * event.
      */
     if (cps->is_userspace_proxy) {
-        usleep(timeout_us);
-        return 0;
+        ret = colo_userspace_proxy_recv(cps, recvbuff, timeout_us);
+        if (ret <= 0) {
+            ret = 0;
+            goto out1;
+        }
+
+        if (!strcmp(recvbuff, "DO_CHECKPOINT")) {
+            ret = 1;
+        } else {
+            LOGD(ERROR, ao->domid, "receive qemu colo-compare checkpoint 
error");
+            ret = 0;
+        }
+        goto out1;
     }
 
     size = colo_proxy_recv(cps, &buff, timeout_us);
@@ -318,4 +433,7 @@ int colo_proxy_checkpoint(libxl__colo_proxy_state *cps,
 out:
     free(buff);
     return ret;
+
+out1:
+    return ret;
 }
diff --git a/tools/libxl/libxl_colo_restore.c b/tools/libxl/libxl_colo_restore.c
index c6d239a..065ea00 100644
--- a/tools/libxl/libxl_colo_restore.c
+++ b/tools/libxl/libxl_colo_restore.c
@@ -613,7 +613,8 @@ static void colo_restore_preresume_cb(libxl__egc *egc,
         }
     }
 
-    colo_proxy_preresume(&crs->cps);
+    if (!crs->cps.is_userspace_proxy)
+        colo_proxy_preresume(&crs->cps);
 
     colo_restore_resume_vm(egc, crcs);
 
@@ -786,9 +787,11 @@ static void colo_setup_checkpoint_devices(libxl__egc *egc,
     cds->ops = colo_restore_ops;
 
     crs->cps.ao = ao;
-    if (colo_proxy_setup(&crs->cps)) {
-        LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest");
-        goto out;
+    if (!crs->cps.is_userspace_proxy) {
+        if (colo_proxy_setup(&crs->cps)) {
+            LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for 
guest");
+            goto out;
+        }
     }
 
     if (init_device_subkind(cds))
diff --git a/tools/libxl/libxl_colo_save.c b/tools/libxl/libxl_colo_save.c
index 91e3fce..b4ca9f6 100644
--- a/tools/libxl/libxl_colo_save.c
+++ b/tools/libxl/libxl_colo_save.c
@@ -86,6 +86,7 @@ void libxl__colo_save_setup(libxl__egc *egc, 
libxl__colo_save_state *css)
     libxl__checkpoint_devices_state *const cds = &dss->cds;
     libxl__srm_save_autogen_callbacks *const callbacks =
         &dss->sws.shs.callbacks.save.a;
+    libxl_device_nic *nics;
 
     STATE_AO_GC(dss->ao);
 
@@ -110,19 +111,25 @@ void libxl__colo_save_setup(libxl__egc *egc, 
libxl__colo_save_state *css)
         css->colo_proxy_script = GCSPRINTF("%s/colo-proxy-setup",
                                            libxl__xen_script_dir_path());
 
-    /* If enable userspace proxy mode, we don't need VIF */
-    if (css->cps.is_userspace_proxy)
-        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD);
-    else
-        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) |
-                                 (1 << LIBXL__DEVICE_KIND_VBD);
-
     cds->ops = colo_ops;
     cds->callback = colo_save_setup_done;
     cds->ao = ao;
     cds->domid = dss->domid;
     cds->concrete_data = css;
 
+    /* If enable userspace proxy mode, we don't need VIF */
+    if (css->cps.is_userspace_proxy) {
+        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD);
+
+        /* Use this args we can connect to qemu colo-compare */
+        nics = libxl_device_nic_list(CTX, cds->domid, &cds->num_nics);
+        css->cps.checkpoint_host = nics->colo_checkpoint_host;
+        css->cps.checkpoint_port = nics->colo_checkpoint_port;
+    } else {
+        cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) |
+                                 (1 << LIBXL__DEVICE_KIND_VBD);
+    }
+
     css->srs.ao = ao;
     css->srs.fd = css->recv_fd;
     css->srs.back_channel = true;
diff --git a/tools/libxl/libxl_nic.c b/tools/libxl/libxl_nic.c
index 33954a2..4b6e8c0 100644
--- a/tools/libxl/libxl_nic.c
+++ b/tools/libxl/libxl_nic.c
@@ -250,6 +250,8 @@ static void libxl__device_nic_add(libxl__egc *egc, uint32_t 
domid,
     MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_indev);
     MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_outdev);
     MAYBE_ADD_COLO_ARGS(filter_sec_rewriter0_queue);
+    MAYBE_ADD_COLO_ARGS(checkpoint_host);
+    MAYBE_ADD_COLO_ARGS(checkpoint_port);
 
 #undef MAYBE_ADD_COLO_ARGS
 
@@ -459,6 +461,8 @@ static int libxl__device_nic_from_xenstore(libxl__gc *gc,
     CHECK_COLO_ARGS(filter_sec_redirector1_indev);
     CHECK_COLO_ARGS(filter_sec_redirector1_outdev);
     CHECK_COLO_ARGS(filter_sec_rewriter0_queue);
+    CHECK_COLO_ARGS(checkpoint_host);
+    CHECK_COLO_ARGS(checkpoint_port);
 
 #undef CHECK_COLO_ARGS
 
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 516bd79..6d28dea 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -675,7 +675,9 @@ libxl_device_nic = Struct("device_nic", [
     ("colo_filter_sec_redirector1_queue", string),
     ("colo_filter_sec_redirector1_indev", string),
     ("colo_filter_sec_redirector1_outdev", string),
-    ("colo_filter_sec_rewriter0_queue", string)
+    ("colo_filter_sec_rewriter0_queue", string),
+    ("colo_checkpoint_host", string),
+    ("colo_checkpoint_port", string)
     ])
 
 libxl_device_pci = Struct("device_pci", [
diff --git a/tools/xl/xl_parse.c b/tools/xl/xl_parse.c
index 77e91c8..0787337 100644
--- a/tools/xl/xl_parse.c
+++ b/tools/xl/xl_parse.c
@@ -551,6 +551,10 @@ int parse_nic_config(libxl_device_nic *nic, XLU_Config 
**config, char *token)
         replace_string(&nic->colo_filter_sec_redirector1_outdev, oparg);
     } else if (MATCH_OPTION("colo_filter_sec_rewriter0_queue", token, oparg)) {
         replace_string(&nic->colo_filter_sec_rewriter0_queue, oparg);
+    } else if (MATCH_OPTION("colo_checkpoint_host", token, oparg)) {
+        replace_string(&nic->colo_checkpoint_host, oparg);
+    } else if (MATCH_OPTION("colo_checkpoint_port", token, oparg)) {
+        replace_string(&nic->colo_checkpoint_port, oparg);
     } else if (MATCH_OPTION("accel", token, oparg)) {
         fprintf(stderr, "the accel parameter for vifs is currently not 
supported\n");
     } else if (MATCH_OPTION("devid", token, oparg)) {
-- 
2.7.4




_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel