[Xen-devel] [PATCH RFC 6/7] libxc/vnuma: vnuma per phys NUMA allocation.

[Elena Ufimtseva <ufimtseva@xxxxxxxxx>]

Allows for vNUMA enabled domains to allocate vnodes on physical NUMA nodes.
Tries to allocate all vnodes on one NUMA node, or on next one if not all
vnodes fit. If no physical numa node found, will let xen decide.

TODO:
take into account cpu pinning if defined in VM config;
take into account automatic NUMA placement mechanism;

Allows for vNUMA enabled domains to allocate vnodes
on physical NUMA nodes. Adds some arch bits.

Signed-off-by: Elena Ufimtseva <ufimtseva@xxxxxxxxx>
---
 tools/libxc/xc_dom.h     |    1 +
 tools/libxc/xc_dom_x86.c |   79 ++++++++++++++++++++++++++++++++++++++++------
 tools/libxc/xg_private.h |    4 +++
 3 files changed, 75 insertions(+), 9 deletions(-)

diff --git a/tools/libxc/xc_dom.h b/tools/libxc/xc_dom.h
index 4375f25..7037614 100644
--- a/tools/libxc/xc_dom.h
+++ b/tools/libxc/xc_dom.h
@@ -371,6 +371,7 @@ static inline xen_pfn_t xc_dom_p2m_guest(struct 
xc_dom_image *dom,
 int arch_setup_meminit(struct xc_dom_image *dom);
 int arch_setup_bootearly(struct xc_dom_image *dom);
 int arch_setup_bootlate(struct xc_dom_image *dom);
+int arch_boot_numa_alloc(struct xc_dom_image *dom);
 
 /*
  * Local variables:
diff --git a/tools/libxc/xc_dom_x86.c b/tools/libxc/xc_dom_x86.c
index 126c0f8..99f7444 100644
--- a/tools/libxc/xc_dom_x86.c
+++ b/tools/libxc/xc_dom_x86.c
@@ -789,27 +789,47 @@ int arch_setup_meminit(struct xc_dom_image *dom)
     else
     {
         /* try to claim pages for early warning of insufficient memory avail */
+        rc = 0;
         if ( dom->claim_enabled ) {
             rc = xc_domain_claim_pages(dom->xch, dom->guest_domid,
                                        dom->total_pages);
             if ( rc )
+            {
+                xc_dom_panic(dom->xch, XC_INTERNAL_ERROR,
+                             "%s: Failed to claim mem for dom\n",
+                             __FUNCTION__);
                 return rc;
+            }
         }
         /* setup initial p2m */
         for ( pfn = 0; pfn < dom->total_pages; pfn++ )
             dom->p2m_host[pfn] = pfn;
         
         /* allocate guest memory */
-        for ( i = rc = allocsz = 0;
-              (i < dom->total_pages) && !rc;
-              i += allocsz )
+        if (dom->nr_vnodes > 0)
         {
-            allocsz = dom->total_pages - i;
-            if ( allocsz > 1024*1024 )
-                allocsz = 1024*1024;
-            rc = xc_domain_populate_physmap_exact(
-                dom->xch, dom->guest_domid, allocsz,
-                0, 0, &dom->p2m_host[i]);
+            rc = arch_boot_numa_alloc(dom);
+            if ( rc )
+            {
+                xc_dom_panic(dom->xch, XC_INTERNAL_ERROR,
+                             "%s: Failed to allocate memory on NUMA nodes\n",
+                             __FUNCTION__);
+                return rc;
+            }
+        }
+        else
+        {
+            for ( i = rc = allocsz = 0;
+                  (i < dom->total_pages) && !rc;
+                  i += allocsz )
+            {
+                allocsz = dom->total_pages - i;
+                if ( allocsz > 1024*1024 )
+                    allocsz = 1024*1024;
+                rc = xc_domain_populate_physmap_exact(
+                    dom->xch, dom->guest_domid, allocsz,
+                    0, 0, &dom->p2m_host[i]);
+            }
         }
 
         /* Ensure no unclaimed pages are left unused.
@@ -817,7 +837,48 @@ int arch_setup_meminit(struct xc_dom_image *dom)
         (void)xc_domain_claim_pages(dom->xch, dom->guest_domid,
                                     0 /* cancels the claim */);
     }
+    return rc;
+}
+
+int arch_boot_numa_alloc(struct xc_dom_image *dom)
+{ 
+    int rc, n;
+    uint64_t guest_pages;
+    unsigned long allocsz, i, k;
+    unsigned long memflags;
 
+    rc = allocsz = k = 0;
+    for(n = 0; n < dom->nr_vnodes; n++)
+    {
+        memflags = 0;
+        if ( dom->vnode_to_pnode[n] != NUMA_NO_NODE )
+        {
+            memflags |= XENMEMF_exact_node(dom->vnode_to_pnode[n]);
+            memflags |= XENMEMF_exact_node_request;
+        }
+        guest_pages = dom->vmemsizes[n] >> PAGE_SHIFT_X86;
+        for ( i = 0;
+            (i < guest_pages) && !rc;
+                i += allocsz )
+        {
+            allocsz = guest_pages - i;
+            if ( allocsz > 1024*1024 )
+                allocsz = 1024*1024;
+                rc = xc_domain_populate_physmap_exact(
+                                    dom->xch, dom->guest_domid, allocsz,
+                                    0, memflags, &dom->p2m_host[i + k]);
+            k += allocsz;
+        }
+        if (rc == 0) printf("%s: allocated %lx pages for vnode %d on pnode %d 
out of %lx\n",
+                       __FUNCTION__,i, n, dom->vnode_to_pnode[n], 
dom->total_pages);
+        else
+        {
+            xc_dom_panic(dom->xch, XC_INTERNAL_ERROR,
+                    "%s: Failed allocation of %lx pages for vnode %d on pnode 
%d out of %lx\n",
+                    __FUNCTION__,i, n, dom->vnode_to_pnode[n], 
dom->total_pages);
+            return -EINVAL;
+        }
+    }
     return rc;
 }
 
diff --git a/tools/libxc/xg_private.h b/tools/libxc/xg_private.h
index db02ccf..538d185 100644
--- a/tools/libxc/xg_private.h
+++ b/tools/libxc/xg_private.h
@@ -127,6 +127,10 @@ typedef uint64_t l4_pgentry_64_t;
 #define ROUNDUP(_x,_w) (((unsigned long)(_x)+(1UL<<(_w))-1) & ~((1UL<<(_w))-1))
 #define NRPAGES(x) (ROUNDUP(x, PAGE_SHIFT) >> PAGE_SHIFT)
 
+#define MAX_ORDER_X86           11
+#define NODE_MIN_SIZE_X86       1024*1024*4
+#define ZONE_ALIGN_X86          (1UL << (MAX_ORDER_X86 + PAGE_SHIFT_X86))
+#define NUMA_NO_NODE            0xFF
 
 /* XXX SMH: following skanky macros rely on variable p2m_size being set */
 /* XXX TJD: also, "guest_width" should be the guest's sizeof(unsigned long) */
-- 
1.7.10.4


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel