[PATCH v2 12/24] tools/libs/guest: Allocate vNUMA memory from associated pNUMA nodes

[Hirokazu Takahashi <taka@xxxxxxxxxxxxx>]

Allocate memory for domU vNUMA nodes from the physical NUMA
nodes specified in the domU configuration file.
---
 tools/libs/guest/xg_dom_arm.c | 62 +++++++++++++++++++++++++++--------
 1 file changed, 49 insertions(+), 13 deletions(-)

diff --git a/tools/libs/guest/xg_dom_arm.c b/tools/libs/guest/xg_dom_arm.c
index c8d0918506..6beab17e48 100644
--- a/tools/libs/guest/xg_dom_arm.c
+++ b/tools/libs/guest/xg_dom_arm.c
@@ -236,7 +236,7 @@ static int set_mode(xc_interface *xch, uint32_t domid, 
const char *guest_type)
  */
 static int populate_one_size(struct xc_dom_image *dom, int pfn_shift,
                              xen_pfn_t base_pfn, xen_pfn_t *nr_pfns,
-                             xen_pfn_t *extents)
+                             xen_pfn_t *extents, unsigned int memflags)
 {
     /* The mask for this level */
     const uint64_t mask = ((uint64_t)1<<(pfn_shift))-1;
@@ -274,7 +274,7 @@ static int populate_one_size(struct xc_dom_image *dom, int 
pfn_shift,
         extents[i] = base_pfn + (i<<pfn_shift);
 
     nr = xc_domain_populate_physmap(dom->xch, dom->guest_domid, count,
-                                    pfn_shift, 0, extents);
+                                    pfn_shift, memflags, extents);
     if ( nr <= 0 ) return nr;
     DOMPRINTF("%s: populated %#x/%#x entries with shift %d",
               __FUNCTION__, nr, count, pfn_shift);
@@ -285,10 +285,12 @@ static int populate_one_size(struct xc_dom_image *dom, 
int pfn_shift,
 }
 
 static int populate_guest_memory(struct xc_dom_image *dom,
-                                 xen_pfn_t base_pfn, xen_pfn_t nr_pfns)
+                                 xen_pfn_t base_pfn, xen_pfn_t nr_pfns,
+                                 unsigned int nid)
 {
     int rc = 0;
     xen_pfn_t allocsz, pfn, *extents;
+    unsigned int memflags = 0U;
 
     extents = calloc(1024*1024,sizeof(xen_pfn_t));
     if ( extents == NULL )
@@ -303,6 +305,16 @@ static int populate_guest_memory(struct xc_dom_image *dom,
               (uint64_t)(base_pfn + nr_pfns) << XC_PAGE_SHIFT,
               (uint64_t)nr_pfns >> (20-XC_PAGE_SHIFT));
 
+    if ( dom->nr_vmemranges > 0 )
+    {
+        memflags = XENMEMF_exact_node(dom->vnode_to_pnode[nid]);
+
+        DOMPRINTF("%s:     on pNODE%u for vNODE%u",
+              __FUNCTION__,
+              dom->vnode_to_pnode[nid],
+              nid);
+    }
+
     for ( pfn = 0; pfn < nr_pfns; pfn += allocsz )
     {
         allocsz = min_t(int, 1024*1024, nr_pfns - pfn);
@@ -312,7 +324,7 @@ static int populate_guest_memory(struct xc_dom_image *dom,
         {
             allocsz = 1;
             rc = populate_one_size(dom, PFN_4K_SHIFT,
-                                   base_pfn + pfn, &allocsz, extents);
+                                   base_pfn + pfn, &allocsz, extents, 
memflags);
             if (rc < 0) break;
             if (rc > 0) continue;
             /* Failed to allocate a single page? */
@@ -321,22 +333,22 @@ static int populate_guest_memory(struct xc_dom_image *dom,
 #endif
 
         rc = populate_one_size(dom, PFN_512G_SHIFT,
-                               base_pfn + pfn, &allocsz, extents);
+                               base_pfn + pfn, &allocsz, extents, memflags);
         if ( rc < 0 ) break;
         if ( rc > 0 ) continue;
 
         rc = populate_one_size(dom, PFN_1G_SHIFT,
-                               base_pfn + pfn, &allocsz, extents);
+                               base_pfn + pfn, &allocsz, extents, memflags);
         if ( rc < 0 ) break;
         if ( rc > 0 ) continue;
 
         rc = populate_one_size(dom, PFN_2M_SHIFT,
-                               base_pfn + pfn, &allocsz, extents);
+                               base_pfn + pfn, &allocsz, extents, memflags);
         if ( rc < 0 ) break;
         if ( rc > 0 ) continue;
 
         rc = populate_one_size(dom, PFN_4K_SHIFT,
-                               base_pfn + pfn, &allocsz, extents);
+                               base_pfn + pfn, &allocsz, extents, memflags);
         if ( rc < 0 ) break;
         if ( rc == 0 )
         {
@@ -415,12 +427,36 @@ static int meminit(struct xc_dom_image *dom)
     assert(ramsize == 0); /* Too much RAM is rejected above */
 
     /* setup initial p2m and allocate guest memory */
-    for ( i = 0; i < GUEST_RAM_BANKS && dom->rambank_size[i]; i++ )
+    if ( dom->nr_vmemranges > 0 )
     {
-        if ((rc = populate_guest_memory(dom,
-                                        bankbase[i] >> XC_PAGE_SHIFT,
-                                        dom->rambank_size[i])))
-            return rc;
+        ramsize = (uint64_t)dom->total_pages << XC_PAGE_SHIFT;
+
+        for ( i = 0; i < dom->nr_vmemranges; i++ )
+        {
+            ramsize -= dom->vmemranges[i].end - dom->vmemranges[i].start;
+        }
+
+        assert(ramsize == 0);
+
+        for ( i = 0; i < dom->nr_vmemranges; i++ )
+        {
+            if ((rc = populate_guest_memory(dom,
+                    dom->vmemranges[i].start  >> XC_PAGE_SHIFT,
+                    (dom->vmemranges[i].end - dom->vmemranges[i].start) >> 
XC_PAGE_SHIFT,
+                    dom->vmemranges[i].nid)))
+                return rc;
+        }
+    }
+    else
+    {
+        for ( i = 0; i < GUEST_RAM_BANKS && dom->rambank_size[i]; i++ )
+        {
+            if ((rc = populate_guest_memory(dom,
+                                            bankbase[i] >> XC_PAGE_SHIFT,
+                                            dom->rambank_size[i],
+                                            0U)))
+                return rc;
+        }
     }
 
     /*
-- 
2.43.0