[PATCH for-4.22 v2 5/5] xen/numa: fix setup of non-aligned memory affinity ranges

[Roger Pau Monne <roger.pau@xxxxxxxxxx>]

The logic to populate memnodemap in populate_memnodemap() assumes that all
ranges are aligned to the hash shift, this however is only true for the
first address in a memory affinity node.  Any subsequent ranges belonging
to the same node might not be aligned to the hash shift value.

Such lack of alignment causes issues to the logic in populate_memnodemap(),
as then the tail of the range might not be properly accounted for and setup
in memnodemap.  Fix this by forcing the start address of all regions to
be aligned to the hash shift; if such alignment causes a region overlap it
would always be between regions on the same node, and hence will never
cause setup issues of the memnodemap array.

Introduce two additional test cases to the user-space NUMA setup unit
testing, first test case is the native memory affinity and memory map of
the system where this issue was found, second test case is a simplification
to demonstrate the original problem more clearly.

Fixes: 1666086b0044 ("x86/NUMA: improve memnode_shift calculation for multi 
node system")
Signed-off-by: Roger Pau Monné <roger.pau@xxxxxxxxxx>
Reviewed-by: Jan Beulich <jbeulich@xxxxxxxx>
---
Changes since v1:
 - Do the rounding after the checking.
---
 tools/include/xen-tools/common-macros.h |  1 +
 tools/tests/numa/test-numa.c            | 45 +++++++++++++++++++++++++
 xen/common/numa.c                       |  6 ++++
 3 files changed, 52 insertions(+)

diff --git a/tools/include/xen-tools/common-macros.h 
b/tools/include/xen-tools/common-macros.h
index 9e2799178235..88b4a0e5a693 100644
--- a/tools/include/xen-tools/common-macros.h
+++ b/tools/include/xen-tools/common-macros.h
@@ -69,6 +69,7 @@
 #endif
 
 #define ROUNDUP(x, a) (((x) + (a) - 1) & ~((a) - 1))
+#define ROUNDDOWN(x, a) ((x) & ~((a) - 1))
 
 #define MASK_EXTR(v, m) (((v) & (m)) / ((m) & -(m)))
 #define MASK_INSR(v, m) (((v) * ((m) & -(m))) & (m))
diff --git a/tools/tests/numa/test-numa.c b/tools/tests/numa/test-numa.c
index bced68d4d7f1..8122e63a88ed 100644
--- a/tools/tests/numa/test-numa.c
+++ b/tools/tests/numa/test-numa.c
@@ -158,6 +158,51 @@ int main(int argc, char **argv)
                 { .start = 0x183f8800000ULL, .end = 0x183faabffffULL },
             },
         },
+        /* System with unaligned affinity memblock. */
+        {
+            .affinity = {
+                { .nid = 0, .start = 0x00000000000ULL, .end = 0x000afffffffULL 
},
+                { .nid = 0, .start = 0x00100000000ULL, .end = 0x0fc4fffffffULL 
},
+                { .nid = 0, .start = 0x10000000000ULL, .end = 0x103ffffffffULL 
},
+                { .nid = 1, .start = 0x10400000000ULL, .end = 0x203ffffffffULL 
},
+            },
+            .ram = {
+                { .start = 0x00000000000ULL, .end = 0x0000009ffffULL },
+                { .start = 0x00000100000ULL, .end = 0x000165bffffULL },
+                { .start = 0x00016600000ULL, .end = 0x0001aa1dfffULL },
+                { .start = 0x0001aa1f000ULL, .end = 0x0001aa53fffULL },
+                { .start = 0x0001aab8000ULL, .end = 0x0001aac6fffULL },
+                { .start = 0x0001aacc000ULL, .end = 0x0006f3fefffULL },
+                { .start = 0x00075dff000ULL, .end = 0x00075dfffffULL },
+                { .start = 0x00076000000ULL, .end = 0x000a7ffffffULL },
+                { .start = 0x00100010000ULL, .end = 0x0fc43ffffffULL },
+                { .start = 0x0fc45000000ULL, .end = 0x0fc47ffffffULL },
+                { .start = 0x0fc49000000ULL, .end = 0x0fc4bffffffULL },
+                { .start = 0x0fc4d000000ULL, .end = 0x0fc4d3bffffULL },
+                { .start = 0x0fc4f000000ULL, .end = 0x0fc4f0fffffULL },
+                { .start = 0x10000000000ULL, .end = 0x203fd7fffffULL },
+            },
+        },
+        /*
+         * Reduction of the issue above: introduce an unaligned middle region
+         * with regards to the hash shift.
+         */
+        {
+            .affinity = {
+                { .nid = 0, .start = 0x00000ULL, .end = 0x00fffULL },
+                /*
+                 * The offset of the region below is not aligned with the hash
+                 * shift: the shift calculation only takes into account the
+                 * start of node address.
+                 */
+                { .nid = 0, .start = 0x01000ULL, .end = 0x04fffULL },
+                { .nid = 1, .start = 0x14000ULL, .end = 0x14fffULL },
+            },
+            .ram = {
+                { .start = 0x00000ULL, .end = 0x04fffULL },
+                { .start = 0x14000ULL, .end = 0x14fffULL },
+            },
+        },
     };
     int ret_code = EXIT_SUCCESS;
 
diff --git a/xen/common/numa.c b/xen/common/numa.c
index 8544a1598218..92f8f1cedce1 100644
--- a/xen/common/numa.c
+++ b/xen/common/numa.c
@@ -405,6 +405,12 @@ static int __init populate_memnodemap(const struct node 
*nodes,
         if ( (epdx >> shift) >= memnodemapsize )
             return 0;
 
+        /*
+         * Round down start address: if start is not aligned to the memnodemap
+         * chunk size the tail remainder might not be added.  Overlaps created
+         * by rounding will fall into the same NUMA region.
+         */
+        spdx = ROUNDDOWN(spdx, 1UL << shift);
         do {
             if ( memnodemap[spdx >> shift] != NUMA_NO_NODE &&
                  (!nodeids || memnodemap[spdx >> shift] != nodeids[i]) )
-- 
2.53.0