[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH] 4/7 xen: Add basic NUMA support - Debug Keyhandler



This patch adds a new keyhandler function, dump_numa_stats, which is
triggered on the Xen debug console by pressing 'n'.  This function will
dump a collection of NUMA-related information.  Such information is
useful when debugging domain memory placement on NUMA systems.

In a future patch we would like to export this information via a
hypercall so it can be collected automatically rather than
interactively.

Here is an example of the output on a Dual Opteron (2 Node, 2 CPU, with
one guest running).

(XEN) 'n' pressed -> dumping numa stats
(XEN) ZONE:XEN NUMA hits 33 misses 4 allocs 41 frees 2579
(XEN) ZONE:DOM NUMA hits 17028 misses 17292 allocs 34321 frees 410828
(XEN) ***********************************
(XEN) stats for dom 0:
(XEN) vcpu[0] -> cpu[0]
(XEN) vcpu[1] -> cpu[1]
(XEN) numa hits  : 677
(XEN) numa misses: 912
(XEN) page allocs: 1589
(XEN) total pages currently possessed: 131072
(XEN) total xen heap pages           : 5
(XEN) pages in node         0: 18520
(XEN) pages in node         1: 112552
(XEN) xenheap pages in node 0: 5
(XEN) xenheap pages in node 1: 0
(XEN) pages not matched: 0
(XEN) ***********************************
(XEN) ***********************************
(XEN) stats for dom 1:
(XEN) vcpu[0] -> cpu[1]
(XEN) vcpu[1] -> cpu[0]
(XEN) vcpu[2] -> cpu[1]
(XEN) vcpu[3] -> cpu[0]
(XEN) numa hits  : 16388
(XEN) numa misses: 16383
(XEN) page allocs: 32771
(XEN) total pages currently possessed: 32768
(XEN) total xen heap pages           : 5
(XEN) pages in node         0: 0
(XEN) pages in node         1: 32768
(XEN) xenheap pages in node 0: 5
(XEN) xenheap pages in node 1: 0
(XEN) pages not matched: 0
(XEN) ***********************************
(XEN) free pages in node 0: 499800
(XEN) free pages in node 1: 246712

-- 
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253   T/L: 678-9253
ryanh@xxxxxxxxxx


diffstat output:
 common/keyhandler.c |   99 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 common/page_alloc.c |   43 ++++++++++++++++++++++
 include/xen/numa.h  |    2 +
 3 files changed, 143 insertions(+), 1 deletion(-)

Signed-off-by: Ryan Harper <ryanh@xxxxxxxxxx>
Signed-off-by: Ryan Grimm <grimm@xxxxxxxxxx>
---
diff -r 31e1def2471e xen/common/keyhandler.c
--- a/xen/common/keyhandler.c   Mon Dec 12 19:46:49 2005
+++ b/xen/common/keyhandler.c   Mon Dec 12 14:16:30 2005
@@ -15,6 +15,16 @@
 
 #define KEY_MAX 256
 #define STR_MAX  64
+#ifdef CONFIG_NUMA
+#include <xen/numa.h>
+/* NUMA/Alloc stats */
+extern unsigned long page_alloc[];  /* total page allocs */
+extern unsigned long page_free[];   /* total page frees */
+extern unsigned long numa_hit[];    /* allocated in intended node */
+extern unsigned long numa_miss[];   /* allocated in non intended node */
+#define MEMZONE_XEN 0
+#define MEMZONE_DOM 1
+#endif
 
 static struct {
     union {
@@ -145,6 +155,91 @@
     read_unlock(&domlist_lock);
 }
 
+#ifdef CONFIG_NUMA
+static void dump_numa_stats(unsigned char key)
+{
+     struct domain   *d  = NULL;
+     struct vcpu     *v  = NULL;
+     struct pfn_info *pg = NULL;
+     int i = 0;
+     u32 free_pages[nodes_detected];
+     
+     printk("'%c' pressed -> dumping numa stats\n", key); 
+     printk("ZONE:XEN NUMA hits %ld misses %ld allocs %ld frees %ld\n",
+            numa_hit[MEMZONE_XEN], 
+            numa_miss[MEMZONE_XEN], 
+            page_alloc[MEMZONE_XEN], 
+            page_free[MEMZONE_XEN]);
+     printk("ZONE:DOM NUMA hits %ld misses %ld allocs %ld frees %ld\n",
+            numa_hit[MEMZONE_DOM], 
+            numa_miss[MEMZONE_DOM], 
+            page_alloc[MEMZONE_DOM], 
+            page_free[MEMZONE_DOM]);
+     /* show per-domain vcpu->cpu mappings and numa stats */
+     read_lock(&domlist_lock);
+ 
+     for_each_domain (d) {
+         u32 domain_page_to_nodes[nodes_detected];
+         u32 xenheap_page_to_nodes[nodes_detected];
+         u32 lost_pages = 0;
+ 
+         memset(domain_page_to_nodes, 0, sizeof(domain_page_to_nodes));
+         memset(xenheap_page_to_nodes, 0, sizeof(xenheap_page_to_nodes));
+ 
+         printk("***********************************\n");
+         printk("stats for dom %d:\n", d->domain_id);
+         /* vcpu->cpu mappings */
+         for_each_vcpu (d, v) 
+             /* make sure vcpu is up */
+             if (!test_bit(_VCPUF_down, &v->vcpu_flags))
+                 printk("vcpu[%d] -> cpu[%d]\n", v->vcpu_id, v->processor);
+         
+         /* numa stats */
+         printk("numa hits  : %ld\n", d->numa_hit);
+         printk("numa misses: %ld\n", d->numa_miss);
+         printk("page allocs: %ld\n", d->page_alloc);
+ 
+         /* page to node mappings */
+         spin_lock(&d->page_alloc_lock);
+         list_for_each_entry(pg, &d->page_list, list) {
+             int nid = page_to_node(pg);
+ 
+             if (nid >= 0)
+                 domain_page_to_nodes[nid]++;
+             else 
+                 lost_pages++;
+         }        
+ 
+         list_for_each_entry(pg, &d->xenpage_list, list) {
+             int nid = page_to_node(pg);
+ 
+             if (nid >= 0)
+                 xenheap_page_to_nodes[nid]++;
+             else 
+                 lost_pages++;
+         }        
+         spin_unlock(&d->page_alloc_lock);
+         printk("total pages currently possessed: %d\n", d->tot_pages);
+         printk("total xen heap pages           : %d\n", d->xenheap_pages); 
+         for (i = 0; i < nodes_detected; i++) {
+             printk("pages in node         %d: %d\n", i, 
+                     domain_page_to_nodes[i]);
+         } 
+         for (i = 0; i < nodes_detected; i++) {
+             printk("xenheap pages in node %d: %d\n", i, 
+                     xenheap_page_to_nodes[i]);
+         } 
+         printk("pages not matched: %d\n", lost_pages); 
+         printk("***********************************\n");
+     }
+     read_unlock(&domlist_lock);
+ 
+     memset(free_pages, 0, sizeof(free_pages));
+ 
+     for ( i = 0; i < nodes_detected; i++) 
+         printk("free pages in node %d: %d\n", i, node_free_pages(i)); 
+}
+#endif
 extern void dump_runq(unsigned char key);
 extern void print_sched_histo(unsigned char key);
 extern void reset_sched_histo(unsigned char key);
@@ -187,6 +282,10 @@
         'l', print_sched_histo, "print sched latency histogram");
     register_keyhandler(
         'L', reset_sched_histo, "reset sched latency histogram");
+#ifdef CONFIG_NUMA
+    register_keyhandler(
+        'n', dump_numa_stats, "dump numa stats");
+#endif
     register_keyhandler(
         'q', do_task_queues, "dump task queues + guest state");
     register_keyhandler(
diff -r 31e1def2471e xen/common/page_alloc.c
--- a/xen/common/page_alloc.c   Mon Dec 12 19:46:49 2005
+++ b/xen/common/page_alloc.c   Mon Dec 12 14:16:30 2005
@@ -308,7 +308,7 @@
    u64 cpu_end;   /* highest end addr near cpu */
    u64 start = 0, end = 0, length = (PAGE_SIZE << list_order);
    struct pfn_info *pg = NULL;
-   struct node_memory_chunk_s *c;
+   struct node_memory_chunk_s *c=NULL;
 
    nid = cpu_to_node[cpu];
    cpu_start = node_min_paddr[nid];
@@ -349,6 +349,47 @@
 
  out:
     return pg;
+}
+
+/* page to node mapping */
+int page_to_node(struct pfn_info *pg)
+{
+    u64 pg_paddr = page_to_phys(pg);
+    u64 node_start_paddr = 0ULL, node_end_paddr = 0ULL;
+    u8 nid = 0;
+    int i = 0;
+
+    for (i = 0; i < num_memory_chunks; i++) {
+        node_start_paddr = node_memory_chunk[i].start_paddr;
+        node_end_paddr   = node_memory_chunk[i].end_paddr;
+        nid              = node_memory_chunk[i].nid;
+        
+        if (pg_paddr >= node_start_paddr && pg_paddr <= node_end_paddr)
+           goto found; 
+    }
+    return -1;
+    
+found:
+    return nid;
+} 
+
+int node_free_pages(int nid) 
+{
+    int i = 0, j = 0, free_pages = 0;
+    struct pfn_info *pg = NULL;
+    spin_lock(&heap_lock);
+    for ( i = 0; i < NR_ZONES; i++) 
+        for ( j = 0; j < MAX_ORDER+1; j++) {
+            /* if page is on free list */
+            list_for_each_entry( pg, &heap[i][j], list) {
+                if ( (pg->count_info & PGC_count_mask) == 0) {
+                    if ( page_to_node(pg) == nid)
+                        free_pages+=1<<j;
+                }
+            }
+        }
+    spin_unlock(&heap_lock);
+    return free_pages;
 }
 #endif
 
diff -r 31e1def2471e xen/include/xen/numa.h
--- a/xen/include/xen/numa.h    Mon Dec 12 19:46:49 2005
+++ b/xen/include/xen/numa.h    Mon Dec 12 14:16:30 2005
@@ -42,5 +36,7 @@
 extern unsigned long numa_miss[];   /* allocated in non intended node */
 
 int numa_init(void);
+int page_to_node(struct pfn_info *pg);
+int node_free_pages(int nid);
 
 #endif /* _LINUX_NUMA_H */

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.