[Xen-devel] [PATCH] fix memory allocation from NUMA node for VT-d.

[Yuji Shimada <shimada-yxb@xxxxxxxxxxxxxxx>]

The memory relating guest domain should be allocated from NUMA node on
which the guest runs.
Because the latency of the same NUMA node is faster than that of a
different one.

This patch fixes memory allocation for Address Translation Structure
of VT-d.

VT-d uses two types of Structures for DMA address translation.
The one is Device Assignment Structure.
The other is Address Translation Structure.

There is only one Device Assignment Structure on a system.
So, it doesn't need to change memory allocation for Device Assignment
Structure. It means using default policy.

On the other hand, Address Translation Structure exists per guest domain.
So, it needs allocating the memory for Address Translation Structure
from NUMA node which guest domain runs.

This patch is useful for a system which has many IOMMUs.

Thanks,

--
Yuji Shimada

Signed-off-by: Yuji Shimada <shimada-yxb@xxxxxxxxxxxxxxx>

diff -r 5fd51e1e9c79 xen/drivers/passthrough/vtd/intremap.c
--- a/xen/drivers/passthrough/vtd/intremap.c    Wed Nov 05 10:57:21 2008 +0000
+++ b/xen/drivers/passthrough/vtd/intremap.c    Tue Nov 18 17:37:31 2008 +0900
@@ -473,7 +473,7 @@
     ir_ctrl = iommu_ir_ctrl(iommu);
     if ( ir_ctrl->iremap_maddr == 0 )
     {
-        ir_ctrl->iremap_maddr = alloc_pgtable_maddr();
+        ir_ctrl->iremap_maddr = alloc_pgtable_maddr(NULL);
         if ( ir_ctrl->iremap_maddr == 0 )
         {
             dprintk(XENLOG_WARNING VTDPREFIX,
diff -r 5fd51e1e9c79 xen/drivers/passthrough/vtd/iommu.c
--- a/xen/drivers/passthrough/vtd/iommu.c       Wed Nov 05 10:57:21 2008 +0000
+++ b/xen/drivers/passthrough/vtd/iommu.c       Tue Nov 18 17:37:31 2008 +0900
@@ -148,7 +148,7 @@
     root = &root_entries[bus];
     if ( !root_present(*root) )
     {
-        maddr = alloc_pgtable_maddr();
+        maddr = alloc_pgtable_maddr(NULL);
         if ( maddr == 0 )
         {
             unmap_vtd_domain_page(root_entries);
@@ -205,7 +205,7 @@
     addr &= (((u64)1) << addr_width) - 1;
     spin_lock_irqsave(&hd->mapping_lock, flags);
     if ( hd->pgd_maddr == 0 )
-        if ( !alloc || ((hd->pgd_maddr = alloc_pgtable_maddr()) == 0) )
+        if ( !alloc || ((hd->pgd_maddr = alloc_pgtable_maddr(domain)) == 0) )
             goto out;
 
     parent = (struct dma_pte *)map_vtd_domain_page(hd->pgd_maddr);
@@ -218,7 +218,7 @@
         {
             if ( !alloc )
                 break;
-            maddr = alloc_pgtable_maddr();
+            maddr = alloc_pgtable_maddr(domain);
             if ( !maddr )
                 break;
             dma_set_pte_addr(*pte, maddr);
@@ -605,7 +605,7 @@
     spin_lock_irqsave(&iommu->register_lock, flags);
 
     if ( iommu->root_maddr == 0 )
-        iommu->root_maddr = alloc_pgtable_maddr();
+        iommu->root_maddr = alloc_pgtable_maddr(NULL);
     if ( iommu->root_maddr == 0 )
     {
         spin_unlock_irqrestore(&iommu->register_lock, flags);
diff -r 5fd51e1e9c79 xen/drivers/passthrough/vtd/qinval.c
--- a/xen/drivers/passthrough/vtd/qinval.c      Wed Nov 05 10:57:21 2008 +0000
+++ b/xen/drivers/passthrough/vtd/qinval.c      Tue Nov 18 17:37:31 2008 +0900
@@ -426,7 +426,7 @@
 
     if ( qi_ctrl->qinval_maddr == 0 )
     {
-        qi_ctrl->qinval_maddr = alloc_pgtable_maddr();
+        qi_ctrl->qinval_maddr = alloc_pgtable_maddr(NULL);
         if ( qi_ctrl->qinval_maddr == 0 )
         {
             dprintk(XENLOG_WARNING VTDPREFIX,
diff -r 5fd51e1e9c79 xen/drivers/passthrough/vtd/vtd.h
--- a/xen/drivers/passthrough/vtd/vtd.h Wed Nov 05 10:57:21 2008 +0000
+++ b/xen/drivers/passthrough/vtd/vtd.h Tue Nov 18 17:37:31 2008 +0900
@@ -101,7 +101,7 @@
 void cacheline_flush(char *);
 void flush_all_cache(void);
 void *map_to_nocache_virt(int nr_iommus, u64 maddr);
-u64 alloc_pgtable_maddr(void);
+u64 alloc_pgtable_maddr(struct domain *d);
 void free_pgtable_maddr(u64 maddr);
 void *map_vtd_domain_page(u64 maddr);
 void unmap_vtd_domain_page(void *va);
diff -r 5fd51e1e9c79 xen/drivers/passthrough/vtd/x86/vtd.c
--- a/xen/drivers/passthrough/vtd/x86/vtd.c     Wed Nov 05 10:57:21 2008 +0000
+++ b/xen/drivers/passthrough/vtd/x86/vtd.c     Tue Nov 18 17:37:31 2008 +0900
@@ -22,6 +22,7 @@
 #include <xen/domain_page.h>
 #include <asm/paging.h>
 #include <xen/iommu.h>
+#include <xen/numa.h>
 #include "../iommu.h"
 #include "../dmar.h"
 #include "../vtd.h"
@@ -37,13 +38,21 @@
 }
 
 /* Allocate page table, return its machine address */
-u64 alloc_pgtable_maddr(void)
+u64 alloc_pgtable_maddr(struct domain *d)
 {
     struct page_info *pg;
     u64 *vaddr;
     unsigned long mfn;
 
-    pg = alloc_domheap_page(NULL, 0);
+    if (d == NULL)
+    {
+        pg = alloc_domheap_page(NULL, 0);
+    }
+    else
+    {
+        pg = alloc_domheap_page(NULL, MEMF_node(domain_to_node(d)));
+    }
+
     if ( !pg )
         return 0;
     mfn = page_to_mfn(pg);


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel