Re: [Xen-devel] [PATCH] fix memory allocation from NUMA node for VT-d.

[Espen Skoglund <espen.skoglund@xxxxxxxxxxxxx>]

Given an FSB based system the IOMMUs sit in the north-bridge.  How
does this work qith QPI?  Where in the system do the different IOMMUs
sit?  Wouldn't it make more sense to allocate memory from one of the
nodes where the IOMMU is attached?  Having the memory allocated from
the node of the guest only helps when the guest needs to update its
page tables.  I'd rather optimize for page table walks in the IOMMU.

        eSk


[Yuji Shimada]
> The memory relating guest domain should be allocated from NUMA node
> on which the guest runs.  Because the latency of the same NUMA node
> is faster than that of a different one.

> This patch fixes memory allocation for Address Translation Structure
> of VT-d.

> VT-d uses two types of Structures for DMA address translation.  The
> one is Device Assignment Structure.  The other is Address
> Translation Structure.

> There is only one Device Assignment Structure on a system.
> So, it doesn't need to change memory allocation for Device Assignment
> Structure. It means using default policy.

> On the other hand, Address Translation Structure exists per guest
> domain.  So, it needs allocating the memory for Address Translation
> Structure from NUMA node which guest domain runs.

> This patch is useful for a system which has many IOMMUs.

> Thanks,

> --
> Yuji Shimada

> Signed-off-by: Yuji Shimada <shimada-yxb@xxxxxxxxxxxxxxx>

> diff -r 5fd51e1e9c79 xen/drivers/passthrough/vtd/intremap.c
> --- a/xen/drivers/passthrough/vtd/intremap.c  Wed Nov 05 10:57:21 2008 +0000
> +++ b/xen/drivers/passthrough/vtd/intremap.c  Tue Nov 18 17:37:31 2008 +0900
> @@ -473,7 +473,7 @@
>      ir_ctrl = iommu_ir_ctrl(iommu);
>      if ( ir_ctrl->iremap_maddr == 0 )
>      {
> -        ir_ctrl->iremap_maddr = alloc_pgtable_maddr();
> +        ir_ctrl->iremap_maddr = alloc_pgtable_maddr(NULL);
>          if ( ir_ctrl->iremap_maddr == 0 )
>          {
>              dprintk(XENLOG_WARNING VTDPREFIX,
> diff -r 5fd51e1e9c79 xen/drivers/passthrough/vtd/iommu.c
> --- a/xen/drivers/passthrough/vtd/iommu.c     Wed Nov 05 10:57:21 2008 +0000
> +++ b/xen/drivers/passthrough/vtd/iommu.c     Tue Nov 18 17:37:31 2008 +0900
> @@ -148,7 +148,7 @@
>      root = &root_entries[bus];
>      if ( !root_present(*root) )
>      {
> -        maddr = alloc_pgtable_maddr();
> +        maddr = alloc_pgtable_maddr(NULL);
>          if ( maddr == 0 )
>          {
>              unmap_vtd_domain_page(root_entries);
> @@ -205,7 +205,7 @@
>      addr &= (((u64)1) << addr_width) - 1;
>      spin_lock_irqsave(&hd->mapping_lock, flags);
>      if ( hd->pgd_maddr == 0 )
> -        if ( !alloc || ((hd->pgd_maddr = alloc_pgtable_maddr()) == 0) )
> +        if ( !alloc || ((hd->pgd_maddr = alloc_pgtable_maddr(domain)) == 0) )
>              goto out;
 
>      parent = (struct dma_pte *)map_vtd_domain_page(hd->pgd_maddr);
> @@ -218,7 +218,7 @@
>          {
>              if ( !alloc )
>                  break;
> -            maddr = alloc_pgtable_maddr();
> +            maddr = alloc_pgtable_maddr(domain);
>              if ( !maddr )
>                  break;
>              dma_set_pte_addr(*pte, maddr);
> @@ -605,7 +605,7 @@
>      spin_lock_irqsave(&iommu->register_lock, flags);
 
>      if ( iommu->root_maddr == 0 )
> -        iommu->root_maddr = alloc_pgtable_maddr();
> +        iommu->root_maddr = alloc_pgtable_maddr(NULL);
>      if ( iommu->root_maddr == 0 )
>      {
>          spin_unlock_irqrestore(&iommu->register_lock, flags);
> diff -r 5fd51e1e9c79 xen/drivers/passthrough/vtd/qinval.c
> --- a/xen/drivers/passthrough/vtd/qinval.c    Wed Nov 05 10:57:21 2008 +0000
> +++ b/xen/drivers/passthrough/vtd/qinval.c    Tue Nov 18 17:37:31 2008 +0900
> @@ -426,7 +426,7 @@
 
>      if ( qi_ctrl->qinval_maddr == 0 )
>      {
> -        qi_ctrl->qinval_maddr = alloc_pgtable_maddr();
> +        qi_ctrl->qinval_maddr = alloc_pgtable_maddr(NULL);
>          if ( qi_ctrl->qinval_maddr == 0 )
>          {
>              dprintk(XENLOG_WARNING VTDPREFIX,
> diff -r 5fd51e1e9c79 xen/drivers/passthrough/vtd/vtd.h
> --- a/xen/drivers/passthrough/vtd/vtd.h       Wed Nov 05 10:57:21 2008 +0000
> +++ b/xen/drivers/passthrough/vtd/vtd.h       Tue Nov 18 17:37:31 2008 +0900
> @@ -101,7 +101,7 @@
>  void cacheline_flush(char *);
>  void flush_all_cache(void);
>  void *map_to_nocache_virt(int nr_iommus, u64 maddr);
> -u64 alloc_pgtable_maddr(void);
> +u64 alloc_pgtable_maddr(struct domain *d);
>  void free_pgtable_maddr(u64 maddr);
>  void *map_vtd_domain_page(u64 maddr);
>  void unmap_vtd_domain_page(void *va);
> diff -r 5fd51e1e9c79 xen/drivers/passthrough/vtd/x86/vtd.c
> --- a/xen/drivers/passthrough/vtd/x86/vtd.c   Wed Nov 05 10:57:21 2008 +0000
> +++ b/xen/drivers/passthrough/vtd/x86/vtd.c   Tue Nov 18 17:37:31 2008 +0900
> @@ -22,6 +22,7 @@
>  #include <xen/domain_page.h>
>  #include <asm/paging.h>
>  #include <xen/iommu.h>
> +#include <xen/numa.h>
>  #include "../iommu.h"
>  #include "../dmar.h"
>  #include "../vtd.h"
> @@ -37,13 +38,21 @@
>  }
 
>  /* Allocate page table, return its machine address */
> -u64 alloc_pgtable_maddr(void)
> +u64 alloc_pgtable_maddr(struct domain *d)
>  {
>      struct page_info *pg;
>      u64 *vaddr;
>      unsigned long mfn;
 
> -    pg = alloc_domheap_page(NULL, 0);
> +    if (d == NULL)
> +    {
> +        pg = alloc_domheap_page(NULL, 0);
> +    }
> +    else
> +    {
> +        pg = alloc_domheap_page(NULL, MEMF_node(domain_to_node(d)));
> +    }
> +
>      if ( !pg )
>          return 0;
>      mfn = page_to_mfn(pg);


> _______________________________________________
> Xen-devel mailing list
> Xen-devel@xxxxxxxxxxxxxxxxxxx
> http://lists.xensource.com/xen-devel


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel