[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH] linux/x86: batch hypercalls when pinning address spaces



Signed-off-by: Jan Beulich <jbeulich@xxxxxxxxxx>

Index: head-2007-03-19/arch/i386/mm/pgtable-xen.c
===================================================================
--- head-2007-03-19.orig/arch/i386/mm/pgtable-xen.c     2007-03-21 
11:51:37.000000000 +0100
+++ head-2007-03-19/arch/i386/mm/pgtable-xen.c  2007-03-23 17:51:03.000000000 
+0100
@@ -574,10 +574,13 @@ static void _pin_lock(struct mm_struct *
        }
 }
 
-static inline void pgd_walk_set_prot(struct page *page, pgprot_t flags)
+#define PIN_BATCH 4
+static DEFINE_PER_CPU(multicall_entry_t[PIN_BATCH], pb_mcl);
+
+static inline unsigned int pgd_walk_set_prot(struct page *page, pgprot_t flags,
+                                             unsigned int cpu, unsigned seq)
 {
        unsigned long pfn = page_to_pfn(page);
-       int rc;
 
        if (PageHighMem(page)) {
                if (pgprot_val(flags) & _PAGE_RW)
@@ -585,12 +588,18 @@ static inline void pgd_walk_set_prot(str
                else
                        set_bit(PG_pinned, &page->flags);
        } else {
-               rc = HYPERVISOR_update_va_mapping(
-                       (unsigned long)__va(pfn << PAGE_SHIFT),
-                       pfn_pte(pfn, flags), 0);
-               if (rc)
-                       BUG();
+               MULTI_update_va_mapping(per_cpu(pb_mcl, cpu) + seq,
+                               (unsigned long)__va(pfn << PAGE_SHIFT),
+                               pfn_pte(pfn, flags), 0);
+               if (unlikely(++seq == PIN_BATCH)) {
+                       if (unlikely(HYPERVISOR_multicall_check(per_cpu(pb_mcl, 
cpu),
+                                                               PIN_BATCH, 
NULL)))
+                               BUG();
+                       seq = 0;
+               }
        }
+
+       return seq;
 }
 
 static void pgd_walk(pgd_t *pgd_base, pgprot_t flags)
@@ -598,37 +607,48 @@ static void pgd_walk(pgd_t *pgd_base, pg
        pgd_t *pgd = pgd_base;
        pud_t *pud;
        pmd_t *pmd;
-       int    g, u, m, rc;
+       int    g, u, m;
+       unsigned int cpu, seq;
 
        if (xen_feature(XENFEAT_auto_translated_physmap))
                return;
 
-       for (g = 0; g < USER_PTRS_PER_PGD; g++, pgd++) {
+       cpu = get_cpu();
+
+       for (g = 0, seq = 0; g < USER_PTRS_PER_PGD; g++, pgd++) {
                if (pgd_none(*pgd))
                        continue;
                pud = pud_offset(pgd, 0);
                if (PTRS_PER_PUD > 1) /* not folded */
-                       pgd_walk_set_prot(virt_to_page(pud),flags);
+                       seq = 
pgd_walk_set_prot(virt_to_page(pud),flags,cpu,seq);
                for (u = 0; u < PTRS_PER_PUD; u++, pud++) {
                        if (pud_none(*pud))
                                continue;
                        pmd = pmd_offset(pud, 0);
                        if (PTRS_PER_PMD > 1) /* not folded */
-                               pgd_walk_set_prot(virt_to_page(pmd),flags);
+                               seq = 
pgd_walk_set_prot(virt_to_page(pmd),flags,cpu,seq);
                        for (m = 0; m < PTRS_PER_PMD; m++, pmd++) {
                                if (pmd_none(*pmd))
                                        continue;
-                               pgd_walk_set_prot(pmd_page(*pmd),flags);
+                               seq = 
pgd_walk_set_prot(pmd_page(*pmd),flags,cpu,seq);
                        }
                }
        }
 
-       rc = HYPERVISOR_update_va_mapping(
-               (unsigned long)pgd_base,
-               pfn_pte(virt_to_phys(pgd_base)>>PAGE_SHIFT, flags),
-               UVMF_TLB_FLUSH);
-       if (rc)
+       if (likely(seq != 0)) {
+               MULTI_update_va_mapping(per_cpu(pb_mcl, cpu) + seq,
+                       (unsigned long)pgd_base,
+                       pfn_pte(virt_to_phys(pgd_base)>>PAGE_SHIFT, flags),
+                       UVMF_TLB_FLUSH);
+               if (unlikely(HYPERVISOR_multicall_check(per_cpu(pb_mcl, cpu),
+                                                       seq + 1, NULL)))
+                       BUG();
+       } else if(HYPERVISOR_update_va_mapping((unsigned long)pgd_base,
+                       pfn_pte(virt_to_phys(pgd_base)>>PAGE_SHIFT, flags),
+                       UVMF_TLB_FLUSH))
                BUG();
+
+       put_cpu();
 }
 
 static void __pgd_pin(pgd_t *pgd)
Index: head-2007-03-19/arch/x86_64/mm/pageattr-xen.c
===================================================================
--- head-2007-03-19.orig/arch/x86_64/mm/pageattr-xen.c  2007-03-21 
11:50:13.000000000 +0100
+++ head-2007-03-19/arch/x86_64/mm/pageattr-xen.c       2007-03-23 
17:51:24.000000000 +0100
@@ -20,17 +20,26 @@ static void _pin_lock(struct mm_struct *
 LIST_HEAD(mm_unpinned);
 DEFINE_SPINLOCK(mm_unpinned_lock);
 
-static inline void mm_walk_set_prot(void *pt, pgprot_t flags)
+#define PIN_BATCH 8
+static DEFINE_PER_CPU(multicall_entry_t[PIN_BATCH], pb_mcl);
+
+static inline unsigned int mm_walk_set_prot(void *pt, pgprot_t flags,
+                                            unsigned int cpu, unsigned int seq)
 {
        struct page *page = virt_to_page(pt);
        unsigned long pfn = page_to_pfn(page);
-       int rc;
 
-       rc = HYPERVISOR_update_va_mapping(
+       MULTI_update_va_mapping(per_cpu(pb_mcl, cpu) + seq,
                (unsigned long)__va(pfn << PAGE_SHIFT),
                pfn_pte(pfn, flags), 0);
-       if (rc)
-               BUG();
+       if (unlikely(++seq == PIN_BATCH)) {
+               if (unlikely(HYPERVISOR_multicall_check(per_cpu(pb_mcl, cpu),
+                                                       PIN_BATCH, NULL)))
+                       BUG();
+               seq = 0;
+       }
+
+       return seq;
 }
 
 static void mm_walk(struct mm_struct *mm, pgprot_t flags)
@@ -40,8 +49,12 @@ static void mm_walk(struct mm_struct *mm
        pmd_t       *pmd;
        pte_t       *pte;
        int          g,u,m;
+       unsigned int cpu, seq;
+       multicall_entry_t *mcl;
 
        pgd = mm->pgd;
+       cpu = get_cpu();
+
        /*
         * Cannot iterate up to USER_PTRS_PER_PGD as these pagetables may not
         * be the 'current' task's pagetables (e.g., current may be 32-bit,
@@ -49,26 +62,45 @@ static void mm_walk(struct mm_struct *mm
         * Subtracting 1 from TASK_SIZE64 means the loop limit is correct
         * regardless of whether TASK_SIZE64 is a multiple of PGDIR_SIZE.
         */
-       for (g = 0; g <= ((TASK_SIZE64-1) / PGDIR_SIZE); g++, pgd++) {
+       for (g = 0, seq = 0; g <= ((TASK_SIZE64-1) / PGDIR_SIZE); g++, pgd++) {
                if (pgd_none(*pgd))
                        continue;
                pud = pud_offset(pgd, 0);
                if (PTRS_PER_PUD > 1) /* not folded */ 
-                       mm_walk_set_prot(pud,flags);
+                       seq = mm_walk_set_prot(pud,flags,cpu,seq);
                for (u = 0; u < PTRS_PER_PUD; u++, pud++) {
                        if (pud_none(*pud))
                                continue;
                        pmd = pmd_offset(pud, 0);
                        if (PTRS_PER_PMD > 1) /* not folded */ 
-                               mm_walk_set_prot(pmd,flags);
+                               seq = mm_walk_set_prot(pmd,flags,cpu,seq);
                        for (m = 0; m < PTRS_PER_PMD; m++, pmd++) {
                                if (pmd_none(*pmd))
                                        continue;
                                pte = pte_offset_kernel(pmd,0);
-                               mm_walk_set_prot(pte,flags);
+                               seq = mm_walk_set_prot(pte,flags,cpu,seq);
                        }
                }
        }
+
+       mcl = per_cpu(pb_mcl, cpu);
+       if (unlikely(seq > PIN_BATCH - 2)) {
+               if (unlikely(HYPERVISOR_multicall_check(mcl, seq, NULL)))
+                       BUG();
+               seq = 0;
+       }
+       MULTI_update_va_mapping(mcl + seq,
+              (unsigned long)__user_pgd(mm->pgd),
+              pfn_pte(virt_to_phys(__user_pgd(mm->pgd))>>PAGE_SHIFT, flags),
+              0);
+       MULTI_update_va_mapping(mcl + seq + 1,
+              (unsigned long)mm->pgd,
+              pfn_pte(virt_to_phys(mm->pgd)>>PAGE_SHIFT, flags),
+              UVMF_TLB_FLUSH);
+       if (unlikely(HYPERVISOR_multicall_check(mcl, seq + 2, NULL)))
+               BUG();
+
+       put_cpu();
 }
 
 void mm_pin(struct mm_struct *mm)
@@ -79,17 +112,6 @@ void mm_pin(struct mm_struct *mm)
        spin_lock(&mm->page_table_lock);
 
        mm_walk(mm, PAGE_KERNEL_RO);
-       if (HYPERVISOR_update_va_mapping(
-               (unsigned long)mm->pgd,
-               pfn_pte(virt_to_phys(mm->pgd)>>PAGE_SHIFT, PAGE_KERNEL_RO),
-               UVMF_TLB_FLUSH))
-               BUG();
-       if (HYPERVISOR_update_va_mapping(
-               (unsigned long)__user_pgd(mm->pgd),
-               pfn_pte(virt_to_phys(__user_pgd(mm->pgd))>>PAGE_SHIFT,
-                       PAGE_KERNEL_RO),
-               UVMF_TLB_FLUSH))
-               BUG();
        xen_pgd_pin(__pa(mm->pgd)); /* kernel */
        xen_pgd_pin(__pa(__user_pgd(mm->pgd))); /* user */
        mm->context.pinned = 1;
@@ -109,17 +131,7 @@ void mm_unpin(struct mm_struct *mm)
 
        xen_pgd_unpin(__pa(mm->pgd));
        xen_pgd_unpin(__pa(__user_pgd(mm->pgd)));
-       if (HYPERVISOR_update_va_mapping(
-               (unsigned long)mm->pgd,
-               pfn_pte(virt_to_phys(mm->pgd)>>PAGE_SHIFT, PAGE_KERNEL), 0))
-               BUG();
-       if (HYPERVISOR_update_va_mapping(
-               (unsigned long)__user_pgd(mm->pgd),
-               pfn_pte(virt_to_phys(__user_pgd(mm->pgd))>>PAGE_SHIFT,
-                       PAGE_KERNEL), 0))
-               BUG();
        mm_walk(mm, PAGE_KERNEL);
-       xen_tlb_flush();
        mm->context.pinned = 0;
        spin_lock(&mm_unpinned_lock);
        list_add(&mm->context.unpinned, &mm_unpinned);



_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.