[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[XEN][PATCH v5 6/8] xen/arm: batch stage-2 removal with range flushes



From: Haseeb Ashraf <haseeb.ashraf@xxxxxxxxxxx>

Removing pages from a guest p2m requires TLB invalidation. The cost is
particularly high when Xen runs under another hypervisor, where each
invalidation can trap to the outer hypervisor.

Add guest_physmap_remove_page_batch() to remove a list of pages under a
single p2m write lock and invalidate each contiguous run of changed GFNs
with one range-flush call. The range helper selects the TLBI method and
uses a full-VMID flush when a range exceeds the supported limit. Patch 7
adds the public hypercall that calls this helper.

An entry is checked at removal time, so a duplicate or missing GFN is
reported as -ENOENT, like single-page removal. Entries that were not
changed are excluded from invalidation; a removal that fails after
changing an entry is included.

The caller must hold a reference on each page until the invalidation is
complete. HAS_PHYSMAP_REMOVE_BATCH is selected for Arm MMU only; the MPU
p2m has no TLB invalidation by range.

Linux reference:
- KVM: arm64: Use TLBI range-based instructions for unmap, which
  similarly defers the stage-2 invalidation and coalesces it into
  ranges:
  
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=7657ea920c54218f123ddc1b572821695b669c13

Signed-off-by: Haseeb Ashraf <haseeb.ashraf@xxxxxxxxxxx>
---
New in v5.

 xen/arch/arm/Kconfig         |  1 +
 xen/arch/arm/p2m.c           | 97 ++++++++++++++++++++++++++++++++----
 xen/common/Kconfig           |  3 ++
 xen/include/xen/p2m-common.h | 16 ++++++
 4 files changed, 106 insertions(+), 11 deletions(-)

diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig
index b9c5c40678..7ad676940b 100644
--- a/xen/arch/arm/Kconfig
+++ b/xen/arch/arm/Kconfig
@@ -85,6 +85,7 @@ config MMU
        bool "MMU"
        select ARCH_PAGING_MEMPOOL
        select HAS_LLC_COLORING if !NUMA && ARM_64
+       select HAS_PHYSMAP_REMOVE_BATCH
        select HAS_PMAP
        select HAS_VMAP
        select HAS_PASSTHROUGH
diff --git a/xen/arch/arm/p2m.c b/xen/arch/arm/p2m.c
index a39351ede9..c4f1f3709c 100644
--- a/xen/arch/arm/p2m.c
+++ b/xen/arch/arm/p2m.c
@@ -123,16 +123,18 @@ int p2m_insert_mapping(struct domain *d, gfn_t start_gfn, 
unsigned long nr,
     return rc;
 }
 
-static inline int p2m_remove_mapping(struct domain *d,
+/*
+ * Remove [start_gfn, start_gfn + nr) from the p2m. The caller must hold the
+ * p2m write lock and is responsible for flushing any removed entries.
+ */
+static int p2m_remove_mapping_locked(struct p2m_domain *p2m,
                                      gfn_t start_gfn,
                                      unsigned long nr,
                                      mfn_t mfn)
 {
-    struct p2m_domain *p2m = p2m_get_hostp2m(d);
     unsigned long i;
-    int rc;
 
-    p2m_write_lock(p2m);
+    ASSERT(p2m_is_write_locked(p2m));
 
     /*
      * Before removing the GFN - MFN mapping for any RAM pages make sure
@@ -150,17 +152,27 @@ static inline int p2m_remove_mapping(struct domain *d,
 
         if ( p2m_is_any_ram(t) &&
              (!mfn_valid(mfn) || !mfn_eq(mfn_add(mfn, i), mfn_return)) )
-        {
-            rc = -EILSEQ;
-            goto out;
-        }
+            return -EILSEQ;
 
         i += (1UL << cur_order) -
              ((gfn_x(start_gfn) + i) & ((1UL << cur_order) - 1));
     }
 
-    rc = p2m_set_entry(p2m, start_gfn, nr, INVALID_MFN,
-                       p2m_invalid, p2m_access_rwx);
+    return p2m_set_entry(p2m, start_gfn, nr, INVALID_MFN,
+                         p2m_invalid, p2m_access_rwx);
+}
+
+static inline int p2m_remove_mapping(struct domain *d,
+                                     gfn_t start_gfn,
+                                     unsigned long nr,
+                                     mfn_t mfn)
+{
+    struct p2m_domain *p2m = p2m_get_hostp2m(d);
+    int rc;
+
+    p2m_write_lock(p2m);
+
+    rc = p2m_remove_mapping_locked(p2m, start_gfn, nr, mfn);
 
     /*
      * The range that may have changed is known. Invalidate it instead of
@@ -175,12 +187,75 @@ static inline int p2m_remove_mapping(struct domain *d,
         p2m->need_flush = false;
     }
 
- out:
     p2m_write_unlock(p2m);
 
     return rc;
 }
 
+#ifdef CONFIG_HAS_PHYSMAP_REMOVE_BATCH
+/* Whether the removal of an entry may have changed the p2m. */
+static bool removal_touched_p2m(int err)
+{
+    /* These are returned before the p2m is modified. */
+    return err != -ENOENT && err != -EILSEQ;
+}
+
+void guest_physmap_remove_page_batch(struct domain *d,
+                                     const xen_pfn_t *gpfns,
+                                     const mfn_t *mfns,
+                                     int *errs,
+                                     unsigned int count)
+{
+    struct p2m_domain *p2m = p2m_get_hostp2m(d);
+    unsigned int i;
+
+    p2m_write_lock(p2m);
+
+    /* A flush that is still pending covers more than this batch. */
+    p2m_tlb_flush_sync(p2m);
+
+    /* __p2m_set_entry() only sets p2m->need_flush for a removed entry. */
+    for ( i = 0; i < count; i++ )
+    {
+        if ( mfn_eq(mfns[i], INVALID_MFN) ||
+             mfn_eq(p2m_get_entry(p2m, _gfn(gpfns[i]), NULL, NULL, NULL, NULL),
+                    INVALID_MFN) )
+            errs[i] = -ENOENT;
+        else
+            errs[i] = p2m_remove_mapping_locked(p2m, _gfn(gpfns[i]), 1,
+                                                mfns[i]);
+    }
+
+    /*
+     * Invalidate each run of consecutive entries that may have changed.
+     * The range helper selects the TLBI method and falls back to a full-VMID
+     * flush for ranges beyond its per-CPU limit. Include an entry whose
+     * removal failed after changing the p2m.
+     */
+    if ( p2m->need_flush )
+    {
+        for ( i = 0; i < count; i++ )
+        {
+            unsigned int first = i;
+
+            if ( !removal_touched_p2m(errs[i]) )
+                continue;
+
+            while ( i + 1 < count && removal_touched_p2m(errs[i + 1]) &&
+                    gpfns[i + 1] == gpfns[i] + 1 )
+                i++;
+
+            p2m_force_tlb_flush_range_sync(p2m, pfn_to_paddr(gpfns[first]),
+                                           i - first + 1);
+        }
+
+        p2m->need_flush = false;
+    }
+
+    p2m_write_unlock(p2m);
+}
+#endif /* CONFIG_HAS_PHYSMAP_REMOVE_BATCH */
+
 int map_regions_p2mt(struct domain *d,
                      gfn_t gfn,
                      unsigned long nr,
diff --git a/xen/common/Kconfig b/xen/common/Kconfig
index a619f2cef1..e687b59e8f 100644
--- a/xen/common/Kconfig
+++ b/xen/common/Kconfig
@@ -152,6 +152,9 @@ config HAS_KEXEC
 config HAS_LLC_COLORING
        bool
 
+config HAS_PHYSMAP_REMOVE_BATCH
+       bool
+
 config HAS_PIRQ
        bool
 
diff --git a/xen/include/xen/p2m-common.h b/xen/include/xen/p2m-common.h
index 0eb0619912..7a78c8b421 100644
--- a/xen/include/xen/p2m-common.h
+++ b/xen/include/xen/p2m-common.h
@@ -12,6 +12,22 @@ int __must_check
 guest_physmap_remove_page(struct domain *d, gfn_t gfn, mfn_t mfn,
                           unsigned int page_order);
 
+#ifdef CONFIG_HAS_PHYSMAP_REMOVE_BATCH
+/*
+ * Remove a batch of pages from a domain's p2m table.
+ *
+ * gpfns[] lists the frames. mfns[] holds, for each page the caller holds a
+ * reference on, its frame, or INVALID_MFN if there is no page. errs[]
+ * receives a result per entry. All entries are removed under one p2m write
+ * lock; changed contiguous runs are invalidated after processing the batch.
+ */
+void guest_physmap_remove_page_batch(struct domain *d,
+                                     const xen_pfn_t *gpfns,
+                                     const mfn_t *mfns,
+                                     int *errs,
+                                     unsigned int count);
+#endif
+
 /* Map MMIO regions in the p2m: start_gfn and nr describe the range in
  *  * the guest physical address space to map, starting from the machine
  *   * frame number mfn. */



 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.