|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [XEN][PATCH v5 6/8] xen/arm: batch stage-2 removal with range flushes
From: Haseeb Ashraf <haseeb.ashraf@xxxxxxxxxxx> Removing pages from a guest p2m requires TLB invalidation. The cost is particularly high when Xen runs under another hypervisor, where each invalidation can trap to the outer hypervisor. Add guest_physmap_remove_page_batch() to remove a list of pages under a single p2m write lock and invalidate each contiguous run of changed GFNs with one range-flush call. The range helper selects the TLBI method and uses a full-VMID flush when a range exceeds the supported limit. Patch 7 adds the public hypercall that calls this helper. An entry is checked at removal time, so a duplicate or missing GFN is reported as -ENOENT, like single-page removal. Entries that were not changed are excluded from invalidation; a removal that fails after changing an entry is included. The caller must hold a reference on each page until the invalidation is complete. HAS_PHYSMAP_REMOVE_BATCH is selected for Arm MMU only; the MPU p2m has no TLB invalidation by range. Linux reference: - KVM: arm64: Use TLBI range-based instructions for unmap, which similarly defers the stage-2 invalidation and coalesces it into ranges: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=7657ea920c54218f123ddc1b572821695b669c13 Signed-off-by: Haseeb Ashraf <haseeb.ashraf@xxxxxxxxxxx> --- New in v5. xen/arch/arm/Kconfig | 1 + xen/arch/arm/p2m.c | 97 ++++++++++++++++++++++++++++++++---- xen/common/Kconfig | 3 ++ xen/include/xen/p2m-common.h | 16 ++++++ 4 files changed, 106 insertions(+), 11 deletions(-) diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig index b9c5c40678..7ad676940b 100644 --- a/xen/arch/arm/Kconfig +++ b/xen/arch/arm/Kconfig @@ -85,6 +85,7 @@ config MMU bool "MMU" select ARCH_PAGING_MEMPOOL select HAS_LLC_COLORING if !NUMA && ARM_64 + select HAS_PHYSMAP_REMOVE_BATCH select HAS_PMAP select HAS_VMAP select HAS_PASSTHROUGH diff --git a/xen/arch/arm/p2m.c b/xen/arch/arm/p2m.c index a39351ede9..c4f1f3709c 100644 --- a/xen/arch/arm/p2m.c +++ b/xen/arch/arm/p2m.c @@ -123,16 +123,18 @@ int p2m_insert_mapping(struct domain *d, gfn_t start_gfn, unsigned long nr, return rc; } -static inline int p2m_remove_mapping(struct domain *d, +/* + * Remove [start_gfn, start_gfn + nr) from the p2m. The caller must hold the + * p2m write lock and is responsible for flushing any removed entries. + */ +static int p2m_remove_mapping_locked(struct p2m_domain *p2m, gfn_t start_gfn, unsigned long nr, mfn_t mfn) { - struct p2m_domain *p2m = p2m_get_hostp2m(d); unsigned long i; - int rc; - p2m_write_lock(p2m); + ASSERT(p2m_is_write_locked(p2m)); /* * Before removing the GFN - MFN mapping for any RAM pages make sure @@ -150,17 +152,27 @@ static inline int p2m_remove_mapping(struct domain *d, if ( p2m_is_any_ram(t) && (!mfn_valid(mfn) || !mfn_eq(mfn_add(mfn, i), mfn_return)) ) - { - rc = -EILSEQ; - goto out; - } + return -EILSEQ; i += (1UL << cur_order) - ((gfn_x(start_gfn) + i) & ((1UL << cur_order) - 1)); } - rc = p2m_set_entry(p2m, start_gfn, nr, INVALID_MFN, - p2m_invalid, p2m_access_rwx); + return p2m_set_entry(p2m, start_gfn, nr, INVALID_MFN, + p2m_invalid, p2m_access_rwx); +} + +static inline int p2m_remove_mapping(struct domain *d, + gfn_t start_gfn, + unsigned long nr, + mfn_t mfn) +{ + struct p2m_domain *p2m = p2m_get_hostp2m(d); + int rc; + + p2m_write_lock(p2m); + + rc = p2m_remove_mapping_locked(p2m, start_gfn, nr, mfn); /* * The range that may have changed is known. Invalidate it instead of @@ -175,12 +187,75 @@ static inline int p2m_remove_mapping(struct domain *d, p2m->need_flush = false; } - out: p2m_write_unlock(p2m); return rc; } +#ifdef CONFIG_HAS_PHYSMAP_REMOVE_BATCH +/* Whether the removal of an entry may have changed the p2m. */ +static bool removal_touched_p2m(int err) +{ + /* These are returned before the p2m is modified. */ + return err != -ENOENT && err != -EILSEQ; +} + +void guest_physmap_remove_page_batch(struct domain *d, + const xen_pfn_t *gpfns, + const mfn_t *mfns, + int *errs, + unsigned int count) +{ + struct p2m_domain *p2m = p2m_get_hostp2m(d); + unsigned int i; + + p2m_write_lock(p2m); + + /* A flush that is still pending covers more than this batch. */ + p2m_tlb_flush_sync(p2m); + + /* __p2m_set_entry() only sets p2m->need_flush for a removed entry. */ + for ( i = 0; i < count; i++ ) + { + if ( mfn_eq(mfns[i], INVALID_MFN) || + mfn_eq(p2m_get_entry(p2m, _gfn(gpfns[i]), NULL, NULL, NULL, NULL), + INVALID_MFN) ) + errs[i] = -ENOENT; + else + errs[i] = p2m_remove_mapping_locked(p2m, _gfn(gpfns[i]), 1, + mfns[i]); + } + + /* + * Invalidate each run of consecutive entries that may have changed. + * The range helper selects the TLBI method and falls back to a full-VMID + * flush for ranges beyond its per-CPU limit. Include an entry whose + * removal failed after changing the p2m. + */ + if ( p2m->need_flush ) + { + for ( i = 0; i < count; i++ ) + { + unsigned int first = i; + + if ( !removal_touched_p2m(errs[i]) ) + continue; + + while ( i + 1 < count && removal_touched_p2m(errs[i + 1]) && + gpfns[i + 1] == gpfns[i] + 1 ) + i++; + + p2m_force_tlb_flush_range_sync(p2m, pfn_to_paddr(gpfns[first]), + i - first + 1); + } + + p2m->need_flush = false; + } + + p2m_write_unlock(p2m); +} +#endif /* CONFIG_HAS_PHYSMAP_REMOVE_BATCH */ + int map_regions_p2mt(struct domain *d, gfn_t gfn, unsigned long nr, diff --git a/xen/common/Kconfig b/xen/common/Kconfig index a619f2cef1..e687b59e8f 100644 --- a/xen/common/Kconfig +++ b/xen/common/Kconfig @@ -152,6 +152,9 @@ config HAS_KEXEC config HAS_LLC_COLORING bool +config HAS_PHYSMAP_REMOVE_BATCH + bool + config HAS_PIRQ bool diff --git a/xen/include/xen/p2m-common.h b/xen/include/xen/p2m-common.h index 0eb0619912..7a78c8b421 100644 --- a/xen/include/xen/p2m-common.h +++ b/xen/include/xen/p2m-common.h @@ -12,6 +12,22 @@ int __must_check guest_physmap_remove_page(struct domain *d, gfn_t gfn, mfn_t mfn, unsigned int page_order); +#ifdef CONFIG_HAS_PHYSMAP_REMOVE_BATCH +/* + * Remove a batch of pages from a domain's p2m table. + * + * gpfns[] lists the frames. mfns[] holds, for each page the caller holds a + * reference on, its frame, or INVALID_MFN if there is no page. errs[] + * receives a result per entry. All entries are removed under one p2m write + * lock; changed contiguous runs are invalidated after processing the batch. + */ +void guest_physmap_remove_page_batch(struct domain *d, + const xen_pfn_t *gpfns, + const mfn_t *mfns, + int *errs, + unsigned int count); +#endif + /* Map MMIO regions in the p2m: start_gfn and nr describe the range in * * the guest physical address space to map, starting from the machine * * frame number mfn. */
|
![]() |
Lists.xenproject.org is hosted with RackSpace, monitoring our |