|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [XEN][PATCH v5 4/8] xen/arm64: use range TLBI for IPA flush with FEAT_TLBIRANGE
From: Haseeb Ashraf <haseeb.ashraf@xxxxxxxxxxx> Invalidating a range of IPAs with TLBI IPAS2E1IS takes one broadcast instruction per page. With FEAT_TLBIRANGE, TLBI RIPAS2E1IS invalidates a range with at most four range operations when it fits the range limit of the instruction. Larger ranges fall back to a full VMID invalidation. When Xen runs on top of another hypervisor, each instruction also traps to it. A range of one page costs one instruction either way. The gain is for multi-page ranges: the whole old block of a split superpage, a removal of several pages, and the batched removal added by patches 6 and 7. Without the feature, a block of 512 pages or more is flushed with the whole VMID, as before. Detect the feature as a CPU capability, from ID_AA64ISAR0_EL1.TLB. When it is present, flush_guest_tlb_range_ipa() covers the range with one operation per scale, the largest first. Only a range that does not fit the largest operation is flushed with VMALLS12E1IS, which is the limit Linux uses (MAX_TLBI_RANGE_PAGES). Without the feature the per-page loop and its TLBI_MAX_PAGE_OPS bound are unchanged. A range operation covers an even number of pages. Extend an odd range by one page to the next even boundary instead of ending with a single page TLBI IPAS2E1IS, which saves one instruction per range. A page more is always safe to invalidate, and extending to the neighbor that makes the range 2 page aligned never leaves the IPA space. The operation is encoded with SYS so that no assembler support for FEAT_TLBIRANGE is required. Stage-2 of Xen does not use FEAT_LPA2, so BADDR holds IPA[48:12]. Linux references: - arm64: tlb: Use the TLBI RANGE feature in arm64, for the range encoding and the scale decomposition: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=d1d3aa98b1d4826a19adfefb69b96142a0cac633 - KVM: arm64: Implement __kvm_tlb_flush_vmid_range(), for the stage-2 IPA range invalidation: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=6354d15052ec88273c24beae4c99e31c3d3889b6 Signed-off-by: Haseeb Ashraf <haseeb.ashraf@xxxxxxxxxxx> --- xen/arch/arm/cpufeature.c | 14 +++ xen/arch/arm/include/asm/arm64/flushtlb.h | 113 ++++++++++++++++++++-- xen/arch/arm/include/asm/cpufeature.h | 3 +- 3 files changed, 121 insertions(+), 9 deletions(-) diff --git a/xen/arch/arm/cpufeature.c b/xen/arch/arm/cpufeature.c index e4e0dd0922..05c4393324 100644 --- a/xen/arch/arm/cpufeature.c +++ b/xen/arch/arm/cpufeature.c @@ -39,6 +39,15 @@ static bool has_sb_instruction(const struct arm_cpu_capabilities *entry) { return system_cpuinfo.isa64.sb; } + +static bool has_tlbi_range(const struct arm_cpu_capabilities *entry) +{ + /* + * ID_AA64ISAR0_EL1.TLB: 0b0001 is FEAT_TLBIOS, 0b0010 adds + * FEAT_TLBIRANGE. + */ + return system_cpuinfo.isa64.tlb >= 2; +} #endif static const struct arm_cpu_capabilities arm_features[] = { @@ -62,6 +71,11 @@ static const struct arm_cpu_capabilities arm_features[] = { .capability = ARM_HAS_SB, .matches = has_sb_instruction, }, + { + .desc = "Range-based TLB invalidation (FEAT_TLBIRANGE)", + .capability = ARM64_HAS_TLBI_RANGE, + .matches = has_tlbi_range, + }, #endif {}, }; diff --git a/xen/arch/arm/include/asm/arm64/flushtlb.h b/xen/arch/arm/include/asm/arm64/flushtlb.h index ff2ce2bae2..7dc8ae608a 100644 --- a/xen/arch/arm/include/asm/arm64/flushtlb.h +++ b/xen/arch/arm/include/asm/arm64/flushtlb.h @@ -133,6 +133,70 @@ static inline void tlbi_complete_broadcast(void) isb(); } +/* + * Range-based TLB invalidation (FEAT_TLBIRANGE), see Arm ARM + * (DDI 0487 M.d), C5.5.22 (TLBI RIPAS2E1IS). + * + * The operand of TLBI RIPAS2E1IS is: + * [47:46] TG translation granule, 0b01 for 4KB + * [45:44] SCALE + * [43:39] NUM + * [38:37] TTL 0b00, no level hint + * [36:0] BADDR IPA[48:12], Xen does not use FEAT_LPA2 for stage-2 + * and the instruction invalidates (NUM + 1) * 2^(5 * SCALE + 1) pages. + * + * These are the same definitions as in Linux, see + * arch/arm64/include/asm/tlbflush.h. + */ +#define TLBI_RANGE_MAX_SCALE 3 +#define TLBI_RANGE_PAGES(num, scale) \ + ((((unsigned long)(num)) + 1) << (5 * (scale) + 1)) +#define TLBI_RANGE_MAX_PAGES TLBI_RANGE_PAGES(31, TLBI_RANGE_MAX_SCALE) +/* Gives -1 to 31 for a range of up to TLBI_RANGE_MAX_PAGES pages. */ +#define TLBI_RANGE_NUM(pages, scale) \ + ((long)((pages) >> (5 * (scale) + 1)) - 1) + +/* + * TLBI RIPAS2E1IS: invalidate a range of IPAs of the current VMID, in the + * inner-shareable domain. It is encoded as SYS #4, C8, C0, #2 so that no + * assembler support for FEAT_TLBIRANGE is required. + */ +static inline void tlbi_ripas2e1is(uint64_t op) +{ + asm_inline volatile( + "sys #4, c8, c0, #2, %0" + : : "r" (op) : "memory"); +} + +/* + * Invalidate an even number @pages, of at most TLBI_RANGE_MAX_PAGES, of + * stage-2 pages starting at @ipa, using at most one range operation per + * scale. The caller issues the barriers. + */ +static inline void tlbi_range_ipas2e1is(paddr_t ipa, unsigned long pages) +{ + int scale = TLBI_RANGE_MAX_SCALE; + + while ( pages > 1 && scale >= 0 ) + { + long num = TLBI_RANGE_NUM(pages, scale); + + if ( num >= 0 ) + { + uint64_t op = ((ipa >> PAGE_SHIFT) & 0x1fffffffffULL) | + ((uint64_t)num << 39) | + ((uint64_t)scale << 44) | + (1ULL << 46); + + tlbi_ripas2e1is(op); + + ipa += (paddr_t)TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; + pages -= TLBI_RANGE_PAGES(num, scale); + } + scale--; + } +} + /* * TLBI IPAS2E1IS: invalidate the stage-2 and combined entries of one IPA of * the current VMID, in the inner-shareable domain. The caller issues the @@ -155,16 +219,42 @@ static inline void tlbi_vmalle1is(void) * Flush a range of IPA's mappings from the TLB of all processors in the * inner-shareable domain. * - * A range of TLBI_MAX_PAGE_OPS pages or more is flushed with the whole - * VMID. Each page of the range costs a broadcast TLBI, and the limit bounds - * their number like MAX_DVM_OPS does in Linux. It is not a tuned value. + * With FEAT_TLBIRANGE, ranges larger than TLBI_RANGE_MAX_PAGES are flushed + * with the whole VMID. Without it, a range of TLBI_MAX_PAGE_OPS pages or + * more uses the whole VMID; each smaller range costs one broadcast TLBI per + * page. These limits bound instruction count; neither is a tuned crossover. */ static inline void flush_guest_tlb_range_ipa(paddr_t ipa, unsigned long size) { unsigned long pages = size >> PAGE_SHIFT; - paddr_t end = ipa + size; + bool range = cpus_have_const_cap(ARM64_HAS_TLBI_RANGE); + + if ( !pages ) + return; - if ( pages >= TLBI_MAX_PAGE_OPS ) + if ( range ) + { + /* + * A range operation covers an even number of pages. Extend an odd + * range by one page, on the side that makes it 2 pages aligned: the + * neighbor is always inside the IPA space, and a page more is safe + * to invalidate. + */ + if ( (pages & 1UL) != 0UL ) + { + if ( (ipa & PAGE_SIZE) != 0UL ) + ipa -= PAGE_SIZE; + pages++; + } + + /* Same limit as Linux, one range operation per scale at most. */ + if ( pages > TLBI_RANGE_MAX_PAGES ) + { + flush_guest_tlb(); + return; + } + } + else if ( pages >= TLBI_MAX_PAGE_OPS ) { flush_guest_tlb(); return; @@ -175,10 +265,17 @@ static inline void flush_guest_tlb_range_ipa(paddr_t ipa, unsigned long size) * stage 2 translations). */ dsb(ishst); /* Ensure prior page-table updates have completed */ - for ( ; ipa < end; ipa += PAGE_SIZE ) + if ( range ) + tlbi_range_ipas2e1is(ipa, pages); + else { - /* Flush stage-2 TLBs for IPA address */ - tlbi_ipas2e1is(ipa); + paddr_t end = ipa + size; + + for ( ; ipa < end; ipa += PAGE_SIZE ) + { + /* Flush stage-2 TLBs for IPA address */ + tlbi_ipas2e1is(ipa); + } } /* diff --git a/xen/arch/arm/include/asm/cpufeature.h b/xen/arch/arm/include/asm/cpufeature.h index 38e01aa94d..a4f7bcbe3e 100644 --- a/xen/arch/arm/include/asm/cpufeature.h +++ b/xen/arch/arm/include/asm/cpufeature.h @@ -78,8 +78,9 @@ #define ARM64_WORKAROUND_1508412 17 #define ARM_HAS_NTLBPA 18 #define ARM32_HAS_TLB_IPA 19 +#define ARM64_HAS_TLBI_RANGE 20 -#define ARM_NCAPS 20 +#define ARM_NCAPS 21 #ifndef __ASSEMBLER__
|
![]() |
Lists.xenproject.org is hosted with RackSpace, monitoring our |