[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[XEN][PATCH v5 4/8] xen/arm64: use range TLBI for IPA flush with FEAT_TLBIRANGE



From: Haseeb Ashraf <haseeb.ashraf@xxxxxxxxxxx>

Invalidating a range of IPAs with TLBI IPAS2E1IS takes one broadcast
instruction per page. With FEAT_TLBIRANGE, TLBI RIPAS2E1IS invalidates
a range with at most four range operations when it fits the range limit
of the instruction. Larger ranges fall back to a full VMID invalidation.
When Xen runs on top of another hypervisor, each instruction also traps
to it.

A range of one page costs one instruction either way. The gain is for
multi-page ranges: the whole old block of a split superpage, a removal
of several pages, and the batched removal added by patches 6 and 7.
Without the feature, a block of 512 pages or more is flushed with the
whole VMID, as before.

Detect the feature as a CPU capability, from ID_AA64ISAR0_EL1.TLB.
When it is present, flush_guest_tlb_range_ipa() covers the range with
one operation per scale, the largest first. Only a range that does not
fit the largest operation is flushed with VMALLS12E1IS, which is the
limit Linux uses (MAX_TLBI_RANGE_PAGES). Without the feature the
per-page loop and its TLBI_MAX_PAGE_OPS bound are unchanged.

A range operation covers an even number of pages. Extend an odd range
by one page to the next even boundary instead of ending with a single
page TLBI IPAS2E1IS, which saves one instruction per range. A page
more is always safe to invalidate, and extending to the neighbor
that makes the range 2 page aligned never leaves the IPA space.

The operation is encoded with SYS so that no assembler support for
FEAT_TLBIRANGE is required. Stage-2 of Xen does not use FEAT_LPA2, so
BADDR holds IPA[48:12].

Linux references:
- arm64: tlb: Use the TLBI RANGE feature in arm64, for the range
  encoding and the scale decomposition:
  
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=d1d3aa98b1d4826a19adfefb69b96142a0cac633
- KVM: arm64: Implement __kvm_tlb_flush_vmid_range(), for the stage-2
  IPA range invalidation:
  
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=6354d15052ec88273c24beae4c99e31c3d3889b6

Signed-off-by: Haseeb Ashraf <haseeb.ashraf@xxxxxxxxxxx>
---
 xen/arch/arm/cpufeature.c                 |  14 +++
 xen/arch/arm/include/asm/arm64/flushtlb.h | 113 ++++++++++++++++++++--
 xen/arch/arm/include/asm/cpufeature.h     |   3 +-
 3 files changed, 121 insertions(+), 9 deletions(-)

diff --git a/xen/arch/arm/cpufeature.c b/xen/arch/arm/cpufeature.c
index e4e0dd0922..05c4393324 100644
--- a/xen/arch/arm/cpufeature.c
+++ b/xen/arch/arm/cpufeature.c
@@ -39,6 +39,15 @@ static bool has_sb_instruction(const struct 
arm_cpu_capabilities *entry)
 {
     return system_cpuinfo.isa64.sb;
 }
+
+static bool has_tlbi_range(const struct arm_cpu_capabilities *entry)
+{
+    /*
+     * ID_AA64ISAR0_EL1.TLB: 0b0001 is FEAT_TLBIOS, 0b0010 adds
+     * FEAT_TLBIRANGE.
+     */
+    return system_cpuinfo.isa64.tlb >= 2;
+}
 #endif
 
 static const struct arm_cpu_capabilities arm_features[] = {
@@ -62,6 +71,11 @@ static const struct arm_cpu_capabilities arm_features[] = {
         .capability = ARM_HAS_SB,
         .matches = has_sb_instruction,
     },
+    {
+        .desc = "Range-based TLB invalidation (FEAT_TLBIRANGE)",
+        .capability = ARM64_HAS_TLBI_RANGE,
+        .matches = has_tlbi_range,
+    },
 #endif
     {},
 };
diff --git a/xen/arch/arm/include/asm/arm64/flushtlb.h 
b/xen/arch/arm/include/asm/arm64/flushtlb.h
index ff2ce2bae2..7dc8ae608a 100644
--- a/xen/arch/arm/include/asm/arm64/flushtlb.h
+++ b/xen/arch/arm/include/asm/arm64/flushtlb.h
@@ -133,6 +133,70 @@ static inline void tlbi_complete_broadcast(void)
     isb();
 }
 
+/*
+ * Range-based TLB invalidation (FEAT_TLBIRANGE), see Arm ARM
+ * (DDI 0487 M.d), C5.5.22 (TLBI RIPAS2E1IS).
+ *
+ * The operand of TLBI RIPAS2E1IS is:
+ *   [47:46] TG     translation granule, 0b01 for 4KB
+ *   [45:44] SCALE
+ *   [43:39] NUM
+ *   [38:37] TTL    0b00, no level hint
+ *   [36:0]  BADDR  IPA[48:12], Xen does not use FEAT_LPA2 for stage-2
+ * and the instruction invalidates (NUM + 1) * 2^(5 * SCALE + 1) pages.
+ *
+ * These are the same definitions as in Linux, see
+ * arch/arm64/include/asm/tlbflush.h.
+ */
+#define TLBI_RANGE_MAX_SCALE    3
+#define TLBI_RANGE_PAGES(num, scale) \
+    ((((unsigned long)(num)) + 1) << (5 * (scale) + 1))
+#define TLBI_RANGE_MAX_PAGES    TLBI_RANGE_PAGES(31, TLBI_RANGE_MAX_SCALE)
+/* Gives -1 to 31 for a range of up to TLBI_RANGE_MAX_PAGES pages. */
+#define TLBI_RANGE_NUM(pages, scale) \
+    ((long)((pages) >> (5 * (scale) + 1)) - 1)
+
+/*
+ * TLBI RIPAS2E1IS: invalidate a range of IPAs of the current VMID, in the
+ * inner-shareable domain. It is encoded as SYS #4, C8, C0, #2 so that no
+ * assembler support for FEAT_TLBIRANGE is required.
+ */
+static inline void tlbi_ripas2e1is(uint64_t op)
+{
+    asm_inline volatile(
+        "sys #4, c8, c0, #2, %0"
+        : : "r" (op) : "memory");
+}
+
+/*
+ * Invalidate an even number @pages, of at most TLBI_RANGE_MAX_PAGES, of
+ * stage-2 pages starting at @ipa, using at most one range operation per
+ * scale. The caller issues the barriers.
+ */
+static inline void tlbi_range_ipas2e1is(paddr_t ipa, unsigned long pages)
+{
+    int scale = TLBI_RANGE_MAX_SCALE;
+
+    while ( pages > 1 && scale >= 0 )
+    {
+        long num = TLBI_RANGE_NUM(pages, scale);
+
+        if ( num >= 0 )
+        {
+            uint64_t op = ((ipa >> PAGE_SHIFT) & 0x1fffffffffULL) |
+                          ((uint64_t)num << 39) |
+                          ((uint64_t)scale << 44) |
+                          (1ULL << 46);
+
+            tlbi_ripas2e1is(op);
+
+            ipa += (paddr_t)TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT;
+            pages -= TLBI_RANGE_PAGES(num, scale);
+        }
+        scale--;
+    }
+}
+
 /*
  * TLBI IPAS2E1IS: invalidate the stage-2 and combined entries of one IPA of
  * the current VMID, in the inner-shareable domain. The caller issues the
@@ -155,16 +219,42 @@ static inline void tlbi_vmalle1is(void)
  * Flush a range of IPA's mappings from the TLB of all processors in the
  * inner-shareable domain.
  *
- * A range of TLBI_MAX_PAGE_OPS pages or more is flushed with the whole
- * VMID. Each page of the range costs a broadcast TLBI, and the limit bounds
- * their number like MAX_DVM_OPS does in Linux. It is not a tuned value.
+ * With FEAT_TLBIRANGE, ranges larger than TLBI_RANGE_MAX_PAGES are flushed
+ * with the whole VMID. Without it, a range of TLBI_MAX_PAGE_OPS pages or
+ * more uses the whole VMID; each smaller range costs one broadcast TLBI per
+ * page. These limits bound instruction count; neither is a tuned crossover.
  */
 static inline void flush_guest_tlb_range_ipa(paddr_t ipa, unsigned long size)
 {
     unsigned long pages = size >> PAGE_SHIFT;
-    paddr_t end = ipa + size;
+    bool range = cpus_have_const_cap(ARM64_HAS_TLBI_RANGE);
+
+    if ( !pages )
+        return;
 
-    if ( pages >= TLBI_MAX_PAGE_OPS )
+    if ( range )
+    {
+        /*
+         * A range operation covers an even number of pages. Extend an odd
+         * range by one page, on the side that makes it 2 pages aligned: the
+         * neighbor is always inside the IPA space, and a page more is safe
+         * to invalidate.
+         */
+        if ( (pages & 1UL) != 0UL )
+        {
+            if ( (ipa & PAGE_SIZE) != 0UL )
+                ipa -= PAGE_SIZE;
+            pages++;
+        }
+
+        /* Same limit as Linux, one range operation per scale at most. */
+        if ( pages > TLBI_RANGE_MAX_PAGES )
+        {
+            flush_guest_tlb();
+            return;
+        }
+    }
+    else if ( pages >= TLBI_MAX_PAGE_OPS )
     {
         flush_guest_tlb();
         return;
@@ -175,10 +265,17 @@ static inline void flush_guest_tlb_range_ipa(paddr_t ipa, 
unsigned long size)
      * stage 2 translations).
      */
     dsb(ishst); /* Ensure prior page-table updates have completed */
-    for ( ; ipa < end; ipa += PAGE_SIZE )
+    if ( range )
+        tlbi_range_ipas2e1is(ipa, pages);
+    else
     {
-        /* Flush stage-2 TLBs for IPA address */
-        tlbi_ipas2e1is(ipa);
+        paddr_t end = ipa + size;
+
+        for ( ; ipa < end; ipa += PAGE_SIZE )
+        {
+            /* Flush stage-2 TLBs for IPA address */
+            tlbi_ipas2e1is(ipa);
+        }
     }
 
     /*
diff --git a/xen/arch/arm/include/asm/cpufeature.h 
b/xen/arch/arm/include/asm/cpufeature.h
index 38e01aa94d..a4f7bcbe3e 100644
--- a/xen/arch/arm/include/asm/cpufeature.h
+++ b/xen/arch/arm/include/asm/cpufeature.h
@@ -78,8 +78,9 @@
 #define ARM64_WORKAROUND_1508412 17
 #define ARM_HAS_NTLBPA 18
 #define ARM32_HAS_TLB_IPA 19
+#define ARM64_HAS_TLBI_RANGE 20
 
-#define ARM_NCAPS           20
+#define ARM_NCAPS           21
 
 #ifndef __ASSEMBLER__
 



 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.