[Xen-devel] [PATCH 3/5] x86: use PDEP/PEXT for maddr/direct-map-offset conversion when available

["Jan Beulich" <JBeulich@xxxxxxxx>]

Both replace 6 instructions by a single one, further reducing code size,
cache, and TLB footprint (in particular on systems supporting BMI2).

Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>

--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -393,11 +393,26 @@ void __init arch_init_memory(void)
 
 const intpte_t pte_flags_mask = ~(PADDR_MASK & PAGE_MASK);
 
+paddr_t __read_mostly ma_real_mask = ~0UL;
+
 #ifndef HAVE_GAS_QUOTED_EXPR_SYM
 intpte_t put_pte_flags_v(unsigned int flags)
 {
     return put_pte_flags_c(flags);
 }
+
+/* Conversion between machine address and direct map offset. */
+paddr_t do2ma(unsigned long off)
+{
+    return (off & ma_va_bottom_mask) |
+           ((off << pfn_pdx_hole_shift) & ma_top_mask);
+}
+
+unsigned long ma2do(paddr_t ma)
+{
+    return (ma & ma_va_bottom_mask) |
+           ((ma & ma_top_mask) >> pfn_pdx_hole_shift);
+}
 #endif
 
 int page_is_ram_type(unsigned long mfn, unsigned long mem_type)
--- a/xen/arch/x86/srat.c
+++ b/xen/arch/x86/srat.c
@@ -446,6 +446,8 @@ void __init srat_parse_regions(u64 addr)
        }
 
        pfn_pdx_hole_setup(mask >> PAGE_SHIFT);
+
+       ma_real_mask = ma_top_mask | ma_va_bottom_mask;
 }
 
 /* Use the information discovered above to actually set up the nodes. */
--- a/xen/include/asm-x86/x86_64/page.h
+++ b/xen/include/asm-x86/x86_64/page.h
@@ -42,6 +42,10 @@
 #include <xen/pdx.h>
 
 extern unsigned long xen_virt_end;
+extern paddr_t ma_real_mask;
+
+paddr_t do2ma(unsigned long);
+unsigned long ma2do(paddr_t);
 
 /*
  * Note: These are solely for the use by page_{get,set}_owner(), and
@@ -52,8 +56,10 @@ extern unsigned long xen_virt_end;
 #define pdx_to_virt(pdx) ((void *)(DIRECTMAP_VIRT_START + \
                                    ((unsigned long)(pdx) << PAGE_SHIFT)))
 
-static inline unsigned long __virt_to_maddr(unsigned long va)
+static always_inline paddr_t __virt_to_maddr(unsigned long va)
 {
+    paddr_t ma;
+
     ASSERT(va < DIRECTMAP_VIRT_END);
     if ( va >= DIRECTMAP_VIRT_START )
         va -= DIRECTMAP_VIRT_START;
@@ -66,16 +72,81 @@ static inline unsigned long __virt_to_ma
 
         va += xen_phys_start - XEN_VIRT_START;
     }
-    return (va & ma_va_bottom_mask) |
-           ((va << pfn_pdx_hole_shift) & ma_top_mask);
+
+#ifdef HAVE_GAS_QUOTED_EXPR_SYM
+#define SYMNAME(pfx...) "\"" #pfx "do2ma_%[ma]_%[off]\""
+    alternative_io_2("call " SYMNAME() "; " ASM_NOP4 "\t"
+                     LINKONCE_PROLOGUE(SYMNAME)
+                     "mov %[shift], %%ecx\n\t"
+                     "mov %[off], %[ma]\n\t"
+                     "and %[bmask], %[ma]\n\t"
+                     "shl %%cl, %[off]\n\t"
+                     "and %[tmask], %[off]\n\t"
+                     "or %[off], %[ma]\n\t"
+                     "ret\n\t"
+                     LINKONCE_EPILOGUE(SYMNAME),
+                     "call " SYMNAME(), X86_FEATURE_ALWAYS,
+                     "pdep %[mask], %[off], %[ma]", X86_FEATURE_BMI2,
+                     ASM_OUTPUT2([ma] "=&r" (ma), [off] "+r" (va)),
+                     [mask] "m" (ma_real_mask),
+                     [shift] "m" (pfn_pdx_hole_shift),
+                     [bmask] "m" (ma_va_bottom_mask),
+                     [tmask] "m" (ma_top_mask)
+                     : "ecx");
+#undef SYMNAME
+#else
+    alternative_io_2("call do2ma; " ASM_NOP4,
+                     "call do2ma", X86_FEATURE_ALWAYS,
+                     /* pdep ma_real_mask(%rip), %rdi, %rax */
+                     ".byte 0xc4, 0xe2, 0xc3, 0xf5, 0x05\n\t"
+                     ".long ma_real_mask - 4 - .",
+                     X86_FEATURE_BMI2,
+                     ASM_OUTPUT2("=a" (ma), "+D" (va)), "m" (ma_real_mask)
+                     : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11");
+#endif
+
+    return ma;
 }
 
-static inline void *__maddr_to_virt(unsigned long ma)
+static always_inline void *__maddr_to_virt(paddr_t ma)
 {
+    unsigned long off;
+
     ASSERT(pfn_to_pdx(ma >> PAGE_SHIFT) < (DIRECTMAP_SIZE >> PAGE_SHIFT));
-    return (void *)(DIRECTMAP_VIRT_START +
-                    ((ma & ma_va_bottom_mask) |
-                     ((ma & ma_top_mask) >> pfn_pdx_hole_shift)));
+
+#ifdef HAVE_GAS_QUOTED_EXPR_SYM
+#define SYMNAME(pfx...) "\"" #pfx "ma2do_%[off]_%[ma]\""
+    alternative_io_2("call " SYMNAME() "; " ASM_NOP4 "\t"
+                     LINKONCE_PROLOGUE(SYMNAME)
+                     "mov %[tmask], %[off]\n\t"
+                     "mov %[shift], %%ecx\n\t"
+                     "and %[ma], %[off]\n\t"
+                     "and %[bmask], %[ma]\n\t"
+                     "shr %%cl, %[off]\n\t"
+                     "or %[ma], %[off]\n\t"
+                     "ret\n\t"
+                     LINKONCE_EPILOGUE(SYMNAME),
+                     "call " SYMNAME(), X86_FEATURE_ALWAYS,
+                     "pext %[mask], %[ma], %[off]", X86_FEATURE_BMI2,
+                     ASM_OUTPUT2([off] "=&r" (off), [ma] "+r" (ma)),
+                     [mask] "m" (ma_real_mask),
+                     [shift] "m" (pfn_pdx_hole_shift),
+                     [bmask] "m" (ma_va_bottom_mask),
+                     [tmask] "m" (ma_top_mask)
+                     : "ecx");
+#undef SYMNAME
+#else
+    alternative_io_2("call ma2do; " ASM_NOP4,
+                     "call ma2do", X86_FEATURE_ALWAYS,
+                     /* pext ma_real_mask(%rip), %rdi, %rax */
+                     ".byte 0xc4, 0xe2, 0xc2, 0xf5, 0x05\n\t"
+                     ".long ma_real_mask - 4 - .",
+                     X86_FEATURE_BMI2,
+                     ASM_OUTPUT2("=a" (off), "+D" (ma)), "m" (ma_real_mask)
+                     : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11");
+#endif
+
+    return (void *)DIRECTMAP_VIRT_START + off;
 }
 
 /* read access (should only be used for debug printk's) */



_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel