Re: [Xen-devel] [PATCH 4/5] x86: use PDEP/PEXT for PFN/PDX conversion when available

["Jan Beulich" <JBeulich@xxxxxxxx>]

Both replace 6 instructions by a single one, further reducing code size,
cache, and TLB footprint (in particular on systems supporting BMI2).

Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>

--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -394,6 +394,7 @@ void __init arch_init_memory(void)
 const intpte_t pte_flags_mask = ~(PADDR_MASK & PAGE_MASK);
 
 paddr_t __read_mostly ma_real_mask = ~0UL;
+unsigned long __read_mostly pfn_real_mask = ~0UL;
 
 #ifndef HAVE_GAS_QUOTED_EXPR_SYM
 intpte_t put_pte_flags_v(unsigned int flags)
@@ -413,6 +414,17 @@ unsigned long ma2do(paddr_t ma)
     return (ma & ma_va_bottom_mask) |
            ((ma & ma_top_mask) >> pfn_pdx_hole_shift);
 }
+
+/* Conversion between PDX and PFN. */
+unsigned long pdx2pfn(unsigned long pdx)
+{
+    return generic_pdx_to_pfn(pdx);
+}
+
+unsigned long pfn2pdx(unsigned long pfn)
+{
+    return generic_pfn_to_pdx(pfn);
+}
 #endif
 
 int page_is_ram_type(unsigned long mfn, unsigned long mem_type)
--- a/xen/arch/x86/srat.c
+++ b/xen/arch/x86/srat.c
@@ -448,6 +448,7 @@ void __init srat_parse_regions(u64 addr)
        pfn_pdx_hole_setup(mask >> PAGE_SHIFT);
 
        ma_real_mask = ma_top_mask | ma_va_bottom_mask;
+       pfn_real_mask = pfn_top_mask | pfn_pdx_bottom_mask;
 }
 
 /* Use the information discovered above to actually set up the nodes. */
--- /dev/null
+++ b/xen/include/asm-arm/pdx.h
@@ -0,0 +1,16 @@
+#ifndef __ASM_ARM_PDX_H__
+#define __ASM_ARM_PDX_H__
+
+#define pdx_to_pfn generic_pdx_to_pfn
+#define pfn_to_pdx generic_pfn_to_pdx
+
+#endif /* __ASM_ARM_PDX_H__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
--- /dev/null
+++ b/xen/include/asm-x86/pdx.h
@@ -0,0 +1,97 @@
+#ifndef __ASM_ARM_PDX_H__
+#define __ASM_ARM_PDX_H__
+
+#include <asm/alternative.h>
+#include <asm/asm_defns.h>
+#include <asm/cpufeature.h>
+
+extern unsigned long pfn_real_mask;
+
+static always_inline unsigned long pdx_to_pfn(unsigned long pdx)
+{
+    unsigned long pfn;
+
+#ifdef HAVE_GAS_QUOTED_EXPR_SYM
+#define SYMNAME(pfx...) "\"" #pfx "pdx2pfn_%[pfn]_%[pdx]\""
+    alternative_io_2("call " SYMNAME() "; " ASM_NOP4 "\t"
+                     LINKONCE_PROLOGUE(SYMNAME)
+                     "mov %[shift], %%ecx\n\t"
+                     "mov %[pdx], %[pfn]\n\t"
+                     "and %[bmask], %[pfn]\n\t"
+                     "shl %%cl, %[pdx]\n\t"
+                     "and %[tmask], %[pdx]\n\t"
+                     "or %[pdx], %[pfn]\n\t"
+                     "ret\n\t"
+                     LINKONCE_EPILOGUE(SYMNAME),
+                     "call " SYMNAME(), X86_FEATURE_ALWAYS,
+                     "pdep %[mask], %[pdx], %[pfn]", X86_FEATURE_BMI2,
+                     ASM_OUTPUT2([pfn] "=&r" (pfn), [pdx] "+r" (pdx)),
+                     [mask] "m" (pfn_real_mask),
+                     [shift] "m" (pfn_pdx_hole_shift),
+                     [bmask] "m" (pfn_pdx_bottom_mask),
+                     [tmask] "m" (pfn_top_mask)
+                     : "ecx");
+#undef SYMNAME
+#else
+    alternative_io_2("call pdx2pfn; " ASM_NOP4,
+                     "call pdx2pfn", X86_FEATURE_ALWAYS,
+                     /* pdep pfn_real_mask(%rip), %rdi, %rax */
+                     ".byte 0xc4, 0xe2, 0xc3, 0xf5, 0x05\n\t"
+                     ".long pfn_real_mask - 4 - .",
+                     X86_FEATURE_BMI2,
+                     ASM_OUTPUT2("=a" (pfn), "+D" (pdx)), "m" (pfn_real_mask)
+                     : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11");
+#endif
+
+    return pfn;
+}
+
+static always_inline unsigned long pfn_to_pdx(unsigned long pfn)
+{
+    unsigned long pdx;
+
+#ifdef HAVE_GAS_QUOTED_EXPR_SYM
+#define SYMNAME(pfx...) "\"" #pfx "pfn2pdx_%[pdx]_%[pfn]\""
+    alternative_io_2("call " SYMNAME() "; " ASM_NOP4 "\t"
+                     LINKONCE_PROLOGUE(SYMNAME)
+                     "mov %[tmask], %[pdx]\n\t"
+                     "mov %[shift], %%ecx\n\t"
+                     "and %[pfn], %[pdx]\n\t"
+                     "and %[bmask], %[pfn]\n\t"
+                     "shr %%cl, %[pdx]\n\t"
+                     "or %[pfn], %[pdx]\n\t"
+                     "ret\n\t"
+                     LINKONCE_EPILOGUE(SYMNAME),
+                     "call " SYMNAME(), X86_FEATURE_ALWAYS,
+                     "pext %[mask], %[pfn], %[pdx]", X86_FEATURE_BMI2,
+                     ASM_OUTPUT2([pdx] "=&r" (pdx), [pfn] "+r" (pfn)),
+                     [mask] "m" (pfn_real_mask),
+                     [shift] "m" (pfn_pdx_hole_shift),
+                     [bmask] "m" (pfn_pdx_bottom_mask),
+                     [tmask] "m" (pfn_top_mask)
+                     : "ecx");
+#undef SYMNAME
+#else
+    alternative_io_2("call pfn2pdx; " ASM_NOP4,
+                     "call pfn2pdx", X86_FEATURE_ALWAYS,
+                     /* pext pfn_real_mask(%rip), %rdi, %rax */
+                     ".byte 0xc4, 0xe2, 0xc2, 0xf5, 0x05\n\t"
+                     ".long pfn_real_mask - 4 - .",
+                     X86_FEATURE_BMI2,
+                     ASM_OUTPUT2("=a" (pdx), "+D" (pfn)), "m" (pfn_real_mask)
+                     : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11");
+#endif
+
+    return pdx;
+}
+
+#endif /* __ASM_ARM_PDX_H__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
--- a/xen/include/xen/pdx.h
+++ b/xen/include/xen/pdx.h
@@ -23,13 +23,13 @@ extern void set_pdx_range(unsigned long
 
 bool __mfn_valid(unsigned long mfn);
 
-static inline unsigned long pfn_to_pdx(unsigned long pfn)
+static inline unsigned long generic_pfn_to_pdx(unsigned long pfn)
 {
     return (pfn & pfn_pdx_bottom_mask) |
            ((pfn & pfn_top_mask) >> pfn_pdx_hole_shift);
 }
 
-static inline unsigned long pdx_to_pfn(unsigned long pdx)
+static inline unsigned long generic_pdx_to_pfn(unsigned long pdx)
 {
     return (pdx & pfn_pdx_bottom_mask) |
            ((pdx << pfn_pdx_hole_shift) & pfn_top_mask);
@@ -37,6 +37,8 @@ static inline unsigned long pdx_to_pfn(u
 
 extern void pfn_pdx_hole_setup(unsigned long);
 
+#include <asm/pdx.h>
+
 #endif /* HAS_PDX */
 #endif /* __XEN_PDX_H__ */
 



_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel