[Xen-devel] [PATCH v3 3/5] x86: use PDEP/PEXT for PFN/PDX conversion when available

["Jan Beulich" <JBeulich@xxxxxxxx>]

Both replace 6 instructions by a single one, further reducing code size,
cache, and TLB footprint (in particular on systems supporting BMI2).

Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>
---
v2: Avoid quoted symbols; use gcc's new V operand modifier instead.
    Re-base.

--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -391,6 +391,7 @@ void __init arch_init_memory(void)
 }
 
 paddr_t __read_mostly ma_real_mask = ~0UL;
+unsigned long __read_mostly pfn_real_mask = ~0UL;
 
 #ifndef CONFIG_INDIRECT_THUNK /* V modifier unavailable? */
 
@@ -407,6 +408,17 @@ unsigned long ma2do(paddr_t ma)
            ((ma & ma_top_mask) >> pfn_pdx_hole_shift);
 }
 
+/* Conversion between PDX and PFN. */
+unsigned long pdx2pfn(unsigned long pdx)
+{
+    return generic_pdx_to_pfn(pdx);
+}
+
+unsigned long pfn2pdx(unsigned long pfn)
+{
+    return generic_pfn_to_pdx(pfn);
+}
+
 #endif
 
 int page_is_ram_type(unsigned long mfn, unsigned long mem_type)
--- a/xen/arch/x86/srat.c
+++ b/xen/arch/x86/srat.c
@@ -452,6 +452,7 @@ void __init srat_parse_regions(u64 addr)
        pfn_pdx_hole_setup(mask >> PAGE_SHIFT);
 
        ma_real_mask = ma_top_mask | ma_va_bottom_mask;
+       pfn_real_mask = pfn_top_mask | pfn_pdx_bottom_mask;
 }
 
 /* Use the information discovered above to actually set up the nodes. */
--- /dev/null
+++ b/xen/include/asm-arm/pdx.h
@@ -0,0 +1,16 @@
+#ifndef __ASM_ARM_PDX_H__
+#define __ASM_ARM_PDX_H__
+
+#define pdx_to_pfn generic_pdx_to_pfn
+#define pfn_to_pdx generic_pfn_to_pdx
+
+#endif /* __ASM_ARM_PDX_H__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
--- /dev/null
+++ b/xen/include/asm-x86/pdx.h
@@ -0,0 +1,93 @@
+#ifndef __ASM_ARM_PDX_H__
+#define __ASM_ARM_PDX_H__
+
+#include <asm/alternative.h>
+#include <asm/asm_defns.h>
+#include <asm/cpufeature.h>
+
+extern unsigned long pfn_real_mask;
+
+static always_inline unsigned long pdx_to_pfn(unsigned long pdx)
+{
+    unsigned long pfn;
+
+#ifdef CONFIG_INDIRECT_THUNK /* V modifier available? */
+#define SYMNAME(pfx...) #pfx "pdx2pfn_%V[pfn]_%V[pdx]"
+    alternative_io("call " SYMNAME() "\n\t"
+                   LINKONCE_PROLOGUE(SYMNAME) "\n\t"
+                   "mov %[shift], %%ecx\n\t"
+                   "mov %[pdx], %[pfn]\n\t"
+                   "and %[bmask], %[pfn]\n\t"
+                   "shl %%cl, %[pdx]\n\t"
+                   "and %[tmask], %[pdx]\n\t"
+                   "or %[pdx], %[pfn]\n\t"
+                   "ret\n\t"
+                   LINKONCE_EPILOGUE(SYMNAME),
+                   "pdep %[mask], %[pdx], %[pfn]", X86_FEATURE_BMI2,
+                   ASM_OUTPUT2([pfn] "=&r" (pfn), [pdx] "+r" (pdx)),
+                   [mask] "m" (pfn_real_mask),
+                   [shift] "m" (pfn_pdx_hole_shift),
+                   [bmask] "m" (pfn_pdx_bottom_mask),
+                   [tmask] "m" (pfn_top_mask)
+                   : "ecx");
+#undef SYMNAME
+#else
+    alternative_io("call pdx2pfn",
+                   /* pdep pfn_real_mask(%rip), %rdi, %rax */
+                   ".byte 0xc4, 0xe2, 0xc3, 0xf5, 0x05\n\t"
+                   ".long pfn_real_mask - 4 - .",
+                   X86_FEATURE_BMI2,
+                   ASM_OUTPUT2("=a" (pfn), "+D" (pdx)), "m" (pfn_real_mask)
+                   : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11");
+#endif
+
+    return pfn;
+}
+
+static always_inline unsigned long pfn_to_pdx(unsigned long pfn)
+{
+    unsigned long pdx;
+
+#ifdef CONFIG_INDIRECT_THUNK /* V modifier available? */
+#define SYMNAME(pfx...) #pfx "pfn2pdx_%V[pdx]_%V[pfn]"
+    alternative_io("call " SYMNAME() "\n\t"
+                   LINKONCE_PROLOGUE(SYMNAME) "\n\t"
+                   "mov %[tmask], %[pdx]\n\t"
+                   "mov %[shift], %%ecx\n\t"
+                   "and %[pfn], %[pdx]\n\t"
+                   "and %[bmask], %[pfn]\n\t"
+                   "shr %%cl, %[pdx]\n\t"
+                   "or %[pfn], %[pdx]\n\t"
+                   "ret\n\t"
+                   LINKONCE_EPILOGUE(SYMNAME),
+                   "pext %[mask], %[pfn], %[pdx]", X86_FEATURE_BMI2,
+                   ASM_OUTPUT2([pdx] "=&r" (pdx), [pfn] "+r" (pfn)),
+                   [mask] "m" (pfn_real_mask),
+                   [shift] "m" (pfn_pdx_hole_shift),
+                   [bmask] "m" (pfn_pdx_bottom_mask),
+                   [tmask] "m" (pfn_top_mask)
+                   : "ecx");
+#undef SYMNAME
+#else
+    alternative_io("call pfn2pdx",
+                   /* pext pfn_real_mask(%rip), %rdi, %rax */
+                   ".byte 0xc4, 0xe2, 0xc2, 0xf5, 0x05\n\t"
+                   ".long pfn_real_mask - 4 - .",
+                   X86_FEATURE_BMI2,
+                   ASM_OUTPUT2("=a" (pdx), "+D" (pfn)), "m" (pfn_real_mask)
+                   : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11");
+#endif
+
+    return pdx;
+}
+
+#endif /* __ASM_ARM_PDX_H__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
--- a/xen/include/xen/pdx.h
+++ b/xen/include/xen/pdx.h
@@ -23,13 +23,13 @@ extern void set_pdx_range(unsigned long
 
 bool __mfn_valid(unsigned long mfn);
 
-static inline unsigned long pfn_to_pdx(unsigned long pfn)
+static inline unsigned long generic_pfn_to_pdx(unsigned long pfn)
 {
     return (pfn & pfn_pdx_bottom_mask) |
            ((pfn & pfn_top_mask) >> pfn_pdx_hole_shift);
 }
 
-static inline unsigned long pdx_to_pfn(unsigned long pdx)
+static inline unsigned long generic_pdx_to_pfn(unsigned long pdx)
 {
     return (pdx & pfn_pdx_bottom_mask) |
            ((pdx << pfn_pdx_hole_shift) & pfn_top_mask);
@@ -40,6 +40,8 @@ static inline unsigned long pdx_to_pfn(u
 
 extern void pfn_pdx_hole_setup(unsigned long);
 
+#include <asm/pdx.h>
+
 #endif /* HAS_PDX */
 #endif /* __XEN_PDX_H__ */
 




_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel