|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v2 5/6] x86: use PDEP/PEXT for PFN/PDX conversion when available
Both replace 6 instructions by a single one, further reducing code size,
cache, and TLB footprint (in particular on systems supporting BMI2).
Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>
---
v2: Avoid quoted symbols; use gcc's new V operand modifier instead.
Re-base.
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -394,6 +394,7 @@ void __init arch_init_memory(void)
const intpte_t pte_flags_mask = ~(PADDR_MASK & PAGE_MASK);
paddr_t __read_mostly ma_real_mask = ~0UL;
+unsigned long __read_mostly pfn_real_mask = ~0UL;
#ifndef CONFIG_INDIRECT_THUNK /* V modifier unavailable? */
intpte_t put_pte_flags_v(unsigned int flags)
@@ -413,6 +414,17 @@ unsigned long ma2do(paddr_t ma)
return (ma & ma_va_bottom_mask) |
((ma & ma_top_mask) >> pfn_pdx_hole_shift);
}
+
+/* Conversion between PDX and PFN. */
+unsigned long pdx2pfn(unsigned long pdx)
+{
+ return generic_pdx_to_pfn(pdx);
+}
+
+unsigned long pfn2pdx(unsigned long pfn)
+{
+ return generic_pfn_to_pdx(pfn);
+}
#endif
int page_is_ram_type(unsigned long mfn, unsigned long mem_type)
--- a/xen/arch/x86/srat.c
+++ b/xen/arch/x86/srat.c
@@ -448,6 +448,7 @@ void __init srat_parse_regions(u64 addr)
pfn_pdx_hole_setup(mask >> PAGE_SHIFT);
ma_real_mask = ma_top_mask | ma_va_bottom_mask;
+ pfn_real_mask = pfn_top_mask | pfn_pdx_bottom_mask;
}
/* Use the information discovered above to actually set up the nodes. */
--- /dev/null
+++ b/xen/include/asm-arm/pdx.h
@@ -0,0 +1,16 @@
+#ifndef __ASM_ARM_PDX_H__
+#define __ASM_ARM_PDX_H__
+
+#define pdx_to_pfn generic_pdx_to_pfn
+#define pfn_to_pdx generic_pfn_to_pdx
+
+#endif /* __ASM_ARM_PDX_H__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
--- /dev/null
+++ b/xen/include/asm-x86/pdx.h
@@ -0,0 +1,93 @@
+#ifndef __ASM_ARM_PDX_H__
+#define __ASM_ARM_PDX_H__
+
+#include <asm/alternative.h>
+#include <asm/asm_defns.h>
+#include <asm/cpufeature.h>
+
+extern unsigned long pfn_real_mask;
+
+static always_inline unsigned long pdx_to_pfn(unsigned long pdx)
+{
+ unsigned long pfn;
+
+#ifdef CONFIG_INDIRECT_THUNK /* V modifier available? */
+#define SYMNAME(pfx...) #pfx "pdx2pfn_%V[pfn]_%V[pdx]"
+ alternative_io("call " SYMNAME() "\n\t"
+ LINKONCE_PROLOGUE(SYMNAME) "\n\t"
+ "mov %[shift], %%ecx\n\t"
+ "mov %[pdx], %[pfn]\n\t"
+ "and %[bmask], %[pfn]\n\t"
+ "shl %%cl, %[pdx]\n\t"
+ "and %[tmask], %[pdx]\n\t"
+ "or %[pdx], %[pfn]\n\t"
+ "ret\n\t"
+ LINKONCE_EPILOGUE(SYMNAME),
+ "pdep %[mask], %[pdx], %[pfn]", X86_FEATURE_BMI2,
+ ASM_OUTPUT2([pfn] "=&r" (pfn), [pdx] "+r" (pdx)),
+ [mask] "m" (pfn_real_mask),
+ [shift] "m" (pfn_pdx_hole_shift),
+ [bmask] "m" (pfn_pdx_bottom_mask),
+ [tmask] "m" (pfn_top_mask)
+ : "ecx");
+#undef SYMNAME
+#else
+ alternative_io("call pdx2pfn",
+ /* pdep pfn_real_mask(%rip), %rdi, %rax */
+ ".byte 0xc4, 0xe2, 0xc3, 0xf5, 0x05\n\t"
+ ".long pfn_real_mask - 4 - .",
+ X86_FEATURE_BMI2,
+ ASM_OUTPUT2("=a" (pfn), "+D" (pdx)), "m" (pfn_real_mask)
+ : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11");
+#endif
+
+ return pfn;
+}
+
+static always_inline unsigned long pfn_to_pdx(unsigned long pfn)
+{
+ unsigned long pdx;
+
+#ifdef CONFIG_INDIRECT_THUNK /* V modifier available? */
+#define SYMNAME(pfx...) #pfx "pfn2pdx_%V[pdx]_%V[pfn]"
+ alternative_io("call " SYMNAME() "\n\t"
+ LINKONCE_PROLOGUE(SYMNAME) "\n\t"
+ "mov %[tmask], %[pdx]\n\t"
+ "mov %[shift], %%ecx\n\t"
+ "and %[pfn], %[pdx]\n\t"
+ "and %[bmask], %[pfn]\n\t"
+ "shr %%cl, %[pdx]\n\t"
+ "or %[pfn], %[pdx]\n\t"
+ "ret\n\t"
+ LINKONCE_EPILOGUE(SYMNAME),
+ "pext %[mask], %[pfn], %[pdx]", X86_FEATURE_BMI2,
+ ASM_OUTPUT2([pdx] "=&r" (pdx), [pfn] "+r" (pfn)),
+ [mask] "m" (pfn_real_mask),
+ [shift] "m" (pfn_pdx_hole_shift),
+ [bmask] "m" (pfn_pdx_bottom_mask),
+ [tmask] "m" (pfn_top_mask)
+ : "ecx");
+#undef SYMNAME
+#else
+ alternative_io("call pfn2pdx",
+ /* pext pfn_real_mask(%rip), %rdi, %rax */
+ ".byte 0xc4, 0xe2, 0xc2, 0xf5, 0x05\n\t"
+ ".long pfn_real_mask - 4 - .",
+ X86_FEATURE_BMI2,
+ ASM_OUTPUT2("=a" (pdx), "+D" (pfn)), "m" (pfn_real_mask)
+ : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11");
+#endif
+
+ return pdx;
+}
+
+#endif /* __ASM_ARM_PDX_H__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
--- a/xen/include/xen/pdx.h
+++ b/xen/include/xen/pdx.h
@@ -23,13 +23,13 @@ extern void set_pdx_range(unsigned long
bool __mfn_valid(unsigned long mfn);
-static inline unsigned long pfn_to_pdx(unsigned long pfn)
+static inline unsigned long generic_pfn_to_pdx(unsigned long pfn)
{
return (pfn & pfn_pdx_bottom_mask) |
((pfn & pfn_top_mask) >> pfn_pdx_hole_shift);
}
-static inline unsigned long pdx_to_pfn(unsigned long pdx)
+static inline unsigned long generic_pdx_to_pfn(unsigned long pdx)
{
return (pdx & pfn_pdx_bottom_mask) |
((pdx << pfn_pdx_hole_shift) & pfn_top_mask);
@@ -37,6 +37,8 @@ static inline unsigned long pdx_to_pfn(u
extern void pfn_pdx_hole_setup(unsigned long);
+#include <asm/pdx.h>
+
#endif /* HAS_PDX */
#endif /* __XEN_PDX_H__ */
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |