[PATCH v3 3/5] x86: allow Kconfig control over psABI level

[Jan Beulich <jbeulich@xxxxxxxx>]

Newer hardware offers more efficient and/or flexible and/or capable
instructions, some of which we can make good use of in the hypervisor
as well. Allow a basic way (no alternatives patching) of enabling their
use. Of course this means that hypervisors thus built won't work
anymore on older, less capable hardware.

Since older compilers (apparently gcc10 / clang11 and older) won't
recognize -march=x86-64-v2 etc, also addd fallback logic passing
-mpopcnt and alike explicitly.

Note that in efi_arch_cpu() the filling of 7c0 and 7d0 are forward-
looking; we only require 7b0, but we need to use cpuid_count() anyway.

Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>
---
TBD: While the added assembly code goes strictly from the CONFIG_*
     settings, EFI code uses the compiler provided manifest constants
     where available (just like the subsequent "x86: short-circuit
     certain cpu_has_* when x86-64-v{2,3} are in effect"). While I
     generally prefer it that way, this comes with a downside: If we
     ever started to use one of the ISA extensions in assembly code
     (and then keyed to the CONFIG_* settings), things could break. I
     don't view us doing this as very likely though, as then we'd need
     to have two (or more) variants of such assembly code, which we
     would likely want to avoid. (What I'd like to avoid is using the
     compiler manifest constants in assembly code: In principle we ought
     to filter out any -march= when constructing AFLAGS, and perhaps
     at least all -m... and -f... options. Compilers might rightfully
     complain about their use as inapplicable, much like we've seen for
     -Wl,... when no linking is done.)

TBD: While we don't document most "cpuid=no-*" options (and hence imply
     their use to be unsupported), using e.g. "cpuid=no-popcnt" with a
     V2=y hypervisor clearly can't have the intended effect, and hence
     might perhaps better be flagged in some way.

TBD: v2 also includes LAHF/SAHF. Since we don't use floating point and
     hence FPU insns, we ought to be okay not explicitly checking for
     it. But there is a certain amount of risk of the compiler finding
     some "smart" use for one or both of the insns. However, if we were
     to check the feature, we'd need to account for the quirk that
     init_amd() also works around.

Whereas the baseline -> v2 step isn't much of a difference (we'll gain
more there by a subsequent patch), v2 -> v3, while presumably (or shall
I say hopefully) faster, yields an overall growth of .text size by (in
my build) about 2k. The primary reason for this appear to be conversions
of SHL-by-immediate to SHLX.

The VGA output attempt in early (MB/MB2) boot code does not appear to
work (anymore?). The serial output may work, but only if - without any
setup in Xen - both sides agree on the serial settings (baud rate etc).
Hence the feature checks added on the legacy paths are of limited use.
---
v3: Add fallback logic and use logic from new prereq change.
v2: Also cover XSAVE. Add early boot feature checking.

--- a/xen/arch/x86/Kconfig
+++ b/xen/arch/x86/Kconfig
@@ -118,6 +118,36 @@ config HVM
 
          If unsure, say Y.
 
+choice
+       prompt "base psABI level"
+       default X86_64_BASELINE
+       help
+         The psABI defines 4 levels of ISA extension sets as a coarse granular
+         way of identifying advanced functionality that would be uniformly
+         available in respectively newer hardware.  While v4 is not really of
+         interest for Xen, the others can be selected here, making the
+         resulting Xen no longer work on older hardware.  This option won't
+         have any effect if the toolchain doesn't support the distinction.
+
+         If unsure, stick to the default.
+
+config X86_64_BASELINE
+       bool "baseline"
+
+config X86_64_V2
+       bool "v2"
+       help
+         This enables POPCNT and CX16, besides other extensions which are of
+         no interest here.
+
+config X86_64_V3
+       bool "v3"
+       help
+         This enables BMI, BMI2, LZCNT, MOVBE, and XSAVE, besides other
+         extensions which are of no interest here.
+
+endchoice
+
 config XEN_SHSTK
        bool "Supervisor Shadow Stacks"
        depends on HAS_AS_CET_SS
--- a/xen/arch/x86/arch.mk
+++ b/xen/arch/x86/arch.mk
@@ -36,6 +36,29 @@ CFLAGS += -mno-red-zone -fpic
 # the SSE setup for variadic function calls.
 CFLAGS += -mno-mmx -mno-sse $(call cc-option,$(CC),-mskip-rax-setup)
 
+# Enable the selected baseline ABI, if supported by the compiler.
+x86-64-$(CONFIG_X86_64_BASELINE) :=
+x86-64-$(CONFIG_X86_64_V2) := x86-64-v2
+x86-64-$(CONFIG_X86_64_V3) := x86-64-v3
+
+ifneq ($(x86-64-y),)
+CFLAGS-psabi := $(call cc-option,$(CC),-march=$(x86-64-y))
+
+ifeq ($(CFLAGS-psabi),)
+# Fall back to using the subset of -m... options that are of interest.
+x86-64-v2 := -mpopcnt -mcx16
+x86-64-v3 := $(x86-64-v2) -mbmi -mbmi2 -mlzcnt -mmovbe -mxsave
+$(call cc-options-add,CFLAGS-psabi,CC,$($(x86-64-y)))
+
+ifneq ($(strip $(CFLAGS-psabi)),$($(x86-64-y)))
+$(warning Options not recognized by $(CC): $(filter-out 
$(CFLAGS-psabi),$($(x86-64-y))))
+XEN_CONFIG_UNSATISFIED += X86_64_Vn
+endif
+endif # CFLAGS-psabi
+
+CFLAGS += $(CFLAGS-psabi)
+endif # x86-64-y
+
 ifeq ($(CONFIG_INDIRECT_THUNK),y)
 # Compile with gcc thunk-extern, indirect-branch-register if available.
 CFLAGS-$(CONFIG_CC_IS_GCC) += -mindirect-branch=thunk-extern
--- a/xen/arch/x86/boot/head.S
+++ b/xen/arch/x86/boot/head.S
@@ -124,6 +124,12 @@ multiboot2_header:
 .Lbad_efi_msg: .asciz "ERR: EFI IA-32 platforms are not supported!"
 .Lbag_alg_msg: .asciz "ERR: Xen must be loaded at a 2Mb boundary!"
 .Lno_nx_msg:   .asciz "ERR: Not an NX-capable CPU!"
+#if defined(CONFIG_X86_64_V2) || defined(CONFIG_X86_64_V3)
+.Lno_x86_64_v2_msg: .asciz "ERR: Not an x86-64-v2 capable CPU!"
+#endif
+#ifdef CONFIG_X86_64_V3
+.Lno_x86_64_v3_msg: .asciz "ERR: Not an x86-64-v3 capable CPU!"
+#endif
 
         .section .init.data, "aw", @progbits
         .align 4
@@ -156,9 +162,20 @@ early_error: /* Here to improve the disa
         jmp     .Lget_vtb
 #ifdef CONFIG_REQUIRE_NX
 .Lno_nx:
+        pop     %ecx
         mov     $sym_offs(.Lno_nx_msg), %ecx
         jmp     .Lget_vtb
 #endif
+#if defined(CONFIG_X86_64_V2) || defined(CONFIG_X86_64_V3)
+.Lno_x86_64_v2:
+        mov     $sym_offs(.Lno_x86_64_v2_msg), %ecx
+        jmp     .Lget_vtb
+#endif
+#ifdef CONFIG_X86_64_V3
+.Lno_x86_64_v3:
+        mov     $sym_offs(.Lno_x86_64_v3_msg), %ecx
+        jmp     .Lget_vtb
+#endif
 .Lmb2_no_st:
         /*
          * Here we are on EFI platform. vga_text_buffer was zapped earlier
@@ -657,6 +674,7 @@ trampoline_setup:
         mov     $1, %eax
         cpuid
         mov     %ecx, CPUINFO_FEATURE_OFFSET(X86_FEATURE_HYPERVISOR) + 
sym_esi(boot_cpu_data)
+        mov     %ecx, %edi
 
         mov     $0x80000000,%eax
         cpuid
@@ -674,6 +692,9 @@ trampoline_setup:
         bt      $cpufeat_bit(X86_FEATURE_LM),%edx
         jnc     .Lbad_cpu
 
+        /* Preserve %ecx for later use. */
+        push    %ecx
+
         /*
          * Check for NX
          *   - If Xen was compiled requiring it simply assert it's
@@ -724,6 +745,36 @@ trampoline_setup:
 .Lno_nx:
 #endif
 
+        /* Restore CPUID[80000001].ecx output. */
+        pop     %ecx
+
+#if defined(CONFIG_X86_64_V2) || defined(CONFIG_X86_64_V3)
+        mov     $cpufeat_mask(X86_FEATURE_POPCNT) | 
cpufeat_mask(X86_FEATURE_CX16), %eax
+        and     %edi, %eax
+        cmp     $cpufeat_mask(X86_FEATURE_POPCNT) | 
cpufeat_mask(X86_FEATURE_CX16), %eax
+        jne     .Lno_x86_64_v2
+#endif
+
+#ifdef CONFIG_X86_64_V3
+        mov     $cpufeat_mask(X86_FEATURE_MOVBE) | 
cpufeat_mask(X86_FEATURE_XSAVE), %eax
+        and     %edi, %eax
+        cmp     $cpufeat_mask(X86_FEATURE_MOVBE) | 
cpufeat_mask(X86_FEATURE_XSAVE), %eax
+        jne     .Lno_x86_64_v3
+        bt      $cpufeat_bit(X86_FEATURE_ABM), %ecx
+        jnc     .Lno_x86_64_v3
+        xor     %eax, %eax
+        cpuid
+        cmp     $7, %eax
+        jb      .Lno_x86_64_v3
+        mov     $7, %eax
+        xor     %ecx, %ecx
+        cpuid
+        mov     $cpufeat_mask(X86_FEATURE_BMI1) | 
cpufeat_mask(X86_FEATURE_BMI2), %eax
+        and     %ebx, %eax
+        cmp     $cpufeat_mask(X86_FEATURE_BMI1) | 
cpufeat_mask(X86_FEATURE_BMI2), %eax
+        jne     .Lno_x86_64_v3
+#endif
+
         /* Stash TSC to calculate a good approximation of time-since-boot */
         rdtsc
         mov     %eax,     sym_esi(boot_tsc_stamp)
--- a/xen/arch/x86/efi/efi-boot.h
+++ b/xen/arch/x86/efi/efi-boot.h
@@ -740,29 +740,53 @@ static void __init efi_arch_handle_modul
 
 static void __init efi_arch_cpu(void)
 {
-    uint32_t eax = cpuid_eax(0x80000000);
+    uint32_t eax = cpuid_eax(0), dummy;
     uint32_t *caps = boot_cpu_data.x86_capability;
 
     boot_tsc_stamp = rdtsc();
 
-    caps[FEATURESET_1c] = cpuid_ecx(1);
+    if ( eax )
+        caps[FEATURESET_1c] = cpuid_ecx(1);
 
-    if ( (eax >> 16) == 0x8000 && eax > 0x80000000 )
-    {
-        caps[FEATURESET_e1d] = cpuid_edx(0x80000001);
-
-        /*
-         * This check purposefully doesn't use cpu_has_nx because
-         * cpu_has_nx bypasses the boot_cpu_data read if Xen was compiled
-         * with CONFIG_REQUIRE_NX
-         */
-        if ( IS_ENABLED(CONFIG_REQUIRE_NX) &&
-             !boot_cpu_has(X86_FEATURE_NX) )
-            blexit(L"This build of Xen requires NX support");
-
-        if ( cpu_has_nx )
-            trampoline_efer |= EFER_NXE;
-    }
+    if ( eax >= 7 )
+        cpuid_count(7, 0, &dummy,
+                    &caps[FEATURESET_7b0],
+                    &caps[FEATURESET_7c0],
+                    &caps[FEATURESET_7d0]);
+
+    eax = cpuid_eax(0x80000000U);
+    if ( (eax >> 16) == 0x8000 && eax > 0x80000000U )
+        cpuid(0x80000001U, &dummy, &dummy,
+              &caps[FEATURESET_e1c], &caps[FEATURESET_e1d]);
+
+    /*
+     * Checks here purposefully don't use cpu_has_* because they bypass
+     * boot_cpu_data reads if Xen was compiled with respective CONFIG_*.
+     */
+#define CHK(ppsym, feat) do {                                        \
+        if ( IS_ENABLED(ppsym) &&                                    \
+             !boot_cpu_has(X86_FEATURE_ ## feat) )                   \
+            blexit(L"This build of Xen requires " #feat " support"); \
+    } while ( false )
+
+    CHK(CONFIG_REQUIRE_NX, NX);
+
+    /* x86-64-v2 */
+    CHK(__POPCNT__, POPCNT);
+    CHK(CONFIG_X86_64_V2, CX16);
+    CHK(CONFIG_X86_64_V3, CX16);
+
+    /* x86-64-v3 */
+    CHK(__ABM__, ABM); /* LZCNT */
+    CHK(__BMI__, BMI1);
+    CHK(__BMI2__, BMI2);
+    CHK(__MOVBE__, MOVBE);
+    CHK(__XSAVE__, XSAVE);
+
+#undef CHK
+
+    if ( cpu_has_nx )
+        trampoline_efer |= EFER_NXE;
 }
 
 static void __init efi_arch_blexit(void)