[PATCH v1 16/17] xen/riscv: add guest load emulation for trapped MMIO accesses

[Oleksii Kurochko <oleksii.kurochko@xxxxxxxxx>]

Introduce emulate_load() to decode and emulate guest load instructions
that fault due to MMIO accesses. This provides the basic infrastructure
required for MMIO emulation on RISC-V.

The instruction decode (decode_trapped_insn() and the mask/match chain
for standard and compressed load encodings) is adapted from Linux's KVM
RISC-V implementation. The completion path differs from KVM's,
since Xen dispatches MMIO synchronously to an in-hypervisor handler via
try_handle_mmio() and has no userspace exit/return step equivalent to
KVM's kvm_io_bus_read() / KVM_EXIT_MMIO / kvm_riscv_vcpu_mmio_return()
split.

A fault taken while re-reading the trapped instruction is handled
depending on the faulting translation stage:
 - A VS-stage fault is the guest's own fault (e.g. it modified its page
   tables from another vCPU) and, as in KVM, is redirected to the
   guest's trap vector, with the cause remapped to
   CAUSE_FETCH_PAGE_FAULT since HLVX reports execute-permission failures
   as load faults.
 - A G-stage fault would mean the P2M mapping of the instruction page
   disappeared after the instruction was fetched. KVM must handle this
   by resuming the guest and retrying, as Linux MM can invalidate
   G-stage mappings at any time. Xen does not remove P2M mappings of a
   running domain at the moment, so this case is asserted unreachable with
   BUG_ON(); it will need to be revisited once such removal is implemented.

When a guest load triggers a page fault, the trapped instruction is
decoded using HTINST or, if unavailable, fetched via unprivileged access.
At the moment only virtual interrupt controller (vINTC) traps are
expected to occur, since it is currently the only backend registered
with the MMIO handler dispatch, so in practice the load is emulated via
the vINTC backend and the guest register state is updated accordingly.

Signed-off-by: Oleksii Kurochko <oleksii.kurochko@xxxxxxxxx>
---
 xen/arch/riscv/include/asm/traps.h |   6 ++
 xen/arch/riscv/traps.c             | 162 ++++++++++++++++++++++++++++-
 2 files changed, 167 insertions(+), 1 deletion(-)

diff --git a/xen/arch/riscv/include/asm/traps.h 
b/xen/arch/riscv/include/asm/traps.h
index 8d4ab664bca9..937295c5c76c 100644
--- a/xen/arch/riscv/include/asm/traps.h
+++ b/xen/arch/riscv/include/asm/traps.h
@@ -4,6 +4,7 @@
 #define ASM__RISCV__TRAPS_H
 
 #include <asm/processor.h>
+#include <asm/riscv_encoding.h>
 
 #ifndef __ASSEMBLER__
 
@@ -13,6 +14,11 @@ struct trap_info {
     register_t stval;
 };
 
+static inline bool is_load_guest_page_fault(unsigned long scause)
+{
+    return (scause == CAUSE_LOAD_GUEST_PAGE_FAULT);
+}
+
 void do_trap(struct cpu_user_regs *cpu_regs);
 void handle_trap(void);
 void trap_init(void);
diff --git a/xen/arch/riscv/traps.c b/xen/arch/riscv/traps.c
index 1c97bd101948..12690ae37aba 100644
--- a/xen/arch/riscv/traps.c
+++ b/xen/arch/riscv/traps.c
@@ -14,7 +14,9 @@
 
 #include <asm/extable.h>
 #include <asm/cpufeature.h>
+#include <asm/guest_access.h>
 #include <asm/intc.h>
+#include <asm/mmio.h>
 #include <asm/processor.h>
 #include <asm/riscv_encoding.h>
 #include <asm/traps.h>
@@ -191,6 +193,11 @@ static void timer_interrupt(void)
     raise_softirq(TIMER_SOFTIRQ);
 }
 
+static always_inline void advance_pc(struct cpu_user_regs *regs, int step)
+{
+    regs->sepc += step;
+}
+
 static always_inline unsigned long get_faulting_gpa(void)
 {
     /*
@@ -210,9 +217,162 @@ static always_inline unsigned long get_faulting_gpa(void)
     return (csr_read(CSR_HTVAL) << 2) | (csr_read(CSR_STVAL) & 0x3);
 }
 
+/*
+ * Determine the trapped instruction which caused a guest MMIO trap.
+ *
+ * Returns true if the trap was redirected to the guest, in which case
+ * the caller must stop emulation and return success. Otherwise *insn
+ * and *insn_len are filled in and the caller should continue decoding.
+ */
+static bool decode_trapped_insn(unsigned long htinst, unsigned long *insn,
+                                unsigned int *insn_len)
+{
+    if ( htinst & 0x1 )
+    {
+        /*
+         * Bit[0] == 1 implies trapped instruction value is
+         * transformed instruction or custom instruction.
+         */
+        *insn = htinst | INSN_16BIT_MASK;
+        *insn_len = (htinst & BIT(1, UL)) ? INSN_LEN(*insn) : 2;
+    }
+    else
+    {
+        struct cpu_user_regs *regs = vcpu_guest_cpu_user_regs(current);
+        struct trap_info utrap = { 0 };
+
+        /*
+         * Bit[0] == 0 implies trapped instruction value is
+         * zero or special value.
+         */
+        *insn = riscv_vcpu_unpriv_read(true, regs->sepc, &utrap);
+        if ( utrap.scause )
+        {
+            /*
+             * A G-stage fault here would mean the P2M mapping of the page
+             * containing the trapped instruction disappeared after it was
+             * fetched. Nothing removes P2M mappings of a running domain yet,
+             * so this cannot happen.
+             *
+             * TODO: Revisit once P2M mappings can be removed at runtime.
+             */
+            BUG_ON(is_load_guest_page_fault(utrap.scause));
+
+            utrap.sepc = regs->sepc;
+            utrap.stval = utrap.sepc;
+
+            riscv_vcpu_trap_redirect(&utrap);
+
+            return true;
+        }
+
+        *insn_len = INSN_LEN(*insn);
+    }
+
+    return false;
+}
+
+/*
+ * Check alignment and dispatch a decoded MMIO access to a registered
+ * handler. On success (0), info->data holds the read value for loads.
+ */
+static int do_mmio(mmio_info_t *info, unsigned long fault_addr,
+                   unsigned int len)
+{
+    /* Fault address should be aligned to length of MMIO */
+    if ( fault_addr & (len - 1) )
+        return -EIO;
+
+    info->gpa = fault_addr;
+    info->len = len;
+
+    switch ( try_handle_mmio(info) )
+    {
+    case IO_HANDLED:
+        return 0;
+    case IO_ABORT:
+        return -EIO;
+    default:
+        return -EOPNOTSUPP;
+    }
+}
+
 static int emulate_load(unsigned long fault_addr, unsigned long htinst)
 {
-    return -EOPNOTSUPP;
+    struct cpu_user_regs *regs = vcpu_guest_cpu_user_regs(current);
+    mmio_info_t info = { .is_write = false };
+    unsigned long insn;
+    unsigned int shift = 0, len, insn_len;
+    bool is_unsigned = false;
+    int rc;
+
+    if ( decode_trapped_insn(htinst, &insn, &insn_len) )
+        return 0;
+
+    /* Decode length of MMIO and whether it is a sign- or zero-extending load 
*/
+    if ( (insn & INSN_MASK_LB) == INSN_MATCH_LB )
+        len = 1;
+    else if ( (insn & INSN_MASK_LBU) == INSN_MATCH_LBU )
+    {
+        len = 1;
+        is_unsigned = true;
+    }
+    else if ( (insn & INSN_MASK_LH) == INSN_MATCH_LH )
+        len = 2;
+    else if ( (insn & INSN_MASK_LHU) == INSN_MATCH_LHU )
+    {
+        len = 2;
+        is_unsigned = true;
+    }
+    else if ( (insn & INSN_MASK_LW) == INSN_MATCH_LW )
+        len = 4;
+#ifndef CONFIG_RISCV_32
+    else if ( (insn & INSN_MASK_LWU) == INSN_MATCH_LWU )
+    {
+        len = 4;
+        is_unsigned = true;
+    }
+#endif
+    else if ( (insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW )
+    {
+        len = 4;
+        insn = RVC_RS2S(insn) << SH_RD;
+    }
+    else if ( (insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
+              RV_X(insn, SH_RD, 5) )
+        len = 4;
+#ifndef CONFIG_RISCV_32
+    else if ( (insn & INSN_MASK_LD) == INSN_MATCH_LD )
+        len = 8;
+    else if ( (insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD )
+    {
+        len = 8;
+        insn = RVC_RS2S(insn) << SH_RD;
+    }
+    else if ( (insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
+              RV_X(insn, SH_RD, 5) )
+        len = 8;
+#endif
+    else
+        return -EOPNOTSUPP;
+
+    if ( !is_unsigned )
+        shift = BITS_PER_BYTE * (sizeof(unsigned long) - len);
+
+#ifdef EMULATE_LOAD_DEBUG
+    gdprintk(XENLOG_DEBUG, "pc=%#02lx, addr=%#02lx, len=%d, shift=%d\n",
+             regs->sepc, fault_addr, len, shift);
+#endif
+
+    rc = do_mmio(&info, fault_addr, len);
+    if ( rc )
+        return rc;
+
+    SET_RD(insn, regs, (long)((unsigned long)info.data << shift) >> shift);
+
+    advance_pc(regs, insn_len);
+
+    return 0;
 }
 
 static int emulate_store(unsigned long fault_addr, unsigned long htinst)
-- 
2.54.0