[PATCH v2 1/2] xen/x86: add nmi continuation framework

[Juergen Gross <jgross@xxxxxxxx>]

Actions in NMI context are rather limited as e.g. locking is rather
fragile.

Add a generic framework to continue processing in softirq context after
leaving NMI processing. This is working for NMIs happening in guest
context as NMI exit handling will issue an IPI to itself in case a
softirq is pending, resulting in the continuation running before the
guest gets control again.

Signed-off-by: Juergen Gross <jgross@xxxxxxxx>
---
 xen/arch/x86/traps.c      | 37 +++++++++++++++++++++++++++++++++++++
 xen/include/asm-x86/nmi.h |  8 +++++++-
 xen/include/xen/softirq.h |  5 ++++-
 3 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index bc5b8f8ea3..f433fe5acb 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -1799,6 +1799,42 @@ void unset_nmi_callback(void)
     nmi_callback = dummy_nmi_callback;
 }
 
+static DEFINE_PER_CPU(void (*)(void *), nmi_cont_func);
+static DEFINE_PER_CPU(void *, nmi_cont_par);
+
+static void nmi_cont_softirq(void)
+{
+    unsigned int cpu = smp_processor_id();
+    void (*func)(void *par) = per_cpu(nmi_cont_func, cpu);
+    void *par = per_cpu(nmi_cont_par, cpu);
+
+    /* Reads must be done before following write (local cpu ordering only). */
+    barrier();
+
+    per_cpu(nmi_cont_func, cpu) = NULL;
+
+    if ( func )
+        func(par);
+}
+
+int set_nmi_continuation(void (*func)(void *par), void *par)
+{
+    unsigned int cpu = smp_processor_id();
+
+    if ( per_cpu(nmi_cont_func, cpu) )
+    {
+        printk("Trying to set NMI continuation while still one active!\n");
+        return -EBUSY;
+    }
+
+    per_cpu(nmi_cont_func, cpu) = func;
+    per_cpu(nmi_cont_par, cpu) = par;
+
+    raise_softirq(NMI_CONT_SOFTIRQ);
+
+    return 0;
+}
+
 void do_device_not_available(struct cpu_user_regs *regs)
 {
 #ifdef CONFIG_PV
@@ -2132,6 +2168,7 @@ void __init trap_init(void)
 
     cpu_init();
 
+    open_softirq(NMI_CONT_SOFTIRQ, nmi_cont_softirq);
     open_softirq(PCI_SERR_SOFTIRQ, pci_serr_softirq);
 }
 
diff --git a/xen/include/asm-x86/nmi.h b/xen/include/asm-x86/nmi.h
index a288f02a50..da40fb6599 100644
--- a/xen/include/asm-x86/nmi.h
+++ b/xen/include/asm-x86/nmi.h
@@ -33,5 +33,11 @@ nmi_callback_t *set_nmi_callback(nmi_callback_t *callback);
 void unset_nmi_callback(void);
 
 DECLARE_PER_CPU(unsigned int, nmi_count);
- 
+
+/**
+ * set_nmi_continuation
+ *
+ * Schedule a function to be started in softirq context after NMI handling.
+ */
+int set_nmi_continuation(void (*func)(void *par), void *par);
 #endif /* ASM_NMI_H */
diff --git a/xen/include/xen/softirq.h b/xen/include/xen/softirq.h
index 1f6c4783da..14c744bbf7 100644
--- a/xen/include/xen/softirq.h
+++ b/xen/include/xen/softirq.h
@@ -3,7 +3,10 @@
 
 /* Low-latency softirqs come first in the following list. */
 enum {
-    TIMER_SOFTIRQ = 0,
+#ifdef CONFIG_X86
+    NMI_CONT_SOFTIRQ,
+#endif
+    TIMER_SOFTIRQ,
     RCU_SOFTIRQ,
     SCHED_SLAVE_SOFTIRQ,
     SCHEDULE_SOFTIRQ,
-- 
2.26.2