[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-changelog] [xen master] x86: show remote CPU state upon fatal NMI or unknown MCE



commit 266b6539bf5671d8bad10900c79eb02d2cd49d15
Author:     Jan Beulich <jbeulich@xxxxxxxx>
AuthorDate: Tue Jun 21 12:06:13 2016 +0200
Commit:     Jan Beulich <jbeulich@xxxxxxxx>
CommitDate: Tue Jun 21 12:06:13 2016 +0200

    x86: show remote CPU state upon fatal NMI or unknown MCE
    
    Quite frequently the watchdog would hit an innocent CPU, e.g. one
    trying to acquire a spin lock a remote CPU holds for extended periods
    of time, or a random CPU in TSC calbration rendezvous. In such cases
    the register and stack dump for that CPU doesn't really help in the
    analysis of the problem.
    
    To keep things reasonable on large systems, only log CS:RIP by default.
    This can be overridden via a new command line option such that full
    register/stack state would get logged.
    
    Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>
    Reviewed-by: Andrew Cooper <andrew.cooper3@xxxxxxxxxx>
---
 docs/misc/xen-command-line.markdown |  8 +++++
 xen/arch/x86/cpu/mcheck/mce.c       |  2 +-
 xen/arch/x86/nmi.c                  |  2 +-
 xen/arch/x86/traps.c                | 59 +++++++++++++++++++++++++++++++++----
 xen/arch/x86/x86_64/entry.S         |  1 +
 xen/include/asm-x86/processor.h     |  2 +-
 6 files changed, 65 insertions(+), 9 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index fed732c..7a271c0 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -198,6 +198,14 @@ Permit Xen to use Address Space Identifiers.  This is an 
optimisation which
 tags the TLB entries with an ID per vcpu.  This allows for guest TLB flushes
 to be performed without the overhead of a complete TLB flush.
 
+### async-show-all
+> `= <boolean>`
+
+> Default: `false`
+
+Forces all CPUs' full state to be logged upon certain fatal asynchronous
+exceptions (watchdog NMIs and unexpected MCEs).
+
 ### ats
 > `= <boolean>`
 
diff --git a/xen/arch/x86/cpu/mcheck/mce.c b/xen/arch/x86/cpu/mcheck/mce.c
index 0244553..edcbe48 100644
--- a/xen/arch/x86/cpu/mcheck/mce.c
+++ b/xen/arch/x86/cpu/mcheck/mce.c
@@ -77,7 +77,7 @@ static void unexpected_machine_check(const struct 
cpu_user_regs *regs)
 {
     console_force_unlock();
     printk("Unexpected Machine Check Exception\n");
-    fatal_trap(regs);
+    fatal_trap(regs, 1);
 }
 
 
diff --git a/xen/arch/x86/nmi.c b/xen/arch/x86/nmi.c
index 426c24e..efc21b2 100644
--- a/xen/arch/x86/nmi.c
+++ b/xen/arch/x86/nmi.c
@@ -488,7 +488,7 @@ bool_t nmi_watchdog_tick(const struct cpu_user_regs *regs)
             console_force_unlock();
             printk("Watchdog timer detects that CPU%d is stuck!\n",
                    smp_processor_id());
-            fatal_trap(regs);
+            fatal_trap(regs, 1);
         }
     } 
     else 
diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index f604b89..767d0b0 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -525,6 +525,25 @@ void vcpu_show_execution_state(struct vcpu *v)
     vcpu_unpause(v);
 }
 
+static cpumask_t show_state_mask;
+static bool_t opt_show_all;
+boolean_param("async-show-all", opt_show_all);
+
+static int nmi_show_execution_state(const struct cpu_user_regs *regs, int cpu)
+{
+    if ( !cpumask_test_cpu(cpu, &show_state_mask) )
+        return 0;
+
+    if ( opt_show_all )
+        show_execution_state(regs);
+    else
+        printk(XENLOG_ERR "CPU%d @ %04x:%08lx (%pS)\n", cpu, regs->cs, 
regs->rip,
+               guest_mode(regs) ? _p(regs->rip) : NULL);
+    cpumask_clear_cpu(cpu, &show_state_mask);
+
+    return 1;
+}
+
 static const char *trapstr(unsigned int trapnr)
 {
     static const char * const strings[] = {
@@ -544,7 +563,7 @@ static const char *trapstr(unsigned int trapnr)
  * are disabled). In such situations we can't do much that is safe. We try to
  * print out some tracing and then we just spin.
  */
-void fatal_trap(const struct cpu_user_regs *regs)
+void fatal_trap(const struct cpu_user_regs *regs, bool_t show_remote)
 {
     static DEFINE_PER_CPU(char, depth);
     unsigned int trapnr = regs->entry_vector;
@@ -570,6 +589,34 @@ void fatal_trap(const struct cpu_user_regs *regs)
             printk("Faulting linear address: %p\n", _p(cr2));
             show_page_walk(cr2);
         }
+
+        if ( show_remote )
+        {
+            unsigned int msecs, pending;
+
+            cpumask_andnot(&show_state_mask, &cpu_online_map,
+                           cpumask_of(smp_processor_id()));
+            set_nmi_callback(nmi_show_execution_state);
+            /* Ensure new callback is set before sending out the NMI. */
+            smp_wmb();
+            smp_send_nmi_allbutself();
+
+            /* Wait at most 10ms for some other CPU to respond. */
+            msecs = 10;
+            pending = cpumask_weight(&show_state_mask);
+            while ( pending && msecs-- )
+            {
+                unsigned int left;
+
+                mdelay(1);
+                left = cpumask_weight(&show_state_mask);
+                if ( left < pending )
+                {
+                    pending = left;
+                    msecs = 10;
+                }
+            }
+        }
     }
 
     panic("FATAL TRAP: vector = %d (%s)\n"
@@ -1711,7 +1758,7 @@ void do_page_fault(struct cpu_user_regs *regs)
         {
             console_start_sync();
             printk("Xen SM%cP violation\n", (pf_type == smep_fault) ? 'E' : 
'A');
-            fatal_trap(regs);
+            fatal_trap(regs, 0);
         }
 
         if ( pf_type != real_fault )
@@ -1782,7 +1829,7 @@ void __init do_early_page_fault(struct cpu_user_regs 
*regs)
         console_start_sync();
         printk("Early fatal page fault at %04x:%p (cr2=%p, ec=%04x)\n",
                regs->cs, _p(regs->eip), _p(cr2), regs->error_code);
-        fatal_trap(regs);
+        fatal_trap(regs, 0);
     }
 }
 
@@ -3598,7 +3645,7 @@ static void pci_serr_error(const struct cpu_user_regs 
*regs)
     default:  /* 'fatal' */
         console_force_unlock();
         printk("\n\nNMI - PCI system error (SERR)\n");
-        fatal_trap(regs);
+        fatal_trap(regs, 0);
     }
 }
 
@@ -3613,7 +3660,7 @@ static void io_check_error(const struct cpu_user_regs 
*regs)
     default:  /* 'fatal' */
         console_force_unlock();
         printk("\n\nNMI - I/O ERROR\n");
-        fatal_trap(regs);
+        fatal_trap(regs, 0);
     }
 
     outb((inb(0x61) & 0x0f) | 0x08, 0x61); /* clear-and-disable IOCK */
@@ -3633,7 +3680,7 @@ static void unknown_nmi_error(const struct cpu_user_regs 
*regs, unsigned char re
         console_force_unlock();
         printk("Uhhuh. NMI received for unknown reason %02x.\n", reason);
         printk("Do you have a strange power saving mode enabled?\n");
-        fatal_trap(regs);
+        fatal_trap(regs, 0);
     }
 }
 
diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S
index 2947c17..ad8c64c 100644
--- a/xen/arch/x86/x86_64/entry.S
+++ b/xen/arch/x86/x86_64/entry.S
@@ -558,6 +558,7 @@ exception_with_ints_disabled:
 
 /* No special register assumptions. */
 FATAL_exception_with_ints_disabled:
+        xorl  %esi,%esi
         movq  %rsp,%rdi
         call  fatal_trap
         BUG   /* fatal_trap() shouldn't return. */
diff --git a/xen/include/asm-x86/processor.h b/xen/include/asm-x86/processor.h
index 318c92c..c69d7a4 100644
--- a/xen/include/asm-x86/processor.h
+++ b/xen/include/asm-x86/processor.h
@@ -541,7 +541,7 @@ void show_registers(const struct cpu_user_regs *regs);
 void show_execution_state(const struct cpu_user_regs *regs);
 #define dump_execution_state() run_in_exception_handler(show_execution_state)
 void show_page_walk(unsigned long addr);
-void noreturn fatal_trap(const struct cpu_user_regs *regs);
+void noreturn fatal_trap(const struct cpu_user_regs *regs, bool_t show_remote);
 
 void compat_show_guest_stack(struct vcpu *v,
                              const struct cpu_user_regs *regs, int lines);
--
generated by git-patchbot for /home/xen/git/xen.git#master

_______________________________________________
Xen-changelog mailing list
Xen-changelog@xxxxxxxxxxxxx
http://lists.xensource.com/xen-changelog

 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.