[Xen-devel] [PATCH 5 of 9] Fine-grained concurrency control structure for the p2m

[Andres Lagar-Cavilla <andres@xxxxxxxxxxxxxxxx>]

 xen/arch/x86/mm/hap/private.h |    1 +
 xen/arch/x86/mm/mm-locks.h    |   20 +-
 xen/arch/x86/mm/p2m-ept.c     |    1 +
 xen/arch/x86/mm/p2m-lock.h    |  613 ++++++++++++++++++++++++++++++++++++++++++
 xen/arch/x86/mm/p2m-pod.c     |    1 +
 xen/arch/x86/mm/p2m-pt.c      |    1 +
 xen/arch/x86/mm/p2m.c         |   24 +-
 xen/include/asm-x86/p2m.h     |    3 +-
 8 files changed, 652 insertions(+), 12 deletions(-)


Introduce a fine-grained concurrency control structure for the p2m. This
allows for locking 2M-aligned chunks of the p2m at a time, exclusively.
Recursive locking is allowed. Global locking of the whole p2m is also
allowed for certain operations. Simple deadlock detection heuristics are
put in place.

Note the patch creates backwards-compatible shortcuts that will lock the
p2m globally. So it should remain functionally identical to what is currently
in place.

Signed-off-by: Andres Lagar-Cavilla <andres@xxxxxxxxxxxxxxxx>

diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/hap/private.h
--- a/xen/arch/x86/mm/hap/private.h
+++ b/xen/arch/x86/mm/hap/private.h
@@ -21,6 +21,7 @@
 #define __HAP_PRIVATE_H__
 
 #include "../mm-locks.h"
+#include "../p2m-lock.h"
 
 /********************************************/
 /*          GUEST TRANSLATION FUNCS         */
diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/mm-locks.h
--- a/xen/arch/x86/mm/mm-locks.h
+++ b/xen/arch/x86/mm/mm-locks.h
@@ -146,14 +146,22 @@ declare_mm_lock(nestedp2m)
 
 /* P2M lock (per-p2m-table)
  * 
- * This protects all updates to the p2m table.  Updates are expected to
- * be safe against concurrent reads, which do *not* require the lock. */
+ * This protects all updates to the p2m table.
+ * 
+ * In 64 bit mode we disable this because the lock becomes fine-grained,
+ * and several code paths cause inversion/deadlock:
+ *   -- PoD sweeps
+ *   -- mem_sharing_unshare_page
+ *   -- generally widespread recursive locking, which we don't support
+ *      (yet, I guess) on an "external" mm lock. */
 
+#ifndef __x86_64__
 declare_mm_lock(p2m)
-#define p2m_lock(p)           mm_lock(p2m, &(p)->lock)
-#define p2m_lock_recursive(p) mm_lock_recursive(p2m, &(p)->lock)
-#define p2m_unlock(p)         mm_unlock(&(p)->lock)
-#define p2m_locked_by_me(p)   mm_locked_by_me(&(p)->lock)
+#define _p2m_lock(p)           mm_lock(p2m, &(p)->lock)
+#define _p2m_lock_recursive(p) mm_lock_recursive(p2m, &(p)->lock)
+#define _p2m_unlock(p)         mm_unlock(&(p)->lock)
+#define _p2m_locked_by_me(p)   mm_locked_by_me(&(p)->lock)
+#endif /* __x86_64__ */
 
 /* PoD lock (per-p2m-table)
  * 
diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/p2m-ept.c
--- a/xen/arch/x86/mm/p2m-ept.c
+++ b/xen/arch/x86/mm/p2m-ept.c
@@ -33,6 +33,7 @@
 #include <xen/softirq.h>
 
 #include "mm-locks.h"
+#include "p2m-lock.h"
 
 #define atomic_read_ept_entry(__pepte)                              \
     ( (ept_entry_t) { .epte = atomic_read64(&(__pepte)->epte) } )
diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/p2m-lock.h
--- /dev/null
+++ b/xen/arch/x86/mm/p2m-lock.h
@@ -0,0 +1,613 @@
+/******************************************************************************
+ * arch/x86/mm/p2m-lock.h
+ *
+ * Fine-grained locking of the p2m. Allow for concurrent updates to different
+ * regions of the p2m. Serially synchronize updates and lookups. Mutex 
+ * access on p2m entries while a CPU is using them.
+ *
+ * Copyright (c) 2011 Andres Lagar-Cavilla, GridCentric Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#ifndef _XEN_P2M_LOCK_H
+#define _XEN_P2M_LOCK_H
+
+#include <xen/config.h>
+#include <xen/lib.h>
+/* See comment about space consideration for spinlocks below */
+#define NDEBUG
+#undef LOCK_PROFILE
+#include <xen/spinlock.h>
+#include <asm/atomic.h>
+#include <xen/xmalloc.h>
+#include <xen/paging.h>
+#include <asm/page.h>
+#include <asm/p2m.h>
+#include "mm-locks.h"
+
+/* Rationale:
+ *
+ * The motivating scenario is one in which you have at least three CPUs 
+ * operating on likely disjoint regions of the p2m: a paging utility, a sharing
+ * utility, and the domU vcpu. With yet another p2m-heavy utility (mem 
+ * access?), and/or a migrate/remus utility, the number of CPUs operating
+ * on disjoint regions increases. Not to mention multi-vcpu domUs.
+ *
+ * Therefore, p2m concurrency control is achieved through a hierarchical 
+ * tree of locks, to allow all these CPUs to work without bothering each other.
+ * (Without disallowing any other cases such as single-vcpu domU)
+ *
+ * Leafs in the tree of locks are represented by spinlocks.
+ *
+ * Inner nodes (or uppper levels), are represented by a spinlock and a count.
+ * The count indicates how many CPUs are locking a node beneath. 
+ *
+ * A cpu holds a leaf by grabbing the spinlock, and not letting go of it. On 
its 
+ * way to the leaf, for each inner node, it grabs the spinlock, increases the 
+ * count, and releases the spinlock.
+ *
+ * Leaf levels are recursive, the same CPU can lock them again.
+ *
+ * A cpu holds an inner node in exclusive mode by busy-waiting until the count 
+ * is zero, grabbing the spinlock, and not letting go of it.
+ *
+ * Unlocks work by releasing the current spinlock, and working your way up:
+ * grab spinlock, decrease count, release.
+ *
+ * No locker can be preempted. For that reason, there are no atomic promotions:
+ * you would end up with promoters deadlocking on their way up the tree.
+ *
+ * Today, there are effectively two levels: the global lock (an inner node), 
and
+ * 2M locks, leaf locks for contiguous, aligned, 2M extents (akin to 
superpages).
+ *
+ * The global level can be held exclusively for big hammer operations such as
+ * log dirty (re)set.
+ *
+ * For non-global locking, the global lock is grabbed non-exclusively. At each 
+ * 1G boundary we allocate, if we hadn't before, the corresponding set of 512 
+ * 2M locks. Allocation of 2M locks is itself protected by a regular
+ * spinlock (this is rare enough). Allocation functions on-demand because
+ * we can't really know a priori the "total" size of the p2m.
+ *
+ * It is expected that every query or modification to the p2m will lock the 
+ * appropriate range. Leafs are recurisve for this reason: commonly you query 
a 
+ * range and then you modify it.
+ *
+ * Conversely, all callers of queries and modifications, once done, need to 
undo
+ * their locking.
+ * 
+ * Because we mimic the page table structure of a 512-radix tree, we run into 
+ * space considerations with the spinlocks in this tree. So we need to be 
careful
+ * about space.
+ *
+ * For 32bit code, we currently bail out and default to one big lock. Sorry 
Atom :(
+ *
+ * Also note that the p2m tree of locks is included in the ordering constraints
+ * enforced by mm-locks.h. It is treated as an "external" lock in that code.
+ *
+ */
+
+#define P2M_ORDER_GLOBAL    ~0U
+
+/* The 32 bit case serves as a concise summary of the external API */
+#ifndef __x86_64__
+/* For 32 bits we default to one big lock */
+typedef struct __p2m_lock {
+    mm_lock_t lock;
+} p2m_lock_t;
+
+static inline int p2m_lock_init(struct p2m_domain *p2m)
+{
+    p2m_lock_t *p2ml = xmalloc(p2m_lock_t);
+    if ( !p2ml )
+        return -ENOMEM;
+    mm_lock_init(&p2ml->lock);
+    p2m->lock = p2ml;
+    return 0;
+}
+
+static inline void get_p2m(struct p2m_domain *p2m, unsigned long gfn, unsigned 
int order)
+{
+    _p2m_lock(p2m->lock);
+}
+
+static inline void put_p2m(struct p2m_domain *p2m, unsigned long gfn, unsigned 
int order)
+{
+    _p2m_unlock(p2m->lock);
+}
+
+static inline void p2m_lock_destroy(struct p2m_domain *p2m)
+{
+    xfree(p2m->lock);
+    p2m->lock = NULL;
+}
+
+/* Backwards compatiblity */
+#define p2m_lock(p)             _p2m_lock((p)->lock)
+#define p2m_lock_recursive(p)   _p2m_lock_recursive((p)->lock)
+#define p2m_locked_by_me(p)     _p2m_locked_by_me((p)->lock)
+#define p2m_unlock(p)           _p2m_unlock((p)->lock)
+
+#else /* __x86_64__ */
+/* If we were to have inner locks (say 1G locks, then the space considerations
+ * outlined below for leaf locks would also apply here. */
+typedef struct p2m_inner_lock {
+    spinlock_t lock;
+    atomic_t   count;
+} p2m_inner_lock_t;
+
+static inline void init_p2m_inner_lock(p2m_inner_lock_t *inner)
+{
+    spin_lock_init(&inner->lock);
+    _atomic_set(inner->count, 0);
+}
+
+/* We cannot risk reusing the code in common/spinlock.c, because it may
+ * have been compiled with LOCK_DEBUG or LOCK_PROFILE. This is unfortunate. */
+static inline void lock_p2m_inner(p2m_inner_lock_t *inner)
+{
+    spin_lock(&inner->lock);
+}
+
+static inline void unlock_p2m_inner(p2m_inner_lock_t *inner)
+{
+    spin_unlock(&inner->lock);
+}
+
+static inline void get_p2m_inner(p2m_inner_lock_t *inner)
+{
+    lock_p2m_inner(inner);
+    atomic_inc(&inner->count);
+    unlock_p2m_inner(inner);
+}
+
+static inline void put_p2m_inner(p2m_inner_lock_t *inner)
+{
+    lock_p2m_inner(inner);
+    atomic_dec(&inner->count);
+    unlock_p2m_inner(inner);
+}
+
+/* XXX Consider starvation here */
+static inline void get_p2m_inner_exclusive(p2m_inner_lock_t *inner)
+{
+    int count;
+retry:
+    while (1)
+    {
+        mb();
+        count = atomic_read(&inner->count);
+        if ( count == 0 )
+            break;
+        cpu_relax();
+    }
+
+    spin_lock(&inner->lock);
+    mb();
+    count = atomic_read(&inner->count);
+    if ( count )
+    {
+        spin_unlock(&inner->lock);
+        goto retry;
+    }
+    /* We leave holding the spinlock */
+}
+
+static inline void put_p2m_inner_exclusive(p2m_inner_lock_t *inner)
+{
+    spin_unlock(&inner->lock);
+}
+
+/* Because we operate under page-table sizing constraints, we need to be 
+ * extremely conscious about the space we're taking up. So we become somewhat 
+ * re-inventers of the wheel, and we disable many things. */
+typedef struct p2m_leaf_lock {
+    raw_spinlock_t raw;
+    u16 recurse_cpu:12;
+    u16 recurse_cnt:4;
+/* Padding to confine each inner lock to its own word */
+#define LEAF_PAD   4
+    uint8_t             pad[LEAF_PAD];
+} __attribute__((packed)) p2m_leaf_lock_t;
+
+/* BUILD_BUG_ON(sizeof(p2m_leaf_lock_t) != sizeof(unsigned long)); */
+
+static inline void init_p2m_leaf_lock(p2m_leaf_lock_t *lock)
+{
+    *lock = (p2m_leaf_lock_t) { _RAW_SPIN_LOCK_UNLOCKED, 0xfffu, 0, { } };
+}
+
+static inline int __p2m_spin_trylock_recursive(p2m_leaf_lock_t *lock)
+{
+    int cpu = smp_processor_id();
+
+    if ( likely(lock->recurse_cpu != cpu) )
+    {
+        if ( !_raw_spin_trylock(&lock->raw) )
+            return 0;
+        preempt_disable();
+        lock->recurse_cpu = cpu;
+    }
+
+    lock->recurse_cnt++;
+    return 1;
+}
+
+static inline void lock_p2m_leaf(p2m_leaf_lock_t *lock)
+{
+    while ( !__p2m_spin_trylock_recursive(lock) )
+        cpu_relax();
+}
+
+static inline void unlock_p2m_leaf(p2m_leaf_lock_t *lock)
+{
+    if ( likely(--lock->recurse_cnt == 0) )
+    {
+        lock->recurse_cpu = 0xfffu;
+        preempt_enable();
+        _raw_spin_unlock(&lock->raw);
+    }
+}
+
+/* Deadlock book-keeping, see below */
+#define MAX_LOCK_DEPTH  16
+
+/* The lock structure */
+typedef struct __p2m_lock 
+{
+    /* To enforce ordering in mm-locks */
+    int unlock_level;
+    /* To protect on-demand allocation of locks 
+     * (yeah you heard that right) */
+    spinlock_t alloc_lock;
+    /* Global lock */
+    p2m_inner_lock_t global;
+    /* 2M locks. Allocate on demand: fun */
+    p2m_leaf_lock_t  **locks_2m;
+    /* Book-keeping for deadlock detection. Could be a per-cpu. */
+    unsigned long deadlock_guard[NR_CPUS][MAX_LOCK_DEPTH + 1];
+    uint8_t lock_depth[NR_CPUS];
+    /* Is anybody holding this exclusively */
+    unsigned int exclusive_holder;
+    /* Order of pages allocates for first level of locks_2m */
+    uint8_t order;
+} p2m_lock_t;
+
+#define EXCLUSIVE_CPU_NULL  ~0U
+
+/* Some deadlock book-keeping. Say CPU A holds a lock on range A, CPU B holds 
a 
+ * lock on range B. Now, CPU A wants to lock range B and vice-versa. Deadlock.
+ * We detect this by remembering the start of the current locked range.
+ * We keep a fairly small stack of guards (8), because we don't anticipate
+ * a great deal of recursive locking because (a) recursive locking is rare 
+ * (b) it is evil (c) only PoD seems to do it (is PoD therefore evil?) */
+
+#define DEADLOCK_NULL   ~0UL
+
+#define CURRENT_GUARD(l)    ((l)->deadlock_guard[current->processor] \
+                                [(l)->lock_depth[current->processor]])
+
+#define DEADLOCK_CHECK(cond, action, _f, _a...) \
+do {                                            \
+    if ( (cond) )                               \
+    {                                           \
+        printk(_f, ##_a);                       \
+        action;                                 \
+    }                                           \
+} while(0)
+
+static inline void push_guard(p2m_lock_t *p2ml, unsigned long gfn)
+{
+    int cpu = current->processor;
+
+    DEADLOCK_CHECK(((p2ml->lock_depth[cpu] + 1) > MAX_LOCK_DEPTH), 
+                    BUG(), "CPU %u exceeded deadlock depth\n", cpu);
+
+    p2ml->lock_depth[cpu]++;
+    p2ml->deadlock_guard[cpu][p2ml->lock_depth[cpu]] = gfn;
+}
+
+static inline void pop_guard(p2m_lock_t *p2ml)
+{
+    int cpu = current->processor;
+
+    DEADLOCK_CHECK((!p2ml->lock_depth[cpu] == 0), BUG(), 
+                    "CPU %u underflow deadlock depth\n", cpu);
+
+    p2ml->lock_depth[cpu]--;
+}
+
+static inline int p2m_lock_init(struct p2m_domain *p2m)
+{
+    unsigned int i;
+    p2m_lock_t *p2ml;
+
+    p2ml = xmalloc(p2m_lock_t);
+    if ( !p2ml ) 
+        return -ENOMEM;
+
+    memset(p2ml, 0, sizeof(p2m_lock_t));
+
+    spin_lock_init(&p2ml->alloc_lock);
+    init_p2m_inner_lock(&p2ml->global);
+
+    p2ml->locks_2m = alloc_xenheap_page();
+    if ( !p2ml->locks_2m )
+    {
+        xfree(p2ml);
+        return -ENOMEM;
+    }
+    memset(p2ml->locks_2m, 0, PAGE_SIZE);
+
+    for (i = 0; i < NR_CPUS; i++)
+        p2ml->deadlock_guard[i][0] = DEADLOCK_NULL;
+
+    p2ml->exclusive_holder = EXCLUSIVE_CPU_NULL;
+
+    p2m->lock = p2ml;
+    return 0;    
+}
+
+/* Conversion macros for aligned boundaries */
+#define gfn_to_superpage(g, o)      (((g) & (~((1 << (o)) - 1))) >> (o))
+#define gfn_to_1g_sp(gfn)           gfn_to_superpage(gfn, PAGE_ORDER_1G)
+#define gfn_to_2m_sp(gfn)           gfn_to_superpage(gfn, PAGE_ORDER_2M)
+#define gfn_1g_to_2m(gfn_1g)        ((gfn_1g) << (PAGE_ORDER_1G - 
PAGE_ORDER_2M))
+#define gfn_1g_to_last_2m(gfn_1g)   (gfn_1g_to_2m(gfn_1g) + \
+                                        ((1 << (PAGE_ORDER_1G - 
PAGE_ORDER_2M)) - 1))
+#define gfn_1g_to_4k(gfn_1g)        ((gfn_1g) << PAGE_ORDER_1G)
+#define gfn_1g_to_last_4k(gfn_1g)   (gfn_1g_to_4k(gfn_1g) + ((1 << 
PAGE_ORDER_1G) - 1))
+
+/* Global lock accessors. Global lock is our only "inner" node. */
+#define p2m_exclusive_locked_by_me(l)    \
+     ((l)->lock->exclusive_holder == current->processor)
+
+static inline void get_p2m_global_exclusive(struct p2m_domain *p2m)
+{
+    p2m_lock_t *p2ml = p2m->lock;
+    DEADLOCK_CHECK((CURRENT_GUARD(p2ml) != DEADLOCK_NULL), BUG(),
+                    "P2M DEADLOCK: cpu %u prev range start %lx trying 
global\n",
+                    (unsigned) current->processor, CURRENT_GUARD(p2ml)); 
+
+    get_p2m_inner_exclusive(&p2ml->global);
+    p2ml->exclusive_holder = current->processor;
+}
+
+static inline void put_p2m_global_exclusive(struct p2m_domain *p2m)
+{
+    p2m_lock_t *p2ml = p2m->lock;
+    p2ml->exclusive_holder = EXCLUSIVE_CPU_NULL;
+    put_p2m_inner_exclusive(&p2ml->global);
+}
+
+/* Not to be confused with shortcut for external use */
+static inline void __get_p2m_global(struct p2m_domain *p2m)
+{
+    get_p2m_inner(&p2m->lock->global);
+}
+
+/* Not to be confused with shortcut for external use */
+static inline void __put_p2m_global(struct p2m_domain *p2m)
+{
+    put_p2m_inner(&p2m->lock->global);
+}
+
+/* 2M lock accessors */
+static inline p2m_leaf_lock_t *__get_2m_lock(p2m_lock_t *p2ml,
+                            unsigned long gfn_1g, unsigned long gfn_2m)
+{
+    p2m_leaf_lock_t *lock_2m_l1;
+    BUG_ON(gfn_1g >= (1 << PAGETABLE_ORDER));
+    BUG_ON(gfn_2m >= (1 << PAGETABLE_ORDER));
+    lock_2m_l1 = p2ml->locks_2m[gfn_1g];
+    BUG_ON(lock_2m_l1 == NULL);
+    return (lock_2m_l1 + gfn_2m);
+}
+
+static inline void get_p2m_2m(struct p2m_domain *p2m, unsigned long gfn_1g,
+                                unsigned long gfn_2m)
+{
+    lock_p2m_leaf(__get_2m_lock(p2m->lock, gfn_1g, gfn_2m));
+}
+
+static inline void put_p2m_2m(struct p2m_domain *p2m, unsigned long gfn_1g,
+                                unsigned long gfn_2m)
+{
+    unlock_p2m_leaf(__get_2m_lock(p2m->lock, gfn_1g, gfn_2m));
+}
+
+/* Allocate 2M locks we may not have allocated yet for this 1G superpage */
+static inline int alloc_locks_2m(struct p2m_domain *p2m, unsigned long gfn_1g)
+{
+    p2m_lock_t *p2ml = p2m->lock;
+
+    /* With a single page for l1, we cover a gfn space of 512GB (39 bits)
+     * Given that current x86_64 processors physically address 40 bits,
+     * we're in no immediate danger of overflowing this table for a domU.
+     * If necessary, the l1 itself can grow subject to proper locking 
+     * on the p2ml->alloc_lock */
+
+    /* Quick test for common case */
+    if ( likely(p2ml->locks_2m[gfn_1g] != NULL) ) 
+        return 0;
+
+    spin_lock(&(p2ml->alloc_lock));
+
+    if ( likely(p2ml->locks_2m[gfn_1g] == NULL) )
+    {
+        unsigned long j;
+        p2m_leaf_lock_t *p = alloc_xenheap_page();
+        if ( !p ) 
+        {
+            spin_unlock(&(p2ml->alloc_lock));
+            return -ENOMEM;
+        }
+
+        for (j = 0; j < (1 << PAGETABLE_ORDER); j++)
+            init_p2m_leaf_lock(&p[j]);
+
+        p2ml->locks_2m[gfn_1g] = p;
+    }
+
+    spin_unlock(&(p2ml->alloc_lock));
+    return 0;
+}
+
+static inline unsigned long __get_last_gfn(unsigned long gfn, unsigned int 
order)
+{
+    /* Underflow */
+    unsigned long last_gfn = gfn + (1 << order) - 1;
+    BUG_ON(last_gfn < gfn);
+    return last_gfn;
+}
+
+static inline void get_p2m(struct p2m_domain *p2m, unsigned long gfn, unsigned 
int order)
+{
+    unsigned long last_gfn, first_1g, last_1g, first_2m, last_2m, i, j;
+    p2m_lock_t *p2ml = p2m->lock;
+
+    /* Holders of the p2m in exclusive mode can lock sub ranges. We make that 
a no-op.
+     * however, locking exclusively again is considered rude and tasteless. */
+    if ( (p2m_exclusive_locked_by_me(p2m)) && (order != P2M_ORDER_GLOBAL) )
+        return;
+        
+    DEADLOCK_CHECK(((CURRENT_GUARD(p2ml) != DEADLOCK_NULL) &&
+                    (CURRENT_GUARD(p2ml) > gfn)), WARN(),
+                    "P2M DEADLOCK: cpu %d prev range start %lx new range start 
%lx",
+                    current->processor, CURRENT_GUARD(p2ml), gfn);
+
+    preempt_disable();
+
+    if ( order == P2M_ORDER_GLOBAL ) {
+        get_p2m_global_exclusive(p2m);
+        goto get_p2m_out;
+    } 
+
+    __get_p2m_global(p2m);
+    /* We're non-preemptible. We've disallowed global p2m locking. We
+     * will now (allocate and) lock all relevant 2M leafs */
+
+    last_gfn    = __get_last_gfn(gfn, order);
+    first_1g    = gfn_to_1g_sp(gfn);
+    last_1g     = gfn_to_1g_sp(last_gfn);
+
+    for (i = first_1g; i <= last_1g; i++) 
+    {
+        first_2m    = (gfn_1g_to_4k(i) > gfn) ? gfn_1g_to_2m(i) : 
gfn_to_2m_sp(gfn);
+        last_2m     = min(gfn_to_2m_sp(last_gfn), gfn_1g_to_last_2m(i));
+
+        if ( alloc_locks_2m(p2m, i) )
+        {
+            /* There really isn't much we can do at this point */
+            panic("Fine-grained p2m locking failed to alloc 2M locks"
+                  " for 1G page %lx, domain %hu\n", i, p2m->domain->domain_id);
+        }
+
+        for (j = first_2m; j <= last_2m; j++)
+        {
+            get_p2m_2m(p2m, i, j & ((1 << PAGETABLE_ORDER) - 1));
+        }
+    }
+
+get_p2m_out:
+    push_guard(p2ml, gfn);
+}
+
+/* Conversely to the get method, we unlock all leafs pro-actively here */
+static inline void put_p2m(struct p2m_domain *p2m, unsigned long gfn, unsigned 
int order)
+{
+    unsigned long last_gfn, first_1g, last_1g, first_2m, last_2m, i, j;
+    p2m_lock_t *p2ml = p2m->lock;
+
+    last_gfn = __get_last_gfn(gfn, order);
+
+    /* See comment about exclusive holders recursively locking sub-ranges in 
get_p2m */
+    if ( (p2m_exclusive_locked_by_me(p2m)) && (order != P2M_ORDER_GLOBAL) )
+        return;
+
+    if ( order == P2M_ORDER_GLOBAL )
+    {
+        put_p2m_global_exclusive(p2m);
+        goto cleanup;
+    }
+
+    first_1g    = gfn_to_1g_sp(gfn);
+    last_1g     = gfn_to_1g_sp(last_gfn);
+
+    for (i = first_1g; i <= last_1g; i++) 
+    {
+        first_2m    = (gfn_1g_to_4k(i) > gfn) ? gfn_1g_to_2m(i) : 
gfn_to_2m_sp(gfn);
+        last_2m     = min(gfn_to_2m_sp(last_gfn), gfn_1g_to_last_2m(i));
+
+        for (j = first_2m; j <= last_2m; j++)
+        {
+            put_p2m_2m(p2m, i, j & ((1 << PAGETABLE_ORDER) - 1));
+        }
+    }
+
+    __put_p2m_global(p2m);
+    
+cleanup:
+    pop_guard(p2ml);
+    preempt_enable();
+}
+
+static inline void p2m_lock_destroy(struct p2m_domain *p2m)
+{
+    unsigned int i;
+    p2m_lock_t *p2ml = p2m->lock;
+
+    get_p2m_global_exclusive(p2m);
+
+    for (i = 0; i < (1 << PAGETABLE_ORDER); i++)
+        if ( p2ml->locks_2m[i] )
+            free_xenheap_page(p2ml->locks_2m[i]);
+
+    free_xenheap_page(p2ml->locks_2m);
+
+    put_p2m_global_exclusive(p2m);
+
+    xfree(p2ml);
+    p2m->lock = NULL;
+}
+
+/* Backwards compatibility */
+#define p2m_lock(p)             get_p2m((p), 0, P2M_ORDER_GLOBAL)
+#define p2m_unlock(p)           put_p2m((p), 0, P2M_ORDER_GLOBAL)
+#define p2m_locked_by_me(p)     p2m_exclusive_locked_by_me((p))
+/* There is no backwards compatibility for this, unless we make the 
+ * global lock recursive */
+#define p2m_lock_recursive(p)   ((void)0) 
+
+#endif /* __x86_64__ */
+
+/* Commonly-used shortcus */
+#define get_p2m_global(p2m)     get_p2m((p2m), 0, P2M_ORDER_GLOBAL)
+#define put_p2m_global(p2m)     put_p2m((p2m), 0, P2M_ORDER_GLOBAL)
+
+#define get_p2m_gfn(p2m, gfn)   get_p2m((p2m), (gfn), 0)
+#define put_p2m_gfn(p2m, gfn)   put_p2m((p2m), (gfn), 0)
+
+#endif /* _XEN_P2M_LOCK_H */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/p2m-pod.c
--- a/xen/arch/x86/mm/p2m-pod.c
+++ b/xen/arch/x86/mm/p2m-pod.c
@@ -34,6 +34,7 @@
 #include <asm/hvm/svm/amd-iommu-proto.h>
 
 #include "mm-locks.h"
+#include "p2m-lock.h"
 
 /* Override macros from asm/page.h to make them work with mfn_t */
 #undef mfn_to_page
diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/p2m-pt.c
--- a/xen/arch/x86/mm/p2m-pt.c
+++ b/xen/arch/x86/mm/p2m-pt.c
@@ -39,6 +39,7 @@
 #include <asm/hvm/svm/amd-iommu-proto.h>
 
 #include "mm-locks.h"
+#include "p2m-lock.h"
 
 /* Override macros from asm/page.h to make them work with mfn_t */
 #undef mfn_to_page
diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/p2m.c
--- a/xen/arch/x86/mm/p2m.c
+++ b/xen/arch/x86/mm/p2m.c
@@ -38,6 +38,7 @@
 #include <asm/hvm/svm/amd-iommu-proto.h>
 
 #include "mm-locks.h"
+#include "p2m-lock.h"
 
 /* turn on/off 1GB host page table support for hap, default on */
 static bool_t __read_mostly opt_hap_1gb = 1;
@@ -69,9 +70,12 @@ boolean_param("hap_2mb", opt_hap_2mb);
 
 
 /* Init the datastructures for later use by the p2m code */
-static void p2m_initialise(struct domain *d, struct p2m_domain *p2m)
+static int p2m_initialise(struct domain *d, struct p2m_domain *p2m)
 {
-    mm_lock_init(&p2m->lock);
+    if (p2m_lock_init(p2m))
+    {
+        return -ENOMEM;
+    }
     mm_lock_init(&p2m->pod.lock);
     INIT_LIST_HEAD(&p2m->np2m_list);
     INIT_PAGE_LIST_HEAD(&p2m->pages);
@@ -89,7 +93,7 @@ static void p2m_initialise(struct domain
     else
         p2m_pt_init(p2m);
 
-    return;
+    return 0;
 }
 
 static int
@@ -103,7 +107,11 @@ p2m_init_nestedp2m(struct domain *d)
         d->arch.nested_p2m[i] = p2m = xzalloc(struct p2m_domain);
         if (p2m == NULL)
             return -ENOMEM;
-        p2m_initialise(d, p2m);
+        if (p2m_initialise(d, p2m))
+        {
+            xfree(p2m);
+            return -ENOMEM;
+        }
         p2m->write_p2m_entry = nestedp2m_write_p2m_entry;
         list_add(&p2m->np2m_list, &p2m_get_hostp2m(d)->np2m_list);
     }
@@ -118,7 +126,11 @@ int p2m_init(struct domain *d)
     p2m_get_hostp2m(d) = p2m = xzalloc(struct p2m_domain);
     if ( p2m == NULL )
         return -ENOMEM;
-    p2m_initialise(d, p2m);
+    if (p2m_initialise(d, p2m))
+    {
+        xfree(p2m);
+        return -ENOMEM;
+    }
 
     /* Must initialise nestedp2m unconditionally
      * since nestedhvm_enabled(d) returns false here.
@@ -331,6 +343,7 @@ static void p2m_teardown_nestedp2m(struc
     uint8_t i;
 
     for (i = 0; i < MAX_NESTEDP2M; i++) {
+        p2m_lock_destroy(d->arch.nested_p2m[i]);
         xfree(d->arch.nested_p2m[i]);
         d->arch.nested_p2m[i] = NULL;
     }
@@ -338,6 +351,7 @@ static void p2m_teardown_nestedp2m(struc
 
 void p2m_final_teardown(struct domain *d)
 {
+    p2m_lock_destroy(d->arch.p2m); 
     /* Iterate over all p2m tables per domain */
     xfree(d->arch.p2m);
     d->arch.p2m = NULL;
diff -r 981073d78f7f -r a23e1262b124 xen/include/asm-x86/p2m.h
--- a/xen/include/asm-x86/p2m.h
+++ b/xen/include/asm-x86/p2m.h
@@ -187,9 +187,10 @@ typedef enum {
 #define p2m_is_broken(_t)   (p2m_to_mask(_t) & P2M_BROKEN_TYPES)
 
 /* Per-p2m-table state */
+struct __p2m_lock;
 struct p2m_domain {
     /* Lock that protects updates to the p2m */
-    mm_lock_t          lock;
+    struct __p2m_lock *lock;
 
     /* Shadow translated domain: p2m mapping */
     pagetable_t        phys_table;

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel