|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v1 7/9] mm: Keep pages available for allocation while scrubbing
Instead of scrubbing pages while holding heap lock we can mark
buddy's head as being scrubbed and drop the lock temporarily.
If someone (most likely alloc_heap_pages()) tries to access
this chunk it will signal the scrubber to abort scrub by setting
head's PAGE_SCRUB_ABORT bit. The scrubber checks this bit after
processing each page and stops its work as soon as it sees it.
Signed-off-by: Boris Ostrovsky <boris.ostrovsky@xxxxxxxxxx>
---
xen/common/page_alloc.c | 78 +++++++++++++++++++++++++++++++++++++++++++--
xen/include/asm-arm/mm.h | 4 ++
xen/include/asm-x86/mm.h | 4 ++
3 files changed, 82 insertions(+), 4 deletions(-)
diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c
index e5e6e70..df28090 100644
--- a/xen/common/page_alloc.c
+++ b/xen/common/page_alloc.c
@@ -685,6 +685,18 @@ static void check_low_mem_virq(void)
}
}
+static void check_and_stop_scrub(struct page_info *head)
+{
+ if ( head->u.free.scrub_state & PAGE_SCRUBBING )
+ {
+ head->u.free.scrub_state |= PAGE_SCRUB_ABORT;
+ smp_mb();
+ spin_lock_kick();
+ while ( ACCESS_ONCE(head->u.free.scrub_state) & PAGE_SCRUB_ABORT )
+ cpu_relax();
+ }
+}
+
/* Allocate 2^@order contiguous pages. */
static struct page_info *alloc_heap_pages(
unsigned int zone_lo, unsigned int zone_hi,
@@ -771,10 +783,15 @@ static struct page_info *alloc_heap_pages(
{
if ( (pg = page_list_remove_head(&heap(node, zone, j))) )
{
- if ( (order == 0) || use_unscrubbed ||
- !test_bit(_PGC_need_scrub, &pg->count_info) )
+ if ( !test_bit(_PGC_need_scrub, &pg[0].count_info) )
goto found;
+ if ( (order == 0) || use_unscrubbed )
+ {
+ check_and_stop_scrub(pg);
+ goto found;
+ }
+
page_list_add_tail(pg, &heap(node, zone, j));
}
}
@@ -911,6 +928,8 @@ static int reserve_offlined_page(struct page_info *head)
cur_head = head;
+ check_and_stop_scrub(head);
+
page_list_del(head, &heap(node, zone, head_order));
while ( cur_head < (head + (1 << head_order)) )
@@ -993,6 +1012,9 @@ static bool_t can_merge(struct page_info *buddy, unsigned
int node,
!!test_bit(_PGC_need_scrub, &buddy->count_info) )
return 0;
+ if ( buddy->u.free.scrub_state & PAGE_SCRUBBING )
+ return 0;
+
return 1;
}
@@ -1048,12 +1070,34 @@ static void merge_chunks(struct page_info *pg, unsigned
int node,
}
#define SCRUB_CHUNK_ORDER 8
+
+struct scrub_wait_state {
+ struct page_info *pg;
+ bool_t drop;
+};
+
+static void scrub_continue(void *data)
+{
+ struct scrub_wait_state *st = (struct scrub_wait_state *)data;
+
+ if ( st->drop )
+ return;
+
+ if ( st->pg->u.free.scrub_state & PAGE_SCRUB_ABORT )
+ {
+ /* There is a waiter for this chunk. Release it. */
+ st->drop = true;
+ st->pg->u.free.scrub_state = 0;
+ }
+}
+
bool_t scrub_free_pages()
{
struct page_info *pg;
unsigned int i, zone;
unsigned int num_scrubbed, scrub_order, start, end;
bool_t preempt, is_frag;
+ struct scrub_wait_state st;
int order, cpu = smp_processor_id();
nodeid_t node = cpu_to_node(cpu), local_node;
static nodemask_t node_scrubbing;
@@ -1092,7 +1136,10 @@ bool_t scrub_free_pages()
if ( !test_bit(_PGC_need_scrub, &pg->count_info) )
break;
- page_list_del(pg, &heap(node, zone, order));
+ ASSERT(!pg->u.free.scrub_state);
+ pg->u.free.scrub_state = PAGE_SCRUBBING;
+
+ spin_unlock(&heap_lock);
scrub_order = MIN(order, SCRUB_CHUNK_ORDER);
num_scrubbed = 0;
@@ -1100,7 +1147,15 @@ bool_t scrub_free_pages()
while ( num_scrubbed < (1 << order) )
{
for ( i = 0; i < (1 << scrub_order); i++ )
+ {
scrub_one_page(&pg[num_scrubbed + i]);
+ if ( ACCESS_ONCE(pg->u.free.scrub_state) &
PAGE_SCRUB_ABORT )
+ {
+ /* Someone wants this chunk. Drop everything. */
+ pg->u.free.scrub_state = 0;
+ goto out_nolock;
+ }
+ }
num_scrubbed += (1 << scrub_order);
if ( softirq_pending(cpu) )
@@ -1110,7 +1165,16 @@ bool_t scrub_free_pages()
break;
}
}
-
+
+ st.pg = pg;
+ st.drop = false;
+ spin_lock_cb(&heap_lock, scrub_continue, &st);
+
+ if ( st.drop )
+ goto out;
+
+ page_list_del(pg, &heap(node, zone, order));
+
start = 0;
end = num_scrubbed;
@@ -1148,6 +1212,8 @@ bool_t scrub_free_pages()
end += (1 << chunk_order);
}
+ pg->u.free.scrub_state = 0;
+
if ( preempt )
goto out;
}
@@ -1156,6 +1222,8 @@ bool_t scrub_free_pages()
out:
spin_unlock(&heap_lock);
+
+ out_nolock:
node_clear(node, node_scrubbing);
return (node_need_scrub[node] != 0);
}
@@ -1194,6 +1262,8 @@ static void free_heap_pages(
if ( page_state_is(&pg[i], offlined) )
tainted = 1;
+ pg[i].u.free.scrub_state=0;
+
/* If a page has no owner it will need no safety TLB flush. */
pg[i].u.free.need_tlbflush = (page_get_owner(&pg[i]) != NULL);
if ( pg[i].u.free.need_tlbflush )
diff --git a/xen/include/asm-arm/mm.h b/xen/include/asm-arm/mm.h
index 52a03a0..1752d44 100644
--- a/xen/include/asm-arm/mm.h
+++ b/xen/include/asm-arm/mm.h
@@ -41,6 +41,10 @@ struct page_info
} inuse;
/* Page is on a free list: ((count_info & PGC_count_mask) == 0). */
struct {
+#define PAGE_SCRUBBING (1<<1)
+#define PAGE_SCRUB_ABORT (1<<2)
+ unsigned char scrub_state;
+
/* Do TLBs need flushing for safety before next page use? */
bool_t need_tlbflush;
} free;
diff --git a/xen/include/asm-x86/mm.h b/xen/include/asm-x86/mm.h
index f3d4443..31e53e9 100644
--- a/xen/include/asm-x86/mm.h
+++ b/xen/include/asm-x86/mm.h
@@ -87,6 +87,10 @@ struct page_info
/* Page is on a free list: ((count_info & PGC_count_mask) == 0). */
struct {
+#define PAGE_SCRUBBING (1<<1)
+#define PAGE_SCRUB_ABORT (1<<2)
+ unsigned char scrub_state;
+
/* Do TLBs need flushing for safety before next page use? */
bool_t need_tlbflush;
} free;
--
1.7.1
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |