Simply trying order-9 allocations until they won't succeed anymore
may consume unnecessarily much memory from the DMA zone (since the
page allocator will try to fulfill the request by using memory from
that zone when only lower order memory blocks are left in all other
zones). To avoid using DMA zone memory, make alloc_chunk() try to
allocate a second smaller chunk and use that one in favor of the
first one if it came from a higher addressed memory. This way, all
memory outside the DMA zone will be consumed before eating into that
zone.

Signed-off-by: Jan Beulich <jbeulich@xxxxxxxxxx>

--- 2010-03-09.orig/xen/arch/x86/domain_build.c	2010-03-12 12:25:18.000000000 +0100
+++ 2010-03-09/xen/arch/x86/domain_build.c	2010-03-12 17:24:11.000000000 +0100
@@ -127,7 +131,8 @@ static struct page_info * __init alloc_c
     struct domain *d, unsigned long max_pages)
 {
     struct page_info *page;
-    unsigned int order;
+    unsigned int order, free_order;
+
     /*
      * Allocate up to 2MB at a time: It prevents allocating very large chunks
      * from DMA pools before the >4GB pool is fully depleted.
@@ -140,6 +145,26 @@ static struct page_info * __init alloc_c
     while ( (page = alloc_domheap_pages(d, order, 0)) == NULL )
         if ( order-- == 0 )
             break;
+    /*
+     * Make a reasonable attempt at finding a smaller chunk at a higher
+     * address, to avoid allocating from low memory as much as possible.
+     */
+    for ( free_order = order; page && order--; )
+    {
+        struct page_info *pg2;
+
+        if ( d->tot_pages + (1 << order) > d->max_pages )
+            continue;
+        pg2 = alloc_domheap_pages(d, order, 0);
+        if ( pg2 > page )
+        {
+            free_domheap_pages(page, free_order);
+            page = pg2;
+            free_order = order;
+        }
+        else if ( pg2 )
+            free_domheap_pages(pg2, order);
+    }
     return page;
 }