[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

Re: [Xen-devel] [PATCH RFC 15/18] arm: Add ability to relocate Xen in over 4GB space



Hello,

On 18/05/16 17:32, Andrii Anisov wrote:
From: Iurii Konovalenko <iurii.konovalenko@xxxxxxxxxxxxxxx>

Move Xen to the end of physical memory

Can you explain why? As Ian campbell said on the previous version sent last year [1], Xen itself (i.e .text, .bss and .data, not the xenheap itself) is at most 2MB. Are your constraints such that at a 4GB-2MB (i.e. 4096M) of lowmem would not be acceptable?

Also, this patch looks very similar to the one you have sent last year [2]. Most of the comments in this thread are still valid, so I will not repeat them here.

Regards,


Signed-off-by: Iurii Konovalenko <iurii.konovalenko@xxxxxxxxxxxxxxx>
---
  xen/Rules.mk                   |  1 +
  xen/arch/arm/arm32/head.S      | 21 ++++++++++++++++++++-
  xen/arch/arm/domain_build.c    |  2 +-
  xen/arch/arm/platforms/omap5.c | 17 ++++++++++++++---
  xen/arch/arm/platforms/rcar2.c |  9 ++++++++-
  xen/arch/arm/setup.c           | 21 ++++++++++
++++++++++-
  xen/arch/arm/smpboot.c         | 33 +++++++++++++++++++++++++++++----
  7 files changed, 93 insertions(+), 11 deletions(-)

diff --git a/xen/Rules.mk b/xen/Rules.mk
index feb08d6..fbd34a5 100644
--- a/xen/Rules.mk
+++ b/xen/Rules.mk
@@ -64,6 +64,7 @@ CFLAGS-$(HAS_PCI)       += -DHAS_PCI
  CFLAGS-$(HAS_IOPORTS)   += -DHAS_IOPORTS
  CFLAGS-$(HAS_PDX)       += -DHAS_PDX
  CFLAGS-$(frame_pointer) += -fno-omit-frame-pointer -DCONFIG_FRAME_POINTER
+CFLAGS-$(ARM32_RELOCATE_OVER_4GB) += -DARM32_RELOCATE_OVER_4GB

  ifneq ($(max_phys_cpus),)
  CFLAGS-y                += -DMAX_PHYS_CPUS=$(max_phys_cpus)
diff --git a/xen/arch/arm/arm32/head.S b/xen/arch/arm/arm32/head.S
index e1f29bd..a644d6d 100644
--- a/xen/arch/arm/arm32/head.S
+++ b/xen/arch/arm/arm32/head.S
@@ -262,8 +262,21 @@ cpu_init_done:
          add   r4, r4, r10            /* r4 := paddr (boot_pagetable) */
          mov   r5, #0                 /* r4:r5 is paddr (boot_pagetable) */
          mcrr  CP64(r4, r5, HTTBR)
+#ifdef ARM32_RELOCATE_OVER_4GB
+        teq   r7, #0
+        beq   1f                     /* construct pagetable if CPU0 */

-        /* Setup boot_pgtable: */
+        /*Skip constructing TLBs for secondary CPUs, use constracted by CPU0*/
+        PRINT("- Skip construction pagetable, using CPU0 table @")
+        mov   r0, r5
+        bl    putn
+        mov   r0, r4
+        bl    putn
+        PRINT("  -\r\n")
+        bne   skip_constructing
+#endif
+
+1:      /* Setup boot_pgtable: */
          ldr   r1, =boot_second
          add   r1, r1, r10            /* r1 := paddr (boot_second) */

@@ -346,6 +359,7 @@ virtphys_clash:
          PRINT("- Unable to build boot page tables - virt and phys addresses clash. 
-\r\n")
          b     fail

+skip_constructing:
  1:
          PRINT("- Turning on paging -\r\n")

@@ -427,6 +441,11 @@ paging:
           * setup in init_secondary_pagetables. */

          ldr   r4, =init_ttbr         /* VA of HTTBR value stashed by CPU 0 */
+#ifdef ARM32_RELOCATE_OVER_4GB
+        ldr   r1, =_start
+        sub r4, r1
+        add r4, #BOOT_RELOC_VIRT_START
+#endif
          ldrd  r4, r5, [r4]           /* Actual value */
          dsb
          mcrr  CP64(r4, r5, HTTBR)
diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
index b48718d..f06792e 100644
--- a/xen/arch/arm/domain_build.c
+++ b/xen/arch/arm/domain_build.c
@@ -1487,7 +1487,7 @@ static void __init find_gnttab_region(struct domain *d,
      if ( (kinfo->gnttab_size >> PAGE_SHIFT) < max_grant_frames )
          panic("Cannot find a space for the grant table region\n");

-#ifdef CONFIG_ARM_32
+#if defined(CONFIG_ARM_32) && !defined(ARM32_RELOCATE_OVER_4GB)
      /*
       * The gnttab region must be under 4GB in order to work with DOM0
       * using short page table.
diff --git a/xen/arch/arm/platforms/omap5.c b/xen/arch/arm/platforms/omap5.c
index a49ba62..fe77397 100644
--- a/xen/arch/arm/platforms/omap5.c
+++ b/xen/arch/arm/platforms/omap5.c
@@ -25,6 +25,10 @@
  #include <xen/vmap.h>
  #include <asm/io.h>

+#ifdef ARM32_RELOCATE_OVER_4GB
+extern paddr_t xen_relocation_offset;
+#endif
+
  static uint16_t num_den[8][2] = {
      {         0,          0 },  /* not used */
      {  26 *  64,  26 *  125 },  /* 12.0 Mhz */
@@ -132,9 +136,16 @@ static int __init omap5_smp_init(void)
      }

      printk("Set AuxCoreBoot1 to %"PRIpaddr" (%p)\n",
-           __pa(init_secondary), init_secondary);
-    writel(__pa(init_secondary), wugen_base + OMAP_AUX_CORE_BOOT_1_OFFSET);
-
+           __pa(init_secondary)
+#ifdef ARM32_RELOCATE_OVER_4GB
+            - xen_relocation_offset
+#endif
+                       , init_secondary);
+    writel(__pa(init_secondary)
+#ifdef ARM32_RELOCATE_OVER_4GB
+            - xen_relocation_offset
+#endif
+                       , wugen_base + OMAP_AUX_CORE_BOOT_1_OFFSET);
      printk("Set AuxCoreBoot0 to 0x20\n");
      writel(0x20, wugen_base + OMAP_AUX_CORE_BOOT_0_OFFSET);

diff --git a/xen/arch/arm/platforms/rcar2.c b/xen/arch/arm/platforms/rcar2.c
index bb25751..26973f6 100644
--- a/xen/arch/arm/platforms/rcar2.c
+++ b/xen/arch/arm/platforms/rcar2.c
@@ -25,6 +25,9 @@
  #define RCAR2_RAM_SIZE                         0x1000
  #define RCAR2_SMP_START_OFFSET                 0xFFC

+#ifdef ARM32_RELOCATE_OVER_4GB
+extern paddr_t xen_relocation_offset;
+#endif
  static int __init rcar2_smp_init(void)
  {
      void __iomem *pram;
@@ -38,7 +41,11 @@ static int __init rcar2_smp_init(void)
      }

      /* setup reset vectors */
-    writel(__pa(init_secondary), pram + RCAR2_SMP_START_OFFSET);
+    writel(__pa(init_secondary)
+#ifdef ARM32_RELOCATE_OVER_4GB
+            - xen_relocation_offset
+#endif
+            , pram + RCAR2_SMP_START_OFFSET);
      iounmap(pram);

      sev();
diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index 48f734f..7e507bc 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -394,9 +394,13 @@ static paddr_t __init get_xen_paddr(void)
          const struct membank *bank = &mi->bank[i];
          paddr_t s, e;

+        //TODO: investigate reason why we need contiguous memory
+        //and how it can affect relocation of Xen in over 4GB space
+#ifndef ARM32_RELOCATE_OVER_4GB
          /* We can only deal with contiguous memory at the moment */
          if ( last_end != bank->start )
              break;
+#endif

          last_end = bank->start + bank->size;

@@ -407,7 +411,7 @@ static paddr_t __init get_xen_paddr(void)
              if ( !e )
                  continue;

-#ifdef CONFIG_ARM_32
+#if defined (CONFIG_ARM_32) && !defined(ARM32_RELOCATE_OVER_4GB)
              /* Xen must be under 4GB */
              if ( e > 0x100000000ULL )
                  e = 0x100000000ULL;
@@ -698,6 +702,9 @@ void __init setup_cache(void)
      cacheline_bytes = 1U << (4 + (ccsid & 0x7));
  }

+#ifdef ARM32_RELOCATE_OVER_4GB
+paddr_t xen_relocation_offset;
+#endif
  /* C entry point for boot CPU */
  void __init start_xen(unsigned long boot_phys_offset,
                        unsigned long fdt_paddr,
@@ -739,8 +746,20 @@ void __init start_xen(unsigned long boot_phys_offset,
                               (paddr_t)(uintptr_t)(_end - _start + 1), NULL);
      BUG_ON(!xen_bootmodule);

+#ifdef ARM32_RELOCATE_OVER_4GB
+    //save physical address of init_secondary
+    xen_relocation_offset = __pa(init_secondary);
+#endif
      xen_paddr = get_xen_paddr();
      setup_pagetables(boot_phys_offset, xen_paddr);
+#ifdef ARM32_RELOCATE_OVER_4GB
+    //Now Xen is relocated
+    //Calculate offset of Xen relocation
+    //It is difference between new physical address of init_secondary an old 
one
+    //This offset is needed in several places when we have to write to old Xen 
location
+    //(secondary CPUs run on old-located Xen and rely on some variables from 
CPU0)
+    xen_relocation_offset = __pa(init_secondary) - xen_relocation_offset;
+#endif

      /* Update Xen's address now that we have relocated. */
      printk("Update BOOTMOD_XEN from %"PRIpaddr"-%"PRIpaddr" => 
%"PRIpaddr"-%"PRIpaddr"\n",
diff --git a/xen/arch/arm/smpboot.c b/xen/arch/arm/smpboot.c
index a96cda2..731144c 100644
--- a/xen/arch/arm/smpboot.c
+++ b/xen/arch/arm/smpboot.c
@@ -31,6 +31,9 @@
  #include <xen/console.h>
  #include <asm/gic.h>
  #include <asm/psci.h>
+#ifdef ARM32_RELOCATE_OVER_4GB
+#include <xen/vmap.h>
+#endif

  cpumask_t cpu_online_map;
  cpumask_t cpu_present_map;
@@ -353,17 +356,33 @@ int __init cpu_up_send_sgi(int cpu)
      return 0;
  }

+#ifdef ARM32_RELOCATE_OVER_4GB
+extern paddr_t xen_relocation_offset;
+#endif
  /* Bring up a remote CPU */
  int __cpu_up(unsigned int cpu)
  {
      int rc;
      s_time_t deadline;
+#ifdef ARM32_RELOCATE_OVER_4GB
+    paddr_t p_info = __pa(&smp_up_cpu) - xen_relocation_offset;
+    unsigned long* info = ioremap_nocache(p_info, sizeof(unsigned long));
+#else
+    unsigned long* info = &smp_up_cpu;
+#endif

      printk("Bringing up CPU%d\n", cpu);

      rc = init_secondary_pagetables(cpu);
      if ( rc < 0 )
+#ifdef ARM32_RELOCATE_OVER_4GB
+    {
+        iounmap(info);
+        return rc;
+    }
+#else
          return rc;
+#endif

      console_start_sync(); /* Secondary may use early_printk */

@@ -374,8 +393,8 @@ int __cpu_up(unsigned int cpu)
      init_data.cpuid = cpu;

      /* Open the gate for this CPU */
-    smp_up_cpu = cpu_logical_map(cpu);
-    clean_dcache(smp_up_cpu);
+    *info = cpu_logical_map(cpu);
+    clean_dcache(*info);

      rc = arch_cpu_up(cpu);

@@ -383,6 +402,9 @@ int __cpu_up(unsigned int cpu)

      if ( rc < 0 )
      {
+#ifdef ARM32_RELOCATE_OVER_4GB
+        iounmap(info);
+#endif
          printk("Failed to bring up CPU%d\n", cpu);
          return rc;
      }
@@ -406,8 +428,11 @@ int __cpu_up(unsigned int cpu)
       */
      init_data.stack = NULL;
      init_data.cpuid = ~0;
-    smp_up_cpu = MPIDR_INVALID;
-    clean_dcache(smp_up_cpu);
+    *info = MPIDR_INVALID;
+    clean_dcache(*info);
+#ifdef ARM32_RELOCATE_OVER_4GB
+    iounmap(info);
+#endif

      if ( !cpu_online(cpu) )
      {


[1] http://lists.xenproject.org/archives/html/xen-devel/2015-05/msg01031.html [2] http://lists.xenproject.org/archives/html/xen-devel/2015-04/msg00662.html

--
Julien Grall

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel

 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.