[xen master] hvmloader: enable MMIO and I/O decode, after all resource allocation

[patchbot@xxxxxxx]

commit a8e0c228c79f3a000e19183090eb41fca173b034
Author:     Harsha Shamsundara Havanur <havanur@xxxxxxxxxx>
AuthorDate: Thu Apr 16 10:58:46 2020 +0200
Commit:     Jan Beulich <jbeulich@xxxxxxxx>
CommitDate: Thu Apr 16 10:58:46 2020 +0200

    hvmloader: enable MMIO and I/O decode, after all resource allocation
    
    It was observed that PCI MMIO and/or IO BARs were programmed with
    memory and I/O decodes (bits 0 and 1 of PCI COMMAND register) enabled,
    during PCI setup phase. This resulted in incorrect memory mapping as
    soon as the lower half of the 64 bit bar is programmed.
    This displaced any RAM mappings under 4G. After the
    upper half is programmed PCI memory mapping is restored to its
    intended high mem location, but the RAM displaced is not restored.
    The OS then continues to boot and function until it tries to access
    the displaced RAM at which point it suffers a page fault and crashes.
    
    This patch address the issue by deferring enablement of memory and
    I/O decode in command register until all the resources, like interrupts
    I/O and/or MMIO BARs for all the PCI device functions are programmed,
    in the descending order of memory requested.
    
    Signed-off-by: Harsha Shamsundara Havanur <havanur@xxxxxxxxxx>
    Reviewed-by: Jan Beulich <jbeulich@xxxxxxxx>
---
 tools/firmware/hvmloader/pci.c | 49 +++++++++++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/tools/firmware/hvmloader/pci.c b/tools/firmware/hvmloader/pci.c
index 0b708bf578..dcd097a866 100644
--- a/tools/firmware/hvmloader/pci.c
+++ b/tools/firmware/hvmloader/pci.c
@@ -84,6 +84,7 @@ void pci_setup(void)
     uint32_t vga_devfn = 256;
     uint16_t class, vendor_id, device_id;
     unsigned int bar, pin, link, isa_irq;
+    uint8_t pci_devfn_decode_type[256] = {};
 
     /* Resources assignable to PCI devices via BARs. */
     struct resource {
@@ -120,6 +121,13 @@ void pci_setup(void)
      */
     bool allow_memory_relocate = 1;
 
+    BUILD_BUG_ON((typeof(*pci_devfn_decode_type))PCI_COMMAND_IO !=
+                 PCI_COMMAND_IO);
+    BUILD_BUG_ON((typeof(*pci_devfn_decode_type))PCI_COMMAND_MEMORY !=
+                 PCI_COMMAND_MEMORY);
+    BUILD_BUG_ON((typeof(*pci_devfn_decode_type))PCI_COMMAND_MASTER !=
+                 PCI_COMMAND_MASTER);
+
     s = xenstore_read(HVM_XS_ALLOW_MEMORY_RELOCATE, NULL);
     if ( s )
         allow_memory_relocate = strtoll(s, NULL, 0);
@@ -208,6 +216,20 @@ void pci_setup(void)
             break;
         }
 
+        /*
+         * It is recommended that BAR programming be done whilst decode
+         * bits are cleared to avoid incorrect mappings being created.
+         * When 64-bit memory BAR is programmed, first by writing the
+         * lower half and then the upper half, which maps to an address
+         * under 4G, as soon as lower half is wriiten, replacing any RAM
+         * mapped in that address, which is not restored back after the
+         * upper half is written and PCI memory is correctly mapped to
+         * its intended high mem address.
+         */
+        cmd = pci_readw(devfn, PCI_COMMAND);
+        cmd &= ~(PCI_COMMAND_MEMORY | PCI_COMMAND_IO);
+        pci_writew(devfn, PCI_COMMAND, cmd);
+
         /* Map the I/O memory and port resources. */
         for ( bar = 0; bar < 7; bar++ )
         {
@@ -289,10 +311,8 @@ void pci_setup(void)
                    devfn>>3, devfn&7, 'A'+pin-1, isa_irq);
         }
 
-        /* Enable bus mastering. */
-        cmd = pci_readw(devfn, PCI_COMMAND);
-        cmd |= PCI_COMMAND_MASTER;
-        pci_writew(devfn, PCI_COMMAND, cmd);
+        /* Enable bus master for this function later */
+        pci_devfn_decode_type[devfn] = PCI_COMMAND_MASTER;
     }
 
     if ( mmio_hole_size )
@@ -497,16 +517,12 @@ void pci_setup(void)
                PRIllx_arg(bar_sz),
                bar_data_upper, bar_data);
                        
-
-        /* Now enable the memory or I/O mapping. */
-        cmd = pci_readw(devfn, PCI_COMMAND);
         if ( (bar_reg == PCI_ROM_ADDRESS) ||
              ((bar_data & PCI_BASE_ADDRESS_SPACE) ==
               PCI_BASE_ADDRESS_SPACE_MEMORY) )
-            cmd |= PCI_COMMAND_MEMORY;
+            pci_devfn_decode_type[devfn] |= PCI_COMMAND_MEMORY;
         else
-            cmd |= PCI_COMMAND_IO;
-        pci_writew(devfn, PCI_COMMAND, cmd);
+            pci_devfn_decode_type[devfn] |= PCI_COMMAND_IO;
     }
 
     if ( pci_hi_mem_start )
@@ -526,10 +542,17 @@ void pci_setup(void)
          * has IO enabled, even if there is no I/O BAR on that
          * particular device.
          */
-        cmd = pci_readw(vga_devfn, PCI_COMMAND);
-        cmd |= PCI_COMMAND_IO;
-        pci_writew(vga_devfn, PCI_COMMAND, cmd);
+        pci_devfn_decode_type[vga_devfn] |= PCI_COMMAND_IO;
     }
+
+    /* Enable bus master, memory and I/O decode for all valid functions. */
+    for ( devfn = 0; devfn < 256; devfn++ )
+        if ( pci_devfn_decode_type[devfn] )
+        {
+            cmd = pci_readw(devfn, PCI_COMMAND);
+            cmd |= pci_devfn_decode_type[devfn];
+            pci_writew(devfn, PCI_COMMAND, cmd);
+        }
 }
 
 /*
--
generated by git-patchbot for /home/xen/git/xen.git#master