[PATCH v1 2/5] drivers/char: Handle Xen relocation in the XHCI console driver

[Marek Marczykowski-Górecki <marmarek@xxxxxxxxxxxxxxxxxxxxxx>]

The XHCI uses DMA for a bunch of configuration structures and also for
transfer rings. Since those buffers live in .bss it's sensitive for Xen
relocation. Use the newly added hooks to handle this case:

In pre-relocation hook wait for all the data already sent to be handled
and pause sending any more.

In the post-relocation hook detect if relocation happened (check if
physical address of one of the structures matches what was programmed
into hardware) and if so - re-initialize all structures carying
physical addresses and program XHCI with new addresses. And then resume
console - this needs to happen in no-relocation case too, to undo
pausing done in pre-relocation.

Move the iommu_add_extra_reserved_device_memory() call post relocation,
as it needs physical addresses. It needs to happen before setting up
IOMMU (specifically before the acpi_iommu_init() call) but that's the
only ordering constraint - moving it is simpler than doing it initially
with pre-relocation addresses and then un-doing during relocation. This
is also the place where calling post-relocation hook unconditionally
(even if relocation didn't actually happened) is helpful - otherwise the
iommu_add_extra_reserved_device_memory() call would need to be done
conditionally in two places.

Finally, move dbc_dma_bufs declaration near top of the file, as it's
used earlier now.

Unfortunately, changes to several registers require flipping DCE (Debug
Capability Enable) to 0 and then back to 1 which results in the device
disconnect for a short time. Linux's xhci_dbc driver appears to do some
synchronization (or buffering?) so if one re-connects to the /dev/USB0
fast enough (for example by running minicom/picocom/etc in a loop), no
messages are lost. But technically there is no guarantee of that...

Signed-off-by: Marek Marczykowski-Górecki <marmarek@xxxxxxxxxxxxxxxxxxxxxx>
---
I tried to avoid flipping DCE, but it seems to be a limitation actually
enforced by the hardware. The XHCI spec says "just" this about a bunch
of registers:

    Software shall initialize this register before setting the Debug
    Capability Enable bit in the Debug Capability Control Register to ‘1’.

As for the implicit console flush in pre-relocation hook, technically it
could be avoided in the no-relocation case, but that would complicate
code structure (see note about reserved device memory). For the
relocation case, avoiding it might be possible, if the driver could
access old pages somehow (as without the flush the device might have
modified them in the meantime), but again - IMO it's not worth it.
Alternative would be flipping DCE also in the no-relocation case, which
is IMO worse than just the flush.
---
 xen/drivers/char/xhci-dbc.c | 89 +++++++++++++++++++++++++++-----------
 1 file changed, 64 insertions(+), 25 deletions(-)

diff --git a/xen/drivers/char/xhci-dbc.c b/xen/drivers/char/xhci-dbc.c
index c45e4b6825cc..c4bb371ff78f 100644
--- a/xen/drivers/char/xhci-dbc.c
+++ b/xen/drivers/char/xhci-dbc.c
@@ -264,6 +264,24 @@ struct dbc {
     uint16_t pci_cr;
 };
 
+/* Those are accessed via DMA. */
+struct dbc_dma_bufs {
+    struct xhci_trb evt_trb[DBC_TRB_RING_CAP];
+    struct xhci_trb out_trb[DBC_TRB_RING_CAP];
+    struct xhci_trb in_trb[DBC_TRB_RING_CAP];
+    uint8_t out_wrk_buf[DBC_WORK_RING_CAP];
+    uint8_t in_wrk_buf[DBC_WORK_RING_CAP];
+    struct xhci_erst_segment erst __aligned(16);
+    struct xhci_dbc_ctx ctx __aligned(16);
+    struct xhci_string_descriptor str_buf[DBC_STRINGS_COUNT];
+    /*
+     * Don't place anything else on this page - it will be
+     * DMA-reachable by the USB controller.
+     */
+};
+static struct dbc_dma_bufs __section(".bss.page_aligned") __aligned(PAGE_SIZE)
+    dbc_dma_bufs;
+
 static void *dbc_sys_map_xhc(uint64_t phys, size_t size)
 {
     size_t i;
@@ -1189,6 +1207,50 @@ static void __init cf_check dbc_uart_init_preirq(struct 
serial_port *port)
     uart->lock = &port->tx_lock;
 }
 
+static void __init cf_check dbc_uart_init_pre_relocate(struct serial_port 
*port)
+{
+    struct dbc_uart *uart = port->uart;
+    struct dbc *dbc = &uart->dbc;
+
+    /* Wait for all the data already sent to be handled. */
+    while ( xhci_trb_ring_size(&dbc->dbc_oring) )
+        dbc_pop_events(dbc);
+    /* Do not send any more data until after relocation. */
+    dbc->suspended = true;
+}
+
+static void __init cf_check dbc_uart_init_post_relocate(struct serial_port 
*port)
+{
+    struct dbc_uart *uart = port->uart;
+    struct dbc *dbc = &uart->dbc;
+
+    if ( readq(&dbc->dbc_reg->erstba) != virt_to_maddr(dbc->dbc_erst) )
+    {
+        /*
+         * Do not use dbc_init_work_ring() to not discard queued data, just
+         * update the DMA address.
+         */
+        dbc->dbc_owork.dma = virt_to_maddr(dbc->dbc_owork.buf);
+        dbc->dbc_iwork.dma = virt_to_maddr(dbc->dbc_iwork.buf);
+
+        if ( !dbc_init_dbc(dbc) )
+        {
+            dbc_error("relocate failed\n");
+            return;
+        }
+
+        dbc_enable_dbc(dbc);
+    }
+
+    dbc->suspended = false;
+
+    iommu_add_extra_reserved_device_memory(
+            PFN_DOWN(virt_to_maddr(&dbc_dma_bufs)),
+            PFN_UP(sizeof(dbc_dma_bufs)),
+            uart->dbc.sbdf,
+            "XHCI console");
+}
+
 static void __init cf_check dbc_uart_init_postirq(struct serial_port *port)
 {
     struct dbc_uart *uart = port->uart;
@@ -1310,6 +1372,8 @@ static void cf_check dbc_uart_resume(struct serial_port 
*port)
 static struct uart_driver dbc_uart_driver = {
     .init_preirq = dbc_uart_init_preirq,
     .init_postirq = dbc_uart_init_postirq,
+    .init_pre_relocate = dbc_uart_init_pre_relocate,
+    .init_post_relocate = dbc_uart_init_post_relocate,
     .tx_ready = dbc_uart_tx_ready,
     .putc = dbc_uart_putc,
     .getc = dbc_uart_getc,
@@ -1318,24 +1382,6 @@ static struct uart_driver dbc_uart_driver = {
     .resume = dbc_uart_resume,
 };
 
-/* Those are accessed via DMA. */
-struct dbc_dma_bufs {
-    struct xhci_trb evt_trb[DBC_TRB_RING_CAP];
-    struct xhci_trb out_trb[DBC_TRB_RING_CAP];
-    struct xhci_trb in_trb[DBC_TRB_RING_CAP];
-    uint8_t out_wrk_buf[DBC_WORK_RING_CAP];
-    uint8_t in_wrk_buf[DBC_WORK_RING_CAP];
-    struct xhci_erst_segment erst __aligned(16);
-    struct xhci_dbc_ctx ctx __aligned(16);
-    struct xhci_string_descriptor str_buf[DBC_STRINGS_COUNT];
-    /*
-     * Don't place anything else on this page - it will be
-     * DMA-reachable by the USB controller.
-     */
-};
-static struct dbc_dma_bufs __section(".bss.page_aligned") __aligned(PAGE_SIZE)
-    dbc_dma_bufs;
-
 static int __init cf_check xhci_parse_dbgp(const char *opt_dbgp)
 {
     struct dbc_uart *uart = &dbc_uart;
@@ -1425,14 +1471,7 @@ void __init xhci_dbc_uart_init(void)
     dbc->dbc_str = dbc_dma_bufs.str_buf;
 
     if ( dbc_open(dbc) )
-    {
-        iommu_add_extra_reserved_device_memory(
-                PFN_DOWN(virt_to_maddr(&dbc_dma_bufs)),
-                PFN_UP(sizeof(dbc_dma_bufs)),
-                uart->dbc.sbdf,
-                "XHCI console");
         serial_register_uart(SERHND_XHCI, &dbc_uart_driver, &dbc_uart);
-    }
 }
 
 #ifdef DBC_DEBUG
-- 
git-series 0.9.1