Re: [PATCH v5 08/11] xen/arm: Enable the existing x86 virtual PCI support for ARM.

[Jan Beulich <jbeulich@xxxxxxxx>]

On 13.10.2021 10:45, Roger Pau Monné wrote:
> On Wed, Oct 06, 2021 at 06:40:34PM +0100, Rahul Singh wrote:
>> The existing VPCI support available for X86 is adapted for Arm.
>> When the device is added to XEN via the hyper call
>> “PHYSDEVOP_pci_device_add”, VPCI handler for the config space
>> access is added to the Xen to emulate the PCI devices config space.
>>
>> A MMIO trap handler for the PCI ECAM space is registered in XEN
>> so that when guest is trying to access the PCI config space,XEN
>> will trap the access and emulate read/write using the VPCI and
>> not the real PCI hardware.
>>
>> For Dom0less systems scan_pci_devices() would be used to discover the
>> PCI device in XEN and VPCI handler will be added during XEN boots.
>>
>> Signed-off-by: Rahul Singh <rahul.singh@xxxxxxx>
>> Reviewed-by: Stefano Stabellini <sstabellini@xxxxxxxxxx>
>> ---
>> Change in v5:
>> - Add pci_cleanup_msi(pdev) in cleanup part.
>> - Added Reviewed-by: Stefano Stabellini <sstabellini@xxxxxxxxxx>
>> Change in v4:
>> - Move addition of XEN_DOMCTL_CDF_vpci flag to separate patch
>> Change in v3:
>> - Use is_pci_passthrough_enabled() in place of pci_passthrough_enabled 
>> variable
>> - Reject XEN_DOMCTL_CDF_vpci for x86 in arch_sanitise_domain_config()
>> - Remove IS_ENABLED(CONFIG_HAS_VPCI) from has_vpci()
>> Change in v2:
>> - Add new XEN_DOMCTL_CDF_vpci flag
>> - modify has_vpci() to include XEN_DOMCTL_CDF_vpci
>> - enable vpci support when pci-passthough option is enabled.
>> ---
>> ---
>>  xen/arch/arm/Makefile         |   1 +
>>  xen/arch/arm/domain.c         |   4 ++
>>  xen/arch/arm/domain_build.c   |   3 +
>>  xen/arch/arm/vpci.c           | 102 ++++++++++++++++++++++++++++++++++
>>  xen/arch/arm/vpci.h           |  36 ++++++++++++
>>  xen/drivers/passthrough/pci.c |  18 ++++++
>>  xen/include/asm-arm/domain.h  |   7 ++-
>>  xen/include/asm-x86/pci.h     |   2 -
>>  xen/include/public/arch-arm.h |   7 +++
>>  xen/include/xen/pci.h         |   2 +
>>  10 files changed, 179 insertions(+), 3 deletions(-)
>>  create mode 100644 xen/arch/arm/vpci.c
>>  create mode 100644 xen/arch/arm/vpci.h
>>
>> diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
>> index 44d7cc81fa..fb9c976ea2 100644
>> --- a/xen/arch/arm/Makefile
>> +++ b/xen/arch/arm/Makefile
>> @@ -7,6 +7,7 @@ ifneq ($(CONFIG_NO_PLAT),y)
>>  obj-y += platforms/
>>  endif
>>  obj-$(CONFIG_TEE) += tee/
>> +obj-$(CONFIG_HAS_VPCI) += vpci.o
>>  
>>  obj-$(CONFIG_HAS_ALTERNATIVE) += alternative.o
>>  obj-y += bootfdt.init.o
>> diff --git a/xen/arch/arm/domain.c b/xen/arch/arm/domain.c
>> index 36138c1b2e..fbb52f78f1 100644
>> --- a/xen/arch/arm/domain.c
>> +++ b/xen/arch/arm/domain.c
>> @@ -39,6 +39,7 @@
>>  #include <asm/vgic.h>
>>  #include <asm/vtimer.h>
>>  
>> +#include "vpci.h"
>>  #include "vuart.h"
>>  
>>  DEFINE_PER_CPU(struct vcpu *, curr_vcpu);
>> @@ -767,6 +768,9 @@ int arch_domain_create(struct domain *d,
>>      if ( is_hardware_domain(d) && (rc = domain_vuart_init(d)) )
>>          goto fail;
>>  
>> +    if ( (rc = domain_vpci_init(d)) != 0 )
>> +        goto fail;
>> +
>>      return 0;
>>  
>>  fail:
>> diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
>> index c5afbe2e05..f4c89bde8c 100644
>> --- a/xen/arch/arm/domain_build.c
>> +++ b/xen/arch/arm/domain_build.c
>> @@ -3053,6 +3053,9 @@ void __init create_dom0(void)
>>      if ( iommu_enabled )
>>          dom0_cfg.flags |= XEN_DOMCTL_CDF_iommu;
>>  
>> +    if ( is_pci_passthrough_enabled() )
>> +        dom0_cfg.flags |= XEN_DOMCTL_CDF_vpci;
>> +
>>      dom0 = domain_create(0, &dom0_cfg, true);
>>      if ( IS_ERR(dom0) || (alloc_dom0_vcpu0(dom0) == NULL) )
>>          panic("Error creating domain 0\n");
>> diff --git a/xen/arch/arm/vpci.c b/xen/arch/arm/vpci.c
>> new file mode 100644
>> index 0000000000..76c12b9281
>> --- /dev/null
>> +++ b/xen/arch/arm/vpci.c
>> @@ -0,0 +1,102 @@
>> +/*
>> + * xen/arch/arm/vpci.c
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License as published by
>> + * the Free Software Foundation; either version 2 of the License, or
>> + * (at your option) any later version.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU General Public License for more details.
>> + */
>> +#include <xen/sched.h>
>> +
>> +#include <asm/mmio.h>
>> +
>> +#define REGISTER_OFFSET(addr)  ( (addr) & 0x00000fff)
>> +
>> +/* Do some sanity checks. */
>> +static bool vpci_mmio_access_allowed(unsigned int reg, unsigned int len)
>> +{
>> +    /* Check access size. */
>> +    if ( len > 8 )
>> +        return false;
>> +
>> +    /* Check that access is size aligned. */
>> +    if ( (reg & (len - 1)) )
>> +        return false;
>> +
>> +    return true;
>> +}
>> +
>> +static int vpci_mmio_read(struct vcpu *v, mmio_info_t *info,
>> +                          register_t *r, void *p)
>> +{
>> +    unsigned int reg;
>> +    pci_sbdf_t sbdf;
>> +    unsigned long data = ~0UL;
>> +    unsigned int size = 1U << info->dabt.size;
>> +
>> +    sbdf.sbdf = MMCFG_BDF(info->gpa);
>> +    reg = REGISTER_OFFSET(info->gpa);
>> +
>> +    if ( !vpci_mmio_access_allowed(reg, size) )
>> +        return 0;
>> +
>> +    data = vpci_read(sbdf, reg, min(4u, size));
>> +    if ( size == 8 )
>> +        data |= (uint64_t)vpci_read(sbdf, reg + 4, 4) << 32;
>> +
>> +    *r = data;
>> +
>> +    return 1;
>> +}
>> +
>> +static int vpci_mmio_write(struct vcpu *v, mmio_info_t *info,
>> +                           register_t r, void *p)
>> +{
>> +    unsigned int reg;
>> +    pci_sbdf_t sbdf;
>> +    unsigned long data = r;
>> +    unsigned int size = 1U << info->dabt.size;
>> +
>> +    sbdf.sbdf = MMCFG_BDF(info->gpa);
>> +    reg = REGISTER_OFFSET(info->gpa);
>> +
>> +    if ( !vpci_mmio_access_allowed(reg, size) )
>> +        return 0;
>> +
>> +    vpci_write(sbdf, reg, min(4u, size), data);
>> +    if ( size == 8 )
>> +        vpci_write(sbdf, reg + 4, 4, data >> 32);
> 
> I think those two helpers (and vpci_mmio_access_allowed) are very
> similar to the existing x86 ones (see vpci_mmcfg_{read,write}), up to
> the point where I would consider moving the shared code to vpci.c as
> vpci_ecam_{read,write} and call them from the arch specific trap
> handlers.

Except that please can we stick to mcfg or mmcfg instead of ecam
in names, as that's how the thing has been named in Xen from its
introduction? I've just grep-ed the code base (case insensitively)
and found no mention of ECAM. There are only a few "became".

Jan