[PATCH v1] xen/arm: smmuv3: Add support for removing devices

[Mykyta Poturai <Mykyta_Poturai@xxxxxxxx>]

Allow for removing devices from SMMUv3. arm_smmu_deassign_dev handles
most of the work by disabling ATS and zeroing STEs. Additionally, unset
the dt_device_is_protected flag and free no longer needed smmu_master.

Tested on QEMU with SRIOV series[1] by repeatedly enabling/disabling
VFs.

[1]: https://patchew.org/Xen/cover.1772806036.git.mykyta._5Fpoturai@xxxxxxxx/

Signed-off-by: Mykyta Poturai <mykyta_poturai@xxxxxxxx>
---
 xen/drivers/passthrough/arm/smmu-v3.c | 59 +++++++++++++++++++++++++++
 xen/include/xen/device_tree.h         |  5 +++
 2 files changed, 64 insertions(+)

diff --git a/xen/drivers/passthrough/arm/smmu-v3.c 
b/xen/drivers/passthrough/arm/smmu-v3.c
index bf153227db..b5b834a7b7 100644
--- a/xen/drivers/passthrough/arm/smmu-v3.c
+++ b/xen/drivers/passthrough/arm/smmu-v3.c
@@ -1493,6 +1493,64 @@ static int arm_smmu_assign_dev(struct domain *d, u8 
devfn, struct device *dev,
 static int arm_smmu_deassign_dev(struct domain *d, uint8_t devfn,
                                 struct device *dev);
 
+static int arm_smmu_remove_device(u8 devfn, struct device *dev)
+{
+       int ret = 0;
+       struct arm_smmu_master *master;
+       struct iommu_fwspec *fwspec;
+
+       fwspec = dev_iommu_fwspec_get(dev);
+       if ( !fwspec )
+               return -ENODEV;
+
+       master = dev_iommu_priv_get(dev);
+       if ( !master )
+               return -ENODEV;
+
+#ifdef CONFIG_HAS_PCI
+       if ( dev_is_pci(dev) )
+       {
+               struct pci_dev *pdev = dev_to_pci(dev);
+
+               if ( pdev->domain )
+               {
+                       ret = arm_smmu_deassign_dev(pdev->domain, devfn, dev);
+                       if ( ret )
+                               printk(XENLOG_WARNING "Failed to deassign 
device %pp from SMMU\n",
+                                       &pdev->sbdf);
+               }
+       }
+#endif
+
+       if ( !dev_is_pci(dev) )
+       {
+               if ( !dt_device_is_protected(dev_to_dt(dev)) )
+               {
+                       dev_err(dev, "Not added to SMMUv3\n");
+                       ret = -ENODEV;
+                       goto out_free;
+               }
+
+               if ( master->domain && master->domain->d )
+               {
+                       ret = arm_smmu_deassign_dev(master->domain->d, 0, dev);
+                       if ( ret )
+                               dev_warn(dev, "Failed to deassign device from 
SMMU\n");
+               }
+               dt_device_unset_protected(dev_to_dt(dev));
+       }
+
+       arm_smmu_disable_pasid(master);
+
+       dev_info(dev, "Removed master device (SMMUv3 %s StreamIds %u)\n",
+                dev_name(fwspec->iommu_dev), fwspec->num_ids);
+
+out_free:
+       xfree(master);
+       dev_iommu_priv_set(dev, NULL);
+       return ret;
+}
+
 static int arm_smmu_add_device(u8 devfn, struct device *dev)
 {
        int i, ret;
@@ -2867,6 +2925,7 @@ static const struct iommu_ops arm_smmu_iommu_ops = {
        .unmap_page             = arm_iommu_unmap_page,
        .dt_xlate               = arm_smmu_dt_xlate,
        .add_device             = arm_smmu_add_device,
+       .remove_device          = arm_smmu_remove_device,
 };
 
 static __init int arm_smmu_dt_init(struct dt_device_node *dev,
diff --git a/xen/include/xen/device_tree.h b/xen/include/xen/device_tree.h
index 06d7643622..1f9608cdcd 100644
--- a/xen/include/xen/device_tree.h
+++ b/xen/include/xen/device_tree.h
@@ -305,6 +305,11 @@ static inline void dt_device_set_protected(struct 
dt_device_node *device)
     device->is_protected = true;
 }
 
+static inline void dt_device_unset_protected(struct dt_device_node *device)
+{
+    device->is_protected = false;
+}
+
 static inline bool dt_device_is_protected(const struct dt_device_node *device)
 {
     return device->is_protected;
-- 
2.51.2