[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [xen master] VT-d / x86: re-arrange cache syncing
commit 3330013e6739661fd21a518b7e85be2ebe08c354 Author: Jan Beulich <jbeulich@xxxxxxxx> AuthorDate: Fri Feb 18 14:18:01 2022 +0100 Commit: Jan Beulich <jbeulich@xxxxxxxx> CommitDate: Fri Feb 18 14:18:01 2022 +0100 VT-d / x86: re-arrange cache syncing The actual function should always have lived in core x86 code; move it there, replacing get_cache_line_size() by readily available (except very early during boot; see the code comment) data. Also rename the function. Drop the respective IOMMU hook, (re)introducing a respective boolean instead. Replace a true and an almost open-coding instance of iommu_sync_cache(). Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx> Reviewed-by: Paul Durrant <paul@xxxxxxx> Reviewed-by: Kevin Tian <kevin.tian@xxxxxxxxx> --- xen/arch/x86/flushtlb.c | 52 ++++++++++++++++++++++++++++++++++ xen/arch/x86/include/asm/cache.h | 6 ++++ xen/arch/x86/include/asm/iommu.h | 14 +++++---- xen/drivers/passthrough/vtd/extern.h | 1 - xen/drivers/passthrough/vtd/iommu.c | 53 ++--------------------------------- xen/drivers/passthrough/vtd/x86/vtd.c | 5 ---- xen/drivers/passthrough/x86/iommu.c | 4 +-- xen/include/xen/iommu.h | 1 - 8 files changed, 70 insertions(+), 66 deletions(-) diff --git a/xen/arch/x86/flushtlb.c b/xen/arch/x86/flushtlb.c index 25798df50f..735ce00316 100644 --- a/xen/arch/x86/flushtlb.c +++ b/xen/arch/x86/flushtlb.c @@ -11,6 +11,7 @@ #include <xen/sched.h> #include <xen/smp.h> #include <xen/softirq.h> +#include <asm/cache.h> #include <asm/flushtlb.h> #include <asm/invpcid.h> #include <asm/nops.h> @@ -265,6 +266,57 @@ unsigned int flush_area_local(const void *va, unsigned int flags) return flags; } +void cache_writeback(const void *addr, unsigned int size) +{ + /* + * This function may be called before current_cpu_data is established. + * Hence a fallback is needed to prevent the loop below becoming infinite. + */ + unsigned int clflush_size = current_cpu_data.x86_clflush_size ?: 16; + const void *end = addr + size; + + addr -= (unsigned long)addr & (clflush_size - 1); + for ( ; addr < end; addr += clflush_size ) + { +/* + * The arguments to a macro must not include preprocessor directives. Doing so + * results in undefined behavior, so we have to create some defines here in + * order to avoid it. + */ +#if defined(HAVE_AS_CLWB) +# define CLWB_ENCODING "clwb %[p]" +#elif defined(HAVE_AS_XSAVEOPT) +# define CLWB_ENCODING "data16 xsaveopt %[p]" /* clwb */ +#else +# define CLWB_ENCODING ".byte 0x66, 0x0f, 0xae, 0x30" /* clwb (%%rax) */ +#endif + +#define BASE_INPUT(addr) [p] "m" (*(const char *)(addr)) +#if defined(HAVE_AS_CLWB) || defined(HAVE_AS_XSAVEOPT) +# define INPUT BASE_INPUT +#else +# define INPUT(addr) "a" (addr), BASE_INPUT(addr) +#endif + /* + * Note regarding the use of NOP_DS_PREFIX: it's faster to do a clflush + * + prefix than a clflush + nop, and hence the prefix is added instead + * of letting the alternative framework fill the gap by appending nops. + */ + alternative_io_2(".byte " __stringify(NOP_DS_PREFIX) "; clflush %[p]", + "data16 clflush %[p]", /* clflushopt */ + X86_FEATURE_CLFLUSHOPT, + CLWB_ENCODING, + X86_FEATURE_CLWB, /* no outputs */, + INPUT(addr)); +#undef INPUT +#undef BASE_INPUT +#undef CLWB_ENCODING + } + + alternative_2("", "sfence", X86_FEATURE_CLFLUSHOPT, + "sfence", X86_FEATURE_CLWB); +} + unsigned int guest_flush_tlb_flags(const struct domain *d) { bool shadow = paging_mode_shadow(d); diff --git a/xen/arch/x86/include/asm/cache.h b/xen/arch/x86/include/asm/cache.h index 1f7173d8c7..424dc5b7b9 100644 --- a/xen/arch/x86/include/asm/cache.h +++ b/xen/arch/x86/include/asm/cache.h @@ -11,4 +11,10 @@ #define __read_mostly __section(".data.read_mostly") +#ifndef __ASSEMBLY__ + +void cache_writeback(const void *addr, unsigned int size); + +#endif + #endif diff --git a/xen/arch/x86/include/asm/iommu.h b/xen/arch/x86/include/asm/iommu.h index de46149b40..8a96ba1f09 100644 --- a/xen/arch/x86/include/asm/iommu.h +++ b/xen/arch/x86/include/asm/iommu.h @@ -19,6 +19,7 @@ #include <xen/mem_access.h> #include <xen/spinlock.h> #include <asm/apicdef.h> +#include <asm/cache.h> #include <asm/processor.h> #include <asm/hvm/vmx/vmcs.h> @@ -134,12 +135,13 @@ extern bool untrusted_msi; int pi_update_irte(const struct pi_desc *pi_desc, const struct pirq *pirq, const uint8_t gvec); -#define iommu_sync_cache(addr, size) ({ \ - const struct iommu_ops *ops = iommu_get_ops(); \ - \ - if ( ops->sync_cache ) \ - iommu_vcall(ops, sync_cache, addr, size); \ -}) +extern bool iommu_non_coherent; + +static inline void iommu_sync_cache(const void *addr, unsigned int size) +{ + if ( iommu_non_coherent ) + cache_writeback(addr, size); +} int __must_check iommu_free_pgtables(struct domain *d); struct page_info *__must_check iommu_alloc_pgtable(struct domain *d); diff --git a/xen/drivers/passthrough/vtd/extern.h b/xen/drivers/passthrough/vtd/extern.h index f97883a780..5e9cc935c4 100644 --- a/xen/drivers/passthrough/vtd/extern.h +++ b/xen/drivers/passthrough/vtd/extern.h @@ -78,7 +78,6 @@ int __must_check qinval_device_iotlb_sync(struct vtd_iommu *iommu, struct pci_dev *pdev, u16 did, u16 size, u64 addr); -unsigned int get_cache_line_size(void); void flush_all_cache(void); uint64_t alloc_pgtable_maddr(unsigned long npages, nodeid_t node); diff --git a/xen/drivers/passthrough/vtd/iommu.c b/xen/drivers/passthrough/vtd/iommu.c index 34ea5f485d..698b09d070 100644 --- a/xen/drivers/passthrough/vtd/iommu.c +++ b/xen/drivers/passthrough/vtd/iommu.c @@ -240,54 +240,6 @@ domid_t did_to_domain_id(const struct vtd_iommu *iommu, unsigned int did) return iommu->domid_map[did]; } -static void sync_cache(const void *addr, unsigned int size) -{ - static unsigned long clflush_size = 0; - const void *end = addr + size; - - if ( clflush_size == 0 ) - clflush_size = get_cache_line_size(); - - addr -= (unsigned long)addr & (clflush_size - 1); - for ( ; addr < end; addr += clflush_size ) -/* - * The arguments to a macro must not include preprocessor directives. Doing so - * results in undefined behavior, so we have to create some defines here in - * order to avoid it. - */ -#if defined(HAVE_AS_CLWB) -# define CLWB_ENCODING "clwb %[p]" -#elif defined(HAVE_AS_XSAVEOPT) -# define CLWB_ENCODING "data16 xsaveopt %[p]" /* clwb */ -#else -# define CLWB_ENCODING ".byte 0x66, 0x0f, 0xae, 0x30" /* clwb (%%rax) */ -#endif - -#define BASE_INPUT(addr) [p] "m" (*(const char *)(addr)) -#if defined(HAVE_AS_CLWB) || defined(HAVE_AS_XSAVEOPT) -# define INPUT BASE_INPUT -#else -# define INPUT(addr) "a" (addr), BASE_INPUT(addr) -#endif - /* - * Note regarding the use of NOP_DS_PREFIX: it's faster to do a clflush - * + prefix than a clflush + nop, and hence the prefix is added instead - * of letting the alternative framework fill the gap by appending nops. - */ - alternative_io_2(".byte " __stringify(NOP_DS_PREFIX) "; clflush %[p]", - "data16 clflush %[p]", /* clflushopt */ - X86_FEATURE_CLFLUSHOPT, - CLWB_ENCODING, - X86_FEATURE_CLWB, /* no outputs */, - INPUT(addr)); -#undef INPUT -#undef BASE_INPUT -#undef CLWB_ENCODING - - alternative_2("", "sfence", X86_FEATURE_CLFLUSHOPT, - "sfence", X86_FEATURE_CLWB); -} - /* Allocate page table, return its machine address */ uint64_t alloc_pgtable_maddr(unsigned long npages, nodeid_t node) { @@ -306,8 +258,7 @@ uint64_t alloc_pgtable_maddr(unsigned long npages, nodeid_t node) clear_page(vaddr); - if ( (iommu_ops.init ? &iommu_ops : &vtd_ops)->sync_cache ) - sync_cache(vaddr, PAGE_SIZE); + iommu_sync_cache(vaddr, PAGE_SIZE); unmap_domain_page(vaddr); cur_pg++; } @@ -1327,7 +1278,7 @@ int __init iommu_alloc(struct acpi_drhd_unit *drhd) iommu->nr_pt_levels = agaw_to_level(agaw); if ( !ecap_coherent(iommu->ecap) ) - vtd_ops.sync_cache = sync_cache; + iommu_non_coherent = true; nr_dom = cap_ndoms(iommu->cap); diff --git a/xen/drivers/passthrough/vtd/x86/vtd.c b/xen/drivers/passthrough/vtd/x86/vtd.c index 6681dccd69..55f0faa521 100644 --- a/xen/drivers/passthrough/vtd/x86/vtd.c +++ b/xen/drivers/passthrough/vtd/x86/vtd.c @@ -47,11 +47,6 @@ void unmap_vtd_domain_page(const void *va) unmap_domain_page(va); } -unsigned int get_cache_line_size(void) -{ - return ((cpuid_ebx(1) >> 8) & 0xff) * 8; -} - void flush_all_cache() { wbinvd(); diff --git a/xen/drivers/passthrough/x86/iommu.c b/xen/drivers/passthrough/x86/iommu.c index 583abed9bd..ad5f44e13d 100644 --- a/xen/drivers/passthrough/x86/iommu.c +++ b/xen/drivers/passthrough/x86/iommu.c @@ -28,6 +28,7 @@ const struct iommu_init_ops *__initdata iommu_init_ops; struct iommu_ops __read_mostly iommu_ops; +bool __read_mostly iommu_non_coherent; enum iommu_intremap __read_mostly iommu_intremap = iommu_intremap_full; @@ -438,8 +439,7 @@ struct page_info *iommu_alloc_pgtable(struct domain *d) p = __map_domain_page(pg); clear_page(p); - if ( hd->platform_ops->sync_cache ) - iommu_vcall(hd->platform_ops, sync_cache, p, PAGE_SIZE); + iommu_sync_cache(p, PAGE_SIZE); unmap_domain_page(p); diff --git a/xen/include/xen/iommu.h b/xen/include/xen/iommu.h index 6b2cdffa4a..b18e7760a2 100644 --- a/xen/include/xen/iommu.h +++ b/xen/include/xen/iommu.h @@ -268,7 +268,6 @@ struct iommu_ops { int (*setup_hpet_msi)(struct msi_desc *); int (*adjust_irq_affinities)(void); - void (*sync_cache)(const void *addr, unsigned int size); void (*clear_root_pgtable)(struct domain *d); int (*update_ire_from_msi)(struct msi_desc *msi_desc, struct msi_msg *msg); #endif /* CONFIG_X86 */ -- generated by git-patchbot for /home/xen/git/xen.git#master
|
![]() |
Lists.xenproject.org is hosted with RackSpace, monitoring our |