|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [PATCH v9 1/3] x86/tlb: introduce a flush HVM ASIDs flag
Introduce a specific flag to request a HVM guest linear TLB flush,
which is an ASID/VPID tickle that forces a guest linear to guest
physical TLB flush for all HVM guests.
This was previously unconditionally done in each pre_flush call, but
that's not required: HVM guests not using shadow don't require linear
TLB flushes as Xen doesn't modify the guest page tables in that case
(ie: when using HAP). Note that shadow paging code already takes care
of issuing the necessary flushes when the shadow page tables are
modified.
In order to keep the previous behavior modify all shadow code TLB
flushes to also flush the guest linear to physical TLB if the guest is
HVM. I haven't looked at each specific shadow code TLB flush in order
to figure out whether it actually requires a guest TLB flush or not,
so there might be room for improvement in that regard.
Also perform ASID/VPID flushes when modifying the p2m tables as it's a
requirement for AMD hardware. Finally keep the flush in
switch_cr3_cr4, as it's not clear whether code could rely on
switch_cr3_cr4 also performing a guest linear TLB flush. A following
patch can remove the ASID/VPID tickle from switch_cr3_cr4 if found to
not be necessary.
Signed-off-by: Roger Pau Monné <roger.pau@xxxxxxxxxx>
---
Changes since v8:
- Don't flush host TLB on HAP changes.
- Introduce a helper for shadow changes that only flushes ASIDs/VPIDs
when the guest is HVM.
- Introduce a helper for HAP that only flushes ASIDs/VPIDs.
Changes since v7:
- Do not perform an ASID flush in filtered_flush_tlb_mask: the
requested flush is related to the page need_tlbflush field and not
to p2m changes (applies to both callers).
Changes since v6:
- Add ASID/VPID flushes when modifying the p2m.
- Keep the ASID/VPID flush in switch_cr3_cr4.
Changes since v5:
- Rename FLUSH_GUESTS_TLB to FLUSH_HVM_ASID_CORE.
- Clarify commit message.
- Define FLUSH_HVM_ASID_CORE to 0 when !CONFIG_HVM.
---
xen/arch/x86/flushtlb.c | 6 ++++--
xen/arch/x86/mm/hap/hap.c | 8 ++++----
xen/arch/x86/mm/hap/nested_hap.c | 2 +-
xen/arch/x86/mm/hap/private.h | 5 +++++
xen/arch/x86/mm/p2m-pt.c | 2 +-
xen/arch/x86/mm/paging.c | 3 ++-
xen/arch/x86/mm/shadow/common.c | 18 +++++++++---------
xen/arch/x86/mm/shadow/hvm.c | 2 +-
xen/arch/x86/mm/shadow/multi.c | 17 +++++++++--------
xen/arch/x86/mm/shadow/private.h | 6 ++++++
xen/include/asm-x86/flushtlb.h | 6 ++++++
11 files changed, 48 insertions(+), 27 deletions(-)
diff --git a/xen/arch/x86/flushtlb.c b/xen/arch/x86/flushtlb.c
index 03f92c23dc..c81e53c0ae 100644
--- a/xen/arch/x86/flushtlb.c
+++ b/xen/arch/x86/flushtlb.c
@@ -59,8 +59,6 @@ static u32 pre_flush(void)
raise_softirq(NEW_TLBFLUSH_CLOCK_PERIOD_SOFTIRQ);
skip_clocktick:
- hvm_flush_guest_tlbs();
-
return t2;
}
@@ -118,6 +116,7 @@ void switch_cr3_cr4(unsigned long cr3, unsigned long cr4)
local_irq_save(flags);
t = pre_flush();
+ hvm_flush_guest_tlbs();
old_cr4 = read_cr4();
ASSERT(!(old_cr4 & X86_CR4_PCIDE) || !(old_cr4 & X86_CR4_PGE));
@@ -221,6 +220,9 @@ unsigned int flush_area_local(const void *va, unsigned int
flags)
do_tlb_flush();
}
+ if ( flags & FLUSH_HVM_ASID_CORE )
+ hvm_flush_guest_tlbs();
+
if ( flags & FLUSH_CACHE )
{
const struct cpuinfo_x86 *c = ¤t_cpu_data;
diff --git a/xen/arch/x86/mm/hap/hap.c b/xen/arch/x86/mm/hap/hap.c
index a6d5e39b02..12856245be 100644
--- a/xen/arch/x86/mm/hap/hap.c
+++ b/xen/arch/x86/mm/hap/hap.c
@@ -118,7 +118,7 @@ int hap_track_dirty_vram(struct domain *d,
p2m_change_type_range(d, begin_pfn, begin_pfn + nr,
p2m_ram_rw, p2m_ram_logdirty);
- flush_tlb_mask(d->dirty_cpumask);
+ hap_flush_tlb_mask(d->dirty_cpumask);
memset(dirty_bitmap, 0xff, size); /* consider all pages dirty */
}
@@ -205,7 +205,7 @@ static int hap_enable_log_dirty(struct domain *d, bool_t
log_global)
* to be read-only, or via hardware-assisted log-dirty.
*/
p2m_change_entry_type_global(d, p2m_ram_rw, p2m_ram_logdirty);
- flush_tlb_mask(d->dirty_cpumask);
+ hap_flush_tlb_mask(d->dirty_cpumask);
}
return 0;
}
@@ -234,7 +234,7 @@ static void hap_clean_dirty_bitmap(struct domain *d)
* be read-only, or via hardware-assisted log-dirty.
*/
p2m_change_entry_type_global(d, p2m_ram_rw, p2m_ram_logdirty);
- flush_tlb_mask(d->dirty_cpumask);
+ hap_flush_tlb_mask(d->dirty_cpumask);
}
/************************************************/
@@ -798,7 +798,7 @@ hap_write_p2m_entry(struct p2m_domain *p2m, unsigned long
gfn, l1_pgentry_t *p,
safe_write_pte(p, new);
if ( old_flags & _PAGE_PRESENT )
- flush_tlb_mask(d->dirty_cpumask);
+ hap_flush_tlb_mask(d->dirty_cpumask);
paging_unlock(d);
diff --git a/xen/arch/x86/mm/hap/nested_hap.c b/xen/arch/x86/mm/hap/nested_hap.c
index abe5958a52..02a5ae75c0 100644
--- a/xen/arch/x86/mm/hap/nested_hap.c
+++ b/xen/arch/x86/mm/hap/nested_hap.c
@@ -84,7 +84,7 @@ nestedp2m_write_p2m_entry(struct p2m_domain *p2m, unsigned
long gfn,
safe_write_pte(p, new);
if (old_flags & _PAGE_PRESENT)
- flush_tlb_mask(p2m->dirty_cpumask);
+ hap_flush_tlb_mask(p2m->dirty_cpumask);
paging_unlock(d);
diff --git a/xen/arch/x86/mm/hap/private.h b/xen/arch/x86/mm/hap/private.h
index 973fbe8be5..7ee8480d83 100644
--- a/xen/arch/x86/mm/hap/private.h
+++ b/xen/arch/x86/mm/hap/private.h
@@ -47,4 +47,9 @@ unsigned long hap_p2m_ga_to_gfn_4_levels(struct vcpu *v,
struct p2m_domain *p2m, unsigned long cr3,
paddr_t ga, uint32_t *pfec, unsigned int *page_order);
+static inline void hap_flush_tlb_mask(const cpumask_t *mask)
+{
+ flush_mask(mask, FLUSH_HVM_ASID_CORE);
+}
+
#endif /* __HAP_PRIVATE_H__ */
diff --git a/xen/arch/x86/mm/p2m-pt.c b/xen/arch/x86/mm/p2m-pt.c
index eb66077496..c90032dc88 100644
--- a/xen/arch/x86/mm/p2m-pt.c
+++ b/xen/arch/x86/mm/p2m-pt.c
@@ -896,7 +896,7 @@ static void p2m_pt_change_entry_type_global(struct
p2m_domain *p2m,
unmap_domain_page(tab);
if ( changed )
- flush_tlb_mask(p2m->domain->dirty_cpumask);
+ flush_mask(p2m->domain->dirty_cpumask, FLUSH_HVM_ASID_CORE);
}
static int p2m_pt_change_entry_type_range(struct p2m_domain *p2m,
diff --git a/xen/arch/x86/mm/paging.c b/xen/arch/x86/mm/paging.c
index 469bb76429..d0bccaf7eb 100644
--- a/xen/arch/x86/mm/paging.c
+++ b/xen/arch/x86/mm/paging.c
@@ -613,7 +613,8 @@ void paging_log_dirty_range(struct domain *d,
p2m_unlock(p2m);
- flush_tlb_mask(d->dirty_cpumask);
+ flush_mask(d->dirty_cpumask, (!hap_enabled(d) ? FLUSH_TLB : 0) |
+ FLUSH_HVM_ASID_CORE);
}
/*
diff --git a/xen/arch/x86/mm/shadow/common.c b/xen/arch/x86/mm/shadow/common.c
index 75dd414a6e..467e0d3fe1 100644
--- a/xen/arch/x86/mm/shadow/common.c
+++ b/xen/arch/x86/mm/shadow/common.c
@@ -363,7 +363,7 @@ static int oos_remove_write_access(struct vcpu *v, mfn_t
gmfn,
}
if ( ftlb )
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
return 0;
}
@@ -939,7 +939,7 @@ static void _shadow_prealloc(struct domain *d, unsigned int
pages)
/* See if that freed up enough space */
if ( d->arch.paging.shadow.free_pages >= pages )
{
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
return;
}
}
@@ -993,7 +993,7 @@ static void shadow_blow_tables(struct domain *d)
pagetable_get_mfn(v->arch.shadow_table[i]), 0);
/* Make sure everyone sees the unshadowings */
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
}
void shadow_blow_tables_per_domain(struct domain *d)
@@ -1102,7 +1102,7 @@ mfn_t shadow_alloc(struct domain *d,
if ( unlikely(!cpumask_empty(&mask)) )
{
perfc_incr(shadow_alloc_tlbflush);
- flush_tlb_mask(&mask);
+ sh_flush_tlb_mask(d, &mask);
}
/* Now safe to clear the page for reuse */
clear_domain_page(page_to_mfn(sp));
@@ -2293,7 +2293,7 @@ void sh_remove_shadows(struct domain *d, mfn_t gmfn, int
fast, int all)
/* Need to flush TLBs now, so that linear maps are safe next time we
* take a fault. */
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
paging_unlock(d);
}
@@ -3008,7 +3008,7 @@ static void sh_unshadow_for_p2m_change(struct domain *d,
unsigned long gfn,
{
sh_remove_all_shadows_and_parents(d, mfn);
if ( sh_remove_all_mappings(d, mfn, _gfn(gfn)) )
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
}
}
@@ -3048,7 +3048,7 @@ static void sh_unshadow_for_p2m_change(struct domain *d,
unsigned long gfn,
}
omfn = mfn_add(omfn, 1);
}
- flush_tlb_mask(&flushmask);
+ sh_flush_tlb_mask(d, &flushmask);
if ( npte )
unmap_domain_page(npte);
@@ -3335,7 +3335,7 @@ int shadow_track_dirty_vram(struct domain *d,
}
}
if ( flush_tlb )
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
goto out;
out_sl1ma:
@@ -3405,7 +3405,7 @@ bool shadow_flush_tlb(bool (*flush_vcpu)(void *ctxt,
struct vcpu *v),
}
/* Flush TLBs on all CPUs with dirty vcpu state. */
- flush_tlb_mask(mask);
+ sh_flush_tlb_mask(d, mask);
/* Done. */
for_each_vcpu ( d, v )
diff --git a/xen/arch/x86/mm/shadow/hvm.c b/xen/arch/x86/mm/shadow/hvm.c
index 1e6024c71f..149f346a48 100644
--- a/xen/arch/x86/mm/shadow/hvm.c
+++ b/xen/arch/x86/mm/shadow/hvm.c
@@ -591,7 +591,7 @@ static void validate_guest_pt_write(struct vcpu *v, mfn_t
gmfn,
if ( rc & SHADOW_SET_FLUSH )
/* Need to flush TLBs to pick up shadow PT changes */
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
if ( rc & SHADOW_SET_ERROR )
{
diff --git a/xen/arch/x86/mm/shadow/multi.c b/xen/arch/x86/mm/shadow/multi.c
index f6b1628742..17af28cdbd 100644
--- a/xen/arch/x86/mm/shadow/multi.c
+++ b/xen/arch/x86/mm/shadow/multi.c
@@ -3067,7 +3067,7 @@ static int sh_page_fault(struct vcpu *v,
perfc_incr(shadow_rm_write_flush_tlb);
smp_wmb();
atomic_inc(&d->arch.paging.shadow.gtable_dirty_version);
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
}
#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
@@ -3576,7 +3576,8 @@ static bool sh_invlpg(struct vcpu *v, unsigned long
linear)
if ( mfn_to_page(sl1mfn)->u.sh.type
== SH_type_fl1_shadow )
{
- flush_tlb_local();
+ flush_local(FLUSH_TLB |
+ (is_hvm_domain(v->domain) ? FLUSH_HVM_ASID_CORE : 0));
return false;
}
@@ -3811,7 +3812,7 @@ sh_update_linear_entries(struct vcpu *v)
* table entry. But, without this change, it would fetch the wrong
* value due to a stale TLB.
*/
- flush_tlb_local();
+ flush_local(FLUSH_TLB | (is_hvm_domain(d) ? FLUSH_HVM_ASID_CORE : 0));
}
}
@@ -4012,7 +4013,7 @@ sh_update_cr3(struct vcpu *v, int do_locking, bool
noflush)
* (old) shadow linear maps in the writeable mapping heuristics. */
#if GUEST_PAGING_LEVELS == 2
if ( sh_remove_write_access(d, gmfn, 2, 0) != 0 )
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
sh_set_toplevel_shadow(v, 0, gmfn, SH_type_l2_shadow);
#elif GUEST_PAGING_LEVELS == 3
/* PAE guests have four shadow_table entries, based on the
@@ -4036,7 +4037,7 @@ sh_update_cr3(struct vcpu *v, int do_locking, bool
noflush)
}
}
if ( flush )
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
/* Now install the new shadows. */
for ( i = 0; i < 4; i++ )
{
@@ -4057,7 +4058,7 @@ sh_update_cr3(struct vcpu *v, int do_locking, bool
noflush)
}
#elif GUEST_PAGING_LEVELS == 4
if ( sh_remove_write_access(d, gmfn, 4, 0) != 0 )
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
sh_set_toplevel_shadow(v, 0, gmfn, SH_type_l4_shadow);
if ( !shadow_mode_external(d) && !is_pv_32bit_domain(d) )
{
@@ -4503,7 +4504,7 @@ static void sh_pagetable_dying(paddr_t gpa)
}
}
if ( flush )
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
/* Remember that we've seen the guest use this interface, so we
* can rely on it using it in future, instead of guessing at
@@ -4540,7 +4541,7 @@ static void sh_pagetable_dying(paddr_t gpa)
mfn_to_page(gmfn)->pagetable_dying = true;
shadow_unhook_mappings(d, smfn, 1/* user pages only */);
/* Now flush the TLB: we removed toplevel mappings. */
- flush_tlb_mask(d->dirty_cpumask);
+ sh_flush_tlb_mask(d, d->dirty_cpumask);
}
/* Remember that we've seen the guest use this interface, so we
diff --git a/xen/arch/x86/mm/shadow/private.h b/xen/arch/x86/mm/shadow/private.h
index e8b028a365..2404ca4ff8 100644
--- a/xen/arch/x86/mm/shadow/private.h
+++ b/xen/arch/x86/mm/shadow/private.h
@@ -818,6 +818,12 @@ static inline int sh_check_page_has_no_refs(struct
page_info *page)
bool shadow_flush_tlb(bool (*flush_vcpu)(void *ctxt, struct vcpu *v),
void *ctxt);
+static inline void sh_flush_tlb_mask(const struct domain *d,
+ const cpumask_t *mask)
+{
+ flush_mask(mask, FLUSH_TLB | (is_hvm_domain(d) ? FLUSH_HVM_ASID_CORE : 0));
+}
+
#endif /* _XEN_SHADOW_PRIVATE_H */
/*
diff --git a/xen/include/asm-x86/flushtlb.h b/xen/include/asm-x86/flushtlb.h
index 2cfe4e6e97..579dc56803 100644
--- a/xen/include/asm-x86/flushtlb.h
+++ b/xen/include/asm-x86/flushtlb.h
@@ -105,6 +105,12 @@ void switch_cr3_cr4(unsigned long cr3, unsigned long cr4);
#define FLUSH_VCPU_STATE 0x1000
/* Flush the per-cpu root page table */
#define FLUSH_ROOT_PGTBL 0x2000
+#if CONFIG_HVM
+ /* Flush all HVM guests linear TLB (using ASID/VPID) */
+#define FLUSH_HVM_ASID_CORE 0x4000
+#else
+#define FLUSH_HVM_ASID_CORE 0
+#endif
/* Flush local TLBs/caches. */
unsigned int flush_area_local(const void *va, unsigned int flags);
--
2.26.0
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |