|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH] x86: consolidate/enhance TLB flushing interface
Folding into a single local handler and a single SMP multiplexor as
well as adding capability to also flush caches through the same
interfaces (a subsequent patch will make use of this).
Once at changing cpuinfo_x86, this patch also removes several unused
fields apparently inherited from Linux.
The changes to xen/include/asm-x86/cpufeature.h apply cleanly only with
the SVM/EFER patch (which was sent several days ago) applied.
Signed-off-by: Jan Beulich <jbeulich@xxxxxxxxxx>
Index: 2007-08-08/xen/arch/x86/cpu/common.c
===================================================================
--- 2007-08-08.orig/xen/arch/x86/cpu/common.c 2007-08-21 14:15:47.000000000
+0200
+++ 2007-08-08/xen/arch/x86/cpu/common.c 2007-08-08 12:03:19.000000000
+0200
@@ -229,7 +229,6 @@ static void __init early_cpu_detect(void
void __devinit generic_identify(struct cpuinfo_x86 * c)
{
u32 tfms, xlvl;
- int junk;
if (have_cpuid_p()) {
/* Get vendor name */
@@ -244,8 +243,8 @@ void __devinit generic_identify(struct c
/* Intel-defined flags: level 0x00000001 */
if ( c->cpuid_level >= 0x00000001 ) {
- u32 capability, excap;
- cpuid(0x00000001, &tfms, &junk, &excap, &capability);
+ u32 capability, excap, ebx;
+ cpuid(0x00000001, &tfms, &ebx, &excap, &capability);
c->x86_capability[0] = capability;
c->x86_capability[4] = excap;
c->x86 = (tfms >> 8) & 15;
@@ -255,6 +254,8 @@ void __devinit generic_identify(struct c
c->x86_model += ((tfms >> 16) & 0xF) << 4;
}
c->x86_mask = tfms & 15;
+ if ( cpu_has(c, X86_FEATURE_CLFLSH) )
+ c->x86_clflush_size = ((ebx >> 8) & 0xff) * 8;
} else {
/* Have CPUID level 0 only - unheard of */
c->x86 = 4;
@@ -272,6 +273,11 @@ void __devinit generic_identify(struct c
}
}
+#ifdef CONFIG_X86_64
+ if ( cpu_has(c, X86_FEATURE_PAGE1GB) )
+ __set_bit(3, &c->invlpg_works_ok);
+#endif
+
early_intel_workaround(c);
#ifdef CONFIG_X86_HT
@@ -313,6 +319,7 @@ void __devinit identify_cpu(struct cpuin
c->x86_vendor_id[0] = '\0'; /* Unset */
c->x86_model_id[0] = '\0'; /* Unset */
c->x86_max_cores = 1;
+ c->x86_clflush_size = 0;
memset(&c->x86_capability, 0, sizeof c->x86_capability);
if (!have_cpuid_p()) {
Index: 2007-08-08/xen/arch/x86/cpu/intel.c
===================================================================
--- 2007-08-08.orig/xen/arch/x86/cpu/intel.c 2007-08-21 14:15:47.000000000
+0200
+++ 2007-08-08/xen/arch/x86/cpu/intel.c 2007-08-21 12:17:32.000000000 +0200
@@ -16,8 +16,6 @@
#define select_idle_routine(x) ((void)0)
-extern int trap_init_f00f_bug(void);
-
#ifdef CONFIG_X86_INTEL_USERCOPY
/*
* Alignment at which movsl is preferred for bulk memory copies.
@@ -97,25 +95,6 @@ static void __devinit init_intel(struct
unsigned int l2 = 0;
char *p = NULL;
-#ifdef CONFIG_X86_F00F_BUG
- /*
- * All current models of Pentium and Pentium with MMX technology CPUs
- * have the F0 0F bug, which lets nonprivileged users lock up the
system.
- * Note that the workaround only should be initialized once...
- */
- c->f00f_bug = 0;
- if ( c->x86 == 5 ) {
- static int f00f_workaround_enabled = 0;
-
- c->f00f_bug = 1;
- if ( !f00f_workaround_enabled ) {
- trap_init_f00f_bug();
- printk(KERN_NOTICE "Intel Pentium with F0 0F bug -
workaround enabled.\n");
- f00f_workaround_enabled = 1;
- }
- }
-#endif
-
select_idle_routine(c);
l2 = init_intel_cacheinfo(c);
@@ -123,6 +102,16 @@ static void __devinit init_intel(struct
if ((c->x86<<8 | c->x86_model<<4 | c->x86_mask) < 0x633)
clear_bit(X86_FEATURE_SEP, c->x86_capability);
+ if ( /* PentiumPro erratum 30 */
+ (c->x86 == 6 && c->x86_model == 1 && c->x86_mask < 9) ||
+ /* Dual-Core Intel Xeon 3000/5100 series erratum 89/90 */
+ /* Quad-Core Intel Xeon 3200/5300 series erratum 89/88 */
+ /* Intel Core2 erratum 89 */
+ (c->x86 == 6 && c->x86_model == 15 ) ||
+ /* Dual-Core Intel Xeon LV/ULV erratum 75 */
+ (c->x86 == 6 && c->x86_model == 14 ) )
+ __clear_bit(2, &c->invlpg_works_ok);
+
/* Names for the Pentium II/Celeron processors
detectable only by also checking the cache size.
Dixon is NOT a Celeron. */
Index: 2007-08-08/xen/arch/x86/flushtlb.c
===================================================================
--- 2007-08-08.orig/xen/arch/x86/flushtlb.c 2007-08-21 14:15:47.000000000
+0200
+++ 2007-08-08/xen/arch/x86/flushtlb.c 2007-08-21 14:00:19.000000000 +0200
@@ -95,26 +95,74 @@ void write_cr3(unsigned long cr3)
local_irq_restore(flags);
}
-void local_flush_tlb(void)
+void flush_one_local(const void *va, unsigned int flags)
{
- unsigned long flags;
- u32 t;
+ const struct cpuinfo_x86 *c = ¤t_cpu_data;
+ unsigned int level = flags & FLUSH_LEVEL_MASK;
+ unsigned long irqfl;
- /* This non-reentrant function is sometimes called in interrupt context. */
- local_irq_save(flags);
-
- t = pre_flush();
+ if ( !c->x86 )
+ c = &boot_cpu_data;
- hvm_flush_guest_tlbs();
+ /* This non-reentrant function is sometimes called in interrupt context. */
+ local_irq_save(irqfl);
-#ifdef USER_MAPPINGS_ARE_GLOBAL
- __pge_off();
- __pge_on();
-#else
- __asm__ __volatile__ ( "mov %0, %%cr3" : : "r" (read_cr3()) : "memory" );
+ if ( flags & (FLUSH_TLB|FLUSH_TLB_GLOBAL) )
+ {
+ if ( level > 0 &&
+ level < CONFIG_PAGING_LEVELS &&
+ test_bit(level, &c->invlpg_works_ok) )
+ __asm__ __volatile__( "invlpg %0"
+ :
+ : "m" (*(const char *)(va))
+ : "memory" );
+ else
+ {
+ u32 t = pre_flush();
+
+ hvm_flush_guest_tlbs();
+
+#ifndef USER_MAPPINGS_ARE_GLOBAL
+ if ( !(flags & FLUSH_TLB_GLOBAL) ||
+ !(mmu_cr4_features & X86_CR4_PGE) )
+ __asm__ __volatile__ ( "mov %0, %%cr3"
+ :
+ : "r" (read_cr3())
+ : "memory" );
+ else
#endif
+ {
+ __pge_off();
+ __pge_on();
+ }
+
+ post_flush(t);
+ }
+ }
+
+ if ( flags & FLUSH_CACHE )
+ {
+ unsigned long sz;
+
+ if ( level > 0 && level < CONFIG_PAGING_LEVELS )
+ sz = 1UL << ((level - 1) * PAGETABLE_ORDER);
+ else
+ sz = ULONG_MAX;
+ if ( c->x86_clflush_size > 0 &&
+ c->x86_cache_size > 0 &&
+ sz < (c->x86_cache_size >> (PAGE_SHIFT - 10)) )
+ {
+ unsigned long i;
+
+ va = (const void *)((unsigned long)va & ~(sz - 1));
+ for ( i = 0; i < sz; i += c->x86_clflush_size )
+ __asm__ __volatile__( "clflush %0"
+ :
+ : "m" (((const char *)va)[i]) );
+ }
+ else
+ wbinvd();
+ }
- post_flush(t);
-
- local_irq_restore(flags);
+ local_irq_restore(irqfl);
}
Index: 2007-08-08/xen/arch/x86/mm.c
===================================================================
--- 2007-08-08.orig/xen/arch/x86/mm.c 2007-08-21 14:15:47.000000000 +0200
+++ 2007-08-08/xen/arch/x86/mm.c 2007-08-21 14:18:00.000000000 +0200
@@ -3497,7 +3497,7 @@ int map_pages_to_xen(
if ( (l2e_get_flags(ol2e) & _PAGE_PRESENT) )
{
- local_flush_tlb_pge();
+ flush_one_local((void *)virt, FLUSH_TLB_GLOBAL|2);
if ( !(l2e_get_flags(ol2e) & _PAGE_PSE) )
free_xen_pagetable(mfn_to_virt(l2e_get_pfn(ol2e)));
}
@@ -3525,7 +3525,7 @@ int map_pages_to_xen(
l2e_get_flags(*pl2e) & ~_PAGE_PSE));
l2e_write_atomic(pl2e, l2e_from_pfn(virt_to_mfn(pl1e),
__PAGE_HYPERVISOR));
- local_flush_tlb_pge();
+ flush_one_local((void *)virt, FLUSH_TLB_GLOBAL|2);
}
pl1e = l2e_to_l1e(*pl2e) + l1_table_offset(virt);
@@ -3608,7 +3608,7 @@ void destroy_xen_mappings(unsigned long
}
}
- flush_tlb_all_pge();
+ flush_global(FLUSH_TLB_GLOBAL);
}
void __set_fixmap(
Index: 2007-08-08/xen/arch/x86/setup.c
===================================================================
--- 2007-08-08.orig/xen/arch/x86/setup.c 2007-08-21 14:15:47.000000000
+0200
+++ 2007-08-08/xen/arch/x86/setup.c 2007-08-21 11:06:23.000000000 +0200
@@ -114,7 +114,7 @@ struct tss_struct init_tss[NR_CPUS];
char __attribute__ ((__section__(".bss.stack_aligned")))
cpu0_stack[STACK_SIZE];
-struct cpuinfo_x86 boot_cpu_data = { 0, 0, 0, 0, -1, 1, 0, 0, -1 };
+struct cpuinfo_x86 boot_cpu_data = { 0, 0, 0, 0, 0x6, 1, -1 };
#if CONFIG_PAGING_LEVELS > 2
unsigned long mmu_cr4_features = X86_CR4_PSE | X86_CR4_PGE | X86_CR4_PAE;
Index: 2007-08-08/xen/arch/x86/smp.c
===================================================================
--- 2007-08-08.orig/xen/arch/x86/smp.c 2007-08-21 14:15:47.000000000 +0200
+++ 2007-08-08/xen/arch/x86/smp.c 2007-08-21 12:20:28.000000000 +0200
@@ -164,7 +164,8 @@ void send_IPI_mask_phys(cpumask_t mask,
static DEFINE_SPINLOCK(flush_lock);
static cpumask_t flush_cpumask;
-static unsigned long flush_va;
+static const void *flush_va;
+static unsigned int flush_flags;
fastcall void smp_invalidate_interrupt(void)
{
@@ -172,26 +173,18 @@ fastcall void smp_invalidate_interrupt(v
perfc_incr(ipis);
irq_enter();
if ( !__sync_lazy_execstate() )
- {
- if ( flush_va == FLUSHVA_ALL )
- local_flush_tlb();
- else
- local_flush_tlb_one(flush_va);
- }
+ flush_one_local(flush_va, flush_flags);
cpu_clear(smp_processor_id(), flush_cpumask);
irq_exit();
}
-void __flush_tlb_mask(cpumask_t mask, unsigned long va)
+void flush_one_mask(cpumask_t mask, const void *va, unsigned int flags)
{
ASSERT(local_irq_is_enabled());
if ( cpu_isset(smp_processor_id(), mask) )
{
- if ( va == FLUSHVA_ALL )
- local_flush_tlb();
- else
- local_flush_tlb_one(va);
+ flush_one_local(va, flags);
cpu_clear(smp_processor_id(), mask);
}
@@ -200,6 +193,7 @@ void __flush_tlb_mask(cpumask_t mask, un
spin_lock(&flush_lock);
flush_cpumask = mask;
flush_va = va;
+ flush_flags = flags;
send_IPI_mask(mask, INVALIDATE_TLB_VECTOR);
while ( !cpus_empty(flush_cpumask) )
cpu_relax();
@@ -215,24 +209,13 @@ void new_tlbflush_clock_period(void)
/* Flush everyone else. We definitely flushed just before entry. */
allbutself = cpu_online_map;
cpu_clear(smp_processor_id(), allbutself);
- __flush_tlb_mask(allbutself, FLUSHVA_ALL);
+ flush_mask(allbutself, FLUSH_TLB);
/* No need for atomicity: we are the only possible updater. */
ASSERT(tlbflush_clock == 0);
tlbflush_clock++;
}
-static void flush_tlb_all_pge_ipi(void *info)
-{
- local_flush_tlb_pge();
-}
-
-void flush_tlb_all_pge(void)
-{
- smp_call_function(flush_tlb_all_pge_ipi, 0, 1, 1);
- local_flush_tlb_pge();
-}
-
void smp_send_event_check_mask(cpumask_t mask)
{
cpu_clear(smp_processor_id(), mask);
Index: 2007-08-08/xen/arch/x86/x86_32/mm.c
===================================================================
--- 2007-08-08.orig/xen/arch/x86/x86_32/mm.c 2007-08-21 14:15:47.000000000
+0200
+++ 2007-08-08/xen/arch/x86/x86_32/mm.c 2007-08-21 09:59:15.000000000 +0200
@@ -149,7 +149,7 @@ void __init zap_low_mappings(l2_pgentry_
/* Now zap mappings in the idle pagetables. */
destroy_xen_mappings(0, HYPERVISOR_VIRT_START);
- flush_tlb_all_pge();
+ flush_global(FLUSH_TLB_GLOBAL);
/* Replace with mapping of the boot trampoline only. */
map_pages_to_xen(BOOT_TRAMPOLINE, BOOT_TRAMPOLINE >> PAGE_SHIFT,
Index: 2007-08-08/xen/arch/x86/x86_64/mm.c
===================================================================
--- 2007-08-08.orig/xen/arch/x86/x86_64/mm.c 2007-08-21 14:15:47.000000000
+0200
+++ 2007-08-08/xen/arch/x86/x86_64/mm.c 2007-08-21 12:33:35.000000000 +0200
@@ -205,7 +205,7 @@ void __init zap_low_mappings(void)
/* Remove aliased mapping of first 1:1 PML4 entry. */
l4e_write(&idle_pg_table[0], l4e_empty());
- local_flush_tlb_pge();
+ flush_local(FLUSH_TLB_GLOBAL);
/* Replace with mapping of the boot trampoline only. */
map_pages_to_xen(BOOT_TRAMPOLINE, BOOT_TRAMPOLINE >> PAGE_SHIFT,
Index: 2007-08-08/xen/include/asm-x86/cpufeature.h
===================================================================
--- 2007-08-08.orig/xen/include/asm-x86/cpufeature.h 2007-08-21
14:15:47.000000000 +0200
+++ 2007-08-08/xen/include/asm-x86/cpufeature.h 2007-08-21 12:07:46.000000000
+0200
@@ -50,6 +50,7 @@
#define X86_FEATURE_NX (1*32+20) /* Execute Disable */
#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */
#define X86_FEATURE_FFXSR (1*32+25) /* FFXSR instruction optimizations */
+#define X86_FEATURE_PAGE1GB (1*32+26) /* 1Gb large page support */
#define X86_FEATURE_RDTSCP (1*32+27) /* RDTSCP */
#define X86_FEATURE_LM (1*32+29) /* Long Mode (x86-64) */
#define X86_FEATURE_3DNOWEXT (1*32+30) /* AMD 3DNow! extensions */
@@ -137,6 +138,7 @@
#define cpu_has_centaur_mcr boot_cpu_has(X86_FEATURE_CENTAUR_MCR)
#define cpu_has_clflush boot_cpu_has(X86_FEATURE_CLFLSH)
#define cpu_has_ffxsr 0
+#define cpu_has_page1gb 0
#else /* __x86_64__ */
#define cpu_has_vme 0
#define cpu_has_de 1
@@ -161,6 +163,7 @@
#define cpu_has_centaur_mcr 0
#define cpu_has_clflush boot_cpu_has(X86_FEATURE_CLFLSH)
#define cpu_has_ffxsr boot_cpu_has(X86_FEATURE_FFXSR)
+#define cpu_has_page1gb boot_cpu_has(X86_FEATURE_PAGE1GB)
#endif
#endif /* __ASM_I386_CPUFEATURE_H */
Index: 2007-08-08/xen/include/asm-x86/flushtlb.h
===================================================================
--- 2007-08-08.orig/xen/include/asm-x86/flushtlb.h 2007-08-21
14:15:47.000000000 +0200
+++ 2007-08-08/xen/include/asm-x86/flushtlb.h 2007-08-21 12:34:36.000000000
+0200
@@ -15,6 +15,29 @@
#include <xen/smp.h>
#include <xen/types.h>
+#define FLUSH_LEVEL_MASK 0x0f
+#define FLUSH_TLB 0x10
+#define FLUSH_TLB_GLOBAL 0x20
+#define FLUSH_CACHE 0x40
+
+void flush_one_local(const void *va, unsigned int flags);
+static inline void flush_local(unsigned int flags)
+{
+ flush_one_local(NULL, flags | FLUSH_LEVEL_MASK);
+}
+#ifdef CONFIG_SMP
+void flush_one_mask(cpumask_t, const void *va, unsigned int flags);
+static inline void flush_mask(cpumask_t mask, unsigned int flags)
+{
+ flush_one_mask(mask, NULL, flags | FLUSH_LEVEL_MASK);
+}
+#else
+#define flush_mask(mask, flags) flush_local(flags)
+#define flush_one_mask(mask, va, flags) flush_one_local(va, flags)
+#endif
+#define flush_global(flags) flush_mask(cpu_online_map, flags)
+#define flush_one_global(va, flags) flush_one_mask(cpu_online_map, va, flags)
+
/* The current time as shown by the virtual TLB clock. */
extern u32 tlbflush_clock;
@@ -72,31 +95,20 @@ static inline unsigned long read_cr3(voi
extern void write_cr3(unsigned long cr3);
/* Flush guest mappings from the TLB and implicitly tick the tlbflush clock. */
-extern void local_flush_tlb(void);
-
-#define local_flush_tlb_pge() \
- do { \
- __pge_off(); \
- local_flush_tlb(); \
- __pge_on(); \
- } while ( 0 )
+#define local_flush_tlb(void) flush_local(FLUSH_TLB)
-#define local_flush_tlb_one(__addr) \
- __asm__ __volatile__("invlpg %0": :"m" (*(char *) (__addr)))
+#define local_flush_tlb_one(v) \
+ flush_one_local((const void *)(v), FLUSH_TLB|1)
#define flush_tlb_all() flush_tlb_mask(cpu_online_map)
#ifndef CONFIG_SMP
-#define flush_tlb_all_pge() local_flush_tlb_pge()
#define flush_tlb_mask(mask) local_flush_tlb()
-#define flush_tlb_one_mask(mask,v) local_flush_tlb_one(_v)
+#define flush_tlb_one_mask(mask,v) local_flush_tlb_one(v)
#else
#include <xen/smp.h>
-#define FLUSHVA_ALL (~0UL)
-extern void flush_tlb_all_pge(void);
-extern void __flush_tlb_mask(cpumask_t mask, unsigned long va);
-#define flush_tlb_mask(mask) __flush_tlb_mask(mask,FLUSHVA_ALL)
-#define flush_tlb_one_mask(mask,v) __flush_tlb_mask(mask,(unsigned long)(v))
+#define flush_tlb_mask(mask) flush_mask(mask, FLUSH_TLB)
+#define flush_tlb_one_mask(mask,v) flush_one_mask(mask,(const void *)(v),
FLUSH_TLB|1)
#endif
#endif /* __FLUSHTLB_H__ */
Index: 2007-08-08/xen/include/asm-x86/processor.h
===================================================================
--- 2007-08-08.orig/xen/include/asm-x86/processor.h 2007-08-21
14:15:47.000000000 +0200
+++ 2007-08-08/xen/include/asm-x86/processor.h 2007-08-21 11:34:25.000000000
+0200
@@ -156,24 +156,20 @@ struct cpuinfo_x86 {
__u8 x86_vendor; /* CPU vendor */
__u8 x86_model;
__u8 x86_mask;
- char wp_works_ok; /* It doesn't on 386's */
- char hlt_works_ok; /* Problems on some 486Dx4's and old 386's */
+ __u8 invlpg_works_ok;
char hard_math;
- char rfu;
int cpuid_level; /* Maximum supported CPUID level, -1=no CPUID */
unsigned int x86_capability[NCAPINTS];
char x86_vendor_id[16];
char x86_model_id[64];
int x86_cache_size; /* in KB - valid for CPUS which support this call */
int x86_cache_alignment; /* In bytes */
- char fdiv_bug;
- char f00f_bug;
char coma_bug;
- char pad0;
int x86_power;
unsigned char x86_max_cores; /* cpuid returned max cores value */
unsigned char booted_cores; /* number of cores as seen by OS */
unsigned char apicid;
+ unsigned short x86_clflush_size;
} __cacheline_aligned;
/*
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |