|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v10 1/6] x86: detect and initialize Cache QoS Monitoring feature
Detect platform QoS feature status and enumerate the resource types,
one of which is to monitor the L3 cache occupancy.
Also introduce a Xen grub command line parameter to control the
QoS feature status.
Reviewed-by: Andrew Cooper <andrew.cooper3@xxxxxxxxxx>
Signed-off-by: Dongxiao Xu <dongxiao.xu@xxxxxxxxx>
Signed-off-by: Jiongxi Li <jiongxi.li@xxxxxxxxx>
---
docs/misc/xen-command-line.markdown | 7 ++
xen/arch/x86/Makefile | 1 +
xen/arch/x86/pqos/Makefile | 2 +
xen/arch/x86/pqos/cqm.c | 179 +++++++++++++++++++++++++++++++++++
xen/arch/x86/pqos/pqos.c | 94 ++++++++++++++++++
xen/arch/x86/setup.c | 3 +
xen/include/asm-x86/cpufeature.h | 1 +
xen/include/asm-x86/pqos.h | 47 +++++++++
8 files changed, 334 insertions(+)
create mode 100644 xen/arch/x86/pqos/Makefile
create mode 100644 xen/arch/x86/pqos/cqm.c
create mode 100644 xen/arch/x86/pqos/pqos.c
create mode 100644 xen/include/asm-x86/pqos.h
diff --git a/docs/misc/xen-command-line.markdown
b/docs/misc/xen-command-line.markdown
index 689ffe6..36620ad 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -788,6 +788,13 @@ This option can be specified more than once (up to 8 times
at present).
### ple\_window
> `= <integer>`
+### pqos (Intel)
+> `= List of ( <boolean> | cqm:<boolean> | cqm_max_rmid:<integer> )`
+
+> Default: `pqos=1,cqm:1,cqm_max_rmid:255`
+
+Configure platform QoS services.
+
### reboot
> `= t[riple] | k[bd] | n[o] [, [w]arm | [c]old]`
diff --git a/xen/arch/x86/Makefile b/xen/arch/x86/Makefile
index d502bdf..87b329b 100644
--- a/xen/arch/x86/Makefile
+++ b/xen/arch/x86/Makefile
@@ -4,6 +4,7 @@ subdir-y += genapic
subdir-y += hvm
subdir-y += mm
subdir-y += oprofile
+subdir-y += pqos
subdir-$(x86_64) += x86_64
diff --git a/xen/arch/x86/pqos/Makefile b/xen/arch/x86/pqos/Makefile
new file mode 100644
index 0000000..92f9b3d
--- /dev/null
+++ b/xen/arch/x86/pqos/Makefile
@@ -0,0 +1,2 @@
+obj-y += pqos.o
+obj-y += cqm.o
diff --git a/xen/arch/x86/pqos/cqm.c b/xen/arch/x86/pqos/cqm.c
new file mode 100644
index 0000000..b46f25b
--- /dev/null
+++ b/xen/arch/x86/pqos/cqm.c
@@ -0,0 +1,179 @@
+/*
+ * pqos.c: Platform QoS related service for guest.
+ *
+ * Copyright (c) 2014, Intel Corporation
+ * Author: Dongxiao Xu <dongxiao.xu@xxxxxxxxx>
+ * Author: Jiongxi Li <jiongxi.li@xxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+#include <asm/processor.h>
+#include <xen/init.h>
+#include <xen/mm.h>
+#include <xen/spinlock.h>
+#include <xen/notifier.h>
+#include <xen/cpu.h>
+#include <asm/pqos.h>
+
+struct pqos_cqm *__read_mostly cqm = NULL;
+
+static int cqm_add_socket(int socket)
+{
+ unsigned int i, order;
+
+ if ( cqm->l3c[socket] )
+ return 0;
+
+ /* Allocate per-socket CQM LLC buffer */
+ order = get_order_from_bytes((cqm->rmid_max + 1) * sizeof(unsigned long));
+ cqm->l3c[socket] = alloc_xenheap_pages(order, 0);
+ if ( !cqm->l3c[socket] )
+ {
+ printk(XENLOG_WARNING
+ "Failed to alloc CQM buffer when adding a new socket.\n");
+ return -1;
+ }
+ /* Track the per-socket CQM LLC buffer MFN */
+ cqm->socket_l3c_mfn[socket] = virt_to_mfn(cqm->l3c[socket]);
+ for ( i = 0; i < (1 << order); i++ )
+ share_xen_page_with_privileged_guests(
+ virt_to_page((void *)cqm->l3c[socket] + i * PAGE_SIZE),
+ XENSHARE_readonly);
+
+ return 0;
+}
+
+static void cqm_remove_socket(int socket)
+{
+ unsigned int order;
+
+ if ( cqm->l3c[socket] )
+ {
+ order = get_order_from_bytes((cqm->rmid_max + 1) * sizeof(unsigned
long));
+ free_xenheap_pages(cqm->l3c[socket], order);
+ }
+
+ cqm->socket_l3c_mfn[socket] = 0;
+}
+
+/* Always return NOTIFY_DONE to avoid CPU online/offline failure by CQM */
+static int cpu_callback(
+ struct notifier_block *nfb, unsigned long action, void *hcpu)
+{
+ unsigned int cpu = (unsigned long)hcpu;
+ int socket = cpu_to_socket(cpu);
+
+ if ( socket < 0 )
+ return NOTIFY_DONE;
+
+ switch ( action )
+ {
+ case CPU_ONLINE:
+ cqm_add_socket(socket);
+ break;
+ case CPU_DEAD:
+ if ( !cpumask_weight(per_cpu(cpu_core_mask, cpu)) )
+ cqm_remove_socket(socket);
+ break;
+ default:
+ break;
+ }
+
+ return NOTIFY_DONE;
+}
+
+static struct notifier_block cpu_nfb = {
+ .notifier_call = cpu_callback
+};
+
+void __init init_cqm(unsigned int rmid_max, unsigned long rmid_mask)
+{
+ unsigned int rmid, cpu, socket;
+ unsigned int eax, edx;
+ unsigned int i, order = 0;
+
+ if ( !rmid_max )
+ return;
+
+ cqm = xzalloc(struct pqos_cqm);
+ if ( !cqm )
+ return;
+
+ cpuid_count(0xf, 1, &eax, &cqm->upscaling_factor, &cqm->rmid_max, &edx);
+ if ( !(edx & QOS_MONITOR_EVTID_L3) )
+ goto out;
+
+ cqm->rmid_mask = rmid_mask;
+ cqm->rmid_inuse = 0;
+ cqm->rmid_min = 1;
+ cqm->rmid_max = min(rmid_max, cqm->rmid_max);
+
+ spin_lock_init(&cqm->cqm_lock);
+
+ /* According to Intel SDM, the possible maximum rmid number is 2^10 = 1024,
+ * thus one page is enough to hold cqm->rmid_to_dom structure */
+ cqm->rmid_to_dom = alloc_xenheap_page();
+ if ( !cqm->rmid_to_dom )
+ goto out;
+ /* Reserve RMID 0 for all domains not being monitored */
+ cqm->rmid_to_dom[0] = DOMID_XEN;
+ for ( rmid = 1; rmid < PAGE_SIZE/sizeof(domid_t); rmid++ )
+ cqm->rmid_to_dom[rmid] = DOMID_INVALID;
+ /* Dom0 tool stack needs to know the RMID to DOMID mapping */
+ share_xen_page_with_privileged_guests(
+ virt_to_page((void *)cqm->rmid_to_dom), XENSHARE_readonly);
+
+ /* Allocate the buffer that holds MFNs of per-socket CQM LLC */
+ order = get_order_from_bytes(sizeof(unsigned long) * NR_CPUS);
+ cqm->socket_l3c_mfn = alloc_xenheap_pages(order, 0);
+ if ( !cqm->socket_l3c_mfn )
+ goto out;
+ memset(cqm->socket_l3c_mfn, 0, PAGE_SIZE * (1 << order));
+ /* Dom0 tool stack will use these MFNs to map each CQM LLC buffer */
+ for ( i = 0; i < (1 << order); i++ )
+ share_xen_page_with_privileged_guests(
+ virt_to_page((void *)cqm->socket_l3c_mfn + i * PAGE_SIZE),
+ XENSHARE_readonly);
+
+ for ( cpu = 0; cpu < NR_CPUS; cpu++ )
+ {
+ socket = cpu_to_socket(cpu);
+ if ( socket < 0 )
+ continue;
+ if ( cqm_add_socket(socket) < 0 )
+ goto out;
+ }
+
+ register_cpu_notifier(&cpu_nfb);
+
+ printk(XENLOG_INFO "Cache QoS Monitoring Enabled.\n");
+
+ return;
+
+out:
+ for ( socket = 0; socket < NR_CPUS; socket++ )
+ cqm_remove_socket(socket);
+ if ( cqm->socket_l3c_mfn )
+ free_xenheap_pages(cqm->socket_l3c_mfn, order);
+ if ( cqm->rmid_to_dom )
+ free_xenheap_page(cqm->rmid_to_dom);
+ xfree(cqm);
+ cqm = NULL;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/arch/x86/pqos/pqos.c b/xen/arch/x86/pqos/pqos.c
new file mode 100644
index 0000000..3f430ba
--- /dev/null
+++ b/xen/arch/x86/pqos/pqos.c
@@ -0,0 +1,94 @@
+/*
+ * pqos.c: Platform QoS related service for guest.
+ *
+ * Copyright (c) 2014, Intel Corporation
+ * Author: Dongxiao Xu <dongxiao.xu@xxxxxxxxx>
+ * Author: Jiongxi Li <jiongxi.li@xxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+#include <xen/init.h>
+#include <xen/cpu.h>
+#include <asm/pqos.h>
+
+static bool_t __initdata opt_pqos = 1;
+static bool_t __initdata opt_cqm = 1;
+static unsigned int __initdata opt_cqm_max_rmid = 255;
+
+static void __init parse_pqos_param(char *s)
+{
+ char *ss, *val_str;
+ int val;
+
+ do {
+ ss = strchr(s, ',');
+ if ( ss )
+ *ss = '\0';
+
+ val = parse_bool(s);
+ if ( val >= 0 )
+ opt_pqos = val;
+ else
+ {
+ val = !!strncmp(s, "no-", 3);
+ if ( !val )
+ s += 3;
+
+ val_str = strchr(s, ':');
+ if ( val_str )
+ *val_str++ = '\0';
+
+ if ( val_str && !strcmp(s, "cqm") &&
+ (val = parse_bool(val_str)) >= 0 )
+ opt_cqm = val;
+ else if ( val_str && !strcmp(s, "cqm_max_rmid") )
+ opt_cqm_max_rmid = simple_strtoul(val_str, NULL, 0);
+ }
+
+ s = ss + 1;
+ } while ( ss );
+}
+
+custom_param("pqos", parse_pqos_param);
+
+static void __init init_qos_monitor(void)
+{
+ unsigned int qm_features;
+ unsigned int eax, ebx, ecx;
+ uint64_t rmid_mask;
+
+ if ( !boot_cpu_has(X86_FEATURE_QOSM) )
+ return;
+
+ cpuid_count(0xf, 0, &eax, &ebx, &ecx, &qm_features);
+
+ rmid_mask = ~(~0ull << get_count_order(ebx));
+
+ if ( opt_cqm && (qm_features & QOS_MONITOR_TYPE_L3) )
+ init_cqm(opt_cqm_max_rmid, rmid_mask);
+}
+
+void __init init_platform_qos(void)
+{
+ if ( !opt_pqos )
+ return;
+
+ init_qos_monitor();
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 4dbf2b7..5c522d2 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -48,6 +48,7 @@
#include <asm/setup.h>
#include <xen/cpu.h>
#include <asm/nmi.h>
+#include <asm/pqos.h>
/* opt_nosmp: If true, secondary processors are ignored. */
static bool_t __initdata opt_nosmp;
@@ -1412,6 +1413,8 @@ void __init noreturn __start_xen(unsigned long mbi_p)
domain_unpause_by_systemcontroller(dom0);
+ init_platform_qos();
+
reset_stack_and_jump(init_done);
}
diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h
index 0c4d6c1..37cd8b7 100644
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -145,6 +145,7 @@
#define X86_FEATURE_ERMS (7*32+ 9) /* Enhanced REP MOVSB/STOSB */
#define X86_FEATURE_INVPCID (7*32+10) /* Invalidate Process Context ID */
#define X86_FEATURE_RTM (7*32+11) /* Restricted Transactional Memory */
+#define X86_FEATURE_QOSM (7*32+12) /* Platform QoS monitoring capability
*/
#define X86_FEATURE_NO_FPU_SEL (7*32+13) /* FPU CS/DS stored as zero */
#define X86_FEATURE_MPX (7*32+14) /* Memory Protection
Extensions */
#define X86_FEATURE_RDSEED (7*32+18) /* RDSEED instruction */
diff --git a/xen/include/asm-x86/pqos.h b/xen/include/asm-x86/pqos.h
new file mode 100644
index 0000000..9baa621
--- /dev/null
+++ b/xen/include/asm-x86/pqos.h
@@ -0,0 +1,47 @@
+/*
+ * pqos.h: Platform QoS related service for guest.
+ *
+ * Copyright (c) 2014, Intel Corporation
+ * Author: Dongxiao Xu <dongxiao.xu@xxxxxxxxx>
+ * Author: Jiongxi Li <jiongxi.li@xxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+#ifndef ASM_PQOS_H
+#define ASM_PQOS_H
+
+#include <public/xen.h>
+#include <xen/spinlock.h>
+
+/* QoS Resource Type Enumeration */
+#define QOS_MONITOR_TYPE_L3 0x2
+
+/* QoS Monitoring Event ID */
+#define QOS_MONITOR_EVTID_L3 0x1
+
+struct pqos_cqm {
+ unsigned long rmid_mask;
+ unsigned int rmid_min;
+ unsigned int rmid_max;
+ unsigned int rmid_inuse;
+ unsigned int upscaling_factor;
+ domid_t *rmid_to_dom;
+ /* socket_l3c_mfn stores cqm->l3c MFNs of each socket */
+ unsigned long *socket_l3c_mfn;
+ /* NR_CPUS is big enough to cover all sockets */
+ unsigned long *l3c[NR_CPUS];
+ spinlock_t cqm_lock;
+};
+extern struct pqos_cqm *cqm;
+
+void __init init_platform_qos(void);
+void __init init_cqm(unsigned int rmid_max, unsigned long rmid_mask);
+
+#endif
--
1.7.9.5
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |