|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [win-pv-devel] [PATCH 4/4] Use per-CPU event channel upcalls if available
A recent patch to Xen introduced a new HVM op to set a per-vcpu event
channel upcall. This patch adds code to make use of the latched interrupts
allocated by the FDO code to enable per-vcpu upcalls and adds an extra
EvtchnBind operation to a new v2 EVTCHN interface so that events can be
steered to a specified CPU.
Signed-off-by: Paul Durrant <paul.durrant@xxxxxxxxxx>
---
include/evtchn_interface.h | 35 ++++++-
include/xen.h | 16 +++
src/xen/event_channel.c | 30 ++++++
src/xen/hvm.c | 28 ++++++
src/xenbus/evtchn.c | 239 ++++++++++++++++++++++++++++++++++++++++++---
src/xenbus/store.c | 9 ++
6 files changed, 341 insertions(+), 16 deletions(-)
diff --git a/include/evtchn_interface.h b/include/evtchn_interface.h
index 5898e08..4ab2b28 100644
--- a/include/evtchn_interface.h
+++ b/include/evtchn_interface.h
@@ -112,6 +112,20 @@ typedef PXENBUS_EVTCHN_CHANNEL
...
);
+/*! \typedef XENBUS_EVTCHN_BIND
+ \brief Bind an event channel to a specific CPU
+
+ \param Interface The interface header
+ \param Channel The channel handle
+ \param Cpu The CPU that should handle events
+*/
+typedef NTSTATUS
+(*XENBUS_EVTCHN_BIND)(
+ IN PINTERFACE Interface,
+ IN PXENBUS_EVTCHN_CHANNEL Channel,
+ IN ULONG Cpu
+ );
+
/*! \typedef XENBUS_EVTCHN_UNMASK
\brief Unmask an event channel
@@ -196,7 +210,24 @@ struct _XENBUS_EVTCHN_INTERFACE_V1 {
XENBUS_EVTCHN_CLOSE EvtchnClose;
};
-typedef struct _XENBUS_EVTCHN_INTERFACE_V1 XENBUS_EVTCHN_INTERFACE,
*PXENBUS_EVTCHN_INTERFACE;
+/*! \struct _XENBUS_EVTCHN_INTERFACE_V2
+ \brief EVTCHN interface version 2
+ \ingroup interfaces
+*/
+struct _XENBUS_EVTCHN_INTERFACE_V2 {
+ INTERFACE Interface;
+ XENBUS_EVTCHN_ACQUIRE EvtchnAcquire;
+ XENBUS_EVTCHN_RELEASE EvtchnRelease;
+ XENBUS_EVTCHN_OPEN EvtchnOpen;
+ XENBUS_EVTCHN_BIND EvtchnBind;
+ XENBUS_EVTCHN_UNMASK EvtchnUnmask;
+ XENBUS_EVTCHN_SEND EvtchnSend;
+ XENBUS_EVTCHN_TRIGGER EvtchnTrigger;
+ XENBUS_EVTCHN_GET_PORT EvtchnGetPort;
+ XENBUS_EVTCHN_CLOSE EvtchnClose;
+};
+
+typedef struct _XENBUS_EVTCHN_INTERFACE_V2 XENBUS_EVTCHN_INTERFACE,
*PXENBUS_EVTCHN_INTERFACE;
/*! \def XENBUS_EVTCHN
\brief Macro at assist in method invocation
@@ -207,7 +238,7 @@ typedef struct _XENBUS_EVTCHN_INTERFACE_V1
XENBUS_EVTCHN_INTERFACE, *PXENBUS_EVT
#endif // _WINDLL
#define XENBUS_EVTCHN_INTERFACE_VERSION_MIN 1
-#define XENBUS_EVTCHN_INTERFACE_VERSION_MAX 1
+#define XENBUS_EVTCHN_INTERFACE_VERSION_MAX 2
#endif // _XENBUS_EVTCHN_INTERFACE_H
diff --git a/include/xen.h b/include/xen.h
index 0dabc74..84197a5 100644
--- a/include/xen.h
+++ b/include/xen.h
@@ -97,6 +97,14 @@ HvmPagetableDying(
IN PHYSICAL_ADDRESS Address
);
+__checkReturn
+XEN_API
+NTSTATUS
+HvmSetEvtchnUpcallVector(
+ IN unsigned int vcpu_id,
+ IN UCHAR Vector
+ );
+
// MEMORY
__checkReturn
@@ -196,6 +204,14 @@ EventChannelReset(
VOID
);
+__checkReturn
+XEN_API
+NTSTATUS
+EventChannelBindVirtualCpu(
+ IN ULONG LocalPort,
+ IN unsigned int vcpu_id
+ );
+
// GRANT TABLE
__checkReturn
diff --git a/src/xen/event_channel.c b/src/xen/event_channel.c
index aa87fd4..94a6b82 100644
--- a/src/xen/event_channel.c
+++ b/src/xen/event_channel.c
@@ -327,3 +327,33 @@ fail1:
return status;
}
+
+__checkReturn
+XEN_API
+NTSTATUS
+EventChannelBindVirtualCpu(
+ IN ULONG LocalPort,
+ IN unsigned int vcpu_id
+ )
+{
+ struct evtchn_bind_vcpu op;
+ LONG_PTR rc;
+ NTSTATUS status;
+
+ op.port = LocalPort;
+ op.vcpu = vcpu_id;
+
+ rc = EventChannelOp(EVTCHNOP_bind_vcpu, &op);
+
+ if (rc < 0) {
+ ERRNO_TO_STATUS(-rc, status);
+ goto fail1;
+ }
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
diff --git a/src/xen/hvm.c b/src/xen/hvm.c
index 8135e97..3e3e12c 100644
--- a/src/xen/hvm.c
+++ b/src/xen/hvm.c
@@ -166,3 +166,31 @@ HvmPagetableDying(
fail1:
return status;
}
+
+__checkReturn
+XEN_API
+NTSTATUS
+HvmSetEvtchnUpcallVector(
+ IN unsigned int vcpu_id,
+ IN UCHAR Vector
+ )
+{
+ struct xen_hvm_set_evtchn_upcall_vector op;
+ LONG_PTR rc;
+ NTSTATUS status;
+
+ op.vcpu = vcpu_id;
+ op.vector = Vector;
+
+ rc = HvmOp(HVMOP_set_evtchn_upcall_vector, &op);
+
+ if (rc < 0) {
+ ERRNO_TO_STATUS(-rc, status);
+ goto fail1;
+ }
+
+ return STATUS_SUCCESS;
+
+fail1:
+ return status;
+}
diff --git a/src/xenbus/evtchn.c b/src/xenbus/evtchn.c
index 44043d4..aa26a31 100644
--- a/src/xenbus/evtchn.c
+++ b/src/xenbus/evtchn.c
@@ -82,13 +82,16 @@ struct _XENBUS_EVTCHN_CHANNEL {
XENBUS_EVTCHN_PARAMETERS Parameters;
BOOLEAN Mask;
ULONG LocalPort;
+ PXENBUS_INTERRUPT Interrupt;
};
struct _XENBUS_EVTCHN_CONTEXT {
PXENBUS_FDO Fdo;
KSPIN_LOCK Lock;
LONG References;
- PXENBUS_INTERRUPT Interrupt;
+ PXENBUS_INTERRUPT LevelSensitiveInterrupt;
+ PXENBUS_INTERRUPT LatchedInterrupt[MAXIMUM_PROCESSORS];
+ KAFFINITY Affinity;
BOOLEAN Enabled;
XENBUS_SUSPEND_INTERFACE SuspendInterface;
PXENBUS_SUSPEND_CALLBACK SuspendCallbackEarly;
@@ -127,15 +130,39 @@ EvtchnInterruptEnable(
IN PXENBUS_EVTCHN_CONTEXT Context
)
{
+ LONG Cpu;
ULONG Line;
NTSTATUS status;
- Trace("<===>\n");
+ Trace("====>\n");
+
+ ASSERT3U(Context->Affinity, ==, 0);
+
+ Cpu = 0;
+ while (Cpu < KeNumberProcessors) {
+ unsigned int vcpu_id;
+ UCHAR Vector;
+
+ vcpu_id = SystemVirtualCpuIndex(Cpu);
+ Vector = FdoGetInterruptVector(Context->Fdo,
+ Context->LatchedInterrupt[Cpu]);
+
+ status = HvmSetEvtchnUpcallVector(vcpu_id, Vector);
+ if (NT_SUCCESS(status)) {
+ Info("CPU %u\n", Cpu);
+ Context->Affinity |= (KAFFINITY)1 << Cpu;
+ }
+
+ Cpu++;
+ }
- Line = FdoGetInterruptLine(Context->Fdo, Context->Interrupt);
+ Line = FdoGetInterruptLine(Context->Fdo,
+ Context->LevelSensitiveInterrupt);
status = HvmSetParam(HVM_PARAM_CALLBACK_IRQ, Line);
ASSERT(NT_SUCCESS(status));
+
+ Trace("<====\n");
}
static VOID
@@ -143,14 +170,29 @@ EvtchnInterruptDisable(
IN PXENBUS_EVTCHN_CONTEXT Context
)
{
+ LONG Cpu;
NTSTATUS status;
UNREFERENCED_PARAMETER(Context);
- Trace("<===>\n");
+ Trace("====>\n");
status = HvmSetParam(HVM_PARAM_CALLBACK_IRQ, 0);
ASSERT(NT_SUCCESS(status));
+
+ Cpu = KeNumberProcessors;
+ while (--Cpu >= 0) {
+ unsigned int vcpu_id;
+
+ vcpu_id = SystemVirtualCpuIndex(Cpu);
+
+ (VOID) HvmSetEvtchnUpcallVector(vcpu_id, 0);
+ Context->Affinity &= ~((KAFFINITY)1 << Cpu);
+ }
+
+ ASSERT3U(Context->Affinity, ==, 0);
+
+ Trace("<====\n");
}
static FORCEINLINE
@@ -159,10 +201,11 @@ _IRQL_saves_
_IRQL_raises_(HIGH_LEVEL)
KIRQL
__EvtchnAcquireInterruptLock(
- IN PXENBUS_EVTCHN_CONTEXT Context
+ IN PXENBUS_EVTCHN_CONTEXT Context,
+ IN PXENBUS_EVTCHN_CHANNEL Channel
)
{
- return FdoAcquireInterruptLock(Context->Fdo, Context->Interrupt);
+ return FdoAcquireInterruptLock(Context->Fdo, Channel->Interrupt);
}
static FORCEINLINE
@@ -170,10 +213,11 @@ __drv_requiresIRQL(HIGH_LEVEL)
VOID
__EvtchnReleaseInterruptLock(
IN PXENBUS_EVTCHN_CONTEXT Context,
+ IN PXENBUS_EVTCHN_CHANNEL Channel,
IN __drv_restoresIRQL KIRQL Irql
)
{
- FdoReleaseInterruptLock(Context->Fdo, Context->Interrupt, Irql);
+ FdoReleaseInterruptLock(Context->Fdo, Channel->Interrupt, Irql);
}
static NTSTATUS
@@ -358,6 +402,10 @@ EvtchnOpen(
LocalPort = Channel->LocalPort;
+ Channel->Interrupt = (Context->Affinity != 0) ? // Latched available
+ Context->LatchedInterrupt[0] :
+ Context->LevelSensitiveInterrupt;
+
status = XENBUS_EVTCHN_ABI(PortEnable,
&Context->EvtchnAbi,
LocalPort);
@@ -422,6 +470,89 @@ fail1:
return NULL;
}
+#define EVTCHN_SWAP_POINTER(_X, _Y) \
+ do { \
+ (_X) = (PVOID)((ULONG_PTR)(_X) ^ (ULONG_PTR)(_Y)); \
+ (_Y) = (PVOID)((ULONG_PTR)(_X) ^ (ULONG_PTR)(_Y)); \
+ (_X) = (PVOID)((ULONG_PTR)(_X) ^ (ULONG_PTR)(_Y)); \
+ } while (FALSE)
+
+static NTSTATUS
+EvtchnBind(
+ IN PINTERFACE Interface,
+ IN PXENBUS_EVTCHN_CHANNEL Channel,
+ IN ULONG Cpu
+ )
+{
+ PXENBUS_EVTCHN_CONTEXT Context = Interface->Context;
+ PXENBUS_INTERRUPT Interrupt;
+ ULONG LocalPort;
+ unsigned int vcpu_id;
+ KIRQL Irql;
+ NTSTATUS status;
+
+ status = STATUS_INVALID_PARAMETER;
+ if (Cpu >= (ULONG)KeNumberProcessors)
+ goto fail1;
+
+ ASSERT(Context->Enabled);
+
+ status = STATUS_NOT_SUPPORTED;
+ if (~Context->Affinity & ((KAFFINITY)1 << Cpu))
+ goto fail2;
+
+ Interrupt = Context->LatchedInterrupt[Cpu];
+
+ if (Channel->Interrupt == Interrupt)
+ goto done;
+
+ KeRaiseIrql(HIGH_LEVEL, &Irql);
+
+ // Make sure we always lock in a consistent order
+ if ((ULONG_PTR)Interrupt < (ULONG_PTR)Channel->Interrupt) {
+ (VOID) FdoAcquireInterruptLock(Context->Fdo, Interrupt);
+ (VOID) FdoAcquireInterruptLock(Context->Fdo, Channel->Interrupt);
+ } else {
+ (VOID) FdoAcquireInterruptLock(Context->Fdo, Channel->Interrupt);
+ (VOID) FdoAcquireInterruptLock(Context->Fdo, Interrupt);
+ }
+
+ LocalPort = Channel->LocalPort;
+ vcpu_id = SystemVirtualCpuIndex(Cpu);
+
+ status = EventChannelBindVirtualCpu(LocalPort, vcpu_id);
+ if (!NT_SUCCESS(status))
+ goto fail3;
+
+ EVTCHN_SWAP_POINTER(Channel->Interrupt, Interrupt);
+
+ FdoReleaseInterruptLock(Context->Fdo, Channel->Interrupt, HIGH_LEVEL);
+ FdoReleaseInterruptLock(Context->Fdo, Interrupt, HIGH_LEVEL);
+
+ KeLowerIrql(Irql);
+
+ Info("[%u]: CPU %u\n", LocalPort, Cpu);
+
+done:
+ return STATUS_SUCCESS;
+
+fail3:
+ Error("fail3\n");
+
+ FdoReleaseInterruptLock(Context->Fdo, Channel->Interrupt, HIGH_LEVEL);
+ FdoReleaseInterruptLock(Context->Fdo, Interrupt, HIGH_LEVEL);
+
+ KeLowerIrql(Irql);
+
+fail2:
+ Error("fail2\n");
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
static BOOLEAN
EvtchnUnmask(
IN PINTERFACE Interface,
@@ -516,7 +647,7 @@ EvtchnTrigger(
ASSERT3U(Channel->Magic, ==, XENBUS_EVTCHN_CHANNEL_MAGIC);
- Irql = __EvtchnAcquireInterruptLock(Context);
+ Irql = __EvtchnAcquireInterruptLock(Context, Channel);
ASSERT3U(KeGetCurrentIrql(), >=, DISPATCH_LEVEL);
@@ -527,7 +658,7 @@ EvtchnTrigger(
DoneSomething = FALSE;
}
- __EvtchnReleaseInterruptLock(Context, Irql);
+ __EvtchnReleaseInterruptLock(Context, Channel, Irql);
return DoneSomething;
}
@@ -575,6 +706,8 @@ EvtchnClose(
ASSERT(NT_SUCCESS(status));
}
+ Channel->Interrupt = NULL;
+
Channel->LocalPort = 0;
RtlZeroMemory(&Channel->Parameters, sizeof (XENBUS_EVTCHN_PARAMETERS));
@@ -617,6 +750,8 @@ EvtchnPollCallback(
BOOLEAN DoneSomething;
NTSTATUS status;
+ DoneSomething = FALSE;
+
status = HashTableLookup(Context->Table,
LocalPort,
(PULONG_PTR)&Channel);
@@ -628,10 +763,20 @@ EvtchnPollCallback(
&Context->EvtchnAbi,
LocalPort);
- DoneSomething = FALSE;
goto done;
}
+ if (Context->Affinity != 0) {
+ ULONG Cpu;
+
+ ASSERT3U(KeGetCurrentIrql(), >=, DISPATCH_LEVEL);
+ Cpu = KeGetCurrentProcessorNumber();
+
+ // Only handle events on the correct CPU
+ if (Channel->Interrupt != Context->LatchedInterrupt[Cpu])
+ goto done;
+ }
+
if (Channel->Mask)
XENBUS_EVTCHN_ABI(PortMask,
&Context->EvtchnAbi,
@@ -767,8 +912,10 @@ EvtchnSuspendCallbackLate(
status = EvtchnAbiAcquire(Context);
ASSERT(NT_SUCCESS(status));
- if (Context->Enabled)
+ if (Context->Enabled) {
+ EvtchnInterruptDisable(Context);
EvtchnInterruptEnable(Context);
+ }
}
static VOID
@@ -862,6 +1009,7 @@ EvtchnAcquire(
PXENBUS_EVTCHN_CONTEXT Context = Interface->Context;
PXENBUS_FDO Fdo = Context->Fdo;
KIRQL Irql;
+ LONG Cpu;
NTSTATUS status;
KeAcquireSpinLock(&Context->Lock, &Irql);
@@ -919,10 +1067,24 @@ EvtchnAcquire(
0,
EvtchnInterruptCallback,
Context,
- &Context->Interrupt);
+ &Context->LevelSensitiveInterrupt);
if (!NT_SUCCESS(status))
goto fail8;
+ Cpu = 0;
+ while (Cpu < KeNumberProcessors) {
+ status = FdoAllocateInterrupt(Fdo,
+ Latched,
+ Cpu,
+ EvtchnInterruptCallback,
+ Context,
+ &Context->LatchedInterrupt[Cpu]);
+ if (!NT_SUCCESS(status))
+ goto fail9;
+
+ Cpu++;
+ }
+
Trace("<====\n");
done:
@@ -930,6 +1092,17 @@ done:
return STATUS_SUCCESS;
+fail9:
+ Error("fail9\n");
+
+ while (--Cpu >= 0) {
+ FdoFreeInterrupt(Fdo, Context->LatchedInterrupt[Cpu]);
+ Context->LatchedInterrupt[Cpu] = NULL;
+ }
+
+ FdoFreeInterrupt(Fdo, Context->LevelSensitiveInterrupt);
+ Context->LevelSensitiveInterrupt = NULL;
+
fail8:
Error("fail8\n");
@@ -990,7 +1163,9 @@ EvtchnRelease(
)
{
PXENBUS_EVTCHN_CONTEXT Context = Interface->Context;
+ PXENBUS_FDO Fdo = Context->Fdo;
KIRQL Irql;
+ LONG Cpu;
KeAcquireSpinLock(&Context->Lock, &Irql);
@@ -1002,8 +1177,14 @@ EvtchnRelease(
if (!IsListEmpty(&Context->List))
BUG("OUTSTANDING EVENT CHANNELS");
- FdoFreeInterrupt(Context->Fdo, Context->Interrupt);
- Context->Interrupt = NULL;
+ Cpu = KeNumberProcessors;
+ while (--Cpu >= 0) {
+ FdoFreeInterrupt(Fdo, Context->LatchedInterrupt[Cpu]);
+ Context->LatchedInterrupt[Cpu] = NULL;
+ }
+
+ FdoFreeInterrupt(Fdo, Context->LevelSensitiveInterrupt);
+ Context->LevelSensitiveInterrupt = NULL;
EvtchnAbiRelease(Context);
@@ -1046,6 +1227,19 @@ static struct _XENBUS_EVTCHN_INTERFACE_V1
EvtchnInterfaceVersion1 = {
EvtchnClose
};
+static struct _XENBUS_EVTCHN_INTERFACE_V2 EvtchnInterfaceVersion2 = {
+ { sizeof (struct _XENBUS_EVTCHN_INTERFACE_V2), 2, NULL, NULL, NULL },
+ EvtchnAcquire,
+ EvtchnRelease,
+ EvtchnOpen,
+ EvtchnBind,
+ EvtchnUnmask,
+ EvtchnSend,
+ EvtchnTrigger,
+ EvtchnGetPort,
+ EvtchnClose
+};
+
NTSTATUS
EvtchnInitialize(
IN PXENBUS_FDO Fdo,
@@ -1171,6 +1365,23 @@ EvtchnGetInterface(
status = STATUS_SUCCESS;
break;
}
+ case 2: {
+ struct _XENBUS_EVTCHN_INTERFACE_V2 *EvtchnInterface;
+
+ EvtchnInterface = (struct _XENBUS_EVTCHN_INTERFACE_V2 *)Interface;
+
+ status = STATUS_BUFFER_OVERFLOW;
+ if (Size < sizeof (struct _XENBUS_EVTCHN_INTERFACE_V2))
+ break;
+
+ *EvtchnInterface = EvtchnInterfaceVersion2;
+
+ ASSERT3U(Interface->Version, ==, Version);
+ Interface->Context = Context;
+
+ status = STATUS_SUCCESS;
+ break;
+ }
default:
status = STATUS_NOT_SUPPORTED;
break;
diff --git a/src/xenbus/store.c b/src/xenbus/store.c
index 22c2a12..04fce3a 100644
--- a/src/xenbus/store.c
+++ b/src/xenbus/store.c
@@ -1848,6 +1848,15 @@ StoreEnable(
FALSE);
ASSERT(Context->Channel != NULL);
+ //
+ // Attempt to use some CPU other than 0 for events from
+ // xenstored.
+ //
+ (VOID) XENBUS_EVTCHN(Bind,
+ &Context->EvtchnInterface,
+ Context->Channel,
+ KeNumberProcessors - 1);
+
Pending = XENBUS_EVTCHN(Unmask,
&Context->EvtchnInterface,
Context->Channel,
--
2.1.1
_______________________________________________
win-pv-devel mailing list
win-pv-devel@xxxxxxxxxxxxxxxxxxxx
http://lists.xenproject.org/cgi-bin/mailman/listinfo/win-pv-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |