|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [win-pv-devel] [PATCH 01/10] Rework transmitter packet code to use an internal cache of objects
Use XENVIF_TRANSMITTER_PACKET_V2 for queueing packets, with a
translation on entry and exit through the interface (for v1)
Renames XENVIF_TRANSMITTER_PACKET to XENVIF_TRANSMITTER_PACKET_V1
XENVIF_TRANSMITTER_PACKET_V2 is allocated from a cache and not
included inline in the miniport reserved area of a NET_BUFFER, like
XENVIF_TRANSMITTER_PACKET_V1.
Signed-off-by: Owen Smith <owen.smith@xxxxxxxxxx>
---
include/vif_interface.h | 37 ++--
src/xenvif/transmitter.c | 477 ++++++++++++++++++++++++++++++++---------------
src/xenvif/transmitter.h | 6 +-
src/xenvif/vif.c | 12 +-
src/xenvif/vif.h | 4 +-
5 files changed, 368 insertions(+), 168 deletions(-)
diff --git a/include/vif_interface.h b/include/vif_interface.h
index 498ed8f..8bde731 100644
--- a/include/vif_interface.h
+++ b/include/vif_interface.h
@@ -257,7 +257,7 @@ struct _XENVIF_TRANSMITTER_PACKET_V1 {
};
};
-typedef struct _XENVIF_TRANSMITTER_PACKET_V1 XENVIF_TRANSMITTER_PACKET,
*PXENVIF_TRANSMITTER_PACKET;
+typedef struct _XENVIF_TRANSMITTER_PACKET_V1 XENVIF_TRANSMITTER_PACKET_V1,
*PXENVIF_TRANSMITTER_PACKET_V1;
#pragma warning(pop)
@@ -265,9 +265,24 @@ typedef struct _XENVIF_TRANSMITTER_PACKET_V1
XENVIF_TRANSMITTER_PACKET, *PXENVIF
C_ASSERT(sizeof (struct _XENVIF_TRANSMITTER_PACKET_V1) <= (3 * sizeof
(PVOID)));
+/*! \struct _XENVIF_TRANSMITTER_PACKER_V2
+ \brief Transmit-side packet structure
+*/
+struct _XENVIF_TRANSMITTER_PACKET_V2 {
+ LIST_ENTRY ListEntry;
+ PMDL Mdl;
+ ULONG Offset;
+ ULONG Length;
+ PVOID Cookie;
+ XENVIF_TRANSMITTER_PACKET_SEND_INFO Send;
+ XENVIF_TRANSMITTER_PACKET_COMPLETION_INFO Completion;
+};
+
+typedef struct _XENVIF_TRANSMITTER_PACKET_V2 XENVIF_TRANSMITTER_PACKET_V2,
*PXENVIF_TRANSMITTER_PACKET_V2;
+
/*! \enum _XENVIF_TRANSMITTER_PACKET_OFFSET
\brief Offsets of packet metadata relative to
- XENVIF_TRANSMITTER_PACKET pointer
+ XENVIF_TRANSMITTER_PACKET_V1 pointer
Because the transmit side packet structure is limited to 3 pointer
types in size, not all information about the packet can be passed in
@@ -380,10 +395,10 @@ typedef VOID
\param ... Additional paramaters required by \a Type
\b XENVIF_TRANSMITTER_RETURN_PACKETS:
- \param Head The head of a chain of XENVIF_TRANSMITTER_PACKET
+ \param Head The head of a chain of XENVIF_TRANSMITTER_PACKET_V1
\b XENVIF_RECEIVER_QUEUE_PACKETS:
- \param List List of XENVIF_TRANSMITTER_PACKET
+ \param List List of XENVIF_RECEIVER_PACKET
\b XENVIF_MAC_STATE_CHANGE:
No additional arguments
@@ -459,7 +474,7 @@ typedef VOID
/*! \typedef XENVIF_VIF_TRANSMITTER_SET_PACKET_OFFSET
\brief Set byte offset of packet information relative to
- XENVIF_TRANSMITTER_PACKET pointer.
+ XENVIF_TRANSMITTER_PACKET_V1 pointer.
See \ref _XENVIF_TRANSMITTER_PACKET_OFFSET.
@@ -474,16 +489,16 @@ typedef NTSTATUS
IN LONG_PTR Value
);
-/*! \typedef XENVIF_VIF_TRANSMITTER_QUEUE_PACKETS
+/*! \typedef XENVIF_VIF_TRANSMITTER_QUEUE_PACKETS_V1
\brief Queue transmit side packets at the provider
\param Interface The interface header
- \param Head The head of a chain of XENVIF_TRANSMITTER_PACKET
+ \param Head The head of a chain of XENVIF_TRANSMITTER_PACKET_V1
*/
typedef NTSTATUS
-(*XENVIF_VIF_TRANSMITTER_QUEUE_PACKETS)(
- IN PINTERFACE Interface,
- IN PXENVIF_TRANSMITTER_PACKET Head
+(*XENVIF_VIF_TRANSMITTER_QUEUE_PACKETS_V1)(
+ IN PINTERFACE Interface,
+ IN PXENVIF_TRANSMITTER_PACKET_V1 Head
);
/*! \typedef XENVIF_VIF_TRANSMITTER_QUERY_OFFLOAD_OPTIONS
@@ -694,7 +709,7 @@ struct _XENVIF_VIF_INTERFACE_V1 {
XENVIF_VIF_RECEIVER_SET_OFFLOAD_OPTIONS ReceiverSetOffloadOptions;
XENVIF_VIF_RECEIVER_QUERY_RING_SIZE ReceiverQueryRingSize;
XENVIF_VIF_TRANSMITTER_SET_PACKET_OFFSET TransmitterSetPacketOffset;
- XENVIF_VIF_TRANSMITTER_QUEUE_PACKETS TransmitterQueuePackets;
+ XENVIF_VIF_TRANSMITTER_QUEUE_PACKETS_V1 TransmitterQueuePackets;
XENVIF_VIF_TRANSMITTER_QUERY_OFFLOAD_OPTIONS
TransmitterQueryOffloadOptions;
XENVIF_VIF_TRANSMITTER_QUERY_LARGE_PACKET_SIZE
TransmitterQueryLargePacketSize;
XENVIF_VIF_TRANSMITTER_QUERY_RING_SIZE TransmitterQueryRingSize;
diff --git a/src/xenvif/transmitter.c b/src/xenvif/transmitter.c
index 8125a5a..38e4cb2 100644
--- a/src/xenvif/transmitter.c
+++ b/src/xenvif/transmitter.c
@@ -88,7 +88,7 @@ typedef struct _XENVIF_TRANSMITTER_FRAGMENT {
#define XENVIF_TRANSMITTER_MAXIMUM_FRAGMENT_ID 0x03FF
typedef struct _XENVIF_TRANSMITTER_STATE {
- PXENVIF_TRANSMITTER_PACKET Packet;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
XENVIF_TRANSMITTER_PACKET_SEND_INFO Send;
PUCHAR StartVa;
XENVIF_PACKET_INFO Info;
@@ -100,8 +100,8 @@ typedef struct _XENVIF_TRANSMITTER_STATE {
#define XENVIF_TRANSMITTER_RING_SIZE (__CONST_RING_SIZE(netif_tx, PAGE_SIZE))
typedef struct _XENVIF_TRANSMITTER_PACKET_LIST {
- PXENVIF_TRANSMITTER_PACKET HeadPacket;
- PXENVIF_TRANSMITTER_PACKET *TailPacket;
+ PLIST_ENTRY HeadEntry;
+ PLIST_ENTRY *TailEntry;
} XENVIF_TRANSMITTER_PACKET_LIST, *PXENVIF_TRANSMITTER_PACKET_LIST;
typedef struct _XENVIF_TRANSMITTER_RING {
@@ -118,7 +118,7 @@ typedef struct _XENVIF_TRANSMITTER_RING {
BOOLEAN Connected;
BOOLEAN Enabled;
BOOLEAN Stopped;
- PXENVIF_TRANSMITTER_PACKET Lock;
+ PLIST_ENTRY Lock;
PKTHREAD LockThread;
XENVIF_TRANSMITTER_PACKET_LIST Queued;
XENVIF_TRANSMITTER_STATE State;
@@ -133,7 +133,7 @@ typedef struct _XENVIF_TRANSMITTER_RING {
ULONG RequestsPushed;
ULONG ResponsesProcessed;
ULONG PacketsSent;
- XENVIF_TRANSMITTER_PACKET_LIST Completed;
+ LIST_ENTRY Completed;
ULONG PacketsCompleted;
PSOCKADDR_INET AddressTable;
ULONG AddressCount;
@@ -146,6 +146,7 @@ struct _XENVIF_TRANSMITTER {
PXENVIF_FRONTEND Frontend;
XENBUS_CACHE_INTERFACE CacheInterface;
XENBUS_RANGE_SET_INTERFACE RangeSetInterface;
+ PXENBUS_CACHE PacketCache;
LIST_ENTRY List;
LONG_PTR Offset[XENVIF_TRANSMITTER_PACKET_OFFSET_COUNT];
ULONG DisableIpVersion4Gso;
@@ -362,7 +363,7 @@ __TransmitterGetFragment(
TRUE);
}
-static FORCEINLINE
+static FORCEINLINE VOID
__TransmitterPutFragment(
IN PXENVIF_TRANSMITTER_RING Ring,
IN PXENVIF_TRANSMITTER_FRAGMENT Fragment
@@ -387,6 +388,67 @@ __TransmitterPutFragment(
TRUE);
}
+static NTSTATUS
+TransmitterPacketCtor(
+ IN PVOID Argument,
+ IN PVOID Object
+ )
+{
+ UNREFERENCED_PARAMETER(Argument);
+ RtlZeroMemory(Object, sizeof(XENVIF_TRANSMITTER_PACKET_V2));
+ return STATUS_SUCCESS;
+}
+
+static VOID
+TransmitterPacketDtor(
+ IN PVOID Argument,
+ IN PVOID Object
+ )
+{
+ UNREFERENCED_PARAMETER(Argument);
+ UNREFERENCED_PARAMETER(Object);
+}
+
+static VOID
+TransmitterPacketLock(
+ IN PVOID Argument
+ )
+{
+ UNREFERENCED_PARAMETER(Argument);
+}
+
+static VOID
+TransmitterPacketUnlock(
+ IN PVOID Argument
+ )
+{
+ UNREFERENCED_PARAMETER(Argument);
+}
+
+static FORCEINLINE PXENVIF_TRANSMITTER_PACKET_V2
+__TransmitterGetPacket(
+ IN PXENVIF_TRANSMITTER Transmitter
+ )
+{
+ return XENBUS_CACHE(Get,
+ &Transmitter->CacheInterface,
+ Transmitter->PacketCache,
+ TRUE);
+}
+
+static FORCEINLINE VOID
+__TransmitterPutPacket(
+ IN PXENVIF_TRANSMITTER Transmitter,
+ IN PXENVIF_TRANSMITTER_PACKET_V2 Packet
+ )
+{
+ XENBUS_CACHE(Put,
+ &Transmitter->CacheInterface,
+ Transmitter->PacketCache,
+ Packet,
+ TRUE);
+}
+
static VOID
TransmitterRingDebugCallback(
IN PVOID Argument,
@@ -523,25 +585,25 @@ fail1:
return FALSE;
}
-#define INCREMENT_PACKET_REFERENCE(_Packet) \
- do { \
- PULONG_PTR Reference = (PULONG_PTR)&(_Packet)->Next; \
- \
- ASSERT(Packet != NULL); \
- (*Reference)++; \
+#define INCREMENT_PACKET_REFERENCE(_Packet) \
+ do { \
+ PULONG_PTR Reference = (PULONG_PTR)&(_Packet)->ListEntry.Flink; \
+ \
+ ASSERT(Packet != NULL); \
+ (*Reference)++; \
} while (FALSE)
-#define DECREMENT_PACKET_REFERENCE(_Packet) \
- do { \
- PULONG_PTR Reference = (PULONG_PTR)&(_Packet)->Next; \
- \
- ASSERT(Packet != NULL); \
- ASSERT(*Reference != 0); \
- --(*Reference); \
+#define DECREMENT_PACKET_REFERENCE(_Packet) \
+ do { \
+ PULONG_PTR Reference = (PULONG_PTR)&(_Packet)->ListEntry.Flink; \
+ \
+ ASSERT(Packet != NULL); \
+ ASSERT(*Reference != 0); \
+ --(*Reference); \
} while (FALSE)
-#define PACKET_REFERENCE(_Packet) \
- (*(PULONG_PTR)&(_Packet)->Next)
+#define PACKET_REFERENCE(_Packet) \
+ (*(PULONG_PTR)&(_Packet)->ListEntry.Flink)
static FORCEINLINE NTSTATUS
__TransmitterRingCopyPayload(
@@ -551,7 +613,7 @@ __TransmitterRingCopyPayload(
PXENVIF_TRANSMITTER Transmitter;
PXENVIF_FRONTEND Frontend;
PXENVIF_TRANSMITTER_STATE State;
- PXENVIF_TRANSMITTER_PACKET Packet;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
XENVIF_PACKET_PAYLOAD Payload;
PXENVIF_TRANSMITTER_FRAGMENT Fragment;
PXENVIF_TRANSMITTER_BUFFER Buffer;
@@ -698,7 +760,7 @@ __TransmitterRingGrantPayload(
PXENVIF_TRANSMITTER Transmitter;
PXENVIF_FRONTEND Frontend;
PXENVIF_TRANSMITTER_STATE State;
- PXENVIF_TRANSMITTER_PACKET Packet;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
PXENVIF_PACKET_PAYLOAD Payload;
PMDL Mdl;
ULONG Offset;
@@ -845,7 +907,7 @@ __TransmitterRingPrepareHeader(
PXENVIF_FRONTEND Frontend;
PXENVIF_MAC Mac;
PXENVIF_TRANSMITTER_STATE State;
- PXENVIF_TRANSMITTER_PACKET Packet;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
PXENVIF_PACKET_PAYLOAD Payload;
PXENVIF_PACKET_INFO Info;
PXENVIF_TRANSMITTER_FRAGMENT Fragment;
@@ -1122,7 +1184,7 @@ __TransmitterRingUnprepareFragments(
while (State->Count != 0) {
PLIST_ENTRY ListEntry;
PXENVIF_TRANSMITTER_FRAGMENT Fragment;
- PXENVIF_TRANSMITTER_PACKET Packet;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
--State->Count;
@@ -1178,19 +1240,10 @@ __TransmitterRingUnprepareFragments(
static FORCEINLINE NTSTATUS
__TransmitterRingPreparePacket(
- IN PXENVIF_TRANSMITTER_RING Ring,
- IN PXENVIF_TRANSMITTER_PACKET Packet
+ IN PXENVIF_TRANSMITTER_RING Ring,
+ IN PXENVIF_TRANSMITTER_PACKET_V2 Packet
)
{
-#define OFFSET_EXISTS(_Ring, _Packet, _Type)
\
- ((_Ring)->Transmitter->Offset[XENVIF_TRANSMITTER_PACKET_ ## _Type ##
_OFFSET] != 0)
-
-#define OFFSET(_Ring, _Packet, _Type)
\
- ((OFFSET_EXISTS(_Ring, _Packet, _Type)) ?
\
- (PVOID)((PUCHAR)(_Packet) +
\
- (_Ring)->Transmitter->Offset[XENVIF_TRANSMITTER_PACKET_ ##
_Type ## _OFFSET]) : \
- NULL)
-
PXENVIF_TRANSMITTER Transmitter;
PXENVIF_TRANSMITTER_STATE State;
PXENVIF_PACKET_PAYLOAD Payload;
@@ -1198,7 +1251,7 @@ __TransmitterRingPreparePacket(
NTSTATUS status;
ASSERT(IsZeroMemory(&Ring->State, sizeof (XENVIF_TRANSMITTER_STATE)));
- ASSERT3P(Packet->Next, ==, NULL);
+ ASSERT3P(Packet->ListEntry.Flink, ==, NULL);
Transmitter = Ring->Transmitter;
@@ -1211,16 +1264,9 @@ __TransmitterRingPreparePacket(
Payload = &State->Payload;
- ASSERT(OFFSET_EXISTS(Ring, Packet, MDL));
- Payload->Mdl = *(PMDL *)OFFSET(Ring, Packet, MDL);
-
- if (OFFSET_EXISTS(Ring, Packet, OFFSET))
- Payload->Offset = *(PULONG)OFFSET(Ring, Packet, OFFSET);
- else
- Payload->Offset = 0;
-
- ASSERT(OFFSET_EXISTS(Ring, Packet, LENGTH));
- Payload->Length = *(PULONG)OFFSET(Ring, Packet, LENGTH);
+ Payload->Mdl = Packet->Mdl;
+ Payload->Offset = Packet->Offset;
+ Payload->Length = Packet->Length;
InitializeListHead(&State->List);
ASSERT3U(State->Count, ==, 0);
@@ -1325,18 +1371,15 @@ fail1:
ASSERT(IsZeroMemory(&Ring->State, sizeof (XENVIF_TRANSMITTER_STATE)));
return status;
-
-#undef OFFSET
-#undef OFFSET_EXISTS
}
-static FORCEINLINE PXENVIF_TRANSMITTER_PACKET
+static FORCEINLINE PXENVIF_TRANSMITTER_PACKET_V2
__TransmitterRingUnpreparePacket(
IN PXENVIF_TRANSMITTER_RING Ring
)
{
PXENVIF_TRANSMITTER_STATE State;
- PXENVIF_TRANSMITTER_PACKET Packet;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
State = &Ring->State;
Packet = State->Packet;
@@ -1695,7 +1738,7 @@ __TransmitterRingPostFragments(
PXENVIF_TRANSMITTER Transmitter;
PXENVIF_FRONTEND Frontend;
PXENVIF_TRANSMITTER_STATE State;
- PXENVIF_TRANSMITTER_PACKET Packet;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
PXENVIF_PACKET_PAYLOAD Payload;
RING_IDX req_prod;
RING_IDX rsp_cons;
@@ -1920,8 +1963,8 @@ __TransmitterRingFakeResponses(
static FORCEINLINE VOID
__TransmitterRingCompletePacket(
- IN PXENVIF_TRANSMITTER_RING Ring,
- IN PXENVIF_TRANSMITTER_PACKET Packet
+ IN PXENVIF_TRANSMITTER_RING Ring,
+ IN PXENVIF_TRANSMITTER_PACKET_V2 Packet
)
{
PXENVIF_TRANSMITTER Transmitter;
@@ -1980,9 +2023,9 @@ __TransmitterRingCompletePacket(
}
}
- *Ring->Completed.TailPacket = Packet;
- ASSERT3P(Packet->Next, ==, NULL);
- Ring->Completed.TailPacket = &Packet->Next;
+ ASSERT3P(Packet->ListEntry.Flink, ==, NULL);
+ ASSERT3P(Packet->ListEntry.Blink, ==, NULL);
+ InsertTailList(&Ring->Completed, &Packet->ListEntry);
Ring->PacketsCompleted++;
}
@@ -2019,7 +2062,7 @@ TransmitterRingPoll(
netif_tx_response_t *rsp;
uint16_t id;
PXENVIF_TRANSMITTER_FRAGMENT Fragment;
- PXENVIF_TRANSMITTER_PACKET Packet;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
rsp = RING_GET_RESPONSE(&Ring->Front, rsp_cons);
rsp_cons++;
@@ -2164,30 +2207,30 @@ __TransmitterRingPushRequests(
static FORCEINLINE ULONG
__TransmitterReversePacketList(
- IN PXENVIF_TRANSMITTER_PACKET *Packet
+ IN PLIST_ENTRY *Entry
)
{
- PXENVIF_TRANSMITTER_PACKET HeadPacket;
+ PLIST_ENTRY HeadEntry;
ULONG Count;
- HeadPacket = NULL;
+ HeadEntry = NULL;
Count = 0;
- while (*Packet != NULL) {
- PXENVIF_TRANSMITTER_PACKET Next;
+ while (*Entry != NULL) {
+ PLIST_ENTRY Next;
- ASSERT(((ULONG_PTR)*Packet & XENVIF_TRANSMITTER_LOCK_BIT) == 0);
+ ASSERT(((ULONG_PTR)*Entry & XENVIF_TRANSMITTER_LOCK_BIT) == 0);
- Next = (*Packet)->Next;
+ Next = (*Entry)->Flink;
- (*Packet)->Next = HeadPacket;
- HeadPacket = *Packet;
+ (*Entry)->Flink = HeadEntry;
+ HeadEntry = *Entry;
- *Packet = Next;
+ *Entry = Next;
Count++;
}
- *Packet = HeadPacket;
+ *Entry = HeadEntry;
return Count;
}
@@ -2199,8 +2242,8 @@ TransmitterRingSwizzle(
{
ULONG_PTR Old;
ULONG_PTR New;
- PXENVIF_TRANSMITTER_PACKET HeadPacket;
- PXENVIF_TRANSMITTER_PACKET *TailPacket;
+ PLIST_ENTRY HeadEntry;
+ PLIST_ENTRY *TailEntry;
ULONG Count;
ASSERT3P(Ring->LockThread, ==, KeGetCurrentThread());
@@ -2209,21 +2252,21 @@ TransmitterRingSwizzle(
Old = (ULONG_PTR)InterlockedExchangePointer(&Ring->Lock, (PVOID)New);
ASSERT(Old & XENVIF_TRANSMITTER_LOCK_BIT);
- HeadPacket = (PVOID)(Old & ~XENVIF_TRANSMITTER_LOCK_BIT);
+ HeadEntry = (PVOID)(Old & ~XENVIF_TRANSMITTER_LOCK_BIT);
- if (HeadPacket == NULL)
+ if (HeadEntry == NULL)
return;
// Packets are held in the atomic packet list in reverse order
// so that the most recent is always head of the list. This is
// necessary to allow addition to the list to be done atomically.
- TailPacket = &HeadPacket->Next;
- Count = __TransmitterReversePacketList(&HeadPacket);
- ASSERT3P(*TailPacket, ==, NULL);
+ TailEntry = &HeadEntry->Flink;
+ Count = __TransmitterReversePacketList(&HeadEntry);
+ ASSERT3P(*TailEntry, ==, NULL);
- *(Ring->Queued.TailPacket) = HeadPacket;
- Ring->Queued.TailPacket = TailPacket;
+ *(Ring->Queued.TailEntry) = HeadEntry;
+ Ring->Queued.TailEntry = TailEntry;
Ring->PacketsQueued += Count;
}
@@ -2240,8 +2283,9 @@ TransmitterRingSchedule(
State = &Ring->State;
for (;;) {
- PXENVIF_TRANSMITTER_PACKET Packet;
- NTSTATUS status;
+ PLIST_ENTRY ListEntry;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
+ NTSTATUS status;
if (State->Count != 0) {
status = __TransmitterRingPostFragments(Ring);
@@ -2290,19 +2334,20 @@ TransmitterRingSchedule(
continue;
}
- Packet = Ring->Queued.HeadPacket;
-
- if (Packet == NULL)
+ ListEntry = Ring->Queued.HeadEntry;
+ if (ListEntry == NULL)
break;
- if (Packet->Next == NULL) {
- Ring->Queued.HeadPacket = NULL;
- Ring->Queued.TailPacket = &Ring->Queued.HeadPacket;
+ if (ListEntry->Flink == NULL) {
+ Ring->Queued.HeadEntry = NULL;
+ Ring->Queued.TailEntry = &Ring->Queued.HeadEntry;
} else {
- Ring->Queued.HeadPacket = Packet->Next;
- Packet->Next = NULL;
+ Ring->Queued.HeadEntry = ListEntry->Flink;
+ ListEntry->Flink = NULL;
}
+ Packet = CONTAINING_RECORD(ListEntry, XENVIF_TRANSMITTER_PACKET_V2,
ListEntry);
+
status = __TransmitterRingPreparePacket(Ring, Packet);
if (!NT_SUCCESS(status)) {
PXENVIF_TRANSMITTER Transmitter;
@@ -2330,11 +2375,46 @@ TransmitterRingSchedule(
ASSERT3U(Ring->PacketsPrepared, ==, Ring->PacketsCopied +
Ring->PacketsGranted + Ring->PacketsFaked);
}
- ASSERT(IMPLY(Ring->Queued.HeadPacket == NULL, Ring->Queued.TailPacket ==
&Ring->Queued.HeadPacket));
+ ASSERT(IMPLY(Ring->Queued.HeadEntry == NULL, Ring->Queued.TailEntry ==
&Ring->Queued.HeadEntry));
__TransmitterRingPushRequests(Ring);
}
+static VOID
+TransmitterReturnPackets(
+ IN PXENVIF_TRANSMITTER Transmitter,
+ IN PLIST_ENTRY List
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ PXENVIF_TRANSMITTER_PACKET_V1 HeadPacket;
+ PXENVIF_TRANSMITTER_PACKET_V1 NextPacket;
+
+ HeadPacket = NULL;
+ NextPacket = NULL;
+ while (!IsListEmpty(List)) {
+ PLIST_ENTRY ListEntry;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
+
+ ListEntry = RemoveTailList(List);
+ ASSERT3P(ListEntry, !=, List);
+
+ Packet = CONTAINING_RECORD(ListEntry, XENVIF_TRANSMITTER_PACKET_V2,
ListEntry);
+
+ HeadPacket = Packet->Cookie;
+ HeadPacket->Next = NextPacket;
+ HeadPacket->Completion = Packet->Completion;
+
+ __TransmitterPutPacket(Transmitter, Packet);
+ NextPacket = HeadPacket;
+ }
+
+ Frontend = Transmitter->Frontend;
+
+ VifTransmitterReturnPackets(PdoGetVifContext(FrontendGetPdo(Frontend)),
+ HeadPacket);
+}
+
static FORCEINLINE BOOLEAN
__drv_requiresIRQL(DISPATCH_LEVEL)
__TransmitterRingTryAcquireLock(
@@ -2427,16 +2507,29 @@ __TransmitterRingTryReleaseLock(
}
static FORCEINLINE VOID
+__AppendTailList(
+ IN OUT PLIST_ENTRY List,
+ IN OUT PLIST_ENTRY Apendee
+ )
+{
+ PLIST_ENTRY HeadEntry = Apendee->Flink;
+
+ if (!IsListEmpty(Apendee)) {
+ RemoveEntryList(Apendee);
+ InitializeListHead(Apendee);
+ AppendTailList(List, HeadEntry);
+ }
+}
+
+static FORCEINLINE VOID
__drv_requiresIRQL(DISPATCH_LEVEL)
__TransmitterRingReleaseLock(
IN PXENVIF_TRANSMITTER_RING Ring
)
{
- PXENVIF_TRANSMITTER_PACKET HeadPacket;
- PXENVIF_TRANSMITTER_PACKET *TailPacket;
-
- HeadPacket = NULL;
- TailPacket = &HeadPacket;
+ LIST_ENTRY List;
+
+ InitializeListHead(&List);
ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
@@ -2449,22 +2542,11 @@ __TransmitterRingReleaseLock(
TransmitterRingSwizzle(Ring);
TransmitterRingSchedule(Ring);
- *TailPacket = Ring->Completed.HeadPacket;
- TailPacket = Ring->Completed.TailPacket;
-
- Ring->Completed.HeadPacket = NULL;
- Ring->Completed.TailPacket = &Ring->Completed.HeadPacket;
+ __AppendTailList(&List, &Ring->Completed);
} while (!__TransmitterRingTryReleaseLock(Ring));
- if (HeadPacket != NULL) {
- PXENVIF_TRANSMITTER Transmitter;
- PXENVIF_FRONTEND Frontend;
-
- Transmitter = Ring->Transmitter;
- Frontend = Transmitter->Frontend;
-
- VifTransmitterReturnPackets(PdoGetVifContext(FrontendGetPdo(Frontend)),
- HeadPacket);
+ if (!IsListEmpty(&List)) {
+ TransmitterReturnPackets(Ring->Transmitter, &List);
}
}
@@ -2624,8 +2706,8 @@ __TransmitterRingInitialize(
(*Ring)->Transmitter = Transmitter;
(*Ring)->Index = Index;
- (*Ring)->Queued.TailPacket = &(*Ring)->Queued.HeadPacket;
- (*Ring)->Completed.TailPacket = &(*Ring)->Completed.HeadPacket;
+ (*Ring)->Queued.TailEntry = &(*Ring)->Queued.HeadEntry;
+ InitializeListHead(&(*Ring)->Completed);
status = RtlStringCbPrintfA(Name,
sizeof (Name),
@@ -2756,8 +2838,8 @@ fail3:
fail2:
Error("fail2\n");
- (*Ring)->Queued.TailPacket = NULL;
- (*Ring)->Completed.TailPacket = NULL;
+ (*Ring)->Queued.TailEntry = NULL;
+ RtlZeroMemory(&(*Ring)->Completed, sizeof(LIST_ENTRY));
(*Ring)->Index = 0;
(*Ring)->Transmitter = NULL;
@@ -2910,7 +2992,7 @@ __TransmitterRingDisable(
{
PXENVIF_TRANSMITTER Transmitter;
PXENVIF_FRONTEND Frontend;
- PXENVIF_TRANSMITTER_PACKET Packet;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
PCHAR Buffer;
XenbusState State;
ULONG Attempt;
@@ -2929,14 +3011,15 @@ __TransmitterRingDisable(
// Put any packet back on the head of the queue
if (Packet != NULL) {
- ASSERT3P(Packet->Next, ==, NULL);
+ ASSERT3P(Packet->ListEntry.Flink, ==, NULL);
+ ASSERT3P(Packet->ListEntry.Blink, ==, NULL);
- Packet->Next = Ring->Queued.HeadPacket;
+ Packet->ListEntry.Flink = Ring->Queued.HeadEntry;
- if (Ring->Queued.TailPacket == &Ring->Queued.HeadPacket)
- Ring->Queued.TailPacket = &Packet->Next;
+ if (Ring->Queued.TailEntry == &Ring->Queued.HeadEntry)
+ Ring->Queued.TailEntry = &Packet->ListEntry.Flink;
- Ring->Queued.HeadPacket = Packet;
+ Ring->Queued.HeadEntry = &Packet->ListEntry;
}
Ring->AddressIndex = 0;
@@ -3076,11 +3159,11 @@ __TransmitterRingTeardown(
Ring->BufferCache);
Ring->BufferCache = NULL;
- ASSERT3P(Ring->Queued.TailPacket, ==, &Ring->Queued.HeadPacket);
- Ring->Queued.TailPacket = NULL;
+ ASSERT3P(Ring->Queued.TailEntry, ==, &Ring->Queued.HeadEntry);
+ Ring->Queued.TailEntry = NULL;
- ASSERT3P(Ring->Completed.TailPacket, ==, &Ring->Completed.HeadPacket);
- Ring->Completed.TailPacket = NULL;
+ ASSERT(IsListEmpty(&Ring->Completed));
+ RtlZeroMemory(&Ring->Completed, sizeof(LIST_ENTRY));
Ring->Index = 0;
Ring->Transmitter = NULL;
@@ -3092,24 +3175,40 @@ __TransmitterRingTeardown(
static FORCEINLINE VOID
__TransmitterRingQueuePackets(
IN PXENVIF_TRANSMITTER_RING Ring,
- IN PXENVIF_TRANSMITTER_PACKET HeadPacket
+ IN PLIST_ENTRY List
)
{
- PXENVIF_TRANSMITTER_PACKET *TailPacket;
+ PLIST_ENTRY ListEntry;
+ PLIST_ENTRY HeadEntry;
+ PLIST_ENTRY *TailEntry;
ULONG_PTR Old;
ULONG_PTR LockBit;
ULONG_PTR New;
- TailPacket = &HeadPacket->Next;
- (VOID) __TransmitterReversePacketList(&HeadPacket);
- ASSERT3P(*TailPacket, ==, NULL);
+ ListEntry = RemoveTailList(List);
+ ASSERT3P(ListEntry, !=, List);
+
+ ListEntry->Flink = NULL;
+ ListEntry->Blink = NULL;
+ HeadEntry = ListEntry;
+
+ TailEntry = &ListEntry->Flink;
+ while (!IsListEmpty(List)) {
+ ListEntry = RemoveTailList(List);
+ ASSERT3P(ListEntry, !=, List);
+
+ ListEntry->Flink = HeadEntry;
+ ListEntry->Blink = NULL;
+ HeadEntry = ListEntry;
+ }
+ ASSERT3P(*TailEntry, ==, NULL);
do {
Old = (ULONG_PTR)Ring->Lock;
LockBit = Old & XENVIF_TRANSMITTER_LOCK_BIT;
- *TailPacket = (PVOID)(Old & ~XENVIF_TRANSMITTER_LOCK_BIT);
- New = (ULONG_PTR)HeadPacket;
+ *TailEntry = (PVOID)(Old & ~XENVIF_TRANSMITTER_LOCK_BIT);
+ New = (ULONG_PTR)HeadEntry;
ASSERT((New & XENVIF_TRANSMITTER_LOCK_BIT) == 0);
New |= LockBit;
} while ((ULONG_PTR)InterlockedCompareExchangePointer(&Ring->Lock,
(PVOID)New, (PVOID)Old) != Old);
@@ -3128,33 +3227,35 @@ __TransmitterRingAbortPackets(
IN PXENVIF_TRANSMITTER_RING Ring
)
{
- PXENVIF_TRANSMITTER_PACKET Packet;
+ PLIST_ENTRY ListEntry;
__TransmitterRingAcquireLock(Ring);
TransmitterRingSwizzle(Ring);
- Packet = Ring->Queued.HeadPacket;
+ ListEntry = Ring->Queued.HeadEntry;
- Ring->Queued.HeadPacket = NULL;
- Ring->Queued.TailPacket = &Ring->Queued.HeadPacket;
+ Ring->Queued.HeadEntry = NULL;
+ Ring->Queued.TailEntry = &Ring->Queued.HeadEntry;
- while (Packet != NULL) {
- PXENVIF_TRANSMITTER_PACKET Next;
+ while (ListEntry != NULL) {
+ PLIST_ENTRY Next;
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
- Next = Packet->Next;
- Packet->Next = NULL;
+ Next = ListEntry->Flink;
+ ListEntry->Flink = NULL;
// Fake that we prapared and sent this packet
Ring->PacketsPrepared++;
Ring->PacketsSent++;
Ring->PacketsFaked++;
+ Packet = CONTAINING_RECORD(ListEntry, XENVIF_TRANSMITTER_PACKET_V2,
ListEntry);
Packet->Completion.Status = XENVIF_TRANSMITTER_PACKET_DROPPED;
__TransmitterRingCompletePacket(Ring, Packet);
- Packet = Next;
+ ListEntry = Next;
}
ASSERT3U(Ring->PacketsSent, ==, Ring->PacketsPrepared -
Ring->PacketsUnprepared);
@@ -3264,13 +3365,27 @@ TransmitterInitialize(
if (!NT_SUCCESS(status))
goto fail3;
+ status = XENBUS_CACHE(Create,
+ &(*Transmitter)->CacheInterface,
+ "packet_cache",
+ sizeof(XENVIF_TRANSMITTER_PACKET_V2),
+ 0,
+ TransmitterPacketCtor,
+ TransmitterPacketDtor,
+ TransmitterPacketLock,
+ TransmitterPacketUnlock,
+ *Transmitter,
+ &(*Transmitter)->PacketCache);
+ if (!NT_SUCCESS(status))
+ goto fail4;
+
Index = 0;
while (Index < Count) {
PXENVIF_TRANSMITTER_RING Ring;
status = __TransmitterRingInitialize(*Transmitter, Index, &Ring);
if (!NT_SUCCESS(status))
- goto fail4;
+ goto fail5;
InsertTailList(&(*Transmitter)->List, &Ring->ListEntry);
Index++;
@@ -3278,8 +3393,8 @@ TransmitterInitialize(
return STATUS_SUCCESS;
-fail4:
- Error("fail4\n");
+fail5:
+ Error("fail5\n");
while (!IsListEmpty(&(*Transmitter)->List)) {
PLIST_ENTRY ListEntry;
@@ -3297,6 +3412,14 @@ fail4:
}
ASSERT3U(Index, ==, 0);
+ XENBUS_CACHE(Destroy,
+ &(*Transmitter)->CacheInterface,
+ (*Transmitter)->PacketCache);
+ (*Transmitter)->PacketCache = NULL;
+
+fail4:
+ Error("fail4\n");
+
XENBUS_CACHE(Release, &(*Transmitter)->CacheInterface);
fail3:
@@ -3530,6 +3653,11 @@ TransmitterTeardown(
__TransmitterRingTeardown(Ring);
}
+ XENBUS_CACHE(Destroy,
+ &Transmitter->CacheInterface,
+ Transmitter->PacketCache);
+ Transmitter->PacketCache = NULL;
+
XENBUS_CACHE(Release, &Transmitter->CacheInterface);
XENBUS_RANGE_SET(Release, &Transmitter->RangeSetInterface);
@@ -3620,20 +3748,77 @@ fail1:
}
VOID
-TransmitterQueuePackets(
- IN PXENVIF_TRANSMITTER Transmitter,
- IN PXENVIF_TRANSMITTER_PACKET HeadPacket
+TransmitterQueuePacketsV1(
+ IN PXENVIF_TRANSMITTER Transmitter,
+ IN PXENVIF_TRANSMITTER_PACKET_V1 HeadPacket
)
{
+#define OFFSET_EXISTS(_Transmitter, _Packet, _Type)
\
+ ((_Transmitter)->Offset[XENVIF_TRANSMITTER_PACKET_ ## _Type ## _OFFSET] !=
0)
+
+#define OFFSET(_Transmitter, _Packet, _Type)
\
+ ((OFFSET_EXISTS(_Transmitter, _Packet, _Type)) ?
\
+ (PVOID)((PUCHAR)(_Packet) +
\
+ (_Transmitter)->Offset[XENVIF_TRANSMITTER_PACKET_ ## _Type ##
_OFFSET]) : \
+ NULL)
+
+ LIST_ENTRY List;
PLIST_ENTRY ListEntry;
PXENVIF_TRANSMITTER_RING Ring;
+ InitializeListHead(&List);
+ while (HeadPacket != NULL) {
+ PXENVIF_TRANSMITTER_PACKET_V2 Packet;
+
+ Packet = __TransmitterGetPacket(Transmitter);
+ if (Packet == NULL) {
+ Warning("Out-of-cached-packets\n");
+ break;
+ }
+
+ Packet->Cookie = HeadPacket;
+ RtlCopyMemory(&Packet->Send, &HeadPacket->Send, sizeof(Packet->Send));
+ RtlZeroMemory(&Packet->Completion, sizeof(Packet->Completion));
+
+ ASSERT(OFFSET_EXISTS(Transmitter, HeadPacket, MDL));
+ Packet->Mdl = *(PMDL *)OFFSET(Transmitter, HeadPacket, MDL);
+ if (OFFSET_EXISTS(Transmitter, HeadPacket, OFFSET))
+ Packet->Offset = *(PULONG)OFFSET(Transmitter, HeadPacket, OFFSET);
+ else
+ Packet->Offset = 0;
+ ASSERT(OFFSET_EXISTS(Transmitter, HeadPacket, LENGTH));
+ Packet->Length = *(PULONG)OFFSET(Transmitter, HeadPacket, LENGTH);
+
+ InsertTailList(&List, &Packet->ListEntry);
+ HeadPacket = HeadPacket->Next;
+ }
+
// We need to hash for a ring eventually. Since there is only a
// single ring for now, we just use that.
ListEntry = Transmitter->List.Flink;
Ring = CONTAINING_RECORD(ListEntry, XENVIF_TRANSMITTER_RING, ListEntry);
- __TransmitterRingQueuePackets(Ring, HeadPacket);
+ __TransmitterRingQueuePackets(Ring, &List);
+
+ // if HeadPacket != NULL, errors occured and need returning
+ if (HeadPacket != NULL) {
+ PXENVIF_TRANSMITTER_PACKET_V1 Packet;
+ PXENVIF_FRONTEND Frontend;
+
+ Frontend = Transmitter->Frontend;
+
+ Packet = HeadPacket;
+ while (Packet != NULL) {
+ Packet->Completion.Status = XENVIF_TRANSMITTER_PACKET_DROPPED;
+ Packet = Packet->Next;
+ }
+
+ VifTransmitterReturnPackets(PdoGetVifContext(FrontendGetPdo(Frontend)),
+ HeadPacket);
+ }
+
+#undef OFFSET
+#undef OFFSET_EXISTS
}
VOID
diff --git a/src/xenvif/transmitter.h b/src/xenvif/transmitter.h
index 5ffb590..661eeae 100644
--- a/src/xenvif/transmitter.h
+++ b/src/xenvif/transmitter.h
@@ -107,9 +107,9 @@ TransmitterAdvertiseAddresses(
);
extern VOID
-TransmitterQueuePackets(
- IN PXENVIF_TRANSMITTER Transmitter,
- IN PXENVIF_TRANSMITTER_PACKET HeadPacket
+TransmitterQueuePacketsV1(
+ IN PXENVIF_TRANSMITTER Transmitter,
+ IN PXENVIF_TRANSMITTER_PACKET_V1 HeadPacket
);
extern VOID
diff --git a/src/xenvif/vif.c b/src/xenvif/vif.c
index 97ce84a..d6da258 100644
--- a/src/xenvif/vif.c
+++ b/src/xenvif/vif.c
@@ -276,8 +276,8 @@ VifReceiverReturnPackets(
static NTSTATUS
VifTransmitterQueuePackets(
- IN PINTERFACE Interface,
- IN PXENVIF_TRANSMITTER_PACKET Head
+ IN PINTERFACE Interface,
+ IN PXENVIF_TRANSMITTER_PACKET_V1 Head
)
{
PXENVIF_VIF_CONTEXT Context = Interface->Context;
@@ -289,8 +289,8 @@ VifTransmitterQueuePackets(
if (Context->Enabled == FALSE)
goto fail1;
- TransmitterQueuePackets(FrontendGetTransmitter(Context->Frontend),
- Head);
+ TransmitterQueuePacketsV1(FrontendGetTransmitter(Context->Frontend),
+ Head);
ReleaseMrswLockShared(&Context->Lock);
@@ -784,8 +784,8 @@ VifReceiverQueuePackets(
VOID
VifTransmitterReturnPackets(
- IN PXENVIF_VIF_CONTEXT Context,
- IN PXENVIF_TRANSMITTER_PACKET Head
+ IN PXENVIF_VIF_CONTEXT Context,
+ IN PXENVIF_TRANSMITTER_PACKET_V1 Head
)
{
Context->Callback(Context->Argument,
diff --git a/src/xenvif/vif.h b/src/xenvif/vif.h
index 17a04db..0ef6687 100644
--- a/src/xenvif/vif.h
+++ b/src/xenvif/vif.h
@@ -70,8 +70,8 @@ VifReceiverQueuePackets(
extern VOID
VifTransmitterReturnPackets(
- IN PXENVIF_VIF_CONTEXT Context,
- IN PXENVIF_TRANSMITTER_PACKET Head
+ IN PXENVIF_VIF_CONTEXT Context,
+ IN PXENVIF_TRANSMITTER_PACKET_V1 Head
);
extern PXENVIF_THREAD
--
1.9.4.msysgit.1
_______________________________________________
win-pv-devel mailing list
win-pv-devel@xxxxxxxxxxxxxxxxxxxx
http://lists.xenproject.org/cgi-bin/mailman/listinfo/win-pv-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |