[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [win-pv-devel] [PATCH 6/7] Implement VIF Interface v2
Use VIF v2 structures internally, and add translation layer for VIF v1 calls. Signed-off-by: Owen Smith <owen.smith@xxxxxxxxxx> --- src/xenvif/transmitter.c | 587 +++++++++++++++++++++++++++++++++-------------- src/xenvif/transmitter.h | 14 ++ src/xenvif/vif.c | 128 ++++++++++- src/xenvif/vif.h | 11 + 4 files changed, 564 insertions(+), 176 deletions(-) diff --git a/src/xenvif/transmitter.c b/src/xenvif/transmitter.c index 41bab7c..914e544 100644 --- a/src/xenvif/transmitter.c +++ b/src/xenvif/transmitter.c @@ -89,7 +89,7 @@ typedef struct _XENVIF_TRANSMITTER_FRAGMENT { #define XENVIF_TRANSMITTER_MAXIMUM_FRAGMENT_ID 0x03FF typedef struct _XENVIF_TRANSMITTER_STATE { - PXENVIF_TRANSMITTER_PACKET_V1 Packet; + PXENVIF_TRANSMITTER_PACKET Packet; XENVIF_TRANSMITTER_PACKET_SEND_INFO Send; PUCHAR StartVa; XENVIF_PACKET_INFO Info; @@ -100,11 +100,6 @@ typedef struct _XENVIF_TRANSMITTER_STATE { #define XENVIF_TRANSMITTER_RING_SIZE (__CONST_RING_SIZE(netif_tx, PAGE_SIZE)) -typedef struct _XENVIF_TRANSMITTER_PACKET_LIST { - PXENVIF_TRANSMITTER_PACKET_V1 HeadPacket; - PXENVIF_TRANSMITTER_PACKET_V1 *TailPacket; -} XENVIF_TRANSMITTER_PACKET_LIST, *PXENVIF_TRANSMITTER_PACKET_LIST; - typedef struct _XENVIF_TRANSMITTER_RING { PXENVIF_TRANSMITTER Transmitter; ULONG Index; @@ -125,7 +120,7 @@ typedef struct _XENVIF_TRANSMITTER_RING { BOOLEAN Stopped; PVOID Lock; PKTHREAD LockThread; - XENVIF_TRANSMITTER_PACKET_LIST Queued; + LIST_ENTRY Queued; XENVIF_TRANSMITTER_STATE State; ULONG PacketsQueued; ULONG PacketsGranted; @@ -138,7 +133,7 @@ typedef struct _XENVIF_TRANSMITTER_RING { ULONG RequestsPushed; ULONG ResponsesProcessed; ULONG PacketsSent; - XENVIF_TRANSMITTER_PACKET_LIST Completed; + LIST_ENTRY Completed; ULONG PacketsCompleted; PSOCKADDR_INET AddressTable; ULONG AddressCount; @@ -158,12 +153,15 @@ struct _XENVIF_TRANSMITTER { ULONG DisableIpVersion4Gso; ULONG DisableIpVersion6Gso; ULONG AlwaysCopy; + KSPIN_LOCK PacketLock; + PXENBUS_CACHE PacketCache; XENBUS_STORE_INTERFACE StoreInterface; XENBUS_DEBUG_INTERFACE DebugInterface; PXENBUS_DEBUG_CALLBACK DebugCallback; }; #define XENVIF_TRANSMITTER_TAG 'NART' +#define XENVIF_PACKET_CACHE_RESERVATION 32 static FORCEINLINE PVOID __TransmitterAllocate( @@ -181,6 +179,72 @@ __TransmitterFree( __FreePoolWithTag(Buffer, XENVIF_TRANSMITTER_TAG); } +static VOID +TransmitterPacketAcquireLock( + IN PVOID Argument + ) +{ + PXENVIF_TRANSMITTER Transmitter = Argument; + KeAcquireSpinLockAtDpcLevel(&Transmitter->PacketLock); +} + +static VOID +TransmitterPacketReleaseLock( + IN PVOID Argument + ) +{ + PXENVIF_TRANSMITTER Transmitter = Argument; + KeReleaseSpinLockFromDpcLevel(&Transmitter->PacketLock); +} + +static NTSTATUS +TransmitterPacketCtor( + IN PVOID Argument, + IN PVOID Object + ) +{ + UNREFERENCED_PARAMETER(Argument); + UNREFERENCED_PARAMETER(Object); + + return STATUS_SUCCESS; +} + +static VOID +TransmitterPacketDtor( + IN PVOID Argument, + IN PVOID Object + ) +{ + UNREFERENCED_PARAMETER(Argument); + UNREFERENCED_PARAMETER(Object); +} + +static FORCEINLINE PXENVIF_TRANSMITTER_PACKET +__TransmitterGetPacket( + IN PXENVIF_TRANSMITTER Transmitter + ) +{ + return XENBUS_CACHE(Get, + &Transmitter->CacheInterface, + Transmitter->PacketCache, + TRUE); +} + +static FORCEINLINE VOID +__TransmitterPutPacket( + IN PXENVIF_TRANSMITTER Transmitter, + IN PXENVIF_TRANSMITTER_PACKET Packet + ) +{ + RtlZeroMemory(Packet, sizeof(XENVIF_TRANSMITTER_PACKET)); + + XENBUS_CACHE(Put, + &Transmitter->CacheInterface, + Transmitter->PacketCache, + Packet, + TRUE); +} + static NTSTATUS TransmitterBufferCtor( IN PVOID Argument, @@ -530,26 +594,6 @@ fail1: return FALSE; } -#define INCREMENT_PACKET_REFERENCE(_Packet) \ - do { \ - PULONG_PTR Reference = (PULONG_PTR)&(_Packet)->Next; \ - \ - ASSERT(Packet != NULL); \ - (*Reference)++; \ - } while (FALSE) - -#define DECREMENT_PACKET_REFERENCE(_Packet) \ - do { \ - PULONG_PTR Reference = (PULONG_PTR)&(_Packet)->Next; \ - \ - ASSERT(Packet != NULL); \ - ASSERT(*Reference != 0); \ - --(*Reference); \ - } while (FALSE) - -#define PACKET_REFERENCE(_Packet) \ - (*(PULONG_PTR)&(_Packet)->Next) - static FORCEINLINE NTSTATUS __TransmitterRingCopyPayload( IN PXENVIF_TRANSMITTER_RING Ring @@ -558,7 +602,7 @@ __TransmitterRingCopyPayload( PXENVIF_TRANSMITTER Transmitter; PXENVIF_FRONTEND Frontend; PXENVIF_TRANSMITTER_STATE State; - PXENVIF_TRANSMITTER_PACKET_V1 Packet; + PXENVIF_TRANSMITTER_PACKET Packet; XENVIF_PACKET_PAYLOAD Payload; PXENVIF_TRANSMITTER_FRAGMENT Fragment; PXENVIF_TRANSMITTER_BUFFER Buffer; @@ -572,7 +616,7 @@ __TransmitterRingCopyPayload( Payload = State->Payload; - ASSERT3U(PACKET_REFERENCE(Packet), ==, 1); + ASSERT3U(Packet->Value, ==, 1); while (Payload.Length != 0) { PMDL Mdl; @@ -587,7 +631,7 @@ __TransmitterRingCopyPayload( goto fail1; Buffer->Context = Packet; - INCREMENT_PACKET_REFERENCE(Packet); + Packet->Value++; Mdl = Buffer->Mdl; @@ -649,14 +693,14 @@ fail2: ASSERT3P(Buffer->Context, ==, Packet); Buffer->Context = NULL; - DECREMENT_PACKET_REFERENCE(Packet); + Packet->Value--; __TransmitterPutBuffer(Ring, Buffer); fail1: Error("fail1 (%08x)\n", status); - while (PACKET_REFERENCE(Packet) != 1) { + while (Packet->Value != 1) { PLIST_ENTRY ListEntry; ASSERT(State->Count != 0); @@ -689,7 +733,7 @@ fail1: ASSERT3P(Buffer->Context, ==, Packet); Buffer->Context = NULL; - DECREMENT_PACKET_REFERENCE(Packet); + Packet->Value--; __TransmitterPutBuffer(Ring, Buffer); } @@ -705,7 +749,7 @@ __TransmitterRingGrantPayload( PXENVIF_TRANSMITTER Transmitter; PXENVIF_FRONTEND Frontend; PXENVIF_TRANSMITTER_STATE State; - PXENVIF_TRANSMITTER_PACKET_V1 Packet; + PXENVIF_TRANSMITTER_PACKET Packet; PXENVIF_PACKET_PAYLOAD Payload; PMDL Mdl; ULONG Offset; @@ -720,7 +764,7 @@ __TransmitterRingGrantPayload( Packet = State->Packet; Payload = &State->Payload; - ASSERT3U(PACKET_REFERENCE(Packet), ==, 1); + ASSERT3U(Packet->Value, ==, 1); Mdl = Payload->Mdl; Offset = Payload->Offset; @@ -749,7 +793,7 @@ __TransmitterRingGrantPayload( Fragment->Type = XENVIF_TRANSMITTER_FRAGMENT_TYPE_PACKET; Fragment->Context = Packet; - INCREMENT_PACKET_REFERENCE(Packet); + Packet->Value++; Pfn = MmGetMdlPfnArray(Mdl)[MdlOffset / PAGE_SIZE]; PageOffset = MdlOffset & (PAGE_SIZE - 1); @@ -801,7 +845,7 @@ fail2: Fragment->Context = NULL; Fragment->Type = XENVIF_TRANSMITTER_FRAGMENT_TYPE_INVALID; - DECREMENT_PACKET_REFERENCE(Packet); + Packet->Value--; __TransmitterPutFragment(Ring, Fragment); } @@ -811,7 +855,7 @@ fail1: ASSERT3P(Fragment, ==, NULL); - while (PACKET_REFERENCE(Packet) != 1) { + while (Packet->Value != 1) { PLIST_ENTRY ListEntry; ASSERT(State->Count != 0); @@ -835,7 +879,7 @@ fail1: Fragment->Context = NULL; Fragment->Type = XENVIF_TRANSMITTER_FRAGMENT_TYPE_INVALID; - DECREMENT_PACKET_REFERENCE(Packet); + Packet->Value; __TransmitterPutFragment(Ring, Fragment); } @@ -852,7 +896,7 @@ __TransmitterRingPrepareHeader( PXENVIF_FRONTEND Frontend; PXENVIF_MAC Mac; PXENVIF_TRANSMITTER_STATE State; - PXENVIF_TRANSMITTER_PACKET_V1 Packet; + PXENVIF_TRANSMITTER_PACKET Packet; PXENVIF_PACKET_PAYLOAD Payload; PXENVIF_PACKET_INFO Info; PXENVIF_TRANSMITTER_FRAGMENT Fragment; @@ -872,7 +916,7 @@ __TransmitterRingPrepareHeader( Payload = &State->Payload; Info = &State->Info; - ASSERT3U(PACKET_REFERENCE(Packet), ==, 0); + ASSERT3U(Packet->Value, ==, 0); Buffer = __TransmitterGetBuffer(Ring); @@ -881,7 +925,7 @@ __TransmitterRingPrepareHeader( goto fail1; Buffer->Context = Packet; - INCREMENT_PACKET_REFERENCE(Packet); + Packet->Value++; Mdl = Buffer->Mdl; @@ -1099,7 +1143,7 @@ fail3: fail2: Error("fail2\n"); - DECREMENT_PACKET_REFERENCE(Packet); + Packet->Value--; Buffer->Context = NULL; __TransmitterPutBuffer(Ring, Buffer); @@ -1107,7 +1151,7 @@ fail2: fail1: Error("fail1 (%08x)\n", status); - ASSERT3U(PACKET_REFERENCE(Packet), ==, 0); + ASSERT3U(Packet->Value, ==, 0); return status; } @@ -1129,7 +1173,7 @@ __TransmitterRingUnprepareFragments( while (State->Count != 0) { PLIST_ENTRY ListEntry; PXENVIF_TRANSMITTER_FRAGMENT Fragment; - PXENVIF_TRANSMITTER_PACKET_V1 Packet; + PXENVIF_TRANSMITTER_PACKET Packet; --State->Count; @@ -1179,25 +1223,16 @@ __TransmitterRingUnprepareFragments( __TransmitterPutFragment(Ring, Fragment); if (Packet != NULL) - DECREMENT_PACKET_REFERENCE(Packet); + Packet->Value--; } } static FORCEINLINE NTSTATUS __TransmitterRingPreparePacket( IN PXENVIF_TRANSMITTER_RING Ring, - IN PXENVIF_TRANSMITTER_PACKET_V1 Packet + IN PXENVIF_TRANSMITTER_PACKET Packet ) { -#define OFFSET_EXISTS(_Ring, _Packet, _Type) \ - ((_Ring)->Transmitter->Offset[XENVIF_TRANSMITTER_PACKET_ ## _Type ## _OFFSET] != 0) - -#define OFFSET(_Ring, _Packet, _Type) \ - ((OFFSET_EXISTS(_Ring, _Packet, _Type)) ? \ - (PVOID)((PUCHAR)(_Packet) + \ - (_Ring)->Transmitter->Offset[XENVIF_TRANSMITTER_PACKET_ ## _Type ## _OFFSET]) : \ - NULL) - PXENVIF_TRANSMITTER Transmitter; PXENVIF_TRANSMITTER_STATE State; PXENVIF_PACKET_PAYLOAD Payload; @@ -1205,7 +1240,6 @@ __TransmitterRingPreparePacket( NTSTATUS status; ASSERT(IsZeroMemory(&Ring->State, sizeof (XENVIF_TRANSMITTER_STATE))); - ASSERT3P(Packet->Next, ==, NULL); Transmitter = Ring->Transmitter; @@ -1217,17 +1251,9 @@ __TransmitterRingPreparePacket( RtlZeroMemory(&Packet->Send, sizeof (XENVIF_TRANSMITTER_PACKET_SEND_INFO)); Payload = &State->Payload; - - ASSERT(OFFSET_EXISTS(Ring, Packet, MDL)); - Payload->Mdl = *(PMDL *)OFFSET(Ring, Packet, MDL); - - if (OFFSET_EXISTS(Ring, Packet, OFFSET)) - Payload->Offset = *(PULONG)OFFSET(Ring, Packet, OFFSET); - else - Payload->Offset = 0; - - ASSERT(OFFSET_EXISTS(Ring, Packet, LENGTH)); - Payload->Length = *(PULONG)OFFSET(Ring, Packet, LENGTH); + Payload->Mdl = Packet->Mdl; + Payload->Offset = Packet->Offset; + Payload->Length = Packet->Length; InitializeListHead(&State->List); ASSERT3U(State->Count, ==, 0); @@ -1236,7 +1262,7 @@ __TransmitterRingPreparePacket( if (!NT_SUCCESS(status)) goto fail1; - ASSERT3U(State->Count, ==, PACKET_REFERENCE(Packet)); + ASSERT3U(State->Count, ==, Packet->Value); Info = &State->Info; @@ -1294,7 +1320,7 @@ __TransmitterRingPreparePacket( if (Transmitter->AlwaysCopy != 0 || (!NT_SUCCESS(status) && status == STATUS_BUFFER_OVERFLOW)) { - ASSERT3U(State->Count, ==, PACKET_REFERENCE(Packet)); + ASSERT3U(State->Count, ==, Packet->Value); status = __TransmitterRingCopyPayload(Ring); } @@ -1303,7 +1329,7 @@ __TransmitterRingPreparePacket( if (!NT_SUCCESS(status)) goto fail2; - ASSERT3U(State->Count, ==, PACKET_REFERENCE(Packet)); + ASSERT3U(State->Count, ==, Packet->Value); Ring->PacketsPrepared++; return STATUS_SUCCESS; @@ -1332,18 +1358,15 @@ fail1: ASSERT(IsZeroMemory(&Ring->State, sizeof (XENVIF_TRANSMITTER_STATE))); return status; - -#undef OFFSET -#undef OFFSET_EXISTS } -static FORCEINLINE PXENVIF_TRANSMITTER_PACKET_V1 +static FORCEINLINE PXENVIF_TRANSMITTER_PACKET __TransmitterRingUnpreparePacket( IN PXENVIF_TRANSMITTER_RING Ring ) { PXENVIF_TRANSMITTER_STATE State; - PXENVIF_TRANSMITTER_PACKET_V1 Packet; + PXENVIF_TRANSMITTER_PACKET Packet; State = &Ring->State; Packet = State->Packet; @@ -1702,7 +1725,7 @@ __TransmitterRingPostFragments( PXENVIF_TRANSMITTER Transmitter; PXENVIF_FRONTEND Frontend; PXENVIF_TRANSMITTER_STATE State; - PXENVIF_TRANSMITTER_PACKET_V1 Packet; + PXENVIF_TRANSMITTER_PACKET Packet; PXENVIF_PACKET_PAYLOAD Payload; RING_IDX req_prod; RING_IDX rsp_cons; @@ -1722,7 +1745,7 @@ __TransmitterRingPostFragments( ASSERT(!IsListEmpty(&State->List)); ASSERT(State->Count != 0); ASSERT3U(State->Count, <=, XEN_NETIF_NR_SLOTS_MIN); - ASSERT(IMPLY(Packet != NULL, State->Count == PACKET_REFERENCE(Packet))); + ASSERT(IMPLY(Packet != NULL, State->Count == Packet->Value)); req_prod = Ring->Front.req_prod_pvt; rsp_cons = Ring->Front.rsp_cons; @@ -1927,8 +1950,8 @@ __TransmitterRingFakeResponses( static FORCEINLINE VOID __TransmitterRingCompletePacket( - IN PXENVIF_TRANSMITTER_RING Ring, - IN PXENVIF_TRANSMITTER_PACKET_V1 Packet + IN PXENVIF_TRANSMITTER_RING Ring, + IN PXENVIF_TRANSMITTER_PACKET Packet ) { PXENVIF_TRANSMITTER Transmitter; @@ -1987,10 +2010,7 @@ __TransmitterRingCompletePacket( } } - *Ring->Completed.TailPacket = Packet; - ASSERT3P(Packet->Next, ==, NULL); - Ring->Completed.TailPacket = &Packet->Next; - + InsertTailList(&Ring->Completed, &Packet->ListEntry); Ring->PacketsCompleted++; } @@ -2026,7 +2046,7 @@ TransmitterRingPoll( netif_tx_response_t *rsp; uint16_t id; PXENVIF_TRANSMITTER_FRAGMENT Fragment; - PXENVIF_TRANSMITTER_PACKET_V1 Packet; + PXENVIF_TRANSMITTER_PACKET Packet; rsp = RING_GET_RESPONSE(&Ring->Front, rsp_cons); rsp_cons++; @@ -2090,7 +2110,7 @@ TransmitterRingPoll( continue; } - DECREMENT_PACKET_REFERENCE(Packet); + Packet->Value--; if (rsp->status != NETIF_RSP_OKAY && Packet->Completion.Status == XENVIF_TRANSMITTER_PACKET_PENDING) { @@ -2111,7 +2131,7 @@ TransmitterRingPoll( RtlZeroMemory(rsp, sizeof (netif_tx_response_t)); - if (PACKET_REFERENCE(Packet) != 0) + if (Packet->Value != 0) continue; if (Packet->Completion.Status == XENVIF_TRANSMITTER_PACKET_PENDING) @@ -2266,32 +2286,44 @@ TransmitterRingSwizzle( { ULONG_PTR Old; ULONG_PTR New; - PXENVIF_TRANSMITTER_PACKET_V1 HeadPacket; - PXENVIF_TRANSMITTER_PACKET_V1 *TailPacket; + PLIST_ENTRY ListEntry; + LIST_ENTRY List; ULONG Count; ASSERT3P(Ring->LockThread, ==, KeGetCurrentThread()); + InitializeListHead(&List); + New = XENVIF_TRANSMITTER_LOCK_BIT; Old = (ULONG_PTR)InterlockedExchangePointer(&Ring->Lock, (PVOID)New); ASSERT(Old & XENVIF_TRANSMITTER_LOCK_BIT); - HeadPacket = (PVOID)(Old & ~XENVIF_TRANSMITTER_LOCK_BIT); + ListEntry = (PVOID)(Old & ~XENVIF_TRANSMITTER_LOCK_BIT); - if (HeadPacket == NULL) + if (ListEntry == NULL) return; // Packets are held in the atomic packet list in reverse order // so that the most recent is always head of the list. This is // necessary to allow addition to the list to be done atomically. - TailPacket = &HeadPacket->Next; - Count = __TransmitterReversePacketList(&HeadPacket); - ASSERT3P(*TailPacket, ==, NULL); + for (Count = 0; ListEntry != NULL; ++Count) { + PLIST_ENTRY NextEntry; + + NextEntry = ListEntry->Blink; + ListEntry->Flink = ListEntry->Blink = ListEntry; + + InsertHeadList(&List, ListEntry); + + ListEntry = NextEntry; + } - *(Ring->Queued.TailPacket) = HeadPacket; - Ring->Queued.TailPacket = TailPacket; - Ring->PacketsQueued += Count; + ListEntry = List.Flink; + if (!IsListEmpty(&List)) { + RemoveEntryList(&List); + AppendTailList(&Ring->Queued, ListEntry); + Ring->PacketsQueued += Count; + } } static DECLSPEC_NOINLINE VOID @@ -2307,8 +2339,9 @@ TransmitterRingSchedule( State = &Ring->State; for (;;) { - PXENVIF_TRANSMITTER_PACKET_V1 Packet; - NTSTATUS status; + PLIST_ENTRY ListEntry; + PXENVIF_TRANSMITTER_PACKET Packet; + NTSTATUS status; if (State->Count != 0) { status = __TransmitterRingPostFragments(Ring); @@ -2357,18 +2390,13 @@ TransmitterRingSchedule( continue; } - Packet = Ring->Queued.HeadPacket; - - if (Packet == NULL) + ListEntry = RemoveHeadList(&Ring->Queued); + if (ListEntry == &Ring->Queued) break; - if (Packet->Next == NULL) { - Ring->Queued.HeadPacket = NULL; - Ring->Queued.TailPacket = &Ring->Queued.HeadPacket; - } else { - Ring->Queued.HeadPacket = Packet->Next; - Packet->Next = NULL; - } + Packet = CONTAINING_RECORD(ListEntry, XENVIF_TRANSMITTER_PACKET, ListEntry); + Packet->ListEntry.Flink = Packet->ListEntry.Blink = NULL; + Packet->Value = 0; status = __TransmitterRingPreparePacket(Ring, Packet); if (!NT_SUCCESS(status)) { @@ -2397,23 +2425,63 @@ TransmitterRingSchedule( ASSERT3U(Ring->PacketsPrepared, ==, Ring->PacketsCopied + Ring->PacketsGranted + Ring->PacketsFaked); } - ASSERT(IMPLY(Ring->Queued.HeadPacket == NULL, Ring->Queued.TailPacket == &Ring->Queued.HeadPacket)); - __TransmitterRingPushRequests(Ring); } static FORCEINLINE VOID __TransmitterReturnPackets( - IN PXENVIF_TRANSMITTER Transmitter, - IN PXENVIF_TRANSMITTER_PACKET_V1 HeadPacket + IN PXENVIF_TRANSMITTER Transmitter, + IN PLIST_ENTRY List ) { - PXENVIF_FRONTEND Frontend; + PXENVIF_FRONTEND Frontend; + PXENVIF_VIF_CONTEXT VifContext; Frontend = Transmitter->Frontend; + VifContext = PdoGetVifContext(FrontendGetPdo(Frontend)); + + switch (VifGetVersion(VifContext)) { + case 1: { + PXENVIF_TRANSMITTER_PACKET_V1 HeadPacket; + PXENVIF_TRANSMITTER_PACKET_V1 NextPacket; - VifTransmitterReturnPacketsV1(PdoGetVifContext(FrontendGetPdo(Frontend)), - HeadPacket); + HeadPacket = NextPacket = NULL; + + while (!IsListEmpty(List)) { + PLIST_ENTRY ListEntry; + PXENVIF_TRANSMITTER_PACKET Packet; + + ListEntry = RemoveTailList(List); + ASSERT3P(ListEntry, !=, List); + + Packet = CONTAINING_RECORD(ListEntry, XENVIF_TRANSMITTER_PACKET, ListEntry); + Packet->ListEntry.Flink = Packet->ListEntry.Blink = NULL; + + HeadPacket = Packet->Cookie; + HeadPacket->Next = NextPacket; + NextPacket = HeadPacket; + + HeadPacket->Completion = Packet->Completion; + + __TransmitterPutPacket(Transmitter, Packet); + } + + if (HeadPacket != NULL) { + VifTransmitterReturnPacketsV1(VifContext, HeadPacket); + } + break; + } + case 2: { + if (!IsListEmpty(List)) { + VifTransmitterReturnPacketsV2(VifContext, List); + } + break; + } + default: { + ASSERT(FALSE); + break; + } + } } static FORCEINLINE BOOLEAN @@ -2513,11 +2581,9 @@ __TransmitterRingReleaseLock( IN PXENVIF_TRANSMITTER_RING Ring ) { - PXENVIF_TRANSMITTER_PACKET_V1 HeadPacket; - PXENVIF_TRANSMITTER_PACKET_V1 *TailPacket; - - HeadPacket = NULL; - TailPacket = &HeadPacket; + LIST_ENTRY List; + + InitializeListHead(&List); ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL); @@ -2527,23 +2593,26 @@ __TransmitterRingReleaseLock( // thread could be simuntaneously adding to the list. do { + PLIST_ENTRY ListEntry; + TransmitterRingSwizzle(Ring); TransmitterRingSchedule(Ring); - *TailPacket = Ring->Completed.HeadPacket; - TailPacket = Ring->Completed.TailPacket; - - Ring->Completed.HeadPacket = NULL; - Ring->Completed.TailPacket = &Ring->Completed.HeadPacket; + ListEntry = Ring->Completed.Flink; + if (!IsListEmpty(&Ring->Completed)) { + RemoveEntryList(&Ring->Completed); + InitializeListHead(&Ring->Completed); + AppendTailList(&List, ListEntry); + } } while (!__TransmitterRingTryReleaseLock(Ring)); - if (HeadPacket != NULL) { + if (!IsListEmpty(&List)) { PXENVIF_TRANSMITTER Transmitter; Transmitter = Ring->Transmitter; __TransmitterReturnPackets(Transmitter, - HeadPacket); + &List); } } @@ -2792,8 +2861,8 @@ __TransmitterRingInitialize( (*Ring)->Transmitter = Transmitter; (*Ring)->Index = Index; - (*Ring)->Queued.TailPacket = &(*Ring)->Queued.HeadPacket; - (*Ring)->Completed.TailPacket = &(*Ring)->Completed.HeadPacket; + InitializeListHead(&(*Ring)->Queued); + InitializeListHead(&(*Ring)->Completed); KeInitializeSpinLock(&(*Ring)->EvtchnLock); KeInitializeDpc(&(*Ring)->Dpc, TransmitterRingDpc, *Ring); @@ -2929,8 +2998,8 @@ fail2: RtlZeroMemory(&(*Ring)->Dpc, sizeof (KDPC)); RtlZeroMemory(&(*Ring)->EvtchnLock, sizeof (KSPIN_LOCK)); - (*Ring)->Queued.TailPacket = NULL; - (*Ring)->Completed.TailPacket = NULL; + RtlZeroMemory(&(*Ring)->Queued, sizeof (LIST_ENTRY)); + RtlZeroMemory(&(*Ring)->Completed, sizeof (LIST_ENTRY)); (*Ring)->Index = 0; (*Ring)->Transmitter = NULL; @@ -3151,7 +3220,7 @@ __TransmitterRingDisable( { PXENVIF_TRANSMITTER Transmitter; PXENVIF_FRONTEND Frontend; - PXENVIF_TRANSMITTER_PACKET_V1 Packet; + PXENVIF_TRANSMITTER_PACKET Packet; PCHAR Buffer; XenbusState State; ULONG Attempt; @@ -3170,14 +3239,7 @@ __TransmitterRingDisable( // Put any packet back on the head of the queue if (Packet != NULL) { - ASSERT3P(Packet->Next, ==, NULL); - - Packet->Next = Ring->Queued.HeadPacket; - - if (Ring->Queued.TailPacket == &Ring->Queued.HeadPacket) - Ring->Queued.TailPacket = &Packet->Next; - - Ring->Queued.HeadPacket = Packet; + InsertHeadList(&Ring->Queued, &Packet->ListEntry); } Ring->AddressIndex = 0; @@ -3336,11 +3398,11 @@ __TransmitterRingTeardown( Ring->BufferCache); Ring->BufferCache = NULL; - ASSERT3P(Ring->Queued.TailPacket, ==, &Ring->Queued.HeadPacket); - Ring->Queued.TailPacket = NULL; + ASSERT(IsListEmpty(&Ring->Queued)); + RtlZeroMemory(&Ring->Queued, sizeof (LIST_ENTRY)); - ASSERT3P(Ring->Completed.TailPacket, ==, &Ring->Completed.HeadPacket); - Ring->Completed.TailPacket = NULL; + ASSERT(IsListEmpty(&Ring->Completed)); + RtlZeroMemory(&Ring->Completed, sizeof (LIST_ENTRY)); Ring->Index = 0; Ring->Transmitter = NULL; @@ -3351,25 +3413,20 @@ __TransmitterRingTeardown( static FORCEINLINE VOID __TransmitterRingQueuePackets( - IN PXENVIF_TRANSMITTER_RING Ring, - IN PXENVIF_TRANSMITTER_PACKET_V1 HeadPacket + IN PXENVIF_TRANSMITTER_RING Ring, + IN PLIST_ENTRY List ) { - PXENVIF_TRANSMITTER_PACKET_V1 *TailPacket; ULONG_PTR Old; ULONG_PTR LockBit; ULONG_PTR New; - TailPacket = &HeadPacket->Next; - (VOID) __TransmitterReversePacketList(&HeadPacket); - ASSERT3P(*TailPacket, ==, NULL); - do { Old = (ULONG_PTR)Ring->Lock; LockBit = Old & XENVIF_TRANSMITTER_LOCK_BIT; - *TailPacket = (PVOID)(Old & ~XENVIF_TRANSMITTER_LOCK_BIT); - New = (ULONG_PTR)HeadPacket; + List->Flink->Blink = (PVOID)(Old & ~XENVIF_TRANSMITTER_LOCK_BIT); + New = (ULONG_PTR)List->Blink; ASSERT((New & XENVIF_TRANSMITTER_LOCK_BIT) == 0); New |= LockBit; } while ((ULONG_PTR)InterlockedCompareExchangePointer(&Ring->Lock, (PVOID)New, (PVOID)Old) != Old); @@ -3388,22 +3445,19 @@ __TransmitterRingAbortPackets( IN PXENVIF_TRANSMITTER_RING Ring ) { - PXENVIF_TRANSMITTER_PACKET_V1 Packet; - __TransmitterRingAcquireLock(Ring); TransmitterRingSwizzle(Ring); - Packet = Ring->Queued.HeadPacket; - - Ring->Queued.HeadPacket = NULL; - Ring->Queued.TailPacket = &Ring->Queued.HeadPacket; - - while (Packet != NULL) { - PXENVIF_TRANSMITTER_PACKET_V1 Next; + while (!IsListEmpty(&Ring->Queued)) { + PLIST_ENTRY ListEntry; + PXENVIF_TRANSMITTER_PACKET Packet; - Next = Packet->Next; - Packet->Next = NULL; + ListEntry = RemoveHeadList(&Ring->Queued); + ASSERT3P(ListEntry, !=, &Ring->Queued); + + Packet = CONTAINING_RECORD(ListEntry, XENVIF_TRANSMITTER_PACKET, ListEntry); + Packet->ListEntry.Flink = Packet->ListEntry.Blink = NULL; // Fake that we prapared and sent this packet Ring->PacketsPrepared++; @@ -3413,8 +3467,6 @@ __TransmitterRingAbortPackets( Packet->Completion.Status = XENVIF_TRANSMITTER_PACKET_DROPPED; __TransmitterRingCompletePacket(Ring, Packet); - - Packet = Next; } ASSERT3U(Ring->PacketsSent, ==, Ring->PacketsPrepared - Ring->PacketsUnprepared); @@ -3507,6 +3559,7 @@ TransmitterInitialize( &(*Transmitter)->EvtchnInterface); (*Transmitter)->Frontend = Frontend; + KeInitializeSpinLock(&(*Transmitter)->PacketLock); status = XENBUS_RANGE_SET(Acquire, &(*Transmitter)->RangeSetInterface); if (!NT_SUCCESS(status)) @@ -3556,6 +3609,9 @@ fail2: (*Transmitter)->Frontend = NULL; + RtlZeroMemory(&(*Transmitter)->PacketLock, + sizeof (KSPIN_LOCK)); + RtlZeroMemory(&(*Transmitter)->CacheInterface, sizeof (XENBUS_CACHE_INTERFACE)); @@ -3605,6 +3661,20 @@ TransmitterConnect( if (!NT_SUCCESS(status)) goto fail3; + status = XENBUS_CACHE(Create, + &Transmitter->CacheInterface, + "packet_cache", + sizeof (XENVIF_TRANSMITTER_PACKET), + XENVIF_PACKET_CACHE_RESERVATION, + TransmitterPacketCtor, + TransmitterPacketDtor, + TransmitterPacketAcquireLock, + TransmitterPacketReleaseLock, + Transmitter, + &Transmitter->PacketCache); + if (!NT_SUCCESS(status)) + goto fail4; + status = XENBUS_STORE(Read, &Transmitter->StoreInterface, NULL, @@ -3630,7 +3700,7 @@ TransmitterConnect( status = __TransmitterRingConnect(Ring); if (!NT_SUCCESS(status)) - goto fail4; + goto fail5; } status = XENBUS_DEBUG(Register, @@ -3640,16 +3710,16 @@ TransmitterConnect( Transmitter, &Transmitter->DebugCallback); if (!NT_SUCCESS(status)) - goto fail5; + goto fail6; return STATUS_SUCCESS; +fail6: + Error("fail6\n"); + fail5: Error("fail5\n"); -fail4: - Error("fail4\n"); - for (Index = 0; Index < MAXIMUM_PROCESSORS; ++Index) { PXENVIF_TRANSMITTER_RING Ring; @@ -3660,6 +3730,14 @@ fail4: __TransmitterRingDisconnect(Ring); } + XENBUS_CACHE(Destroy, + &Transmitter->CacheInterface, + Transmitter->PacketCache); + Transmitter->PacketCache = NULL; + +fail4: + Error("fail4\n"); + XENBUS_EVTCHN(Release, &Transmitter->EvtchnInterface); fail3: @@ -3772,6 +3850,11 @@ TransmitterDisconnect( __TransmitterRingDisconnect(Ring); } + XENBUS_CACHE(Destroy, + &Transmitter->CacheInterface, + Transmitter->PacketCache); + Transmitter->PacketCache = NULL; + XENBUS_STORE(Release, &Transmitter->StoreInterface); XENBUS_DEBUG(Release, &Transmitter->DebugInterface); @@ -3811,6 +3894,9 @@ TransmitterTeardown( Transmitter->Frontend = NULL; Transmitter->Split = FALSE; + RtlZeroMemory(&Transmitter->PacketLock, + sizeof (KSPIN_LOCK)); + RtlZeroMemory(&Transmitter->CacheInterface, sizeof (XENBUS_CACHE_INTERFACE)); @@ -3893,12 +3979,165 @@ fail1: return status; } +static BOOLEAN +__TransmitterGetPacketHeadersPullup( + IN PVOID Argument, + IN PUCHAR DestinationVa, + IN OUT PXENVIF_PACKET_PAYLOAD Payload, + IN ULONG Length + ) +{ + PMDL Mdl; + ULONG Offset; + + UNREFERENCED_PARAMETER(Argument); + + Mdl = Payload->Mdl; + Offset = Payload->Offset; + + if (Payload->Length < Length) + goto fail1; + + Payload->Length -= Length; + + while (Length != 0) { + PUCHAR MdlMappedSystemVa; + ULONG MdlByteCount; + ULONG CopyLength; + + ASSERT(Mdl != NULL); + + MdlMappedSystemVa = MmGetSystemAddressForMdlSafe(Mdl, NormalPagePriority); + ASSERT(MdlMappedSystemVa != NULL); + + MdlMappedSystemVa += Offset; + + MdlByteCount = Mdl->ByteCount - Offset; + + CopyLength = __min(MdlByteCount, Length); + + RtlCopyMemory(DestinationVa, MdlMappedSystemVa, CopyLength); + DestinationVa += CopyLength; + + Offset += CopyLength; + Length -= CopyLength; + + MdlByteCount -= CopyLength; + if (MdlByteCount == 0) { + Mdl = Mdl->Next; + Offset = 0; + } + } + + Payload->Mdl = Mdl; + Payload->Offset = Offset; + + return TRUE; + +fail1: + Error("fail1\n"); + + return FALSE; +} + +NTSTATUS +TransmitterGetPacketHeaders( + IN PXENVIF_TRANSMITTER Transmitter, + IN PXENVIF_TRANSMITTER_PACKET Packet, + OUT PVOID Headers, + OUT PXENVIF_PACKET_INFO Info + ) +{ + XENVIF_PACKET_PAYLOAD Payload; + NTSTATUS status; + + Payload.Mdl = Packet->Mdl; + Payload.Offset = Packet->Offset; + Payload.Length = Packet->Length; + + status = ParsePacket(Headers, + __TransmitterGetPacketHeadersPullup, + Transmitter, + &Payload, + Info); + if (!NT_SUCCESS(status)) + goto fail1; + + return STATUS_SUCCESS; + +fail1: + return status; +} + VOID TransmitterQueuePacketsV1( IN PXENVIF_TRANSMITTER Transmitter, IN PXENVIF_TRANSMITTER_PACKET_V1 HeadPacket ) { +#define OFFSET_EXISTS(_Transmitter, _Packet, _Type) \ + ((_Transmitter)->Offset[XENVIF_TRANSMITTER_PACKET_ ## _Type ## _OFFSET] != 0) + +#define OFFSET(_Transmitter, _Packet, _Type) \ + ((OFFSET_EXISTS(_Transmitter, _Packet, _Type)) ? \ + (PVOID)((PUCHAR)(_Packet) + \ + (_Transmitter)->Offset[XENVIF_TRANSMITTER_PACKET_ ## _Type ## _OFFSET]) : \ + NULL) + + LIST_ENTRY List; + PXENVIF_FRONTEND Frontend; + PXENVIF_VIF_CONTEXT VifContext; + + Frontend = Transmitter->Frontend; + VifContext = PdoGetVifContext(FrontendGetPdo(Frontend)); + + ASSERT3U(VifGetVersion(VifContext), ==, 1); + + InitializeListHead(&List); + + while (HeadPacket != NULL) { + PXENVIF_TRANSMITTER_PACKET Packet; + + Packet = __TransmitterGetPacket(Transmitter); + if (Packet == NULL) + break; + + Packet->Cookie = HeadPacket; + Packet->Send = HeadPacket->Send; + + ASSERT(OFFSET_EXISTS(Transmitter, HeadPacket, MDL)); + Packet->Mdl = *(PMDL *)OFFSET(Transmitter, HeadPacket, MDL); + + if (OFFSET_EXISTS(Transmitter, HeadPacket, OFFSET)) + Packet->Offset = *(PULONG)OFFSET(Transmitter, HeadPacket, OFFSET); + else + Packet->Offset = 0; + + ASSERT(OFFSET_EXISTS(Transmitter, HeadPacket, LENGTH)); + Packet->Length = *(PULONG)OFFSET(Transmitter, HeadPacket, LENGTH); + + InsertTailList(&List, &Packet->ListEntry); + + HeadPacket = HeadPacket->Next; + } + + if (!IsListEmpty(&List)) + TransmitterQueuePacketsV2(Transmitter, &List); + + if (HeadPacket != NULL) + VifTransmitterReturnPacketsV1(VifContext, + HeadPacket); + +#undef OFFSET +#undef OFFSET_EXISTS +} + +VOID +TransmitterQueuePacketsV2( + IN PXENVIF_TRANSMITTER Transmitter, + IN PLIST_ENTRY List + ) +{ PXENVIF_TRANSMITTER_RING Ring; // We need to hash for a ring eventually. Since there is only a @@ -3906,7 +4145,7 @@ TransmitterQueuePacketsV1( Ring = Transmitter->Rings[0]; ASSERT3P(Ring, !=, NULL); - __TransmitterRingQueuePackets(Ring, HeadPacket); + __TransmitterRingQueuePackets(Ring, List); } VOID diff --git a/src/xenvif/transmitter.h b/src/xenvif/transmitter.h index 6a7fcd5..2ed2786 100644 --- a/src/xenvif/transmitter.h +++ b/src/xenvif/transmitter.h @@ -114,6 +114,12 @@ TransmitterQueuePacketsV1( ); extern VOID +TransmitterQueuePacketsV2( + IN PXENVIF_TRANSMITTER Transmitter, + IN PLIST_ENTRY List + ); + +extern VOID TransmitterQueryOffloadOptions( IN PXENVIF_TRANSMITTER Transmitter, OUT PXENVIF_VIF_OFFLOAD_OPTIONS Options @@ -133,4 +139,12 @@ TransmitterSetPacketOffset( IN LONG_PTR Value ); +extern NTSTATUS +TransmitterGetPacketHeaders( + IN PXENVIF_TRANSMITTER Transmitter, + IN PXENVIF_TRANSMITTER_PACKET Packet, + OUT PVOID Headers, + OUT PXENVIF_PACKET_INFO Info + ); + #endif // _XENVIF_TRANSMITTER_H diff --git a/src/xenvif/vif.c b/src/xenvif/vif.c index b858d8f..24855b6 100644 --- a/src/xenvif/vif.c +++ b/src/xenvif/vif.c @@ -47,7 +47,8 @@ struct _XENVIF_VIF_CONTEXT { XENVIF_MRSW_LOCK Lock; LONG References; PXENVIF_FRONTEND Frontend; - BOOLEAN Enabled; + BOOLEAN Enabled; + ULONG Version; XENVIF_VIF_CALLBACK Callback; PVOID Argument; PXENVIF_THREAD MacThread; @@ -248,6 +249,7 @@ VifTransmitterSetPacketOffset( AcquireMrswLockShared(&Context->Lock); + ASSERT3U(VifGetVersion(Context), ==, 1); status = TransmitterSetPacketOffset(FrontendGetTransmitter(Context->Frontend), Type, Value); @@ -273,6 +275,29 @@ VifReceiverReturnPackets( ReleaseMrswLockShared(&Context->Lock); } +static NTSTATUS +VifTransmitterGetPacketHeaders( + IN PINTERFACE Interface, + IN PXENVIF_TRANSMITTER_PACKET Packet, + OUT PVOID Headers, + OUT PXENVIF_PACKET_INFO Info + ) +{ + PXENVIF_VIF_CONTEXT Context = Interface->Context; + NTSTATUS status; + + AcquireMrswLockShared(&Context->Lock); + + ASSERT3U(VifGetVersion(Context), >=, 2); + status = TransmitterGetPacketHeaders(FrontendGetTransmitter(Context->Frontend), + Packet, + Headers, + Info); + + ReleaseMrswLockShared(&Context->Lock); + + return status; +} static NTSTATUS VifTransmitterQueuePacketsV1( @@ -289,6 +314,7 @@ VifTransmitterQueuePacketsV1( if (Context->Enabled == FALSE) goto fail1; + ASSERT3U(VifGetVersion(Context), ==, 1); TransmitterQueuePacketsV1(FrontendGetTransmitter(Context->Frontend), Head); @@ -302,6 +328,35 @@ fail1: return status; } +static NTSTATUS +VifTransmitterQueuePacketsV2( + IN PINTERFACE Interface, + IN PLIST_ENTRY List + ) +{ + PXENVIF_VIF_CONTEXT Context = Interface->Context; + NTSTATUS status; + + AcquireMrswLockShared(&Context->Lock); + + status = STATUS_UNSUCCESSFUL; + if (Context->Enabled == FALSE) + goto fail1; + + ASSERT3U(VifGetVersion(Context), >=, 2); + TransmitterQueuePacketsV2(FrontendGetTransmitter(Context->Frontend), + List); + + ReleaseMrswLockShared(&Context->Lock); + + return STATUS_SUCCESS; + +fail1: + ReleaseMrswLockShared(&Context->Lock); + + return status; +} + static VOID VifTransmitterQueryOffloadOptions( IN PINTERFACE Interface, @@ -656,6 +711,31 @@ static struct _XENVIF_VIF_INTERFACE_V1 VifInterfaceVersion1 = { VifMacQueryFilterLevel }; +static struct _XENVIF_VIF_INTERFACE_V2 VifInterfaceVersion2 = { + { sizeof (struct _XENVIF_VIF_INTERFACE_V2), 2, NULL, NULL, NULL }, + VifAcquire, + VifRelease, + VifEnable, + VifDisable, + VifQueryStatistic, + VifReceiverReturnPackets, + VifReceiverSetOffloadOptions, + VifReceiverQueryRingSize, + VifTransmitterGetPacketHeaders, + VifTransmitterQueuePacketsV2, + VifTransmitterQueryOffloadOptions, + VifTransmitterQueryLargePacketSize, + VifTransmitterQueryRingSize, + VifMacQueryState, + VifMacQueryMaximumFrameSize, + VifMacQueryPermanentAddress, + VifMacQueryCurrentAddress, + VifMacQueryMulticastAddresses, + VifMacSetMulticastAddresses, + VifMacSetFilterLevel, + VifMacQueryFilterLevel +}; + NTSTATUS VifInitialize( IN PXENVIF_PDO Pdo, @@ -737,11 +817,30 @@ VifGetInterface( status = STATUS_SUCCESS; break; } + case 2: { + struct _XENVIF_VIF_INTERFACE_V2 *VifInterface; + + VifInterface = (struct _XENVIF_VIF_INTERFACE_V2 *)Interface; + + status = STATUS_BUFFER_OVERFLOW; + if (Size < sizeof (struct _XENVIF_VIF_INTERFACE_V2)) + break; + + *VifInterface = VifInterfaceVersion2; + + ASSERT3U(Interface->Version, ==, Version); + Interface->Context = Context; + + status = STATUS_SUCCESS; + break; + } default: status = STATUS_NOT_SUPPORTED; break; } + Context->Version = Version; + return status; } @@ -753,6 +852,7 @@ VifTeardown( Trace("====>\n"); Context->Pdo = NULL; + Context->Version = 0; ThreadAlert(Context->MacThread); ThreadJoin(Context->MacThread); @@ -788,15 +888,39 @@ VifTransmitterReturnPacketsV1( IN PXENVIF_TRANSMITTER_PACKET_V1 Head ) { + ASSERT3U(VifGetVersion(Context), ==, 1); + Context->Callback(Context->Argument, XENVIF_TRANSMITTER_RETURN_PACKETS, Head); } -extern PXENVIF_THREAD +VOID +VifTransmitterReturnPacketsV2( + IN PXENVIF_VIF_CONTEXT Context, + IN PLIST_ENTRY List + ) +{ + ASSERT3U(VifGetVersion(Context), >=, 2); + + Context->Callback(Context->Argument, + XENVIF_TRANSMITTER_RETURN_PACKETS, + List); +} + +PXENVIF_THREAD VifGetMacThread( IN PXENVIF_VIF_CONTEXT Context ) { return Context->MacThread; } + +ULONG +VifGetVersion( + IN PXENVIF_VIF_CONTEXT Context + ) +{ + return Context->Version; +} + diff --git a/src/xenvif/vif.h b/src/xenvif/vif.h index a26aa1b..fc7aabc 100644 --- a/src/xenvif/vif.h +++ b/src/xenvif/vif.h @@ -74,10 +74,21 @@ VifTransmitterReturnPacketsV1( IN PXENVIF_TRANSMITTER_PACKET_V1 Head ); +extern VOID +VifTransmitterReturnPacketsV2( + IN PXENVIF_VIF_CONTEXT Context, + IN PLIST_ENTRY List + ); + extern PXENVIF_THREAD VifGetMacThread( IN PXENVIF_VIF_CONTEXT Context ); +extern ULONG +VifGetVersion( + IN PXENVIF_VIF_CONTEXT Context + ); + #endif // _XENVIF_VIF_H -- 1.9.4.msysgit.1 _______________________________________________ win-pv-devel mailing list win-pv-devel@xxxxxxxxxxxxxxxxxxxx http://lists.xenproject.org/cgi-bin/mailman/listinfo/win-pv-devel
|
Lists.xenproject.org is hosted with RackSpace, monitoring our |