|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [win-pv-devel] [RFC PATCH 4/6] Add initial multi-queue support
Create a queue per CPU, up-to the lower limit specified by xenstore and
registry override. Requests are queued on the same processor's DPC as
StartIo runs on. It is assumed that any request dispatched to the
backend on a particular queue will always be completed on the same
queue.
Signed-off-by: owensm <owen.smith@xxxxxxxxxx>
---
src/xenvbd/protocol.c | 2100 ++++++++++++++++++++++++++++---------------------
1 file changed, 1222 insertions(+), 878 deletions(-)
diff --git a/src/xenvbd/protocol.c b/src/xenvbd/protocol.c
index 39767eb..8aa6ec8 100644
--- a/src/xenvbd/protocol.c
+++ b/src/xenvbd/protocol.c
@@ -56,15 +56,10 @@
#define XENVBD_MAX_PROTOCOL_PAGE_ORDER (4)
#define XENVBD_MAX_PROTOCOL_PAGES (1 << XENVBD_MAX_PROTOCOL_PAGE_ORDER)
-struct _XENVBD_PROTOCOL {
- PXENVBD_FRONTEND Frontend;
- BOOLEAN Connected;
- BOOLEAN Enabled;
-
- XENBUS_CACHE_INTERFACE CacheInterface;
- XENBUS_STORE_INTERFACE StoreInterface;
- XENBUS_EVTCHN_INTERFACE EvtchnInterface;
- XENBUS_DEBUG_INTERFACE DebugInterface;
+typedef struct _XENVBD_RING {
+ PXENVBD_PROTOCOL Protocol;
+ ULONG Index;
+ PCHAR Path;
PXENBUS_DEBUG_CALLBACK DebugCallback;
@@ -72,14 +67,10 @@ struct _XENVBD_PROTOCOL {
PMDL Mdl;
blkif_sring_t* Shared;
blkif_front_ring_t Front;
- ULONG Order;
PVOID Grants[XENVBD_MAX_PROTOCOL_PAGES];
PXENBUS_EVTCHN_CHANNEL Channel;
KDPC Dpc;
- PXENBUS_CACHE RequestCache;
- PXENBUS_CACHE SegmentCache;
- PXENBUS_CACHE IndirectCache;
XENVBD_QUEUE PreparedReqs;
XENVBD_QUEUE SubmittedReqs;
XENVBD_QUEUE ShutdownSrbs;
@@ -95,6 +86,29 @@ struct _XENVBD_PROTOCOL {
ULONG BlkOpBarrier;
ULONG BlkOpDiscard;
ULONG BlkOpFlush;
+} XENVBD_RING, *PXENVBD_RING;
+
+struct _XENVBD_PROTOCOL {
+ PXENVBD_FRONTEND Frontend;
+ BOOLEAN Connected;
+ BOOLEAN Enabled;
+
+ XENBUS_CACHE_INTERFACE CacheInterface;
+ XENBUS_STORE_INTERFACE StoreInterface;
+ XENBUS_EVTCHN_INTERFACE EvtchnInterface;
+ XENBUS_DEBUG_INTERFACE DebugInterface;
+
+ PXENBUS_DEBUG_CALLBACK DebugCallback;
+ KSPIN_LOCK Lock;
+
+ PXENBUS_CACHE RequestCache;
+ PXENBUS_CACHE SegmentCache;
+ PXENBUS_CACHE IndirectCache;
+
+ ULONG Order;
+ ULONG NumQueues;
+ PXENVBD_RING *Rings;
+
ULONG64 SegsGranted;
ULONG64 SegsBounced;
};
@@ -138,12 +152,13 @@ xen_wmb()
}
static FORCEINLINE VOID
-__ProtocolInsert(
- IN PXENVBD_PROTOCOL Protocol,
+__RingInsert(
+ IN PXENVBD_RING Ring,
IN PXENVBD_REQUEST Request,
IN blkif_request_t* req
)
{
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol->Frontend);
switch (Request->Operation) {
@@ -234,7 +249,7 @@ __ProtocolInsert(
ASSERT(FALSE);
break;
}
- ++Protocol->Submitted;
+ ++Ring->Submitted;
}
static PXENVBD_INDIRECT
@@ -400,15 +415,16 @@ ProtocolPutRequest(
}
static FORCEINLINE PXENVBD_REQUEST
-ProtocolFindRequest(
- IN PXENVBD_PROTOCOL Protocol,
+RingFindRequest(
+ IN PXENVBD_RING Ring,
IN ULONG64 Id
)
{
KIRQL Irql;
PLIST_ENTRY ListEntry;
PXENVBD_REQUEST Request;
- PXENVBD_QUEUE Queue = &Protocol->SubmittedReqs;
+ PXENVBD_QUEUE Queue = &Ring->SubmittedReqs;
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
KeAcquireSpinLock(&Queue->Lock, &Irql);
@@ -425,40 +441,41 @@ ProtocolFindRequest(
}
KeReleaseSpinLock(&Queue->Lock, Irql);
- Warning("Target[%d] : Tag %llx not found in submitted list (%u items)\n",
+ Warning("Target[%d][%u] : Tag %llx not found in submitted list (%u
items)\n",
FrontendGetTargetId(Protocol->Frontend),
+ Ring->Index,
Id,
QueueCount(Queue));
return NULL;
}
static FORCEINLINE VOID
-__ProtocolIncBlkifOpCount(
- IN PXENVBD_PROTOCOL Protocol,
+__RingIncBlkifOpCount(
+ IN PXENVBD_RING Ring,
IN PXENVBD_REQUEST Request
)
{
switch (Request->Operation) {
case BLKIF_OP_READ:
if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
- ++Protocol->BlkOpIndirectRead;
+ ++Ring->BlkOpIndirectRead;
else
- ++Protocol->BlkOpRead;
+ ++Ring->BlkOpRead;
break;
case BLKIF_OP_WRITE:
if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
- ++Protocol->BlkOpIndirectWrite;
+ ++Ring->BlkOpIndirectWrite;
else
- ++Protocol->BlkOpWrite;
+ ++Ring->BlkOpWrite;
break;
case BLKIF_OP_WRITE_BARRIER:
- ++Protocol->BlkOpBarrier;
+ ++Ring->BlkOpBarrier;
break;
case BLKIF_OP_DISCARD:
- ++Protocol->BlkOpDiscard;
+ ++Ring->BlkOpDiscard;
break;
case BLKIF_OP_FLUSH_DISKCACHE:
- ++Protocol->BlkOpFlush;
+ ++Ring->BlkOpFlush;
break;
default:
ASSERT(FALSE);
@@ -467,7 +484,7 @@ __ProtocolIncBlkifOpCount(
}
static FORCEINLINE ULONG
-__ProtocolSectorsPerPage(
+__SectorsPerPage(
IN ULONG SectorSize
)
{
@@ -476,7 +493,7 @@ __ProtocolSectorsPerPage(
}
static FORCEINLINE VOID
-__ProtocolOperation(
+__Operation(
IN UCHAR CdbOp,
OUT PUCHAR ProtocolOp,
OUT PBOOLEAN ReadOnly
@@ -496,22 +513,8 @@ __ProtocolOperation(
}
}
-static FORCEINLINE MM_PAGE_PRIORITY
-__ProtocolPriority(
- IN PXENVBD_PROTOCOL Protocol
- )
-{
- PXENVBD_CAPS Caps = FrontendGetCaps(Protocol->Frontend);
- if (!(Caps->Paging ||
- Caps->Hibernation ||
- Caps->DumpFile))
- return NormalPagePriority;
-
- return HighPagePriority;
-}
-
static FORCEINLINE VOID
-ProtocolRequestCopyOutput(
+__RequestCopyOutput(
IN PXENVBD_REQUEST Request
)
{
@@ -534,532 +537,136 @@ ProtocolRequestCopyOutput(
}
}
-static BOOLEAN
-ProtocolPrepareSegment(
- IN PXENVBD_PROTOCOL Protocol,
- IN PXENVBD_SEGMENT Segment,
- IN PXENVBD_SRBEXT SrbExt,
- IN BOOLEAN ReadOnly,
- IN ULONG SectorsLeft,
- OUT PULONG SectorsNow
+static FORCEINLINE VOID
+RingQueueRequestList(
+ IN PXENVBD_RING Ring,
+ IN PLIST_ENTRY List
)
{
- PFN_NUMBER Pfn;
- ULONG Offset;
- ULONG Length;
- NTSTATUS Status;
- PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol->Frontend);
- const ULONG SectorSize =
FrontendGetDiskInfo(Protocol->Frontend)->SectorSize;
- const ULONG SectorsPerPage = __ProtocolSectorsPerPage(SectorSize);
- PXENVBD_TARGET Target = FrontendGetTarget(Protocol->Frontend);
- PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
-
- Pfn = AdapterGetNextSGEntry(Adapter,
- SrbExt,
- 0,
- &Offset,
- &Length);
- if ((Offset & (SectorSize - 1)) == 0 &&
- (Length & (SectorSize - 1)) == 0) {
- ++Protocol->SegsGranted;
- // get first sector, last sector and count
- Segment->FirstSector = (UCHAR)((Offset + SectorSize - 1) /
SectorSize);
- *SectorsNow = __min(SectorsLeft, SectorsPerPage -
Segment->FirstSector);
- Segment->LastSector = (UCHAR)(Segment->FirstSector + *SectorsNow -
1);
-
- ASSERT3U((Length / SectorSize), ==, *SectorsNow);
- } else {
- PXENVBD_BOUNCE Bounce;
- PMDL Mdl;
-
- ++Protocol->SegsBounced;
- // get first sector, last sector and count
- Segment->FirstSector = 0;
- *SectorsNow = __min(SectorsLeft, SectorsPerPage);
- Segment->LastSector = (UCHAR)(*SectorsNow - 1);
-
- Bounce = AdapterGetBounce(Adapter);
- if (Bounce == NULL)
- goto fail1;
- Segment->Bounce = Bounce;
-
-#pragma warning(push)
-#pragma warning(disable:28145)
- Mdl = &Bounce->SourceMdl;
- Mdl->Next = NULL;
- Mdl->Size = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER));
- Mdl->MdlFlags = MDL_PAGES_LOCKED;
- Mdl->Process = NULL;
- Mdl->MappedSystemVa = NULL;
- Mdl->StartVa = NULL;
- Mdl->ByteCount = Length;
- Mdl->ByteOffset = Offset;
- Bounce->SourcePfn[0] = Pfn;
-
- if (Length < *SectorsNow * SectorSize) {
- Pfn = AdapterGetNextSGEntry(Adapter,
- SrbExt,
- Length,
- &Offset,
- &Length);
- Mdl->Size += sizeof(PFN_NUMBER);
- Mdl->ByteCount += Length;
- Bounce->SourcePfn[1] = Pfn;
- }
-#pragma warning(pop)
-
- ASSERT((Mdl->ByteCount & (SectorSize - 1)) == 0);
- ASSERT3U(Mdl->ByteCount, <=, PAGE_SIZE);
- ASSERT3U(*SectorsNow, ==, (Mdl->ByteCount / SectorSize));
-
- Bounce->SourcePtr = MmMapLockedPagesSpecifyCache(Mdl,
- KernelMode,
- MmCached,
- NULL,
- FALSE,
-
__ProtocolPriority(Protocol));
- if (Bounce->SourcePtr == NULL)
- goto fail2;
-
- ASSERT3P(MmGetMdlPfnArray(Mdl)[0], ==, Bounce->SourcePfn[0]);
- ASSERT3P(MmGetMdlPfnArray(Mdl)[1], ==, Bounce->SourcePfn[1]);
+ for (;;) {
+ PXENVBD_REQUEST Request;
+ PLIST_ENTRY ListEntry;
- // copy contents in
- if (ReadOnly) { // Operation == BLKIF_OP_WRITE
- RtlCopyMemory(Bounce->BouncePtr,
- Bounce->SourcePtr,
- MmGetMdlByteCount(&Bounce->SourceMdl));
- }
+ ListEntry = RemoveHeadList(List);
+ if (ListEntry == List)
+ break;
- Pfn = MmGetMdlPfnArray(Bounce->BounceMdl)[0];
+ Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
+ __RingIncBlkifOpCount(Ring, Request);
+ QueueAppend(&Ring->PreparedReqs, &Request->ListEntry);
}
-
- // Grant segment's page
- Status = GranterGet(Granter, Pfn, ReadOnly, &Segment->Grant);
- if (!NT_SUCCESS(Status))
- goto fail3;
-
- return TRUE;
-
-fail3:
-fail2:
-fail1:
- return FALSE;
}
static BOOLEAN
-ProtocolPrepareBlkifReadWrite(
- IN PXENVBD_PROTOCOL Protocol,
- IN PXENVBD_REQUEST Request,
- IN PXENVBD_SRBEXT SrbExt,
- IN ULONG MaxSegments,
- IN ULONG64 SectorStart,
- IN ULONG SectorsLeft,
- OUT PULONG SectorsDone
+RingSubmit(
+ IN PXENVBD_RING Ring,
+ IN PXENVBD_REQUEST Request
)
{
- PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
- UCHAR Operation;
- BOOLEAN ReadOnly;
- ULONG Index;
- __ProtocolOperation(Cdb_OperationEx(Srb), &Operation, &ReadOnly);
-
- Request->Operation = Operation;
- Request->NrSegments = 0;
- Request->FirstSector = SectorStart;
-
- for (Index = 0;
- Index < MaxSegments &&
- SectorsLeft > 0;
- ++Index) {
- PXENVBD_SEGMENT Segment;
- ULONG SectorsNow;
-
- Segment = ProtocolGetSegment(Protocol);
- if (Segment == NULL)
- goto fail1;
-
- InsertTailList(&Request->Segments, &Segment->ListEntry);
- ++Request->NrSegments;
-
- if (!ProtocolPrepareSegment(Protocol,
- Segment,
- SrbExt,
- ReadOnly,
- SectorsLeft,
- &SectorsNow))
- goto fail2;
+ KIRQL Irql;
+ blkif_request_t* req;
+ BOOLEAN Notify;
+ PXENVBD_PROTOCOL Protocol;
- *SectorsDone += SectorsNow;
- SectorsLeft -= SectorsNow;
+ KeAcquireSpinLock(&Ring->Lock, &Irql);
+ if (RING_FULL(&Ring->Front)) {
+ KeReleaseSpinLock(&Ring->Lock, Irql);
+ return FALSE;
}
- ASSERT3U(Request->NrSegments, >, 0);
- ASSERT3U(Request->NrSegments, <=, MaxSegments);
-
- return TRUE;
-
-fail2:
-fail1:
- return FALSE;
-}
-static BOOLEAN
-ProtocolPrepareBlkifIndirect(
- IN PXENVBD_PROTOCOL Protocol,
- IN PXENVBD_REQUEST Request
- )
-{
- ULONG Index;
- ULONG NrSegments = 0;
+ req = RING_GET_REQUEST(&Ring->Front, Ring->Front.req_prod_pvt);
+ __RingInsert(Ring, Request, req);
+ KeMemoryBarrier();
+ ++Ring->Front.req_prod_pvt;
- for (Index = 0;
- Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
- NrSegments < Request->NrSegments;
- ++Index) {
- PXENVBD_INDIRECT Indirect;
+ RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&Ring->Front, Notify);
+ KeReleaseSpinLock(&Ring->Lock, Irql);
- Indirect = ProtocolGetIndirect(Protocol);
- if (Indirect == NULL)
- goto fail1;
- InsertTailList(&Request->Indirects, &Indirect->ListEntry);
+ if (Notify) {
+ Protocol = Ring->Protocol;
+ if (!Protocol->Enabled)
+ return TRUE;
- NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE;
+ XENBUS_EVTCHN(Send,
+ &Protocol->EvtchnInterface,
+ Ring->Channel);
}
return TRUE;
-
-fail1:
- return FALSE;
}
-static FORCEINLINE ULONG
-ProtocolUseIndirect(
- IN PXENVBD_PROTOCOL Protocol,
- IN ULONG SectorsLeft
+static FORCEINLINE BOOLEAN
+RingSubmitRequests(
+ IN PXENVBD_RING Ring
)
{
- const ULONG SectorsPerPage =
__ProtocolSectorsPerPage(FrontendGetDiskInfo(Protocol->Frontend)->SectorSize);
- const ULONG MaxIndirectSegs =
FrontendGetFeatures(Protocol->Frontend)->Indirect;
-
- if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST)
- return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported
-
- if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * SectorsPerPage)
- return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single
BLKIF_OP_{READ/WRITE}
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
- return MaxIndirectSegs;
-}
+ if (!Protocol->Enabled) {
+ if (QueueCount(&Ring->PreparedReqs))
+ Warning("Target[%d][%u] : Paused, not submitting new requests
(%u)\n",
+ FrontendGetTargetId(Protocol->Frontend),
+ Ring->Index,
+ QueueCount(&Ring->PreparedReqs));
+ return FALSE;
+ }
-static FORCEINLINE VOID
-ProtocolQueueRequestList(
- IN PXENVBD_PROTOCOL Protocol,
- IN PLIST_ENTRY List
- )
-{
for (;;) {
PXENVBD_REQUEST Request;
PLIST_ENTRY ListEntry;
- ListEntry = RemoveHeadList(List);
- if (ListEntry == List)
+ ListEntry = QueuePop(&Ring->PreparedReqs);
+ if (ListEntry == NULL)
break;
Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
- __ProtocolIncBlkifOpCount(Protocol, Request);
- QueueAppend(&Protocol->PreparedReqs, &Request->ListEntry);
+
+ QueueAppend(&Ring->SubmittedReqs, &Request->ListEntry);
+ KeMemoryBarrier();
+
+ if (RingSubmit(Ring, Request))
+ continue;
+
+ QueueRemove(&Ring->SubmittedReqs, &Request->ListEntry);
+ QueueUnPop(&Ring->PreparedReqs, &Request->ListEntry);
+ break;
}
+
+ return QueueCount(&Ring->PreparedReqs) != 0;
}
static FORCEINLINE VOID
-ProtocolCancelRequestList(
- IN PXENVBD_PROTOCOL Protocol,
- IN PLIST_ENTRY List
+RingCompleteShutdown(
+ IN PXENVBD_RING Ring
)
{
+ PXENVBD_PROTOCOL Protocol;
+ PXENVBD_TARGET Target;
+ PXENVBD_ADAPTER Adapter;
+
+ if (QueueCount(&Ring->ShutdownSrbs) == 0)
+ return;
+
+ if (QueueCount(&Ring->PreparedReqs) ||
+ QueueCount(&Ring->SubmittedReqs))
+ return;
+
+ Protocol = Ring->Protocol;
+ Target = FrontendGetTarget(Protocol->Frontend);
+ Adapter = TargetGetAdapter(Target);
for (;;) {
- PXENVBD_REQUEST Request;
- PLIST_ENTRY ListEntry;
+ PXENVBD_SRBEXT SrbExt;
+ PSCSI_REQUEST_BLOCK Srb;
+ PLIST_ENTRY ListEntry;
- ListEntry = RemoveHeadList(List);
- if (ListEntry == List)
+ ListEntry = QueuePop(&Ring->ShutdownSrbs);
+ if (ListEntry == NULL)
break;
-
- Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
- ProtocolPutRequest(Protocol, Request);
- }
-}
-
-static BOOLEAN
-ProtocolPrepareReadWrite(
- IN PXENVBD_PROTOCOL Protocol,
- IN PXENVBD_SRBEXT SrbExt
- )
-{
- PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
- ULONG64 SectorStart = Cdb_LogicalBlock(Srb);
- ULONG SectorsLeft = Cdb_TransferBlock(Srb);
- LIST_ENTRY List;
-
- Srb->SrbStatus = SRB_STATUS_PENDING;
-
- InitializeListHead(&List);
- SrbExt->RequestCount = 0;
-
- while (SectorsLeft > 0) {
- ULONG MaxSegments;
- ULONG SectorsDone = 0;
- PXENVBD_REQUEST Request;
-
- Request = ProtocolGetRequest(Protocol);
- if (Request == NULL)
- goto fail1;
- InsertTailList(&List, &Request->ListEntry);
- InterlockedIncrement(&SrbExt->RequestCount);
-
- Request->SrbExt = SrbExt;
- MaxSegments = ProtocolUseIndirect(Protocol, SectorsLeft);
-
- if (!ProtocolPrepareBlkifReadWrite(Protocol,
- Request,
- SrbExt,
- MaxSegments,
- SectorStart,
- SectorsLeft,
- &SectorsDone))
- goto fail2;
-
- if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) {
- if (!ProtocolPrepareBlkifIndirect(Protocol, Request))
- goto fail3;
- }
-
- SectorsLeft -= SectorsDone;
- SectorStart += SectorsDone;
- }
-
- ProtocolQueueRequestList(Protocol, &List);
- return TRUE;
-
-fail3:
-fail2:
-fail1:
- ProtocolCancelRequestList(Protocol, &List);
- SrbExt->RequestCount = 0;
- Srb->SrbStatus = SRB_STATUS_ERROR;
- return FALSE;
-}
-
-static BOOLEAN
-ProtocolPrepareSyncCache(
- IN PXENVBD_PROTOCOL Protocol,
- IN PXENVBD_SRBEXT SrbExt
- )
-{
- PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
- PXENVBD_REQUEST Request;
- LIST_ENTRY List;
- UCHAR Operation;
-
- Srb->SrbStatus = SRB_STATUS_PENDING;
-
- if (FrontendGetDiskInfo(Protocol->Frontend)->FlushCache)
- Operation = BLKIF_OP_FLUSH_DISKCACHE;
- else
- Operation = BLKIF_OP_WRITE_BARRIER;
-
- InitializeListHead(&List);
- SrbExt->RequestCount = 0;
-
- Request = ProtocolGetRequest(Protocol);
- if (Request == NULL)
- goto fail1;
- InsertTailList(&List, &Request->ListEntry);
- InterlockedIncrement(&SrbExt->RequestCount);
-
- Request->SrbExt = SrbExt;
- Request->Operation = Operation;
- Request->FirstSector = Cdb_LogicalBlock(Srb);
-
- ProtocolQueueRequestList(Protocol, &List);
- return TRUE;
-
-fail1:
- ProtocolCancelRequestList(Protocol, &List);
- SrbExt->RequestCount = 0;
- Srb->SrbStatus = SRB_STATUS_ERROR;
- return FALSE;
-}
-
-static BOOLEAN
-ProtocolPrepareUnmap(
- IN PXENVBD_PROTOCOL Protocol,
- IN PXENVBD_SRBEXT SrbExt
- )
-{
- PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
- PUNMAP_LIST_HEADER Unmap = Srb->DataBuffer;
- ULONG Count =
_byteswap_ushort(*(PUSHORT)Unmap->BlockDescrDataLength) /
sizeof(UNMAP_BLOCK_DESCRIPTOR);
- ULONG Index;
- LIST_ENTRY List;
-
- Srb->SrbStatus = SRB_STATUS_PENDING;
-
- InitializeListHead(&List);
- SrbExt->RequestCount = 0;
-
- for (Index = 0; Index < Count; ++Index) {
- PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index];
- PXENVBD_REQUEST Request;
-
- Request = ProtocolGetRequest(Protocol);
- if (Request == NULL)
- goto fail1;
- InsertTailList(&List, &Request->ListEntry);
- InterlockedIncrement(&SrbExt->RequestCount);
-
- Request->SrbExt = SrbExt;
- Request->Operation = BLKIF_OP_DISCARD;
- Request->FirstSector =
_byteswap_uint64(*(PULONG64)Descr->StartingLba);
- Request->NrSectors = _byteswap_ulong(*(PULONG)Descr->LbaCount);
- Request->Flags = 0;
- }
-
- ProtocolQueueRequestList(Protocol, &List);
- return TRUE;
-
-fail1:
- ProtocolCancelRequestList(Protocol, &List);
- SrbExt->RequestCount = 0;
- Srb->SrbStatus = SRB_STATUS_ERROR;
- return FALSE;
-}
-
-static FORCEINLINE BOOLEAN
-ProtocolPrepareRequest(
- IN PXENVBD_PROTOCOL Protocol,
- IN PXENVBD_SRBEXT SrbExt
- )
-{
- switch (Cdb_OperationEx(SrbExt->Srb)) {
- case SCSIOP_READ:
- case SCSIOP_WRITE:
- return ProtocolPrepareReadWrite(Protocol, SrbExt);
-
- case SCSIOP_SYNCHRONIZE_CACHE:
- return ProtocolPrepareSyncCache(Protocol, SrbExt);
-
- case SCSIOP_UNMAP:
- return ProtocolPrepareUnmap(Protocol, SrbExt);
-
- default:
- ASSERT(FALSE);
- return FALSE;
- }
-}
-
-static BOOLEAN
-ProtocolSubmit(
- IN PXENVBD_PROTOCOL Protocol,
- IN PXENVBD_REQUEST Request
- )
-{
- KIRQL Irql;
- blkif_request_t* req;
- BOOLEAN Notify;
-
- KeAcquireSpinLock(&Protocol->Lock, &Irql);
- if (RING_FULL(&Protocol->Front)) {
- KeReleaseSpinLock(&Protocol->Lock, Irql);
- return FALSE;
- }
-
- req = RING_GET_REQUEST(&Protocol->Front, Protocol->Front.req_prod_pvt);
- __ProtocolInsert(Protocol, Request, req);
- KeMemoryBarrier();
- ++Protocol->Front.req_prod_pvt;
-
- RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&Protocol->Front, Notify);
- KeReleaseSpinLock(&Protocol->Lock, Irql);
-
- if (Notify) {
- if (!Protocol->Enabled)
- return TRUE;
-
- XENBUS_EVTCHN(Send,
- &Protocol->EvtchnInterface,
- Protocol->Channel);
- }
-
- return TRUE;
-}
-
-static FORCEINLINE BOOLEAN
-ProtocolSubmitRequests(
- IN PXENVBD_PROTOCOL Protocol
- )
-{
- if (!Protocol->Enabled) {
- if (QueueCount(&Protocol->PreparedReqs))
- Warning("Target[%d] : Paused, not submitting new requests (%u)\n",
- FrontendGetTargetId(Protocol->Frontend),
- QueueCount(&Protocol->PreparedReqs));
- return FALSE;
- }
-
- for (;;) {
- PXENVBD_REQUEST Request;
- PLIST_ENTRY ListEntry;
-
- ListEntry = QueuePop(&Protocol->PreparedReqs);
- if (ListEntry == NULL)
- break;
-
- Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
-
- QueueAppend(&Protocol->SubmittedReqs, &Request->ListEntry);
- KeMemoryBarrier();
-
- if (ProtocolSubmit(Protocol, Request))
- continue;
-
- QueueRemove(&Protocol->SubmittedReqs, &Request->ListEntry);
- QueueUnPop(&Protocol->PreparedReqs, &Request->ListEntry);
- break;
- }
-
- return QueueCount(&Protocol->PreparedReqs) != 0;
-}
-
-static FORCEINLINE VOID
-ProtocolCompleteShutdown(
- IN PXENVBD_PROTOCOL Protocol
- )
-{
- PXENVBD_TARGET Target;
- PXENVBD_ADAPTER Adapter;
-
- if (QueueCount(&Protocol->ShutdownSrbs) == 0)
- return;
-
- if (QueueCount(&Protocol->PreparedReqs) ||
- QueueCount(&Protocol->SubmittedReqs))
- return;
-
- Target = FrontendGetTarget(Protocol->Frontend);
- Adapter = TargetGetAdapter(Target);
- for (;;) {
- PXENVBD_SRBEXT SrbExt;
- PSCSI_REQUEST_BLOCK Srb;
- PLIST_ENTRY ListEntry;
-
- ListEntry = QueuePop(&Protocol->ShutdownSrbs);
- if (ListEntry == NULL)
- break;
- SrbExt = CONTAINING_RECORD(ListEntry, XENVBD_SRBEXT, ListEntry);
- Srb = SrbExt->Srb;
-
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
- AdapterCompleteSrb(Adapter, SrbExt);
+ SrbExt = CONTAINING_RECORD(ListEntry, XENVBD_SRBEXT, ListEntry);
+ Srb = SrbExt->Srb;
+
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+ AdapterCompleteSrb(Adapter, SrbExt);
}
}
@@ -1081,8 +688,8 @@ __BlkifOperationName(
}
static VOID
-ProtocolCompleteResponse(
- IN PXENVBD_PROTOCOL Protocol,
+RingCompleteResponse(
+ IN PXENVBD_RING Ring,
IN ULONG64 Id,
IN SHORT Status
)
@@ -1090,8 +697,9 @@ ProtocolCompleteResponse(
PXENVBD_REQUEST Request;
PSCSI_REQUEST_BLOCK Srb;
PXENVBD_SRBEXT SrbExt;
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
- Request = ProtocolFindRequest(Protocol, Id);
+ Request = RingFindRequest(Ring, Id);
if (Request == NULL)
return;
@@ -1100,7 +708,7 @@ ProtocolCompleteResponse(
switch (Status) {
case BLKIF_RSP_OKAY:
- ProtocolRequestCopyOutput(Request);
+ __RequestCopyOutput(Request);
break;
case BLKIF_RSP_EOPNOTSUPP:
@@ -1112,8 +720,9 @@ ProtocolCompleteResponse(
case BLKIF_RSP_ERROR:
default:
- Warning("Target[%d] : %s BLKIF_RSP_ERROR (Tag %llx)\n",
+ Warning("Target[%d][%u] : %s BLKIF_RSP_ERROR (Tag %llx)\n",
FrontendGetTargetId(Protocol->Frontend),
+ Ring->Index,
__BlkifOperationName(Request->Operation),
Id);
Srb->SrbStatus = SRB_STATUS_ERROR;
@@ -1142,14 +751,15 @@ ProtocolCompleteResponse(
}
static BOOLEAN
-ProtocolPoll(
- IN PXENVBD_PROTOCOL Protocol
+RingPoll(
+ IN PXENVBD_RING Ring
)
{
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
BOOLEAN Retry = FALSE;
ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
- KeAcquireSpinLockAtDpcLevel(&Protocol->Lock);
+ KeAcquireSpinLockAtDpcLevel(&Ring->Lock);
// Guard against this locked region being called after the
// lock on FrontendSetState
@@ -1162,8 +772,8 @@ ProtocolPoll(
KeMemoryBarrier();
- rsp_prod = Protocol->Shared->rsp_prod;
- rsp_cons = Protocol->Front.rsp_cons;
+ rsp_prod = Ring->Shared->rsp_prod;
+ rsp_cons = Ring->Front.rsp_cons;
KeMemoryBarrier();
@@ -1173,105 +783,1018 @@ ProtocolPoll(
while (rsp_cons != rsp_prod && !Retry) {
blkif_response_t* rsp;
- rsp = RING_GET_RESPONSE(&Protocol->Front, rsp_cons);
+ rsp = RING_GET_RESPONSE(&Ring->Front, rsp_cons);
++rsp_cons;
- ++Protocol->Received;
+ ++Ring->Received;
- ProtocolCompleteResponse(Protocol, rsp->id, rsp->status);
+ RingCompleteResponse(Ring, rsp->id, rsp->status);
RtlZeroMemory(rsp, sizeof(union blkif_sring_entry));
- if (rsp_cons - Protocol->Front.rsp_cons >
RING_SIZE(&Protocol->Front) / 4)
+ if (rsp_cons - Ring->Front.rsp_cons > RING_SIZE(&Ring->Front) / 4)
Retry = TRUE;
}
KeMemoryBarrier();
- Protocol->Front.rsp_cons = rsp_cons;
- Protocol->Shared->rsp_event = rsp_cons + 1;
+ Ring->Front.rsp_cons = rsp_cons;
+ Ring->Shared->rsp_event = rsp_cons + 1;
}
done:
- KeReleaseSpinLockFromDpcLevel(&Protocol->Lock);
+ KeReleaseSpinLockFromDpcLevel(&Ring->Lock);
return Retry;
}
__drv_requiresIRQL(DISPATCH_LEVEL)
static BOOLEAN
-ProtocolNotifyResponses(
- IN PXENVBD_PROTOCOL Protocol
+RingNotifyResponses(
+ IN PXENVBD_RING Ring
)
{
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
BOOLEAN Retry = FALSE;
if (!Protocol->Enabled)
return FALSE;
- Retry |= ProtocolPoll(Protocol);
- Retry |= ProtocolSubmitRequests(Protocol);
+ Retry |= RingPoll(Ring);
+ Retry |= RingSubmitRequests(Ring);
- ProtocolCompleteShutdown(Protocol);
+ RingCompleteShutdown(Ring);
return Retry;
}
-KSERVICE_ROUTINE ProtocolInterrupt;
+KSERVICE_ROUTINE RingInterrupt;
BOOLEAN
-ProtocolInterrupt(
+RingInterrupt(
IN PKINTERRUPT Interrupt,
IN PVOID Context
)
{
- PXENVBD_PROTOCOL Protocol = Context;
+ PXENVBD_RING Ring = Context;
+ PXENVBD_PROTOCOL Protocol;
UNREFERENCED_PARAMETER(Interrupt);
+ ASSERT(Ring != NULL);
+ Protocol = Ring->Protocol;
ASSERT(Protocol != NULL);
- ++Protocol->Events;
+ ++Ring->Events;
if (!Protocol->Connected)
return TRUE;
- if (KeInsertQueueDpc(&Protocol->Dpc, NULL, NULL))
- ++Protocol->Dpcs;
+ if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
+ ++Ring->Dpcs;
return TRUE;
}
-KDEFERRED_ROUTINE ProtocolDpc;
+KDEFERRED_ROUTINE RingDpc;
VOID
-ProtocolDpc(
+RingDpc(
__in PKDPC Dpc,
__in_opt PVOID Context,
__in_opt PVOID Arg1,
__in_opt PVOID Arg2
)
{
- PXENVBD_PROTOCOL Protocol = Context;
+ PXENVBD_RING Ring = Context;
+ PXENVBD_PROTOCOL Protocol;
UNREFERENCED_PARAMETER(Dpc);
UNREFERENCED_PARAMETER(Arg1);
UNREFERENCED_PARAMETER(Arg2);
+ ASSERT(Ring != NULL);
+ Protocol = Ring->Protocol;
ASSERT(Protocol != NULL);
for (;;) {
KIRQL Irql;
BOOLEAN Retry;
- KeRaiseIrql(DISPATCH_LEVEL, &Irql);
- Retry = ProtocolNotifyResponses(Protocol);
- KeLowerIrql(Irql);
+ KeRaiseIrql(DISPATCH_LEVEL, &Irql);
+ Retry = RingNotifyResponses(Ring);
+ KeLowerIrql(Irql);
+
+ if (!Retry)
+ break;
+ }
+
+ XENBUS_EVTCHN(Unmask,
+ &Protocol->EvtchnInterface,
+ Ring->Channel,
+ FALSE);
+}
+
+static DECLSPEC_NOINLINE VOID
+RingDebugCallback(
+ IN PVOID Argument,
+ IN BOOLEAN Crashing
+ )
+{
+ PXENVBD_RING Ring = Argument;
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol->Frontend);
+ ULONG Index;
+
+ UNREFERENCED_PARAMETER(Crashing);
+
+ XENBUS_DEBUG(Printf,
+ &Protocol->DebugInterface,
+ "Submitted: %u Received: %u\n",
+ Ring->Submitted,
+ Ring->Received);
+
+ XENBUS_DEBUG(Printf,
+ &Protocol->DebugInterface,
+ "Events: %u Dpcs: %u\n",
+ Ring->Events,
+ Ring->Dpcs);
+
+ XENBUS_DEBUG(Printf,
+ &Protocol->DebugInterface,
+ "Shared : 0x%p\n",
+ Ring->Shared);
+
+ if (Ring->Shared) {
+ XENBUS_DEBUG(Printf,
+ &Protocol->DebugInterface,
+ "Shared: %d / %d - %d / %d\n",
+ Ring->Shared->req_prod,
+ Ring->Shared->req_event,
+ Ring->Shared->rsp_prod,
+ Ring->Shared->rsp_event);
+ }
+
+ XENBUS_DEBUG(Printf,
+ &Protocol->DebugInterface,
+ "Front: %d / %d (%d)\n",
+ Ring->Front.req_prod_pvt,
+ Ring->Front.rsp_cons,
+ Ring->Front.nr_ents);
+
+ for (Index = 0; Index < (1ul << Protocol->Order); ++Index) {
+ XENBUS_DEBUG(Printf,
+ &Protocol->DebugInterface,
+ "Grants[%-2d]: 0x%p (%u)\n",
+ Index,
+ Ring->Grants[Index],
+ GranterReference(Granter, Ring->Grants[Index]));
+ }
+
+ if (Ring->Channel) {
+ ULONG Port = XENBUS_EVTCHN(GetPort,
+ &Protocol->EvtchnInterface,
+ Ring->Channel);
+
+ XENBUS_DEBUG(Printf,
+ &Protocol->DebugInterface,
+ "Channel : %p (%d)\n",
+ Ring->Channel,
+ Port);
+ }
+
+ XENBUS_DEBUG(Printf,
+ &Protocol->DebugInterface,
+ "BLKIF_OPs: READ=%u WRITE=%u\n",
+ Ring->BlkOpRead,
+ Ring->BlkOpWrite);
+ XENBUS_DEBUG(Printf,
+ &Protocol->DebugInterface,
+ "BLKIF_OPs: INDIRECT_READ=%u INDIRECT_WRITE=%u\n",
+ Ring->BlkOpIndirectRead,
+ Ring->BlkOpIndirectWrite);
+ XENBUS_DEBUG(Printf,
+ &Protocol->DebugInterface,
+ "BLKIF_OPs: BARRIER=%u DISCARD=%u FLUSH=%u\n",
+ Ring->BlkOpBarrier,
+ Ring->BlkOpDiscard,
+ Ring->BlkOpFlush);
+
+ QueueDebugCallback(&Ring->PreparedReqs,
+ "Prepared ",
+ &Protocol->DebugInterface);
+ QueueDebugCallback(&Ring->SubmittedReqs,
+ "Submitted",
+ &Protocol->DebugInterface);
+ QueueDebugCallback(&Ring->ShutdownSrbs,
+ "Shutdown ",
+ &Protocol->DebugInterface);
+}
+
+static NTSTATUS
+RingCreate(
+ IN PXENVBD_PROTOCOL Protocol,
+ IN ULONG Index,
+ OUT PXENVBD_RING *Ring
+ )
+{
+ PXENVBD_FRONTEND Frontend = Protocol->Frontend;
+ PROCESSOR_NUMBER ProcNumber;
+ ULONG Length;
+ NTSTATUS status;
+
+ *Ring = __ProtocolAllocate(sizeof(XENVBD_RING));
+
+ status = STATUS_NO_MEMORY;
+ if (*Ring == NULL)
+ goto fail1;
+
+ (*Ring)->Protocol = Protocol;
+ (*Ring)->Index = Index;
+
+ Length = 1 + (ULONG)strlen(FrontendGetFrontendPath(Frontend)) +
(ULONG)strlen("queue-xx");
+
+ (*Ring)->Path = __ProtocolAllocate(sizeof(CHAR) * Length);
+
+ status = STATUS_NO_MEMORY;
+ if ((*Ring)->Path == NULL)
+ goto fail2;
+
+ status = RtlStringCchPrintfA((*Ring)->Path,
+ Length,
+ "%s/queue-%u",
+ FrontendGetFrontendPath(Frontend),
+ Index);
+ if (!NT_SUCCESS(status))
+ goto fail3;
+
+ KeInitializeSpinLock(&(*Ring)->Lock);
+
+ KeInitializeThreadedDpc(&(*Ring)->Dpc, RingDpc, *Ring);
+ KeSetImportanceDpc(&(*Ring)->Dpc, MediumHighImportance);
+
+ status = KeGetProcessorNumberFromIndex(Index, &ProcNumber);
+ ASSERT(NT_SUCCESS(status));
+
+ KeSetTargetProcessorDpcEx(&(*Ring)->Dpc, &ProcNumber);
+
+ QueueInit(&(*Ring)->PreparedReqs);
+ QueueInit(&(*Ring)->SubmittedReqs);
+ QueueInit(&(*Ring)->ShutdownSrbs);
+
+ return STATUS_SUCCESS;
+
+fail3:
+ Error("fail3\n");
+ __ProtocolFree((*Ring)->Path);
+ (*Ring)->Path = NULL;
+fail2:
+ Error("fail2\n");
+ (*Ring)->Index = 0;
+ (*Ring)->Protocol = NULL;
+
+ ASSERT(IsZeroMemory(*Ring, sizeof(XENVBD_RING)));
+ __ProtocolFree(*Ring);
+fail1:
+ Error("fail1 (%08x)\n", status);
+ *Ring = NULL;
+ return status;
+}
+
+static VOID
+RingDestroy(
+ IN PXENVBD_RING Ring
+ )
+{
+ RtlZeroMemory(&Ring->PreparedReqs, sizeof(XENVBD_QUEUE));
+ RtlZeroMemory(&Ring->SubmittedReqs, sizeof(XENVBD_QUEUE));
+ RtlZeroMemory(&Ring->ShutdownSrbs, sizeof(XENVBD_QUEUE));
+
+ RtlZeroMemory(&Ring->Dpc, sizeof(KDPC));
+ RtlZeroMemory(&Ring->Lock, sizeof(KSPIN_LOCK));
+
+ Ring->BlkOpRead = 0;
+ Ring->BlkOpWrite = 0;
+ Ring->BlkOpIndirectRead = 0;
+ Ring->BlkOpIndirectWrite = 0;
+ Ring->BlkOpBarrier = 0;
+ Ring->BlkOpDiscard = 0;
+ Ring->BlkOpFlush = 0;
+
+ __ProtocolFree(Ring->Path);
+ Ring->Path = NULL;
+
+ Ring->Index = 0;
+ Ring->Protocol = NULL;
+
+ ASSERT(IsZeroMemory(Ring, sizeof(XENVBD_RING)));
+ __ProtocolFree(Ring);
+}
+
+static NTSTATUS
+RingConnect(
+ IN PXENVBD_RING Ring
+ )
+{
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol->Frontend);
+ CHAR Name[MAX_NAME_LEN + 1];
+ ULONG Index;
+ NTSTATUS status;
+
+ Ring->Mdl = __AllocatePages(1 << Protocol->Order);
+
+ status = STATUS_NO_MEMORY;
+ if (Ring->Mdl == NULL)
+ goto fail1;
+
+ Ring->Shared = MmGetSystemAddressForMdlSafe(Ring->Mdl,
+ NormalPagePriority);
+ ASSERT(Ring->Shared != NULL);
+
+#pragma warning(push)
+#pragma warning(disable: 4305)
+#pragma warning(disable: 4311) // 'type cast' pointer truncation from
'blkif_sring_entry[1]' to 'long'
+ SHARED_RING_INIT(Ring->Shared);
+ FRONT_RING_INIT(&Ring->Front, Ring->Shared, PAGE_SIZE << Protocol->Order);
+#pragma warning(pop)
+
+ for (Index = 0; Index < (1ul << Protocol->Order); ++Index) {
+ status = GranterGet(Granter,
+ MmGetMdlPfnArray(Ring->Mdl)[Index],
+ FALSE,
+ &Ring->Grants[Index]);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+ }
+
+ Ring->Channel = XENBUS_EVTCHN(Open,
+ &Protocol->EvtchnInterface,
+ XENBUS_EVTCHN_TYPE_UNBOUND,
+ RingInterrupt,
+ Ring,
+ FrontendGetBackendDomain(Protocol->Frontend),
+ TRUE);
+ status = STATUS_NO_MEMORY;
+ if (Ring->Channel == NULL)
+ goto fail3;
+
+ XENBUS_EVTCHN(Unmask,
+ &Protocol->EvtchnInterface,
+ Ring->Channel,
+ FALSE);
+
+ status = RtlStringCchPrintfA(Name,
+ MAX_NAME_LEN,
+ __MODULE__"%s",
+ Ring->Path);
+ if (!NT_SUCCESS(status))
+ goto fail4;
+
+ status = XENBUS_DEBUG(Register,
+ &Protocol->DebugInterface,
+ Name,
+ RingDebugCallback,
+ Ring,
+ &Ring->DebugCallback);
+ if (!NT_SUCCESS(status))
+ goto fail5;
+
+ return STATUS_SUCCESS;
+
+fail5:
+ Error("fail5\n");
+fail4:
+ Error("fail4\n");
+ XENBUS_EVTCHN(Close,
+ &Protocol->EvtchnInterface,
+ Ring->Channel);
+ Ring->Channel = NULL;
+fail3:
+ Error("fail3\n");
+fail2:
+ Error("fail2\n");
+ for (Index = 0; Index < (1ul << Protocol->Order); ++Index) {
+ if (Ring->Grants[Index] == NULL)
+ continue;
+
+ GranterPut(Granter, Ring->Grants[Index]);
+ Ring->Grants[Index] = NULL;
+ }
+
+ RtlZeroMemory(&Ring->Front, sizeof(blkif_front_ring_t));
+
+ __FreePages(Ring->Mdl);
+ Ring->Shared = NULL;
+ Ring->Mdl = NULL;
+fail1:
+ Error("fail1 (%08x)\n", status);
+ return status;
+}
+
+static NTSTATUS
+RingStoreWrite(
+ IN PXENVBD_RING Ring,
+ IN PVOID Transaction
+ )
+{
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol->Frontend);
+ PCHAR Path;
+ ULONG Port;
+ NTSTATUS status;
+
+ if (FrontendGetMultiQueueMaxQueues(Protocol->Frontend) > 1)
+ Path = Ring->Path;
+ else
+ Path = FrontendGetFrontendPath(Protocol->Frontend);
+
+ if (Protocol->Order == 0) {
+ status = XENBUS_STORE(Printf,
+ &Protocol->StoreInterface,
+ Transaction,
+ Path,
+ "ring-ref",
+ "%u",
+ GranterReference(Granter, Ring->Grants[0]));
+ if (!NT_SUCCESS(status))
+ return status;
+ }
+ else {
+ ULONG Index;
+
+ for (Index = 0; Index < (1ul << Protocol->Order); ++Index) {
+ CHAR Name[MAX_NAME_LEN + 1];
+
+ status = RtlStringCchPrintfA(Name,
+ MAX_NAME_LEN,
+ "ring-ref%u",
+ Index);
+ if (!NT_SUCCESS(status))
+ return status;
+
+ status = XENBUS_STORE(Printf,
+ &Protocol->StoreInterface,
+ Transaction,
+ Path,
+ Name,
+ "%u",
+ GranterReference(Granter,
Ring->Grants[Index]));
+ if (!NT_SUCCESS(status))
+ return status;
+ }
+ }
+
+ Port = XENBUS_EVTCHN(GetPort,
+ &Protocol->EvtchnInterface,
+ Ring->Channel);
+
+ status = XENBUS_STORE(Printf,
+ &Protocol->StoreInterface,
+ Transaction,
+ Path,
+ "event-channel",
+ "%u",
+ Port);
+ if (!NT_SUCCESS(status))
+ return status;
+
+ return STATUS_SUCCESS;
+}
+
+static VOID
+RingDisconnect(
+ IN PXENVBD_RING Ring
+ )
+{
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol->Frontend);
+ ULONG Index;
+
+ ASSERT3U(Ring->Submitted, == , Ring->Received);
+
+ XENBUS_DEBUG(Deregister,
+ &Protocol->DebugInterface,
+ Ring->DebugCallback);
+ Ring->DebugCallback = NULL;
+
+ XENBUS_EVTCHN(Close,
+ &Protocol->EvtchnInterface,
+ Ring->Channel);
+ Ring->Channel = NULL;
+
+ for (Index = 0; Index < (1ul << Protocol->Order); ++Index) {
+ if (Ring->Grants[Index] == NULL)
+ continue;
+
+ GranterPut(Granter, Ring->Grants[Index]);
+ Ring->Grants[Index] = NULL;
+ }
+
+ RtlZeroMemory(&Ring->Front, sizeof(blkif_front_ring_t));
+
+ __FreePages(Ring->Mdl);
+ Ring->Shared = NULL;
+ Ring->Mdl = NULL;
+ Ring->Events = 0;
+ Ring->Dpcs = 0;
+ Ring->Submitted = 0;
+ Ring->Received = 0;
+}
+
+static VOID
+RingEnable(
+ IN PXENVBD_RING Ring
+ )
+{
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
+
+ XENBUS_EVTCHN(Trigger,
+ &Protocol->EvtchnInterface,
+ Ring->Channel);
+}
+
+static VOID
+RingDisable(
+ IN PXENVBD_RING Ring
+ )
+{
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
+ ULONG Count;
+ KIRQL Irql;
+ PXENVBD_TARGET Target = FrontendGetTarget(Protocol->Frontend);
+ PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
+
+ // poll ring and send event channel notification every 1ms (for up to 3
minutes)
+ Count = 0;
+ while (QueueCount(&Ring->SubmittedReqs)) {
+ if (Count > 180000)
+ break;
+ KeRaiseIrql(DISPATCH_LEVEL, &Irql);
+ RingPoll(Ring);
+ KeLowerIrql(Irql);
+ XENBUS_EVTCHN(Send,
+ &Protocol->EvtchnInterface,
+ Ring->Channel);
+ StorPortStallExecution(1000); // 1000 micro-seconds
+ ++Count;
+ }
+
+ Verbose("Target[%d][%u] : %u Submitted requests left (%u iterrations)\n",
+ FrontendGetTargetId(Protocol->Frontend),
+ Ring->Index,
+ QueueCount(&Ring->SubmittedReqs),
+ Count);
+
+ // Fail PreparedReqs
+ for (;;) {
+ PXENVBD_SRBEXT SrbExt;
+ PSCSI_REQUEST_BLOCK Srb;
+ PXENVBD_REQUEST Request;
+ PLIST_ENTRY ListEntry;
+
+ ListEntry = QueuePop(&Ring->PreparedReqs);
+ if (ListEntry == NULL)
+ break;
+ Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
+ SrbExt = Request->SrbExt;
+ Srb = SrbExt->Srb;
+
+ Srb->SrbStatus = SRB_STATUS_ABORTED;
+ Srb->ScsiStatus = 0x40; // SCSI_ABORTED
+
+ ProtocolPutRequest(Protocol, Request);
+
+ if (InterlockedDecrement(&SrbExt->RequestCount) == 0)
+ AdapterCompleteSrb(Adapter, SrbExt);
+ }
+}
+
+static VOID
+RingTrigger(
+ IN PXENVBD_RING Ring
+ )
+{
+ PXENVBD_PROTOCOL Protocol = Ring->Protocol;
+
+ XENBUS_EVTCHN(Trigger,
+ &Protocol->EvtchnInterface,
+ Ring->Channel);
+}
+
+static FORCEINLINE MM_PAGE_PRIORITY
+__ProtocolPriority(
+ IN PXENVBD_PROTOCOL Protocol
+ )
+{
+ PXENVBD_CAPS Caps = FrontendGetCaps(Protocol->Frontend);
+ if (!(Caps->Paging ||
+ Caps->Hibernation ||
+ Caps->DumpFile))
+ return NormalPagePriority;
+
+ return HighPagePriority;
+}
+
+static BOOLEAN
+ProtocolPrepareSegment(
+ IN PXENVBD_PROTOCOL Protocol,
+ IN PXENVBD_SEGMENT Segment,
+ IN PXENVBD_SRBEXT SrbExt,
+ IN BOOLEAN ReadOnly,
+ IN ULONG SectorsLeft,
+ OUT PULONG SectorsNow
+ )
+{
+ PFN_NUMBER Pfn;
+ ULONG Offset;
+ ULONG Length;
+ NTSTATUS Status;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol->Frontend);
+ const ULONG SectorSize =
FrontendGetDiskInfo(Protocol->Frontend)->SectorSize;
+ const ULONG SectorsPerPage = __SectorsPerPage(SectorSize);
+ PXENVBD_TARGET Target = FrontendGetTarget(Protocol->Frontend);
+ PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
+
+ Pfn = AdapterGetNextSGEntry(Adapter,
+ SrbExt,
+ 0,
+ &Offset,
+ &Length);
+ if ((Offset & (SectorSize - 1)) == 0 &&
+ (Length & (SectorSize - 1)) == 0) {
+ ++Protocol->SegsGranted;
+ // get first sector, last sector and count
+ Segment->FirstSector = (UCHAR)((Offset + SectorSize - 1) / SectorSize);
+ *SectorsNow = __min(SectorsLeft, SectorsPerPage -
Segment->FirstSector);
+ Segment->LastSector = (UCHAR)(Segment->FirstSector + *SectorsNow - 1);
+
+ ASSERT3U((Length / SectorSize), == , *SectorsNow);
+ }
+ else {
+ PXENVBD_BOUNCE Bounce;
+ PMDL Mdl;
+
+ ++Protocol->SegsBounced;
+ // get first sector, last sector and count
+ Segment->FirstSector = 0;
+ *SectorsNow = __min(SectorsLeft, SectorsPerPage);
+ Segment->LastSector = (UCHAR)(*SectorsNow - 1);
+
+ Bounce = AdapterGetBounce(Adapter);
+ if (Bounce == NULL)
+ goto fail1;
+ Segment->Bounce = Bounce;
+
+#pragma warning(push)
+#pragma warning(disable:28145)
+ Mdl = &Bounce->SourceMdl;
+ Mdl->Next = NULL;
+ Mdl->Size = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER));
+ Mdl->MdlFlags = MDL_PAGES_LOCKED;
+ Mdl->Process = NULL;
+ Mdl->MappedSystemVa = NULL;
+ Mdl->StartVa = NULL;
+ Mdl->ByteCount = Length;
+ Mdl->ByteOffset = Offset;
+ Bounce->SourcePfn[0] = Pfn;
+
+ if (Length < *SectorsNow * SectorSize) {
+ Pfn = AdapterGetNextSGEntry(Adapter,
+ SrbExt,
+ Length,
+ &Offset,
+ &Length);
+ Mdl->Size += sizeof(PFN_NUMBER);
+ Mdl->ByteCount += Length;
+ Bounce->SourcePfn[1] = Pfn;
+ }
+#pragma warning(pop)
+
+ ASSERT((Mdl->ByteCount & (SectorSize - 1)) == 0);
+ ASSERT3U(Mdl->ByteCount, <= , PAGE_SIZE);
+ ASSERT3U(*SectorsNow, == , (Mdl->ByteCount / SectorSize));
+
+ Bounce->SourcePtr = MmMapLockedPagesSpecifyCache(Mdl,
+ KernelMode,
+ MmCached,
+ NULL,
+ FALSE,
+
__ProtocolPriority(Protocol));
+ if (Bounce->SourcePtr == NULL)
+ goto fail2;
+
+ ASSERT3P(MmGetMdlPfnArray(Mdl)[0], == , Bounce->SourcePfn[0]);
+ ASSERT3P(MmGetMdlPfnArray(Mdl)[1], == , Bounce->SourcePfn[1]);
+
+ // copy contents in
+ if (ReadOnly) { // Operation == BLKIF_OP_WRITE
+ RtlCopyMemory(Bounce->BouncePtr,
+ Bounce->SourcePtr,
+ MmGetMdlByteCount(&Bounce->SourceMdl));
+ }
+
+ Pfn = MmGetMdlPfnArray(Bounce->BounceMdl)[0];
+ }
+
+ // Grant segment's page
+ Status = GranterGet(Granter, Pfn, ReadOnly, &Segment->Grant);
+ if (!NT_SUCCESS(Status))
+ goto fail3;
+
+ return TRUE;
+
+fail3:
+fail2:
+fail1:
+ return FALSE;
+}
+
+static BOOLEAN
+ProtocolPrepareBlkifReadWrite(
+ IN PXENVBD_PROTOCOL Protocol,
+ IN PXENVBD_REQUEST Request,
+ IN PXENVBD_SRBEXT SrbExt,
+ IN ULONG MaxSegments,
+ IN ULONG64 SectorStart,
+ IN ULONG SectorsLeft,
+ OUT PULONG SectorsDone
+ )
+{
+ PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
+ UCHAR Operation;
+ BOOLEAN ReadOnly;
+ ULONG Index;
+ __Operation(Cdb_OperationEx(Srb), &Operation, &ReadOnly);
+
+ Request->Operation = Operation;
+ Request->NrSegments = 0;
+ Request->FirstSector = SectorStart;
+
+ for (Index = 0;
+ Index < MaxSegments &&
+ SectorsLeft > 0;
+ ++Index) {
+ PXENVBD_SEGMENT Segment;
+ ULONG SectorsNow;
+
+ Segment = ProtocolGetSegment(Protocol);
+ if (Segment == NULL)
+ goto fail1;
+
+ InsertTailList(&Request->Segments, &Segment->ListEntry);
+ ++Request->NrSegments;
+
+ if (!ProtocolPrepareSegment(Protocol,
+ Segment,
+ SrbExt,
+ ReadOnly,
+ SectorsLeft,
+ &SectorsNow))
+ goto fail2;
+
+ *SectorsDone += SectorsNow;
+ SectorsLeft -= SectorsNow;
+ }
+ ASSERT3U(Request->NrSegments, >, 0);
+ ASSERT3U(Request->NrSegments, <= , MaxSegments);
+
+ return TRUE;
+
+fail2:
+fail1:
+ return FALSE;
+}
+
+static BOOLEAN
+ProtocolPrepareBlkifIndirect(
+ IN PXENVBD_PROTOCOL Protocol,
+ IN PXENVBD_REQUEST Request
+ )
+{
+ ULONG Index;
+ ULONG NrSegments = 0;
+
+ for (Index = 0;
+ Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
+ NrSegments < Request->NrSegments;
+ ++Index) {
+ PXENVBD_INDIRECT Indirect;
+
+ Indirect = ProtocolGetIndirect(Protocol);
+ if (Indirect == NULL)
+ goto fail1;
+ InsertTailList(&Request->Indirects, &Indirect->ListEntry);
+
+ NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE;
+ }
+
+ return TRUE;
+
+fail1:
+ return FALSE;
+}
+
+static FORCEINLINE ULONG
+ProtocolUseIndirect(
+ IN PXENVBD_PROTOCOL Protocol,
+ IN ULONG SectorsLeft
+ )
+{
+ const ULONG SectorsPerPage =
__SectorsPerPage(FrontendGetDiskInfo(Protocol->Frontend)->SectorSize);
+ const ULONG MaxIndirectSegs =
FrontendGetFeatures(Protocol->Frontend)->Indirect;
+
+ if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST)
+ return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported
+
+ if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * SectorsPerPage)
+ return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single
BLKIF_OP_{READ/WRITE}
+
+ return MaxIndirectSegs;
+}
+
+static FORCEINLINE VOID
+ProtocolCancelRequestList(
+ IN PXENVBD_PROTOCOL Protocol,
+ IN PLIST_ENTRY List
+ )
+{
+ for (;;) {
+ PXENVBD_REQUEST Request;
+ PLIST_ENTRY ListEntry;
+
+ ListEntry = RemoveHeadList(List);
+ if (ListEntry == List)
+ break;
+
+ Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
+ ProtocolPutRequest(Protocol, Request);
+ }
+}
+
+static BOOLEAN
+ProtocolPrepareReadWrite(
+ IN PXENVBD_PROTOCOL Protocol,
+ IN PXENVBD_SRBEXT SrbExt,
+ IN PLIST_ENTRY List
+ )
+{
+ PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
+ ULONG64 SectorStart = Cdb_LogicalBlock(Srb);
+ ULONG SectorsLeft = Cdb_TransferBlock(Srb);
+
+ Srb->SrbStatus = SRB_STATUS_PENDING;
+
+ SrbExt->RequestCount = 0;
+
+ while (SectorsLeft > 0) {
+ ULONG MaxSegments;
+ ULONG SectorsDone = 0;
+ PXENVBD_REQUEST Request;
+
+ Request = ProtocolGetRequest(Protocol);
+ if (Request == NULL)
+ goto fail1;
+ InsertTailList(List, &Request->ListEntry);
+ InterlockedIncrement(&SrbExt->RequestCount);
+
+ Request->SrbExt = SrbExt;
+ MaxSegments = ProtocolUseIndirect(Protocol, SectorsLeft);
+
+ if (!ProtocolPrepareBlkifReadWrite(Protocol,
+ Request,
+ SrbExt,
+ MaxSegments,
+ SectorStart,
+ SectorsLeft,
+ &SectorsDone))
+ goto fail2;
+
+ if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) {
+ if (!ProtocolPrepareBlkifIndirect(Protocol, Request))
+ goto fail3;
+ }
+
+ SectorsLeft -= SectorsDone;
+ SectorStart += SectorsDone;
+ }
+
+ return TRUE;
+
+fail3:
+fail2:
+fail1:
+ ProtocolCancelRequestList(Protocol, List);
+ SrbExt->RequestCount = 0;
+ Srb->SrbStatus = SRB_STATUS_ERROR;
+ return FALSE;
+}
+
+static BOOLEAN
+ProtocolPrepareSyncCache(
+ IN PXENVBD_PROTOCOL Protocol,
+ IN PXENVBD_SRBEXT SrbExt,
+ IN PLIST_ENTRY List
+ )
+{
+ PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
+ PXENVBD_REQUEST Request;
+ UCHAR Operation;
+
+ Srb->SrbStatus = SRB_STATUS_PENDING;
+
+ if (FrontendGetDiskInfo(Protocol->Frontend)->FlushCache)
+ Operation = BLKIF_OP_FLUSH_DISKCACHE;
+ else
+ Operation = BLKIF_OP_WRITE_BARRIER;
+
+ SrbExt->RequestCount = 0;
+
+ Request = ProtocolGetRequest(Protocol);
+ if (Request == NULL)
+ goto fail1;
+ InsertTailList(List, &Request->ListEntry);
+ InterlockedIncrement(&SrbExt->RequestCount);
+
+ Request->SrbExt = SrbExt;
+ Request->Operation = Operation;
+ Request->FirstSector = Cdb_LogicalBlock(Srb);
+
+ return TRUE;
+
+fail1:
+ ProtocolCancelRequestList(Protocol, List);
+ SrbExt->RequestCount = 0;
+ Srb->SrbStatus = SRB_STATUS_ERROR;
+ return FALSE;
+}
+
+static BOOLEAN
+ProtocolPrepareUnmap(
+ IN PXENVBD_PROTOCOL Protocol,
+ IN PXENVBD_SRBEXT SrbExt,
+ IN PLIST_ENTRY List
+ )
+{
+ PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
+ PUNMAP_LIST_HEADER Unmap = Srb->DataBuffer;
+ ULONG Count =
_byteswap_ushort(*(PUSHORT)Unmap->BlockDescrDataLength) /
sizeof(UNMAP_BLOCK_DESCRIPTOR);
+ ULONG Index;
+
+ Srb->SrbStatus = SRB_STATUS_PENDING;
+
+ SrbExt->RequestCount = 0;
+
+ for (Index = 0; Index < Count; ++Index) {
+ PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index];
+ PXENVBD_REQUEST Request;
+
+ Request = ProtocolGetRequest(Protocol);
+ if (Request == NULL)
+ goto fail1;
+ InsertTailList(List, &Request->ListEntry);
+ InterlockedIncrement(&SrbExt->RequestCount);
+
+ Request->SrbExt = SrbExt;
+ Request->Operation = BLKIF_OP_DISCARD;
+ Request->FirstSector = _byteswap_uint64(*(PULONG64)Descr->StartingLba);
+ Request->NrSectors = _byteswap_ulong(*(PULONG)Descr->LbaCount);
+ Request->Flags = 0;
+ }
+
+ return TRUE;
+
+fail1:
+ ProtocolCancelRequestList(Protocol, List);
+ SrbExt->RequestCount = 0;
+ Srb->SrbStatus = SRB_STATUS_ERROR;
+ return FALSE;
+}
+
+static FORCEINLINE BOOLEAN
+ProtocolPrepareRequest(
+ IN PXENVBD_PROTOCOL Protocol,
+ IN PXENVBD_SRBEXT SrbExt,
+ IN PLIST_ENTRY List
+ )
+{
+ switch (Cdb_OperationEx(SrbExt->Srb)) {
+ case SCSIOP_READ:
+ case SCSIOP_WRITE:
+ return ProtocolPrepareReadWrite(Protocol,
+ SrbExt,
+ List);
+
+ case SCSIOP_SYNCHRONIZE_CACHE:
+ return ProtocolPrepareSyncCache(Protocol,
+ SrbExt,
+ List);
+
+ case SCSIOP_UNMAP:
+ return ProtocolPrepareUnmap(Protocol,
+ SrbExt,
+ List);
- if (!Retry)
- break;
+ default:
+ ASSERT(FALSE);
+ return FALSE;
}
-
- XENBUS_EVTCHN(Unmask,
- &Protocol->EvtchnInterface,
- Protocol->Channel,
- FALSE);
}
static DECLSPEC_NOINLINE VOID
@@ -1281,102 +1804,19 @@ ProtocolDebugCallback(
)
{
PXENVBD_PROTOCOL Protocol = Argument;
- PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol->Frontend);
- ULONG Index;
UNREFERENCED_PARAMETER(Crashing);
- XENBUS_DEBUG(Printf,
- &Protocol->DebugInterface,
- "Submitted: %u Received: %u\n",
- Protocol->Submitted,
- Protocol->Received);
-
- XENBUS_DEBUG(Printf,
- &Protocol->DebugInterface,
- "Events: %u Dpcs: %u\n",
- Protocol->Events,
- Protocol->Dpcs);
-
- XENBUS_DEBUG(Printf,
- &Protocol->DebugInterface,
- "Shared : 0x%p\n",
- Protocol->Shared);
-
- if (Protocol->Shared) {
- XENBUS_DEBUG(Printf,
- &Protocol->DebugInterface,
- "Shared: %d / %d - %d / %d\n",
- Protocol->Shared->req_prod,
- Protocol->Shared->req_event,
- Protocol->Shared->rsp_prod,
- Protocol->Shared->rsp_event);
- }
-
- XENBUS_DEBUG(Printf,
- &Protocol->DebugInterface,
- "Front: %d / %d (%d)\n",
- Protocol->Front.req_prod_pvt,
- Protocol->Front.rsp_cons,
- Protocol->Front.nr_ents);
-
XENBUS_DEBUG(Printf,
&Protocol->DebugInterface,
"Order: %d\n",
Protocol->Order);
- for (Index = 0; Index < (1ul << Protocol->Order); ++Index) {
- XENBUS_DEBUG(Printf,
- &Protocol->DebugInterface,
- "Grants[%-2d]: 0x%p (%u)\n",
- Index,
- Protocol->Grants[Index],
- GranterReference(Granter, Protocol->Grants[Index]));
- }
-
- if (Protocol->Channel) {
- ULONG Port = XENBUS_EVTCHN(GetPort,
- &Protocol->EvtchnInterface,
- Protocol->Channel);
-
- XENBUS_DEBUG(Printf,
- &Protocol->DebugInterface,
- "Channel : %p (%d)\n",
- Protocol->Channel,
- Port);
- }
-
- XENBUS_DEBUG(Printf,
- &Protocol->DebugInterface,
- "BLKIF_OPs: READ=%u WRITE=%u\n",
- Protocol->BlkOpRead,
- Protocol->BlkOpWrite);
- XENBUS_DEBUG(Printf,
- &Protocol->DebugInterface,
- "BLKIF_OPs: INDIRECT_READ=%u INDIRECT_WRITE=%u\n",
- Protocol->BlkOpIndirectRead,
- Protocol->BlkOpIndirectWrite);
- XENBUS_DEBUG(Printf,
- &Protocol->DebugInterface,
- "BLKIF_OPs: BARRIER=%u DISCARD=%u FLUSH=%u\n",
- Protocol->BlkOpBarrier,
- Protocol->BlkOpDiscard,
- Protocol->BlkOpFlush);
XENBUS_DEBUG(Printf,
&Protocol->DebugInterface,
"Segments Granted=%llu Bounced=%llu\n",
Protocol->SegsGranted,
Protocol->SegsBounced);
-
- QueueDebugCallback(&Protocol->PreparedReqs,
- "Prepared ",
- &Protocol->DebugInterface);
- QueueDebugCallback(&Protocol->SubmittedReqs,
- "Submitted",
- &Protocol->DebugInterface);
- QueueDebugCallback(&Protocol->ShutdownSrbs,
- "Shutdown ",
- &Protocol->DebugInterface);
}
static DECLSPEC_NOINLINE VOID
@@ -1391,7 +1831,7 @@ ProtocolAcquireLock(
static DECLSPEC_NOINLINE VOID
ProtocolReleaseLock(
IN PVOID Argument
- )
+)
{
PXENVBD_PROTOCOL Protocol = Argument;
KeReleaseSpinLockFromDpcLevel(&Protocol->Lock);
@@ -1426,7 +1866,7 @@ static DECLSPEC_NOINLINE NTSTATUS
ProtocolSegmentCtor(
IN PVOID Argument,
IN PVOID Object
- )
+)
{
UNREFERENCED_PARAMETER(Argument);
UNREFERENCED_PARAMETER(Object);
@@ -1488,12 +1928,13 @@ ProtocolIndirectDtor(
NTSTATUS
ProtocolCreate(
IN PXENVBD_FRONTEND Frontend,
- OUT PXENVBD_PROTOCOL* Protocol
+ OUT PXENVBD_PROTOCOL* Protocol
)
{
PXENVBD_TARGET Target = FrontendGetTarget(Frontend);
PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
CHAR Name[MAX_NAME_LEN];
+ ULONG Index;
NTSTATUS status;
*Protocol = __ProtocolAllocate(sizeof(XENVBD_PROTOCOL));
@@ -1504,12 +1945,6 @@ ProtocolCreate(
(*Protocol)->Frontend = Frontend;
KeInitializeSpinLock(&(*Protocol)->Lock);
- KeInitializeThreadedDpc(&(*Protocol)->Dpc, ProtocolDpc, *Protocol);
- KeSetImportanceDpc(&(*Protocol)->Dpc, MediumHighImportance);
-
- QueueInit(&(*Protocol)->PreparedReqs);
- QueueInit(&(*Protocol)->SubmittedReqs);
- QueueInit(&(*Protocol)->ShutdownSrbs);
AdapterGetCacheInterface(Adapter, &(*Protocol)->CacheInterface);
@@ -1580,8 +2015,41 @@ ProtocolCreate(
if (!NT_SUCCESS(status))
goto fail8;
+ (*Protocol)->NumQueues = FrontendGetMultiQueueMaxQueues(Frontend);
+ if ((*Protocol)->NumQueues == 0)
+ (*Protocol)->NumQueues = 1;
+ (*Protocol)->Rings = __ProtocolAllocate(sizeof(PXENVBD_RING) *
(*Protocol)->NumQueues);
+
+ status = STATUS_NO_MEMORY;
+ if ((*Protocol)->Rings == NULL)
+ goto fail9;
+
+ for (Index = 0; Index < (*Protocol)->NumQueues; ++Index) {
+ status = RingCreate(*Protocol,
+ Index,
+ &(*Protocol)->Rings[Index]);
+ if (!NT_SUCCESS(status))
+ goto fail10;
+ }
+
return STATUS_SUCCESS;
+fail10:
+ Error("fail10\n");
+ for (Index = 0; Index < (*Protocol)->NumQueues; ++Index) {
+ if ((*Protocol)->Rings[Index])
+ RingDestroy((*Protocol)->Rings[Index]);
+ (*Protocol)->Rings[Index] = NULL;
+ }
+ __ProtocolFree((*Protocol)->Rings);
+ (*Protocol)->Rings = NULL;
+fail9:
+ Error("fail9\n");
+ (*Protocol)->NumQueues = 0;
+ XENBUS_CACHE(Destroy,
+ &(*Protocol)->CacheInterface,
+ (*Protocol)->IndirectCache);
+ (*Protocol)->IndirectCache = NULL;
fail8:
Error("fail8\n");
fail7:
@@ -1608,13 +2076,8 @@ fail2:
Error("fail2\n");
RtlZeroMemory(&(*Protocol)->CacheInterface,
- sizeof (XENBUS_CACHE_INTERFACE));
+ sizeof(XENBUS_CACHE_INTERFACE));
- RtlZeroMemory(&(*Protocol)->PreparedReqs, sizeof(XENVBD_QUEUE));
- RtlZeroMemory(&(*Protocol)->SubmittedReqs, sizeof(XENVBD_QUEUE));
- RtlZeroMemory(&(*Protocol)->ShutdownSrbs, sizeof(XENVBD_QUEUE));
-
- RtlZeroMemory(&(*Protocol)->Dpc, sizeof(KDPC));
RtlZeroMemory(&(*Protocol)->Lock, sizeof(KSPIN_LOCK));
(*Protocol)->Frontend = NULL;
@@ -1631,6 +2094,19 @@ ProtocolDestroy(
IN PXENVBD_PROTOCOL Protocol
)
{
+ ULONG Index;
+
+ for (Index = 0; Index < Protocol->NumQueues; ++Index) {
+ if (Protocol->Rings[Index])
+ RingDestroy(Protocol->Rings[Index]);
+ Protocol->Rings[Index] = 0;
+ }
+
+ __ProtocolFree(Protocol->Rings);
+ Protocol->Rings = NULL;
+
+ Protocol->NumQueues = 0;
+
XENBUS_CACHE(Destroy,
&Protocol->CacheInterface,
Protocol->IndirectCache);
@@ -1650,26 +2126,14 @@ ProtocolDestroy(
&Protocol->CacheInterface);
RtlZeroMemory(&Protocol->CacheInterface,
- sizeof (XENBUS_CACHE_INTERFACE));
+ sizeof(XENBUS_CACHE_INTERFACE));
- RtlZeroMemory(&Protocol->PreparedReqs, sizeof(XENVBD_QUEUE));
- RtlZeroMemory(&Protocol->SubmittedReqs, sizeof(XENVBD_QUEUE));
- RtlZeroMemory(&Protocol->ShutdownSrbs, sizeof(XENVBD_QUEUE));
+ Protocol->SegsGranted = 0;
+ Protocol->SegsBounced = 0;
- RtlZeroMemory(&Protocol->Dpc, sizeof(KDPC));
RtlZeroMemory(&Protocol->Lock, sizeof(KSPIN_LOCK));
Protocol->Frontend = NULL;
- Protocol->BlkOpRead = 0;
- Protocol->BlkOpWrite = 0;
- Protocol->BlkOpIndirectRead = 0;
- Protocol->BlkOpIndirectWrite = 0;
- Protocol->BlkOpBarrier = 0;
- Protocol->BlkOpDiscard = 0;
- Protocol->BlkOpFlush = 0;
- Protocol->SegsGranted = 0;
- Protocol->SegsBounced = 0;
-
ASSERT(IsZeroMemory(Protocol, sizeof(XENVBD_PROTOCOL)));
__ProtocolFree(Protocol);
}
@@ -1681,7 +2145,6 @@ ProtocolConnect(
{
PXENVBD_TARGET Target = FrontendGetTarget(Protocol->Frontend);
PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
- PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol->Frontend);
PCHAR Buffer;
ULONG Index;
NTSTATUS status;
@@ -1730,48 +2193,12 @@ ProtocolConnect(
Protocol->Order = 0;
}
- Protocol->Mdl = __AllocatePages(1 << Protocol->Order);
-
- status = STATUS_NO_MEMORY;
- if (Protocol->Mdl == NULL)
- goto fail4;
-
- Protocol->Shared = MmGetSystemAddressForMdlSafe(Protocol->Mdl,
- NormalPagePriority);
- ASSERT(Protocol->Shared != NULL);
-
-#pragma warning(push)
-#pragma warning(disable: 4305)
-#pragma warning(disable: 4311) // 'type cast' pointer truncation from
'blkif_sring_entry[1]' to 'long'
- SHARED_RING_INIT(Protocol->Shared);
- FRONT_RING_INIT(&Protocol->Front, Protocol->Shared, PAGE_SIZE <<
Protocol->Order);
-#pragma warning(pop)
-
- for (Index = 0; Index < (1ul << Protocol->Order); ++Index) {
- status = GranterGet(Granter,
- MmGetMdlPfnArray(Protocol->Mdl)[Index],
- FALSE,
- &Protocol->Grants[Index]);
+ for (Index = 0; Index < Protocol->NumQueues; ++Index) {
+ status = RingConnect(Protocol->Rings[Index]);
if (!NT_SUCCESS(status))
- goto fail5;
+ goto fail4;
}
- Protocol->Channel = XENBUS_EVTCHN(Open,
- &Protocol->EvtchnInterface,
- XENBUS_EVTCHN_TYPE_UNBOUND,
- ProtocolInterrupt,
- Protocol,
- FrontendGetBackendDomain(Protocol->Frontend),
- TRUE);
- status = STATUS_NO_MEMORY;
- if (Protocol->Channel == NULL)
- goto fail6;
-
- XENBUS_EVTCHN(Unmask,
- &Protocol->EvtchnInterface,
- Protocol->Channel,
- FALSE);
-
status = XENBUS_DEBUG(Register,
&Protocol->DebugInterface,
__MODULE__"|PROTOCOL",
@@ -1779,38 +2206,20 @@ ProtocolConnect(
Protocol,
&Protocol->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail7;
+ goto fail5;
Protocol->Connected = TRUE;
return STATUS_SUCCESS;
-fail7:
- Error("fail7\n");
- XENBUS_EVTCHN(Close,
- &Protocol->EvtchnInterface,
- Protocol->Channel);
- Protocol->Channel = NULL;
-fail6:
- Error("fail6\n");
fail5:
Error("fail5\n");
- for (Index = 0; Index < (1ul << Protocol->Order); ++Index) {
- if (Protocol->Grants[Index] == NULL)
- continue;
-
- GranterPut(Granter, Protocol->Grants[Index]);
- Protocol->Grants[Index] = NULL;
- }
-
- RtlZeroMemory(&Protocol->Front, sizeof(blkif_front_ring_t));
-
- __FreePages(Protocol->Mdl);
- Protocol->Shared = NULL;
- Protocol->Mdl = NULL;
-
- Protocol->Order = 0;
+ Index = Protocol->NumQueues;
fail4:
Error("fail4\n");
+ while (Index-- != 0) {
+ RingDisconnect(Protocol->Rings[Index]);
+ }
+ Protocol->Order = 0;
XENBUS_DEBUG(Release, &Protocol->DebugInterface);
fail3:
Error("fail3\n");
@@ -1834,26 +2243,20 @@ fail1:
NTSTATUS
ProtocolStoreWrite(
IN PXENVBD_PROTOCOL Protocol,
- IN PVOID Transaction
+ IN PVOID Transaction
)
{
- PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol->Frontend);
- ULONG Port;
- NTSTATUS status;
+ ULONG Index;
+ NTSTATUS status;
- if (Protocol->Order == 0) {
- status = XENBUS_STORE(Printf,
- &Protocol->StoreInterface,
- Transaction,
- FrontendGetFrontendPath(Protocol->Frontend),
- "ring-ref",
- "%u",
- GranterReference(Granter, Protocol->Grants[0]));
+ for (Index = 0; Index < Protocol->NumQueues; ++Index) {
+ status = RingStoreWrite(Protocol->Rings[Index],
+ Transaction);
if (!NT_SUCCESS(status))
return status;
- } else {
- ULONG Index;
+ }
+ if (Protocol->Order != 0) {
status = XENBUS_STORE(Printf,
&Protocol->StoreInterface,
Transaction,
@@ -1863,49 +2266,24 @@ ProtocolStoreWrite(
Protocol->Order);
if (!NT_SUCCESS(status))
return status;
-
- for (Index = 0; Index < (1ul << Protocol->Order); ++Index) {
- CHAR Name[MAX_NAME_LEN+1];
-
- status = RtlStringCchPrintfA(Name,
- MAX_NAME_LEN,
- "ring-ref%u",
- Index);
- if (!NT_SUCCESS(status))
- return status;
-
- status = XENBUS_STORE(Printf,
- &Protocol->StoreInterface,
- Transaction,
- FrontendGetFrontendPath(Protocol->Frontend),
- Name,
- "%u",
- GranterReference(Granter,
Protocol->Grants[Index]));
- if (!NT_SUCCESS(status))
- return status;
- }
}
status = XENBUS_STORE(Printf,
&Protocol->StoreInterface,
Transaction,
FrontendGetFrontendPath(Protocol->Frontend),
- "protocol",
- XEN_IO_PROTO_ABI);
+ "multi-queue-num-queues",
+ "%u",
+ Protocol->NumQueues);
if (!NT_SUCCESS(status))
return status;
- Port = XENBUS_EVTCHN(GetPort,
- &Protocol->EvtchnInterface,
- Protocol->Channel);
-
status = XENBUS_STORE(Printf,
&Protocol->StoreInterface,
Transaction,
FrontendGetFrontendPath(Protocol->Frontend),
- "event-channel",
- "%u",
- Port);
+ "protocol",
+ XEN_IO_PROTO_ABI);
if (!NT_SUCCESS(status))
return status;
@@ -1917,12 +2295,13 @@ ProtocolEnable(
IN PXENVBD_PROTOCOL Protocol
)
{
+ ULONG Index;
+
ASSERT(Protocol->Enabled == FALSE);
Protocol->Enabled = TRUE;
- XENBUS_EVTCHN(Trigger,
- &Protocol->EvtchnInterface,
- Protocol->Channel);
+ for (Index = 0; Index < Protocol->NumQueues; ++Index)
+ RingEnable(Protocol->Rings[Index]);
}
VOID
@@ -1930,56 +2309,13 @@ ProtocolDisable(
IN PXENVBD_PROTOCOL Protocol
)
{
- ULONG Count;
- KIRQL Irql;
- PXENVBD_TARGET Target = FrontendGetTarget(Protocol->Frontend);
- PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
+ ULONG Index;
ASSERT(Protocol->Enabled == TRUE);
Protocol->Enabled = FALSE;
- // poll ring and send event channel notification every 1ms (for up to 3
minutes)
- Count = 0;
- while (QueueCount(&Protocol->SubmittedReqs)) {
- if (Count > 180000)
- break;
- KeRaiseIrql(DISPATCH_LEVEL, &Irql);
- ProtocolPoll(Protocol);
- KeLowerIrql(Irql);
- XENBUS_EVTCHN(Send,
- &Protocol->EvtchnInterface,
- Protocol->Channel);
- StorPortStallExecution(1000); // 1000 micro-seconds
- ++Count;
- }
-
- Verbose("Target[%d] : %u Submitted requests left (%u iterrations)\n",
- FrontendGetTargetId(Protocol->Frontend),
- QueueCount(&Protocol->SubmittedReqs),
- Count);
-
- // Fail PreparedReqs
- for (;;) {
- PXENVBD_SRBEXT SrbExt;
- PSCSI_REQUEST_BLOCK Srb;
- PXENVBD_REQUEST Request;
- PLIST_ENTRY ListEntry;
-
- ListEntry = QueuePop(&Protocol->PreparedReqs);
- if (ListEntry == NULL)
- break;
- Request = CONTAINING_RECORD(ListEntry, XENVBD_REQUEST, ListEntry);
- SrbExt = Request->SrbExt;
- Srb = SrbExt->Srb;
-
- Srb->SrbStatus = SRB_STATUS_ABORTED;
- Srb->ScsiStatus = 0x40; // SCSI_ABORTED
-
- ProtocolPutRequest(Protocol, Request);
-
- if (InterlockedDecrement(&SrbExt->RequestCount) == 0)
- AdapterCompleteSrb(Adapter, SrbExt);
- }
+ for (Index = 0; Index < Protocol->NumQueues; ++Index)
+ RingDisable(Protocol->Rings[Index]);
}
VOID
@@ -1987,10 +2323,8 @@ ProtocolDisconnect(
IN PXENVBD_PROTOCOL Protocol
)
{
- PXENVBD_GRANTER Granter = FrontendGetGranter(Protocol->Frontend);
ULONG Index;
- ASSERT3U(Protocol->Submitted, ==, Protocol->Received);
ASSERT(Protocol->Connected);
Protocol->Connected = FALSE;
@@ -1999,24 +2333,8 @@ ProtocolDisconnect(
Protocol->DebugCallback);
Protocol->DebugCallback = NULL;
- XENBUS_EVTCHN(Close,
- &Protocol->EvtchnInterface,
- Protocol->Channel);
- Protocol->Channel = NULL;
-
- for (Index = 0; Index < (1ul << Protocol->Order); ++Index) {
- if (Protocol->Grants[Index] == NULL)
- continue;
-
- GranterPut(Granter, Protocol->Grants[Index]);
- Protocol->Grants[Index] = NULL;
- }
-
- RtlZeroMemory(&Protocol->Front, sizeof(blkif_front_ring_t));
-
- __FreePages(Protocol->Mdl);
- Protocol->Shared = NULL;
- Protocol->Mdl = NULL;
+ for (Index = 0; Index < Protocol->NumQueues; ++Index)
+ RingDisconnect(Protocol->Rings[Index]);
Protocol->Order = 0;
@@ -2030,11 +2348,6 @@ ProtocolDisconnect(
sizeof(XENBUS_EVTCHN_INTERFACE));
RtlZeroMemory(&Protocol->StoreInterface,
sizeof(XENBUS_STORE_INTERFACE));
-
- Protocol->Events = 0;
- Protocol->Dpcs = 0;
- Protocol->Submitted = 0;
- Protocol->Received = 0;
}
VOID
@@ -2042,12 +2355,29 @@ ProtocolTrigger(
IN PXENVBD_PROTOCOL Protocol
)
{
+ ULONG Index;
+
if (!Protocol->Enabled)
return;
- XENBUS_EVTCHN(Trigger,
- &Protocol->EvtchnInterface,
- Protocol->Channel);
+ for (Index = 0; Index < Protocol->NumQueues; ++Index)
+ RingTrigger(Protocol->Rings[Index]);
+}
+
+static FORCEINLINE PXENVBD_RING
+__ProtocolGetRing(
+ IN PXENVBD_PROTOCOL Protocol
+ )
+{
+ ULONG Index;
+
+ if (Protocol->NumQueues == 0)
+ return Protocol->Rings[0];
+
+ Index = KeGetCurrentProcessorNumberEx(NULL) % Protocol->NumQueues;
+ ASSERT(Index < Protocol->NumQueues);
+
+ return Protocol->Rings[Index];
}
BOOLEAN
@@ -2057,17 +2387,26 @@ ProtocolQueueRequest(
)
{
PSCSI_REQUEST_BLOCK Srb = SrbExt->Srb;
+ PXENVBD_RING Ring;
+ LIST_ENTRY List;
+
+ InitializeListHead(&List);
if (!Protocol->Enabled)
goto fail1;
- if (!ProtocolPrepareRequest(Protocol, SrbExt))
+ if (!ProtocolPrepareRequest(Protocol, SrbExt, &List))
goto fail2;
- if (ProtocolSubmitRequests(Protocol)) {
+ Ring = __ProtocolGetRing(Protocol);
+ ASSERT(Ring != NULL);
+
+ RingQueueRequestList(Ring, &List);
+
+ if (RingSubmitRequests(Ring)) {
// more prepared-reqs to submit
- if (KeInsertQueueDpc(&Protocol->Dpc, NULL, NULL))
- ++Protocol->Dpcs;
+ if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
+ ++Ring->Dpcs;
}
return TRUE;
@@ -2081,15 +2420,20 @@ fail1:
VOID
ProtocolQueueShutdown(
IN PXENVBD_PROTOCOL Protocol,
- IN PXENVBD_SRBEXT SrbExt
+ IN PXENVBD_SRBEXT SrbExt
)
{
- QueueAppend(&Protocol->ShutdownSrbs,
+ PXENVBD_RING Ring;
+
+ Ring = __ProtocolGetRing(Protocol);
+ ASSERT(Ring != NULL);
+
+ QueueAppend(&Ring->ShutdownSrbs,
&SrbExt->ListEntry);
if (!Protocol->Enabled)
return;
- if (KeInsertQueueDpc(&Protocol->Dpc, NULL, NULL))
- ++Protocol->Dpcs;
+ if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
+ ++Ring->Dpcs;
}
--
2.16.2.windows.1
_______________________________________________
win-pv-devel mailing list
win-pv-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/win-pv-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |