[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[win-pv-devel] [PATCH 3/7] Move request prepare/cleanup to ring



From: Owen Smith <owen.smith@xxxxxxxxxx>

Keep the blkif protocol related functions in the ring code, and
keep the target code related to SCSI operations

Signed-off-by: Owen Smith <owen.smith@xxxxxxxxxx>
---
 src/xenvbd/frontend.c |   42 +-
 src/xenvbd/frontend.h |   12 +-
 src/xenvbd/ring.c     | 1550 +++++++++++++++++++++++++++++++++++++++++++------
 src/xenvbd/ring.h     |   22 +-
 src/xenvbd/target.c   | 1281 +---------------------------------------
 src/xenvbd/target.h   |   12 -
 6 files changed, 1434 insertions(+), 1485 deletions(-)

diff --git a/src/xenvbd/frontend.c b/src/xenvbd/frontend.c
index fca360a..5014593 100644
--- a/src/xenvbd/frontend.c
+++ b/src/xenvbd/frontend.c
@@ -260,21 +260,6 @@ out:
 
 //=============================================================================
 __drv_requiresIRQL(DISPATCH_LEVEL)
-BOOLEAN
-FrontendNotifyResponses(
-    __in  PXENVBD_FRONTEND        Frontend
-    )
-{
-    BOOLEAN     Retry = FALSE;
-
-    Retry |= RingPoll(Frontend->Ring);
-    Retry |= TargetSubmitRequests(Frontend->Target);
-
-    return Retry;
-}
-
-//=============================================================================
-__drv_requiresIRQL(DISPATCH_LEVEL)
 static NTSTATUS
 __UpdateBackendPath(
     __in  PXENVBD_FRONTEND        Frontend
@@ -1574,6 +1559,33 @@ FrontendBackend(
 }
 
 NTSTATUS
+FrontendReset(
+    IN  PXENVBD_FRONTEND    Frontend
+    )
+{
+    XENVBD_STATE            PreviousState = Frontend->State;
+    NTSTATUS                status;
+
+    if (PreviousState != XENVBD_ENABLED)
+        goto done;
+
+    status = FrontendSetState(Frontend, XENVBD_CLOSED);
+    if (!NT_SUCCESS(status))
+        goto fail1;
+
+    status = FrontendSetState(Frontend, PreviousState);
+    if (!NT_SUCCESS(status))
+        goto fail2;
+
+done:
+    return STATUS_SUCCESS;
+
+fail2:
+fail1:
+    return status;
+}
+
+NTSTATUS
 FrontendCreate(
     IN  PXENVBD_TARGET          Target,
     IN  PCHAR                   DeviceId,
diff --git a/src/xenvbd/frontend.h b/src/xenvbd/frontend.h
index 49c121c..d608fef 100644
--- a/src/xenvbd/frontend.h
+++ b/src/xenvbd/frontend.h
@@ -108,13 +108,6 @@ FrontendWriteUsage(
     __in  PXENVBD_FRONTEND        Frontend
     );
 
-// Ring
-__drv_requiresIRQL(DISPATCH_LEVEL)
-extern BOOLEAN
-FrontendNotifyResponses(
-    __in  PXENVBD_FRONTEND        Frontend
-    );
-
 // Init/Term
 __checkReturn
 __drv_maxIRQL(DISPATCH_LEVEL)
@@ -137,6 +130,11 @@ FrontendSetState(
     );
 
 extern NTSTATUS
+FrontendReset(
+    IN  PXENVBD_FRONTEND    Frontend
+    );
+
+extern NTSTATUS
 FrontendCreate(
     IN  PXENVBD_TARGET          Target,
     IN  PCHAR                   DeviceId,
diff --git a/src/xenvbd/ring.c b/src/xenvbd/ring.c
index eb5fcc7..ffd271d 100755
--- a/src/xenvbd/ring.c
+++ b/src/xenvbd/ring.c
@@ -30,9 +30,11 @@
  */ 
 
 #include <ntddk.h>
+#include <storport.h>
 #include <stdlib.h>
 #include <ntstrsafe.h>
 
+#include <xencdb.h>
 #include <store_interface.h>
 #include <evtchn_interface.h>
 #include <debug_interface.h>
@@ -44,6 +46,8 @@
 #include "srbext.h"
 #include "driver.h"
 #include "granter.h"
+#include "queue.h"
+#include "buffer.h"
 
 #include "util.h"
 #include "debug.h"
@@ -53,6 +57,14 @@
 #define XENVBD_MAX_RING_PAGE_ORDER  (4)
 #define XENVBD_MAX_RING_PAGES       (1 << XENVBD_MAX_RING_PAGE_ORDER)
 
+typedef struct _XENVBD_LOOKASIDE {
+    LONG                        Used;
+    LONG                        Max;
+    ULONG                       Failed;
+    ULONG                       Size;
+    NPAGED_LOOKASIDE_LIST       List;
+} XENVBD_LOOKASIDE, *PXENVBD_LOOKASIDE;
+
 struct _XENVBD_RING {
     PXENVBD_FRONTEND                Frontend;
     BOOLEAN                         Connected;
@@ -75,16 +87,112 @@ struct _XENVBD_RING {
     KDPC                            TimerDpc;
     KTIMER                          Timer;
 
+    XENVBD_LOOKASIDE                RequestList;
+    XENVBD_LOOKASIDE                SegmentList;
+    XENVBD_LOOKASIDE                IndirectList;
+    XENVBD_QUEUE                    FreshSrbs;
+    XENVBD_QUEUE                    PreparedReqs;
+    XENVBD_QUEUE                    SubmittedReqs;
+    XENVBD_QUEUE                    ShutdownSrbs;
+    ULONG                           NextTag;
+
     ULONG                           Submitted;
     ULONG                           Received;
     ULONG                           Events;
     ULONG                           Dpcs;
+    ULONG                           BlkOpRead;
+    ULONG                           BlkOpWrite;
+    ULONG                           BlkOpIndirectRead;
+    ULONG                           BlkOpIndirectWrite;
+    ULONG                           BlkOpBarrier;
+    ULONG                           BlkOpDiscard;
+    ULONG                           BlkOpFlush;
+    ULONG64                         SegsGranted;
+    ULONG64                         SegsBounced;
 };
 
 #define MAX_NAME_LEN                64
 #define RING_POOL_TAG               'gnRX'
+#define REQUEST_POOL_TAG            'qeRX'
+#define SEGMENT_POOL_TAG            'geSX'
+#define INDIRECT_POOL_TAG           'dnIX'
 #define XEN_IO_PROTO_ABI            "x86_64-abi"
 
+static FORCEINLINE VOID
+__LookasideInit(
+    IN OUT  PXENVBD_LOOKASIDE   Lookaside,
+    IN  ULONG                   Size,
+    IN  ULONG                   Tag
+    )
+{
+    RtlZeroMemory(Lookaside, sizeof(XENVBD_LOOKASIDE));
+    Lookaside->Size = Size;
+    ExInitializeNPagedLookasideList(&Lookaside->List, NULL, NULL, 0,
+                                    Size, Tag, 0);
+}
+
+static FORCEINLINE VOID
+__LookasideTerm(
+    IN  PXENVBD_LOOKASIDE       Lookaside
+    )
+{
+    ASSERT3U(Lookaside->Used, ==, 0);
+    ExDeleteNPagedLookasideList(&Lookaside->List);
+    RtlZeroMemory(Lookaside, sizeof(XENVBD_LOOKASIDE));
+}
+
+static FORCEINLINE PVOID
+__LookasideAlloc(
+    IN  PXENVBD_LOOKASIDE       Lookaside
+    )
+{
+    LONG    Result;
+    PVOID   Buffer;
+
+    Buffer = ExAllocateFromNPagedLookasideList(&Lookaside->List);
+    if (Buffer == NULL) {
+        ++Lookaside->Failed;
+        return NULL;
+    }
+
+    RtlZeroMemory(Buffer, Lookaside->Size);
+    Result = InterlockedIncrement(&Lookaside->Used);
+    ASSERT3S(Result, >, 0);
+    if (Result > Lookaside->Max)
+        Lookaside->Max = Result;
+
+    return Buffer;
+}
+
+static FORCEINLINE VOID
+__LookasideFree(
+    IN  PXENVBD_LOOKASIDE       Lookaside,
+    IN  PVOID                   Buffer
+    )
+{
+    LONG            Result;
+
+    ExFreeToNPagedLookasideList(&Lookaside->List, Buffer);
+    Result = InterlockedDecrement(&Lookaside->Used);
+    ASSERT3S(Result, >=, 0);
+}
+
+static FORCEINLINE VOID
+__LookasideDebug(
+    IN  PXENVBD_LOOKASIDE           Lookaside,
+    IN  PXENBUS_DEBUG_INTERFACE     Debug,
+    IN  PCHAR                       Name
+    )
+{
+    XENBUS_DEBUG(Printf, Debug,
+                 "LOOKASIDE: %s: %u / %u (%u failed)\n",
+                 Name, Lookaside->Used,
+                 Lookaside->Max, Lookaside->Failed);
+
+    Lookaside->Max = Lookaside->Used;
+    Lookaside->Failed = 0;
+}
+
 static FORCEINLINE PVOID
 __RingAllocate(
     IN  ULONG                       Length
@@ -177,84 +285,1101 @@ __RingInsert(
             PLIST_ENTRY                 SegEntry;
             blkif_request_indirect_t*   req_indirect;
 
-            req_indirect = (blkif_request_indirect_t*)req;
-            req_indirect->operation         = BLKIF_OP_INDIRECT;
-            req_indirect->indirect_op       = Request->Operation;
-            req_indirect->nr_segments       = Request->NrSegments;
-            req_indirect->id                = __RingGetTag(Ring, Request);
-            req_indirect->sector_number     = Request->FirstSector;
-            req_indirect->handle            = 
(USHORT)FrontendGetDeviceId(Ring->Frontend);
+            req_indirect = (blkif_request_indirect_t*)req;
+            req_indirect->operation         = BLKIF_OP_INDIRECT;
+            req_indirect->indirect_op       = Request->Operation;
+            req_indirect->nr_segments       = Request->NrSegments;
+            req_indirect->id                = __RingGetTag(Ring, Request);
+            req_indirect->sector_number     = Request->FirstSector;
+            req_indirect->handle            = 
(USHORT)FrontendGetDeviceId(Ring->Frontend);
+
+            for (PageIdx = 0,
+                 PageEntry = Request->Indirects.Flink,
+                 SegEntry = Request->Segments.Flink;
+                    PageIdx < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
+                    PageEntry != &Request->Indirects &&
+                    SegEntry != &Request->Segments;
+                        ++PageIdx, PageEntry = PageEntry->Flink) {
+                PXENVBD_INDIRECT Page = CONTAINING_RECORD(PageEntry, 
XENVBD_INDIRECT, Entry);
+
+                req_indirect->indirect_grefs[PageIdx] = 
GranterReference(Granter, Page->Grant);
+
+                for (SegIdx = 0;
+                        SegIdx < XENVBD_MAX_SEGMENTS_PER_PAGE &&
+                        SegEntry != &Request->Segments;
+                            ++SegIdx, SegEntry = SegEntry->Flink) {
+                    PXENVBD_SEGMENT Segment = CONTAINING_RECORD(SegEntry, 
XENVBD_SEGMENT, Entry);
+
+                    Page->Page[SegIdx].GrantRef = GranterReference(Granter, 
Segment->Grant);
+                    Page->Page[SegIdx].First    = Segment->FirstSector;
+                    Page->Page[SegIdx].Last     = Segment->LastSector;
+                }
+            }
+        } else {
+            // Direct
+            ULONG           Index;
+            PLIST_ENTRY     Entry;
+
+            req->operation                  = Request->Operation;
+            req->nr_segments                = (UCHAR)Request->NrSegments;
+            req->handle                     = 
(USHORT)FrontendGetDeviceId(Ring->Frontend);
+            req->id                         = __RingGetTag(Ring, Request);
+            req->sector_number              = Request->FirstSector;
+
+            for (Index = 0, Entry = Request->Segments.Flink;
+                    Index < BLKIF_MAX_SEGMENTS_PER_REQUEST &&
+                    Entry != &Request->Segments;
+                        ++Index, Entry = Entry->Flink) {
+                PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry, 
XENVBD_SEGMENT, Entry);
+                req->seg[Index].gref        = GranterReference(Granter, 
Segment->Grant);
+                req->seg[Index].first_sect  = Segment->FirstSector;
+                req->seg[Index].last_sect   = Segment->LastSector;
+            }
+        }
+        break;
+
+    case BLKIF_OP_WRITE_BARRIER:
+    case BLKIF_OP_FLUSH_DISKCACHE:
+        req->operation                  = Request->Operation;
+        req->nr_segments                = 0;
+        req->handle                     = 
(USHORT)FrontendGetDeviceId(Ring->Frontend);
+        req->id                         = __RingGetTag(Ring, Request);
+        req->sector_number              = Request->FirstSector;
+        break;
+
+    case BLKIF_OP_DISCARD: {
+        blkif_request_discard_t*        req_discard;
+        req_discard = (blkif_request_discard_t*)req;
+        req_discard->operation          = BLKIF_OP_DISCARD;
+        req_discard->flag               = Request->Flags;
+        req_discard->handle             = 
(USHORT)FrontendGetDeviceId(Ring->Frontend);
+        req_discard->id                 = __RingGetTag(Ring, Request);
+        req_discard->sector_number      = Request->FirstSector;
+        req_discard->nr_sectors         = Request->NrSectors;
+        } break;
+
+    default:
+        ASSERT(FALSE);
+        break;
+    }
+    ++Ring->Submitted;
+}
+
+static PXENVBD_INDIRECT
+RingGetIndirect(
+    IN  PXENVBD_RING    Ring
+    )
+{
+    PXENVBD_INDIRECT    Indirect;
+    NTSTATUS            status;
+    PXENVBD_GRANTER     Granter = FrontendGetGranter(Ring->Frontend);
+
+    Indirect = __LookasideAlloc(&Ring->IndirectList);
+    if (Indirect == NULL)
+        goto fail1;
+
+    RtlZeroMemory(Indirect, sizeof(XENVBD_INDIRECT));
+
+    Indirect->Mdl = __AllocatePage();
+    if (Indirect->Mdl == NULL)
+        goto fail2;
+
+    Indirect->Page = MmGetSystemAddressForMdlSafe(Indirect->Mdl,
+                                                  NormalPagePriority);
+
+    status = GranterGet(Granter,
+                        MmGetMdlPfnArray(Indirect->Mdl)[0],
+                        TRUE,
+                        &Indirect->Grant);
+    if (!NT_SUCCESS(status))
+        goto fail3;
+
+    return Indirect;
+
+fail3:
+    __FreePage(Indirect->Mdl);
+fail2:
+    __LookasideFree(&Ring->IndirectList, Indirect);
+fail1:
+    return NULL;
+}
+
+static VOID
+RingPutIndirect(
+    IN  PXENVBD_RING        Ring,
+    IN  PXENVBD_INDIRECT    Indirect
+    )
+{
+    PXENVBD_GRANTER         Granter = FrontendGetGranter(Ring->Frontend);
+
+    if (Indirect->Grant)
+        GranterPut(Granter, Indirect->Grant);
+    if (Indirect->Page)
+        __FreePage(Indirect->Mdl);
+
+    RtlZeroMemory(Indirect, sizeof(XENVBD_INDIRECT));
+    __LookasideFree(&Ring->IndirectList, Indirect);
+}
+
+static PXENVBD_SEGMENT
+RingGetSegment(
+    IN  PXENVBD_RING    Ring
+    )
+{
+    PXENVBD_SEGMENT     Segment;
+
+    Segment = __LookasideAlloc(&Ring->SegmentList);
+    if (Segment == NULL)
+        goto fail1;
+
+    RtlZeroMemory(Segment, sizeof(XENVBD_SEGMENT));
+    return Segment;
+
+fail1:
+    return NULL;
+}
+
+static VOID
+RingPutSegment(
+    IN  PXENVBD_RING    Ring,
+    IN  PXENVBD_SEGMENT Segment
+    )
+{
+    PXENVBD_GRANTER     Granter = FrontendGetGranter(Ring->Frontend);
+
+    if (Segment->Grant)
+        GranterPut(Granter, Segment->Grant);
+
+    if (Segment->BufferId)
+        BufferPut(Segment->BufferId);
+
+    if (Segment->Buffer)
+        MmUnmapLockedPages(Segment->Buffer, &Segment->Mdl);
+
+    RtlZeroMemory(Segment, sizeof(XENVBD_SEGMENT));
+    __LookasideFree(&Ring->SegmentList, Segment);
+}
+
+static PXENVBD_REQUEST
+RingGetRequest(
+    IN  PXENVBD_RING    Ring
+    )
+{
+    PXENVBD_REQUEST     Request;
+
+    Request = __LookasideAlloc(&Ring->RequestList);
+    if (Request == NULL)
+        goto fail1;
+
+    RtlZeroMemory(Request, sizeof(XENVBD_REQUEST));
+    Request->Id = (ULONG)InterlockedIncrement((PLONG)&Ring->NextTag);
+    InitializeListHead(&Request->Segments);
+    InitializeListHead(&Request->Indirects);
+
+    return Request;
+
+fail1:
+    return NULL;
+}
+
+static VOID
+RingPutRequest(
+    IN  PXENVBD_RING    Ring,
+    IN  PXENVBD_REQUEST Request
+    )
+{
+    PLIST_ENTRY         Entry;
+
+    for (;;) {
+        PXENVBD_SEGMENT Segment;
+
+        Entry = RemoveHeadList(&Request->Segments);
+        if (Entry == &Request->Segments)
+            break;
+        Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, Entry);
+        RingPutSegment(Ring, Segment);
+    }
+
+    for (;;) {
+        PXENVBD_INDIRECT    Indirect;
+
+        Entry = RemoveHeadList(&Request->Indirects);
+        if (Entry == &Request->Indirects)
+            break;
+        Indirect = CONTAINING_RECORD(Entry, XENVBD_INDIRECT, Entry);
+        RingPutIndirect(Ring, Indirect);
+    }
+
+    RtlZeroMemory(Request, sizeof(XENVBD_REQUEST));
+    __LookasideFree(&Ring->RequestList, Request);
+}
+
+static FORCEINLINE PXENVBD_REQUEST
+RingRequestFromTag(
+    IN  PXENVBD_RING    Ring,
+    IN  ULONG           Tag
+    )
+{
+    KIRQL               Irql;
+    PLIST_ENTRY         Entry;
+    PXENVBD_QUEUE       Queue = &Ring->SubmittedReqs;
+
+    KeAcquireSpinLock(&Queue->Lock, &Irql);
+
+    for (Entry = Queue->List.Flink; Entry != &Queue->List; Entry = 
Entry->Flink) {
+        PXENVBD_REQUEST Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, 
Entry);
+        if (Request->Id == Tag) {
+            RemoveEntryList(&Request->Entry);
+            --Queue->Current;
+            KeReleaseSpinLock(&Queue->Lock, Irql);
+            return Request;
+        }
+    }
+
+    KeReleaseSpinLock(&Queue->Lock, Irql);
+    Warning("Target[%d] : Tag %x not found in submitted list (%u items)\n",
+            FrontendGetTargetId(Ring->Frontend),
+            Tag,
+            QueueCount(Queue));
+    return NULL;
+}
+
+static FORCEINLINE VOID
+__RingIncBlkifOpCount(
+    IN  PXENVBD_RING    Ring,
+    IN  PXENVBD_REQUEST Request
+    )
+{
+    switch (Request->Operation) {
+    case BLKIF_OP_READ:
+        if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
+            ++Ring->BlkOpIndirectRead;
+        else
+            ++Ring->BlkOpRead;
+        break;
+    case BLKIF_OP_WRITE:
+        if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
+            ++Ring->BlkOpIndirectWrite;
+        else
+            ++Ring->BlkOpWrite;
+        break;
+    case BLKIF_OP_WRITE_BARRIER:
+        ++Ring->BlkOpBarrier;
+        break;
+    case BLKIF_OP_DISCARD:
+        ++Ring->BlkOpDiscard;
+        break;
+    case BLKIF_OP_FLUSH_DISKCACHE:
+        ++Ring->BlkOpFlush;
+        break;
+    default:
+        ASSERT(FALSE);
+        break;
+    }
+}
+
+static FORCEINLINE ULONG
+__RingSectorsPerPage(
+    IN  ULONG   SectorSize
+    )
+{
+    ASSERT3U(SectorSize, !=, 0);
+    return PAGE_SIZE / SectorSize;
+}
+
+static FORCEINLINE VOID
+__RingOperation(
+    IN  UCHAR       CdbOp,
+    OUT PUCHAR      RingOp,
+    OUT PBOOLEAN    ReadOnly
+    )
+{
+    switch (CdbOp) {
+    case SCSIOP_READ:
+        *RingOp     = BLKIF_OP_READ;
+        *ReadOnly   = FALSE;
+        break;
+    case SCSIOP_WRITE:
+        *RingOp     = BLKIF_OP_WRITE;
+        *ReadOnly   = TRUE;
+        break;
+    default:
+        ASSERT(FALSE);
+    }
+}
+
+static FORCEINLINE MM_PAGE_PRIORITY
+__RingPriority(
+    IN  PXENVBD_RING    Ring
+    )
+{
+    PXENVBD_CAPS        Caps = FrontendGetCaps(Ring->Frontend);
+    if (!(Caps->Paging ||
+          Caps->Hibernation ||
+          Caps->DumpFile))
+        return NormalPagePriority;
+
+    return HighPagePriority;
+}
+
+static FORCEINLINE VOID
+RingRequestCopyOutput(
+    IN  PXENVBD_REQUEST Request
+    )
+{
+    PLIST_ENTRY         Entry;
+
+    if (Request->Operation != BLKIF_OP_READ)
+        return;
+
+    for (Entry = Request->Segments.Flink;
+            Entry != &Request->Segments;
+            Entry = Entry->Flink) {
+        PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, 
Entry);
+
+        if (Segment->BufferId)
+            BufferCopyOut(Segment->BufferId, Segment->Buffer, Segment->Length);
+    }
+}
+
+static BOOLEAN
+RingPrepareSegment(
+    IN  PXENVBD_RING    Ring,
+    IN  PXENVBD_SEGMENT Segment,
+    IN  PXENVBD_SRBEXT  SrbExt,
+    IN  BOOLEAN         ReadOnly,
+    IN  ULONG           SectorsLeft,
+    OUT PULONG          SectorsNow
+    )
+{
+    PFN_NUMBER          Pfn;
+    ULONG               Offset;
+    ULONG               Length;
+    NTSTATUS            Status;
+    PXENVBD_GRANTER     Granter = FrontendGetGranter(Ring->Frontend);
+    const ULONG         SectorSize = 
FrontendGetDiskInfo(Ring->Frontend)->SectorSize;
+    const ULONG         SectorsPerPage = __RingSectorsPerPage(SectorSize);
+    PXENVBD_TARGET      Target = FrontendGetTarget(Ring->Frontend);
+
+    Pfn = AdapterGetNextSGEntry(TargetGetAdapter(Target),
+                                SrbExt,
+                                0,
+                                &Offset,
+                                &Length);
+    if ((Offset & (SectorSize - 1)) == 0 &&
+        (Length & (SectorSize - 1)) == 0) {
+        ++Ring->SegsGranted;
+        // get first sector, last sector and count
+        Segment->FirstSector    = (UCHAR)((Offset + SectorSize - 1) / 
SectorSize);
+        *SectorsNow             = __min(SectorsLeft, SectorsPerPage - 
Segment->FirstSector);
+        Segment->LastSector     = (UCHAR)(Segment->FirstSector + *SectorsNow - 
1);
+        Segment->BufferId       = NULL; // granted, ensure its null
+        Segment->Buffer         = NULL; // granted, ensure its null
+        Segment->Length         = 0;    // granted, ensure its 0
+
+        ASSERT3U((Length / SectorSize), ==, *SectorsNow);
+    } else {
+        PMDL        Mdl;
+
+        ++Ring->SegsBounced;
+        // get first sector, last sector and count
+        Segment->FirstSector    = 0;
+        *SectorsNow             = __min(SectorsLeft, SectorsPerPage);
+        Segment->LastSector     = (UCHAR)(*SectorsNow - 1);
+
+        // map SGList to Virtual Address. Populates Segment->Buffer and 
Segment->Length
+#pragma warning(push)
+#pragma warning(disable:28145)
+        Mdl = &Segment->Mdl;
+        Mdl->Next           = NULL;
+        Mdl->Size           = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER));
+        Mdl->MdlFlags       = MDL_PAGES_LOCKED;
+        Mdl->Process        = NULL;
+        Mdl->MappedSystemVa = NULL;
+        Mdl->StartVa        = NULL;
+        Mdl->ByteCount      = Length;
+        Mdl->ByteOffset     = Offset;
+        Segment->Pfn[0]     = Pfn;
+
+        if (Length < *SectorsNow * SectorSize) {
+            Pfn = AdapterGetNextSGEntry(TargetGetAdapter(Target),
+                                        SrbExt,
+                                        Length,
+                                        &Offset,
+                                        &Length);
+            Mdl->Size       += sizeof(PFN_NUMBER);
+            Mdl->ByteCount  = Mdl->ByteCount + Length;
+            Segment->Pfn[1] = Pfn;
+        }
+#pragma warning(pop)
+
+        ASSERT((Mdl->ByteCount & (SectorSize - 1)) == 0);
+        ASSERT3U(Mdl->ByteCount, <=, PAGE_SIZE);
+        ASSERT3U(*SectorsNow, ==, (Mdl->ByteCount / SectorSize));
+
+        Segment->Length = __min(Mdl->ByteCount, PAGE_SIZE);
+        Segment->Buffer = MmMapLockedPagesSpecifyCache(Mdl, KernelMode,
+                                MmCached, NULL, FALSE, __RingPriority(Ring));
+        if (!Segment->Buffer)
+            goto fail1;
+
+        ASSERT3P(MmGetMdlPfnArray(Mdl)[0], ==, Segment->Pfn[0]);
+        ASSERT3P(MmGetMdlPfnArray(Mdl)[1], ==, Segment->Pfn[1]);
+
+        // get a buffer
+        if (!BufferGet(Segment, &Segment->BufferId, &Pfn))
+            goto fail2;
+
+        // copy contents in
+        if (ReadOnly) { // Operation == BLKIF_OP_WRITE
+            BufferCopyIn(Segment->BufferId, Segment->Buffer, Segment->Length);
+        }
+    }
+
+    // Grant segment's page
+    Status = GranterGet(Granter, Pfn, ReadOnly, &Segment->Grant);
+    if (!NT_SUCCESS(Status))
+        goto fail3;
+
+    return TRUE;
+
+fail3:
+fail2:
+fail1:
+    return FALSE;
+}
+
+static BOOLEAN
+RingPrepareBlkifReadWrite(
+    IN  PXENVBD_RING    Ring,
+    IN  PXENVBD_REQUEST Request,
+    IN  PXENVBD_SRBEXT  SrbExt,
+    IN  ULONG           MaxSegments,
+    IN  ULONG64         SectorStart,
+    IN  ULONG           SectorsLeft,
+    OUT PULONG          SectorsDone
+    )
+{
+    UCHAR               Operation;
+    BOOLEAN             ReadOnly;
+    ULONG               Index;
+    __RingOperation(Cdb_OperationEx(Request->Srb), &Operation, &ReadOnly);
+
+    Request->Operation  = Operation;
+    Request->NrSegments = 0;
+    Request->FirstSector = SectorStart;
+
+    for (Index = 0;
+                Index < MaxSegments &&
+                SectorsLeft > 0;
+                        ++Index) {
+        PXENVBD_SEGMENT Segment;
+        ULONG           SectorsNow;
+
+        Segment = RingGetSegment(Ring);
+        if (Segment == NULL)
+            goto fail1;
+
+        InsertTailList(&Request->Segments, &Segment->Entry);
+        ++Request->NrSegments;
+
+        if (!RingPrepareSegment(Ring,
+                                Segment,
+                                SrbExt,
+                                ReadOnly,
+                                SectorsLeft,
+                                &SectorsNow))
+            goto fail2;
+
+        *SectorsDone += SectorsNow;
+        SectorsLeft  -= SectorsNow;
+    }
+    ASSERT3U(Request->NrSegments, >, 0);
+    ASSERT3U(Request->NrSegments, <=, MaxSegments);
+
+    return TRUE;
+
+fail2:
+fail1:
+    return FALSE;
+}
+
+static BOOLEAN
+RingPrepareBlkifIndirect(
+    IN  PXENVBD_RING    Ring,
+    IN  PXENVBD_REQUEST Request
+    )
+{
+    ULONG               Index;
+    ULONG               NrSegments = 0;
+
+    for (Index = 0;
+            Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
+            NrSegments < Request->NrSegments;
+                ++Index) {
+        PXENVBD_INDIRECT    Indirect;
+
+        Indirect = RingGetIndirect(Ring);
+        if (Indirect == NULL)
+            goto fail1;
+        InsertTailList(&Request->Indirects, &Indirect->Entry);
+
+        NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE;
+    }
+
+    return TRUE;
+
+fail1:
+    return FALSE;
+}
+
+static FORCEINLINE ULONG
+RingUseIndirect(
+    IN  PXENVBD_RING    Ring,
+    IN  ULONG           SectorsLeft
+    )
+{
+    const ULONG SectorsPerPage = 
__RingSectorsPerPage(FrontendGetDiskInfo(Ring->Frontend)->SectorSize);
+    const ULONG MaxIndirectSegs = 
FrontendGetFeatures(Ring->Frontend)->Indirect;
+
+    if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST)
+        return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported
+
+    if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * SectorsPerPage)
+        return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single 
BLKIF_OP_{READ/WRITE}
+
+    return MaxIndirectSegs;
+}
+
+static FORCEINLINE ULONG
+RingQueueRequestList(
+    IN  PXENVBD_RING    Ring,
+    IN  PLIST_ENTRY     List
+    )
+{
+    ULONG               Count = 0;
+    for (;;) {
+        PXENVBD_REQUEST Request;
+        PLIST_ENTRY     Entry;
+
+        Entry = RemoveHeadList(List);
+        if (Entry == List)
+            break;
+
+        ++Count;
+        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+        __RingIncBlkifOpCount(Ring, Request);
+        QueueAppend(&Ring->PreparedReqs, &Request->Entry);
+    }
+    return Count;
+}
+
+static FORCEINLINE VOID
+RingCancelRequestList(
+    IN  PXENVBD_RING    Ring,
+    IN  PLIST_ENTRY     List
+    )
+{
+    for (;;) {
+        PXENVBD_REQUEST Request;
+        PLIST_ENTRY     Entry;
+
+        Entry = RemoveHeadList(List);
+        if (Entry == List)
+            break;
+
+        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+        RingPutRequest(Ring, Request);
+    }
+}
+
+static BOOLEAN
+RingPrepareReadWrite(
+    IN  PXENVBD_RING        Ring,
+    IN  PSCSI_REQUEST_BLOCK Srb
+    )
+{
+    PXENVBD_SRBEXT          SrbExt = GetSrbExt(Srb);
+    ULONG64                 SectorStart = Cdb_LogicalBlock(Srb);
+    ULONG                   SectorsLeft = Cdb_TransferBlock(Srb);
+    LIST_ENTRY              List;
+    ULONG                   DebugCount;
+
+    Srb->SrbStatus = SRB_STATUS_PENDING;
+
+    InitializeListHead(&List);
+    SrbExt->Count = 0;
+
+    while (SectorsLeft > 0) {
+        ULONG           MaxSegments;
+        ULONG           SectorsDone = 0;
+        PXENVBD_REQUEST Request;
+
+        Request = RingGetRequest(Ring);
+        if (Request == NULL)
+            goto fail1;
+        InsertTailList(&List, &Request->Entry);
+        InterlockedIncrement(&SrbExt->Count);
+
+        Request->Srb    = Srb;
+        MaxSegments = RingUseIndirect(Ring, SectorsLeft);
+
+        if (!RingPrepareBlkifReadWrite(Ring,
+                                       Request,
+                                       SrbExt,
+                                       MaxSegments,
+                                       SectorStart,
+                                       SectorsLeft,
+                                       &SectorsDone))
+            goto fail2;
+
+        if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) {
+            if (!RingPrepareBlkifIndirect(Ring, Request))
+                goto fail3;
+        }
+
+        SectorsLeft -= SectorsDone;
+        SectorStart += SectorsDone;
+    }
+
+    DebugCount = RingQueueRequestList(Ring, &List);
+    if (DebugCount != (ULONG)SrbExt->Count) {
+        Trace("[%u] %d != %u\n",
+              FrontendGetTargetId(Ring->Frontend),
+              SrbExt->Count,
+              DebugCount);
+    }
+    return TRUE;
+
+fail3:
+fail2:
+fail1:
+    RingCancelRequestList(Ring, &List);
+    SrbExt->Count = 0;
+    Srb->SrbStatus = SRB_STATUS_ERROR;
+    return FALSE;
+}
+
+static BOOLEAN
+RingPrepareSyncCache(
+    IN  PXENVBD_RING        Ring,
+    IN  PSCSI_REQUEST_BLOCK Srb
+    )
+{
+    PXENVBD_SRBEXT          SrbExt = GetSrbExt(Srb);
+    PXENVBD_REQUEST         Request;
+    LIST_ENTRY              List;
+    UCHAR                   Operation;
+    ULONG                   DebugCount;
+
+    Srb->SrbStatus = SRB_STATUS_PENDING;
+
+    if (FrontendGetDiskInfo(Ring->Frontend)->FlushCache)
+        Operation = BLKIF_OP_FLUSH_DISKCACHE;
+    else
+        Operation = BLKIF_OP_WRITE_BARRIER;
+
+    InitializeListHead(&List);
+    SrbExt->Count = 0;
+
+    Request = RingGetRequest(Ring);
+    if (Request == NULL)
+        goto fail1;
+    InsertTailList(&List, &Request->Entry);
+    InterlockedIncrement(&SrbExt->Count);
+
+    Request->Srb        = Srb;
+    Request->Operation  = Operation;
+    Request->FirstSector = Cdb_LogicalBlock(Srb);
+
+    DebugCount = RingQueueRequestList(Ring, &List);
+    if (DebugCount != (ULONG)SrbExt->Count) {
+        Trace("[%u] %d != %u\n",
+              FrontendGetTargetId(Ring->Frontend),
+              SrbExt->Count,
+              DebugCount);
+    }
+    return TRUE;
+
+fail1:
+    RingCancelRequestList(Ring, &List);
+    SrbExt->Count = 0;
+    Srb->SrbStatus = SRB_STATUS_ERROR;
+    return FALSE;
+}
+
+static BOOLEAN
+RingPrepareUnmap(
+    IN  PXENVBD_RING        Ring,
+    IN  PSCSI_REQUEST_BLOCK Srb
+    )
+{
+    PXENVBD_SRBEXT          SrbExt = GetSrbExt(Srb);
+    PUNMAP_LIST_HEADER      Unmap = Srb->DataBuffer;
+       ULONG                   Count = 
_byteswap_ushort(*(PUSHORT)Unmap->BlockDescrDataLength) / 
sizeof(UNMAP_BLOCK_DESCRIPTOR);
+    ULONG                   Index;
+    LIST_ENTRY              List;
+    ULONG                   DebugCount;
+
+    Srb->SrbStatus = SRB_STATUS_PENDING;
+
+    InitializeListHead(&List);
+    SrbExt->Count = 0;
+
+    for (Index = 0; Index < Count; ++Index) {
+        PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index];
+        PXENVBD_REQUEST         Request;
+
+        Request = RingGetRequest(Ring);
+        if (Request == NULL)
+            goto fail1;
+        InsertTailList(&List, &Request->Entry);
+        InterlockedIncrement(&SrbExt->Count);
+
+        Request->Srb            = Srb;
+        Request->Operation      = BLKIF_OP_DISCARD;
+        Request->FirstSector    = 
_byteswap_uint64(*(PULONG64)Descr->StartingLba);
+        Request->NrSectors      = _byteswap_ulong(*(PULONG)Descr->LbaCount);
+        Request->Flags          = 0;
+    }
+
+    DebugCount = RingQueueRequestList(Ring, &List);
+    if (DebugCount != (ULONG)SrbExt->Count) {
+        Trace("[%u] %d != %u\n",
+              FrontendGetTargetId(Ring->Frontend),
+              SrbExt->Count,
+              DebugCount);
+    }
+    return TRUE;
+
+fail1:
+    RingCancelRequestList(Ring, &List);
+    SrbExt->Count = 0;
+    Srb->SrbStatus = SRB_STATUS_ERROR;
+    return FALSE;
+}
+
+static FORCEINLINE BOOLEAN
+RingPrepareFresh(
+    IN  PXENVBD_RING    Ring
+    )
+{
+    PXENVBD_SRBEXT      SrbExt;
+    PLIST_ENTRY         Entry;
+
+    Entry = QueuePop(&Ring->FreshSrbs);
+    if (Entry == NULL)
+        return FALSE;   // fresh queue is empty
+
+    SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
+
+    switch (Cdb_OperationEx(SrbExt->Srb)) {
+    case SCSIOP_READ:
+    case SCSIOP_WRITE:
+        if (RingPrepareReadWrite(Ring, SrbExt->Srb))
+            return TRUE;    // prepared this SRB
+        break;
+    case SCSIOP_SYNCHRONIZE_CACHE:
+        if (RingPrepareSyncCache(Ring, SrbExt->Srb))
+            return TRUE;    // prepared this SRB
+        break;
+    case SCSIOP_UNMAP:
+        if (RingPrepareUnmap(Ring, SrbExt->Srb))
+            return TRUE;    // prepared this SRB
+        break;
+    default:
+        ASSERT(FALSE);
+        break;
+    }
+    QueueUnPop(&Ring->FreshSrbs, &SrbExt->Entry);
+
+    return FALSE;       // prepare failed
+}
+
+static BOOLEAN
+RingSubmit(
+    IN  PXENVBD_RING    Ring,
+    IN  PXENVBD_REQUEST Request
+    )
+{
+    KIRQL               Irql;
+    blkif_request_t*    req;
+    BOOLEAN             Notify;
+
+    KeAcquireSpinLock(&Ring->Lock, &Irql);
+    if (RING_FULL(&Ring->Front)) {
+        KeReleaseSpinLock(&Ring->Lock, Irql);
+        return FALSE;
+    }
+
+    req = RING_GET_REQUEST(&Ring->Front, Ring->Front.req_prod_pvt);
+    __RingInsert(Ring, Request, req);
+    KeMemoryBarrier();
+    ++Ring->Front.req_prod_pvt;
+
+    RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&Ring->Front, Notify);
+    KeReleaseSpinLock(&Ring->Lock, Irql);
+
+    if (Notify) {
+        if (!Ring->Enabled)
+            return TRUE;
+
+        XENBUS_EVTCHN(Send,
+                      &Ring->EvtchnInterface,
+                      Ring->Channel);
+    }
+
+    return TRUE;
+}
+
+static FORCEINLINE BOOLEAN
+RingSubmitPrepared(
+    IN  PXENVBD_RING    Ring
+    )
+{
+    if (!Ring->Enabled) {
+        if (QueueCount(&Ring->PreparedReqs))
+            Warning("Target[%d] : Paused, not submitting new requests (%u)\n",
+                    FrontendGetTargetId(Ring->Frontend),
+                    QueueCount(&Ring->PreparedReqs));
+        return FALSE;
+    }
+
+    for (;;) {
+        PXENVBD_REQUEST Request;
+        PLIST_ENTRY     Entry;
+
+        Entry = QueuePop(&Ring->PreparedReqs);
+        if (Entry == NULL)
+            break;
+
+        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+
+        QueueAppend(&Ring->SubmittedReqs, &Request->Entry);
+        KeMemoryBarrier();
+
+        if (RingSubmit(Ring, Request))
+            continue;
+
+        QueueRemove(&Ring->SubmittedReqs, &Request->Entry);
+        QueueUnPop(&Ring->PreparedReqs, &Request->Entry);
+        return FALSE;   // ring full
+    }
+
+    return TRUE;
+}
+
+static FORCEINLINE VOID
+RingCompleteShutdown(
+    IN  PXENVBD_RING    Ring
+    )
+{
+    PXENVBD_TARGET      Target;
+    PXENVBD_ADAPTER     Adapter;
+
+    if (QueueCount(&Ring->ShutdownSrbs) == 0)
+        return;
+
+    if (QueueCount(&Ring->FreshSrbs) ||
+        QueueCount(&Ring->PreparedReqs) ||
+        QueueCount(&Ring->SubmittedReqs))
+        return;
+
+    Target = FrontendGetTarget(Ring->Frontend);
+    Adapter = TargetGetAdapter(Target);
+    for (;;) {
+        PXENVBD_SRBEXT  SrbExt;
+        PLIST_ENTRY     Entry = QueuePop(&Ring->ShutdownSrbs);
+        if (Entry == NULL)
+            break;
+        SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
+        SrbExt->Srb->SrbStatus = SRB_STATUS_SUCCESS;
+        AdapterCompleteSrb(Adapter, SrbExt);
+    }
+}
+
+static FORCEINLINE PCHAR
+__BlkifOperationName(
+    IN  UCHAR   Operation
+    )
+{
+    switch (Operation) {
+    case BLKIF_OP_READ:             return "READ";
+    case BLKIF_OP_WRITE:            return "WRITE";
+    case BLKIF_OP_WRITE_BARRIER:    return "WRITE_BARRIER";
+    case BLKIF_OP_FLUSH_DISKCACHE:  return "FLUSH_DISKCACHE";
+    case BLKIF_OP_RESERVED_1:       return "RESERVED_1";
+    case BLKIF_OP_DISCARD:          return "DISCARD";
+    case BLKIF_OP_INDIRECT:         return "INDIRECT";
+    default:                        return "<unknown>";
+    }
+}
+
+static BOOLEAN
+RingSubmitRequests(
+    IN  PXENVBD_RING    Ring
+    )
+{
+    BOOLEAN             Retry = FALSE;
+
+    for (;;) {
+        // submit all prepared requests (0 or more requests)
+        // return TRUE if submitted 0 or more requests from prepared queue
+        // return FALSE iff ring is full
+        if (!RingSubmitPrepared(Ring))
+            break;
+
+        // prepare a single SRB (into 1 or more requests)
+        // return TRUE if prepare succeeded
+        // return FALSE if prepare failed or fresh queue empty
+        if (!RingPrepareFresh(Ring))
+            break;
+
+        // back off, check DPC timeout and try again
+        Retry = TRUE;
+        break;
+    }
+
+    // if no requests/SRBs outstanding, complete any shutdown SRBs
+    if (!Retry)
+        RingCompleteShutdown(Ring);
+
+    return Retry;
+}
+
+static VOID
+RingCompleteResponse(
+    IN  PXENVBD_RING    Ring,
+    IN  ULONG           Tag,
+    IN  SHORT           Status
+    )
+{
+    PXENVBD_REQUEST     Request;
+    PSCSI_REQUEST_BLOCK Srb;
+    PXENVBD_SRBEXT      SrbExt;
+
+    Request = RingRequestFromTag(Ring, Tag);
+    if (Request == NULL)
+        return;
+
+    Srb     = Request->Srb;
+    SrbExt  = GetSrbExt(Srb);
+    ASSERT3P(SrbExt, !=, NULL);
+
+    switch (Status) {
+    case BLKIF_RSP_OKAY:
+        RingRequestCopyOutput(Request);
+        break;
+
+    case BLKIF_RSP_EOPNOTSUPP:
+        // Remove appropriate feature support
+        FrontendRemoveFeature(Ring->Frontend, Request->Operation);
+        // Succeed this SRB, subsiquent SRBs will be succeeded instead of 
being passed to the backend.
+        Srb->SrbStatus = SRB_STATUS_SUCCESS;
+        break;
+
+    case BLKIF_RSP_ERROR:
+    default:
+        Warning("Target[%d] : %s BLKIF_RSP_ERROR (Tag %x)\n",
+                FrontendGetTargetId(Ring->Frontend),
+                __BlkifOperationName(Request->Operation),
+                Tag);
+        Srb->SrbStatus = SRB_STATUS_ERROR;
+        break;
+    }
+
+    RingPutRequest(Ring, Request);
+
+    // complete srb
+    if (InterlockedDecrement(&SrbExt->Count) == 0) {
+        PXENVBD_TARGET  Target = FrontendGetTarget(Ring->Frontend);
+        PXENVBD_ADAPTER Adapter = TargetGetAdapter(Target);
+
+        if (Srb->SrbStatus == SRB_STATUS_PENDING) {
+            // SRB has not hit a failure condition (BLKIF_RSP_ERROR | 
BLKIF_RSP_EOPNOTSUPP)
+            // from any of its responses. SRB must have succeeded
+            Srb->SrbStatus = SRB_STATUS_SUCCESS;
+            Srb->ScsiStatus = 0x00; // SCSI_GOOD
+        } else {
+            // Srb->SrbStatus has already been set by 1 or more requests with 
Status != BLKIF_RSP_OKAY
+            Srb->ScsiStatus = 0x40; // SCSI_ABORTED
+        }
+
+        AdapterCompleteSrb(Adapter, SrbExt);
+    }
+}
+
+static BOOLEAN
+RingPoll(
+    IN  PXENVBD_RING    Ring
+    )
+{
+    BOOLEAN             Retry = FALSE;
+
+    ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+    KeAcquireSpinLockAtDpcLevel(&Ring->Lock);
+
+    // Guard against this locked region being called after the 
+    // lock on FrontendSetState
+    if (Ring->Enabled == FALSE)
+        goto done;
+
+    for (;;) {
+        ULONG   rsp_prod;
+        ULONG   rsp_cons;
+
+        KeMemoryBarrier();
+
+        rsp_prod = Ring->Shared->rsp_prod;
+        rsp_cons = Ring->Front.rsp_cons;
+
+        KeMemoryBarrier();
 
-            for (PageIdx = 0,
-                 PageEntry = Request->Indirects.Flink,
-                 SegEntry = Request->Segments.Flink;
-                    PageIdx < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
-                    PageEntry != &Request->Indirects &&
-                    SegEntry != &Request->Segments;
-                        ++PageIdx, PageEntry = PageEntry->Flink) {
-                PXENVBD_INDIRECT Page = CONTAINING_RECORD(PageEntry, 
XENVBD_INDIRECT, Entry);
+        if (rsp_cons == rsp_prod || Retry)
+            break;
 
-                req_indirect->indirect_grefs[PageIdx] = 
GranterReference(Granter, Page->Grant);
+        while (rsp_cons != rsp_prod && !Retry) {
+            blkif_response_t*   Response;
+            ULONG               Tag;
 
-                for (SegIdx = 0;
-                        SegIdx < XENVBD_MAX_SEGMENTS_PER_PAGE &&
-                        SegEntry != &Request->Segments;
-                            ++SegIdx, SegEntry = SegEntry->Flink) {
-                    PXENVBD_SEGMENT Segment = CONTAINING_RECORD(SegEntry, 
XENVBD_SEGMENT, Entry);
+            Response = RING_GET_RESPONSE(&Ring->Front, rsp_cons);
+            ++rsp_cons;
 
-                    Page->Page[SegIdx].GrantRef = GranterReference(Granter, 
Segment->Grant);
-                    Page->Page[SegIdx].First    = Segment->FirstSector;
-                    Page->Page[SegIdx].Last     = Segment->LastSector;
-                }
+            if (__RingPutTag(Ring, Response->id, &Tag)) {
+                ++Ring->Received;
+                RingCompleteResponse(Ring, Tag, Response->status);
             }
-        } else {
-            // Direct
-            ULONG           Index;
-            PLIST_ENTRY     Entry;
 
-            req->operation                  = Request->Operation;
-            req->nr_segments                = (UCHAR)Request->NrSegments;
-            req->handle                     = 
(USHORT)FrontendGetDeviceId(Ring->Frontend);
-            req->id                         = __RingGetTag(Ring, Request);
-            req->sector_number              = Request->FirstSector;
+            RtlZeroMemory(Response, sizeof(union blkif_sring_entry));
 
-            for (Index = 0, Entry = Request->Segments.Flink;
-                    Index < BLKIF_MAX_SEGMENTS_PER_REQUEST &&
-                    Entry != &Request->Segments;
-                        ++Index, Entry = Entry->Flink) {
-                PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry, 
XENVBD_SEGMENT, Entry);
-                req->seg[Index].gref        = GranterReference(Granter, 
Segment->Grant);
-                req->seg[Index].first_sect  = Segment->FirstSector;
-                req->seg[Index].last_sect   = Segment->LastSector;
-            }
+            if (rsp_cons - Ring->Front.rsp_cons > RING_SIZE(&Ring->Front) / 4)
+                Retry = TRUE;
         }
-        break;
-
-    case BLKIF_OP_WRITE_BARRIER:
-    case BLKIF_OP_FLUSH_DISKCACHE:
-        req->operation                  = Request->Operation;
-        req->nr_segments                = 0;
-        req->handle                     = 
(USHORT)FrontendGetDeviceId(Ring->Frontend);
-        req->id                         = __RingGetTag(Ring, Request);
-        req->sector_number              = Request->FirstSector;
-        break;
 
-    case BLKIF_OP_DISCARD: {
-        blkif_request_discard_t*        req_discard;
-        req_discard = (blkif_request_discard_t*)req;
-        req_discard->operation          = BLKIF_OP_DISCARD;
-        req_discard->flag               = Request->Flags;
-        req_discard->handle             = 
(USHORT)FrontendGetDeviceId(Ring->Frontend);
-        req_discard->id                 = __RingGetTag(Ring, Request);
-        req_discard->sector_number      = Request->FirstSector;
-        req_discard->nr_sectors         = Request->NrSectors;
-        } break;
+        KeMemoryBarrier();
 
-    default:
-        ASSERT(FALSE);
-        break;
+        Ring->Front.rsp_cons = rsp_cons;
+        Ring->Shared->rsp_event = rsp_cons + 1;
     }
-    ++Ring->Submitted;
+
+done:
+    KeReleaseSpinLockFromDpcLevel(&Ring->Lock);
+
+    return Retry;
+}
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static BOOLEAN
+RingNotifyResponses(
+    IN  PXENVBD_RING    Ring
+    )
+{
+    BOOLEAN             Retry = FALSE;
+
+    Retry |= RingPoll(Ring);
+    Retry |= RingSubmitRequests(Ring);
+
+    return Retry;
 }
 
 KSERVICE_ROUTINE    RingInterrupt;
@@ -334,7 +1459,7 @@ RingDpc(
         return;
 
     for (;;) {
-        if (!FrontendNotifyResponses(Ring->Frontend)) {
+        if (!RingNotifyResponses(Ring)) {
             XENBUS_EVTCHN(Unmask,
                           &Ring->EvtchnInterface,
                           Ring->Channel,
@@ -425,6 +1550,51 @@ RingDebugCallback(
                      Ring->Channel,
                      Port);
     }
+
+    XENBUS_DEBUG(Printf,
+                 &Ring->DebugInterface,
+                 "TARGET: BLKIF_OPs: READ=%u WRITE=%u\n",
+                 Ring->BlkOpRead,
+                 Ring->BlkOpWrite);
+    XENBUS_DEBUG(Printf,
+                 &Ring->DebugInterface,
+                 "TARGET: BLKIF_OPs: INDIRECT_READ=%u INDIRECT_WRITE=%u\n",
+                 Ring->BlkOpIndirectRead,
+                 Ring->BlkOpIndirectWrite);
+    XENBUS_DEBUG(Printf,
+                 &Ring->DebugInterface,
+                 "TARGET: BLKIF_OPs: BARRIER=%u DISCARD=%u FLUSH=%u\n",
+                 Ring->BlkOpBarrier,
+                 Ring->BlkOpDiscard,
+                 Ring->BlkOpFlush);
+    XENBUS_DEBUG(Printf,
+                 &Ring->DebugInterface,
+                 "TARGET: Segments Granted=%llu Bounced=%llu\n",
+                 Ring->SegsGranted,
+                 Ring->SegsBounced);
+
+    __LookasideDebug(&Ring->RequestList,
+                     &Ring->DebugInterface,
+                     "REQUESTs");
+    __LookasideDebug(&Ring->SegmentList,
+                     &Ring->DebugInterface,
+                     "SEGMENTs");
+    __LookasideDebug(&Ring->IndirectList,
+                     &Ring->DebugInterface,
+                     "INDIRECTs");
+
+    QueueDebugCallback(&Ring->FreshSrbs,
+                       "Fresh    ",
+                       &Ring->DebugInterface);
+    QueueDebugCallback(&Ring->PreparedReqs,
+                       "Prepared ",
+                       &Ring->DebugInterface);
+    QueueDebugCallback(&Ring->SubmittedReqs,
+                       "Submitted",
+                       &Ring->DebugInterface);
+    QueueDebugCallback(&Ring->ShutdownSrbs,
+                       "Shutdown ",
+                       &Ring->DebugInterface);
 }
 
 NTSTATUS
@@ -447,6 +1617,14 @@ RingCreate(
     KeInitializeDpc(&(*Ring)->TimerDpc, RingDpc, *Ring);
     KeInitializeTimer(&(*Ring)->Timer);
 
+    QueueInit(&(*Ring)->FreshSrbs);
+    QueueInit(&(*Ring)->PreparedReqs);
+    QueueInit(&(*Ring)->SubmittedReqs);
+    QueueInit(&(*Ring)->ShutdownSrbs);
+    __LookasideInit(&(*Ring)->RequestList, sizeof(XENVBD_REQUEST), 
REQUEST_POOL_TAG);
+    __LookasideInit(&(*Ring)->SegmentList, sizeof(XENVBD_SEGMENT), 
SEGMENT_POOL_TAG);
+    __LookasideInit(&(*Ring)->IndirectList, sizeof(XENVBD_INDIRECT), 
INDIRECT_POOL_TAG);
+
     return STATUS_SUCCESS;
 
 fail1:
@@ -459,6 +1637,14 @@ RingDestroy(
     IN  PXENVBD_RING    Ring
     )
 {
+    __LookasideTerm(&Ring->IndirectList);
+    __LookasideTerm(&Ring->SegmentList);
+    __LookasideTerm(&Ring->RequestList);
+    RtlZeroMemory(&Ring->FreshSrbs, sizeof(XENVBD_QUEUE));
+    RtlZeroMemory(&Ring->PreparedReqs, sizeof(XENVBD_QUEUE));
+    RtlZeroMemory(&Ring->SubmittedReqs, sizeof(XENVBD_QUEUE));
+    RtlZeroMemory(&Ring->ShutdownSrbs, sizeof(XENVBD_QUEUE));
+
     RtlZeroMemory(&Ring->Timer, sizeof(KTIMER));
     RtlZeroMemory(&Ring->TimerDpc, sizeof(KDPC));
     RtlZeroMemory(&Ring->Dpc, sizeof(KDPC));
@@ -717,9 +1903,66 @@ RingDisable(
     IN  PXENVBD_RING    Ring
     )
 {
+    ULONG               Count;
+    KIRQL               Irql;
+    PXENVBD_TARGET      Target = FrontendGetTarget(Ring->Frontend);
+    PXENVBD_ADAPTER     Adapter = TargetGetAdapter(Target);
+
     ASSERT(Ring->Enabled == TRUE);
     Ring->Enabled = FALSE;
 
+    // poll ring and send event channel notification every 1ms (for up to 3 
minutes)
+    Count = 0;
+    while (QueueCount(&Ring->SubmittedReqs)) {
+        if (Count > 180000)
+            break;
+        KeRaiseIrql(DISPATCH_LEVEL, &Irql);
+        RingPoll(Ring);
+        KeLowerIrql(Irql);
+        XENBUS_EVTCHN(Send,
+                      &Ring->EvtchnInterface,
+                      Ring->Channel);
+        StorPortStallExecution(1000);   // 1000 micro-seconds
+        ++Count;
+    }
+
+    Verbose("Target[%d] : %u Submitted requests left (%u iterrations)\n",
+            FrontendGetTargetId(Ring->Frontend),
+            QueueCount(&Ring->SubmittedReqs),
+            Count);
+
+    // Abort Fresh SRBs
+    for (;;) {
+        PXENVBD_SRBEXT  SrbExt;
+        PLIST_ENTRY     Entry = QueuePop(&Ring->FreshSrbs);
+        if (Entry == NULL)
+            break;
+        SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
+
+        SrbExt->Srb->SrbStatus = SRB_STATUS_ABORTED;
+        SrbExt->Srb->ScsiStatus = 0x40; // SCSI_ABORTED;
+        AdapterCompleteSrb(Adapter, SrbExt);
+    }
+
+    // Fail PreparedReqs
+    for (;;) {
+        PXENVBD_SRBEXT  SrbExt;
+        PXENVBD_REQUEST Request;
+        PLIST_ENTRY     Entry = QueuePop(&Ring->PreparedReqs);
+        if (Entry == NULL)
+            break;
+        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+        SrbExt = GetSrbExt(Request->Srb);
+
+        SrbExt->Srb->SrbStatus = SRB_STATUS_ABORTED;
+        RingPutRequest(Ring, Request);
+
+        if (InterlockedDecrement(&SrbExt->Count) == 0) {
+            SrbExt->Srb->ScsiStatus = 0x40; // SCSI_ABORTED
+            AdapterCompleteSrb(Adapter, SrbExt);
+        }
+    }
+
     //
     // No new timers can be scheduled once Enabled goes to FALSE.
     // Cancel any existing ones.
@@ -782,101 +2025,28 @@ RingDisconnect(
     Ring->Received = 0;
 }
 
-BOOLEAN
-RingPoll(
+VOID
+RingTrigger(
     IN  PXENVBD_RING    Ring
     )
 {
-    PXENVBD_TARGET      Target = FrontendGetTarget(Ring->Frontend);
-    BOOLEAN             Retry = FALSE;
-
-    ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
-    KeAcquireSpinLockAtDpcLevel(&Ring->Lock);
-
-    // Guard against this locked region being called after the 
-    // lock on FrontendSetState
-    if (Ring->Enabled == FALSE)
-        goto done;
-
-    for (;;) {
-        ULONG   rsp_prod;
-        ULONG   rsp_cons;
-
-        KeMemoryBarrier();
-
-        rsp_prod = Ring->Shared->rsp_prod;
-        rsp_cons = Ring->Front.rsp_cons;
-
-        KeMemoryBarrier();
-
-        if (rsp_cons == rsp_prod || Retry)
-            break;
-
-        while (rsp_cons != rsp_prod && !Retry) {
-            blkif_response_t*   Response;
-            ULONG               Tag;
-
-            Response = RING_GET_RESPONSE(&Ring->Front, rsp_cons);
-            ++rsp_cons;
-
-            if (__RingPutTag(Ring, Response->id, &Tag)) {
-                ++Ring->Received;
-                TargetCompleteResponse(Target, Tag, Response->status);
-            }
-
-            RtlZeroMemory(Response, sizeof(union blkif_sring_entry));
-
-            if (rsp_cons - Ring->Front.rsp_cons > RING_SIZE(&Ring->Front) / 4)
-                Retry = TRUE;
-        }
-
-        KeMemoryBarrier();
-
-        Ring->Front.rsp_cons = rsp_cons;
-        Ring->Shared->rsp_event = rsp_cons + 1;
-    }
-
-done:
-    KeReleaseSpinLockFromDpcLevel(&Ring->Lock);
+    if (!Ring->Enabled)
+        return;
 
-    return Retry;
+    XENBUS_EVTCHN(Trigger,
+                  &Ring->EvtchnInterface,
+                  Ring->Channel);
 }
 
-BOOLEAN
-RingSubmit(
+VOID
+RingQueueRequest(
     IN  PXENVBD_RING    Ring,
-    IN  PXENVBD_REQUEST Request
+    IN  PXENVBD_SRBEXT  SrbExt
     )
 {
-    KIRQL               Irql;
-    blkif_request_t*    req;
-    BOOLEAN             Notify;
-
-    KeAcquireSpinLock(&Ring->Lock, &Irql);
-    if (RING_FULL(&Ring->Front)) {
-        KeReleaseSpinLock(&Ring->Lock, Irql);
-        return FALSE;
-    }
-
-    req = RING_GET_REQUEST(&Ring->Front, Ring->Front.req_prod_pvt);
-    __RingInsert(Ring, Request, req);
-    KeMemoryBarrier();
-    ++Ring->Front.req_prod_pvt;
-
-    RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&Ring->Front, Notify);
-    KeReleaseSpinLock(&Ring->Lock, Irql);
-
-    if (Notify)
-        RingSend(Ring);
-
-    return TRUE;
-}
+    QueueAppend(&Ring->FreshSrbs,
+                &SrbExt->Entry);
 
-VOID
-RingKick(
-    IN  PXENVBD_RING    Ring
-    )
-{
     if (!Ring->Enabled)
         return;
 
@@ -885,27 +2055,79 @@ RingKick(
 }
 
 VOID
-RingTrigger(
-    IN  PXENVBD_RING    Ring
+RingQueueShutdown(
+    IN  PXENVBD_RING    Ring,
+    IN  PXENVBD_SRBEXT  SrbExt
     )
 {
+    QueueAppend(&Ring->ShutdownSrbs,
+                &SrbExt->Entry);
+
     if (!Ring->Enabled)
         return;
 
-    XENBUS_EVTCHN(Trigger,
-                  &Ring->EvtchnInterface,
-                  Ring->Channel);
+    if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
+           ++Ring->Dpcs;
 }
 
 VOID
-RingSend(
+RingReQueueRequests(
     IN  PXENVBD_RING    Ring
     )
 {
-    if (!Ring->Enabled)
-        return;
+    LIST_ENTRY      List;
 
-    XENBUS_EVTCHN(Send,
-                  &Ring->EvtchnInterface,
-                  Ring->Channel);
+    InitializeListHead(&List);
+
+    // pop all submitted requests, cleanup and add associated SRB to a list
+    for (;;) {
+        PXENVBD_SRBEXT  SrbExt;
+        PXENVBD_REQUEST Request;
+        PLIST_ENTRY     Entry = QueuePop(&Ring->SubmittedReqs);
+        if (Entry == NULL)
+            break;
+        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+        SrbExt = GetSrbExt(Request->Srb);
+
+        RingPutRequest(Ring, Request);
+
+        if (InterlockedDecrement(&SrbExt->Count) == 0) {
+            InsertTailList(&List, &SrbExt->Entry);
+        }
+    }
+
+    // pop all prepared requests, cleanup and add associated SRB to a list
+    for (;;) {
+        PXENVBD_SRBEXT  SrbExt;
+        PXENVBD_REQUEST Request;
+        PLIST_ENTRY     Entry = QueuePop(&Ring->PreparedReqs);
+        if (Entry == NULL)
+            break;
+        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+        SrbExt = GetSrbExt(Request->Srb);
+
+        RingPutRequest(Ring, Request);
+
+        if (InterlockedDecrement(&SrbExt->Count) == 0) {
+            InsertTailList(&List, &SrbExt->Entry);
+        }
+    }
+
+    // foreach SRB in list, put on start of FreshSrbs
+    for (;;) {
+        PXENVBD_SRBEXT  SrbExt;
+        PLIST_ENTRY     Entry = RemoveTailList(&List);
+        if (Entry == &List)
+            break;
+        SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
+
+        QueueUnPop(&Ring->FreshSrbs, &SrbExt->Entry);
+    }
+
+    // now the first set of requests popped off submitted list is the next SRB
+    // to be popped off the fresh list
+
+    Verbose("Target[%d] : %d Fresh SRBs\n",
+            FrontendGetTargetId(Ring->Frontend),
+            QueueCount(&Ring->FreshSrbs));
 }
diff --git a/src/xenvbd/ring.h b/src/xenvbd/ring.h
index 3ac1c71..a39d317 100755
--- a/src/xenvbd/ring.h
+++ b/src/xenvbd/ring.h
@@ -74,29 +74,25 @@ RingDisconnect(
     IN  PXENVBD_RING    Ring
     );
 
-extern BOOLEAN
-RingPoll(
+extern VOID
+RingTrigger(
     IN  PXENVBD_RING    Ring
     );
 
-extern BOOLEAN
-RingSubmit(
-    IN  PXENVBD_RING    Ring,
-    IN  PXENVBD_REQUEST Request
-    );
-
 extern VOID
-RingKick(
-    IN  PXENVBD_RING    Ring
+RingQueueRequest(
+    IN  PXENVBD_RING    Ring,
+    IN  PXENVBD_SRBEXT  SrbExt
     );
 
 extern VOID
-RingTrigger(
-    IN  PXENVBD_RING    Ring
+RingQueueShutdown(
+    IN  PXENVBD_RING    Ring,
+    IN  PXENVBD_SRBEXT  SrbExt
     );
 
 extern VOID
-RingSend(
+RingReQueueRequests(
     IN  PXENVBD_RING    Ring
     );
 
diff --git a/src/xenvbd/target.c b/src/xenvbd/target.c
index c0fa4e9..2e30860 100644
--- a/src/xenvbd/target.c
+++ b/src/xenvbd/target.c
@@ -51,15 +51,6 @@
 
 #define TARGET_SIGNATURE           'odpX'
 
-typedef struct _XENVBD_LOOKASIDE {
-    KEVENT                      Empty;
-    LONG                        Used;
-    LONG                        Max;
-    ULONG                       Failed;
-    ULONG                       Size;
-    NPAGED_LOOKASIDE_LIST       List;
-} XENVBD_LOOKASIDE, *PXENVBD_LOOKASIDE;
-
 struct _XENVBD_TARGET {
     ULONG                       Signature;
     PXENVBD_ADAPTER                 Adapter;
@@ -76,48 +67,16 @@ struct _XENVBD_TARGET {
     PXENBUS_DEBUG_CALLBACK      DebugCallback;
     PXENBUS_SUSPEND_CALLBACK    SuspendCallback;
 
-    // State
-    LONG                        Paused;
-
     // Eject
     BOOLEAN                     WrittenEjected;
     BOOLEAN                     EjectRequested;
     BOOLEAN                     EjectPending;
     BOOLEAN                     Missing;
     const CHAR*                 Reason;
-
-    // SRBs
-    XENVBD_LOOKASIDE            RequestList;
-    XENVBD_LOOKASIDE            SegmentList;
-    XENVBD_LOOKASIDE            IndirectList;
-    XENVBD_QUEUE                FreshSrbs;
-    XENVBD_QUEUE                PreparedReqs;
-    XENVBD_QUEUE                SubmittedReqs;
-    XENVBD_QUEUE                ShutdownSrbs;
-    ULONG                       NextTag;
-
-    // Stats - SRB Counts by BLKIF_OP_
-    ULONG                       BlkOpRead;
-    ULONG                       BlkOpWrite;
-    ULONG                       BlkOpIndirectRead;
-    ULONG                       BlkOpIndirectWrite;
-    ULONG                       BlkOpBarrier;
-    ULONG                       BlkOpDiscard;
-    ULONG                       BlkOpFlush;
-    // Stats - Failures
-    ULONG                       FailedMaps;
-    ULONG                       FailedBounces;
-    ULONG                       FailedGrants;
-    // Stats - Segments
-    ULONG64                     SegsGranted;
-    ULONG64                     SegsBounced;
 };
 
 //=============================================================================
 #define TARGET_POOL_TAG            'odPX'
-#define REQUEST_POOL_TAG        'qeRX'
-#define SEGMENT_POOL_TAG        'geSX'
-#define INDIRECT_POOL_TAG       'dnIX'
 
 __checkReturn
 __drv_allocatesMem(mem)
@@ -142,89 +101,6 @@ __TargetFree(
 }
 
 //=============================================================================
-// Lookasides
-static FORCEINLINE VOID
-__LookasideInit(
-    IN OUT  PXENVBD_LOOKASIDE   Lookaside,
-    IN  ULONG                   Size,
-    IN  ULONG                   Tag
-    )
-{
-    RtlZeroMemory(Lookaside, sizeof(XENVBD_LOOKASIDE));
-    Lookaside->Size = Size;
-    KeInitializeEvent(&Lookaside->Empty, SynchronizationEvent, TRUE);
-    ExInitializeNPagedLookasideList(&Lookaside->List, NULL, NULL, 0,
-                                    Size, Tag, 0);
-}
-
-static FORCEINLINE VOID
-__LookasideTerm(
-    IN  PXENVBD_LOOKASIDE       Lookaside
-    )
-{
-    ASSERT3U(Lookaside->Used, ==, 0);
-    ExDeleteNPagedLookasideList(&Lookaside->List);
-    RtlZeroMemory(Lookaside, sizeof(XENVBD_LOOKASIDE));
-}
-
-static FORCEINLINE PVOID
-__LookasideAlloc(
-    IN  PXENVBD_LOOKASIDE       Lookaside
-    )
-{
-    LONG    Result;
-    PVOID   Buffer;
-
-    Buffer = ExAllocateFromNPagedLookasideList(&Lookaside->List);
-    if (Buffer == NULL) {
-        ++Lookaside->Failed;
-        return NULL;
-    }
-
-    RtlZeroMemory(Buffer, Lookaside->Size);
-    Result = InterlockedIncrement(&Lookaside->Used);
-    ASSERT3S(Result, >, 0);
-    if (Result > Lookaside->Max)
-        Lookaside->Max = Result;
-    KeClearEvent(&Lookaside->Empty);
-
-    return Buffer;
-}
-
-static FORCEINLINE VOID
-__LookasideFree(
-    IN  PXENVBD_LOOKASIDE       Lookaside,
-    IN  PVOID                   Buffer
-    )
-{
-    LONG            Result;
-
-    ExFreeToNPagedLookasideList(&Lookaside->List, Buffer);
-    Result = InterlockedDecrement(&Lookaside->Used);
-    ASSERT3S(Result, >=, 0);
-
-    if (Result == 0) {
-        KeSetEvent(&Lookaside->Empty, IO_NO_INCREMENT, FALSE);
-    }
-}
-
-static FORCEINLINE VOID
-__LookasideDebug(
-    IN  PXENVBD_LOOKASIDE           Lookaside,
-    IN  PXENBUS_DEBUG_INTERFACE     Debug,
-    IN  PCHAR                       Name
-    )
-{
-    XENBUS_DEBUG(Printf, Debug,
-                 "LOOKASIDE: %s: %u / %u (%u failed)\n",
-                 Name, Lookaside->Used,
-                 Lookaside->Max, Lookaside->Failed);
-
-    Lookaside->Max = Lookaside->Used;
-    Lookaside->Failed = 0;
-}
-
-//=============================================================================
 // Debug
 static FORCEINLINE PCHAR
 __PnpStateName(
@@ -337,1011 +213,6 @@ TargetSetDeviceObject(
     Target->DeviceObject = DeviceObject;
 }
 
-static FORCEINLINE BOOLEAN
-TargetIsPaused(
-    IN  PXENVBD_TARGET  Target
-    )
-{
-    BOOLEAN             Paused;
-    KIRQL               Irql;
-
-    KeAcquireSpinLock(&Target->Lock, &Irql);
-    Paused = (Target->Paused > 0);
-    KeReleaseSpinLock(&Target->Lock, Irql);
-
-    return Paused;
-}
-
-static FORCEINLINE ULONG
-TargetSectorSize(
-    __in PXENVBD_TARGET             Target
-    )
-{
-    return FrontendGetDiskInfo(Target->Frontend)->SectorSize;
-}
-
-//=============================================================================
-static PXENVBD_INDIRECT
-TargetGetIndirect(
-    IN  PXENVBD_TARGET             Target
-    )
-{
-    PXENVBD_INDIRECT    Indirect;
-    NTSTATUS            status;
-    PXENVBD_GRANTER     Granter = FrontendGetGranter(Target->Frontend);
-
-    Indirect = __LookasideAlloc(&Target->IndirectList);
-    if (Indirect == NULL)
-        goto fail1;
-
-    RtlZeroMemory(Indirect, sizeof(XENVBD_INDIRECT));
-
-    Indirect->Mdl = __AllocatePage();
-    if (Indirect->Mdl == NULL)
-        goto fail2;
-
-    Indirect->Page = MmGetSystemAddressForMdlSafe(Indirect->Mdl,
-                                                  NormalPagePriority);
-
-    status = GranterGet(Granter,
-                        MmGetMdlPfnArray(Indirect->Mdl)[0],
-                        TRUE,
-                        &Indirect->Grant);
-    if (!NT_SUCCESS(status))
-        goto fail3;
-
-    return Indirect;
-
-fail3:
-    __FreePage(Indirect->Mdl);
-fail2:
-    __LookasideFree(&Target->IndirectList, Indirect);
-fail1:
-    return NULL;
-}
-
-static VOID
-TargetPutIndirect(
-    IN  PXENVBD_TARGET             Target,
-    IN  PXENVBD_INDIRECT        Indirect
-    )
-{
-    PXENVBD_GRANTER Granter = FrontendGetGranter(Target->Frontend);
-
-    if (Indirect->Grant)
-        GranterPut(Granter, Indirect->Grant);
-    if (Indirect->Page)
-        __FreePage(Indirect->Mdl);
-
-    RtlZeroMemory(Indirect, sizeof(XENVBD_INDIRECT));
-    __LookasideFree(&Target->IndirectList, Indirect);
-}
-
-static PXENVBD_SEGMENT
-TargetGetSegment(
-    IN  PXENVBD_TARGET             Target
-    )
-{
-    PXENVBD_SEGMENT             Segment;
-
-    Segment = __LookasideAlloc(&Target->SegmentList);
-    if (Segment == NULL)
-        goto fail1;
-
-    RtlZeroMemory(Segment, sizeof(XENVBD_SEGMENT));
-    return Segment;
-
-fail1:
-    return NULL;
-}
-
-static VOID
-TargetPutSegment(
-    IN  PXENVBD_TARGET             Target,
-    IN  PXENVBD_SEGMENT         Segment
-    )
-{
-    PXENVBD_GRANTER Granter = FrontendGetGranter(Target->Frontend);
-
-    if (Segment->Grant)
-        GranterPut(Granter, Segment->Grant);
-
-    if (Segment->BufferId)
-        BufferPut(Segment->BufferId);
-
-    if (Segment->Buffer)
-        MmUnmapLockedPages(Segment->Buffer, &Segment->Mdl);
-
-    RtlZeroMemory(Segment, sizeof(XENVBD_SEGMENT));
-    __LookasideFree(&Target->SegmentList, Segment);
-}
-
-static PXENVBD_REQUEST
-TargetGetRequest(
-    IN  PXENVBD_TARGET             Target
-    )
-{
-    PXENVBD_REQUEST             Request;
-
-    Request = __LookasideAlloc(&Target->RequestList);
-    if (Request == NULL)
-        goto fail1;
-
-    RtlZeroMemory(Request, sizeof(XENVBD_REQUEST));
-    Request->Id = (ULONG)InterlockedIncrement((PLONG)&Target->NextTag);
-    InitializeListHead(&Request->Segments);
-    InitializeListHead(&Request->Indirects);
-
-    return Request;
-
-fail1:
-    return NULL;
-}
-
-static VOID
-TargetPutRequest(
-    IN  PXENVBD_TARGET             Target,
-    IN  PXENVBD_REQUEST         Request
-    )
-{
-    PLIST_ENTRY     Entry;
-
-    for (;;) {
-        PXENVBD_SEGMENT Segment;
-
-        Entry = RemoveHeadList(&Request->Segments);
-        if (Entry == &Request->Segments)
-            break;
-        Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, Entry);
-        TargetPutSegment(Target, Segment);
-    }
-
-    for (;;) {
-        PXENVBD_INDIRECT    Indirect;
-
-        Entry = RemoveHeadList(&Request->Indirects);
-        if (Entry == &Request->Indirects)
-            break;
-        Indirect = CONTAINING_RECORD(Entry, XENVBD_INDIRECT, Entry);
-        TargetPutIndirect(Target, Indirect);
-    }
-
-    RtlZeroMemory(Request, sizeof(XENVBD_REQUEST));
-    __LookasideFree(&Target->RequestList, Request);
-}
-
-static FORCEINLINE PXENVBD_REQUEST
-TargetRequestFromTag(
-    IN  PXENVBD_TARGET             Target,
-    IN  ULONG                   Tag
-    )
-{
-    KIRQL           Irql;
-    PLIST_ENTRY     Entry;
-    PXENVBD_QUEUE   Queue = &Target->SubmittedReqs;
-
-    KeAcquireSpinLock(&Queue->Lock, &Irql);
-
-    for (Entry = Queue->List.Flink; Entry != &Queue->List; Entry = 
Entry->Flink) {
-        PXENVBD_REQUEST Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, 
Entry);
-        if (Request->Id == Tag) {
-            RemoveEntryList(&Request->Entry);
-            --Queue->Current;
-            KeReleaseSpinLock(&Queue->Lock, Irql);
-            return Request;
-        }
-    }
-
-    KeReleaseSpinLock(&Queue->Lock, Irql);
-    Warning("Target[%d] : Tag %x not found in submitted list (%u items)\n",
-            TargetGetTargetId(Target), Tag, QueueCount(Queue));
-    return NULL;
-}
-
-static FORCEINLINE VOID
-__TargetIncBlkifOpCount(
-    __in PXENVBD_TARGET             Target,
-    __in PXENVBD_REQUEST         Request
-    )
-{
-    switch (Request->Operation) {
-    case BLKIF_OP_READ:
-        if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
-            ++Target->BlkOpIndirectRead;
-        else
-            ++Target->BlkOpRead;
-        break;
-    case BLKIF_OP_WRITE:
-        if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
-            ++Target->BlkOpIndirectWrite;
-        else
-            ++Target->BlkOpWrite;
-        break;
-    case BLKIF_OP_WRITE_BARRIER:
-        ++Target->BlkOpBarrier;
-        break;
-    case BLKIF_OP_DISCARD:
-        ++Target->BlkOpDiscard;
-        break;
-    case BLKIF_OP_FLUSH_DISKCACHE:
-        ++Target->BlkOpFlush;
-        break;
-    default:
-        ASSERT(FALSE);
-        break;
-    }
-}
-
-static FORCEINLINE ULONG
-__SectorsPerPage(
-    __in ULONG                   SectorSize
-    )
-{
-    ASSERT3U(SectorSize, !=, 0);
-    return PAGE_SIZE / SectorSize;
-}
-
-static FORCEINLINE VOID
-__Operation(
-    __in UCHAR                   CdbOp,
-    __out PUCHAR                 RingOp,
-    __out PBOOLEAN               ReadOnly
-    )
-{
-    switch (CdbOp) {
-    case SCSIOP_READ:
-        *RingOp     = BLKIF_OP_READ;
-        *ReadOnly   = FALSE;
-        break;
-    case SCSIOP_WRITE:
-        *RingOp     = BLKIF_OP_WRITE;
-        *ReadOnly   = TRUE;
-        break;
-    default:
-        ASSERT(FALSE);
-    }
-}
-
-static FORCEINLINE MM_PAGE_PRIORITY
-__TargetPriority(
-    __in PXENVBD_TARGET             Target
-    )
-{
-    PXENVBD_CAPS   Caps = FrontendGetCaps(Target->Frontend);
-    if (!(Caps->Paging ||
-          Caps->Hibernation ||
-          Caps->DumpFile))
-        return NormalPagePriority;
-
-    return HighPagePriority;
-}
-
-static FORCEINLINE VOID
-RequestCopyOutput(
-    __in PXENVBD_REQUEST         Request
-    )
-{
-    PLIST_ENTRY     Entry;
-
-    if (Request->Operation != BLKIF_OP_READ)
-        return;
-
-    for (Entry = Request->Segments.Flink;
-            Entry != &Request->Segments;
-            Entry = Entry->Flink) {
-        PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, 
Entry);
-
-        if (Segment->BufferId)
-            BufferCopyOut(Segment->BufferId, Segment->Buffer, Segment->Length);
-    }
-}
-
-static BOOLEAN
-PrepareSegment(
-    IN  PXENVBD_TARGET          Target,
-    IN  PXENVBD_SEGMENT         Segment,
-    IN  PXENVBD_SRBEXT          SrbExt,
-    IN  BOOLEAN                 ReadOnly,
-    IN  ULONG                   SectorsLeft,
-    OUT PULONG                  SectorsNow
-    )
-{
-    PFN_NUMBER      Pfn;
-    ULONG           Offset;
-    ULONG           Length;
-    NTSTATUS        Status;
-    PXENVBD_GRANTER Granter = FrontendGetGranter(Target->Frontend);
-    const ULONG     SectorSize = TargetSectorSize(Target);
-    const ULONG     SectorsPerPage = __SectorsPerPage(SectorSize);
-
-    Pfn = AdapterGetNextSGEntry(TargetGetAdapter(Target),
-                                SrbExt,
-                                0,
-                                &Offset,
-                                &Length);
-    if ((Offset & (SectorSize - 1)) == 0 &&
-        (Length & (SectorSize - 1)) == 0) {
-        ++Target->SegsGranted;
-        // get first sector, last sector and count
-        Segment->FirstSector    = (UCHAR)((Offset + SectorSize - 1) / 
SectorSize);
-        *SectorsNow             = __min(SectorsLeft, SectorsPerPage - 
Segment->FirstSector);
-        Segment->LastSector     = (UCHAR)(Segment->FirstSector + *SectorsNow - 
1);
-        Segment->BufferId       = NULL; // granted, ensure its null
-        Segment->Buffer         = NULL; // granted, ensure its null
-        Segment->Length         = 0;    // granted, ensure its 0
-
-        ASSERT3U((Length / SectorSize), ==, *SectorsNow);
-    } else {
-        PMDL        Mdl;
-
-        ++Target->SegsBounced;
-        // get first sector, last sector and count
-        Segment->FirstSector    = 0;
-        *SectorsNow             = __min(SectorsLeft, SectorsPerPage);
-        Segment->LastSector     = (UCHAR)(*SectorsNow - 1);
-
-        // map SGList to Virtual Address. Populates Segment->Buffer and 
Segment->Length
-#pragma warning(push)
-#pragma warning(disable:28145)
-        Mdl = &Segment->Mdl;
-        Mdl->Next           = NULL;
-        Mdl->Size           = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER));
-        Mdl->MdlFlags       = MDL_PAGES_LOCKED;
-        Mdl->Process        = NULL;
-        Mdl->MappedSystemVa = NULL;
-        Mdl->StartVa        = NULL;
-        Mdl->ByteCount      = Length;
-        Mdl->ByteOffset     = Offset;
-        Segment->Pfn[0]     = Pfn;
-
-        if (Length < *SectorsNow * SectorSize) {
-            Pfn = AdapterGetNextSGEntry(TargetGetAdapter(Target),
-                                        SrbExt,
-                                        Length,
-                                        &Offset,
-                                        &Length);
-            Mdl->Size       += sizeof(PFN_NUMBER);
-            Mdl->ByteCount  = Mdl->ByteCount + Length;
-            Segment->Pfn[1] = Pfn;
-        }
-#pragma warning(pop)
-
-        ASSERT((Mdl->ByteCount & (SectorSize - 1)) == 0);
-        ASSERT3U(Mdl->ByteCount, <=, PAGE_SIZE);
-        ASSERT3U(*SectorsNow, ==, (Mdl->ByteCount / SectorSize));
-
-        Segment->Length = __min(Mdl->ByteCount, PAGE_SIZE);
-        Segment->Buffer = MmMapLockedPagesSpecifyCache(Mdl, KernelMode,
-                                MmCached, NULL, FALSE, 
__TargetPriority(Target));
-        if (!Segment->Buffer) {
-            ++Target->FailedMaps;
-            goto fail1;
-        }
-
-        ASSERT3P(MmGetMdlPfnArray(Mdl)[0], ==, Segment->Pfn[0]);
-        ASSERT3P(MmGetMdlPfnArray(Mdl)[1], ==, Segment->Pfn[1]);
-
-        // get a buffer
-        if (!BufferGet(Segment, &Segment->BufferId, &Pfn)) {
-            ++Target->FailedBounces;
-            goto fail2;
-        }
-
-        // copy contents in
-        if (ReadOnly) { // Operation == BLKIF_OP_WRITE
-            BufferCopyIn(Segment->BufferId, Segment->Buffer, Segment->Length);
-        }
-    }
-
-    // Grant segment's page
-    Status = GranterGet(Granter, Pfn, ReadOnly, &Segment->Grant);
-    if (!NT_SUCCESS(Status)) {
-        ++Target->FailedGrants;
-        goto fail3;
-    }
-
-    return TRUE;
-
-fail3:
-fail2:
-fail1:
-    return FALSE;
-}
-
-static BOOLEAN
-PrepareBlkifReadWrite(
-    IN  PXENVBD_TARGET          Target,
-    IN  PXENVBD_REQUEST         Request,
-    IN  PXENVBD_SRBEXT          SrbExt,
-    IN  ULONG                   MaxSegments,
-    IN  ULONG64                 SectorStart,
-    IN  ULONG                   SectorsLeft,
-    OUT PULONG                  SectorsDone
-    )
-{
-    UCHAR           Operation;
-    BOOLEAN         ReadOnly;
-    ULONG           Index;
-    __Operation(Cdb_OperationEx(Request->Srb), &Operation, &ReadOnly);
-
-    Request->Operation  = Operation;
-    Request->NrSegments = 0;
-    Request->FirstSector = SectorStart;
-
-    for (Index = 0;
-                Index < MaxSegments &&
-                SectorsLeft > 0;
-                        ++Index) {
-        PXENVBD_SEGMENT Segment;
-        ULONG           SectorsNow;
-
-        Segment = TargetGetSegment(Target);
-        if (Segment == NULL)
-            goto fail1;
-
-        InsertTailList(&Request->Segments, &Segment->Entry);
-        ++Request->NrSegments;
-
-        if (!PrepareSegment(Target,
-                            Segment,
-                            SrbExt,
-                            ReadOnly,
-                            SectorsLeft,
-                            &SectorsNow))
-            goto fail2;
-
-        *SectorsDone += SectorsNow;
-        SectorsLeft  -= SectorsNow;
-    }
-    ASSERT3U(Request->NrSegments, >, 0);
-    ASSERT3U(Request->NrSegments, <=, MaxSegments);
-
-    return TRUE;
-
-fail2:
-fail1:
-    return FALSE;
-}
-
-static BOOLEAN
-PrepareBlkifIndirect(
-    IN  PXENVBD_TARGET             Target,
-    IN  PXENVBD_REQUEST         Request
-    )
-{
-    ULONG           Index;
-    ULONG           NrSegments = 0;
-
-    for (Index = 0;
-            Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
-            NrSegments < Request->NrSegments;
-                ++Index) {
-        PXENVBD_INDIRECT    Indirect;
-
-        Indirect = TargetGetIndirect(Target);
-        if (Indirect == NULL)
-            goto fail1;
-        InsertTailList(&Request->Indirects, &Indirect->Entry);
-
-        NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE;
-    }
-
-    return TRUE;
-
-fail1:
-    return FALSE;
-}
-
-static FORCEINLINE ULONG
-UseIndirect(
-    IN  PXENVBD_TARGET             Target,
-    IN  ULONG                   SectorsLeft
-    )
-{
-    const ULONG SectorsPerPage = __SectorsPerPage(TargetSectorSize(Target));
-    const ULONG MaxIndirectSegs = 
FrontendGetFeatures(Target->Frontend)->Indirect;
-
-    if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST)
-        return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported
-
-    if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * SectorsPerPage)
-        return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single 
BLKIF_OP_{READ/WRITE}
-
-    return MaxIndirectSegs;
-}
-
-static FORCEINLINE ULONG
-TargetQueueRequestList(
-    IN  PXENVBD_TARGET     Target,
-    IN  PLIST_ENTRY     List
-    )
-{
-    ULONG               Count = 0;
-    for (;;) {
-        PXENVBD_REQUEST Request;
-        PLIST_ENTRY     Entry;
-
-        Entry = RemoveHeadList(List);
-        if (Entry == List)
-            break;
-
-        ++Count;
-        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
-        __TargetIncBlkifOpCount(Target, Request);
-        QueueAppend(&Target->PreparedReqs, &Request->Entry);
-    }
-    return Count;
-}
-
-static FORCEINLINE VOID
-TargetCancelRequestList(
-    IN  PXENVBD_TARGET     Target,
-    IN  PLIST_ENTRY     List
-    )
-{
-    for (;;) {
-        PXENVBD_REQUEST Request;
-        PLIST_ENTRY     Entry;
-
-        Entry = RemoveHeadList(List);
-        if (Entry == List)
-            break;
-
-        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
-        TargetPutRequest(Target, Request);
-    }
-}
-
-__checkReturn
-static BOOLEAN
-PrepareReadWrite(
-    __in PXENVBD_TARGET             Target,
-    __in PSCSI_REQUEST_BLOCK     Srb
-    )
-{
-    PXENVBD_SRBEXT  SrbExt = GetSrbExt(Srb);
-    ULONG64         SectorStart = Cdb_LogicalBlock(Srb);
-    ULONG           SectorsLeft = Cdb_TransferBlock(Srb);
-    LIST_ENTRY      List;
-    ULONG           DebugCount;
-
-    Srb->SrbStatus = SRB_STATUS_PENDING;
-
-    InitializeListHead(&List);
-    SrbExt->Count = 0;
-
-    while (SectorsLeft > 0) {
-        ULONG           MaxSegments;
-        ULONG           SectorsDone = 0;
-        PXENVBD_REQUEST Request;
-
-        Request = TargetGetRequest(Target);
-        if (Request == NULL)
-            goto fail1;
-        InsertTailList(&List, &Request->Entry);
-        InterlockedIncrement(&SrbExt->Count);
-
-        Request->Srb    = Srb;
-        MaxSegments = UseIndirect(Target, SectorsLeft);
-
-        if (!PrepareBlkifReadWrite(Target,
-                                   Request,
-                                   SrbExt,
-                                   MaxSegments,
-                                   SectorStart,
-                                   SectorsLeft,
-                                   &SectorsDone))
-            goto fail2;
-
-        if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) {
-            if (!PrepareBlkifIndirect(Target, Request))
-                goto fail3;
-        }
-
-        SectorsLeft -= SectorsDone;
-        SectorStart += SectorsDone;
-    }
-
-    DebugCount = TargetQueueRequestList(Target, &List);
-    if (DebugCount != (ULONG)SrbExt->Count) {
-        Trace("[%u] %d != %u\n", TargetGetTargetId(Target), SrbExt->Count, 
DebugCount);
-    }
-    return TRUE;
-
-fail3:
-fail2:
-fail1:
-    TargetCancelRequestList(Target, &List);
-    SrbExt->Count = 0;
-    Srb->SrbStatus = SRB_STATUS_ERROR;
-    return FALSE;
-}
-
-__checkReturn
-static BOOLEAN
-PrepareSyncCache(
-    __in PXENVBD_TARGET             Target,
-    __in PSCSI_REQUEST_BLOCK     Srb
-    )
-{
-    PXENVBD_SRBEXT      SrbExt = GetSrbExt(Srb);
-    PXENVBD_REQUEST     Request;
-    LIST_ENTRY          List;
-    UCHAR               Operation;
-    ULONG               DebugCount;
-
-    Srb->SrbStatus = SRB_STATUS_PENDING;
-
-    if (FrontendGetDiskInfo(Target->Frontend)->FlushCache)
-        Operation = BLKIF_OP_FLUSH_DISKCACHE;
-    else
-        Operation = BLKIF_OP_WRITE_BARRIER;
-
-    InitializeListHead(&List);
-    SrbExt->Count = 0;
-
-    Request = TargetGetRequest(Target);
-    if (Request == NULL)
-        goto fail1;
-    InsertTailList(&List, &Request->Entry);
-    InterlockedIncrement(&SrbExt->Count);
-
-    Request->Srb        = Srb;
-    Request->Operation  = Operation;
-    Request->FirstSector = Cdb_LogicalBlock(Srb);
-
-    DebugCount = TargetQueueRequestList(Target, &List);
-    if (DebugCount != (ULONG)SrbExt->Count) {
-        Trace("[%u] %d != %u\n", TargetGetTargetId(Target), SrbExt->Count, 
DebugCount);
-    }
-    return TRUE;
-
-fail1:
-    TargetCancelRequestList(Target, &List);
-    SrbExt->Count = 0;
-    Srb->SrbStatus = SRB_STATUS_ERROR;
-    return FALSE;
-}
-
-__checkReturn
-static BOOLEAN
-PrepareUnmap(
-    __in PXENVBD_TARGET             Target,
-    __in PSCSI_REQUEST_BLOCK     Srb
-    )
-{
-    PXENVBD_SRBEXT      SrbExt = GetSrbExt(Srb);
-    PUNMAP_LIST_HEADER  Unmap = Srb->DataBuffer;
-       ULONG               Count = 
_byteswap_ushort(*(PUSHORT)Unmap->BlockDescrDataLength) / 
sizeof(UNMAP_BLOCK_DESCRIPTOR);
-    ULONG               Index;
-    LIST_ENTRY          List;
-    ULONG               DebugCount;
-
-    Srb->SrbStatus = SRB_STATUS_PENDING;
-
-    InitializeListHead(&List);
-    SrbExt->Count = 0;
-
-    for (Index = 0; Index < Count; ++Index) {
-        PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index];
-        PXENVBD_REQUEST         Request;
-
-        Request = TargetGetRequest(Target);
-        if (Request == NULL)
-            goto fail1;
-        InsertTailList(&List, &Request->Entry);
-        InterlockedIncrement(&SrbExt->Count);
-
-        Request->Srb            = Srb;
-        Request->Operation      = BLKIF_OP_DISCARD;
-        Request->FirstSector    = 
_byteswap_uint64(*(PULONG64)Descr->StartingLba);
-        Request->NrSectors      = _byteswap_ulong(*(PULONG)Descr->LbaCount);
-        Request->Flags          = 0;
-    }
-
-    DebugCount = TargetQueueRequestList(Target, &List);
-    if (DebugCount != (ULONG)SrbExt->Count) {
-        Trace("[%u] %d != %u\n", TargetGetTargetId(Target), SrbExt->Count, 
DebugCount);
-    }
-    return TRUE;
-
-fail1:
-    TargetCancelRequestList(Target, &List);
-    SrbExt->Count = 0;
-    Srb->SrbStatus = SRB_STATUS_ERROR;
-    return FALSE;
-}
-
-//=============================================================================
-// Queue-Related
-static FORCEINLINE VOID
-__TargetPauseDataPath(
-    __in PXENVBD_TARGET             Target,
-    __in BOOLEAN                 Timeout
-    )
-{
-    KIRQL               Irql;
-    ULONG               Requests;
-    ULONG               Count = 0;
-    PXENVBD_RING        Ring = FrontendGetRing(Target->Frontend);
-
-    KeAcquireSpinLock(&Target->Lock, &Irql);
-    ++Target->Paused;
-    KeReleaseSpinLock(&Target->Lock, Irql);
-
-    Requests = QueueCount(&Target->SubmittedReqs);
-    KeMemoryBarrier();
-
-    Verbose("Target[%d] : Waiting for %d Submitted requests\n", 
TargetGetTargetId(Target), Requests);
-
-    // poll ring and send event channel notification every 1ms (for up to 3 
minutes)
-    while (QueueCount(&Target->SubmittedReqs)) {
-        if (Timeout && Count > 180000)
-            break;
-        KeRaiseIrql(DISPATCH_LEVEL, &Irql);
-        RingPoll(Ring);
-        KeLowerIrql(Irql);
-        RingSend(Ring);         // let backend know it needs to do some work
-        StorPortStallExecution(1000);   // 1000 micro-seconds
-        ++Count;
-    }
-
-    Verbose("Target[%d] : %u/%u Submitted requests left (%u iterrations)\n",
-            TargetGetTargetId(Target), QueueCount(&Target->SubmittedReqs), 
Requests, Count);
-
-    // Abort Fresh SRBs
-    for (;;) {
-        PXENVBD_SRBEXT  SrbExt;
-        PLIST_ENTRY     Entry = QueuePop(&Target->FreshSrbs);
-        if (Entry == NULL)
-            break;
-        SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
-
-        Verbose("Target[%d] : FreshSrb 0x%p -> SCSI_ABORTED\n", 
TargetGetTargetId(Target), SrbExt->Srb);
-        SrbExt->Srb->SrbStatus = SRB_STATUS_ABORTED;
-        SrbExt->Srb->ScsiStatus = 0x40; // SCSI_ABORTED;
-        AdapterCompleteSrb(TargetGetAdapter(Target), SrbExt);
-    }
-
-    // Fail PreparedReqs
-    for (;;) {
-        PXENVBD_SRBEXT  SrbExt;
-        PXENVBD_REQUEST Request;
-        PLIST_ENTRY     Entry = QueuePop(&Target->PreparedReqs);
-        if (Entry == NULL)
-            break;
-        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
-        SrbExt = GetSrbExt(Request->Srb);
-
-        Verbose("Target[%d] : PreparedReq 0x%p -> FAILED\n", 
TargetGetTargetId(Target), Request);
-
-        SrbExt->Srb->SrbStatus = SRB_STATUS_ABORTED;
-        TargetPutRequest(Target, Request);
-
-        if (InterlockedDecrement(&SrbExt->Count) == 0) {
-            SrbExt->Srb->ScsiStatus = 0x40; // SCSI_ABORTED
-            AdapterCompleteSrb(TargetGetAdapter(Target), SrbExt);
-        }
-    }
-}
-
-static FORCEINLINE VOID
-__TargetUnpauseDataPath(
-    __in PXENVBD_TARGET             Target
-    )
-{
-    KIRQL   Irql;
-
-    KeAcquireSpinLock(&Target->Lock, &Irql);
-    --Target->Paused;
-    KeReleaseSpinLock(&Target->Lock, Irql);
-}
-
-static FORCEINLINE BOOLEAN
-TargetPrepareFresh(
-    IN  PXENVBD_TARGET         Target
-    )
-{
-    PXENVBD_SRBEXT  SrbExt;
-    PLIST_ENTRY     Entry;
-
-    Entry = QueuePop(&Target->FreshSrbs);
-    if (Entry == NULL)
-        return FALSE;   // fresh queue is empty
-
-    SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
-
-    switch (Cdb_OperationEx(SrbExt->Srb)) {
-    case SCSIOP_READ:
-    case SCSIOP_WRITE:
-        if (PrepareReadWrite(Target, SrbExt->Srb))
-            return TRUE;    // prepared this SRB
-        break;
-    case SCSIOP_SYNCHRONIZE_CACHE:
-        if (PrepareSyncCache(Target, SrbExt->Srb))
-            return TRUE;    // prepared this SRB
-        break;
-    case SCSIOP_UNMAP:
-        if (PrepareUnmap(Target, SrbExt->Srb))
-            return TRUE;    // prepared this SRB
-        break;
-    default:
-        ASSERT(FALSE);
-        break;
-    }
-    QueueUnPop(&Target->FreshSrbs, &SrbExt->Entry);
-
-    return FALSE;       // prepare failed
-}
-
-static FORCEINLINE BOOLEAN
-TargetSubmitPrepared(
-    __in PXENVBD_TARGET             Target
-    )
-{
-    PXENVBD_RING   Ring = FrontendGetRing(Target->Frontend);
-    if (TargetIsPaused(Target)) {
-        if (QueueCount(&Target->PreparedReqs))
-            Warning("Target[%d] : Paused, not submitting new requests (%u)\n",
-                    TargetGetTargetId(Target),
-                    QueueCount(&Target->PreparedReqs));
-        return FALSE;
-    }
-
-    for (;;) {
-        PXENVBD_REQUEST Request;
-        PLIST_ENTRY     Entry;
-
-        Entry = QueuePop(&Target->PreparedReqs);
-        if (Entry == NULL)
-            break;
-
-        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
-
-        QueueAppend(&Target->SubmittedReqs, &Request->Entry);
-        KeMemoryBarrier();
-
-        if (RingSubmit(Ring, Request))
-            continue;
-
-        QueueRemove(&Target->SubmittedReqs, &Request->Entry);
-        QueueUnPop(&Target->PreparedReqs, &Request->Entry);
-        return FALSE;   // ring full
-    }
-
-    return TRUE;
-}
-
-static FORCEINLINE VOID
-TargetCompleteShutdown(
-    __in PXENVBD_TARGET             Target
-    )
-{
-    if (QueueCount(&Target->ShutdownSrbs) == 0)
-        return;
-
-    if (QueueCount(&Target->FreshSrbs) ||
-        QueueCount(&Target->PreparedReqs) ||
-        QueueCount(&Target->SubmittedReqs))
-        return;
-
-    for (;;) {
-        PXENVBD_SRBEXT  SrbExt;
-        PLIST_ENTRY     Entry = QueuePop(&Target->ShutdownSrbs);
-        if (Entry == NULL)
-            break;
-        SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
-        SrbExt->Srb->SrbStatus = SRB_STATUS_SUCCESS;
-        AdapterCompleteSrb(TargetGetAdapter(Target), SrbExt);
-    }
-}
-
-static FORCEINLINE PCHAR
-BlkifOperationName(
-    IN  UCHAR                   Operation
-    )
-{
-    switch (Operation) {
-    case BLKIF_OP_READ:             return "READ";
-    case BLKIF_OP_WRITE:            return "WRITE";
-    case BLKIF_OP_WRITE_BARRIER:    return "WRITE_BARRIER";
-    case BLKIF_OP_FLUSH_DISKCACHE:  return "FLUSH_DISKCACHE";
-    case BLKIF_OP_RESERVED_1:       return "RESERVED_1";
-    case BLKIF_OP_DISCARD:          return "DISCARD";
-    case BLKIF_OP_INDIRECT:         return "INDIRECT";
-    default:                        return "<unknown>";
-    }
-}
-
-BOOLEAN
-TargetSubmitRequests(
-    IN  PXENVBD_TARGET  Target
-    )
-{
-    BOOLEAN             Retry = FALSE;
-
-    for (;;) {
-        // submit all prepared requests (0 or more requests)
-        // return TRUE if submitted 0 or more requests from prepared queue
-        // return FALSE iff ring is full
-        if (!TargetSubmitPrepared(Target))
-            break;
-
-        // prepare a single SRB (into 1 or more requests)
-        // return TRUE if prepare succeeded
-        // return FALSE if prepare failed or fresh queue empty
-        if (!TargetPrepareFresh(Target))
-            break;
-
-        // back off, check DPC timeout and try again
-        Retry = TRUE;
-        break;
-    }
-
-    // if no requests/SRBs outstanding, complete any shutdown SRBs
-    if (!Retry)
-        TargetCompleteShutdown(Target);
-
-    return Retry;
-}
-
-VOID
-TargetCompleteResponse(
-    IN  PXENVBD_TARGET  Target,
-    IN  ULONG           Tag,
-    IN  SHORT           Status
-    )
-{
-    PXENVBD_REQUEST     Request;
-    PSCSI_REQUEST_BLOCK Srb;
-    PXENVBD_SRBEXT      SrbExt;
-
-    Request = TargetRequestFromTag(Target, Tag);
-    if (Request == NULL)
-        return;
-
-    Srb     = Request->Srb;
-    SrbExt  = GetSrbExt(Srb);
-    ASSERT3P(SrbExt, !=, NULL);
-
-    switch (Status) {
-    case BLKIF_RSP_OKAY:
-        RequestCopyOutput(Request);
-        break;
-
-    case BLKIF_RSP_EOPNOTSUPP:
-        // Remove appropriate feature support
-        FrontendRemoveFeature(Target->Frontend, Request->Operation);
-        // Succeed this SRB, subsiquent SRBs will be succeeded instead of 
being passed to the backend.
-        Srb->SrbStatus = SRB_STATUS_SUCCESS;
-        break;
-
-    case BLKIF_RSP_ERROR:
-    default:
-        Warning("Target[%d] : %s BLKIF_RSP_ERROR (Tag %x)\n",
-                TargetGetTargetId(Target), 
BlkifOperationName(Request->Operation), Tag);
-        Srb->SrbStatus = SRB_STATUS_ERROR;
-        break;
-    }
-
-    TargetPutRequest(Target, Request);
-
-    // complete srb
-    if (InterlockedDecrement(&SrbExt->Count) == 0) {
-        if (Srb->SrbStatus == SRB_STATUS_PENDING) {
-            // SRB has not hit a failure condition (BLKIF_RSP_ERROR | 
BLKIF_RSP_EOPNOTSUPP)
-            // from any of its responses. SRB must have succeeded
-            Srb->SrbStatus = SRB_STATUS_SUCCESS;
-            Srb->ScsiStatus = 0x00; // SCSI_GOOD
-        } else {
-            // Srb->SrbStatus has already been set by 1 or more requests with 
Status != BLKIF_RSP_OKAY
-            Srb->ScsiStatus = 0x40; // SCSI_ABORTED
-        }
-
-        AdapterCompleteSrb(TargetGetAdapter(Target), SrbExt);
-    }
-}
-
 //=============================================================================
 // SRBs
 __checkReturn
@@ -1407,9 +278,7 @@ TargetReadWrite(
         return TRUE; // Complete now
     }
 
-    QueueAppend(&Target->FreshSrbs, &SrbExt->Entry);
-    RingKick(Ring);
-
+    RingQueueRequest(Ring, SrbExt);
     return FALSE;
 }
 
@@ -1437,9 +306,7 @@ TargetSyncCache(
         return TRUE;
     }
 
-    QueueAppend(&Target->FreshSrbs, &SrbExt->Entry);
-    RingKick(Ring);
-
+    RingQueueRequest(Ring, SrbExt);
     return FALSE;
 }
 
@@ -1466,9 +333,7 @@ TargetUnmap(
         return TRUE;
     }
 
-    QueueAppend(&Target->FreshSrbs, &SrbExt->Entry);
-    RingKick(Ring);
-
+    RingQueueRequest(Ring, SrbExt);
     return FALSE;
 }
 
@@ -1872,15 +737,7 @@ TargetReset(
 {
     Verbose("[%u] =====>\n", TargetGetTargetId(Target));
 
-    __TargetPauseDataPath(Target, TRUE);
-
-    if (QueueCount(&Target->SubmittedReqs)) {
-        Error("Target[%d] : backend has %u outstanding requests after a 
TargetReset\n",
-              TargetGetTargetId(Target),
-              QueueCount(&Target->SubmittedReqs));
-    }
-
-    __TargetUnpauseDataPath(Target);
+    FrontendReset(Target->Frontend);
 
     Verbose("[%u] <=====\n", TargetGetTargetId(Target));
 }
@@ -1891,8 +748,7 @@ TargetFlush(
     IN  PXENVBD_SRBEXT  SrbExt
     )
 {
-    QueueAppend(&Target->ShutdownSrbs, &SrbExt->Entry);
-    RingKick(FrontendGetRing(Target->Frontend));
+    RingQueueShutdown(FrontendGetRing(Target->Frontend), SrbExt);
 }
 
 VOID
@@ -1901,8 +757,7 @@ TargetShutdown(
     IN  PXENVBD_SRBEXT  SrbExt
     )
 {
-    QueueAppend(&Target->ShutdownSrbs, &SrbExt->Entry);
-    RingKick(FrontendGetRing(Target->Frontend));
+    RingQueueShutdown(FrontendGetRing(Target->Frontend), SrbExt);
 }
 
 VOID
@@ -2156,58 +1011,8 @@ TargetDebugCallback(
                  "TARGET: %s\n",
                  Target->Missing ? Target->Reason : "Not Missing");
 
-    XENBUS_DEBUG(Printf,
-                 &Target->DebugInterface,
-                 "TARGET: BLKIF_OPs: READ=%u WRITE=%u\n",
-                 Target->BlkOpRead, Target->BlkOpWrite);
-    XENBUS_DEBUG(Printf,
-                 &Target->DebugInterface,
-                 "TARGET: BLKIF_OPs: INDIRECT_READ=%u INDIRECT_WRITE=%u\n",
-                 Target->BlkOpIndirectRead, Target->BlkOpIndirectWrite);
-    XENBUS_DEBUG(Printf,
-                 &Target->DebugInterface,
-                 "TARGET: BLKIF_OPs: BARRIER=%u DISCARD=%u FLUSH=%u\n",
-                 Target->BlkOpBarrier, Target->BlkOpDiscard, 
Target->BlkOpFlush);
-    XENBUS_DEBUG(Printf,
-                 &Target->DebugInterface,
-                 "TARGET: Failed: Maps=%u Bounces=%u Grants=%u\n",
-                 Target->FailedMaps, Target->FailedBounces, 
Target->FailedGrants);
-    XENBUS_DEBUG(Printf,
-                 &Target->DebugInterface,
-                 "TARGET: Segments Granted=%llu Bounced=%llu\n",
-                 Target->SegsGranted, Target->SegsBounced);
-
-    __LookasideDebug(&Target->RequestList,
-                     &Target->DebugInterface,
-                     "REQUESTs");
-    __LookasideDebug(&Target->SegmentList,
-                     &Target->DebugInterface,
-                     "SEGMENTs");
-    __LookasideDebug(&Target->IndirectList,
-                     &Target->DebugInterface,
-                     "INDIRECTs");
-
-    QueueDebugCallback(&Target->FreshSrbs,
-                       "Fresh    ",
-                       &Target->DebugInterface);
-    QueueDebugCallback(&Target->PreparedReqs,
-                       "Prepared ",
-                       &Target->DebugInterface);
-    QueueDebugCallback(&Target->SubmittedReqs,
-                       "Submitted",
-                       &Target->DebugInterface);
-    QueueDebugCallback(&Target->ShutdownSrbs,
-                       "Shutdown ",
-                       &Target->DebugInterface);
-
     FrontendDebugCallback(Target->Frontend,
                           &Target->DebugInterface);
-
-    Target->BlkOpRead = Target->BlkOpWrite = 0;
-    Target->BlkOpIndirectRead = Target->BlkOpIndirectWrite = 0;
-    Target->BlkOpBarrier = Target->BlkOpDiscard = Target->BlkOpFlush = 0;
-    Target->FailedMaps = Target->FailedBounces = Target->FailedGrants = 0;
-    Target->SegsGranted = Target->SegsBounced = 0;
 }
 
 static DECLSPEC_NOINLINE VOID
@@ -2216,61 +1021,8 @@ TargetSuspendCallback(
     )
 {
     PXENVBD_TARGET  Target = Argument;
-    LIST_ENTRY      List;
-
-    InitializeListHead(&List);
-
-    // pop all submitted requests, cleanup and add associated SRB to a list
-    for (;;) {
-        PXENVBD_SRBEXT  SrbExt;
-        PXENVBD_REQUEST Request;
-        PLIST_ENTRY     Entry = QueuePop(&Target->SubmittedReqs);
-        if (Entry == NULL)
-            break;
-        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
-        SrbExt = GetSrbExt(Request->Srb);
-
-        TargetPutRequest(Target, Request);
-
-        if (InterlockedDecrement(&SrbExt->Count) == 0) {
-            InsertTailList(&List, &SrbExt->Entry);
-        }
-    }
-
-    // pop all prepared requests, cleanup and add associated SRB to a list
-    for (;;) {
-        PXENVBD_SRBEXT  SrbExt;
-        PXENVBD_REQUEST Request;
-        PLIST_ENTRY     Entry = QueuePop(&Target->PreparedReqs);
-        if (Entry == NULL)
-            break;
-        Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
-        SrbExt = GetSrbExt(Request->Srb);
-
-        TargetPutRequest(Target, Request);
-
-        if (InterlockedDecrement(&SrbExt->Count) == 0) {
-            InsertTailList(&List, &SrbExt->Entry);
-        }
-    }
-
-    // foreach SRB in list, put on start of FreshSrbs
-    for (;;) {
-        PXENVBD_SRBEXT  SrbExt;
-        PLIST_ENTRY     Entry = RemoveTailList(&List);
-        if (Entry == &List)
-            break;
-        SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
 
-        QueueUnPop(&Target->FreshSrbs, &SrbExt->Entry);
-    }
-
-    // now the first set of requests popped off submitted list is the next SRB
-    // to be popped off the fresh list
-
-    Verbose("Target[%d] : %d Fresh SRBs\n",
-            TargetGetTargetId(Target),
-            QueueCount(&Target->FreshSrbs));
+    RingReQueueRequests(FrontendGetRing(Target->Frontend));
 
     Verbose("Target[%d] : %s (%s)\n",
             TargetGetTargetId(Target),
@@ -2332,8 +1084,6 @@ TargetD3ToD0(
     if (!NT_SUCCESS(status))
         goto fail6;
 
-    __TargetUnpauseDataPath(Target);
-
     return STATUS_SUCCESS;
 
 fail6:
@@ -2388,8 +1138,6 @@ TargetD0ToD3(
 
     Verbose("Target[%d] : D0->D3\n", TargetId);
 
-    __TargetPauseDataPath(Target, FALSE);
-
     (VOID) FrontendSetState(Target->Frontend, XENVBD_CLOSED);
 
     FrontendD0ToD3(Target->Frontend);
@@ -2471,18 +1219,10 @@ TargetCreate(
     Target->Signature       = TARGET_SIGNATURE;
     Target->Adapter         = Adapter;
     Target->DeviceObject    = NULL; // filled in later
-    Target->Paused          = 1; // Paused until D3->D0 transition
     Target->DevicePnpState  = Present;
     Target->DevicePowerState = PowerDeviceD3;
 
     KeInitializeSpinLock(&Target->Lock);
-    QueueInit(&Target->FreshSrbs);
-    QueueInit(&Target->PreparedReqs);
-    QueueInit(&Target->SubmittedReqs);
-    QueueInit(&Target->ShutdownSrbs);
-    __LookasideInit(&Target->RequestList, sizeof(XENVBD_REQUEST), 
REQUEST_POOL_TAG);
-    __LookasideInit(&Target->SegmentList, sizeof(XENVBD_SEGMENT), 
SEGMENT_POOL_TAG);
-    __LookasideInit(&Target->IndirectList, sizeof(XENVBD_INDIRECT), 
INDIRECT_POOL_TAG);
 
     status = FrontendCreate(Target, DeviceId, TargetId, &Target->Frontend);
     if (!NT_SUCCESS(status))
@@ -2504,9 +1244,6 @@ fail3:
 
 fail2:
     Error("Fail2\n");
-    __LookasideTerm(&Target->IndirectList);
-    __LookasideTerm(&Target->SegmentList);
-    __LookasideTerm(&Target->RequestList);
     __TargetFree(Target);
 
 fail1:
@@ -2530,10 +1267,6 @@ TargetDestroy(
     FrontendDestroy(Target->Frontend);
     Target->Frontend = NULL;
 
-    __LookasideTerm(&Target->IndirectList);
-    __LookasideTerm(&Target->SegmentList);
-    __LookasideTerm(&Target->RequestList);
-
     RtlZeroMemory(Target, sizeof(XENVBD_TARGET));
     __TargetFree(Target);
 
diff --git a/src/xenvbd/target.h b/src/xenvbd/target.h
index b790eb4..dd360f5 100644
--- a/src/xenvbd/target.h
+++ b/src/xenvbd/target.h
@@ -93,18 +93,6 @@ TargetSetDeviceObject(
     IN  PDEVICE_OBJECT  DeviceObject
     );
 
-extern BOOLEAN
-TargetSubmitRequests(
-    IN  PXENVBD_TARGET  Target
-    );
-
-extern VOID
-TargetCompleteResponse(
-    IN  PXENVBD_TARGET  Target,
-    IN  ULONG           Tag,
-    IN  SHORT           Status
-    );
-
 extern VOID
 TargetPrepareIo(
     IN  PXENVBD_TARGET  Target,
-- 
2.8.3


_______________________________________________
win-pv-devel mailing list
win-pv-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/cgi-bin/mailman/listinfo/win-pv-devel

 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.