|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] RE: [RFC PATCH] Use a FreeList for CACHE entries
RFC patch - there are still outstanding bugs that need addressing, but I wanted
to get comments on the issue and proposed fix
Owen
-----Original Message-----
From: win-pv-devel <win-pv-devel-bounces@xxxxxxxxxxxxxxxxxxxx> On Behalf Of
Owen Smith
Sent: 28 July 2022 15:53
To: win-pv-devel@xxxxxxxxxxxxxxxxxxxx
Cc: Owen Smith <owen.smith@xxxxxxxxxx>
Subject: [RFC PATCH] Use a FreeList for CACHE entries
[CAUTION - EXTERNAL EMAIL] DO NOT reply, click links, or open attachments
unless you have verified the sender and know the content is safe.
The CACHE slabs can reserve a large number of resources, particularly when the
object constructor allocates memory or reserves grant table entries, which
leads to resource starvation far quicker than expected.
For Example, the GNTTAB object constructor will reserve a grant entry, while
the GNTTAB cache will assign 253 objects per slab (i.e. a slab will refer to
253 grantref_t's). The XenVif receiver ring will require 257 GNTTAB objects, so
2 slabs will be allocated (meaning 506 grantref_t's are reserved per receive
queue, while only 257 will ever be used).
Add CACHE v3 that allows callers to specify how many objects are created in a
single pass (default 1) which will allow CACHE objects to grow at a more
managable pace, while maintaining overhead for bursty operations to limit the
number of the slow-path allocation cycles.
Note: RFC patch, there are issues that still need investigating
Signed-off-by: Owen Smith <owen.smith@xxxxxxxxxx>
---
include/cache_interface.h | 75 ++++-
include/revision.h | 3 +-
src/xenbus/cache.c | 608 ++++++++++++--------------------------
src/xenbus/gnttab.c | 1 +
4 files changed, 257 insertions(+), 430 deletions(-)
diff --git a/include/cache_interface.h b/include/cache_interface.h index
ce50f4e..839212c 100644
--- a/include/cache_interface.h
+++ b/include/cache_interface.h
@@ -123,6 +123,23 @@ typedef VOID
IN PVOID Argument
);
+/*! \typedef XENBUS_CACHE_CREATE_V1
+ \brief Create a cache of objects of the given \a Size
+
+ \param Interface The interface header
+ \param Name A name for the cache which will be used in debug output
+ \param Size The size of each object in bytes
+ \param Reservation The target minimum population of the cache
+ \param Ctor A callback which is invoked when a new object created
+ \param Dtor A callback which is invoked when an object is destroyed
+ \param AcquireLock A callback invoked to acquire a spinlock
+ \param ReleaseLock A callback invoked to release the spinlock
+ \param Argument An optional context argument passed to the callbacks
+ \param Cache A pointer to a cache handle to be initialized
+
+ If a non-zero \a Reservation is specified then this method will fail
+ unless that number of objects can be immediately created.
+*/
typedef NTSTATUS
(*XENBUS_CACHE_CREATE_V1)(
IN PINTERFACE Interface,
@@ -137,6 +154,39 @@ typedef NTSTATUS
OUT PXENBUS_CACHE *Cache
);
+/*! \typedef XENBUS_CACHE_CREATE_V2
+ \brief Create a cache of objects of the given \a Size
+
+ \param Interface The interface header
+ \param Name A name for the cache which will be used in debug output
+ \param Size The size of each object in bytes
+ \param Reservation The target minimum population of the cache
+ \param Cap The maximum population of the cache
+ \param Ctor A callback which is invoked when a new object created
+ \param Dtor A callback which is invoked when an object is destroyed
+ \param AcquireLock A callback invoked to acquire a spinlock
+ \param ReleaseLock A callback invoked to release the spinlock
+ \param Argument An optional context argument passed to the callbacks
+ \param Cache A pointer to a cache handle to be initialized
+
+ If a non-zero \a Reservation is specified then this method will fail
+ unless that number of objects can be immediately created.
+*/
+typedef NTSTATUS
+(*XENBUS_CACHE_CREATE_V2)(
+ IN PINTERFACE Interface,
+ IN const CHAR *Name,
+ IN ULONG Size,
+ IN ULONG Reservation,
+ IN ULONG Cap,
+ IN XENBUS_CACHE_CTOR Ctor,
+ IN XENBUS_CACHE_DTOR Dtor,
+ IN XENBUS_CACHE_ACQUIRE_LOCK AcquireLock,
+ IN XENBUS_CACHE_RELEASE_LOCK ReleaseLock,
+ IN PVOID Argument OPTIONAL,
+ OUT PXENBUS_CACHE *Cache
+ );
+
/*! \typedef XENBUS_CACHE_CREATE
\brief Create a cache of objects of the given \a Size
@@ -145,6 +195,7 @@ typedef NTSTATUS
\param Size The size of each object in bytes
\param Reservation The target minimum population of the cache
\param Cap The maximum population of the cache
+ \param Stride The number of objects to initialize in one go.
\param Ctor A callback which is invoked when a new object created
\param Dtor A callback which is invoked when an object is destroyed
\param AcquireLock A callback invoked to acquire a spinlock @@ -154,7
+205,7 @@ typedef NTSTATUS
If a non-zero \a Reservation is specified then this method will fail
unless that number of objects can be immediately created.
-*/
+*/
typedef NTSTATUS
(*XENBUS_CACHE_CREATE)(
IN PINTERFACE Interface,
@@ -162,6 +213,7 @@ typedef NTSTATUS
IN ULONG Size,
IN ULONG Reservation,
IN ULONG Cap,
+ IN ULONG Stride,
IN XENBUS_CACHE_CTOR Ctor,
IN XENBUS_CACHE_DTOR Dtor,
IN XENBUS_CACHE_ACQUIRE_LOCK AcquireLock,
@@ -234,20 +286,33 @@ struct _XENBUS_CACHE_INTERFACE_V1 { };
/*! \struct _XENBUS_CACHE_INTERFACE_V2
- \brief CACHE interface version 1
+ \brief CACHE interface version 2
\ingroup interfaces
*/
struct _XENBUS_CACHE_INTERFACE_V2 {
INTERFACE Interface;
XENBUS_CACHE_ACQUIRE CacheAcquire;
XENBUS_CACHE_RELEASE CacheRelease;
- XENBUS_CACHE_CREATE CacheCreate;
+ XENBUS_CACHE_CREATE_V2 CacheCreate;
XENBUS_CACHE_GET CacheGet;
XENBUS_CACHE_PUT CachePut;
XENBUS_CACHE_DESTROY CacheDestroy;
};
-typedef struct _XENBUS_CACHE_INTERFACE_V2 XENBUS_CACHE_INTERFACE,
*PXENBUS_CACHE_INTERFACE;
+/*! \struct _XENBUS_CACHE_INTERFACE_V3
+ \brief CACHE interface version 3
+ \ingroup interfaces
+*/
+struct _XENBUS_CACHE_INTERFACE_V3 {
+ INTERFACE Interface;
+ XENBUS_CACHE_ACQUIRE CacheAcquire;
+ XENBUS_CACHE_RELEASE CacheRelease;
+ XENBUS_CACHE_CREATE CacheCreate;
+ XENBUS_CACHE_GET CacheGet;
+ XENBUS_CACHE_PUT CachePut;
+ XENBUS_CACHE_DESTROY CacheDestroy;
+};
+typedef struct _XENBUS_CACHE_INTERFACE_V3 XENBUS_CACHE_INTERFACE,
+*PXENBUS_CACHE_INTERFACE;
/*! \def XENBUS_CACHE
\brief Macro at assist in method invocation @@ -258,6 +323,6 @@ typedef
struct _XENBUS_CACHE_INTERFACE_V2 XENBUS_CACHE_INTERFACE, *PXENBUS_CACHE
#endif // _WINDLL
#define XENBUS_CACHE_INTERFACE_VERSION_MIN 1 -#define
XENBUS_CACHE_INTERFACE_VERSION_MAX 2
+#define XENBUS_CACHE_INTERFACE_VERSION_MAX 3
#endif // _XENBUS_CACHE_INTERFACE_H
diff --git a/include/revision.h b/include/revision.h index 3e3779f..e3fd789
100644
--- a/include/revision.h
+++ b/include/revision.h
@@ -56,6 +56,7 @@
DEFINE_REVISION(0x09000006, 1, 3, 8, 1, 2, 1, 2, 4, 1, 1, 1), \
DEFINE_REVISION(0x09000007, 1, 3, 8, 1, 2, 1, 2, 4, 1, 1, 2), \
DEFINE_REVISION(0x09000008, 1, 3, 9, 1, 2, 1, 2, 4, 1, 1, 2), \
- DEFINE_REVISION(0x09000009, 1, 4, 9, 1, 2, 1, 2, 4, 1, 1, 2)
+ DEFINE_REVISION(0x09000009, 1, 4, 9, 1, 2, 1, 2, 4, 1, 1, 2), \
+ DEFINE_REVISION(0x0900000A, 1, 4, 9, 1, 2, 1, 3, 4, 1, 1,
+ 2)
#endif // _REVISION_H
diff --git a/src/xenbus/cache.c b/src/xenbus/cache.c index 13fb0e5..4f47459
100644
--- a/src/xenbus/cache.c
+++ b/src/xenbus/cache.c
@@ -55,15 +55,12 @@ typedef struct _XENBUS_CACHE_MAGAZINE {
#define XENBUS_CACHE_SLAB_MAGIC 'BALS'
-typedef struct _XENBUS_CACHE_SLAB {
+typedef struct _XENBUS_CACHE_ENTRY {
ULONG Magic;
PXENBUS_CACHE Cache;
LIST_ENTRY ListEntry;
- USHORT MaximumOccupancy;
- USHORT CurrentOccupancy;
- ULONG *Mask;
UCHAR Buffer[1];
-} XENBUS_CACHE_SLAB, *PXENBUS_CACHE_SLAB;
+} XENBUS_CACHE_ENTRY, *PXENBUS_CACHE_ENTRY;
#define BITS_PER_ULONG (sizeof (ULONG) * 8)
#define MAXNAMELEN 128
@@ -74,20 +71,19 @@ struct _XENBUS_CACHE {
ULONG Size;
ULONG Reservation;
ULONG Cap;
+ ULONG Stride;
NTSTATUS (*Ctor)(PVOID, PVOID);
VOID (*Dtor)(PVOID, PVOID);
VOID (*AcquireLock)(PVOID);
VOID (*ReleaseLock)(PVOID);
PVOID Argument;
- LIST_ENTRY SlabList;
- PLIST_ENTRY Cursor;
- ULONG Count;
PXENBUS_CACHE_MAGAZINE Magazine;
ULONG MagazineCount;
- LONG CurrentSlabs;
- LONG MaximumSlabs;
+ LONG FreeSize;
+ LONG FreeMaximum;
LONG CurrentObjects;
LONG MaximumObjects;
+ LIST_ENTRY FreeList;
};
struct _XENBUS_CACHE_CONTEXT {
@@ -195,333 +191,103 @@ CachePutObjectToMagazine(
return STATUS_UNSUCCESSFUL;
}
-static VOID
-CacheInsertSlab(
- IN PXENBUS_CACHE Cache,
- IN PXENBUS_CACHE_SLAB New
- )
-{
-#define INSERT_BEFORE(_ListEntry, _New) \
- do { \
- (_New)->Blink = (_ListEntry)->Blink; \
- (_ListEntry)->Blink->Flink = (_New); \
- \
- (_ListEntry)->Blink = (_New); \
- (_New)->Flink = (_ListEntry); \
- } while (FALSE)
-
- PLIST_ENTRY ListEntry;
-
- ASSERT(New->CurrentOccupancy < New->MaximumOccupancy);
-
- Cache->Cursor = NULL;
-
- for (ListEntry = Cache->SlabList.Flink;
- ListEntry != &Cache->SlabList;
- ListEntry = ListEntry->Flink) {
- PXENBUS_CACHE_SLAB Slab;
-
- Slab = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_SLAB, ListEntry);
-
- if (Slab->CurrentOccupancy < New->CurrentOccupancy) {
- INSERT_BEFORE(ListEntry, &New->ListEntry);
- goto done;
- }
-
- if (Slab->CurrentOccupancy < Slab->MaximumOccupancy &&
- Cache->Cursor == NULL)
- Cache->Cursor = ListEntry;
- }
-
- InsertTailList(&Cache->SlabList, &New->ListEntry);
-
-done:
- if (Cache->Cursor == NULL)
- Cache->Cursor = &New->ListEntry;
-
-#undef INSERT_BEFORE
-}
-
-#if DBG
-static VOID
-CacheAudit(
- IN PXENBUS_CACHE Cache
- )
-{
- ULONG CurrentOccupancy = ULONG_MAX;
- PLIST_ENTRY ListEntry;
-
- //
- // The cursror should point at the first slab that is not fully
- // occupied.
- //
- for (ListEntry = Cache->SlabList.Flink;
- ListEntry != &Cache->SlabList;
- ListEntry = ListEntry->Flink) {
- PXENBUS_CACHE_SLAB Slab;
-
- Slab = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_SLAB, ListEntry);
-
- if (Slab->CurrentOccupancy < Slab->MaximumOccupancy) {
- ASSERT3P(Cache->Cursor, ==, ListEntry);
- break;
- }
- }
-
- // Slabs should be kept in order of maximum to minimum occupancy
- for (ListEntry = Cache->SlabList.Flink;
- ListEntry != &Cache->SlabList;
- ListEntry = ListEntry->Flink) {
- PXENBUS_CACHE_SLAB Slab;
-
- Slab = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_SLAB, ListEntry);
-
- ASSERT3U(Slab->CurrentOccupancy, <=, CurrentOccupancy);
-
- CurrentOccupancy = Slab->CurrentOccupancy;
- }
-}
-#else
-#define CacheAudit(_Cache) ((VOID)(_Cache)) -#endif
-
-// Must be called with lock held
static NTSTATUS
-CacheCreateSlab(
- IN PXENBUS_CACHE Cache
+CacheFill(
+ IN PXENBUS_CACHE Cache,
+ IN ULONG Count,
+ IN BOOLEAN Locked
)
{
- PXENBUS_CACHE_SLAB Slab;
- ULONG NumberOfBytes;
- ULONG Count;
+ KIRQL Irql;
+ PXENBUS_CACHE_ENTRY Entry;
ULONG Size;
- LONG Index;
- LONG SlabCount;
NTSTATUS status;
- NumberOfBytes = P2ROUNDUP(FIELD_OFFSET(XENBUS_CACHE_SLAB, Buffer) +
- Cache->Size,
- PAGE_SIZE);
- Count = (NumberOfBytes - FIELD_OFFSET(XENBUS_CACHE_SLAB, Buffer)) /
- Cache->Size;
- ASSERT(Count != 0);
-
- status = STATUS_INSUFFICIENT_RESOURCES;
- if (Cache->Count + Count > Cache->Cap)
- goto fail1;
-
- Slab = __CacheAllocate(NumberOfBytes);
- ASSERT3P(Slab, ==, PAGE_ALIGN(Slab));
-
- status = STATUS_NO_MEMORY;
- if (Slab == NULL)
- goto fail2;
-
- RtlZeroMemory(Slab, NumberOfBytes);
+ KeRaiseIrql(DISPATCH_LEVEL, &Irql);
+ if (!Locked)
+ __CacheAcquireLock(Cache);
- Slab->Magic = XENBUS_CACHE_SLAB_MAGIC;
- Slab->Cache = Cache;
- Slab->MaximumOccupancy = (USHORT)Count;
+ Size = P2ROUNDUP(FIELD_OFFSET(XENBUS_CACHE_ENTRY, Buffer) +
+ Cache->Size,
+ sizeof(ULONG_PTR));
- Size = P2ROUNDUP(Count, BITS_PER_ULONG);
- Size /= 8;
+ status = STATUS_SUCCESS;
+ while (Cache->FreeSize < (LONG)Count) {
+ LONG ObjectCount;
- Slab->Mask = __CacheAllocate(Size);
- if (Slab->Mask == NULL)
- goto fail3;
+ status = STATUS_NO_MEMORY;
+ Entry = __CacheAllocate(Size);
+ if (Entry == NULL)
+ goto fail1;
- for (Index = 0; Index < (LONG)Slab->MaximumOccupancy; Index++) {
- PVOID Object = (PVOID)&Slab->Buffer[Index * Cache->Size];
+ Entry->Magic = XENBUS_CACHE_SLAB_MAGIC;
+ Entry->Cache = Cache;
+ InitializeListHead(&Entry->ListEntry);
- status = __CacheCtor(Cache, Object);
+ status = __CacheCtor(Cache, Entry->Buffer);
if (!NT_SUCCESS(status))
- goto fail4;
- }
-
- CacheInsertSlab(Cache, Slab);
- Cache->Count += Count;
-
- SlabCount = InterlockedIncrement(&Cache->CurrentSlabs);
- if (SlabCount > Cache->MaximumSlabs)
- Cache->MaximumSlabs = SlabCount;
-
- return STATUS_SUCCESS;
-
-fail4:
- Error("fail4\n");
+ goto fail2;
- while (--Index >= 0) {
- PVOID Object = (PVOID)&Slab->Buffer[Index * Cache->Size];
+ InsertTailList(&Cache->FreeList, &Entry->ListEntry);
+ ObjectCount = InterlockedIncrement(&Cache->FreeSize);
- __CacheDtor(Cache, Object);
+ if (ObjectCount > Cache->FreeMaximum)
+ Cache->FreeMaximum = ObjectCount;
}
- __CacheFree(Slab->Mask);
+ if (!Locked)
+ __CacheReleaseLock(Cache);
-fail3:
- Error("fail3\n");
+ KeLowerIrql(Irql);
- __CacheFree(Slab);
+ return STATUS_SUCCESS;
fail2:
- Error("fail2\n");
+ __CacheFree(Entry);
fail1:
- Error("fail1 (%08x)\n", status);
+ if (!Locked)
+ __CacheReleaseLock(Cache);
+
+ KeLowerIrql(Irql);
return status;
}
-// Must be called with lock held
static VOID
-CacheDestroySlab(
- IN PXENBUS_CACHE Cache,
- IN PXENBUS_CACHE_SLAB Slab
+CacheSpill(
+ IN PXENBUS_CACHE Cache,
+ IN ULONG Count,
+ IN BOOLEAN Locked
)
{
- LONG Index;
-
- ASSERT3U(Slab->CurrentOccupancy, ==, 0);
-
- ASSERT3U(Cache->Count, >=, Slab->MaximumOccupancy);
- Cache->Count -= Slab->MaximumOccupancy;
-
- //
- // The only reason the cursor should be pointing at this slab is
- // if it is the only one in the list.
- //
- if (Cache->Cursor == &Slab->ListEntry)
- Cache->Cursor = &Cache->SlabList;
-
- RemoveEntryList(&Slab->ListEntry);
-
- ASSERT(Cache->Cursor != &Cache->SlabList ||
- IsListEmpty(&Cache->SlabList));
-
- Index = Slab->MaximumOccupancy;
- while (--Index >= 0) {
- PVOID Object = (PVOID)&Slab->Buffer[Index * Cache->Size];
-
- __CacheDtor(Cache, Object);
- }
+ KIRQL Irql;
+ PLIST_ENTRY ListEntry;
- ASSERT(Cache->CurrentSlabs != 0);
- InterlockedDecrement(&Cache->CurrentSlabs);
+ KeRaiseIrql(DISPATCH_LEVEL, &Irql);
+ if (!Locked)
+ __CacheAcquireLock(Cache);
- __CacheFree(Slab->Mask);
- __CacheFree(Slab);
-}
+ while (!IsListEmpty(&Cache->FreeList)) {
+ PXENBUS_CACHE_ENTRY Entry;
-static FORCEINLINE ULONG
-__CacheMaskScan(
- IN ULONG *Mask,
- IN ULONG Maximum
- )
-{
- ULONG Size;
- ULONG Index;
-
- Size = P2ROUNDUP(Maximum, BITS_PER_ULONG);
- Size /= sizeof (ULONG);
- ASSERT(Size != 0);
+ if (Cache->FreeSize <= (LONG)Count)
+ break;
- for (Index = 0; Index < Size; Index++) {
- ULONG Free = ~Mask[Index];
- ULONG Bit;
+ ListEntry = RemoveTailList(&Cache->FreeList);
+ ASSERT3P(ListEntry, !=, &Cache->FreeList);
+ InterlockedDecrement(&Cache->FreeSize);
- if (!_BitScanForward(&Bit, Free))
- continue;
+ Entry = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_ENTRY,
+ ListEntry);
- Bit += Index * BITS_PER_ULONG;
- if (Bit < Maximum)
- return Bit;
+ __CacheDtor(Cache, Entry->Buffer);
+ __CacheFree(Entry);
}
- return Maximum;
-}
-
-static FORCEINLINE VOID
-__CacheMaskSet(
- IN ULONG *Mask,
- IN ULONG Bit
- )
-{
- ULONG Index = Bit / BITS_PER_ULONG;
-
- Mask[Index] |= 1u << (Bit % BITS_PER_ULONG);
-}
-
-static FORCEINLINE BOOLEAN
-__CacheMaskTest(
- IN ULONG *Mask,
- IN ULONG Bit
- )
-{
- ULONG Index = Bit / BITS_PER_ULONG;
-
- return (Mask[Index] & (1u << (Bit % BITS_PER_ULONG))) ? TRUE : FALSE;
-}
-
-static FORCEINLINE VOID
-__CacheMaskClear(
- IN ULONG *Mask,
- IN ULONG Bit
- )
-{
- ULONG Index = Bit / BITS_PER_ULONG;
-
- Mask[Index] &= ~(1u << (Bit % BITS_PER_ULONG));
-}
-
-// Must be called with lock held
-static PVOID
-CacheGetObjectFromSlab(
- IN PXENBUS_CACHE_SLAB Slab
- )
-{
- PXENBUS_CACHE Cache;
- ULONG Index;
- PVOID Object;
-
- Cache = Slab->Cache;
-
- ASSERT3U(Slab->CurrentOccupancy, <=, Slab->MaximumOccupancy);
- if (Slab->CurrentOccupancy == Slab->MaximumOccupancy)
- return NULL;
-
- Index = __CacheMaskScan(Slab->Mask, Slab->MaximumOccupancy);
- BUG_ON(Index >= Slab->MaximumOccupancy);
-
- __CacheMaskSet(Slab->Mask, Index);
- Slab->CurrentOccupancy++;
-
- Object = (PVOID)&Slab->Buffer[Index * Cache->Size];
- ASSERT3U(Index, ==, (ULONG)((PUCHAR)Object - &Slab->Buffer[0]) /
- Cache->Size);
-
- return Object;
-}
-
-// Must be called with lock held
-static VOID
-CachePutObjectToSlab(
- IN PXENBUS_CACHE_SLAB Slab,
- IN PVOID Object
- )
-{
- PXENBUS_CACHE Cache;
- ULONG Index;
-
- Cache = Slab->Cache;
-
- Index = (ULONG)((PUCHAR)Object - &Slab->Buffer[0]) / Cache->Size;
- BUG_ON(Index >= Slab->MaximumOccupancy);
-
- ASSERT(Slab->CurrentOccupancy != 0);
- --Slab->CurrentOccupancy;
+ if (!Locked)
+ __CacheReleaseLock(Cache);
- ASSERT(__CacheMaskTest(Slab->Mask, Index));
- __CacheMaskClear(Slab->Mask, Index);
+ KeLowerIrql(Irql);
}
static PVOID
@@ -552,34 +318,22 @@ CacheGet(
if (!Locked)
__CacheAcquireLock(Cache);
-again:
- if (Cache->Cursor != &Cache->SlabList) {
- PLIST_ENTRY ListEntry = Cache->Cursor;
- PXENBUS_CACHE_SLAB Slab;
-
- Slab = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_SLAB, ListEntry);
-
- Object = CacheGetObjectFromSlab(Slab);
- ASSERT(Object != NULL);
-
- if (Slab->CurrentOccupancy == Slab->MaximumOccupancy)
- Cache->Cursor = Slab->ListEntry.Flink;
+ if (IsListEmpty(&Cache->FreeList)) {
+ (VOID) CacheFill(Cache, Cache->Stride, TRUE);
}
- if (Object == NULL) {
- NTSTATUS status;
+ if (!IsListEmpty(&Cache->FreeList)) {
+ PLIST_ENTRY ListEntry;
+ PXENBUS_CACHE_ENTRY Entry;
- ASSERT3P(Cache->Cursor, ==, &Cache->SlabList);
+ ListEntry = RemoveHeadList(&Cache->FreeList);
+ ASSERT3P(ListEntry, !=, &Cache->FreeList);
+ InterlockedDecrement(&Cache->FreeSize);
- status = CacheCreateSlab(Cache);
- if (NT_SUCCESS(status)) {
- ASSERT(Cache->Cursor != &Cache->SlabList);
- goto again;
- }
+ Entry = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_ENTRY, ListEntry);
+ Object = Entry->Buffer;
}
- CacheAudit(Cache);
-
if (!Locked)
__CacheReleaseLock(Cache);
@@ -606,7 +360,8 @@ CachePut(
KIRQL Irql;
ULONG Index;
PXENBUS_CACHE_MAGAZINE Magazine;
- PXENBUS_CACHE_SLAB Slab;
+ PXENBUS_CACHE_ENTRY Entry;
+ LONG ObjectCount;
NTSTATUS status;
UNREFERENCED_PARAMETER(Interface);
@@ -622,19 +377,22 @@ CachePut(
if (NT_SUCCESS(status))
goto done;
- Slab = (PXENBUS_CACHE_SLAB)PAGE_ALIGN(Object);
- ASSERT3U(Slab->Magic, ==, XENBUS_CACHE_SLAB_MAGIC);
+ Entry = CONTAINING_RECORD(Object, XENBUS_CACHE_ENTRY, Buffer);
+ ASSERT3U(Entry->Magic, ==, XENBUS_CACHE_SLAB_MAGIC);
if (!Locked)
__CacheAcquireLock(Cache);
- CachePutObjectToSlab(Slab, Object);
+ InsertHeadList(&Cache->FreeList, &Entry->ListEntry);
+ ObjectCount = InterlockedIncrement(&Cache->FreeSize);
+ if (ObjectCount > Cache->FreeMaximum)
+ Cache->FreeMaximum = ObjectCount;
- /* Re-insert to keep slab list ordered */
- RemoveEntryList(&Slab->ListEntry);
- CacheInsertSlab(Cache, Slab);
-
- CacheAudit(Cache);
+ if (Cache->FreeSize > (LONG)(Cache->Reservation + Cache->Stride))
+ CacheSpill(Cache,
+ __max((Cache->Reservation + Cache->FreeSize -
Cache->Stride),
+ 1),
+ TRUE);
if (!Locked)
__CacheReleaseLock(Cache);
@@ -646,74 +404,6 @@ done:
KeLowerIrql(Irql);
}
-static NTSTATUS
-CacheFill(
- IN PXENBUS_CACHE Cache,
- IN ULONG Count
- )
-{
- KIRQL Irql;
- NTSTATUS status;
-
- KeRaiseIrql(DISPATCH_LEVEL, &Irql);
- __CacheAcquireLock(Cache);
-
- status = STATUS_SUCCESS;
- while (Cache->Count < Count) {
- status = CacheCreateSlab(Cache);
- if (!NT_SUCCESS(status))
- break;
- }
-
- CacheAudit(Cache);
-
- __CacheReleaseLock(Cache);
- KeLowerIrql(Irql);
-
- return status;
-}
-
-static VOID
-CacheSpill(
- IN PXENBUS_CACHE Cache,
- IN ULONG Count
- )
-{
- KIRQL Irql;
- PLIST_ENTRY ListEntry;
-
- KeRaiseIrql(DISPATCH_LEVEL, &Irql);
- __CacheAcquireLock(Cache);
-
- if (Cache->Count <= Count)
- goto done;
-
- while (!IsListEmpty(&Cache->SlabList)) {
- PXENBUS_CACHE_SLAB Slab;
-
- // Actual list removal is done in CacheDestroySlab()
- ListEntry = Cache->SlabList.Blink;
- ASSERT(ListEntry != &Cache->SlabList);
-
- Slab = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_SLAB, ListEntry);
-
- if (Slab->CurrentOccupancy != 0)
- break;
-
- ASSERT(Cache->Count >= Slab->MaximumOccupancy);
- if (Cache->Count - Slab->MaximumOccupancy < Count)
- break;
-
- CacheDestroySlab(Cache, Slab);
- }
-
- CacheAudit(Cache);
-
-done:
- __CacheReleaseLock(Cache);
- KeLowerIrql(Irql);
-}
-
static FORCEINLINE VOID
__CacheFlushMagazines(
IN PXENBUS_CACHE Cache
@@ -730,12 +420,16 @@ __CacheFlushMagazines(
PVOID Object;
while ((Object = CacheGetObjectFromMagazine(Magazine)) != NULL) {
- PXENBUS_CACHE_SLAB Slab;
+ PXENBUS_CACHE_ENTRY Entry;
+ LONG ObjectCount;
- Slab = (PXENBUS_CACHE_SLAB)PAGE_ALIGN(Object);
- ASSERT3U(Slab->Magic, ==, XENBUS_CACHE_SLAB_MAGIC);
+ Entry = CONTAINING_RECORD(Object, XENBUS_CACHE_ENTRY, Buffer);
+ ASSERT3U(Entry->Magic, ==, XENBUS_CACHE_SLAB_MAGIC);
- CachePutObjectToSlab(Slab, Object);
+ InsertHeadList(&Cache->FreeList, &Entry->ListEntry);
+ ObjectCount = InterlockedIncrement(&Cache->FreeSize);
+ if (ObjectCount > Cache->FreeMaximum)
+ Cache->FreeMaximum = ObjectCount;
}
}
@@ -750,6 +444,7 @@ CacheCreate(
IN ULONG Size,
IN ULONG Reservation,
IN ULONG Cap,
+ IN ULONG Stride,
IN NTSTATUS (*Ctor)(PVOID, PVOID),
IN VOID (*Dtor)(PVOID, PVOID),
IN VOID (*AcquireLock)(PVOID),
@@ -781,24 +476,28 @@ CacheCreate(
if (Cap == 0)
Cap = ULONG_MAX;
+ if (Stride == 0)
+ Stride = 1;
+ if (Stride > Cap)
+ Stride = Cap;
(*Cache)->Size = Size;
(*Cache)->Reservation = Reservation;
(*Cache)->Cap = Cap;
+ (*Cache)->Stride = Stride;
(*Cache)->Ctor = Ctor;
(*Cache)->Dtor = Dtor;
(*Cache)->AcquireLock = AcquireLock;
(*Cache)->ReleaseLock = ReleaseLock;
(*Cache)->Argument = Argument;
- InitializeListHead(&(*Cache)->SlabList);
- (*Cache)->Cursor = &(*Cache)->SlabList;
+ InitializeListHead(&(*Cache)->FreeList);
status = STATUS_INVALID_PARAMETER;
if ((*Cache)->Reservation > (*Cache)->Cap)
goto fail3;
- status = CacheFill(*Cache, (*Cache)->Reservation);
+ status = CacheFill(*Cache, (*Cache)->Reservation, FALSE);
if (!NT_SUCCESS(status))
goto fail4;
@@ -822,7 +521,7 @@ fail5:
(*Cache)->MagazineCount = 0;
- CacheSpill(*Cache, 0);
+ CacheSpill(*Cache, 0, FALSE);
fail4:
Error("fail4\n");
@@ -830,15 +529,15 @@ fail4:
fail3:
Error("fail3\n");
- (*Cache)->Cursor = NULL;
- ASSERT(IsListEmpty(&(*Cache)->SlabList));
- RtlZeroMemory(&(*Cache)->SlabList, sizeof (LIST_ENTRY));
+ ASSERT(IsListEmpty(&(*Cache)->FreeList));
+ RtlZeroMemory(&(*Cache)->FreeList, sizeof (LIST_ENTRY));
(*Cache)->Argument = NULL;
(*Cache)->ReleaseLock = NULL;
(*Cache)->AcquireLock = NULL;
(*Cache)->Dtor = NULL;
(*Cache)->Ctor = NULL;
+ (*Cache)->Stride = 0;
(*Cache)->Cap = 0;
(*Cache)->Reservation = 0;
(*Cache)->Size = 0;
@@ -876,6 +575,36 @@ CacheCreateVersion1(
Size,
Reservation,
0,
+ 0,
+ Ctor,
+ Dtor,
+ AcquireLock,
+ ReleaseLock,
+ Argument,
+ Cache);
+}
+
+static NTSTATUS
+CacheCreateVersion2(
+ IN PINTERFACE Interface,
+ IN const CHAR *Name,
+ IN ULONG Size,
+ IN ULONG Reservation,
+ IN ULONG Cap,
+ IN NTSTATUS (*Ctor)(PVOID, PVOID),
+ IN VOID (*Dtor)(PVOID, PVOID),
+ IN VOID (*AcquireLock)(PVOID),
+ IN VOID (*ReleaseLock)(PVOID),
+ IN PVOID Argument,
+ OUT PXENBUS_CACHE *Cache
+ )
+{
+ return CacheCreate(Interface,
+ Name,
+ Size,
+ Reservation,
+ Cap,
+ 0,
Ctor,
Dtor,
AcquireLock,
@@ -908,17 +637,16 @@ CacheDestroy(
Cache->Magazine = NULL;
Cache->MagazineCount = 0;
- CacheSpill(Cache, 0);
+ CacheSpill(Cache, 0, FALSE);
ASSERT(Cache->CurrentObjects == 0);
Cache->MaximumObjects = 0;
- ASSERT(Cache->CurrentSlabs == 0);
- Cache->MaximumSlabs = 0;
+ ASSERT(Cache->FreeSize == 0);
+ Cache->FreeSize = 0;
- Cache->Cursor = NULL;
- ASSERT(IsListEmpty(&Cache->SlabList));
- RtlZeroMemory(&Cache->SlabList, sizeof (LIST_ENTRY));
+ ASSERT(IsListEmpty(&Cache->FreeList));
+ RtlZeroMemory(&Cache->FreeList, sizeof (LIST_ENTRY));
Cache->Argument = NULL;
Cache->ReleaseLock = NULL;
@@ -963,14 +691,15 @@ CacheDebugCallback(
XENBUS_DEBUG(Printf,
&Context->DebugInterface,
- "- %s: Count = %d, Reservation = %d, Objects = %d /
%d, Slabs = %d / %d\n",
+ "- %s: Reservation = %d, Cap = %d, Stride =
+ %d, Objects = %d / %d, Free = %d / %d\n",
Cache->Name,
- Cache->Count,
Cache->Reservation,
+ Cache->Cap,
+ Cache->Stride,
Cache->CurrentObjects,
Cache->MaximumObjects,
- Cache->CurrentSlabs,
- Cache->MaximumSlabs);
+ Cache->FreeSize,
+ Cache->FreeMaximum);
}
}
}
@@ -1024,11 +753,15 @@ CacheMonitor(
Cache = CONTAINING_RECORD(ListEntry, XENBUS_CACHE, ListEntry);
- if (Cache->Count < Cache->Reservation)
- CacheFill(Cache, Cache->Reservation);
- else if (Cache->Count > Cache->Reservation)
+ if (Cache->CurrentObjects + Cache->FreeSize <
(LONG)Cache->Reservation)
+ CacheFill(Cache,
+ __max((Cache->Reservation - Cache->CurrentObjects),
Cache->Stride),
+ FALSE);
+ else if (Cache->CurrentObjects + Cache->FreeSize >
+ (LONG)Cache->Reservation)
CacheSpill(Cache,
- __max(Cache->Reservation, (Cache->Count / 2)));
+ __max((LONG)Cache->Reservation,
+ (Cache->CurrentObjects + Cache->FreeSize /
2)),
+ FALSE);
}
loop:
@@ -1136,12 +869,22 @@ static struct _XENBUS_CACHE_INTERFACE_V2
CacheInterfaceVersion2 = {
{ sizeof (struct _XENBUS_CACHE_INTERFACE_V2), 2, NULL, NULL, NULL },
CacheAcquire,
CacheRelease,
+ CacheCreateVersion2,
+ CacheGet,
+ CachePut,
+ CacheDestroy
+};
+
+static struct _XENBUS_CACHE_INTERFACE_V3 CacheInterfaceVersion3 = {
+ { sizeof (struct _XENBUS_CACHE_INTERFACE_V3), 3, NULL, NULL, NULL },
+ CacheAcquire,
+ CacheRelease,
CacheCreate,
CacheGet,
CachePut,
CacheDestroy
};
-
+
NTSTATUS
CacheInitialize(
IN PXENBUS_FDO Fdo,
@@ -1240,6 +983,23 @@ CacheGetInterface(
status = STATUS_SUCCESS;
break;
}
+ case 3: {
+ struct _XENBUS_CACHE_INTERFACE_V3 *CacheInterface;
+
+ CacheInterface = (struct _XENBUS_CACHE_INTERFACE_V3
+ *)Interface;
+
+ status = STATUS_BUFFER_OVERFLOW;
+ if (Size < sizeof (struct _XENBUS_CACHE_INTERFACE_V3))
+ break;
+
+ *CacheInterface = CacheInterfaceVersion3;
+
+ ASSERT3U(Interface->Version, ==, Version);
+ Interface->Context = Context;
+
+ status = STATUS_SUCCESS;
+ break;
+ }
default:
status = STATUS_NOT_SUPPORTED;
break;
diff --git a/src/xenbus/gnttab.c b/src/xenbus/gnttab.c index 33373c2..10920b8
100644
--- a/src/xenbus/gnttab.c
+++ b/src/xenbus/gnttab.c
@@ -363,6 +363,7 @@ GnttabCreateCache(
sizeof (XENBUS_GNTTAB_ENTRY),
Reservation,
Cap,
+ 1,
GnttabEntryCtor,
GnttabEntryDtor,
GnttabAcquireLock,
--
2.32.0.windows.1
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |