|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [PATCH] Rework CACHE to use a FreeList
The slab allocation method will allogate about a PAGE worth of objects, and
every object will be initialized. If the objects initializer allocates any
resources, this can result in resource starvation. A particular bad example of
this is the grant table cache, where a page of gnttab objects is 253 objects.
This is highlighted by xenvif's queues, where the receiver requires 257 grant
references (1 for the ring, and 256 for the ring slots) which results in 2
slabs,
or 506 gnttab objects, reserving 506 grant references.
Use a FreeList to contain individual objects that are not in use. This trades
an increase in smaller allocations for reducing the wastage of unused objects.
Signed-off-by: Owen Smith <owen.smith@xxxxxxxxxx>
---
src/xenbus/cache.c | 674 ++++++++++++---------------------------------
1 file changed, 178 insertions(+), 496 deletions(-)
diff --git a/src/xenbus/cache.c b/src/xenbus/cache.c
index 13fb0e5..d5500a4 100644
--- a/src/xenbus/cache.c
+++ b/src/xenbus/cache.c
@@ -52,20 +52,12 @@ typedef struct _XENBUS_CACHE_MAGAZINE {
PVOID Slot[XENBUS_CACHE_MAGAZINE_SLOTS];
} XENBUS_CACHE_MAGAZINE, *PXENBUS_CACHE_MAGAZINE;
-
-#define XENBUS_CACHE_SLAB_MAGIC 'BALS'
-
-typedef struct _XENBUS_CACHE_SLAB {
- ULONG Magic;
+typedef struct _XENBUS_CACHE_HEADER {
PXENBUS_CACHE Cache;
LIST_ENTRY ListEntry;
- USHORT MaximumOccupancy;
- USHORT CurrentOccupancy;
- ULONG *Mask;
UCHAR Buffer[1];
-} XENBUS_CACHE_SLAB, *PXENBUS_CACHE_SLAB;
+} XENBUS_CACHE_HEADER, *PXENBUS_CACHE_HEADER;
-#define BITS_PER_ULONG (sizeof (ULONG) * 8)
#define MAXNAMELEN 128
struct _XENBUS_CACHE {
@@ -79,13 +71,12 @@ struct _XENBUS_CACHE {
VOID (*AcquireLock)(PVOID);
VOID (*ReleaseLock)(PVOID);
PVOID Argument;
- LIST_ENTRY SlabList;
- PLIST_ENTRY Cursor;
- ULONG Count;
- PXENBUS_CACHE_MAGAZINE Magazine;
+ PXENBUS_CACHE_MAGAZINE Magazines;
ULONG MagazineCount;
- LONG CurrentSlabs;
- LONG MaximumSlabs;
+ KSPIN_LOCK FreeLock;
+ LIST_ENTRY FreeList;
+ LONG FreeSize;
+ LONG Count;
LONG CurrentObjects;
LONG MaximumObjects;
};
@@ -156,372 +147,197 @@ __CacheDtor(
Cache->Dtor(Cache->Argument, Object);
}
-static PVOID
-CacheGetObjectFromMagazine(
- IN PXENBUS_CACHE_MAGAZINE Magazine
- )
-{
- ULONG Index;
-
- for (Index = 0; Index < XENBUS_CACHE_MAGAZINE_SLOTS; Index++) {
- PVOID Object;
-
- if (Magazine->Slot[Index] != NULL) {
- Object = Magazine->Slot[Index];
- Magazine->Slot[Index] = NULL;
-
- return Object;
- }
- }
-
- return NULL;
-}
-
+// Lock must be held
static NTSTATUS
-CachePutObjectToMagazine(
- IN PXENBUS_CACHE_MAGAZINE Magazine,
- IN PVOID Object
- )
-{
- ULONG Index;
-
- for (Index = 0; Index < XENBUS_CACHE_MAGAZINE_SLOTS; Index++) {
- if (Magazine->Slot[Index] == NULL) {
- Magazine->Slot[Index] = Object;
- return STATUS_SUCCESS;
- }
- }
-
- return STATUS_UNSUCCESSFUL;
-}
-
-static VOID
-CacheInsertSlab(
+CacheFill(
IN PXENBUS_CACHE Cache,
- IN PXENBUS_CACHE_SLAB New
- )
-{
-#define INSERT_BEFORE(_ListEntry, _New) \
- do { \
- (_New)->Blink = (_ListEntry)->Blink; \
- (_ListEntry)->Blink->Flink = (_New); \
- \
- (_ListEntry)->Blink = (_New); \
- (_New)->Flink = (_ListEntry); \
- } while (FALSE)
-
- PLIST_ENTRY ListEntry;
-
- ASSERT(New->CurrentOccupancy < New->MaximumOccupancy);
-
- Cache->Cursor = NULL;
-
- for (ListEntry = Cache->SlabList.Flink;
- ListEntry != &Cache->SlabList;
- ListEntry = ListEntry->Flink) {
- PXENBUS_CACHE_SLAB Slab;
-
- Slab = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_SLAB, ListEntry);
-
- if (Slab->CurrentOccupancy < New->CurrentOccupancy) {
- INSERT_BEFORE(ListEntry, &New->ListEntry);
- goto done;
- }
-
- if (Slab->CurrentOccupancy < Slab->MaximumOccupancy &&
- Cache->Cursor == NULL)
- Cache->Cursor = ListEntry;
- }
-
- InsertTailList(&Cache->SlabList, &New->ListEntry);
-
-done:
- if (Cache->Cursor == NULL)
- Cache->Cursor = &New->ListEntry;
-
-#undef INSERT_BEFORE
-}
-
-#if DBG
-static VOID
-CacheAudit(
- IN PXENBUS_CACHE Cache
+ IN ULONG Target
)
{
- ULONG CurrentOccupancy = ULONG_MAX;
- PLIST_ENTRY ListEntry;
-
- //
- // The cursror should point at the first slab that is not fully
- // occupied.
- //
- for (ListEntry = Cache->SlabList.Flink;
- ListEntry != &Cache->SlabList;
- ListEntry = ListEntry->Flink) {
- PXENBUS_CACHE_SLAB Slab;
-
- Slab = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_SLAB, ListEntry);
-
- if (Slab->CurrentOccupancy < Slab->MaximumOccupancy) {
- ASSERT3P(Cache->Cursor, ==, ListEntry);
- break;
- }
- }
-
- // Slabs should be kept in order of maximum to minimum occupancy
- for (ListEntry = Cache->SlabList.Flink;
- ListEntry != &Cache->SlabList;
- ListEntry = ListEntry->Flink) {
- PXENBUS_CACHE_SLAB Slab;
-
- Slab = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_SLAB, ListEntry);
-
- ASSERT3U(Slab->CurrentOccupancy, <=, CurrentOccupancy);
-
- CurrentOccupancy = Slab->CurrentOccupancy;
- }
-}
-#else
-#define CacheAudit(_Cache) ((VOID)(_Cache))
-#endif
-
-// Must be called with lock held
-static NTSTATUS
-CacheCreateSlab(
- IN PXENBUS_CACHE Cache
- )
-{
- PXENBUS_CACHE_SLAB Slab;
- ULONG NumberOfBytes;
- ULONG Count;
- ULONG Size;
- LONG Index;
- LONG SlabCount;
- NTSTATUS status;
-
- NumberOfBytes = P2ROUNDUP(FIELD_OFFSET(XENBUS_CACHE_SLAB, Buffer) +
- Cache->Size,
- PAGE_SIZE);
- Count = (NumberOfBytes - FIELD_OFFSET(XENBUS_CACHE_SLAB, Buffer)) /
- Cache->Size;
- ASSERT(Count != 0);
-
- status = STATUS_INSUFFICIENT_RESOURCES;
- if (Cache->Count + Count > Cache->Cap)
- goto fail1;
-
- Slab = __CacheAllocate(NumberOfBytes);
- ASSERT3P(Slab, ==, PAGE_ALIGN(Slab));
-
- status = STATUS_NO_MEMORY;
- if (Slab == NULL)
- goto fail2;
-
- RtlZeroMemory(Slab, NumberOfBytes);
+ ULONG Size;
+ PXENBUS_CACHE_HEADER Header;
+ NTSTATUS status;
- Slab->Magic = XENBUS_CACHE_SLAB_MAGIC;
- Slab->Cache = Cache;
- Slab->MaximumOccupancy = (USHORT)Count;
+ Size = P2ROUNDUP(FIELD_OFFSET(XENBUS_CACHE_HEADER, Buffer) + Cache->Size,
+ sizeof(ULONG_PTR));
- Size = P2ROUNDUP(Count, BITS_PER_ULONG);
- Size /= 8;
+ while (Cache->Count < (LONG)Target) {
+ status = STATUS_NO_MEMORY;
- Slab->Mask = __CacheAllocate(Size);
- if (Slab->Mask == NULL)
- goto fail3;
+ Header = __CacheAllocate(Size);
+ if (Header == NULL)
+ goto fail1;
- for (Index = 0; Index < (LONG)Slab->MaximumOccupancy; Index++) {
- PVOID Object = (PVOID)&Slab->Buffer[Index * Cache->Size];
+ Header->Cache = Cache;
+ InitializeListHead(&Header->ListEntry);
- status = __CacheCtor(Cache, Object);
+ status = __CacheCtor(Cache, Header->Buffer);
if (!NT_SUCCESS(status))
- goto fail4;
- }
-
- CacheInsertSlab(Cache, Slab);
- Cache->Count += Count;
-
- SlabCount = InterlockedIncrement(&Cache->CurrentSlabs);
- if (SlabCount > Cache->MaximumSlabs)
- Cache->MaximumSlabs = SlabCount;
-
- return STATUS_SUCCESS;
-
-fail4:
- Error("fail4\n");
+ goto fail2;
- while (--Index >= 0) {
- PVOID Object = (PVOID)&Slab->Buffer[Index * Cache->Size];
+ ExInterlockedInsertTailList(&Cache->FreeList,
+ &Header->ListEntry,
+ &Cache->FreeLock);
+ InterlockedIncrement(&Cache->FreeSize);
- __CacheDtor(Cache, Object);
+ InterlockedIncrement(&Cache->Count);
}
- __CacheFree(Slab->Mask);
-
-fail3:
- Error("fail3\n");
-
- __CacheFree(Slab);
+ return STATUS_SUCCESS;
fail2:
- Error("fail2\n");
+ __CacheFree(Header);
fail1:
- Error("fail1 (%08x)\n", status);
-
return status;
}
-// Must be called with lock held
+// Lock must be held
static VOID
-CacheDestroySlab(
+CacheSpill(
IN PXENBUS_CACHE Cache,
- IN PXENBUS_CACHE_SLAB Slab
+ IN ULONG Target
)
{
- LONG Index;
+ PLIST_ENTRY ListEntry;
+ PXENBUS_CACHE_HEADER Header;
- ASSERT3U(Slab->CurrentOccupancy, ==, 0);
+ while (Cache->Count > (LONG)Target) {
- ASSERT3U(Cache->Count, >=, Slab->MaximumOccupancy);
- Cache->Count -= Slab->MaximumOccupancy;
+ ListEntry = ExInterlockedRemoveHeadList(&Cache->FreeList,
+ &Cache->FreeLock);
+ if (ListEntry == NULL)
+ break;
- //
- // The only reason the cursor should be pointing at this slab is
- // if it is the only one in the list.
- //
- if (Cache->Cursor == &Slab->ListEntry)
- Cache->Cursor = &Cache->SlabList;
+ ASSERT3S(Cache->FreeSize, >, 0);
+ InterlockedDecrement(&Cache->FreeSize);
- RemoveEntryList(&Slab->ListEntry);
+ Header = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_HEADER, ListEntry);
- ASSERT(Cache->Cursor != &Cache->SlabList ||
- IsListEmpty(&Cache->SlabList));
+ __CacheDtor(Cache, Header->Buffer);
- Index = Slab->MaximumOccupancy;
- while (--Index >= 0) {
- PVOID Object = (PVOID)&Slab->Buffer[Index * Cache->Size];
+ __CacheFree(Header);
- __CacheDtor(Cache, Object);
+ InterlockedDecrement(&Cache->Count);
}
-
- ASSERT(Cache->CurrentSlabs != 0);
- InterlockedDecrement(&Cache->CurrentSlabs);
-
- __CacheFree(Slab->Mask);
- __CacheFree(Slab);
}
-static FORCEINLINE ULONG
-__CacheMaskScan(
- IN ULONG *Mask,
- IN ULONG Maximum
+static PVOID
+CacheGetFromMagazine(
+ IN PXENBUS_CACHE_MAGAZINE Magazine
)
{
- ULONG Size;
- ULONG Index;
-
- Size = P2ROUNDUP(Maximum, BITS_PER_ULONG);
- Size /= sizeof (ULONG);
- ASSERT(Size != 0);
+ ULONG Index;
- for (Index = 0; Index < Size; Index++) {
- ULONG Free = ~Mask[Index];
- ULONG Bit;
+ for (Index = 0; Index < XENBUS_CACHE_MAGAZINE_SLOTS; ++Index) {
+ PVOID Object;
- if (!_BitScanForward(&Bit, Free))
- continue;
+ if (Magazine->Slot[Index] != NULL) {
+ Object = Magazine->Slot[Index];
+ Magazine->Slot[Index] = NULL;
- Bit += Index * BITS_PER_ULONG;
- if (Bit < Maximum)
- return Bit;
+ return Object;
+ }
}
- return Maximum;
-}
-
-static FORCEINLINE VOID
-__CacheMaskSet(
- IN ULONG *Mask,
- IN ULONG Bit
- )
-{
- ULONG Index = Bit / BITS_PER_ULONG;
-
- Mask[Index] |= 1u << (Bit % BITS_PER_ULONG);
+ return NULL;
}
-static FORCEINLINE BOOLEAN
-__CacheMaskTest(
- IN ULONG *Mask,
- IN ULONG Bit
+static NTSTATUS
+CachePutToMagazine(
+ IN PXENBUS_CACHE_MAGAZINE Magazine,
+ IN PVOID Object
)
{
- ULONG Index = Bit / BITS_PER_ULONG;
-
- return (Mask[Index] & (1u << (Bit % BITS_PER_ULONG))) ? TRUE : FALSE;
-}
+ ULONG Index;
-static FORCEINLINE VOID
-__CacheMaskClear(
- IN ULONG *Mask,
- IN ULONG Bit
- )
-{
- ULONG Index = Bit / BITS_PER_ULONG;
+ for (Index = 0; Index < XENBUS_CACHE_MAGAZINE_SLOTS; ++Index) {
+ if (Magazine->Slot[Index] == NULL) {
+ Magazine->Slot[Index] = Object;
+ return STATUS_SUCCESS;
+ }
+ }
- Mask[Index] &= ~(1u << (Bit % BITS_PER_ULONG));
+ return STATUS_UNSUCCESSFUL;
}
-// Must be called with lock held
+// Lock must be held
static PVOID
-CacheGetObjectFromSlab(
- IN PXENBUS_CACHE_SLAB Slab
+CacheGetFromFreeList(
+ IN PXENBUS_CACHE Cache
)
{
- PXENBUS_CACHE Cache;
- ULONG Index;
- PVOID Object;
+ PLIST_ENTRY ListEntry;
+ PXENBUS_CACHE_HEADER Header;
+ NTSTATUS status;
- Cache = Slab->Cache;
+ ListEntry = ExInterlockedRemoveHeadList(&Cache->FreeList,
+ &Cache->FreeLock);
+ if (ListEntry == NULL) {
+ status = CacheFill(Cache,
+ Cache->Count + 1);
+ if (!NT_SUCCESS(status))
+ goto fail1;
- ASSERT3U(Slab->CurrentOccupancy, <=, Slab->MaximumOccupancy);
- if (Slab->CurrentOccupancy == Slab->MaximumOccupancy)
- return NULL;
+ ListEntry = ExInterlockedRemoveHeadList(&Cache->FreeList,
+ &Cache->FreeLock);
+ }
+ if (ListEntry == NULL)
+ goto fail2;
- Index = __CacheMaskScan(Slab->Mask, Slab->MaximumOccupancy);
- BUG_ON(Index >= Slab->MaximumOccupancy);
+ ASSERT3S(Cache->FreeSize, >, 0);
+ InterlockedDecrement(&Cache->FreeSize);
- __CacheMaskSet(Slab->Mask, Index);
- Slab->CurrentOccupancy++;
+ Header = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_HEADER, ListEntry);
+ ASSERT3P(Header->Cache, ==, Cache);
- Object = (PVOID)&Slab->Buffer[Index * Cache->Size];
- ASSERT3U(Index, ==, (ULONG)((PUCHAR)Object - &Slab->Buffer[0]) /
- Cache->Size);
+ return Header->Buffer;
- return Object;
+fail2:
+fail1:
+ return NULL;
}
-// Must be called with lock held
+// Lock must be held
static VOID
-CachePutObjectToSlab(
- IN PXENBUS_CACHE_SLAB Slab,
+CachePutToFreeList(
+ IN PXENBUS_CACHE Cache,
IN PVOID Object
)
{
- PXENBUS_CACHE Cache;
+ PXENBUS_CACHE_HEADER Header;
+
+ Header = CONTAINING_RECORD(Object, XENBUS_CACHE_HEADER, Buffer);
+ ASSERT3P(Header->Cache, ==, Cache);
+
+ ExInterlockedInsertTailList(&Cache->FreeList,
+ &Header->ListEntry,
+ &Cache->FreeLock);
+ InterlockedIncrement(&Cache->FreeSize);
+}
+
+static VOID
+__CacheFlushMagazines(
+ IN PXENBUS_CACHE Cache
+ )
+{
+ KIRQL Irql;
ULONG Index;
- Cache = Slab->Cache;
+ KeRaiseIrql(DISPATCH_LEVEL, &Irql);
+ __CacheAcquireLock(Cache);
- Index = (ULONG)((PUCHAR)Object - &Slab->Buffer[0]) / Cache->Size;
- BUG_ON(Index >= Slab->MaximumOccupancy);
+ for (Index = 0; Index < Cache->MagazineCount; ++Index) {
+ PXENBUS_CACHE_MAGAZINE Magazine = &Cache->Magazines[Index];
+ PVOID Object;
- ASSERT(Slab->CurrentOccupancy != 0);
- --Slab->CurrentOccupancy;
+ while ((Object = CacheGetFromMagazine(Magazine)) != NULL) {
+ CachePutToFreeList(Cache, Object);
+ }
+ }
- ASSERT(__CacheMaskTest(Slab->Mask, Index));
- __CacheMaskClear(Slab->Mask, Index);
+ __CacheReleaseLock(Cache);
+ KeLowerIrql(Irql);
}
static PVOID
@@ -532,62 +348,35 @@ CacheGet(
)
{
KIRQL Irql;
- ULONG Index;
PXENBUS_CACHE_MAGAZINE Magazine;
PVOID Object;
- LONG ObjectCount;
+ ULONG Index;
UNREFERENCED_PARAMETER(Interface);
+ Object = NULL;
+
KeRaiseIrql(DISPATCH_LEVEL, &Irql);
Index = KeGetCurrentProcessorNumberEx(NULL);
ASSERT3U(Index, <, Cache->MagazineCount);
- Magazine = &Cache->Magazine[Index];
-
- Object = CacheGetObjectFromMagazine(Magazine);
- if (Object != NULL)
- goto done;
-
- if (!Locked)
- __CacheAcquireLock(Cache);
-
-again:
- if (Cache->Cursor != &Cache->SlabList) {
- PLIST_ENTRY ListEntry = Cache->Cursor;
- PXENBUS_CACHE_SLAB Slab;
-
- Slab = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_SLAB, ListEntry);
-
- Object = CacheGetObjectFromSlab(Slab);
- ASSERT(Object != NULL);
-
- if (Slab->CurrentOccupancy == Slab->MaximumOccupancy)
- Cache->Cursor = Slab->ListEntry.Flink;
- }
+ Magazine = &Cache->Magazines[Index];
+ Object = CacheGetFromMagazine(Magazine);
if (Object == NULL) {
- NTSTATUS status;
+ if (!Locked)
+ __CacheAcquireLock(Cache);
- ASSERT3P(Cache->Cursor, ==, &Cache->SlabList);
+ Object = CacheGetFromFreeList(Cache);
- status = CacheCreateSlab(Cache);
- if (NT_SUCCESS(status)) {
- ASSERT(Cache->Cursor != &Cache->SlabList);
- goto again;
- }
+ if (!Locked)
+ __CacheReleaseLock(Cache);
}
- CacheAudit(Cache);
-
- if (!Locked)
- __CacheReleaseLock(Cache);
-
-done:
if (Object != NULL) {
- ObjectCount = InterlockedIncrement(&Cache->CurrentObjects);
- if (ObjectCount > Cache->MaximumObjects)
- Cache->MaximumObjects = ObjectCount;
+ LONG Count = InterlockedIncrement(&Cache->CurrentObjects);
+ if (Count > Cache->MaximumObjects)
+ Cache->MaximumObjects = Count;
}
KeLowerIrql(Irql);
@@ -606,7 +395,6 @@ CachePut(
KIRQL Irql;
ULONG Index;
PXENBUS_CACHE_MAGAZINE Magazine;
- PXENBUS_CACHE_SLAB Slab;
NTSTATUS status;
UNREFERENCED_PARAMETER(Interface);
@@ -615,131 +403,23 @@ CachePut(
Index = KeGetCurrentProcessorNumberEx(NULL);
ASSERT3U(Index, <, Cache->MagazineCount);
- Magazine = &Cache->Magazine[Index];
-
- status = CachePutObjectToMagazine(Magazine, Object);
-
- if (NT_SUCCESS(status))
- goto done;
-
- Slab = (PXENBUS_CACHE_SLAB)PAGE_ALIGN(Object);
- ASSERT3U(Slab->Magic, ==, XENBUS_CACHE_SLAB_MAGIC);
+ Magazine = &Cache->Magazines[Index];
- if (!Locked)
- __CacheAcquireLock(Cache);
+ status = CachePutToMagazine(Magazine, Object);
- CachePutObjectToSlab(Slab, Object);
+ if (!NT_SUCCESS(status)) {
+ if (!Locked)
+ __CacheAcquireLock(Cache);
- /* Re-insert to keep slab list ordered */
- RemoveEntryList(&Slab->ListEntry);
- CacheInsertSlab(Cache, Slab);
-
- CacheAudit(Cache);
-
- if (!Locked)
- __CacheReleaseLock(Cache);
-
-done:
- ASSERT(Cache->CurrentObjects != 0);
- InterlockedDecrement(&Cache->CurrentObjects);
-
- KeLowerIrql(Irql);
-}
+ CachePutToFreeList(Cache, Object);
-static NTSTATUS
-CacheFill(
- IN PXENBUS_CACHE Cache,
- IN ULONG Count
- )
-{
- KIRQL Irql;
- NTSTATUS status;
-
- KeRaiseIrql(DISPATCH_LEVEL, &Irql);
- __CacheAcquireLock(Cache);
-
- status = STATUS_SUCCESS;
- while (Cache->Count < Count) {
- status = CacheCreateSlab(Cache);
- if (!NT_SUCCESS(status))
- break;
+ if (!Locked)
+ __CacheReleaseLock(Cache);
}
- CacheAudit(Cache);
-
- __CacheReleaseLock(Cache);
- KeLowerIrql(Irql);
-
- return status;
-}
-
-static VOID
-CacheSpill(
- IN PXENBUS_CACHE Cache,
- IN ULONG Count
- )
-{
- KIRQL Irql;
- PLIST_ENTRY ListEntry;
-
- KeRaiseIrql(DISPATCH_LEVEL, &Irql);
- __CacheAcquireLock(Cache);
-
- if (Cache->Count <= Count)
- goto done;
-
- while (!IsListEmpty(&Cache->SlabList)) {
- PXENBUS_CACHE_SLAB Slab;
-
- // Actual list removal is done in CacheDestroySlab()
- ListEntry = Cache->SlabList.Blink;
- ASSERT(ListEntry != &Cache->SlabList);
-
- Slab = CONTAINING_RECORD(ListEntry, XENBUS_CACHE_SLAB, ListEntry);
-
- if (Slab->CurrentOccupancy != 0)
- break;
-
- ASSERT(Cache->Count >= Slab->MaximumOccupancy);
- if (Cache->Count - Slab->MaximumOccupancy < Count)
- break;
-
- CacheDestroySlab(Cache, Slab);
- }
-
- CacheAudit(Cache);
-
-done:
- __CacheReleaseLock(Cache);
- KeLowerIrql(Irql);
-}
-
-static FORCEINLINE VOID
-__CacheFlushMagazines(
- IN PXENBUS_CACHE Cache
- )
-{
- KIRQL Irql;
- ULONG Index;
-
- KeRaiseIrql(DISPATCH_LEVEL, &Irql);
- __CacheAcquireLock(Cache);
-
- for (Index = 0; Index < Cache->MagazineCount; Index++) {
- PXENBUS_CACHE_MAGAZINE Magazine = &Cache->Magazine[Index];
- PVOID Object;
-
- while ((Object = CacheGetObjectFromMagazine(Magazine)) != NULL) {
- PXENBUS_CACHE_SLAB Slab;
-
- Slab = (PXENBUS_CACHE_SLAB)PAGE_ALIGN(Object);
- ASSERT3U(Slab->Magic, ==, XENBUS_CACHE_SLAB_MAGIC);
-
- CachePutObjectToSlab(Slab, Object);
- }
- }
+ ASSERT3U(Cache->CurrentObjects, >, 0);
+ InterlockedDecrement(&Cache->CurrentObjects);
- __CacheReleaseLock(Cache);
KeLowerIrql(Irql);
}
@@ -791,8 +471,8 @@ CacheCreate(
(*Cache)->ReleaseLock = ReleaseLock;
(*Cache)->Argument = Argument;
- InitializeListHead(&(*Cache)->SlabList);
- (*Cache)->Cursor = &(*Cache)->SlabList;
+ InitializeListHead(&(*Cache)->FreeList);
+ KeInitializeSpinLock(&(*Cache)->FreeLock);
status = STATUS_INVALID_PARAMETER;
if ((*Cache)->Reservation > (*Cache)->Cap)
@@ -803,10 +483,10 @@ CacheCreate(
goto fail4;
(*Cache)->MagazineCount =
KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
- (*Cache)->Magazine = __CacheAllocate(sizeof (XENBUS_CACHE_MAGAZINE) *
(*Cache)->MagazineCount);
+ (*Cache)->Magazines = __CacheAllocate(sizeof (XENBUS_CACHE_MAGAZINE) *
(*Cache)->MagazineCount);
status = STATUS_NO_MEMORY;
- if ((*Cache)->Magazine == NULL)
+ if ((*Cache)->Magazines == NULL)
goto fail5;
KeAcquireSpinLock(&Context->Lock, &Irql);
@@ -830,9 +510,9 @@ fail4:
fail3:
Error("fail3\n");
- (*Cache)->Cursor = NULL;
- ASSERT(IsListEmpty(&(*Cache)->SlabList));
- RtlZeroMemory(&(*Cache)->SlabList, sizeof (LIST_ENTRY));
+ ASSERT(IsListEmpty(&(*Cache)->FreeList));
+ RtlZeroMemory(&(*Cache)->FreeList, sizeof (LIST_ENTRY));
+ RtlZeroMemory(&(*Cache)->FreeLock, sizeof (KSPIN_LOCK));
(*Cache)->Argument = NULL;
(*Cache)->ReleaseLock = NULL;
@@ -903,22 +583,22 @@ CacheDestroy(
__CacheFlushMagazines(Cache);
- ASSERT(IsZeroMemory(Cache->Magazine, sizeof (XENBUS_CACHE_MAGAZINE) *
Cache->MagazineCount));
- __CacheFree(Cache->Magazine);
- Cache->Magazine = NULL;
+ ASSERT(IsZeroMemory(Cache->Magazines, sizeof (XENBUS_CACHE_MAGAZINE) *
Cache->MagazineCount));
+ __CacheFree(Cache->Magazines);
+ Cache->Magazines = NULL;
Cache->MagazineCount = 0;
CacheSpill(Cache, 0);
- ASSERT(Cache->CurrentObjects == 0);
+ ASSERT3S(Cache->Count, ==, 0);
+ ASSERT3S(Cache->CurrentObjects, ==, 0);
Cache->MaximumObjects = 0;
- ASSERT(Cache->CurrentSlabs == 0);
- Cache->MaximumSlabs = 0;
+ ASSERT(IsListEmpty(&Cache->FreeList));
+ ASSERT(Cache->FreeSize == 0);
- Cache->Cursor = NULL;
- ASSERT(IsListEmpty(&Cache->SlabList));
- RtlZeroMemory(&Cache->SlabList, sizeof (LIST_ENTRY));
+ RtlZeroMemory(&Cache->FreeList, sizeof (LIST_ENTRY));
+ RtlZeroMemory(&Cache->FreeLock, sizeof (KSPIN_LOCK));
Cache->Argument = NULL;
Cache->ReleaseLock = NULL;
@@ -963,14 +643,13 @@ CacheDebugCallback(
XENBUS_DEBUG(Printf,
&Context->DebugInterface,
- "- %s: Count = %d, Reservation = %d, Objects = %d /
%d, Slabs = %d / %d\n",
+ "- %s: Reservation = %d, Cap = %d, Count = %d,
Objects = %d / %d\n",
Cache->Name,
- Cache->Count,
Cache->Reservation,
+ Cache->Cap,
+ Cache->Count,
Cache->CurrentObjects,
- Cache->MaximumObjects,
- Cache->CurrentSlabs,
- Cache->MaximumSlabs);
+ Cache->MaximumObjects);
}
}
}
@@ -1024,11 +703,14 @@ CacheMonitor(
Cache = CONTAINING_RECORD(ListEntry, XENBUS_CACHE, ListEntry);
- if (Cache->Count < Cache->Reservation)
- CacheFill(Cache, Cache->Reservation);
- else if (Cache->Count > Cache->Reservation)
+ if (Cache->Count < (LONG)Cache->Reservation) {
+ CacheFill(Cache,
+ Cache->Reservation);
+ } else if (Cache->Count > (LONG)Cache->Reservation) {
CacheSpill(Cache,
- __max(Cache->Reservation, (Cache->Count / 2)));
+ __max((LONG)Cache->Reservation,
+ (Cache->Count / 2)));
+ }
}
loop:
--
2.32.0.windows.1
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |