1 files changed, 122 insertions, 299 deletions

diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c
index a8eec9c..12cd4d3 100644
--- a/sys/kern/kern_malloc.c
+++ b/sys/kern/kern_malloc.c
@@ -52,6 +52,8 @@
 #include <vm/vm_extern.h>
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
+#include <vm/uma.h>
+#include <vm/uma_int.h>
 
 #if defined(INVARIANTS) && defined(__i386__)
 #include <machine/cpu.h>
@@ -80,50 +82,42 @@ SYSINIT(kmem, SI_SUB_KMEM, SI_ORDER_FIRST, kmeminit, NULL)
 static MALLOC_DEFINE(M_FREE, "free", "should be on free list");
 
 static struct malloc_type *kmemstatistics;
-static struct kmembuckets bucket[MINBUCKET + 16];
-static struct kmemusage *kmemusage;
 static char *kmembase;
 static char *kmemlimit;
 
-static struct mtx malloc_mtx;
-
-u_int vm_kmem_size;
-
-#ifdef INVARIANTS
-/*
- * This structure provides a set of masks to catch unaligned frees.
- */
-static long addrmask[] = { 0,
-	0x00000001, 0x00000003, 0x00000007, 0x0000000f,
-	0x0000001f, 0x0000003f, 0x0000007f, 0x000000ff,
-	0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff,
-	0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff,
+#define KMEM_ZSHIFT	4
+#define KMEM_ZBASE	16
+#define KMEM_ZMASK	(KMEM_ZBASE - 1)
+
+#define KMEM_ZMAX	65536
+#define KMEM_ZSIZE	(KMEM_ZMAX >> KMEM_ZSHIFT)
+static uma_zone_t kmemzones[KMEM_ZSIZE + 1];
+
+
+/* These won't be powers of two for long */
+struct {
+	int size;
+	char *name;
+} kmemsizes[] = {
+	{16, "16"},
+	{32, "32"},
+	{64, "64"},
+	{128, "128"},
+	{256, "256"},
+	{512, "512"},
+	{1024, "1024"},
+	{2048, "2048"},
+	{4096, "4096"},
+	{8192, "8192"},
+	{16384, "16384"},
+	{32768, "32768"},
+	{65536, "65536"},
+	{0, NULL},
 };
 
-/*
- * The WEIRD_ADDR is used as known text to copy into free objects so
- * that modifications after frees can be detected.
- */
-#define WEIRD_ADDR	0xdeadc0de
-#define MAX_COPY	64
+static struct mtx malloc_mtx;
 
-/*
- * Normally the first word of the structure is used to hold the list
- * pointer for free objects. However, when running with diagnostics,
- * we use the third and fourth fields, so as to catch modifications
- * in the most commonly trashed first two words.
- */
-struct freelist {
-	long	spare0;
-	struct malloc_type *type;
-	long	spare1;
-	caddr_t	next;
-};
-#else /* !INVARIANTS */
-struct freelist {
-	caddr_t	next;
-};
-#endif /* INVARIANTS */
+u_int vm_kmem_size;
 
 /*
  *	malloc:
@@ -139,17 +133,10 @@ malloc(size, type, flags)
 	struct malloc_type *type;
 	int flags;
 {
-	register struct kmembuckets *kbp;
-	register struct kmemusage *kup;
-	register struct freelist *freep;
-	long indx, npg, allocsize;
 	int s;
-	caddr_t va, cp, savedlist;
-#ifdef INVARIANTS
-	long *end, *lp;
-	int copysize;
-	const char *savedtype;
-#endif
+	long indx;
+	caddr_t va;
+	uma_zone_t zone;
 	register struct malloc_type *ksp = type;
 
 #if defined(INVARIANTS)
@@ -157,124 +144,52 @@ malloc(size, type, flags)
 		KASSERT(curthread->td_intr_nesting_level == 0,
 		   ("malloc(M_WAITOK) in interrupt context"));
 #endif
-	indx = BUCKETINDX(size);
-	kbp = &bucket[indx];
 	s = splmem();
-	mtx_lock(&malloc_mtx);
+	/* mtx_lock(&malloc_mtx); XXX */
 	while (ksp->ks_memuse >= ksp->ks_limit) {
 		if (flags & M_NOWAIT) {
 			splx(s);
-			mtx_unlock(&malloc_mtx);
+			/* mtx_unlock(&malloc_mtx); XXX */
 			return ((void *) NULL);
 		}
 		if (ksp->ks_limblocks < 65535)
 			ksp->ks_limblocks++;
-		msleep((caddr_t)ksp, &malloc_mtx, PSWP+2, type->ks_shortdesc,
+		msleep((caddr_t)ksp, /* &malloc_mtx */ NULL, PSWP+2, type->ks_shortdesc,
 		    0);
 	}
-	ksp->ks_size |= 1 << indx;
-#ifdef INVARIANTS
-	copysize = 1 << indx < MAX_COPY ? 1 << indx : MAX_COPY;
-#endif
-	if (kbp->kb_next == NULL) {
-		kbp->kb_last = NULL;
-		if (size > MAXALLOCSAVE)
-			allocsize = roundup(size, PAGE_SIZE);
-		else
-			allocsize = 1 << indx;
-		npg = btoc(allocsize);
-
-		mtx_unlock(&malloc_mtx);
-		va = (caddr_t) kmem_malloc(kmem_map, (vm_size_t)ctob(npg), flags);
-
+	/* mtx_unlock(&malloc_mtx); XXX */
+
+	if (size <= KMEM_ZMAX) {
+		indx = size;
+		if (indx & KMEM_ZMASK)
+			indx = (indx & ~KMEM_ZMASK) + KMEM_ZBASE;
+		zone = kmemzones[indx >> KMEM_ZSHIFT];
+		indx = zone->uz_size;
+		va = uma_zalloc(zone, flags);
 		if (va == NULL) {
-			splx(s);
-			return ((void *) NULL);
-		}
-		/*
-		 * Enter malloc_mtx after the error check to avoid having to
-		 * immediately exit it again if there is an error.
-		 */
-		mtx_lock(&malloc_mtx);
-
-		kbp->kb_total += kbp->kb_elmpercl;
-		kup = btokup(va);
-		kup->ku_indx = indx;
-		if (allocsize > MAXALLOCSAVE) {
-			if (npg > 65535)
-				panic("malloc: allocation too large");
-			kup->ku_pagecnt = npg;
-			ksp->ks_memuse += allocsize;
+			/* mtx_lock(&malloc_mtx); XXX */
 			goto out;
 		}
-		kup->ku_freecnt = kbp->kb_elmpercl;
-		kbp->kb_totalfree += kbp->kb_elmpercl;
-		/*
-		 * Just in case we blocked while allocating memory,
-		 * and someone else also allocated memory for this
-		 * bucket, don't assume the list is still empty.
-		 */
-		savedlist = kbp->kb_next;
-		kbp->kb_next = cp = va + (npg * PAGE_SIZE) - allocsize;
-		for (;;) {
-			freep = (struct freelist *)cp;
-#ifdef INVARIANTS
-			/*
-			 * Copy in known text to detect modification
-			 * after freeing.
-			 */
-			end = (long *)&cp[copysize];
-			for (lp = (long *)cp; lp < end; lp++)
-				*lp = WEIRD_ADDR;
-			freep->type = M_FREE;
-#endif /* INVARIANTS */
-			if (cp <= va)
-				break;
-			cp -= allocsize;
-			freep->next = cp;
+		ksp->ks_size |= indx;
+	} else {
+		/* XXX This is not the next power of two so this will break ks_size */
+		indx = roundup(size, PAGE_SIZE);
+		zone = NULL;
+		va = uma_large_malloc(size, flags);
+		if (va == NULL) {
+			/* mtx_lock(&malloc_mtx); XXX */
+			goto out;
 		}
-		freep->next = savedlist;
-		if (kbp->kb_last == NULL)
-			kbp->kb_last = (caddr_t)freep;
-	}
-	va = kbp->kb_next;
-	kbp->kb_next = ((struct freelist *)va)->next;
-#ifdef INVARIANTS
-	freep = (struct freelist *)va;
-	savedtype = (const char *) freep->type->ks_shortdesc;
-	freep->type = (struct malloc_type *)WEIRD_ADDR;
-	if ((intptr_t)(void *)&freep->next & 0x2)
-		freep->next = (caddr_t)((WEIRD_ADDR >> 16)|(WEIRD_ADDR << 16));
-	else
-		freep->next = (caddr_t)WEIRD_ADDR;
-	end = (long *)&va[copysize];
-	for (lp = (long *)va; lp < end; lp++) {
-		if (*lp == WEIRD_ADDR)
-			continue;
-		printf("%s %ld of object %p size %lu %s %s (0x%lx != 0x%lx)\n",
-			"Data modified on freelist: word",
-			(long)(lp - (long *)va), (void *)va, size,
-			"previous type", savedtype, *lp, (u_long)WEIRD_ADDR);
-		break;
 	}
-	freep->spare0 = 0;
-#endif /* INVARIANTS */
-	kup = btokup(va);
-	if (kup->ku_indx != indx)
-		panic("malloc: wrong bucket");
-	if (kup->ku_freecnt == 0)
-		panic("malloc: lost data");
-	kup->ku_freecnt--;
-	kbp->kb_totalfree--;
-	ksp->ks_memuse += 1 << indx;
-out:
-	kbp->kb_calls++;
+	/* mtx_lock(&malloc_mtx); XXX */
+	ksp->ks_memuse += indx;
 	ksp->ks_inuse++;
+out:
 	ksp->ks_calls++;
 	if (ksp->ks_memuse > ksp->ks_maxused)
 		ksp->ks_maxused = ksp->ks_memuse;
 	splx(s);
-	mtx_unlock(&malloc_mtx);
+	/* mtx_unlock(&malloc_mtx); XXX */
 	/* XXX: Do idle pre-zeroing.  */
 	if (va != NULL && (flags & M_ZERO))
 		bzero(va, size);
@@ -293,124 +208,41 @@ free(addr, type)
 	void *addr;
 	struct malloc_type *type;
 {
-	register struct kmembuckets *kbp;
-	register struct kmemusage *kup;
-	register struct freelist *freep;
-	long size;
+	uma_slab_t slab;
+	void *mem;
+	u_long size;
 	int s;
-#ifdef INVARIANTS
-	struct freelist *fp;
-	long *end, *lp, alloc, copysize;
-#endif
 	register struct malloc_type *ksp = type;
 
 	/* free(NULL, ...) does nothing */
 	if (addr == NULL)
 		return;
 
-	KASSERT(kmembase <= (char *)addr && (char *)addr < kmemlimit,
-	    ("free: address %p out of range", (void *)addr));
-	kup = btokup(addr);
-	size = 1 << kup->ku_indx;
-	kbp = &bucket[kup->ku_indx];
+	size = 0;
 	s = splmem();
-	mtx_lock(&malloc_mtx);
-#ifdef INVARIANTS
-	/*
-	 * Check for returns of data that do not point to the
-	 * beginning of the allocation.
-	 */
-	if (size > PAGE_SIZE)
-		alloc = addrmask[BUCKETINDX(PAGE_SIZE)];
-	else
-		alloc = addrmask[kup->ku_indx];
-	if (((uintptr_t)(void *)addr & alloc) != 0)
-		panic("free: unaligned addr %p, size %ld, type %s, mask %ld",
-		    (void *)addr, size, type->ks_shortdesc, alloc);
-#endif /* INVARIANTS */
-	if (size > MAXALLOCSAVE) {
-		mtx_unlock(&malloc_mtx);
-		kmem_free(kmem_map, (vm_offset_t)addr, ctob(kup->ku_pagecnt));
-		mtx_lock(&malloc_mtx);
-
-		size = kup->ku_pagecnt << PAGE_SHIFT;
-		ksp->ks_memuse -= size;
-		kup->ku_indx = 0;
-		kup->ku_pagecnt = 0;
-		if (ksp->ks_memuse + size >= ksp->ks_limit &&
-		    ksp->ks_memuse < ksp->ks_limit)
-			wakeup((caddr_t)ksp);
-		ksp->ks_inuse--;
-		kbp->kb_total -= 1;
-		splx(s);
-		mtx_unlock(&malloc_mtx);
-		return;
-	}
-	freep = (struct freelist *)addr;
-#ifdef INVARIANTS
-	/*
-	 * Check for multiple frees. Use a quick check to see if
-	 * it looks free before laboriously searching the freelist.
-	 */
-	if (freep->spare0 == WEIRD_ADDR) {
-		fp = (struct freelist *)kbp->kb_next;
-		while (fp) {
-			if (fp->spare0 != WEIRD_ADDR)
-				panic("free: free item %p modified", fp);
-			else if (addr == (caddr_t)fp)
-				panic("free: multiple freed item %p", addr);
-			fp = (struct freelist *)fp->next;
-		}
-	}
-	/*
-	 * Copy in known text to detect modification after freeing
-	 * and to make it look free. Also, save the type being freed
-	 * so we can list likely culprit if modification is detected
-	 * when the object is reallocated.
-	 */
-	copysize = size < MAX_COPY ? size : MAX_COPY;
-	end = (long *)&((caddr_t)addr)[copysize];
-	for (lp = (long *)addr; lp < end; lp++)
-		*lp = WEIRD_ADDR;
-	freep->type = type;
-#endif /* INVARIANTS */
-	kup->ku_freecnt++;
-	if (kup->ku_freecnt >= kbp->kb_elmpercl) {
-		if (kup->ku_freecnt > kbp->kb_elmpercl)
-			panic("free: multiple frees");
-		else if (kbp->kb_totalfree > kbp->kb_highwat)
-			kbp->kb_couldfree++;
+
+	mem = (void *)((u_long)addr & (~UMA_SLAB_MASK));
+	slab = hash_sfind(mallochash, mem);
+
+	if (slab == NULL)
+		panic("free: address %p(%p) has not been allocated.\n", addr, mem);
+
+	if (!(slab->us_flags & UMA_SLAB_MALLOC)) {
+		size = slab->us_zone->uz_size;
+		uma_zfree_arg(slab->us_zone, addr, slab);
+	} else {
+		size = slab->us_size;
+		uma_large_free(slab);
 	}
-	kbp->kb_totalfree++;
+	/* mtx_lock(&malloc_mtx); XXX */
+
 	ksp->ks_memuse -= size;
 	if (ksp->ks_memuse + size >= ksp->ks_limit &&
 	    ksp->ks_memuse < ksp->ks_limit)
 		wakeup((caddr_t)ksp);
 	ksp->ks_inuse--;
-#ifdef OLD_MALLOC_MEMORY_POLICY
-	if (kbp->kb_next == NULL)
-		kbp->kb_next = addr;
-	else
-		((struct freelist *)kbp->kb_last)->next = addr;
-	freep->next = NULL;
-	kbp->kb_last = addr;
-#else
-	/*
-	 * Return memory to the head of the queue for quick reuse.  This
-	 * can improve performance by improving the probability of the
-	 * item being in the cache when it is reused.
-	 */
-	if (kbp->kb_next == NULL) {
-		kbp->kb_next = addr;
-		kbp->kb_last = addr;
-		freep->next = NULL;
-	} else {
-		freep->next = kbp->kb_next;
-		kbp->kb_next = addr;
-	}
-#endif
 	splx(s);
-	mtx_unlock(&malloc_mtx);
+	/* mtx_unlock(&malloc_mtx); XXX */
 }
 
 /*
@@ -423,7 +255,7 @@ realloc(addr, size, type, flags)
 	struct malloc_type *type;
 	int flags;
 {
-	struct kmemusage *kup;
+	uma_slab_t slab;
 	unsigned long alloc;
 	void *newaddr;
 
@@ -431,15 +263,18 @@ realloc(addr, size, type, flags)
 	if (addr == NULL)
 		return (malloc(size, type, flags));
 
+	slab = hash_sfind(mallochash,
+	    (void *)((u_long)addr & ~(UMA_SLAB_MASK)));
+
 	/* Sanity check */
-	KASSERT(kmembase <= (char *)addr && (char *)addr < kmemlimit,
+	KASSERT(slab != NULL,
 	    ("realloc: address %p out of range", (void *)addr));
 
 	/* Get the size of the original block */
-	kup = btokup(addr);
-	alloc = 1 << kup->ku_indx;
-	if (alloc > MAXALLOCSAVE)
-		alloc = kup->ku_pagecnt << PAGE_SHIFT;
+	if (slab->us_zone)
+		alloc = slab->us_zone->uz_size;
+	else
+		alloc = slab->us_size;
 
 	/* Reuse the original block if appropriate */
 	if (size <= alloc
@@ -484,16 +319,11 @@ kmeminit(dummy)
 	register long indx;
 	u_long npg;
 	u_long mem_size;
-
-#if	((MAXALLOCSAVE & (MAXALLOCSAVE - 1)) != 0)
-#error "kmeminit: MAXALLOCSAVE not power of 2"
-#endif
-#if	(MAXALLOCSAVE > MINALLOCSIZE * 32768)
-#error "kmeminit: MAXALLOCSAVE too big"
-#endif
-#if	(MAXALLOCSAVE < PAGE_SIZE)
-#error "kmeminit: MAXALLOCSAVE too small"
-#endif
+	void *hashmem;
+	u_long hashsize;
+	int highbit;
+	int bits;
+	int i;
 
 	mtx_init(&malloc_mtx, "malloc", MTX_DEF);
 
@@ -544,17 +374,36 @@ kmeminit(dummy)
 	npg = (nmbufs * MSIZE + nmbclusters * MCLBYTES + nmbcnt *
 	    sizeof(u_int) + vm_kmem_size) / PAGE_SIZE;
 
-	kmemusage = (struct kmemusage *) kmem_alloc(kernel_map,
-		(vm_size_t)(npg * sizeof(struct kmemusage)));
 	kmem_map = kmem_suballoc(kernel_map, (vm_offset_t *)&kmembase,
 		(vm_offset_t *)&kmemlimit, (vm_size_t)(npg * PAGE_SIZE));
 	kmem_map->system_map = 1;
-	for (indx = 0; indx < MINBUCKET + 16; indx++) {
-		if (1 << indx >= PAGE_SIZE)
-			bucket[indx].kb_elmpercl = 1;
-		else
-			bucket[indx].kb_elmpercl = PAGE_SIZE / (1 << indx);
-		bucket[indx].kb_highwat = 5 * bucket[indx].kb_elmpercl;
+
+	hashsize = npg * sizeof(void *);
+
+	highbit = 0;
+	bits = 0;
+	/* The hash size must be a power of two */
+	for (i = 0; i < 8 * sizeof(hashsize); i++)
+		if (hashsize & (1 << i)) {
+			highbit = i;
+			bits++;
+		}
+	if (bits > 1) 
+		hashsize = 1 << (highbit);
+
+	hashmem = (void *)kmem_alloc(kernel_map, (vm_size_t)hashsize);
+	uma_startup2(hashmem, hashsize / sizeof(void *));
+
+	for (i = 0, indx = 0; kmemsizes[indx].size != 0; indx++) {
+		uma_zone_t zone;
+		int size = kmemsizes[indx].size;
+		char *name = kmemsizes[indx].name;
+
+		zone = uma_zcreate(name, size, NULL, NULL, NULL, NULL,
+		    UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
+		for (;i <= size; i+= KMEM_ZBASE)
+			kmemzones[i >> KMEM_ZSHIFT] = zone;
+		
 	}
 }
 
@@ -588,12 +437,6 @@ malloc_uninit(data)
 {
 	struct malloc_type *type = (struct malloc_type *)data;
 	struct malloc_type *t;
-#ifdef INVARIANTS
-	struct kmembuckets *kbp;
-	struct freelist *freep;
-	long indx;
-	int s;
-#endif
 
 	if (type->ks_magic != M_MAGIC)
 		panic("malloc type lacks magic");
@@ -604,26 +447,6 @@ malloc_uninit(data)
 	if (type->ks_limit == 0)
 		panic("malloc_uninit on uninitialized type");
 
-#ifdef INVARIANTS
-	s = splmem();
-	mtx_lock(&malloc_mtx);
-	for (indx = 0; indx < MINBUCKET + 16; indx++) {
-		kbp = bucket + indx;
-		freep = (struct freelist*)kbp->kb_next;
-		while (freep) {
-			if (freep->type == type)
-				freep->type = M_FREE;
-			freep = (struct freelist*)freep->next;
-		}
-	}
-	splx(s);
-	mtx_unlock(&malloc_mtx);
-
-	if (type->ks_memuse != 0)
-		printf("malloc_uninit: %ld bytes of '%s' still allocated\n",
-		    type->ks_memuse, type->ks_shortdesc);
-#endif
-
 	if (type == kmemstatistics)
 		kmemstatistics = type->ks_next;
 	else {