diff options
| -rw-r--r-- | include/linux/mm_types.h | 5 | ||||
| -rw-r--r-- | include/linux/slub_def.h | 1 | ||||
| -rw-r--r-- | mm/slub.c | 52 |
3 files changed, 13 insertions, 45 deletions
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 0cdc8fb..8776679 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -37,10 +37,7 @@ struct page { * to show when page is mapped * & limit reverse map searches. */ - struct { /* SLUB uses */ - short unsigned int inuse; - short unsigned int offset; - }; + unsigned int inuse; /* SLUB: Nr of objects */ }; union { struct { diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 0a7ae25..92e10cf 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -15,6 +15,7 @@ struct kmem_cache_cpu { void **freelist; struct page *page; int node; + unsigned int offset; /* Lots of wasted space */ } ____cacheline_aligned_in_smp; @@ -200,11 +200,6 @@ static inline void ClearSlabDebug(struct page *page) #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) #endif -/* - * The page->inuse field is 16 bit thus we have this limitation - */ -#define MAX_OBJECTS_PER_SLAB 65535 - /* Internal SLUB flags */ #define __OBJECT_POISON 0x80000000 /* Poison object */ #define __SYSFS_ADD_DEFERRED 0x40000000 /* Not yet visible via sysfs */ @@ -729,11 +724,6 @@ static int check_slab(struct kmem_cache *s, struct page *page) slab_err(s, page, "Not a valid slab page"); return 0; } - if (page->offset * sizeof(void *) != s->offset) { - slab_err(s, page, "Corrupted offset %lu", - (unsigned long)(page->offset * sizeof(void *))); - return 0; - } if (page->inuse > s->objects) { slab_err(s, page, "inuse %u > max %u", s->name, page->inuse, s->objects); @@ -872,8 +862,6 @@ bad: slab_fix(s, "Marking all objects used"); page->inuse = s->objects; page->freelist = NULL; - /* Fix up fields that may be corrupted */ - page->offset = s->offset / sizeof(void *); } return 0; } @@ -1104,7 +1092,6 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) n = get_node(s, page_to_nid(page)); if (n) atomic_long_inc(&n->nr_slabs); - page->offset = s->offset / sizeof(void *); page->slab = s; page->flags |= 1 << PG_slab; if (s->flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_POISON | @@ -1398,10 +1385,10 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) /* Retrieve object from cpu_freelist */ object = c->freelist; - c->freelist = c->freelist[page->offset]; + c->freelist = c->freelist[c->offset]; /* And put onto the regular freelist */ - object[page->offset] = page->freelist; + object[c->offset] = page->freelist; page->freelist = object; page->inuse--; } @@ -1497,7 +1484,7 @@ load_freelist: goto debug; object = c->page->freelist; - c->freelist = object[c->page->offset]; + c->freelist = object[c->offset]; c->page->inuse = s->objects; c->page->freelist = NULL; c->node = page_to_nid(c->page); @@ -1549,7 +1536,7 @@ debug: goto another_slab; c->page->inuse++; - c->page->freelist = object[c->page->offset]; + c->page->freelist = object[c->offset]; slab_unlock(c->page); return object; } @@ -1580,7 +1567,7 @@ static void __always_inline *slab_alloc(struct kmem_cache *s, else { object = c->freelist; - c->freelist = object[c->page->offset]; + c->freelist = object[c->offset]; } local_irq_restore(flags); @@ -1613,7 +1600,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_node); * handling required then we can return immediately. */ static void __slab_free(struct kmem_cache *s, struct page *page, - void *x, void *addr) + void *x, void *addr, unsigned int offset) { void *prior; void **object = (void *)x; @@ -1623,7 +1610,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page, if (unlikely(SlabDebug(page))) goto debug; checks_ok: - prior = object[page->offset] = page->freelist; + prior = object[offset] = page->freelist; page->freelist = object; page->inuse--; @@ -1684,10 +1671,10 @@ static void __always_inline slab_free(struct kmem_cache *s, debug_check_no_locks_freed(object, s->objsize); c = get_cpu_slab(s, smp_processor_id()); if (likely(page == c->page && !SlabDebug(page))) { - object[page->offset] = c->freelist; + object[c->offset] = c->freelist; c->freelist = object; } else - __slab_free(s, page, x, addr); + __slab_free(s, page, x, addr, c->offset); local_irq_restore(flags); } @@ -1774,14 +1761,6 @@ static inline int slab_order(int size, int min_objects, int rem; int min_order = slub_min_order; - /* - * If we would create too many object per slab then reduce - * the slab order even if it goes below slub_min_order. - */ - while (min_order > 0 && - (PAGE_SIZE << min_order) >= MAX_OBJECTS_PER_SLAB * size) - min_order--; - for (order = max(min_order, fls(min_objects * size - 1) - PAGE_SHIFT); order <= max_order; order++) { @@ -1796,9 +1775,6 @@ static inline int slab_order(int size, int min_objects, if (rem <= slab_size / fract_leftover) break; - /* If the next size is too high then exit now */ - if (slab_size * 2 >= MAX_OBJECTS_PER_SLAB * size) - break; } return order; @@ -1878,6 +1854,7 @@ static void init_kmem_cache_cpu(struct kmem_cache *s, { c->page = NULL; c->freelist = NULL; + c->offset = s->offset / sizeof(void *); c->node = 0; } @@ -2110,14 +2087,7 @@ static int calculate_sizes(struct kmem_cache *s) */ s->objects = (PAGE_SIZE << s->order) / size; - /* - * Verify that the number of objects is within permitted limits. - * The page->inuse field is only 16 bit wide! So we cannot have - * more than 64k objects per slab. - */ - if (!s->objects || s->objects > MAX_OBJECTS_PER_SLAB) - return 0; - return 1; + return !!s->objects; } |
