slub: Move map/flag clearing to __free_slab

[linux-2.6-omap-h63xx.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 96d63eb3ab179528efd1ea8c64cbc6358212a967..f924cffb29e793e8f8713d2071d473df1658cb01 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1125,6 +1125,8 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
                NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
                -pages);
 
+       __ClearPageSlab(page);
+       reset_page_mapcount(page);
        __free_pages(page, s->order);
 }
 
@@ -1154,8 +1156,6 @@ static void discard_slab(struct kmem_cache *s, struct page *page)
        struct kmem_cache_node *n = get_node(s, page_to_nid(page));
 
        atomic_long_dec(&n->nr_slabs);
-       reset_page_mapcount(page);
-       __ClearPageSlab(page);
        free_slab(s, page);
 }
 
@@ -1470,6 +1470,9 @@ static void *__slab_alloc(struct kmem_cache *s,
        void **object;
        struct page *new;
 
+       /* We handle __GFP_ZERO in the caller */
+       gfpflags &= ~__GFP_ZERO;
+
        if (!c->page)
                goto new_slab;
 
@@ -1536,9 +1539,15 @@ new_slab:
         * That is only possible if certain conditions are met that are being
         * checked when a slab is created.
         */
-       if (!(gfpflags & __GFP_NORETRY) && (s->flags & __PAGE_ALLOC_FALLBACK))
-               return kmalloc_large(s->objsize, gfpflags);
-
+       if (!(gfpflags & __GFP_NORETRY) &&
+                               (s->flags & __PAGE_ALLOC_FALLBACK)) {
+               if (gfpflags & __GFP_WAIT)
+                       local_irq_enable();
+               object = kmalloc_large(s->objsize, gfpflags);
+               if (gfpflags & __GFP_WAIT)
+                       local_irq_disable();
+               return object;
+       }
        return NULL;
 debug:
        if (!alloc_debug_processing(s, c->page, object, addr))
@@ -1877,6 +1886,9 @@ static void init_kmem_cache_cpu(struct kmem_cache *s,
        c->node = 0;
        c->offset = s->offset / sizeof(void *);
        c->objsize = s->objsize;
+#ifdef CONFIG_SLUB_STATS
+       memset(c->stat, 0, NR_SLUB_STAT_ITEMS * sizeof(unsigned));
+#endif
 }
 
 static void init_kmem_cache_node(struct kmem_cache_node *n)
@@ -2400,10 +2412,6 @@ EXPORT_SYMBOL(kmem_cache_destroy);
 struct kmem_cache kmalloc_caches[PAGE_SHIFT + 1] __cacheline_aligned;
 EXPORT_SYMBOL(kmalloc_caches);
 
-#ifdef CONFIG_ZONE_DMA
-static struct kmem_cache *kmalloc_caches_dma[PAGE_SHIFT + 1];
-#endif
-
 static int __init setup_slub_min_order(char *str)
 {
        get_option(&str, &slub_min_order);
@@ -2463,6 +2471,7 @@ panic:
 }
 
 #ifdef CONFIG_ZONE_DMA
+static struct kmem_cache *kmalloc_caches_dma[PAGE_SHIFT + 1];
 
 static void sysfs_add_func(struct work_struct *w)
 {
@@ -2679,19 +2688,6 @@ void kfree(const void *x)
 }
 EXPORT_SYMBOL(kfree);
 
-static unsigned long count_partial(struct kmem_cache_node *n)
-{
-       unsigned long flags;
-       unsigned long x = 0;
-       struct page *page;
-
-       spin_lock_irqsave(&n->list_lock, flags);
-       list_for_each_entry(page, &n->partial, lru)
-               x += page->inuse;
-       spin_unlock_irqrestore(&n->list_lock, flags);
-       return x;
-}
-
 /*
  * kmem_cache_shrink removes empty slabs from the partial lists and sorts
  * the remaining slabs by the number of items in use. The slabs with the
@@ -3170,6 +3166,21 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
        return slab_alloc(s, gfpflags, node, caller);
 }
 
+#if (defined(CONFIG_SYSFS) && defined(CONFIG_SLUB_DEBUG)) || defined(CONFIG_SLABINFO)
+static unsigned long count_partial(struct kmem_cache_node *n)
+{
+       unsigned long flags;
+       unsigned long x = 0;
+       struct page *page;
+
+       spin_lock_irqsave(&n->list_lock, flags);
+       list_for_each_entry(page, &n->partial, lru)
+               x += page->inuse;
+       spin_unlock_irqrestore(&n->list_lock, flags);
+       return x;
+}
+#endif
+
 #if defined(CONFIG_SYSFS) && defined(CONFIG_SLUB_DEBUG)
 static int validate_slab(struct kmem_cache *s, struct page *page,
                                                unsigned long *map)
@@ -3968,10 +3979,12 @@ static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
 
        len = sprintf(buf, "%lu", sum);
 
+#ifdef CONFIG_SMP
        for_each_online_cpu(cpu) {
                if (data[cpu] && len < PAGE_SIZE - 20)
-                       len += sprintf(buf + len, " c%d=%u", cpu, data[cpu]);
+                       len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
        }
+#endif
        kfree(data);
        return len + sprintf(buf + len, "\n");
 }