From eb91f1d0a531289e18f5587dc197d12a251c66a3 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Fri, 12 Jun 2009 14:56:09 +0300 Subject: slab: fix gfp flag in setup_cpu_cache() Fixes the following warning during bootup when compiling with CONFIG_SLAB: [ 0.000000] ------------[ cut here ]------------ [ 0.000000] WARNING: at kernel/lockdep.c:2282 lockdep_trace_alloc+0x91/0xb9() [ 0.000000] Hardware name: [ 0.000000] Modules linked in: [ 0.000000] Pid: 0, comm: swapper Not tainted 2.6.30 #491 [ 0.000000] Call Trace: [ 0.000000] [] ? lockdep_trace_alloc+0x91/0xb9 [ 0.000000] [] warn_slowpath_common+0x7c/0xa9 [ 0.000000] [] warn_slowpath_null+0x14/0x16 [ 0.000000] [] lockdep_trace_alloc+0x91/0xb9 [ 0.000000] [] kmem_cache_alloc_node_notrace+0x26/0xdf [ 0.000000] [] ? setup_cpu_cache+0x7e/0x210 [ 0.000000] [] setup_cpu_cache+0x113/0x210 [ 0.000000] [] kmem_cache_create+0x409/0x486 [ 0.000000] [] kmem_cache_init+0x232/0x593 [ 0.000000] [] ? mem_init+0x156/0x161 [ 0.000000] [] start_kernel+0x1cc/0x3b9 [ 0.000000] [] x86_64_start_reservations+0xaa/0xae [ 0.000000] [] x86_64_start_kernel+0xe1/0xe8 [ 0.000000] ---[ end trace 4eaa2a86a8e2da22 ]--- Signed-off-by: Pekka Enberg --- mm/slab.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index f46b65d124e..cd76964b53b 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -2102,7 +2102,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) for_each_online_node(node) { cachep->nodelists[node] = kmalloc_node(sizeof(struct kmem_list3), - GFP_KERNEL, node); + gfp, node); BUG_ON(!cachep->nodelists[node]); kmem_list3_init(cachep->nodelists[node]); } -- cgit v1.2.3 From 7e85ee0c1d15ca5f8bff0f514f158eba1742dd87 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Fri, 12 Jun 2009 14:03:06 +0300 Subject: slab,slub: don't enable interrupts during early boot As explained by Benjamin Herrenschmidt: Oh and btw, your patch alone doesn't fix powerpc, because it's missing a whole bunch of GFP_KERNEL's in the arch code... You would have to grep the entire kernel for things that check slab_is_available() and even then you'll be missing some. For example, slab_is_available() didn't always exist, and so in the early days on powerpc, we used a mem_init_done global that is set form mem_init() (not perfect but works in practice). And we still have code using that to do the test. Therefore, mask out __GFP_WAIT, __GFP_IO, and __GFP_FS in the slab allocators in early boot code to avoid enabling interrupts. Signed-off-by: Pekka Enberg --- mm/slab.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index cd76964b53b..453efcb1c98 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -303,6 +303,12 @@ struct kmem_list3 { int free_touched; /* updated without locking */ }; +/* + * The slab allocator is initialized with interrupts disabled. Therefore, make + * sure early boot allocations don't accidentally enable interrupts. + */ +static gfp_t slab_gfp_mask __read_mostly = SLAB_GFP_BOOT_MASK; + /* * Need this for bootstrapping a per node allocator. */ @@ -1654,6 +1660,14 @@ void __init kmem_cache_init(void) */ } +void __init kmem_cache_init_late(void) +{ + /* + * Interrupts are enabled now so all GFP allocations are safe. + */ + slab_gfp_mask = __GFP_BITS_MASK; +} + static int __init cpucache_init(void) { int cpu; @@ -3354,6 +3368,8 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, unsigned long save_flags; void *ptr; + flags &= slab_gfp_mask; + lockdep_trace_alloc(flags); if (slab_should_failslab(cachep, flags)) @@ -3434,6 +3450,8 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller) unsigned long save_flags; void *objp; + flags &= slab_gfp_mask; + lockdep_trace_alloc(flags); if (slab_should_failslab(cachep, flags)) -- cgit v1.2.3 From 8429db5c6336083594036c30f49401405d536911 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Fri, 12 Jun 2009 15:58:59 +0300 Subject: slab: setup cpu caches later on when interrupts are enabled Fixes the following boot-time warning: [ 0.000000] ------------[ cut here ]------------ [ 0.000000] WARNING: at kernel/smp.c:369 smp_call_function_many+0x56/0x1bc() [ 0.000000] Hardware name: [ 0.000000] Modules linked in: [ 0.000000] Pid: 0, comm: swapper Not tainted 2.6.30 #492 [ 0.000000] Call Trace: [ 0.000000] [] ? _spin_unlock+0x4f/0x5c [ 0.000000] [] ? smp_call_function_many+0x56/0x1bc [ 0.000000] [] warn_slowpath_common+0x7c/0xa9 [ 0.000000] [] warn_slowpath_null+0x14/0x16 [ 0.000000] [] smp_call_function_many+0x56/0x1bc [ 0.000000] [] ? do_ccupdate_local+0x0/0x54 [ 0.000000] [] ? do_ccupdate_local+0x0/0x54 [ 0.000000] [] smp_call_function+0x3d/0x68 [ 0.000000] [] ? do_ccupdate_local+0x0/0x54 [ 0.000000] [] on_each_cpu+0x31/0x7c [ 0.000000] [] do_tune_cpucache+0x119/0x454 [ 0.000000] [] ? lockdep_init_map+0x94/0x10b [ 0.000000] [] ? kmem_cache_init+0x421/0x593 [ 0.000000] [] enable_cpucache+0x68/0xad [ 0.000000] [] kmem_cache_init+0x434/0x593 [ 0.000000] [] ? mem_init+0x156/0x161 [ 0.000000] [] start_kernel+0x1cc/0x3b9 [ 0.000000] [] x86_64_start_reservations+0xaa/0xae [ 0.000000] [] x86_64_start_kernel+0xe1/0xe8 [ 0.000000] ---[ end trace 4eaa2a86a8e2da22 ]--- Cc: Christoph Lameter Cc: Nick Piggin Signed-off-by: Pekka Enberg --- mm/slab.c | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index 453efcb1c98..18e3164de09 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -759,6 +759,7 @@ static enum { NONE, PARTIAL_AC, PARTIAL_L3, + EARLY, FULL } g_cpucache_up; @@ -767,7 +768,7 @@ static enum { */ int slab_is_available(void) { - return g_cpucache_up == FULL; + return g_cpucache_up >= EARLY; } static DEFINE_PER_CPU(struct delayed_work, reap_work); @@ -1631,19 +1632,27 @@ void __init kmem_cache_init(void) } } - /* 6) resize the head arrays to their final sizes */ - { - struct kmem_cache *cachep; - mutex_lock(&cache_chain_mutex); - list_for_each_entry(cachep, &cache_chain, next) - if (enable_cpucache(cachep, GFP_NOWAIT)) - BUG(); - mutex_unlock(&cache_chain_mutex); - } + g_cpucache_up = EARLY; /* Annotate slab for lockdep -- annotate the malloc caches */ init_lock_keys(); +} + +void __init kmem_cache_init_late(void) +{ + struct kmem_cache *cachep; + + /* + * Interrupts are enabled now so all GFP allocations are safe. + */ + slab_gfp_mask = __GFP_BITS_MASK; + /* 6) resize the head arrays to their final sizes */ + mutex_lock(&cache_chain_mutex); + list_for_each_entry(cachep, &cache_chain, next) + if (enable_cpucache(cachep, GFP_NOWAIT)) + BUG(); + mutex_unlock(&cache_chain_mutex); /* Done! */ g_cpucache_up = FULL; @@ -1660,14 +1669,6 @@ void __init kmem_cache_init(void) */ } -void __init kmem_cache_init_late(void) -{ - /* - * Interrupts are enabled now so all GFP allocations are safe. - */ - slab_gfp_mask = __GFP_BITS_MASK; -} - static int __init cpucache_init(void) { int cpu; -- cgit v1.2.3