aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--arch/x86/Kconfig4
-rw-r--r--mm/slub.c93
2 files changed, 92 insertions, 5 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index c95482b6b6d..9d0acedf5f3 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -52,6 +52,10 @@ config HAVE_LATENCYTOP_SUPPORT
config SEMAPHORE_SLEEPERS
def_bool y
+config FAST_CMPXCHG_LOCAL
+ bool
+ default y
+
config MMU
def_bool y
diff --git a/mm/slub.c b/mm/slub.c
index 5995626e0cf..20ab8f0a4eb 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -149,6 +149,13 @@ static inline void ClearSlabDebug(struct page *page)
/* Enable to test recovery from slab corruption on boot */
#undef SLUB_RESILIENCY_TEST
+/*
+ * Currently fastpath is not supported if preemption is enabled.
+ */
+#if defined(CONFIG_FAST_CMPXCHG_LOCAL) && !defined(CONFIG_PREEMPT)
+#define SLUB_FASTPATH
+#endif
+
#if PAGE_SHIFT <= 12
/*
@@ -1493,7 +1500,11 @@ static void *__slab_alloc(struct kmem_cache *s,
{
void **object;
struct page *new;
+#ifdef SLUB_FASTPATH
+ unsigned long flags;
+ local_irq_save(flags);
+#endif
if (!c->page)
goto new_slab;
@@ -1512,7 +1523,12 @@ load_freelist:
c->page->inuse = s->objects;
c->page->freelist = c->page->end;
c->node = page_to_nid(c->page);
+unlock_out:
slab_unlock(c->page);
+out:
+#ifdef SLUB_FASTPATH
+ local_irq_restore(flags);
+#endif
return object;
another_slab:
@@ -1542,7 +1558,8 @@ new_slab:
c->page = new;
goto load_freelist;
}
- return NULL;
+ object = NULL;
+ goto out;
debug:
object = c->page->freelist;
if (!alloc_debug_processing(s, c->page, object, addr))
@@ -1551,8 +1568,7 @@ debug:
c->page->inuse++;
c->page->freelist = object[c->offset];
c->node = -1;
- slab_unlock(c->page);
- return object;
+ goto unlock_out;
}
/*
@@ -1569,9 +1585,36 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
gfp_t gfpflags, int node, void *addr)
{
void **object;
- unsigned long flags;
struct kmem_cache_cpu *c;
+/*
+ * The SLUB_FASTPATH path is provisional and is currently disabled if the
+ * kernel is compiled with preemption or if the arch does not support
+ * fast cmpxchg operations. There are a couple of coming changes that will
+ * simplify matters and allow preemption. Ultimately we may end up making
+ * SLUB_FASTPATH the default.
+ *
+ * 1. The introduction of the per cpu allocator will avoid array lookups
+ * through get_cpu_slab(). A special register can be used instead.
+ *
+ * 2. The introduction of per cpu atomic operations (cpu_ops) means that
+ * we can realize the logic here entirely with per cpu atomics. The
+ * per cpu atomic ops will take care of the preemption issues.
+ */
+
+#ifdef SLUB_FASTPATH
+ c = get_cpu_slab(s, raw_smp_processor_id());
+ do {
+ object = c->freelist;
+ if (unlikely(is_end(object) || !node_match(c, node))) {
+ object = __slab_alloc(s, gfpflags, node, addr, c);
+ break;
+ }
+ } while (cmpxchg_local(&c->freelist, object, object[c->offset])
+ != object);
+#else
+ unsigned long flags;
+
local_irq_save(flags);
c = get_cpu_slab(s, smp_processor_id());
if (unlikely(is_end(c->freelist) || !node_match(c, node)))
@@ -1583,6 +1626,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
c->freelist = object[c->offset];
}
local_irq_restore(flags);
+#endif
if (unlikely((gfpflags & __GFP_ZERO) && object))
memset(object, 0, c->objsize);
@@ -1618,6 +1662,11 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
void *prior;
void **object = (void *)x;
+#ifdef SLUB_FASTPATH
+ unsigned long flags;
+
+ local_irq_save(flags);
+#endif
slab_lock(page);
if (unlikely(SlabDebug(page)))
@@ -1643,6 +1692,9 @@ checks_ok:
out_unlock:
slab_unlock(page);
+#ifdef SLUB_FASTPATH
+ local_irq_restore(flags);
+#endif
return;
slab_empty:
@@ -1653,6 +1705,9 @@ slab_empty:
remove_partial(s, page);
slab_unlock(page);
+#ifdef SLUB_FASTPATH
+ local_irq_restore(flags);
+#endif
discard_slab(s, page);
return;
@@ -1677,9 +1732,36 @@ static __always_inline void slab_free(struct kmem_cache *s,
struct page *page, void *x, void *addr)
{
void **object = (void *)x;
- unsigned long flags;
struct kmem_cache_cpu *c;
+#ifdef SLUB_FASTPATH
+ void **freelist;
+
+ c = get_cpu_slab(s, raw_smp_processor_id());
+ debug_check_no_locks_freed(object, s->objsize);
+ do {
+ freelist = c->freelist;
+ barrier();
+ /*
+ * If the compiler would reorder the retrieval of c->page to
+ * come before c->freelist then an interrupt could
+ * change the cpu slab before we retrieve c->freelist. We
+ * could be matching on a page no longer active and put the
+ * object onto the freelist of the wrong slab.
+ *
+ * On the other hand: If we already have the freelist pointer
+ * then any change of cpu_slab will cause the cmpxchg to fail
+ * since the freelist pointers are unique per slab.
+ */
+ if (unlikely(page != c->page || c->node < 0)) {
+ __slab_free(s, page, x, addr, c->offset);
+ break;
+ }
+ object[c->offset] = freelist;
+ } while (cmpxchg_local(&c->freelist, freelist, object) != freelist);
+#else
+ unsigned long flags;
+
local_irq_save(flags);
debug_check_no_locks_freed(object, s->objsize);
c = get_cpu_slab(s, smp_processor_id());
@@ -1690,6 +1772,7 @@ static __always_inline void slab_free(struct kmem_cache *s,
__slab_free(s, page, x, addr, c->offset);
local_irq_restore(flags);
+#endif
}
void kmem_cache_free(struct kmem_cache *s, void *x)