cgroups: add cpu_relax() calls in css_tryget() and cgroup_clear_css_refs()

css_tryget() and cgroup_clear_css_refs() contain polling loops; these
loops should have cpu_relax calls in them to reduce cross-cache traffic.

Signed-off-by: Paul Menage <menage@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index e267e62827bbc4e564c930576fcee8b70a8c4a79..e4e8e117d27d9171f23cdc54b69011758243c3da 100644 (file)
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -99,6 +99,7 @@ static inline bool css_tryget(struct cgroup_subsys_state *css)
        while (!atomic_inc_not_zero(&css->refcnt)) {
                if (test_bit(CSS_REMOVED, &css->flags))
                        return false;
+               cpu_relax();
        }
        return true;
 }

diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index 0066092de19a7f0522810f11194a80d4a31409df..492215d67fa5c3f153f6b1de42416ec4a1653298 100644 (file)
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -2509,7 +2509,7 @@ static int cgroup_clear_css_refs(struct cgroup *cgrp)
        for_each_subsys(cgrp->root, ss) {
                struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
                int refcnt;
-               do {
+               while (1) {
                        /* We can only remove a CSS with a refcnt==1 */
                        refcnt = atomic_read(&css->refcnt);
                        if (refcnt > 1) {
@@ -2523,7 +2523,10 @@ static int cgroup_clear_css_refs(struct cgroup *cgrp)
                         * css_tryget() to spin until we set the
                         * CSS_REMOVED bits or abort
                         */
-               } while (atomic_cmpxchg(&css->refcnt, refcnt, 0) != refcnt);
+                       if (atomic_cmpxchg(&css->refcnt, refcnt, 0) == refcnt)
+                               break;
+                       cpu_relax();
+               }
        }
  done:
        for_each_subsys(cgrp->root, ss) {