Merge branch 'linus' into core/softlockup

[linux-2.6-omap-h63xx.git] / kernel / rcupdate.c

diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index d92a76a881aa47a9aa059c3221910834f6c50cf8..2c7b8457d0d234203517f7aa6a1ddc6b724ef447 100644 (file)
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -44,6 +44,7 @@
 #include <linux/cpu.h>
 #include <linux/mutex.h>
 #include <linux/module.h>
+#include <linux/kernel_stat.h>
 
 enum rcu_barrier {
        RCU_BARRIER_STD,
@@ -55,6 +56,7 @@ static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL};
 static atomic_t rcu_barrier_cpu_count;
 static DEFINE_MUTEX(rcu_barrier_mutex);
 static struct completion rcu_barrier_completion;
+int rcu_scheduler_active __read_mostly;
 
 /*
  * Awaken the corresponding synchronize_rcu() instance now that a
@@ -80,6 +82,10 @@ void wakeme_after_rcu(struct rcu_head  *head)
 void synchronize_rcu(void)
 {
        struct rcu_synchronize rcu;
+
+       if (rcu_blocking_is_gp())
+               return;
+
        init_completion(&rcu.completion);
        /* Will wake me after RCU finished. */
        call_rcu(&rcu.head, wakeme_after_rcu);
@@ -116,6 +122,8 @@ static void rcu_barrier_func(void *type)
        }
 }
 
+static inline void wait_migrated_callbacks(void);
+
 /*
  * Orchestrate the specified type of RCU barrier, waiting for all
  * RCU callbacks of the specified type to complete.
@@ -141,6 +149,7 @@ static void _rcu_barrier(enum rcu_barrier type)
                complete(&rcu_barrier_completion);
        wait_for_completion(&rcu_barrier_completion);
        mutex_unlock(&rcu_barrier_mutex);
+       wait_migrated_callbacks();
 }
 
 /**
@@ -170,8 +179,55 @@ void rcu_barrier_sched(void)
 }
 EXPORT_SYMBOL_GPL(rcu_barrier_sched);
 
+static atomic_t rcu_migrate_type_count = ATOMIC_INIT(0);
+static struct rcu_head rcu_migrate_head[3];
+static DECLARE_WAIT_QUEUE_HEAD(rcu_migrate_wq);
+
+static void rcu_migrate_callback(struct rcu_head *notused)
+{
+       if (atomic_dec_and_test(&rcu_migrate_type_count))
+               wake_up(&rcu_migrate_wq);
+}
+
+static inline void wait_migrated_callbacks(void)
+{
+       wait_event(rcu_migrate_wq, !atomic_read(&rcu_migrate_type_count));
+}
+
+static int __cpuinit rcu_barrier_cpu_hotplug(struct notifier_block *self,
+               unsigned long action, void *hcpu)
+{
+       if (action == CPU_DYING) {
+               /*
+                * preempt_disable() in on_each_cpu() prevents stop_machine(),
+                * so when "on_each_cpu(rcu_barrier_func, (void *)type, 1);"
+                * returns, all online cpus have queued rcu_barrier_func(),
+                * and the dead cpu(if it exist) queues rcu_migrate_callback()s.
+                *
+                * These callbacks ensure _rcu_barrier() waits for all
+                * RCU callbacks of the specified type to complete.
+                */
+               atomic_set(&rcu_migrate_type_count, 3);
+               call_rcu_bh(rcu_migrate_head, rcu_migrate_callback);
+               call_rcu_sched(rcu_migrate_head + 1, rcu_migrate_callback);
+               call_rcu(rcu_migrate_head + 2, rcu_migrate_callback);
+       } else if (action == CPU_POST_DEAD) {
+               /* rcu_migrate_head is protected by cpu_add_remove_lock */
+               wait_migrated_callbacks();
+       }
+
+       return NOTIFY_OK;
+}
+
 void __init rcu_init(void)
 {
        __rcu_init();
+       hotcpu_notifier(rcu_barrier_cpu_hotplug, 0);
 }
 
+void rcu_scheduler_starting(void)
+{
+       WARN_ON(num_online_cpus() != 1);
+       WARN_ON(nr_context_switches() > 0);
+       rcu_scheduler_active = 1;
+}