sched: sched_clock_idle_[sleep|wakeup]_event()

construct a more or less wall-clock time out of sched_clock(), by
using ACPI-idle's existing knowledge about how much time we spent
idling. This allows the rq clock to work around TSC-stops-in-C2,
TSC-gets-corrupted-in-C3 type of problems.

( Besides the scheduler's statistics this also benefits blktrace and
  printk-timestamps as well. )

Furthermore, the precise before-C2/C3-sleep and after-C2/C3-wakeup
callbacks allow the scheduler to get out the most of the period where
the CPU has a reliable TSC. This results in slightly more precise
task statistics.

the ACPI bits were acked by Len.

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Acked-by: Len Brown <len.brown@intel.com>

diff --git a/arch/i386/kernel/tsc.c b/arch/i386/kernel/tsc.c
index debd7dbb4158bc9df0aab0a7f9211922b368ba3b..a39280b4dd3a53f107a23ac85df1b01d5ac6b7c4 100644 (file)
--- a/arch/i386/kernel/tsc.c
+++ b/arch/i386/kernel/tsc.c
@@ -292,7 +292,6 @@ static struct clocksource clocksource_tsc = {
 
 void mark_tsc_unstable(char *reason)
 {
-       sched_clock_unstable_event();
        if (!tsc_unstable) {
                tsc_unstable = 1;
                tsc_enabled = 0;

diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index a8634a0655fca429b0a52f3c43b8d57a8ccc3095..d9b8af763e1ec59e64671655fda51e0601b55da9 100644 (file)
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -63,6 +63,7 @@
 ACPI_MODULE_NAME("processor_idle");
 #define ACPI_PROCESSOR_FILE_POWER      "power"
 #define US_TO_PM_TIMER_TICKS(t)                ((t * (PM_TIMER_FREQUENCY/1000)) / 1000)
+#define PM_TIMER_TICK_NS               (1000000000ULL/PM_TIMER_FREQUENCY)
 #define C2_OVERHEAD                    4       /* 1us (3.579 ticks per us) */
 #define C3_OVERHEAD                    4       /* 1us (3.579 ticks per us) */
 static void (*pm_idle_save) (void) __read_mostly;
@@ -462,6 +463,9 @@ static void acpi_processor_idle(void)
                 * TBD: Can't get time duration while in C1, as resumes
                 *      go to an ISR rather than here.  Need to instrument
                 *      base interrupt handler.
+                *
+                * Note: the TSC better not stop in C1, sched_clock() will
+                *       skew otherwise.
                 */
                sleep_ticks = 0xFFFFFFFF;
                break;
@@ -469,6 +473,8 @@ static void acpi_processor_idle(void)
        case ACPI_STATE_C2:
                /* Get start time (ticks) */
                t1 = inl(acpi_gbl_FADT.xpm_timer_block.address);
+               /* Tell the scheduler that we are going deep-idle: */
+               sched_clock_idle_sleep_event();
                /* Invoke C2 */
                acpi_state_timer_broadcast(pr, cx, 1);
                acpi_cstate_enter(cx);
@@ -479,17 +485,22 @@ static void acpi_processor_idle(void)
                /* TSC halts in C2, so notify users */
                mark_tsc_unstable("possible TSC halt in C2");
 #endif
+               /* Compute time (ticks) that we were actually asleep */
+               sleep_ticks = ticks_elapsed(t1, t2);
+
+               /* Tell the scheduler how much we idled: */
+               sched_clock_idle_wakeup_event(sleep_ticks*PM_TIMER_TICK_NS);
+
                /* Re-enable interrupts */
                local_irq_enable();
+               /* Do not account our idle-switching overhead: */
+               sleep_ticks -= cx->latency_ticks + C2_OVERHEAD;
+
                current_thread_info()->status |= TS_POLLING;
-               /* Compute time (ticks) that we were actually asleep */
-               sleep_ticks =
-                   ticks_elapsed(t1, t2) - cx->latency_ticks - C2_OVERHEAD;
                acpi_state_timer_broadcast(pr, cx, 0);
                break;
 
        case ACPI_STATE_C3:
-
                /*
                 * disable bus master
                 * bm_check implies we need ARB_DIS
@@ -518,6 +529,8 @@ static void acpi_processor_idle(void)
                t1 = inl(acpi_gbl_FADT.xpm_timer_block.address);
                /* Invoke C3 */
                acpi_state_timer_broadcast(pr, cx, 1);
+               /* Tell the scheduler that we are going deep-idle: */
+               sched_clock_idle_sleep_event();
                acpi_cstate_enter(cx);
                /* Get end time (ticks) */
                t2 = inl(acpi_gbl_FADT.xpm_timer_block.address);
@@ -531,12 +544,17 @@ static void acpi_processor_idle(void)
                /* TSC halts in C3, so notify users */
                mark_tsc_unstable("TSC halts in C3");
 #endif
+               /* Compute time (ticks) that we were actually asleep */
+               sleep_ticks = ticks_elapsed(t1, t2);
+               /* Tell the scheduler how much we idled: */
+               sched_clock_idle_wakeup_event(sleep_ticks*PM_TIMER_TICK_NS);
+
                /* Re-enable interrupts */
                local_irq_enable();
+               /* Do not account our idle-switching overhead: */
+               sleep_ticks -= cx->latency_ticks + C3_OVERHEAD;
+
                current_thread_info()->status |= TS_POLLING;
-               /* Compute time (ticks) that we were actually asleep */
-               sleep_ticks =
-                   ticks_elapsed(t1, t2) - cx->latency_ticks - C3_OVERHEAD;
                acpi_state_timer_broadcast(pr, cx, 0);
                break;
 

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 682ef87da6eb953348986927598eeaf8b2264ec4..1845b2e99a871eca1c2bf439468102d4f5f2fecc 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1388,7 +1388,8 @@ extern void sched_exec(void);
 #define sched_exec()   {}
 #endif
 
-extern void sched_clock_unstable_event(void);
+extern void sched_clock_idle_sleep_event(void);
+extern void sched_clock_idle_wakeup_event(u64 delta_ns);
 
 #ifdef CONFIG_HOTPLUG_CPU
 extern void idle_task_exit(void);

diff --git a/kernel/sched.c b/kernel/sched.c
index 45e17b83b7f10ba8a99bf04e3a3bbd2571a345f3..48e7586168ef45635c1244ffc7dc4c07a213f65c 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -262,7 +262,8 @@ struct rq {
        s64 clock_max_delta;
 
        unsigned int clock_warps, clock_overflows;
-       unsigned int clock_unstable_events;
+       u64 idle_clock;
+       unsigned int clock_deep_idle_events;
        u64 tick_timestamp;
 
        atomic_t nr_iowait;
@@ -556,18 +557,40 @@ static inline struct rq *this_rq_lock(void)
 }
 
 /*
- * CPU frequency is/was unstable - start new by setting prev_clock_raw:
+ * We are going deep-idle (irqs are disabled):
  */
-void sched_clock_unstable_event(void)
+void sched_clock_idle_sleep_event(void)
 {
-       unsigned long flags;
-       struct rq *rq;
+       struct rq *rq = cpu_rq(smp_processor_id());
 
-       rq = task_rq_lock(current, &flags);
-       rq->prev_clock_raw = sched_clock();
-       rq->clock_unstable_events++;
-       task_rq_unlock(rq, &flags);
+       spin_lock(&rq->lock);
+       __update_rq_clock(rq);
+       spin_unlock(&rq->lock);
+       rq->clock_deep_idle_events++;
+}
+EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event);
+
+/*
+ * We just idled delta nanoseconds (called with irqs disabled):
+ */
+void sched_clock_idle_wakeup_event(u64 delta_ns)
+{
+       struct rq *rq = cpu_rq(smp_processor_id());
+       u64 now = sched_clock();
+
+       rq->idle_clock += delta_ns;
+       /*
+        * Override the previous timestamp and ignore all
+        * sched_clock() deltas that occured while we idled,
+        * and use the PM-provided delta_ns to advance the
+        * rq clock:
+        */
+       spin_lock(&rq->lock);
+       rq->prev_clock_raw = now;
+       rq->clock += delta_ns;
+       spin_unlock(&rq->lock);
 }
+EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
 
 /*
  * resched_task - mark a task 'to be rescheduled now'.

diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c
index 87e524762b858725207b540c2f4e51a2f230fde3..ab18f45f2ab23f6b76f6d8bf9a99e170892c9542 100644 (file)
--- a/kernel/sched_debug.c
+++ b/kernel/sched_debug.c
@@ -154,10 +154,11 @@ static void print_cpu(struct seq_file *m, int cpu)
        P(next_balance);
        P(curr->pid);
        P(clock);
+       P(idle_clock);
        P(prev_clock_raw);
        P(clock_warps);
        P(clock_overflows);
-       P(clock_unstable_events);
+       P(clock_deep_idle_events);
        P(clock_max_delta);
        P(cpu_load[0]);
        P(cpu_load[1]);