]> www.pilppa.org Git - linux-2.6-omap-h63xx.git/blobdiff - kernel/hrtimer.c
hrtimer: make the nanosleep() syscall use the per process slack
[linux-2.6-omap-h63xx.git] / kernel / hrtimer.c
index ab80515008f4740fd980b1089d922811978ccff4..9a4c901855668fd0daecdd398456a01f348d32ca 100644 (file)
@@ -300,11 +300,10 @@ EXPORT_SYMBOL_GPL(ktime_sub_ns);
  */
 u64 ktime_divns(const ktime_t kt, s64 div)
 {
-       u64 dclc, inc, dns;
+       u64 dclc;
        int sft = 0;
 
-       dclc = dns = ktime_to_ns(kt);
-       inc = div;
+       dclc = ktime_to_ns(kt);
        /* Make sure the divisor is less than 2^32: */
        while (div >> 32) {
                sft++;
@@ -518,7 +517,7 @@ static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base)
                if (!base->first)
                        continue;
                timer = rb_entry(base->first, struct hrtimer, node);
-               expires = ktime_sub(timer->expires, base->offset);
+               expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
                if (expires.tv64 < cpu_base->expires_next.tv64)
                        cpu_base->expires_next = expires;
        }
@@ -540,10 +539,10 @@ static int hrtimer_reprogram(struct hrtimer *timer,
                             struct hrtimer_clock_base *base)
 {
        ktime_t *expires_next = &__get_cpu_var(hrtimer_bases).expires_next;
-       ktime_t expires = ktime_sub(timer->expires, base->offset);
+       ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
        int res;
 
-       WARN_ON_ONCE(timer->expires.tv64 < 0);
+       WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
 
        /*
         * When the callback is running, we do not reprogram the clock event
@@ -623,7 +622,7 @@ static void retrigger_next_event(void *arg)
 void clock_was_set(void)
 {
        /* Retrigger the CPU local events everywhere */
-       on_each_cpu(retrigger_next_event, NULL, 0, 1);
+       on_each_cpu(retrigger_next_event, NULL, 1);
 }
 
 /*
@@ -632,8 +631,6 @@ void clock_was_set(void)
  */
 void hres_timers_resume(void)
 {
-       WARN_ON_ONCE(num_online_cpus() > 1);
-
        /* Retrigger the CPU local events: */
        retrigger_next_event(NULL);
 }
@@ -797,7 +794,7 @@ u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
        u64 orun = 1;
        ktime_t delta;
 
-       delta = ktime_sub(now, timer->expires);
+       delta = ktime_sub(now, hrtimer_get_expires(timer));
 
        if (delta.tv64 < 0)
                return 0;
@@ -809,8 +806,8 @@ u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
                s64 incr = ktime_to_ns(interval);
 
                orun = ktime_divns(delta, incr);
-               timer->expires = ktime_add_ns(timer->expires, incr * orun);
-               if (timer->expires.tv64 > now.tv64)
+               hrtimer_add_expires_ns(timer, incr * orun);
+               if (hrtimer_get_expires_tv64(timer) > now.tv64)
                        return orun;
                /*
                 * This (and the ktime_add() below) is the
@@ -818,7 +815,7 @@ u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
                 */
                orun++;
        }
-       timer->expires = ktime_add_safe(timer->expires, interval);
+       hrtimer_add_expires(timer, interval);
 
        return orun;
 }
@@ -850,7 +847,8 @@ static void enqueue_hrtimer(struct hrtimer *timer,
                 * We dont care about collisions. Nodes with
                 * the same expiry time stay together.
                 */
-               if (timer->expires.tv64 < entry->expires.tv64) {
+               if (hrtimer_get_expires_tv64(timer) <
+                               hrtimer_get_expires_tv64(entry)) {
                        link = &(*link)->rb_left;
                } else {
                        link = &(*link)->rb_right;
@@ -947,9 +945,10 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)
 }
 
 /**
- * hrtimer_start - (re)start an relative timer on the current CPU
+ * hrtimer_start_range_ns - (re)start an relative timer on the current CPU
  * @timer:     the timer to be added
  * @tim:       expiry time
+ * @delta_ns:  "slack" range for the timer
  * @mode:      expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
  *
  * Returns:
@@ -957,7 +956,8 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)
  *  1 when the timer was active
  */
 int
-hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
+hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_ns,
+                       const enum hrtimer_mode mode)
 {
        struct hrtimer_clock_base *base, *new_base;
        unsigned long flags;
@@ -985,7 +985,7 @@ hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
 #endif
        }
 
-       timer->expires = tim;
+       hrtimer_set_expires_range_ns(timer, tim, delta_ns);
 
        timer_stats_hrtimer_set_start_info(timer);
 
@@ -1018,8 +1018,26 @@ hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
 
        return ret;
 }
+EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
+
+/**
+ * hrtimer_start - (re)start an relative timer on the current CPU
+ * @timer:     the timer to be added
+ * @tim:       expiry time
+ * @mode:      expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
+ *
+ * Returns:
+ *  0 on success
+ *  1 when the timer was active
+ */
+int
+hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
+{
+       return hrtimer_start_range_ns(timer, tim, 0, mode);
+}
 EXPORT_SYMBOL_GPL(hrtimer_start);
 
+
 /**
  * hrtimer_try_to_cancel - try to deactivate a timer
  * @timer:     hrtimer to stop
@@ -1079,14 +1097,14 @@ ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
        ktime_t rem;
 
        base = lock_hrtimer_base(timer, &flags);
-       rem = ktime_sub(timer->expires, base->get_time());
+       rem = hrtimer_expires_remaining(timer);
        unlock_hrtimer_base(timer, &flags);
 
        return rem;
 }
 EXPORT_SYMBOL_GPL(hrtimer_get_remaining);
 
-#if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_NO_HZ)
+#ifdef CONFIG_NO_HZ
 /**
  * hrtimer_get_next_event - get the time until next expiry event
  *
@@ -1111,7 +1129,7 @@ ktime_t hrtimer_get_next_event(void)
                                continue;
 
                        timer = rb_entry(base->first, struct hrtimer, node);
-                       delta.tv64 = timer->expires.tv64;
+                       delta.tv64 = hrtimer_get_expires_tv64(timer);
                        delta = ktime_sub(delta, base->get_time());
                        if (delta.tv64 < mindelta.tv64)
                                mindelta.tv64 = delta.tv64;
@@ -1311,10 +1329,23 @@ void hrtimer_interrupt(struct clock_event_device *dev)
 
                        timer = rb_entry(node, struct hrtimer, node);
 
-                       if (basenow.tv64 < timer->expires.tv64) {
+                       /*
+                        * The immediate goal for using the softexpires is
+                        * minimizing wakeups, not running timers at the
+                        * earliest interrupt after their soft expiration.
+                        * This allows us to avoid using a Priority Search
+                        * Tree, which can answer a stabbing querry for
+                        * overlapping intervals and instead use the simple
+                        * BST we already have.
+                        * We don't add extra wakeups by delaying timers that
+                        * are right-of a not yet expired timer, because that
+                        * timer will have to trigger a wakeup anyway.
+                        */
+
+                       if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) {
                                ktime_t expires;
 
-                               expires = ktime_sub(timer->expires,
+                               expires = ktime_sub(hrtimer_get_expires(timer),
                                                    base->offset);
                                if (expires.tv64 < expires_next.tv64)
                                        expires_next = expires;
@@ -1417,7 +1448,8 @@ void hrtimer_run_queues(void)
                        struct hrtimer *timer;
 
                        timer = rb_entry(node, struct hrtimer, node);
-                       if (base->softirq_time.tv64 <= timer->expires.tv64)
+                       if (base->softirq_time.tv64 <=
+                                       hrtimer_get_expires_tv64(timer))
                                break;
 
                        if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) {
@@ -1465,7 +1497,7 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod
 
        do {
                set_current_state(TASK_INTERRUPTIBLE);
-               hrtimer_start(&t->timer, t->timer.expires, mode);
+               hrtimer_start_expires(&t->timer, mode);
                if (!hrtimer_active(&t->timer))
                        t->task = NULL;
 
@@ -1487,7 +1519,7 @@ static int update_rmtp(struct hrtimer *timer, struct timespec __user *rmtp)
        struct timespec rmt;
        ktime_t rem;
 
-       rem = ktime_sub(timer->expires, timer->base->get_time());
+       rem = hrtimer_expires_remaining(timer);
        if (rem.tv64 <= 0)
                return 0;
        rmt = ktime_to_timespec(rem);
@@ -1506,7 +1538,7 @@ long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
 
        hrtimer_init_on_stack(&t.timer, restart->nanosleep.index,
                                HRTIMER_MODE_ABS);
-       t.timer.expires.tv64 = restart->nanosleep.expires;
+       hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
 
        if (do_nanosleep(&t, HRTIMER_MODE_ABS))
                goto out;
@@ -1531,9 +1563,14 @@ long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
        struct restart_block *restart;
        struct hrtimer_sleeper t;
        int ret = 0;
+       unsigned long slack;
+
+       slack = current->timer_slack_ns;
+       if (rt_task(current))
+               slack = 0;
 
        hrtimer_init_on_stack(&t.timer, clockid, mode);
-       t.timer.expires = timespec_to_ktime(*rqtp);
+       hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack);
        if (do_nanosleep(&t, mode))
                goto out;
 
@@ -1553,7 +1590,7 @@ long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
        restart->fn = hrtimer_nanosleep_restart;
        restart->nanosleep.index = t.timer.base->index;
        restart->nanosleep.rmtp = rmtp;
-       restart->nanosleep.expires = t.timer.expires.tv64;
+       restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
 
        ret = -ERESTART_RESTARTBLOCK;
 out:
@@ -1677,7 +1714,107 @@ void __init hrtimers_init(void)
                          (void *)(long)smp_processor_id());
        register_cpu_notifier(&hrtimers_nb);
 #ifdef CONFIG_HIGH_RES_TIMERS
-       open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq, NULL);
+       open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq);
 #endif
 }
 
+/**
+ * schedule_hrtimeout_range - sleep until timeout
+ * @expires:   timeout value (ktime_t)
+ * @delta:     slack in expires timeout (ktime_t)
+ * @mode:      timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
+ *
+ * Make the current task sleep until the given expiry time has
+ * elapsed. The routine will return immediately unless
+ * the current task state has been set (see set_current_state()).
+ *
+ * The @delta argument gives the kernel the freedom to schedule the
+ * actual wakeup to a time that is both power and performance friendly.
+ * The kernel give the normal best effort behavior for "@expires+@delta",
+ * but may decide to fire the timer earlier, but no earlier than @expires.
+ *
+ * You can set the task state as follows -
+ *
+ * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
+ * pass before the routine returns.
+ *
+ * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
+ * delivered to the current task.
+ *
+ * The current task state is guaranteed to be TASK_RUNNING when this
+ * routine returns.
+ *
+ * Returns 0 when the timer has expired otherwise -EINTR
+ */
+int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta,
+                              const enum hrtimer_mode mode)
+{
+       struct hrtimer_sleeper t;
+
+       /*
+        * Optimize when a zero timeout value is given. It does not
+        * matter whether this is an absolute or a relative time.
+        */
+       if (expires && !expires->tv64) {
+               __set_current_state(TASK_RUNNING);
+               return 0;
+       }
+
+       /*
+        * A NULL parameter means "inifinte"
+        */
+       if (!expires) {
+               schedule();
+               __set_current_state(TASK_RUNNING);
+               return -EINTR;
+       }
+
+       hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, mode);
+       hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
+
+       hrtimer_init_sleeper(&t, current);
+
+       hrtimer_start_expires(&t.timer, mode);
+       if (!hrtimer_active(&t.timer))
+               t.task = NULL;
+
+       if (likely(t.task))
+               schedule();
+
+       hrtimer_cancel(&t.timer);
+       destroy_hrtimer_on_stack(&t.timer);
+
+       __set_current_state(TASK_RUNNING);
+
+       return !t.task ? 0 : -EINTR;
+}
+EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
+
+/**
+ * schedule_hrtimeout - sleep until timeout
+ * @expires:   timeout value (ktime_t)
+ * @mode:      timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
+ *
+ * Make the current task sleep until the given expiry time has
+ * elapsed. The routine will return immediately unless
+ * the current task state has been set (see set_current_state()).
+ *
+ * You can set the task state as follows -
+ *
+ * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
+ * pass before the routine returns.
+ *
+ * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
+ * delivered to the current task.
+ *
+ * The current task state is guaranteed to be TASK_RUNNING when this
+ * routine returns.
+ *
+ * Returns 0 when the timer has expired otherwise -EINTR
+ */
+int __sched schedule_hrtimeout(ktime_t *expires,
+                              const enum hrtimer_mode mode)
+{
+       return schedule_hrtimeout_range(expires, 0, mode);
+}
+EXPORT_SYMBOL_GPL(schedule_hrtimeout);