futex: Add bitset conditional wait/wakeup functionality

To allow the implementation of optimized rw-locks in user space, glibc
needs a possibility to select waiters for wakeup depending on a bitset
mask.

This requires two new futex OPs: FUTEX_WAIT_BITS and FUTEX_WAKE_BITS
These OPs are basically the same as FUTEX_WAIT and FUTEX_WAKE plus an
additional argument - a bitset. Further the FUTEX_WAIT_BITS OP is
expecting an absolute timeout value instead of the relative one, which
is used for the FUTEX_WAIT OP.

FUTEX_WAIT_BITS calls into the kernel with a bitset. The bitset is
stored in the futex_q structure, which is used to enqueue the waiter
into the hashed futex waitqueue.

FUTEX_WAKE_BITS also calls into the kernel with a bitset. The wakeup
function logically ANDs the bitset with the bitset stored in each
waiters futex_q structure. If the result is zero (i.e. none of the set
bits in the bitsets is matching), then the waiter is not woken up. If
the result is not zero (i.e. one of the set bits in the bitsets is
matching), then the waiter is woken.

The bitset provided by the caller must be non zero. In case the
provided bitset is zero the kernel returns EINVAL.

Internaly the new OPs are only extensions to the existing FUTEX_WAIT
and FUTEX_WAKE functions. The existing OPs hand a bitset with all bits
set into the futex_wait() and futex_wake() functions.

Signed-off-by: Thomas Gleixner <tgxl@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/include/linux/futex.h b/include/linux/futex.h
index 1a15f8e237a7cf2404b1aa5bf16ccbb4f52214b4..90048fb28a38041a8a3975a59a30502f0b5a23bd 100644 (file)
--- a/include/linux/futex.h
+++ b/include/linux/futex.h
@@ -21,6 +21,8 @@ union ktime;
 #define FUTEX_LOCK_PI          6
 #define FUTEX_UNLOCK_PI                7
 #define FUTEX_TRYLOCK_PI       8
+#define FUTEX_WAIT_BITSET      9
+#define FUTEX_WAKE_BITSET      10
 
 #define FUTEX_PRIVATE_FLAG     128
 #define FUTEX_CMD_MASK         ~FUTEX_PRIVATE_FLAG
@@ -33,6 +35,8 @@ union ktime;
 #define FUTEX_LOCK_PI_PRIVATE  (FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG)
 #define FUTEX_UNLOCK_PI_PRIVATE        (FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG)
 #define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAIT_BITSET_PRIVATE      (FUTEX_WAIT_BITS | FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAKE_BITSET_PRIVATE      (FUTEX_WAKE_BITS | FUTEX_PRIVATE_FLAG)
 
 /*
  * Support for robust futexes: the kernel cleans up held futexes at
@@ -111,6 +115,12 @@ struct robust_list_head {
  */
 #define ROBUST_LIST_LIMIT      2048
 
+/*
+ * bitset with all bits set for the FUTEX_xxx_BITSET OPs to request a
+ * match of any bit.
+ */
+#define FUTEX_BITSET_MATCH_ANY 0xffffffff
+
 #ifdef __KERNEL__
 long do_futex(u32 __user *uaddr, int op, u32 val, union ktime *timeout,
              u32 __user *uaddr2, u32 val2, u32 val3);

diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h
index dfbdfb9836f469ff78184c9b99825a569953c1fd..421323e5a2d6ce3e6d82b09cacf308e59c66bac7 100644 (file)
--- a/include/linux/thread_info.h
+++ b/include/linux/thread_info.h
@@ -23,6 +23,7 @@ struct restart_block {
                        u32 *uaddr;
                        u32 val;
                        u32 flags;
+                       u32 bitset;
                        u64 time;
                } futex;
        };

diff --git a/kernel/futex.c b/kernel/futex.c
index 0006d64c448e6267ce791e8a0599bc796f0b7092..a6baaec44b8f89009bafe2597fa90a6697702afa 100644 (file)
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -109,6 +109,9 @@ struct futex_q {
        /* Optional priority inheritance state: */
        struct futex_pi_state *pi_state;
        struct task_struct *task;
+
+       /* Bitset for the optional bitmasked wakeup */
+       u32 bitset;
 };
 
 /*
@@ -722,7 +725,7 @@ double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
  * to this virtual address:
  */
 static int futex_wake(u32 __user *uaddr, struct rw_semaphore *fshared,
-                     int nr_wake)
+                     int nr_wake, u32 bitset)
 {
        struct futex_hash_bucket *hb;
        struct futex_q *this, *next;
@@ -730,6 +733,9 @@ static int futex_wake(u32 __user *uaddr, struct rw_semaphore *fshared,
        union futex_key key;
        int ret;
 
+       if (!bitset)
+               return -EINVAL;
+
        futex_lock_mm(fshared);
 
        ret = get_futex_key(uaddr, fshared, &key);
@@ -746,6 +752,11 @@ static int futex_wake(u32 __user *uaddr, struct rw_semaphore *fshared,
                                ret = -EINVAL;
                                break;
                        }
+
+                       /* Check if one of the bits is set in both bitsets */
+                       if (!(this->bitset & bitset))
+                               continue;
+
                        wake_futex(this);
                        if (++ret >= nr_wake)
                                break;
@@ -1156,7 +1167,7 @@ static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
 static long futex_wait_restart(struct restart_block *restart);
 
 static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
-                     u32 val, ktime_t *abs_time)
+                     u32 val, ktime_t *abs_time, u32 bitset)
 {
        struct task_struct *curr = current;
        DECLARE_WAITQUEUE(wait, curr);
@@ -1167,7 +1178,11 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
        struct hrtimer_sleeper t;
        int rem = 0;
 
+       if (!bitset)
+               return -EINVAL;
+
        q.pi_state = NULL;
+       q.bitset = bitset;
  retry:
        futex_lock_mm(fshared);
 
@@ -1295,6 +1310,7 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
                restart->futex.uaddr = (u32 *)uaddr;
                restart->futex.val = val;
                restart->futex.time = abs_time->tv64;
+               restart->futex.bitset = bitset;
                restart->futex.flags = 0;
 
                if (fshared)
@@ -1321,7 +1337,8 @@ static long futex_wait_restart(struct restart_block *restart)
        restart->fn = do_no_restart_syscall;
        if (restart->futex.flags & FLAGS_SHARED)
                fshared = &current->mm->mmap_sem;
-       return (long)futex_wait(uaddr, fshared, restart->futex.val, &t);
+       return (long)futex_wait(uaddr, fshared, restart->futex.val, &t,
+                               restart->futex.bitset);
 }
 
 
@@ -1942,7 +1959,8 @@ retry:
                 * PI futexes happens in exit_pi_state():
                 */
                if (!pi && (uval & FUTEX_WAITERS))
-                               futex_wake(uaddr, &curr->mm->mmap_sem, 1);
+                       futex_wake(uaddr, &curr->mm->mmap_sem, 1,
+                                  FUTEX_BITSET_MATCH_ANY);
        }
        return 0;
 }
@@ -2042,10 +2060,14 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
 
        switch (cmd) {
        case FUTEX_WAIT:
-               ret = futex_wait(uaddr, fshared, val, timeout);
+               val3 = FUTEX_BITSET_MATCH_ANY;
+       case FUTEX_WAIT_BITSET:
+               ret = futex_wait(uaddr, fshared, val, timeout, val3);
                break;
        case FUTEX_WAKE:
-               ret = futex_wake(uaddr, fshared, val);
+               val3 = FUTEX_BITSET_MATCH_ANY;
+       case FUTEX_WAKE_BITSET:
+               ret = futex_wake(uaddr, fshared, val, val3);
                break;
        case FUTEX_FD:
                /* non-zero val means F_SETOWN(getpid()) & F_SETSIG(val) */
@@ -2085,7 +2107,8 @@ asmlinkage long sys_futex(u32 __user *uaddr, int op, u32 val,
        u32 val2 = 0;
        int cmd = op & FUTEX_CMD_MASK;
 
-       if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI)) {
+       if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
+                     cmd == FUTEX_WAIT_BITSET)) {
                if (copy_from_user(&ts, utime, sizeof(ts)) != 0)
                        return -EFAULT;
                if (!timespec_valid(&ts))

diff --git a/kernel/futex_compat.c b/kernel/futex_compat.c
index 0a43def6fee7de877f43bdd0d7276efc05b8a48e..133d558db452e38b68c664e9560457f7c3902fc9 100644 (file)
--- a/kernel/futex_compat.c
+++ b/kernel/futex_compat.c
@@ -167,7 +167,8 @@ asmlinkage long compat_sys_futex(u32 __user *uaddr, int op, u32 val,
        int val2 = 0;
        int cmd = op & FUTEX_CMD_MASK;
 
-       if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI)) {
+       if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
+                     cmd == FUTEX_WAIT_BITSET)) {
                if (get_compat_timespec(&ts, utime))
                        return -EFAULT;
                if (!timespec_valid(&ts))