[PATCH] mm: pagefault_{disable,enable}()

Introduce pagefault_{disable,enable}() and use these where previously we did
manual preempt increments/decrements to make the pagefault handler do the
atomic thing.

Currently they still rely on the increased preempt count, but do not rely on
the disabled preemption, this might go away in the future.

(NOTE: the extra barrier() in pagefault_disable might fix some holes on
       machines which have too many registers for their own good)

[heiko.carstens@de.ibm.com: s390 fix]
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/arch/frv/kernel/futex.c b/arch/frv/kernel/futex.c
index eae874a970c62f6750b0c6d41a56ab76078f7ac0..53dc5ed1ebda65d24b33c359bfaefbc584be5999 100644 (file)
--- a/arch/frv/kernel/futex.c
+++ b/arch/frv/kernel/futex.c
@@ -200,7 +200,7 @@ int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
        if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        switch (op) {
        case FUTEX_OP_SET:
@@ -223,7 +223,7 @@ int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
                break;
        }
 
-       dec_preempt_count();
+       pagefault_enable();
 
        if (!ret) {
                switch (cmp) {

diff --git a/arch/i386/mm/highmem.c b/arch/i386/mm/highmem.c
index f9f647cdbc7ba3550e83d82c0c9e6996b3ae8878..178bbfe6cbac7d9e80e30c28ab60caa53bdd4eed 100644 (file)
--- a/arch/i386/mm/highmem.c
+++ b/arch/i386/mm/highmem.c
@@ -32,7 +32,7 @@ void *kmap_atomic(struct page *page, enum km_type type)
        unsigned long vaddr;
 
        /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */
-       inc_preempt_count();
+       pagefault_disable();
        if (!PageHighMem(page))
                return page_address(page);
 
@@ -52,8 +52,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type)
 
 #ifdef CONFIG_DEBUG_HIGHMEM
        if (vaddr >= PAGE_OFFSET && vaddr < (unsigned long)high_memory) {
-               dec_preempt_count();
-               preempt_check_resched();
+               pagefault_enable();
                return;
        }
 
@@ -68,8 +67,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type)
         */
        kpte_clear_flush(kmap_pte-idx, vaddr);
 
-       dec_preempt_count();
-       preempt_check_resched();
+       pagefault_enable();
 }
 
 /* This is the same as kmap_atomic() but can map memory that doesn't
@@ -80,7 +78,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
        enum fixed_addresses idx;
        unsigned long vaddr;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        idx = type + KM_TYPE_NR*smp_processor_id();
        vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);

diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c
index 99ebf3ccc222b78ef8743b9f0bb5b6cba15d894c..675502ada5a27d5f958e6b58bea8e4219ff77ff1 100644 (file)
--- a/arch/mips/mm/highmem.c
+++ b/arch/mips/mm/highmem.c
@@ -39,7 +39,7 @@ void *__kmap_atomic(struct page *page, enum km_type type)
        unsigned long vaddr;
 
        /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */
-       inc_preempt_count();
+       pagefault_disable();
        if (!PageHighMem(page))
                return page_address(page);
 
@@ -62,8 +62,7 @@ void __kunmap_atomic(void *kvaddr, enum km_type type)
        enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id();
 
        if (vaddr < FIXADDR_START) { // FIXME
-               dec_preempt_count();
-               preempt_check_resched();
+               pagefault_enable();
                return;
        }
 
@@ -78,8 +77,7 @@ void __kunmap_atomic(void *kvaddr, enum km_type type)
        local_flush_tlb_one(vaddr);
 #endif
 
-       dec_preempt_count();
-       preempt_check_resched();
+       pagefault_enable();
 }
 
 #ifndef CONFIG_LIMITED_DMA
@@ -92,7 +90,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
        enum fixed_addresses idx;
        unsigned long vaddr;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        idx = type + KM_TYPE_NR*smp_processor_id();
        vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);

diff --git a/arch/s390/lib/uaccess_std.c b/arch/s390/lib/uaccess_std.c
index 2d549ed2e11399dfe17975fcae72aa512b9f52fe..bbaca66fa29356af1b0c7ea5589622f28aec4519 100644 (file)
--- a/arch/s390/lib/uaccess_std.c
+++ b/arch/s390/lib/uaccess_std.c
@@ -11,7 +11,7 @@
 
 #include <linux/errno.h>
 #include <linux/mm.h>
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
 #include <asm/futex.h>
 
 #ifndef __s390x__
@@ -258,7 +258,7 @@ int futex_atomic_op(int op, int __user *uaddr, int oparg, int *old)
 {
        int oldval = 0, newval, ret;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        switch (op) {
        case FUTEX_OP_SET:
@@ -284,7 +284,7 @@ int futex_atomic_op(int op, int __user *uaddr, int oparg, int *old)
        default:
                ret = -ENOSYS;
        }
-       dec_preempt_count();
+       pagefault_enable();
        *old = oldval;
        return ret;
 }

diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c
index 4d8ed9c651823ae82b3d30c6a5d947eb422a18b9..01fc6c25429255c27be0e6a94e36179e2693b86d 100644 (file)
--- a/arch/sparc/mm/highmem.c
+++ b/arch/sparc/mm/highmem.c
@@ -35,7 +35,7 @@ void *kmap_atomic(struct page *page, enum km_type type)
        unsigned long vaddr;
 
        /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */
-       inc_preempt_count();
+       pagefault_disable();
        if (!PageHighMem(page))
                return page_address(page);
 
@@ -70,8 +70,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type)
        unsigned long idx = type + KM_TYPE_NR*smp_processor_id();
 
        if (vaddr < FIXADDR_START) { // FIXME
-               dec_preempt_count();
-               preempt_check_resched();
+               pagefault_enable();
                return;
        }
 
@@ -97,8 +96,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type)
 #endif
 #endif
 
-       dec_preempt_count();
-       preempt_check_resched();
+       pagefault_enable();
 }
 
 /* We may be fed a pagetable here by ptep_to_xxx and others. */

diff --git a/include/asm-frv/highmem.h b/include/asm-frv/highmem.h
index 0f390f41f81680a70c12fe26cdae314644466344..ff4d6cdeb1522811bd4131da61d68575ff55502b 100644 (file)
--- a/include/asm-frv/highmem.h
+++ b/include/asm-frv/highmem.h
@@ -115,7 +115,7 @@ static inline void *kmap_atomic(struct page *page, enum km_type type)
 {
        unsigned long paddr;
 
-       inc_preempt_count();
+       pagefault_disable();
        paddr = page_to_phys(page);
 
        switch (type) {
@@ -170,8 +170,7 @@ static inline void kunmap_atomic(void *kvaddr, enum km_type type)
        default:
                BUG();
        }
-       dec_preempt_count();
-       preempt_check_resched();
+       pagefault_enable();
 }
 
 #endif /* !__ASSEMBLY__ */

diff --git a/include/asm-generic/futex.h b/include/asm-generic/futex.h
index df893c160318a6ac84ae076bc7ac5b98c33722ac..f422df0956a27025deb4c6cdbbc5a47646f74c9e 100644 (file)
--- a/include/asm-generic/futex.h
+++ b/include/asm-generic/futex.h
@@ -21,7 +21,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
        if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        switch (op) {
        case FUTEX_OP_SET:
@@ -33,7 +33,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                ret = -ENOSYS;
        }
 
-       dec_preempt_count();
+       pagefault_enable();
 
        if (!ret) {
                switch (cmp) {

diff --git a/include/asm-i386/futex.h b/include/asm-i386/futex.h
index 946d97cfea23841ccbf919fe7cba1df0fb7abe45..438ef0ec7101c2f4b2fcf4fb711f99e81494712e 100644 (file)
--- a/include/asm-i386/futex.h
+++ b/include/asm-i386/futex.h
@@ -56,7 +56,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
        if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        if (op == FUTEX_OP_SET)
                __futex_atomic_op1("xchgl %0, %2", ret, oldval, uaddr, oparg);
@@ -88,7 +88,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                }
        }
 
-       dec_preempt_count();
+       pagefault_enable();
 
        if (!ret) {
                switch (cmp) {

diff --git a/include/asm-ia64/futex.h b/include/asm-ia64/futex.h
index 07d77f3a8cbe278bbad15d99e80442b86cbd7d83..8a98a26541391ed17cbcebf97b0b98ef5a1efa31 100644 (file)
--- a/include/asm-ia64/futex.h
+++ b/include/asm-ia64/futex.h
@@ -59,7 +59,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
        if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        switch (op) {
        case FUTEX_OP_SET:
@@ -83,7 +83,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                ret = -ENOSYS;
        }
 
-       dec_preempt_count();
+       pagefault_enable();
 
        if (!ret) {
                switch (cmp) {

diff --git a/include/asm-mips/futex.h b/include/asm-mips/futex.h
index 927a216bd530d2f07cc6c5392089b666935f461b..47e5679c235303f5812eb1d4cd39a2140af89016 100644 (file)
--- a/include/asm-mips/futex.h
+++ b/include/asm-mips/futex.h
@@ -88,7 +88,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
        if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        switch (op) {
        case FUTEX_OP_SET:
@@ -115,7 +115,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                ret = -ENOSYS;
        }
 
-       dec_preempt_count();
+       pagefault_enable();
 
        if (!ret) {
                switch (cmp) {

diff --git a/include/asm-parisc/futex.h b/include/asm-parisc/futex.h
index d84bbb283fd17c7a8e9fb92d4a30a660efe86263..dbee6e60aa8139e66897ba65ba4ac9155bf00dab 100644 (file)
--- a/include/asm-parisc/futex.h
+++ b/include/asm-parisc/futex.h
@@ -21,7 +21,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
        if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        switch (op) {
        case FUTEX_OP_SET:
@@ -33,7 +33,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                ret = -ENOSYS;
        }
 
-       dec_preempt_count();
+       pagefault_enable();
 
        if (!ret) {
                switch (cmp) {

diff --git a/include/asm-powerpc/futex.h b/include/asm-powerpc/futex.h
index 936422e54891f0cab44229639f8321e9d47a270e..3f3673fd3ff34abef169c139c71302084b2cc874 100644 (file)
--- a/include/asm-powerpc/futex.h
+++ b/include/asm-powerpc/futex.h
@@ -43,7 +43,7 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
        if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        switch (op) {
        case FUTEX_OP_SET:
@@ -65,7 +65,7 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                ret = -ENOSYS;
        }
 
-       dec_preempt_count();
+       pagefault_enable();
 
        if (!ret) {
                switch (cmp) {

diff --git a/include/asm-ppc/highmem.h b/include/asm-ppc/highmem.h
index 1d2c4ef81c2233673fa64e8569e78b08ed923121..f7b21ee302b4f1b7fcd1e923fa14c3da0e708f6e 100644 (file)
--- a/include/asm-ppc/highmem.h
+++ b/include/asm-ppc/highmem.h
@@ -79,7 +79,7 @@ static inline void *kmap_atomic(struct page *page, enum km_type type)
        unsigned long vaddr;
 
        /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */
-       inc_preempt_count();
+       pagefault_disable();
        if (!PageHighMem(page))
                return page_address(page);
 
@@ -101,8 +101,7 @@ static inline void kunmap_atomic(void *kvaddr, enum km_type type)
        unsigned int idx = type + KM_TYPE_NR*smp_processor_id();
 
        if (vaddr < KMAP_FIX_BEGIN) { // FIXME
-               dec_preempt_count();
-               preempt_check_resched();
+               pagefault_enable();
                return;
        }
 
@@ -115,8 +114,7 @@ static inline void kunmap_atomic(void *kvaddr, enum km_type type)
        pte_clear(&init_mm, vaddr, kmap_pte+idx);
        flush_tlb_page(NULL, vaddr);
 #endif
-       dec_preempt_count();
-       preempt_check_resched();
+       pagefault_enable();
 }
 
 static inline struct page *kmap_atomic_to_page(void *ptr)

diff --git a/include/asm-sparc64/futex.h b/include/asm-sparc64/futex.h
index 7392fc4a954e2d603805c319071bc1cbb89e2d19..876312fe82ccf9dbf894a100a26978a5f8267f10 100644 (file)
--- a/include/asm-sparc64/futex.h
+++ b/include/asm-sparc64/futex.h
@@ -45,7 +45,7 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
        if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
                oparg = 1 << oparg;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        switch (op) {
        case FUTEX_OP_SET:
@@ -67,7 +67,7 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
                ret = -ENOSYS;
        }
 
-       dec_preempt_count();
+       pagefault_enable();
 
        if (!ret) {
                switch (cmp) {

diff --git a/include/asm-x86_64/futex.h b/include/asm-x86_64/futex.h
index 9804bf07b092f4bb154f19284047a9a9711bc989..5cdfb08013c38889dc8a42a2dff4c4a3c1833ff8 100644 (file)
--- a/include/asm-x86_64/futex.h
+++ b/include/asm-x86_64/futex.h
@@ -55,7 +55,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
        if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       inc_preempt_count();
+       pagefault_disable();
 
        switch (op) {
        case FUTEX_OP_SET:
@@ -78,7 +78,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                ret = -ENOSYS;
        }
 
-       dec_preempt_count();
+       pagefault_enable();
 
        if (!ret) {
                switch (cmp) {

diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index a48d7f11c7be06772327604dc40c3f91b0b57145..67918c22339c5334836ed102e3be0e3ffd5695ed 100644 (file)
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -1,8 +1,43 @@
 #ifndef __LINUX_UACCESS_H__
 #define __LINUX_UACCESS_H__
 
+#include <linux/preempt.h>
 #include <asm/uaccess.h>
 
+/*
+ * These routines enable/disable the pagefault handler in that
+ * it will not take any locks and go straight to the fixup table.
+ *
+ * They have great resemblance to the preempt_disable/enable calls
+ * and in fact they are identical; this is because currently there is
+ * no other way to make the pagefault handlers do this. So we do
+ * disable preemption but we don't necessarily care about that.
+ */
+static inline void pagefault_disable(void)
+{
+       inc_preempt_count();
+       /*
+        * make sure to have issued the store before a pagefault
+        * can hit.
+        */
+       barrier();
+}
+
+static inline void pagefault_enable(void)
+{
+       /*
+        * make sure to issue those last loads/stores before enabling
+        * the pagefault handler again.
+        */
+       barrier();
+       dec_preempt_count();
+       /*
+        * make sure we do..
+        */
+       barrier();
+       preempt_check_resched();
+}
+
 #ifndef ARCH_HAS_NOCACHE_UACCESS
 
 static inline unsigned long __copy_from_user_inatomic_nocache(void *to,
@@ -35,9 +70,9 @@ static inline unsigned long __copy_from_user_nocache(void *to,
        ({                                              \
                long ret;                               \
                                                        \
-               inc_preempt_count();                    \
+               pagefault_disable();                    \
                ret = __get_user(retval, addr);         \
-               dec_preempt_count();                    \
+               pagefault_enable();                     \
                ret;                                    \
        })
 

diff --git a/kernel/futex.c b/kernel/futex.c
index 93ef30ba209fc865c7507459f0995849d5d6e336..af7b81cbde30391285ffa77beb4716c7f868ba21 100644 (file)
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -282,9 +282,9 @@ static inline int get_futex_value_locked(u32 *dest, u32 __user *from)
 {
        int ret;
 
-       inc_preempt_count();
+       pagefault_disable();
        ret = __copy_from_user_inatomic(dest, from, sizeof(u32));
-       dec_preempt_count();
+       pagefault_enable();
 
        return ret ? -EFAULT : 0;
 }
@@ -585,9 +585,9 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
        if (!(uval & FUTEX_OWNER_DIED)) {
                newval = FUTEX_WAITERS | new_owner->pid;
 
-               inc_preempt_count();
+               pagefault_disable();
                curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
-               dec_preempt_count();
+               pagefault_enable();
                if (curval == -EFAULT)
                        return -EFAULT;
                if (curval != uval)
@@ -618,9 +618,9 @@ static int unlock_futex_pi(u32 __user *uaddr, u32 uval)
         * There is no waiter, so we unlock the futex. The owner died
         * bit has not to be preserved here. We are the owner:
         */
-       inc_preempt_count();
+       pagefault_disable();
        oldval = futex_atomic_cmpxchg_inatomic(uaddr, uval, 0);
-       dec_preempt_count();
+       pagefault_enable();
 
        if (oldval == -EFAULT)
                return oldval;
@@ -1158,9 +1158,9 @@ static int futex_lock_pi(u32 __user *uaddr, int detect, unsigned long sec,
         */
        newval = current->pid;
 
-       inc_preempt_count();
+       pagefault_disable();
        curval = futex_atomic_cmpxchg_inatomic(uaddr, 0, newval);
-       dec_preempt_count();
+       pagefault_enable();
 
        if (unlikely(curval == -EFAULT))
                goto uaddr_faulted;
@@ -1183,9 +1183,9 @@ static int futex_lock_pi(u32 __user *uaddr, int detect, unsigned long sec,
        uval = curval;
        newval = uval | FUTEX_WAITERS;
 
-       inc_preempt_count();
+       pagefault_disable();
        curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
-       dec_preempt_count();
+       pagefault_enable();
 
        if (unlikely(curval == -EFAULT))
                goto uaddr_faulted;
@@ -1215,10 +1215,10 @@ static int futex_lock_pi(u32 __user *uaddr, int detect, unsigned long sec,
                        newval = current->pid |
                                FUTEX_OWNER_DIED | FUTEX_WAITERS;
 
-                       inc_preempt_count();
+                       pagefault_disable();
                        curval = futex_atomic_cmpxchg_inatomic(uaddr,
                                                               uval, newval);
-                       dec_preempt_count();
+                       pagefault_enable();
 
                        if (unlikely(curval == -EFAULT))
                                goto uaddr_faulted;
@@ -1390,9 +1390,9 @@ retry_locked:
         * anyone else up:
         */
        if (!(uval & FUTEX_OWNER_DIED)) {
-               inc_preempt_count();
+               pagefault_disable();
                uval = futex_atomic_cmpxchg_inatomic(uaddr, current->pid, 0);
-               dec_preempt_count();
+               pagefault_enable();
        }
 
        if (unlikely(uval == -EFAULT))