fix SMP data race in pagetable setup vs walking

[linux-2.6-omap-h63xx.git] / mm / memory.c

diff --git a/mm/memory.c b/mm/memory.c
index 48c122d42ed743dcc90178170b4aff380ec3ad8c..fb5608a120edc77cffac3050ce7e5ebbcff94b8f 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -311,6 +311,21 @@ int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
        if (!new)
                return -ENOMEM;
 
+       /*
+        * Ensure all pte setup (eg. pte page lock and page clearing) are
+        * visible before the pte is made visible to other CPUs by being
+        * put into page tables.
+        *
+        * The other side of the story is the pointer chasing in the page
+        * table walking code (when walking the page table without locking;
+        * ie. most of the time). Fortunately, these data accesses consist
+        * of a chain of data-dependent loads, meaning most CPUs (alpha
+        * being the notable exception) will already guarantee loads are
+        * seen in-order. See the alpha page table accessors for the
+        * smp_read_barrier_depends() barriers in page table walking code.
+        */
+       smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
+
        spin_lock(&mm->page_table_lock);
        if (!pmd_present(*pmd)) {       /* Has another populated it ? */
                mm->nr_ptes++;
@@ -329,6 +344,8 @@ int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)
        if (!new)
                return -ENOMEM;
 
+       smp_wmb(); /* See comment in __pte_alloc */
+
        spin_lock(&init_mm.page_table_lock);
        if (!pmd_present(*pmd)) {       /* Has another populated it ? */
                pmd_populate_kernel(&init_mm, pmd, new);
@@ -2619,6 +2636,8 @@ int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
        if (!new)
                return -ENOMEM;
 
+       smp_wmb(); /* See comment in __pte_alloc */
+
        spin_lock(&mm->page_table_lock);
        if (pgd_present(*pgd))          /* Another has populated it */
                pud_free(mm, new);
@@ -2640,6 +2659,8 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
        if (!new)
                return -ENOMEM;
 
+       smp_wmb(); /* See comment in __pte_alloc */
+
        spin_lock(&mm->page_table_lock);
 #ifndef __ARCH_HAS_4LEVEL_HACK
        if (pud_present(*pud))          /* Another has populated it */