x86, cpa: make the kernel physical mapping initialization a two pass sequence

[linux-2.6-omap-h63xx.git] / arch / x86 / mm / init_64.c

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index d3746efb060d1602a9de63d3be955522f0cd2a5d..1ba945eb6282b29c99c7adc598711e91263fcaf4 100644 (file)
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -270,6 +270,8 @@ static __ref void unmap_low_page(void *adr)
        early_iounmap(adr, PAGE_SIZE);
 }
 
+static int physical_mapping_iter;
+
 static unsigned long __meminit
 phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end)
 {
@@ -290,16 +292,19 @@ phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end)
                }
 
                if (pte_val(*pte))
-                       continue;
+                       goto repeat_set_pte;
 
                if (0)
                        printk("   pte=%p addr=%lx pte=%016lx\n",
                               pte, addr, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL).pte);
+               pages++;
+repeat_set_pte:
                set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL));
                last_map_addr = (addr & PAGE_MASK) + PAGE_SIZE;
-               pages++;
        }
-       update_page_count(PG_LEVEL_4K, pages);
+
+       if (physical_mapping_iter == 1)
+               update_page_count(PG_LEVEL_4K, pages);
 
        return last_map_addr;
 }
@@ -318,7 +323,6 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
 {
        unsigned long pages = 0;
        unsigned long last_map_addr = end;
-       unsigned long start = address;
 
        int i = pmd_index(address);
 
@@ -341,15 +345,14 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
                                last_map_addr = phys_pte_update(pmd, address,
                                                                end);
                                spin_unlock(&init_mm.page_table_lock);
+                               continue;
                        }
-                       /* Count entries we're using from level2_ident_pgt */
-                       if (start == 0)
-                               pages++;
-                       continue;
+                       goto repeat_set_pte;
                }
 
                if (page_size_mask & (1<<PG_LEVEL_2M)) {
                        pages++;
+repeat_set_pte:
                        spin_lock(&init_mm.page_table_lock);
                        set_pte((pte_t *)pmd,
                                pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL_LARGE));
@@ -366,7 +369,8 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
                pmd_populate_kernel(&init_mm, pmd, __va(pte_phys));
                spin_unlock(&init_mm.page_table_lock);
        }
-       update_page_count(PG_LEVEL_2M, pages);
+       if (physical_mapping_iter == 1)
+               update_page_count(PG_LEVEL_2M, pages);
        return last_map_addr;
 }
 
@@ -405,14 +409,18 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
                }
 
                if (pud_val(*pud)) {
-                       if (!pud_large(*pud))
+                       if (!pud_large(*pud)) {
                                last_map_addr = phys_pmd_update(pud, addr, end,
                                                         page_size_mask);
-                       continue;
+                               continue;
+                       }
+
+                       goto repeat_set_pte;
                }
 
                if (page_size_mask & (1<<PG_LEVEL_1G)) {
                        pages++;
+repeat_set_pte:
                        spin_lock(&init_mm.page_table_lock);
                        set_pte((pte_t *)pud,
                                pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_LARGE));
@@ -430,7 +438,9 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
                spin_unlock(&init_mm.page_table_lock);
        }
        __flush_tlb_all();
-       update_page_count(PG_LEVEL_1G, pages);
+
+       if (physical_mapping_iter == 1)
+               update_page_count(PG_LEVEL_1G, pages);
 
        return last_map_addr;
 }
@@ -494,15 +504,54 @@ static void __init init_gbpages(void)
                direct_gbpages = 0;
 }
 
+static int is_kernel(unsigned long pfn)
+{
+       unsigned long pg_addresss = pfn << PAGE_SHIFT;
+
+       if (pg_addresss >= (unsigned long) __pa(_text) &&
+           pg_addresss <= (unsigned long) __pa(_end))
+               return 1;
+
+       return 0;
+}
+
 static unsigned long __init kernel_physical_mapping_init(unsigned long start,
                                                unsigned long end,
                                                unsigned long page_size_mask)
 {
 
-       unsigned long next, last_map_addr = end;
+       unsigned long next, last_map_addr;
+       u64 cached_supported_pte_mask = __supported_pte_mask;
+       unsigned long cache_start = start;
+       unsigned long cache_end = end;
+
+       /*
+        * First iteration will setup identity mapping using large/small pages
+        * based on page_size_mask, with other attributes same as set by
+        * the early code in head_64.S
+        *
+        * Second iteration will setup the appropriate attributes
+        * as desired for the kernel identity mapping.
+        *
+        * This two pass mechanism conforms to the TLB app note which says:
+        *
+        *     "Software should not write to a paging-structure entry in a way
+        *      that would change, for any linear address, both the page size
+        *      and either the page frame or attributes."
+        *
+        * For now, only difference between very early PTE attributes used in
+        * head_64.S and here is _PAGE_NX.
+        */
+       BUILD_BUG_ON((__PAGE_KERNEL_LARGE & ~__PAGE_KERNEL_IDENT_LARGE_EXEC)
+                    != _PAGE_NX);
+       __supported_pte_mask &= ~(_PAGE_NX);
+       physical_mapping_iter = 1;
 
-       start = (unsigned long)__va(start);
-       end = (unsigned long)__va(end);
+repeat:
+       last_map_addr = cache_end;
+
+       start = (unsigned long)__va(cache_start);
+       end = (unsigned long)__va(cache_end);
 
        for (; start < end; start = next) {
                pgd_t *pgd = pgd_offset_k(start);
@@ -514,11 +563,21 @@ static unsigned long __init kernel_physical_mapping_init(unsigned long start,
                        next = end;
 
                if (pgd_val(*pgd)) {
+                       /*
+                        * Static identity mappings will be overwritten
+                        * with run-time mappings. For example, this allows
+                        * the static 0-1GB identity mapping to be mapped
+                        * non-executable with this.
+                        */
+                       if (is_kernel(pte_pfn(*((pte_t *) pgd))))
+                               goto realloc;
+
                        last_map_addr = phys_pud_update(pgd, __pa(start),
                                                 __pa(end), page_size_mask);
                        continue;
                }
 
+realloc:
                pud = alloc_low_page(&pud_phys);
                last_map_addr = phys_pud_init(pud, __pa(start), __pa(next),
                                                 page_size_mask);
@@ -528,6 +587,16 @@ static unsigned long __init kernel_physical_mapping_init(unsigned long start,
                pgd_populate(&init_mm, pgd, __va(pud_phys));
                spin_unlock(&init_mm.page_table_lock);
        }
+       __flush_tlb_all();
+
+       if (physical_mapping_iter == 1) {
+               physical_mapping_iter = 2;
+               /*
+                * Second iteration will set the actual desired PTE attributes.
+                */
+               __supported_pte_mask = cached_supported_pte_mask;
+               goto repeat;
+       }
 
        return last_map_addr;
 }