hugetlbfs: handle pages higher order than MAX_ORDER

[linux-2.6-omap-h63xx.git] / mm / hugetlb.c

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 38633864a93e874009d9e6aa00619cf7471b0d9e..e6afe527bd09388872a2e07590b32105f050a579 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -7,6 +7,7 @@
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/mm.h>
+#include <linux/seq_file.h>
 #include <linux/sysctl.h>
 #include <linux/highmem.h>
 #include <linux/mmu_notifier.h>
@@ -262,7 +263,7 @@ struct resv_map {
        struct list_head regions;
 };
 
-struct resv_map *resv_map_alloc(void)
+static struct resv_map *resv_map_alloc(void)
 {
        struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
        if (!resv_map)
@@ -274,7 +275,7 @@ struct resv_map *resv_map_alloc(void)
        return resv_map;
 }
 
-void resv_map_release(struct kref *ref)
+static void resv_map_release(struct kref *ref)
 {
        struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
 
@@ -289,7 +290,7 @@ static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
        if (!(vma->vm_flags & VM_SHARED))
                return (struct resv_map *)(get_vma_private_data(vma) &
                                                        ~HPAGE_RESV_MASK);
-       return 0;
+       return NULL;
 }
 
 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
@@ -353,11 +354,26 @@ static int vma_has_reserves(struct vm_area_struct *vma)
        return 0;
 }
 
+static void clear_gigantic_page(struct page *page,
+                       unsigned long addr, unsigned long sz)
+{
+       int i;
+       struct page *p = page;
+
+       might_sleep();
+       for (i = 0; i < sz/PAGE_SIZE; i++, p = mem_map_next(p, page, i)) {
+               cond_resched();
+               clear_user_highpage(p, addr + i * PAGE_SIZE);
+       }
+}
 static void clear_huge_page(struct page *page,
                        unsigned long addr, unsigned long sz)
 {
        int i;
 
+       if (unlikely(sz > MAX_ORDER_NR_PAGES))
+               return clear_gigantic_page(page, addr, sz);
+
        might_sleep();
        for (i = 0; i < sz/PAGE_SIZE; i++) {
                cond_resched();
@@ -365,12 +381,32 @@ static void clear_huge_page(struct page *page,
        }
 }
 
+static void copy_gigantic_page(struct page *dst, struct page *src,
+                          unsigned long addr, struct vm_area_struct *vma)
+{
+       int i;
+       struct hstate *h = hstate_vma(vma);
+       struct page *dst_base = dst;
+       struct page *src_base = src;
+       might_sleep();
+       for (i = 0; i < pages_per_huge_page(h); ) {
+               cond_resched();
+               copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
+
+               i++;
+               dst = mem_map_next(dst, dst_base, i);
+               src = mem_map_next(src, src_base, i);
+       }
+}
 static void copy_huge_page(struct page *dst, struct page *src,
                           unsigned long addr, struct vm_area_struct *vma)
 {
        int i;
        struct hstate *h = hstate_vma(vma);
 
+       if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES))
+               return copy_gigantic_page(dst, src, addr, vma);
+
        might_sleep();
        for (i = 0; i < pages_per_huge_page(h); i++) {
                cond_resched();
@@ -1455,15 +1491,15 @@ int hugetlb_overcommit_handler(struct ctl_table *table, int write,
 
 #endif /* CONFIG_SYSCTL */
 
-int hugetlb_report_meminfo(char *buf)
+void hugetlb_report_meminfo(struct seq_file *m)
 {
        struct hstate *h = &default_hstate;
-       return sprintf(buf,
-                       "HugePages_Total: %5lu\n"
-                       "HugePages_Free:  %5lu\n"
-                       "HugePages_Rsvd:  %5lu\n"
-                       "HugePages_Surp:  %5lu\n"
-                       "Hugepagesize:    %5lu kB\n",
+       seq_printf(m,
+                       "HugePages_Total:   %5lu\n"
+                       "HugePages_Free:    %5lu\n"
+                       "HugePages_Rsvd:    %5lu\n"
+                       "HugePages_Surp:    %5lu\n"
+                       "Hugepagesize:   %8lu kB\n",
                        h->nr_huge_pages,
                        h->free_huge_pages,
                        h->resv_huge_pages,
@@ -1747,10 +1783,8 @@ void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
  * from other VMAs and let the children be SIGKILLed if they are faulting the
  * same region.
  */
-int unmap_ref_private(struct mm_struct *mm,
-                                       struct vm_area_struct *vma,
-                                       struct page *page,
-                                       unsigned long address)
+static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
+                               struct page *page, unsigned long address)
 {
        struct vm_area_struct *iter_vma;
        struct address_space *mapping;
@@ -2073,6 +2107,14 @@ follow_huge_pud(struct mm_struct *mm, unsigned long address,
        return NULL;
 }
 
+static int huge_zeropage_ok(pte_t *ptep, int write, int shared)
+{
+       if (!ptep || write || shared)
+               return 0;
+       else
+               return huge_pte_none(huge_ptep_get(ptep));
+}
+
 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
                        struct page **pages, struct vm_area_struct **vmas,
                        unsigned long *position, int *length, int i,
@@ -2082,6 +2124,8 @@ int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
        unsigned long vaddr = *position;
        int remainder = *length;
        struct hstate *h = hstate_vma(vma);
+       int zeropage_ok = 0;
+       int shared = vma->vm_flags & VM_SHARED;
 
        spin_lock(&mm->page_table_lock);
        while (vaddr < vma->vm_end && remainder) {
@@ -2094,8 +2138,11 @@ int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 * first, for the page indexing below to work.
                 */
                pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
+               if (huge_zeropage_ok(pte, write, shared))
+                       zeropage_ok = 1;
 
-               if (!pte || huge_pte_none(huge_ptep_get(pte)) ||
+               if (!pte ||
+                   (huge_pte_none(huge_ptep_get(pte)) && !zeropage_ok) ||
                    (write && !pte_write(huge_ptep_get(pte)))) {
                        int ret;
 
@@ -2115,8 +2162,11 @@ int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
                page = pte_page(huge_ptep_get(pte));
 same_page:
                if (pages) {
-                       get_page(page);
-                       pages[i] = page + pfn_offset;
+                       if (zeropage_ok)
+                               pages[i] = ZERO_PAGE(0);
+                       else
+                               pages[i] = mem_map_offset(page, pfn_offset);
+                       get_page(pages[i]);
                }
 
                if (vmas)