[IA64] Percpu quicklist for combined allocator for pgd/pmd/pte.

[linux-2.6-omap-h63xx.git] / arch / ia64 / mm / init.c

diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c
index 65cf839573ea88c27568695061cbd60f0e66d1b5..4892be53e227a806ee5e24e00b2c373267028378 100644 (file)
--- a/arch/ia64/mm/init.c
+++ b/arch/ia64/mm/init.c
@@ -39,6 +39,9 @@
 
 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
+DEFINE_PER_CPU(unsigned long *, __pgtable_quicklist);
+DEFINE_PER_CPU(long, __pgtable_quicklist_size);
+
 extern void ia64_tlb_init (void);
 
 unsigned long MAX_DMA_ADDRESS = PAGE_OFFSET + 0x100000000UL;
@@ -50,27 +53,53 @@ struct page *vmem_map;
 EXPORT_SYMBOL(vmem_map);
 #endif
 
-static int pgt_cache_water[2] = { 25, 50 };
-
-struct page *zero_page_memmap_ptr;             /* map entry for zero page */
+struct page *zero_page_memmap_ptr;     /* map entry for zero page */
 EXPORT_SYMBOL(zero_page_memmap_ptr);
 
+#define MIN_PGT_PAGES                  25UL
+#define MAX_PGT_FREES_PER_PASS         16
+#define PGT_FRACTION_OF_NODE_MEM       16
+
+static inline long
+max_pgt_pages(void)
+{
+       u64 node_free_pages, max_pgt_pages;
+
+#ifndef        CONFIG_NUMA
+       node_free_pages = nr_free_pages();
+#else
+       node_free_pages = nr_free_pages_pgdat(NODE_DATA(numa_node_id()));
+#endif
+       max_pgt_pages = node_free_pages / PGT_FRACTION_OF_NODE_MEM;
+       max_pgt_pages = max(max_pgt_pages, MIN_PGT_PAGES);
+       return max_pgt_pages;
+}
+
+static inline long
+min_pages_to_free(void)
+{
+       long pages_to_free;
+
+       pages_to_free = pgtable_quicklist_size - max_pgt_pages();
+       pages_to_free = min(pages_to_free, MAX_PGT_FREES_PER_PASS);
+       return pages_to_free;
+}
+
 void
-check_pgt_cache (void)
+check_pgt_cache(void)
 {
-       int low, high;
+       long pages_to_free;
 
-       low = pgt_cache_water[0];
-       high = pgt_cache_water[1];
+       if (unlikely(pgtable_quicklist_size <= MIN_PGT_PAGES))
+               return;
 
        preempt_disable();
-       if (pgtable_cache_size > (u64) high) {
-               do {
-                       if (pgd_quicklist)
-                               free_page((unsigned long)pgd_alloc_one_fast(NULL));
-                       if (pmd_quicklist)
-                               free_page((unsigned long)pmd_alloc_one_fast(NULL, 0));
-               } while (pgtable_cache_size > (u64) low);
+       while (unlikely((pages_to_free = min_pages_to_free()) > 0)) {
+               while (pages_to_free--) {
+                       free_page((unsigned long)pgtable_quicklist_alloc());
+               }
+               preempt_enable();
+               preempt_disable();
        }
        preempt_enable();
 }
@@ -523,11 +552,14 @@ void
 mem_init (void)
 {
        long reserved_pages, codesize, datasize, initsize;
-       unsigned long num_pgt_pages;
        pg_data_t *pgdat;
        int i;
        static struct kcore_list kcore_mem, kcore_vmem, kcore_kernel;
 
+       BUG_ON(PTRS_PER_PGD * sizeof(pgd_t) != PAGE_SIZE);
+       BUG_ON(PTRS_PER_PMD * sizeof(pmd_t) != PAGE_SIZE);
+       BUG_ON(PTRS_PER_PTE * sizeof(pte_t) != PAGE_SIZE);
+
 #ifdef CONFIG_PCI
        /*
         * This needs to be called _after_ the command line has been parsed but _before_
@@ -564,18 +596,6 @@ mem_init (void)
               num_physpages << (PAGE_SHIFT - 10), codesize >> 10,
               reserved_pages << (PAGE_SHIFT - 10), datasize >> 10, initsize >> 10);
 
-       /*
-        * Allow for enough (cached) page table pages so that we can map the entire memory
-        * at least once.  Each task also needs a couple of page tables pages, so add in a
-        * fudge factor for that (don't use "threads-max" here; that would be wrong!).
-        * Don't allow the cache to be more than 10% of total memory, though.
-        */
-#      define NUM_TASKS        500     /* typical number of tasks */
-       num_pgt_pages = nr_free_pages() / PTRS_PER_PGD + NUM_TASKS;
-       if (num_pgt_pages > nr_free_pages() / 10)
-               num_pgt_pages = nr_free_pages() / 10;
-       if (num_pgt_pages > (u64) pgt_cache_water[1])
-               pgt_cache_water[1] = num_pgt_pages;
 
        /*
         * For fsyscall entrpoints with no light-weight handler, use the ordinary