4 * This file contains the various mmu fetch and update operations.
5 * The most important job they must perform is the mapping between the
6 * domain's pfn and the overall machine mfns.
8 * Xen allows guests to directly update the pagetable, in a controlled
9 * fashion. In other words, the guest modifies the same pagetable
10 * that the CPU actually uses, which eliminates the overhead of having
11 * a separate shadow pagetable.
13 * In order to allow this, it falls on the guest domain to map its
14 * notion of a "physical" pfn - which is just a domain-local linear
15 * address - into a real "machine address" which the CPU's MMU can
18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
19 * inserted directly into the pagetable. When creating a new
20 * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
21 * when reading the content back with __(pgd|pmd|pte)_val, it converts
22 * the mfn back into a pfn.
24 * The other constraint is that all pages which make up a pagetable
25 * must be mapped read-only in the guest. This prevents uncontrolled
26 * guest updates to the pagetable. Xen strictly enforces this, and
27 * will disallow any pagetable update which will end up mapping a
28 * pagetable page RW, and will disallow using any writable page as a
31 * Naively, when loading %cr3 with the base of a new pagetable, Xen
32 * would need to validate the whole pagetable before going on.
33 * Naturally, this is quite slow. The solution is to "pin" a
34 * pagetable, which enforces all the constraints on the pagetable even
35 * when it is not actively in use. This menas that Xen can be assured
36 * that it is still valid when you do load it into %cr3, and doesn't
37 * need to revalidate it.
39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
41 #include <linux/sched.h>
42 #include <linux/highmem.h>
43 #include <linux/debugfs.h>
44 #include <linux/bug.h>
46 #include <asm/pgtable.h>
47 #include <asm/tlbflush.h>
48 #include <asm/fixmap.h>
49 #include <asm/mmu_context.h>
50 #include <asm/paravirt.h>
51 #include <asm/linkage.h>
53 #include <asm/xen/hypercall.h>
54 #include <asm/xen/hypervisor.h>
57 #include <xen/interface/xen.h>
59 #include "multicalls.h"
63 #define MMU_UPDATE_HISTO 30
65 #ifdef CONFIG_XEN_DEBUG_FS
69 u32 pgd_update_pinned;
70 u32 pgd_update_batched;
73 u32 pud_update_pinned;
74 u32 pud_update_batched;
77 u32 pmd_update_pinned;
78 u32 pmd_update_batched;
81 u32 pte_update_pinned;
82 u32 pte_update_batched;
85 u32 mmu_update_extended;
86 u32 mmu_update_histo[MMU_UPDATE_HISTO];
89 u32 prot_commit_batched;
92 u32 set_pte_at_batched;
93 u32 set_pte_at_pinned;
94 u32 set_pte_at_current;
95 u32 set_pte_at_kernel;
100 static inline void check_zero(void)
102 if (unlikely(zero_stats)) {
103 memset(&mmu_stats, 0, sizeof(mmu_stats));
108 #define ADD_STATS(elem, val) \
109 do { check_zero(); mmu_stats.elem += (val); } while(0)
111 #else /* !CONFIG_XEN_DEBUG_FS */
113 #define ADD_STATS(elem, val) do { (void)(val); } while(0)
115 #endif /* CONFIG_XEN_DEBUG_FS */
118 * Just beyond the highest usermode address. STACK_TOP_MAX has a
119 * redzone above it, so round it up to a PGD boundary.
121 #define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
124 #define P2M_ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))
125 #define TOP_ENTRIES (MAX_DOMAIN_PAGES / P2M_ENTRIES_PER_PAGE)
127 /* Placeholder for holes in the address space */
128 static unsigned long p2m_missing[P2M_ENTRIES_PER_PAGE] __page_aligned_data =
129 { [ 0 ... P2M_ENTRIES_PER_PAGE-1 ] = ~0UL };
131 /* Array of pointers to pages containing p2m entries */
132 static unsigned long *p2m_top[TOP_ENTRIES] __page_aligned_data =
133 { [ 0 ... TOP_ENTRIES - 1] = &p2m_missing[0] };
135 /* Arrays of p2m arrays expressed in mfns used for save/restore */
136 static unsigned long p2m_top_mfn[TOP_ENTRIES] __page_aligned_bss;
138 static unsigned long p2m_top_mfn_list[TOP_ENTRIES / P2M_ENTRIES_PER_PAGE]
141 static inline unsigned p2m_top_index(unsigned long pfn)
143 BUG_ON(pfn >= MAX_DOMAIN_PAGES);
144 return pfn / P2M_ENTRIES_PER_PAGE;
147 static inline unsigned p2m_index(unsigned long pfn)
149 return pfn % P2M_ENTRIES_PER_PAGE;
152 /* Build the parallel p2m_top_mfn structures */
153 void xen_setup_mfn_list_list(void)
157 for(pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn += P2M_ENTRIES_PER_PAGE) {
158 unsigned topidx = p2m_top_index(pfn);
160 p2m_top_mfn[topidx] = virt_to_mfn(p2m_top[topidx]);
163 for(idx = 0; idx < ARRAY_SIZE(p2m_top_mfn_list); idx++) {
164 unsigned topidx = idx * P2M_ENTRIES_PER_PAGE;
165 p2m_top_mfn_list[idx] = virt_to_mfn(&p2m_top_mfn[topidx]);
168 BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
170 HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
171 virt_to_mfn(p2m_top_mfn_list);
172 HYPERVISOR_shared_info->arch.max_pfn = xen_start_info->nr_pages;
175 /* Set up p2m_top to point to the domain-builder provided p2m pages */
176 void __init xen_build_dynamic_phys_to_machine(void)
178 unsigned long *mfn_list = (unsigned long *)xen_start_info->mfn_list;
179 unsigned long max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages);
182 for(pfn = 0; pfn < max_pfn; pfn += P2M_ENTRIES_PER_PAGE) {
183 unsigned topidx = p2m_top_index(pfn);
185 p2m_top[topidx] = &mfn_list[pfn];
189 unsigned long get_phys_to_machine(unsigned long pfn)
191 unsigned topidx, idx;
193 if (unlikely(pfn >= MAX_DOMAIN_PAGES))
194 return INVALID_P2M_ENTRY;
196 topidx = p2m_top_index(pfn);
197 idx = p2m_index(pfn);
198 return p2m_top[topidx][idx];
200 EXPORT_SYMBOL_GPL(get_phys_to_machine);
202 static void alloc_p2m(unsigned long **pp, unsigned long *mfnp)
207 p = (void *)__get_free_page(GFP_KERNEL | __GFP_NOFAIL);
210 for(i = 0; i < P2M_ENTRIES_PER_PAGE; i++)
211 p[i] = INVALID_P2M_ENTRY;
213 if (cmpxchg(pp, p2m_missing, p) != p2m_missing)
214 free_page((unsigned long)p);
216 *mfnp = virt_to_mfn(p);
219 void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
221 unsigned topidx, idx;
223 if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
224 BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
228 if (unlikely(pfn >= MAX_DOMAIN_PAGES)) {
229 BUG_ON(mfn != INVALID_P2M_ENTRY);
233 topidx = p2m_top_index(pfn);
234 if (p2m_top[topidx] == p2m_missing) {
235 /* no need to allocate a page to store an invalid entry */
236 if (mfn == INVALID_P2M_ENTRY)
238 alloc_p2m(&p2m_top[topidx], &p2m_top_mfn[topidx]);
241 idx = p2m_index(pfn);
242 p2m_top[topidx][idx] = mfn;
245 xmaddr_t arbitrary_virt_to_machine(void *vaddr)
247 unsigned long address = (unsigned long)vaddr;
249 pte_t *pte = lookup_address(address, &level);
250 unsigned offset = address & ~PAGE_MASK;
254 return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
257 void make_lowmem_page_readonly(void *vaddr)
260 unsigned long address = (unsigned long)vaddr;
263 pte = lookup_address(address, &level);
266 ptev = pte_wrprotect(*pte);
268 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
272 void make_lowmem_page_readwrite(void *vaddr)
275 unsigned long address = (unsigned long)vaddr;
278 pte = lookup_address(address, &level);
281 ptev = pte_mkwrite(*pte);
283 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
288 static bool xen_page_pinned(void *ptr)
290 struct page *page = virt_to_page(ptr);
292 return PagePinned(page);
295 static void xen_extend_mmu_update(const struct mmu_update *update)
297 struct multicall_space mcs;
298 struct mmu_update *u;
300 mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u));
302 if (mcs.mc != NULL) {
303 ADD_STATS(mmu_update_extended, 1);
304 ADD_STATS(mmu_update_histo[mcs.mc->args[1]], -1);
308 if (mcs.mc->args[1] < MMU_UPDATE_HISTO)
309 ADD_STATS(mmu_update_histo[mcs.mc->args[1]], 1);
311 ADD_STATS(mmu_update_histo[0], 1);
313 ADD_STATS(mmu_update, 1);
314 mcs = __xen_mc_entry(sizeof(*u));
315 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
316 ADD_STATS(mmu_update_histo[1], 1);
323 void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
331 /* ptr may be ioremapped for 64-bit pagetable setup */
332 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
333 u.val = pmd_val_ma(val);
334 xen_extend_mmu_update(&u);
336 ADD_STATS(pmd_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
338 xen_mc_issue(PARAVIRT_LAZY_MMU);
343 void xen_set_pmd(pmd_t *ptr, pmd_t val)
345 ADD_STATS(pmd_update, 1);
347 /* If page is not pinned, we can just update the entry
349 if (!xen_page_pinned(ptr)) {
354 ADD_STATS(pmd_update_pinned, 1);
356 xen_set_pmd_hyper(ptr, val);
360 * Associate a virtual page frame with a given physical page frame
361 * and protection flags for that frame.
363 void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
365 set_pte_vaddr(vaddr, mfn_pte(mfn, flags));
368 void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
369 pte_t *ptep, pte_t pteval)
371 /* updates to init_mm may be done without lock */
375 ADD_STATS(set_pte_at, 1);
376 // ADD_STATS(set_pte_at_pinned, xen_page_pinned(ptep));
377 ADD_STATS(set_pte_at_current, mm == current->mm);
378 ADD_STATS(set_pte_at_kernel, mm == &init_mm);
380 if (mm == current->mm || mm == &init_mm) {
381 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
382 struct multicall_space mcs;
383 mcs = xen_mc_entry(0);
385 MULTI_update_va_mapping(mcs.mc, addr, pteval, 0);
386 ADD_STATS(set_pte_at_batched, 1);
387 xen_mc_issue(PARAVIRT_LAZY_MMU);
390 if (HYPERVISOR_update_va_mapping(addr, pteval, 0) == 0)
393 xen_set_pte(ptep, pteval);
400 pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
402 /* Just return the pte as-is. We preserve the bits on commit */
406 void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
407 pte_t *ptep, pte_t pte)
413 u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
414 u.val = pte_val_ma(pte);
415 xen_extend_mmu_update(&u);
417 ADD_STATS(prot_commit, 1);
418 ADD_STATS(prot_commit_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
420 xen_mc_issue(PARAVIRT_LAZY_MMU);
423 /* Assume pteval_t is equivalent to all the other *val_t types. */
424 static pteval_t pte_mfn_to_pfn(pteval_t val)
426 if (val & _PAGE_PRESENT) {
427 unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
428 pteval_t flags = val & PTE_FLAGS_MASK;
429 val = ((pteval_t)mfn_to_pfn(mfn) << PAGE_SHIFT) | flags;
435 static pteval_t pte_pfn_to_mfn(pteval_t val)
437 if (val & _PAGE_PRESENT) {
438 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
439 pteval_t flags = val & PTE_FLAGS_MASK;
440 val = ((pteval_t)pfn_to_mfn(pfn) << PAGE_SHIFT) | flags;
446 pteval_t xen_pte_val(pte_t pte)
448 return pte_mfn_to_pfn(pte.pte);
451 pgdval_t xen_pgd_val(pgd_t pgd)
453 return pte_mfn_to_pfn(pgd.pgd);
456 pte_t xen_make_pte(pteval_t pte)
458 pte = pte_pfn_to_mfn(pte);
459 return native_make_pte(pte);
462 pgd_t xen_make_pgd(pgdval_t pgd)
464 pgd = pte_pfn_to_mfn(pgd);
465 return native_make_pgd(pgd);
468 pmdval_t xen_pmd_val(pmd_t pmd)
470 return pte_mfn_to_pfn(pmd.pmd);
473 void xen_set_pud_hyper(pud_t *ptr, pud_t val)
481 /* ptr may be ioremapped for 64-bit pagetable setup */
482 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
483 u.val = pud_val_ma(val);
484 xen_extend_mmu_update(&u);
486 ADD_STATS(pud_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
488 xen_mc_issue(PARAVIRT_LAZY_MMU);
493 void xen_set_pud(pud_t *ptr, pud_t val)
495 ADD_STATS(pud_update, 1);
497 /* If page is not pinned, we can just update the entry
499 if (!xen_page_pinned(ptr)) {
504 ADD_STATS(pud_update_pinned, 1);
506 xen_set_pud_hyper(ptr, val);
509 void xen_set_pte(pte_t *ptep, pte_t pte)
511 ADD_STATS(pte_update, 1);
512 // ADD_STATS(pte_update_pinned, xen_page_pinned(ptep));
513 ADD_STATS(pte_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
515 #ifdef CONFIG_X86_PAE
516 ptep->pte_high = pte.pte_high;
518 ptep->pte_low = pte.pte_low;
524 #ifdef CONFIG_X86_PAE
525 void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
527 set_64bit((u64 *)ptep, native_pte_val(pte));
530 void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
533 smp_wmb(); /* make sure low gets written first */
537 void xen_pmd_clear(pmd_t *pmdp)
539 set_pmd(pmdp, __pmd(0));
541 #endif /* CONFIG_X86_PAE */
543 pmd_t xen_make_pmd(pmdval_t pmd)
545 pmd = pte_pfn_to_mfn(pmd);
546 return native_make_pmd(pmd);
549 #if PAGETABLE_LEVELS == 4
550 pudval_t xen_pud_val(pud_t pud)
552 return pte_mfn_to_pfn(pud.pud);
555 pud_t xen_make_pud(pudval_t pud)
557 pud = pte_pfn_to_mfn(pud);
559 return native_make_pud(pud);
562 pgd_t *xen_get_user_pgd(pgd_t *pgd)
564 pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);
565 unsigned offset = pgd - pgd_page;
566 pgd_t *user_ptr = NULL;
568 if (offset < pgd_index(USER_LIMIT)) {
569 struct page *page = virt_to_page(pgd_page);
570 user_ptr = (pgd_t *)page->private;
578 static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
582 u.ptr = virt_to_machine(ptr).maddr;
583 u.val = pgd_val_ma(val);
584 xen_extend_mmu_update(&u);
588 * Raw hypercall-based set_pgd, intended for in early boot before
589 * there's a page structure. This implies:
590 * 1. The only existing pagetable is the kernel's
591 * 2. It is always pinned
592 * 3. It has no user pagetable attached to it
594 void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
600 __xen_set_pgd_hyper(ptr, val);
602 xen_mc_issue(PARAVIRT_LAZY_MMU);
607 void xen_set_pgd(pgd_t *ptr, pgd_t val)
609 pgd_t *user_ptr = xen_get_user_pgd(ptr);
611 ADD_STATS(pgd_update, 1);
613 /* If page is not pinned, we can just update the entry
615 if (!xen_page_pinned(ptr)) {
618 WARN_ON(xen_page_pinned(user_ptr));
624 ADD_STATS(pgd_update_pinned, 1);
625 ADD_STATS(pgd_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
627 /* If it's pinned, then we can at least batch the kernel and
628 user updates together. */
631 __xen_set_pgd_hyper(ptr, val);
633 __xen_set_pgd_hyper(user_ptr, val);
635 xen_mc_issue(PARAVIRT_LAZY_MMU);
637 #endif /* PAGETABLE_LEVELS == 4 */
640 * (Yet another) pagetable walker. This one is intended for pinning a
641 * pagetable. This means that it walks a pagetable and calls the
642 * callback function on each page it finds making up the page table,
643 * at every level. It walks the entire pagetable, but it only bothers
644 * pinning pte pages which are below limit. In the normal case this
645 * will be STACK_TOP_MAX, but at boot we need to pin up to
648 * For 32-bit the important bit is that we don't pin beyond there,
649 * because then we start getting into Xen's ptes.
651 * For 64-bit, we must skip the Xen hole in the middle of the address
652 * space, just after the big x86-64 virtual hole.
654 static int xen_pgd_walk(struct mm_struct *mm,
655 int (*func)(struct mm_struct *mm, struct page *,
659 pgd_t *pgd = mm->pgd;
661 unsigned hole_low, hole_high;
662 unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
663 unsigned pgdidx, pudidx, pmdidx;
665 /* The limit is the last byte to be touched */
667 BUG_ON(limit >= FIXADDR_TOP);
669 if (xen_feature(XENFEAT_auto_translated_physmap))
673 * 64-bit has a great big hole in the middle of the address
674 * space, which contains the Xen mappings. On 32-bit these
675 * will end up making a zero-sized hole and so is a no-op.
677 hole_low = pgd_index(USER_LIMIT);
678 hole_high = pgd_index(PAGE_OFFSET);
680 pgdidx_limit = pgd_index(limit);
682 pudidx_limit = pud_index(limit);
687 pmdidx_limit = pmd_index(limit);
692 for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) {
695 if (pgdidx >= hole_low && pgdidx < hole_high)
698 if (!pgd_val(pgd[pgdidx]))
701 pud = pud_offset(&pgd[pgdidx], 0);
703 if (PTRS_PER_PUD > 1) /* not folded */
704 flush |= (*func)(mm, virt_to_page(pud), PT_PUD);
706 for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) {
709 if (pgdidx == pgdidx_limit &&
710 pudidx > pudidx_limit)
713 if (pud_none(pud[pudidx]))
716 pmd = pmd_offset(&pud[pudidx], 0);
718 if (PTRS_PER_PMD > 1) /* not folded */
719 flush |= (*func)(mm, virt_to_page(pmd), PT_PMD);
721 for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) {
724 if (pgdidx == pgdidx_limit &&
725 pudidx == pudidx_limit &&
726 pmdidx > pmdidx_limit)
729 if (pmd_none(pmd[pmdidx]))
732 pte = pmd_page(pmd[pmdidx]);
733 flush |= (*func)(mm, pte, PT_PTE);
739 /* Do the top level last, so that the callbacks can use it as
740 a cue to do final things like tlb flushes. */
741 flush |= (*func)(mm, virt_to_page(pgd), PT_PGD);
746 /* If we're using split pte locks, then take the page's lock and
747 return a pointer to it. Otherwise return NULL. */
748 static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)
750 spinlock_t *ptl = NULL;
752 #if USE_SPLIT_PTLOCKS
753 ptl = __pte_lockptr(page);
754 spin_lock_nest_lock(ptl, &mm->page_table_lock);
760 static void xen_pte_unlock(void *v)
766 static void xen_do_pin(unsigned level, unsigned long pfn)
768 struct mmuext_op *op;
769 struct multicall_space mcs;
771 mcs = __xen_mc_entry(sizeof(*op));
774 op->arg1.mfn = pfn_to_mfn(pfn);
775 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
778 static int xen_pin_page(struct mm_struct *mm, struct page *page,
781 unsigned pgfl = TestSetPagePinned(page);
785 flush = 0; /* already pinned */
786 else if (PageHighMem(page))
787 /* kmaps need flushing if we found an unpinned
791 void *pt = lowmem_page_address(page);
792 unsigned long pfn = page_to_pfn(page);
793 struct multicall_space mcs = __xen_mc_entry(0);
799 * We need to hold the pagetable lock between the time
800 * we make the pagetable RO and when we actually pin
801 * it. If we don't, then other users may come in and
802 * attempt to update the pagetable by writing it,
803 * which will fail because the memory is RO but not
804 * pinned, so Xen won't do the trap'n'emulate.
806 * If we're using split pte locks, we can't hold the
807 * entire pagetable's worth of locks during the
808 * traverse, because we may wrap the preempt count (8
809 * bits). The solution is to mark RO and pin each PTE
810 * page while holding the lock. This means the number
811 * of locks we end up holding is never more than a
812 * batch size (~32 entries, at present).
814 * If we're not using split pte locks, we needn't pin
815 * the PTE pages independently, because we're
816 * protected by the overall pagetable lock.
820 ptl = xen_pte_lock(page, mm);
822 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
823 pfn_pte(pfn, PAGE_KERNEL_RO),
824 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
827 xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn);
829 /* Queue a deferred unlock for when this batch
831 xen_mc_callback(xen_pte_unlock, ptl);
838 /* This is called just after a mm has been created, but it has not
839 been used yet. We need to make sure that its pagetable is all
840 read-only, and can be pinned. */
841 static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)
845 if (xen_pgd_walk(mm, xen_pin_page, USER_LIMIT)) {
846 /* re-enable interrupts for kmap_flush_unused */
854 pgd_t *user_pgd = xen_get_user_pgd(pgd);
856 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));
859 xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD);
860 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(user_pgd)));
863 #else /* CONFIG_X86_32 */
864 #ifdef CONFIG_X86_PAE
865 /* Need to make sure unshared kernel PMD is pinnable */
866 xen_pin_page(mm, virt_to_page(pgd_page(pgd[pgd_index(TASK_SIZE)])),
869 xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));
870 #endif /* CONFIG_X86_64 */
874 static void xen_pgd_pin(struct mm_struct *mm)
876 __xen_pgd_pin(mm, mm->pgd);
880 * On save, we need to pin all pagetables to make sure they get their
881 * mfns turned into pfns. Search the list for any unpinned pgds and pin
882 * them (unpinned pgds are not currently in use, probably because the
883 * process is under construction or destruction).
885 * Expected to be called in stop_machine() ("equivalent to taking
886 * every spinlock in the system"), so the locking doesn't really
887 * matter all that much.
889 void xen_mm_pin_all(void)
894 spin_lock_irqsave(&pgd_lock, flags);
896 list_for_each_entry(page, &pgd_list, lru) {
897 if (!PagePinned(page)) {
898 __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page));
899 SetPageSavePinned(page);
903 spin_unlock_irqrestore(&pgd_lock, flags);
907 * The init_mm pagetable is really pinned as soon as its created, but
908 * that's before we have page structures to store the bits. So do all
909 * the book-keeping now.
911 static __init int xen_mark_pinned(struct mm_struct *mm, struct page *page,
918 void __init xen_mark_init_mm_pinned(void)
920 xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP);
923 static int xen_unpin_page(struct mm_struct *mm, struct page *page,
926 unsigned pgfl = TestClearPagePinned(page);
928 if (pgfl && !PageHighMem(page)) {
929 void *pt = lowmem_page_address(page);
930 unsigned long pfn = page_to_pfn(page);
931 spinlock_t *ptl = NULL;
932 struct multicall_space mcs;
935 * Do the converse to pin_page. If we're using split
936 * pte locks, we must be holding the lock for while
937 * the pte page is unpinned but still RO to prevent
938 * concurrent updates from seeing it in this
939 * partially-pinned state.
941 if (level == PT_PTE) {
942 ptl = xen_pte_lock(page, mm);
945 xen_do_pin(MMUEXT_UNPIN_TABLE, pfn);
948 mcs = __xen_mc_entry(0);
950 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
951 pfn_pte(pfn, PAGE_KERNEL),
952 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
955 /* unlock when batch completed */
956 xen_mc_callback(xen_pte_unlock, ptl);
960 return 0; /* never need to flush on unpin */
963 /* Release a pagetables pages back as normal RW */
964 static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)
968 xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
972 pgd_t *user_pgd = xen_get_user_pgd(pgd);
975 xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(user_pgd)));
976 xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD);
981 #ifdef CONFIG_X86_PAE
982 /* Need to make sure unshared kernel PMD is unpinned */
983 xen_unpin_page(mm, virt_to_page(pgd_page(pgd[pgd_index(TASK_SIZE)])),
987 xen_pgd_walk(mm, xen_unpin_page, USER_LIMIT);
992 static void xen_pgd_unpin(struct mm_struct *mm)
994 __xen_pgd_unpin(mm, mm->pgd);
998 * On resume, undo any pinning done at save, so that the rest of the
999 * kernel doesn't see any unexpected pinned pagetables.
1001 void xen_mm_unpin_all(void)
1003 unsigned long flags;
1006 spin_lock_irqsave(&pgd_lock, flags);
1008 list_for_each_entry(page, &pgd_list, lru) {
1009 if (PageSavePinned(page)) {
1010 BUG_ON(!PagePinned(page));
1011 __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page));
1012 ClearPageSavePinned(page);
1016 spin_unlock_irqrestore(&pgd_lock, flags);
1019 void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
1021 spin_lock(&next->page_table_lock);
1023 spin_unlock(&next->page_table_lock);
1026 void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
1028 spin_lock(&mm->page_table_lock);
1030 spin_unlock(&mm->page_table_lock);
1035 /* Another cpu may still have their %cr3 pointing at the pagetable, so
1036 we need to repoint it somewhere else before we can unpin it. */
1037 static void drop_other_mm_ref(void *info)
1039 struct mm_struct *mm = info;
1040 struct mm_struct *active_mm;
1042 #ifdef CONFIG_X86_64
1043 active_mm = read_pda(active_mm);
1045 active_mm = __get_cpu_var(cpu_tlbstate).active_mm;
1048 if (active_mm == mm)
1049 leave_mm(smp_processor_id());
1051 /* If this cpu still has a stale cr3 reference, then make sure
1052 it has been flushed. */
1053 if (x86_read_percpu(xen_current_cr3) == __pa(mm->pgd)) {
1054 load_cr3(swapper_pg_dir);
1055 arch_flush_lazy_cpu_mode();
1059 static void xen_drop_mm_ref(struct mm_struct *mm)
1064 if (current->active_mm == mm) {
1065 if (current->mm == mm)
1066 load_cr3(swapper_pg_dir);
1068 leave_mm(smp_processor_id());
1069 arch_flush_lazy_cpu_mode();
1072 /* Get the "official" set of cpus referring to our pagetable. */
1073 mask = mm->cpu_vm_mask;
1075 /* It's possible that a vcpu may have a stale reference to our
1076 cr3, because its in lazy mode, and it hasn't yet flushed
1077 its set of pending hypercalls yet. In this case, we can
1078 look at its actual current cr3 value, and force it to flush
1080 for_each_online_cpu(cpu) {
1081 if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))
1085 if (!cpus_empty(mask))
1086 smp_call_function_mask(mask, drop_other_mm_ref, mm, 1);
1089 static void xen_drop_mm_ref(struct mm_struct *mm)
1091 if (current->active_mm == mm)
1092 load_cr3(swapper_pg_dir);
1097 * While a process runs, Xen pins its pagetables, which means that the
1098 * hypervisor forces it to be read-only, and it controls all updates
1099 * to it. This means that all pagetable updates have to go via the
1100 * hypervisor, which is moderately expensive.
1102 * Since we're pulling the pagetable down, we switch to use init_mm,
1103 * unpin old process pagetable and mark it all read-write, which
1104 * allows further operations on it to be simple memory accesses.
1106 * The only subtle point is that another CPU may be still using the
1107 * pagetable because of lazy tlb flushing. This means we need need to
1108 * switch all CPUs off this pagetable before we can unpin it.
1110 void xen_exit_mmap(struct mm_struct *mm)
1112 get_cpu(); /* make sure we don't move around */
1113 xen_drop_mm_ref(mm);
1116 spin_lock(&mm->page_table_lock);
1118 /* pgd may not be pinned in the error exit path of execve */
1119 if (xen_page_pinned(mm->pgd))
1122 spin_unlock(&mm->page_table_lock);
1125 #ifdef CONFIG_XEN_DEBUG_FS
1127 static struct dentry *d_mmu_debug;
1129 static int __init xen_mmu_debugfs(void)
1131 struct dentry *d_xen = xen_init_debugfs();
1136 d_mmu_debug = debugfs_create_dir("mmu", d_xen);
1138 debugfs_create_u8("zero_stats", 0644, d_mmu_debug, &zero_stats);
1140 debugfs_create_u32("pgd_update", 0444, d_mmu_debug, &mmu_stats.pgd_update);
1141 debugfs_create_u32("pgd_update_pinned", 0444, d_mmu_debug,
1142 &mmu_stats.pgd_update_pinned);
1143 debugfs_create_u32("pgd_update_batched", 0444, d_mmu_debug,
1144 &mmu_stats.pgd_update_pinned);
1146 debugfs_create_u32("pud_update", 0444, d_mmu_debug, &mmu_stats.pud_update);
1147 debugfs_create_u32("pud_update_pinned", 0444, d_mmu_debug,
1148 &mmu_stats.pud_update_pinned);
1149 debugfs_create_u32("pud_update_batched", 0444, d_mmu_debug,
1150 &mmu_stats.pud_update_pinned);
1152 debugfs_create_u32("pmd_update", 0444, d_mmu_debug, &mmu_stats.pmd_update);
1153 debugfs_create_u32("pmd_update_pinned", 0444, d_mmu_debug,
1154 &mmu_stats.pmd_update_pinned);
1155 debugfs_create_u32("pmd_update_batched", 0444, d_mmu_debug,
1156 &mmu_stats.pmd_update_pinned);
1158 debugfs_create_u32("pte_update", 0444, d_mmu_debug, &mmu_stats.pte_update);
1159 // debugfs_create_u32("pte_update_pinned", 0444, d_mmu_debug,
1160 // &mmu_stats.pte_update_pinned);
1161 debugfs_create_u32("pte_update_batched", 0444, d_mmu_debug,
1162 &mmu_stats.pte_update_pinned);
1164 debugfs_create_u32("mmu_update", 0444, d_mmu_debug, &mmu_stats.mmu_update);
1165 debugfs_create_u32("mmu_update_extended", 0444, d_mmu_debug,
1166 &mmu_stats.mmu_update_extended);
1167 xen_debugfs_create_u32_array("mmu_update_histo", 0444, d_mmu_debug,
1168 mmu_stats.mmu_update_histo, 20);
1170 debugfs_create_u32("set_pte_at", 0444, d_mmu_debug, &mmu_stats.set_pte_at);
1171 debugfs_create_u32("set_pte_at_batched", 0444, d_mmu_debug,
1172 &mmu_stats.set_pte_at_batched);
1173 debugfs_create_u32("set_pte_at_current", 0444, d_mmu_debug,
1174 &mmu_stats.set_pte_at_current);
1175 debugfs_create_u32("set_pte_at_kernel", 0444, d_mmu_debug,
1176 &mmu_stats.set_pte_at_kernel);
1178 debugfs_create_u32("prot_commit", 0444, d_mmu_debug, &mmu_stats.prot_commit);
1179 debugfs_create_u32("prot_commit_batched", 0444, d_mmu_debug,
1180 &mmu_stats.prot_commit_batched);
1184 fs_initcall(xen_mmu_debugfs);
1186 #endif /* CONFIG_XEN_DEBUG_FS */