4 * This file contains the various mmu fetch and update operations.
5 * The most important job they must perform is the mapping between the
6 * domain's pfn and the overall machine mfns.
8 * Xen allows guests to directly update the pagetable, in a controlled
9 * fashion. In other words, the guest modifies the same pagetable
10 * that the CPU actually uses, which eliminates the overhead of having
11 * a separate shadow pagetable.
13 * In order to allow this, it falls on the guest domain to map its
14 * notion of a "physical" pfn - which is just a domain-local linear
15 * address - into a real "machine address" which the CPU's MMU can
18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
19 * inserted directly into the pagetable. When creating a new
20 * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
21 * when reading the content back with __(pgd|pmd|pte)_val, it converts
22 * the mfn back into a pfn.
24 * The other constraint is that all pages which make up a pagetable
25 * must be mapped read-only in the guest. This prevents uncontrolled
26 * guest updates to the pagetable. Xen strictly enforces this, and
27 * will disallow any pagetable update which will end up mapping a
28 * pagetable page RW, and will disallow using any writable page as a
31 * Naively, when loading %cr3 with the base of a new pagetable, Xen
32 * would need to validate the whole pagetable before going on.
33 * Naturally, this is quite slow. The solution is to "pin" a
34 * pagetable, which enforces all the constraints on the pagetable even
35 * when it is not actively in use. This menas that Xen can be assured
36 * that it is still valid when you do load it into %cr3, and doesn't
37 * need to revalidate it.
39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
41 #include <linux/sched.h>
42 #include <linux/highmem.h>
43 #include <linux/debugfs.h>
44 #include <linux/bug.h>
46 #include <asm/pgtable.h>
47 #include <asm/tlbflush.h>
48 #include <asm/fixmap.h>
49 #include <asm/mmu_context.h>
50 #include <asm/paravirt.h>
51 #include <asm/linkage.h>
53 #include <asm/xen/hypercall.h>
54 #include <asm/xen/hypervisor.h>
57 #include <xen/interface/xen.h>
59 #include "multicalls.h"
63 #define MMU_UPDATE_HISTO 30
65 #ifdef CONFIG_XEN_DEBUG_FS
69 u32 pgd_update_pinned;
70 u32 pgd_update_batched;
73 u32 pud_update_pinned;
74 u32 pud_update_batched;
77 u32 pmd_update_pinned;
78 u32 pmd_update_batched;
81 u32 pte_update_pinned;
82 u32 pte_update_batched;
85 u32 mmu_update_extended;
86 u32 mmu_update_histo[MMU_UPDATE_HISTO];
89 u32 prot_commit_batched;
92 u32 set_pte_at_batched;
93 u32 set_pte_at_pinned;
94 u32 set_pte_at_current;
95 u32 set_pte_at_kernel;
100 static inline void check_zero(void)
102 if (unlikely(zero_stats)) {
103 memset(&mmu_stats, 0, sizeof(mmu_stats));
108 #define ADD_STATS(elem, val) \
109 do { check_zero(); mmu_stats.elem += (val); } while(0)
111 #else /* !CONFIG_XEN_DEBUG_FS */
113 #define ADD_STATS(elem, val) do { (void)(val); } while(0)
115 #endif /* CONFIG_XEN_DEBUG_FS */
118 * Just beyond the highest usermode address. STACK_TOP_MAX has a
119 * redzone above it, so round it up to a PGD boundary.
121 #define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
124 #define P2M_ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))
125 #define TOP_ENTRIES (MAX_DOMAIN_PAGES / P2M_ENTRIES_PER_PAGE)
127 /* Placeholder for holes in the address space */
128 static unsigned long p2m_missing[P2M_ENTRIES_PER_PAGE] __page_aligned_data =
129 { [ 0 ... P2M_ENTRIES_PER_PAGE-1 ] = ~0UL };
131 /* Array of pointers to pages containing p2m entries */
132 static unsigned long *p2m_top[TOP_ENTRIES] __page_aligned_data =
133 { [ 0 ... TOP_ENTRIES - 1] = &p2m_missing[0] };
135 /* Arrays of p2m arrays expressed in mfns used for save/restore */
136 static unsigned long p2m_top_mfn[TOP_ENTRIES] __page_aligned_bss;
138 static unsigned long p2m_top_mfn_list[TOP_ENTRIES / P2M_ENTRIES_PER_PAGE]
141 static inline unsigned p2m_top_index(unsigned long pfn)
143 BUG_ON(pfn >= MAX_DOMAIN_PAGES);
144 return pfn / P2M_ENTRIES_PER_PAGE;
147 static inline unsigned p2m_index(unsigned long pfn)
149 return pfn % P2M_ENTRIES_PER_PAGE;
152 /* Build the parallel p2m_top_mfn structures */
153 void xen_setup_mfn_list_list(void)
157 for(pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn += P2M_ENTRIES_PER_PAGE) {
158 unsigned topidx = p2m_top_index(pfn);
160 p2m_top_mfn[topidx] = virt_to_mfn(p2m_top[topidx]);
163 for(idx = 0; idx < ARRAY_SIZE(p2m_top_mfn_list); idx++) {
164 unsigned topidx = idx * P2M_ENTRIES_PER_PAGE;
165 p2m_top_mfn_list[idx] = virt_to_mfn(&p2m_top_mfn[topidx]);
168 BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
170 HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
171 virt_to_mfn(p2m_top_mfn_list);
172 HYPERVISOR_shared_info->arch.max_pfn = xen_start_info->nr_pages;
175 /* Set up p2m_top to point to the domain-builder provided p2m pages */
176 void __init xen_build_dynamic_phys_to_machine(void)
178 unsigned long *mfn_list = (unsigned long *)xen_start_info->mfn_list;
179 unsigned long max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages);
182 for(pfn = 0; pfn < max_pfn; pfn += P2M_ENTRIES_PER_PAGE) {
183 unsigned topidx = p2m_top_index(pfn);
185 p2m_top[topidx] = &mfn_list[pfn];
189 unsigned long get_phys_to_machine(unsigned long pfn)
191 unsigned topidx, idx;
193 if (unlikely(pfn >= MAX_DOMAIN_PAGES))
194 return INVALID_P2M_ENTRY;
196 topidx = p2m_top_index(pfn);
197 idx = p2m_index(pfn);
198 return p2m_top[topidx][idx];
200 EXPORT_SYMBOL_GPL(get_phys_to_machine);
202 static void alloc_p2m(unsigned long **pp, unsigned long *mfnp)
207 p = (void *)__get_free_page(GFP_KERNEL | __GFP_NOFAIL);
210 for(i = 0; i < P2M_ENTRIES_PER_PAGE; i++)
211 p[i] = INVALID_P2M_ENTRY;
213 if (cmpxchg(pp, p2m_missing, p) != p2m_missing)
214 free_page((unsigned long)p);
216 *mfnp = virt_to_mfn(p);
219 void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
221 unsigned topidx, idx;
223 if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
224 BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
228 if (unlikely(pfn >= MAX_DOMAIN_PAGES)) {
229 BUG_ON(mfn != INVALID_P2M_ENTRY);
233 topidx = p2m_top_index(pfn);
234 if (p2m_top[topidx] == p2m_missing) {
235 /* no need to allocate a page to store an invalid entry */
236 if (mfn == INVALID_P2M_ENTRY)
238 alloc_p2m(&p2m_top[topidx], &p2m_top_mfn[topidx]);
241 idx = p2m_index(pfn);
242 p2m_top[topidx][idx] = mfn;
245 xmaddr_t arbitrary_virt_to_machine(void *vaddr)
247 unsigned long address = (unsigned long)vaddr;
249 pte_t *pte = lookup_address(address, &level);
250 unsigned offset = address & ~PAGE_MASK;
254 return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
257 void make_lowmem_page_readonly(void *vaddr)
260 unsigned long address = (unsigned long)vaddr;
263 pte = lookup_address(address, &level);
266 ptev = pte_wrprotect(*pte);
268 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
272 void make_lowmem_page_readwrite(void *vaddr)
275 unsigned long address = (unsigned long)vaddr;
278 pte = lookup_address(address, &level);
281 ptev = pte_mkwrite(*pte);
283 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
288 static bool xen_page_pinned(void *ptr)
290 struct page *page = virt_to_page(ptr);
292 return PagePinned(page);
295 static void xen_extend_mmu_update(const struct mmu_update *update)
297 struct multicall_space mcs;
298 struct mmu_update *u;
300 mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u));
302 if (mcs.mc != NULL) {
303 ADD_STATS(mmu_update_extended, 1);
304 ADD_STATS(mmu_update_histo[mcs.mc->args[1]], -1);
308 if (mcs.mc->args[1] < MMU_UPDATE_HISTO)
309 ADD_STATS(mmu_update_histo[mcs.mc->args[1]], 1);
311 ADD_STATS(mmu_update_histo[0], 1);
313 ADD_STATS(mmu_update, 1);
314 mcs = __xen_mc_entry(sizeof(*u));
315 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
316 ADD_STATS(mmu_update_histo[1], 1);
323 void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
331 /* ptr may be ioremapped for 64-bit pagetable setup */
332 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
333 u.val = pmd_val_ma(val);
334 xen_extend_mmu_update(&u);
336 ADD_STATS(pmd_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
338 xen_mc_issue(PARAVIRT_LAZY_MMU);
343 void xen_set_pmd(pmd_t *ptr, pmd_t val)
345 ADD_STATS(pmd_update, 1);
347 /* If page is not pinned, we can just update the entry
349 if (!xen_page_pinned(ptr)) {
354 ADD_STATS(pmd_update_pinned, 1);
356 xen_set_pmd_hyper(ptr, val);
360 * Associate a virtual page frame with a given physical page frame
361 * and protection flags for that frame.
363 void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
365 set_pte_vaddr(vaddr, mfn_pte(mfn, flags));
368 void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
369 pte_t *ptep, pte_t pteval)
371 /* updates to init_mm may be done without lock */
375 ADD_STATS(set_pte_at, 1);
376 // ADD_STATS(set_pte_at_pinned, xen_page_pinned(ptep));
377 ADD_STATS(set_pte_at_current, mm == current->mm);
378 ADD_STATS(set_pte_at_kernel, mm == &init_mm);
380 if (mm == current->mm || mm == &init_mm) {
381 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
382 struct multicall_space mcs;
383 mcs = xen_mc_entry(0);
385 MULTI_update_va_mapping(mcs.mc, addr, pteval, 0);
386 ADD_STATS(set_pte_at_batched, 1);
387 xen_mc_issue(PARAVIRT_LAZY_MMU);
390 if (HYPERVISOR_update_va_mapping(addr, pteval, 0) == 0)
393 xen_set_pte(ptep, pteval);
400 pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
402 /* Just return the pte as-is. We preserve the bits on commit */
406 void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
407 pte_t *ptep, pte_t pte)
413 u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
414 u.val = pte_val_ma(pte);
415 xen_extend_mmu_update(&u);
417 ADD_STATS(prot_commit, 1);
418 ADD_STATS(prot_commit_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
420 xen_mc_issue(PARAVIRT_LAZY_MMU);
423 /* Assume pteval_t is equivalent to all the other *val_t types. */
424 static pteval_t pte_mfn_to_pfn(pteval_t val)
426 if (val & _PAGE_PRESENT) {
427 unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
428 pteval_t flags = val & PTE_FLAGS_MASK;
429 val = ((pteval_t)mfn_to_pfn(mfn) << PAGE_SHIFT) | flags;
435 static pteval_t pte_pfn_to_mfn(pteval_t val)
437 if (val & _PAGE_PRESENT) {
438 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
439 pteval_t flags = val & PTE_FLAGS_MASK;
440 val = ((pteval_t)pfn_to_mfn(pfn) << PAGE_SHIFT) | flags;
446 pteval_t xen_pte_val(pte_t pte)
448 return pte_mfn_to_pfn(pte.pte);
451 pgdval_t xen_pgd_val(pgd_t pgd)
453 return pte_mfn_to_pfn(pgd.pgd);
456 pte_t xen_make_pte(pteval_t pte)
458 pte = pte_pfn_to_mfn(pte);
459 return native_make_pte(pte);
462 pgd_t xen_make_pgd(pgdval_t pgd)
464 pgd = pte_pfn_to_mfn(pgd);
465 return native_make_pgd(pgd);
468 pmdval_t xen_pmd_val(pmd_t pmd)
470 return pte_mfn_to_pfn(pmd.pmd);
473 void xen_set_pud_hyper(pud_t *ptr, pud_t val)
481 /* ptr may be ioremapped for 64-bit pagetable setup */
482 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
483 u.val = pud_val_ma(val);
484 xen_extend_mmu_update(&u);
486 ADD_STATS(pud_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
488 xen_mc_issue(PARAVIRT_LAZY_MMU);
493 void xen_set_pud(pud_t *ptr, pud_t val)
495 ADD_STATS(pud_update, 1);
497 /* If page is not pinned, we can just update the entry
499 if (!xen_page_pinned(ptr)) {
504 ADD_STATS(pud_update_pinned, 1);
506 xen_set_pud_hyper(ptr, val);
509 void xen_set_pte(pte_t *ptep, pte_t pte)
511 ADD_STATS(pte_update, 1);
512 // ADD_STATS(pte_update_pinned, xen_page_pinned(ptep));
513 ADD_STATS(pte_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
515 #ifdef CONFIG_X86_PAE
516 ptep->pte_high = pte.pte_high;
518 ptep->pte_low = pte.pte_low;
524 #ifdef CONFIG_X86_PAE
525 void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
527 set_64bit((u64 *)ptep, native_pte_val(pte));
530 void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
533 smp_wmb(); /* make sure low gets written first */
537 void xen_pmd_clear(pmd_t *pmdp)
539 set_pmd(pmdp, __pmd(0));
541 #endif /* CONFIG_X86_PAE */
543 pmd_t xen_make_pmd(pmdval_t pmd)
545 pmd = pte_pfn_to_mfn(pmd);
546 return native_make_pmd(pmd);
549 #if PAGETABLE_LEVELS == 4
550 pudval_t xen_pud_val(pud_t pud)
552 return pte_mfn_to_pfn(pud.pud);
555 pud_t xen_make_pud(pudval_t pud)
557 pud = pte_pfn_to_mfn(pud);
559 return native_make_pud(pud);
562 pgd_t *xen_get_user_pgd(pgd_t *pgd)
564 pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);
565 unsigned offset = pgd - pgd_page;
566 pgd_t *user_ptr = NULL;
568 if (offset < pgd_index(USER_LIMIT)) {
569 struct page *page = virt_to_page(pgd_page);
570 user_ptr = (pgd_t *)page->private;
578 static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
582 u.ptr = virt_to_machine(ptr).maddr;
583 u.val = pgd_val_ma(val);
584 xen_extend_mmu_update(&u);
588 * Raw hypercall-based set_pgd, intended for in early boot before
589 * there's a page structure. This implies:
590 * 1. The only existing pagetable is the kernel's
591 * 2. It is always pinned
592 * 3. It has no user pagetable attached to it
594 void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
600 __xen_set_pgd_hyper(ptr, val);
602 xen_mc_issue(PARAVIRT_LAZY_MMU);
607 void xen_set_pgd(pgd_t *ptr, pgd_t val)
609 pgd_t *user_ptr = xen_get_user_pgd(ptr);
611 ADD_STATS(pgd_update, 1);
613 /* If page is not pinned, we can just update the entry
615 if (!xen_page_pinned(ptr)) {
618 WARN_ON(xen_page_pinned(user_ptr));
624 ADD_STATS(pgd_update_pinned, 1);
625 ADD_STATS(pgd_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
627 /* If it's pinned, then we can at least batch the kernel and
628 user updates together. */
631 __xen_set_pgd_hyper(ptr, val);
633 __xen_set_pgd_hyper(user_ptr, val);
635 xen_mc_issue(PARAVIRT_LAZY_MMU);
637 #endif /* PAGETABLE_LEVELS == 4 */
640 * (Yet another) pagetable walker. This one is intended for pinning a
641 * pagetable. This means that it walks a pagetable and calls the
642 * callback function on each page it finds making up the page table,
643 * at every level. It walks the entire pagetable, but it only bothers
644 * pinning pte pages which are below limit. In the normal case this
645 * will be STACK_TOP_MAX, but at boot we need to pin up to
648 * For 32-bit the important bit is that we don't pin beyond there,
649 * because then we start getting into Xen's ptes.
651 * For 64-bit, we must skip the Xen hole in the middle of the address
652 * space, just after the big x86-64 virtual hole.
654 static int xen_pgd_walk(pgd_t *pgd, int (*func)(struct page *, enum pt_level),
658 unsigned hole_low, hole_high;
659 unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
660 unsigned pgdidx, pudidx, pmdidx;
662 /* The limit is the last byte to be touched */
664 BUG_ON(limit >= FIXADDR_TOP);
666 if (xen_feature(XENFEAT_auto_translated_physmap))
670 * 64-bit has a great big hole in the middle of the address
671 * space, which contains the Xen mappings. On 32-bit these
672 * will end up making a zero-sized hole and so is a no-op.
674 hole_low = pgd_index(USER_LIMIT);
675 hole_high = pgd_index(PAGE_OFFSET);
677 pgdidx_limit = pgd_index(limit);
679 pudidx_limit = pud_index(limit);
684 pmdidx_limit = pmd_index(limit);
689 for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) {
692 if (pgdidx >= hole_low && pgdidx < hole_high)
695 if (!pgd_val(pgd[pgdidx]))
698 pud = pud_offset(&pgd[pgdidx], 0);
700 if (PTRS_PER_PUD > 1) /* not folded */
701 flush |= (*func)(virt_to_page(pud), PT_PUD);
703 for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) {
706 if (pgdidx == pgdidx_limit &&
707 pudidx > pudidx_limit)
710 if (pud_none(pud[pudidx]))
713 pmd = pmd_offset(&pud[pudidx], 0);
715 if (PTRS_PER_PMD > 1) /* not folded */
716 flush |= (*func)(virt_to_page(pmd), PT_PMD);
718 for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) {
721 if (pgdidx == pgdidx_limit &&
722 pudidx == pudidx_limit &&
723 pmdidx > pmdidx_limit)
726 if (pmd_none(pmd[pmdidx]))
729 pte = pmd_page(pmd[pmdidx]);
730 flush |= (*func)(pte, PT_PTE);
736 /* Do the top level last, so that the callbacks can use it as
737 a cue to do final things like tlb flushes. */
738 flush |= (*func)(virt_to_page(pgd), PT_PGD);
743 /* If we're using split pte locks, then take the page's lock and
744 return a pointer to it. Otherwise return NULL. */
745 static spinlock_t *xen_pte_lock(struct page *page)
747 spinlock_t *ptl = NULL;
749 #if NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS
750 ptl = __pte_lockptr(page);
757 static void xen_pte_unlock(void *v)
763 static void xen_do_pin(unsigned level, unsigned long pfn)
765 struct mmuext_op *op;
766 struct multicall_space mcs;
768 mcs = __xen_mc_entry(sizeof(*op));
771 op->arg1.mfn = pfn_to_mfn(pfn);
772 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
775 static int xen_pin_page(struct page *page, enum pt_level level)
777 unsigned pgfl = TestSetPagePinned(page);
781 flush = 0; /* already pinned */
782 else if (PageHighMem(page))
783 /* kmaps need flushing if we found an unpinned
787 void *pt = lowmem_page_address(page);
788 unsigned long pfn = page_to_pfn(page);
789 struct multicall_space mcs = __xen_mc_entry(0);
795 * We need to hold the pagetable lock between the time
796 * we make the pagetable RO and when we actually pin
797 * it. If we don't, then other users may come in and
798 * attempt to update the pagetable by writing it,
799 * which will fail because the memory is RO but not
800 * pinned, so Xen won't do the trap'n'emulate.
802 * If we're using split pte locks, we can't hold the
803 * entire pagetable's worth of locks during the
804 * traverse, because we may wrap the preempt count (8
805 * bits). The solution is to mark RO and pin each PTE
806 * page while holding the lock. This means the number
807 * of locks we end up holding is never more than a
808 * batch size (~32 entries, at present).
810 * If we're not using split pte locks, we needn't pin
811 * the PTE pages independently, because we're
812 * protected by the overall pagetable lock.
816 ptl = xen_pte_lock(page);
818 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
819 pfn_pte(pfn, PAGE_KERNEL_RO),
820 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
823 xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn);
825 /* Queue a deferred unlock for when this batch
827 xen_mc_callback(xen_pte_unlock, ptl);
834 /* This is called just after a mm has been created, but it has not
835 been used yet. We need to make sure that its pagetable is all
836 read-only, and can be pinned. */
837 void xen_pgd_pin(pgd_t *pgd)
841 if (xen_pgd_walk(pgd, xen_pin_page, USER_LIMIT)) {
842 /* re-enable interrupts for kmap_flush_unused */
850 pgd_t *user_pgd = xen_get_user_pgd(pgd);
852 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));
855 xen_pin_page(virt_to_page(user_pgd), PT_PGD);
856 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(user_pgd)));
859 #else /* CONFIG_X86_32 */
860 #ifdef CONFIG_X86_PAE
861 /* Need to make sure unshared kernel PMD is pinnable */
862 xen_pin_page(virt_to_page(pgd_page(pgd[pgd_index(TASK_SIZE)])), PT_PMD);
864 xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));
865 #endif /* CONFIG_X86_64 */
870 * On save, we need to pin all pagetables to make sure they get their
871 * mfns turned into pfns. Search the list for any unpinned pgds and pin
872 * them (unpinned pgds are not currently in use, probably because the
873 * process is under construction or destruction).
875 void xen_mm_pin_all(void)
880 spin_lock_irqsave(&pgd_lock, flags);
882 list_for_each_entry(page, &pgd_list, lru) {
883 if (!PagePinned(page)) {
884 xen_pgd_pin((pgd_t *)page_address(page));
885 SetPageSavePinned(page);
889 spin_unlock_irqrestore(&pgd_lock, flags);
893 * The init_mm pagetable is really pinned as soon as its created, but
894 * that's before we have page structures to store the bits. So do all
895 * the book-keeping now.
897 static __init int xen_mark_pinned(struct page *page, enum pt_level level)
903 void __init xen_mark_init_mm_pinned(void)
905 xen_pgd_walk(init_mm.pgd, xen_mark_pinned, FIXADDR_TOP);
908 static int xen_unpin_page(struct page *page, enum pt_level level)
910 unsigned pgfl = TestClearPagePinned(page);
912 if (pgfl && !PageHighMem(page)) {
913 void *pt = lowmem_page_address(page);
914 unsigned long pfn = page_to_pfn(page);
915 spinlock_t *ptl = NULL;
916 struct multicall_space mcs;
919 * Do the converse to pin_page. If we're using split
920 * pte locks, we must be holding the lock for while
921 * the pte page is unpinned but still RO to prevent
922 * concurrent updates from seeing it in this
923 * partially-pinned state.
925 if (level == PT_PTE) {
926 ptl = xen_pte_lock(page);
929 xen_do_pin(MMUEXT_UNPIN_TABLE, pfn);
932 mcs = __xen_mc_entry(0);
934 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
935 pfn_pte(pfn, PAGE_KERNEL),
936 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
939 /* unlock when batch completed */
940 xen_mc_callback(xen_pte_unlock, ptl);
944 return 0; /* never need to flush on unpin */
947 /* Release a pagetables pages back as normal RW */
948 static void xen_pgd_unpin(pgd_t *pgd)
952 xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
956 pgd_t *user_pgd = xen_get_user_pgd(pgd);
959 xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(user_pgd)));
960 xen_unpin_page(virt_to_page(user_pgd), PT_PGD);
965 #ifdef CONFIG_X86_PAE
966 /* Need to make sure unshared kernel PMD is unpinned */
967 xen_unpin_page(virt_to_page(pgd_page(pgd[pgd_index(TASK_SIZE)])), PT_PMD);
970 xen_pgd_walk(pgd, xen_unpin_page, USER_LIMIT);
976 * On resume, undo any pinning done at save, so that the rest of the
977 * kernel doesn't see any unexpected pinned pagetables.
979 void xen_mm_unpin_all(void)
984 spin_lock_irqsave(&pgd_lock, flags);
986 list_for_each_entry(page, &pgd_list, lru) {
987 if (PageSavePinned(page)) {
988 BUG_ON(!PagePinned(page));
989 xen_pgd_unpin((pgd_t *)page_address(page));
990 ClearPageSavePinned(page);
994 spin_unlock_irqrestore(&pgd_lock, flags);
997 void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
999 spin_lock(&next->page_table_lock);
1000 xen_pgd_pin(next->pgd);
1001 spin_unlock(&next->page_table_lock);
1004 void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
1006 spin_lock(&mm->page_table_lock);
1007 xen_pgd_pin(mm->pgd);
1008 spin_unlock(&mm->page_table_lock);
1013 /* Another cpu may still have their %cr3 pointing at the pagetable, so
1014 we need to repoint it somewhere else before we can unpin it. */
1015 static void drop_other_mm_ref(void *info)
1017 struct mm_struct *mm = info;
1018 struct mm_struct *active_mm;
1020 #ifdef CONFIG_X86_64
1021 active_mm = read_pda(active_mm);
1023 active_mm = __get_cpu_var(cpu_tlbstate).active_mm;
1026 if (active_mm == mm)
1027 leave_mm(smp_processor_id());
1029 /* If this cpu still has a stale cr3 reference, then make sure
1030 it has been flushed. */
1031 if (x86_read_percpu(xen_current_cr3) == __pa(mm->pgd)) {
1032 load_cr3(swapper_pg_dir);
1033 arch_flush_lazy_cpu_mode();
1037 static void xen_drop_mm_ref(struct mm_struct *mm)
1042 if (current->active_mm == mm) {
1043 if (current->mm == mm)
1044 load_cr3(swapper_pg_dir);
1046 leave_mm(smp_processor_id());
1047 arch_flush_lazy_cpu_mode();
1050 /* Get the "official" set of cpus referring to our pagetable. */
1051 mask = mm->cpu_vm_mask;
1053 /* It's possible that a vcpu may have a stale reference to our
1054 cr3, because its in lazy mode, and it hasn't yet flushed
1055 its set of pending hypercalls yet. In this case, we can
1056 look at its actual current cr3 value, and force it to flush
1058 for_each_online_cpu(cpu) {
1059 if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))
1063 if (!cpus_empty(mask))
1064 smp_call_function_mask(mask, drop_other_mm_ref, mm, 1);
1067 static void xen_drop_mm_ref(struct mm_struct *mm)
1069 if (current->active_mm == mm)
1070 load_cr3(swapper_pg_dir);
1075 * While a process runs, Xen pins its pagetables, which means that the
1076 * hypervisor forces it to be read-only, and it controls all updates
1077 * to it. This means that all pagetable updates have to go via the
1078 * hypervisor, which is moderately expensive.
1080 * Since we're pulling the pagetable down, we switch to use init_mm,
1081 * unpin old process pagetable and mark it all read-write, which
1082 * allows further operations on it to be simple memory accesses.
1084 * The only subtle point is that another CPU may be still using the
1085 * pagetable because of lazy tlb flushing. This means we need need to
1086 * switch all CPUs off this pagetable before we can unpin it.
1088 void xen_exit_mmap(struct mm_struct *mm)
1090 get_cpu(); /* make sure we don't move around */
1091 xen_drop_mm_ref(mm);
1094 spin_lock(&mm->page_table_lock);
1096 /* pgd may not be pinned in the error exit path of execve */
1097 if (xen_page_pinned(mm->pgd))
1098 xen_pgd_unpin(mm->pgd);
1100 spin_unlock(&mm->page_table_lock);
1103 #ifdef CONFIG_XEN_DEBUG_FS
1105 static struct dentry *d_mmu_debug;
1107 static int __init xen_mmu_debugfs(void)
1109 struct dentry *d_xen = xen_init_debugfs();
1114 d_mmu_debug = debugfs_create_dir("mmu", d_xen);
1116 debugfs_create_u8("zero_stats", 0644, d_mmu_debug, &zero_stats);
1118 debugfs_create_u32("pgd_update", 0444, d_mmu_debug, &mmu_stats.pgd_update);
1119 debugfs_create_u32("pgd_update_pinned", 0444, d_mmu_debug,
1120 &mmu_stats.pgd_update_pinned);
1121 debugfs_create_u32("pgd_update_batched", 0444, d_mmu_debug,
1122 &mmu_stats.pgd_update_pinned);
1124 debugfs_create_u32("pud_update", 0444, d_mmu_debug, &mmu_stats.pud_update);
1125 debugfs_create_u32("pud_update_pinned", 0444, d_mmu_debug,
1126 &mmu_stats.pud_update_pinned);
1127 debugfs_create_u32("pud_update_batched", 0444, d_mmu_debug,
1128 &mmu_stats.pud_update_pinned);
1130 debugfs_create_u32("pmd_update", 0444, d_mmu_debug, &mmu_stats.pmd_update);
1131 debugfs_create_u32("pmd_update_pinned", 0444, d_mmu_debug,
1132 &mmu_stats.pmd_update_pinned);
1133 debugfs_create_u32("pmd_update_batched", 0444, d_mmu_debug,
1134 &mmu_stats.pmd_update_pinned);
1136 debugfs_create_u32("pte_update", 0444, d_mmu_debug, &mmu_stats.pte_update);
1137 // debugfs_create_u32("pte_update_pinned", 0444, d_mmu_debug,
1138 // &mmu_stats.pte_update_pinned);
1139 debugfs_create_u32("pte_update_batched", 0444, d_mmu_debug,
1140 &mmu_stats.pte_update_pinned);
1142 debugfs_create_u32("mmu_update", 0444, d_mmu_debug, &mmu_stats.mmu_update);
1143 debugfs_create_u32("mmu_update_extended", 0444, d_mmu_debug,
1144 &mmu_stats.mmu_update_extended);
1145 xen_debugfs_create_u32_array("mmu_update_histo", 0444, d_mmu_debug,
1146 mmu_stats.mmu_update_histo, 20);
1148 debugfs_create_u32("set_pte_at", 0444, d_mmu_debug, &mmu_stats.set_pte_at);
1149 debugfs_create_u32("set_pte_at_batched", 0444, d_mmu_debug,
1150 &mmu_stats.set_pte_at_batched);
1151 debugfs_create_u32("set_pte_at_current", 0444, d_mmu_debug,
1152 &mmu_stats.set_pte_at_current);
1153 debugfs_create_u32("set_pte_at_kernel", 0444, d_mmu_debug,
1154 &mmu_stats.set_pte_at_kernel);
1156 debugfs_create_u32("prot_commit", 0444, d_mmu_debug, &mmu_stats.prot_commit);
1157 debugfs_create_u32("prot_commit_batched", 0444, d_mmu_debug,
1158 &mmu_stats.prot_commit_batched);
1162 fs_initcall(xen_mmu_debugfs);
1164 #endif /* CONFIG_XEN_DEBUG_FS */