2 * Re-map IO memory to kernel address space so that we can access it.
3 * This is needed for high PCI addresses that aren't mapped in the
4 * 640k-1MB IO memory area on PC's
6 * (C) Copyright 1995 1996 Linus Torvalds
9 #include <linux/bootmem.h>
10 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/slab.h>
14 #include <linux/vmalloc.h>
16 #include <asm/cacheflush.h>
18 #include <asm/fixmap.h>
19 #include <asm/pgtable.h>
20 #include <asm/tlbflush.h>
21 #include <asm/pgalloc.h>
30 unsigned long __phys_addr(unsigned long x)
32 if (x >= __START_KERNEL_map)
33 return x - __START_KERNEL_map + phys_base;
34 return x - PAGE_OFFSET;
36 EXPORT_SYMBOL(__phys_addr);
40 int page_is_ram(unsigned long pagenr)
42 unsigned long addr, end;
46 * A special case is the first 4Kb of memory;
47 * This is a BIOS owned area, not kernel ram, but generally
48 * not listed as such in the E820 table.
54 for (i = 0; i < e820.nr_map; i++) {
58 if (e820.map[i].type != E820_RAM)
60 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
61 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
64 * Sanity check: Some BIOSen report areas as RAM that
65 * are not. Notably the 640->1Mb area, which is the
68 if (addr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
69 end < (BIOS_END >> PAGE_SHIFT))
72 if ((pagenr >= addr) && (pagenr < end))
79 * Fix up the linear direct mapping of the kernel to avoid cache attribute
82 static int ioremap_change_attr(unsigned long vaddr, unsigned long size,
83 enum ioremap_mode mode)
85 unsigned long nrpages = size >> PAGE_SHIFT;
89 case IOR_MODE_UNCACHED:
91 err = set_memory_uc(vaddr, nrpages);
94 err = set_memory_wb(vaddr, nrpages);
102 * Remap an arbitrary physical address space into the kernel virtual
103 * address space. Needed when the kernel wants to access high addresses
106 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
107 * have to convert them into an offset in a page-aligned mapping, but the
108 * caller shouldn't need to know that small detail.
110 static void __iomem *__ioremap(unsigned long phys_addr, unsigned long size,
111 enum ioremap_mode mode)
113 unsigned long pfn, offset, last_addr, vaddr;
114 struct vm_struct *area;
117 /* Don't allow wraparound or zero size */
118 last_addr = phys_addr + size - 1;
119 if (!size || last_addr < phys_addr)
123 * Don't remap the low PCI/ISA area, it's always mapped..
125 if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
126 return (__force void __iomem *)phys_to_virt(phys_addr);
129 * Don't allow anybody to remap normal RAM that we're using..
131 for (pfn = phys_addr >> PAGE_SHIFT; pfn < max_pfn_mapped &&
132 (pfn << PAGE_SHIFT) < last_addr; pfn++) {
133 if (page_is_ram(pfn) && pfn_valid(pfn) &&
134 !PageReserved(pfn_to_page(pfn)))
139 case IOR_MODE_UNCACHED:
141 prot = PAGE_KERNEL_NOCACHE;
143 case IOR_MODE_CACHED:
149 * Mappings have to be page-aligned
151 offset = phys_addr & ~PAGE_MASK;
152 phys_addr &= PAGE_MASK;
153 size = PAGE_ALIGN(last_addr+1) - phys_addr;
158 area = get_vm_area(size, VM_IOREMAP);
161 area->phys_addr = phys_addr;
162 vaddr = (unsigned long) area->addr;
163 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) {
164 remove_vm_area((void *)(vaddr & PAGE_MASK));
168 if (ioremap_change_attr(vaddr, size, mode) < 0) {
173 return (void __iomem *) (vaddr + offset);
177 * ioremap_nocache - map bus memory into CPU space
178 * @offset: bus address of the memory
179 * @size: size of the resource to map
181 * ioremap_nocache performs a platform specific sequence of operations to
182 * make bus memory CPU accessible via the readb/readw/readl/writeb/
183 * writew/writel functions and the other mmio helpers. The returned
184 * address is not guaranteed to be usable directly as a virtual
187 * This version of ioremap ensures that the memory is marked uncachable
188 * on the CPU as well as honouring existing caching rules from things like
189 * the PCI bus. Note that there are other caches and buffers on many
190 * busses. In particular driver authors should read up on PCI writes
192 * It's useful if some control registers are in such an area and
193 * write combining or read caching is not desirable:
195 * Must be freed with iounmap.
197 void __iomem *ioremap_nocache(unsigned long phys_addr, unsigned long size)
199 return __ioremap(phys_addr, size, IOR_MODE_UNCACHED);
201 EXPORT_SYMBOL(ioremap_nocache);
203 void __iomem *ioremap_cache(unsigned long phys_addr, unsigned long size)
205 return __ioremap(phys_addr, size, IOR_MODE_CACHED);
207 EXPORT_SYMBOL(ioremap_cache);
210 * iounmap - Free a IO remapping
211 * @addr: virtual address from ioremap_*
213 * Caller must ensure there is only one unmapping for the same pointer.
215 void iounmap(volatile void __iomem *addr)
217 struct vm_struct *p, *o;
219 if ((void __force *)addr <= high_memory)
223 * __ioremap special-cases the PCI/ISA range by not instantiating a
224 * vm_area and by simply returning an address into the kernel mapping
225 * of ISA space. So handle that here.
227 if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
228 addr < phys_to_virt(ISA_END_ADDRESS))
231 addr = (volatile void __iomem *)
232 (PAGE_MASK & (unsigned long __force)addr);
234 /* Use the vm area unlocked, assuming the caller
235 ensures there isn't another iounmap for the same address
236 in parallel. Reuse of the virtual address is prevented by
237 leaving it in the global lists until we're done with it.
238 cpa takes care of the direct mappings. */
239 read_lock(&vmlist_lock);
240 for (p = vmlist; p; p = p->next) {
244 read_unlock(&vmlist_lock);
247 printk(KERN_ERR "iounmap: bad address %p\n", addr);
252 /* Finally remove it */
253 o = remove_vm_area((void *)addr);
254 BUG_ON(p != o || o == NULL);
257 EXPORT_SYMBOL(iounmap);
261 int __initdata early_ioremap_debug;
263 static int __init early_ioremap_debug_setup(char *str)
265 early_ioremap_debug = 1;
269 early_param("early_ioremap_debug", early_ioremap_debug_setup);
271 static __initdata int after_paging_init;
272 static __initdata pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)]
273 __attribute__((aligned(PAGE_SIZE)));
275 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
277 /* Don't assume we're using swapper_pg_dir at this point */
278 pgd_t *base = __va(read_cr3());
279 pgd_t *pgd = &base[pgd_index(addr)];
280 pud_t *pud = pud_offset(pgd, addr);
281 pmd_t *pmd = pmd_offset(pud, addr);
286 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
288 return &bm_pte[pte_index(addr)];
291 void __init early_ioremap_init(void)
295 if (early_ioremap_debug)
296 printk(KERN_INFO "early_ioremap_init()\n");
298 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
299 memset(bm_pte, 0, sizeof(bm_pte));
300 pmd_populate_kernel(&init_mm, pmd, bm_pte);
303 * The boot-ioremap range spans multiple pmds, for which
304 * we are not prepared:
306 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
308 printk(KERN_WARNING "pmd %p != %p\n",
309 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
310 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
311 fix_to_virt(FIX_BTMAP_BEGIN));
312 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
313 fix_to_virt(FIX_BTMAP_END));
315 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
316 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
321 void __init early_ioremap_clear(void)
325 if (early_ioremap_debug)
326 printk(KERN_INFO "early_ioremap_clear()\n");
328 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
330 paravirt_release_pt(__pa(bm_pte) >> PAGE_SHIFT);
334 void __init early_ioremap_reset(void)
336 enum fixed_addresses idx;
337 unsigned long addr, phys;
340 after_paging_init = 1;
341 for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
342 addr = fix_to_virt(idx);
343 pte = early_ioremap_pte(addr);
344 if (pte_present(*pte)) {
345 phys = pte_val(*pte) & PAGE_MASK;
346 set_fixmap(idx, phys);
351 static void __init __early_set_fixmap(enum fixed_addresses idx,
352 unsigned long phys, pgprot_t flags)
354 unsigned long addr = __fix_to_virt(idx);
357 if (idx >= __end_of_fixed_addresses) {
361 pte = early_ioremap_pte(addr);
362 if (pgprot_val(flags))
363 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
365 pte_clear(NULL, addr, pte);
366 __flush_tlb_one(addr);
369 static inline void __init early_set_fixmap(enum fixed_addresses idx,
372 if (after_paging_init)
373 set_fixmap(idx, phys);
375 __early_set_fixmap(idx, phys, PAGE_KERNEL);
378 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
380 if (after_paging_init)
383 __early_set_fixmap(idx, 0, __pgprot(0));
387 int __initdata early_ioremap_nested;
389 static int __init check_early_ioremap_leak(void)
391 if (!early_ioremap_nested)
395 "Debug warning: early ioremap leak of %d areas detected.\n",
396 early_ioremap_nested);
398 "please boot with early_ioremap_debug and report the dmesg.\n");
403 late_initcall(check_early_ioremap_leak);
405 void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
407 unsigned long offset, last_addr;
408 unsigned int nrpages, nesting;
409 enum fixed_addresses idx0, idx;
411 WARN_ON(system_state != SYSTEM_BOOTING);
413 nesting = early_ioremap_nested;
414 if (early_ioremap_debug) {
415 printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
416 phys_addr, size, nesting);
420 /* Don't allow wraparound or zero size */
421 last_addr = phys_addr + size - 1;
422 if (!size || last_addr < phys_addr) {
427 if (nesting >= FIX_BTMAPS_NESTING) {
431 early_ioremap_nested++;
433 * Mappings have to be page-aligned
435 offset = phys_addr & ~PAGE_MASK;
436 phys_addr &= PAGE_MASK;
437 size = PAGE_ALIGN(last_addr) - phys_addr;
440 * Mappings have to fit in the FIX_BTMAP area.
442 nrpages = size >> PAGE_SHIFT;
443 if (nrpages > NR_FIX_BTMAPS) {
451 idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
453 while (nrpages > 0) {
454 early_set_fixmap(idx, phys_addr);
455 phys_addr += PAGE_SIZE;
459 if (early_ioremap_debug)
460 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
462 return (void *) (offset + fix_to_virt(idx0));
465 void __init early_iounmap(void *addr, unsigned long size)
467 unsigned long virt_addr;
468 unsigned long offset;
469 unsigned int nrpages;
470 enum fixed_addresses idx;
471 unsigned int nesting;
473 nesting = --early_ioremap_nested;
474 WARN_ON(nesting < 0);
476 if (early_ioremap_debug) {
477 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
482 virt_addr = (unsigned long)addr;
483 if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
487 offset = virt_addr & ~PAGE_MASK;
488 nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
490 idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
491 while (nrpages > 0) {
492 early_clear_fixmap(idx);
498 void __this_fixmap_does_not_exist(void)
503 #endif /* CONFIG_X86_32 */