arch/x86/mm/ioremap.c

   1 /*
   2  * Re-map IO memory to kernel address space so that we can access it.
   3  * This is needed for high PCI addresses that aren't mapped in the
   4  * 640k-1MB IO memory area on PC's
   5  *
   6  * (C) Copyright 1995 1996 Linus Torvalds
   7  */
   8
   9 #include <linux/bootmem.h>
  10 #include <linux/init.h>
  11 #include <linux/io.h>
  12 #include <linux/module.h>
  13 #include <linux/slab.h>
  14 #include <linux/vmalloc.h>
  15
  16 #include <asm/cacheflush.h>
  17 #include <asm/e820.h>
  18 #include <asm/fixmap.h>
  19 #include <asm/pgtable.h>
  20 #include <asm/tlbflush.h>
  21 #include <asm/pgalloc.h>
  22 #include <asm/pat.h>
  23
  24 #ifdef CONFIG_X86_64
  25
  26 unsigned long __phys_addr(unsigned long x)
  27 {
  28         if (x >= __START_KERNEL_map)
  29                 return x - __START_KERNEL_map + phys_base;
  30         return x - PAGE_OFFSET;
  31 }
  32 EXPORT_SYMBOL(__phys_addr);
  33
  34 static inline int phys_addr_valid(unsigned long addr)
  35 {
  36         return addr < (1UL << boot_cpu_data.x86_phys_bits);
  37 }
  38
  39 #else
  40
  41 static inline int phys_addr_valid(unsigned long addr)
  42 {
  43         return 1;
  44 }
  45
  46 #endif
  47
  48 int page_is_ram(unsigned long pagenr)
  49 {
  50         unsigned long addr, end;
  51         int i;
  52
  53         /*
  54          * A special case is the first 4Kb of memory;
  55          * This is a BIOS owned area, not kernel ram, but generally
  56          * not listed as such in the E820 table.
  57          */
  58         if (pagenr == 0)
  59                 return 0;
  60
  61         /*
  62          * Second special case: Some BIOSen report the PC BIOS
  63          * area (640->1Mb) as ram even though it is not.
  64          */
  65         if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
  66                     pagenr < (BIOS_END >> PAGE_SHIFT))
  67                 return 0;
  68
  69         for (i = 0; i < e820.nr_map; i++) {
  70                 /*
  71                  * Not usable memory:
  72                  */
  73                 if (e820.map[i].type != E820_RAM)
  74                         continue;
  75                 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
  76                 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
  77
  78
  79                 if ((pagenr >= addr) && (pagenr < end))
  80                         return 1;
  81         }
  82         return 0;
  83 }
  84
  85 /*
  86  * Fix up the linear direct mapping of the kernel to avoid cache attribute
  87  * conflicts.
  88  */
  89 int ioremap_change_attr(unsigned long vaddr, unsigned long size,
  90                                unsigned long prot_val)
  91 {
  92         unsigned long nrpages = size >> PAGE_SHIFT;
  93         int err;
  94
  95         switch (prot_val) {
  96         case _PAGE_CACHE_UC:
  97         default:
  98                 err = _set_memory_uc(vaddr, nrpages);
  99                 break;
 100         case _PAGE_CACHE_WC:
 101                 err = _set_memory_wc(vaddr, nrpages);
 102                 break;
 103         case _PAGE_CACHE_WB:
 104                 err = _set_memory_wb(vaddr, nrpages);
 105                 break;
 106         }
 107
 108         return err;
 109 }
 110
 111 /*
 112  * Remap an arbitrary physical address space into the kernel virtual
 113  * address space. Needed when the kernel wants to access high addresses
 114  * directly.
 115  *
 116  * NOTE! We need to allow non-page-aligned mappings too: we will obviously
 117  * have to convert them into an offset in a page-aligned mapping, but the
 118  * caller shouldn't need to know that small detail.
 119  */
 120 static void __iomem *__ioremap(resource_size_t phys_addr, unsigned long size,
 121                                unsigned long prot_val)
 122 {
 123         unsigned long pfn, offset, last_addr, vaddr;
 124         struct vm_struct *area;
 125         unsigned long new_prot_val;
 126         pgprot_t prot;
 127
 128         /* Don't allow wraparound or zero size */
 129         last_addr = phys_addr + size - 1;
 130         if (!size || last_addr < phys_addr)
 131                 return NULL;
 132
 133         if (!phys_addr_valid(phys_addr)) {
 134                 printk(KERN_WARNING "ioremap: invalid physical address %lx\n",
 135                        phys_addr);
 136                 WARN_ON_ONCE(1);
 137                 return NULL;
 138         }
 139
 140         /*
 141          * Don't remap the low PCI/ISA area, it's always mapped..
 142          */
 143         if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
 144                 return (__force void __iomem *)phys_to_virt(phys_addr);
 145
 146         /*
 147          * Don't allow anybody to remap normal RAM that we're using..
 148          */
 149         for (pfn = phys_addr >> PAGE_SHIFT;
 150                                 (pfn << PAGE_SHIFT) < last_addr; pfn++) {
 151
 152                 int is_ram = page_is_ram(pfn);
 153
 154                 if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
 155                         return NULL;
 156                 WARN_ON_ONCE(is_ram);
 157         }
 158
 159         /*
 160          * Mappings have to be page-aligned
 161          */
 162         offset = phys_addr & ~PAGE_MASK;
 163         phys_addr &= PAGE_MASK;
 164         size = PAGE_ALIGN(last_addr+1) - phys_addr;
 165
 166         if (reserve_memtype(phys_addr, phys_addr + size,
 167                             prot_val, &new_prot_val)) {
 168                 /*
 169                  * Do not fallback to certain memory types with certain
 170                  * requested type:
 171                  * - request is uncached, return cannot be write-back
 172                  * - request is uncached, return cannot be write-combine
 173                  * - request is write-combine, return cannot be write-back
 174                  */
 175                 if ((prot_val == _PAGE_CACHE_UC &&
 176                      (new_prot_val == _PAGE_CACHE_WB ||
 177                       new_prot_val == _PAGE_CACHE_WC)) ||
 178                     (prot_val == _PAGE_CACHE_WC &&
 179                      new_prot_val == _PAGE_CACHE_WB)) {
 180                         free_memtype(phys_addr, phys_addr + size);
 181                         return NULL;
 182                 }
 183                 prot_val = new_prot_val;
 184         }
 185
 186         switch (prot_val) {
 187         case _PAGE_CACHE_UC:
 188         default:
 189                 prot = PAGE_KERNEL_NOCACHE;
 190                 break;
 191         case _PAGE_CACHE_WC:
 192                 prot = PAGE_KERNEL_WC;
 193                 break;
 194         case _PAGE_CACHE_WB:
 195                 prot = PAGE_KERNEL;
 196                 break;
 197         }
 198
 199         /*
 200          * Ok, go for it..
 201          */
 202         area = get_vm_area(size, VM_IOREMAP);
 203         if (!area)
 204                 return NULL;
 205         area->phys_addr = phys_addr;
 206         vaddr = (unsigned long) area->addr;
 207         if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) {
 208                 free_memtype(phys_addr, phys_addr + size);
 209                 free_vm_area(area);
 210                 return NULL;
 211         }
 212
 213         if (ioremap_change_attr(vaddr, size, prot_val) < 0) {
 214                 free_memtype(phys_addr, phys_addr + size);
 215                 vunmap(area->addr);
 216                 return NULL;
 217         }
 218
 219         return (void __iomem *) (vaddr + offset);
 220 }
 221
 222 /**
 223  * ioremap_nocache     -   map bus memory into CPU space
 224  * @offset:    bus address of the memory
 225  * @size:      size of the resource to map
 226  *
 227  * ioremap_nocache performs a platform specific sequence of operations to
 228  * make bus memory CPU accessible via the readb/readw/readl/writeb/
 229  * writew/writel functions and the other mmio helpers. The returned
 230  * address is not guaranteed to be usable directly as a virtual
 231  * address.
 232  *
 233  * This version of ioremap ensures that the memory is marked uncachable
 234  * on the CPU as well as honouring existing caching rules from things like
 235  * the PCI bus. Note that there are other caches and buffers on many
 236  * busses. In particular driver authors should read up on PCI writes
 237  *
 238  * It's useful if some control registers are in such an area and
 239  * write combining or read caching is not desirable:
 240  *
 241  * Must be freed with iounmap.
 242  */
 243 void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
 244 {
 245         return __ioremap(phys_addr, size, _PAGE_CACHE_UC);
 246 }
 247 EXPORT_SYMBOL(ioremap_nocache);
 248
 249 /**
 250  * ioremap_wc   -       map memory into CPU space write combined
 251  * @offset:     bus address of the memory
 252  * @size:       size of the resource to map
 253  *
 254  * This version of ioremap ensures that the memory is marked write combining.
 255  * Write combining allows faster writes to some hardware devices.
 256  *
 257  * Must be freed with iounmap.
 258  */
 259 void __iomem *ioremap_wc(unsigned long phys_addr, unsigned long size)
 260 {
 261         if (pat_wc_enabled)
 262                 return __ioremap(phys_addr, size, _PAGE_CACHE_WC);
 263         else
 264                 return ioremap_nocache(phys_addr, size);
 265 }
 266 EXPORT_SYMBOL(ioremap_wc);
 267
 268 void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
 269 {
 270         return __ioremap(phys_addr, size, _PAGE_CACHE_WB);
 271 }
 272 EXPORT_SYMBOL(ioremap_cache);
 273
 274 /**
 275  * iounmap - Free a IO remapping
 276  * @addr: virtual address from ioremap_*
 277  *
 278  * Caller must ensure there is only one unmapping for the same pointer.
 279  */
 280 void iounmap(volatile void __iomem *addr)
 281 {
 282         struct vm_struct *p, *o;
 283
 284         if ((void __force *)addr <= high_memory)
 285                 return;
 286
 287         /*
 288          * __ioremap special-cases the PCI/ISA range by not instantiating a
 289          * vm_area and by simply returning an address into the kernel mapping
 290          * of ISA space.   So handle that here.
 291          */
 292         if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
 293             addr < phys_to_virt(ISA_END_ADDRESS))
 294                 return;
 295
 296         addr = (volatile void __iomem *)
 297                 (PAGE_MASK & (unsigned long __force)addr);
 298
 299         /* Use the vm area unlocked, assuming the caller
 300            ensures there isn't another iounmap for the same address
 301            in parallel. Reuse of the virtual address is prevented by
 302            leaving it in the global lists until we're done with it.
 303            cpa takes care of the direct mappings. */
 304         read_lock(&vmlist_lock);
 305         for (p = vmlist; p; p = p->next) {
 306                 if (p->addr == addr)
 307                         break;
 308         }
 309         read_unlock(&vmlist_lock);
 310
 311         if (!p) {
 312                 printk(KERN_ERR "iounmap: bad address %p\n", addr);
 313                 dump_stack();
 314                 return;
 315         }
 316
 317         free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
 318
 319         /* Finally remove it */
 320         o = remove_vm_area((void *)addr);
 321         BUG_ON(p != o || o == NULL);
 322         kfree(p);
 323 }
 324 EXPORT_SYMBOL(iounmap);
 325
 326 #ifdef CONFIG_X86_32
 327
 328 int __initdata early_ioremap_debug;
 329
 330 static int __init early_ioremap_debug_setup(char *str)
 331 {
 332         early_ioremap_debug = 1;
 333
 334         return 0;
 335 }
 336 early_param("early_ioremap_debug", early_ioremap_debug_setup);
 337
 338 static __initdata int after_paging_init;
 339 static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)]
 340                 __section(.bss.page_aligned);
 341
 342 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
 343 {
 344         /* Don't assume we're using swapper_pg_dir at this point */
 345         pgd_t *base = __va(read_cr3());
 346         pgd_t *pgd = &base[pgd_index(addr)];
 347         pud_t *pud = pud_offset(pgd, addr);
 348         pmd_t *pmd = pmd_offset(pud, addr);
 349
 350         return pmd;
 351 }
 352
 353 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
 354 {
 355         return &bm_pte[pte_index(addr)];
 356 }
 357
 358 void __init early_ioremap_init(void)
 359 {
 360         pmd_t *pmd;
 361
 362         if (early_ioremap_debug)
 363                 printk(KERN_INFO "early_ioremap_init()\n");
 364
 365         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
 366         memset(bm_pte, 0, sizeof(bm_pte));
 367         pmd_populate_kernel(&init_mm, pmd, bm_pte);
 368
 369         /*
 370          * The boot-ioremap range spans multiple pmds, for which
 371          * we are not prepared:
 372          */
 373         if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
 374                 WARN_ON(1);
 375                 printk(KERN_WARNING "pmd %p != %p\n",
 376                        pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
 377                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
 378                         fix_to_virt(FIX_BTMAP_BEGIN));
 379                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END):   %08lx\n",
 380                         fix_to_virt(FIX_BTMAP_END));
 381
 382                 printk(KERN_WARNING "FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
 383                 printk(KERN_WARNING "FIX_BTMAP_BEGIN:     %d\n",
 384                        FIX_BTMAP_BEGIN);
 385         }
 386 }
 387
 388 void __init early_ioremap_clear(void)
 389 {
 390         pmd_t *pmd;
 391
 392         if (early_ioremap_debug)
 393                 printk(KERN_INFO "early_ioremap_clear()\n");
 394
 395         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
 396         pmd_clear(pmd);
 397         paravirt_release_pt(__pa(bm_pte) >> PAGE_SHIFT);
 398         __flush_tlb_all();
 399 }
 400
 401 void __init early_ioremap_reset(void)
 402 {
 403         enum fixed_addresses idx;
 404         unsigned long addr, phys;
 405         pte_t *pte;
 406
 407         after_paging_init = 1;
 408         for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
 409                 addr = fix_to_virt(idx);
 410                 pte = early_ioremap_pte(addr);
 411                 if (pte_present(*pte)) {
 412                         phys = pte_val(*pte) & PAGE_MASK;
 413                         set_fixmap(idx, phys);
 414                 }
 415         }
 416 }
 417
 418 static void __init __early_set_fixmap(enum fixed_addresses idx,
 419                                    unsigned long phys, pgprot_t flags)
 420 {
 421         unsigned long addr = __fix_to_virt(idx);
 422         pte_t *pte;
 423
 424         if (idx >= __end_of_fixed_addresses) {
 425                 BUG();
 426                 return;
 427         }
 428         pte = early_ioremap_pte(addr);
 429         if (pgprot_val(flags))
 430                 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
 431         else
 432                 pte_clear(NULL, addr, pte);
 433         __flush_tlb_one(addr);
 434 }
 435
 436 static inline void __init early_set_fixmap(enum fixed_addresses idx,
 437                                         unsigned long phys)
 438 {
 439         if (after_paging_init)
 440                 set_fixmap(idx, phys);
 441         else
 442                 __early_set_fixmap(idx, phys, PAGE_KERNEL);
 443 }
 444
 445 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
 446 {
 447         if (after_paging_init)
 448                 clear_fixmap(idx);
 449         else
 450                 __early_set_fixmap(idx, 0, __pgprot(0));
 451 }
 452
 453
 454 int __initdata early_ioremap_nested;
 455
 456 static int __init check_early_ioremap_leak(void)
 457 {
 458         if (!early_ioremap_nested)
 459                 return 0;
 460
 461         printk(KERN_WARNING
 462                "Debug warning: early ioremap leak of %d areas detected.\n",
 463                early_ioremap_nested);
 464         printk(KERN_WARNING
 465                "please boot with early_ioremap_debug and report the dmesg.\n");
 466         WARN_ON(1);
 467
 468         return 1;
 469 }
 470 late_initcall(check_early_ioremap_leak);
 471
 472 void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
 473 {
 474         unsigned long offset, last_addr;
 475         unsigned int nrpages, nesting;
 476         enum fixed_addresses idx0, idx;
 477
 478         WARN_ON(system_state != SYSTEM_BOOTING);
 479
 480         nesting = early_ioremap_nested;
 481         if (early_ioremap_debug) {
 482                 printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
 483                        phys_addr, size, nesting);
 484                 dump_stack();
 485         }
 486
 487         /* Don't allow wraparound or zero size */
 488         last_addr = phys_addr + size - 1;
 489         if (!size || last_addr < phys_addr) {
 490                 WARN_ON(1);
 491                 return NULL;
 492         }
 493
 494         if (nesting >= FIX_BTMAPS_NESTING) {
 495                 WARN_ON(1);
 496                 return NULL;
 497         }
 498         early_ioremap_nested++;
 499         /*
 500          * Mappings have to be page-aligned
 501          */
 502         offset = phys_addr & ~PAGE_MASK;
 503         phys_addr &= PAGE_MASK;
 504         size = PAGE_ALIGN(last_addr) - phys_addr;
 505
 506         /*
 507          * Mappings have to fit in the FIX_BTMAP area.
 508          */
 509         nrpages = size >> PAGE_SHIFT;
 510         if (nrpages > NR_FIX_BTMAPS) {
 511                 WARN_ON(1);
 512                 return NULL;
 513         }
 514
 515         /*
 516          * Ok, go for it..
 517          */
 518         idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
 519         idx = idx0;
 520         while (nrpages > 0) {
 521                 early_set_fixmap(idx, phys_addr);
 522                 phys_addr += PAGE_SIZE;
 523                 --idx;
 524                 --nrpages;
 525         }
 526         if (early_ioremap_debug)
 527                 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
 528
 529         return (void *) (offset + fix_to_virt(idx0));
 530 }
 531
 532 void __init early_iounmap(void *addr, unsigned long size)
 533 {
 534         unsigned long virt_addr;
 535         unsigned long offset;
 536         unsigned int nrpages;
 537         enum fixed_addresses idx;
 538         unsigned int nesting;
 539
 540         nesting = --early_ioremap_nested;
 541         WARN_ON(nesting < 0);
 542
 543         if (early_ioremap_debug) {
 544                 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
 545                        size, nesting);
 546                 dump_stack();
 547         }
 548
 549         virt_addr = (unsigned long)addr;
 550         if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
 551                 WARN_ON(1);
 552                 return;
 553         }
 554         offset = virt_addr & ~PAGE_MASK;
 555         nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
 556
 557         idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
 558         while (nrpages > 0) {
 559                 early_clear_fixmap(idx);
 560                 --idx;
 561                 --nrpages;
 562         }
 563 }
 564
 565 void __this_fixmap_does_not_exist(void)
 566 {
 567         WARN_ON(1);
 568 }
 569
 570 #endif /* CONFIG_X86_32 */