arch/arm/plat-omap/dsp/dsp_mem.c

   1 /*
   2  * linux/arch/arm/mach-omap/dsp/dsp_mem.c
   3  *
   4  * OMAP DSP memory driver
   5  *
   6  * Copyright (C) 2002-2005 Nokia Corporation
   7  *
   8  * Written by Toshihiro Kobayashi <toshihiro.kobayashi@nokia.com>
   9  *
  10  * Conversion to mempool API and ARM MMU section mapping
  11  * by Paul Mundt <paul.mundt@nokia.com>
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program; if not, write to the Free Software
  25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  26  *
  27  * Toshihiro Kobayashi <toshihiro.kobayashi@nokia.com>
  28  * 2005/06/09:  DSP Gateway version 3.3
  29  */
  30
  31 #include <linux/module.h>
  32 #include <linux/init.h>
  33 #include <linux/major.h>
  34 #include <linux/fs.h>
  35 #include <linux/bootmem.h>
  36 #include <linux/fb.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/delay.h>
  39 #include <linux/mempool.h>
  40 #include <linux/platform_device.h>
  41 #include <linux/clk.h>
  42 #include <asm/uaccess.h>
  43 #include <asm/io.h>
  44 #include <asm/ioctls.h>
  45 #include <asm/irq.h>
  46 #include <asm/pgalloc.h>
  47 #include <asm/pgtable.h>
  48 #include <asm/arch/tc.h>
  49 #include <asm/arch/omapfb.h>
  50 #include <asm/arch/dsp.h>
  51 #include <asm/arch/dsp_common.h>
  52 #include "uaccess_dsp.h"
  53 #include "ipbuf.h"
  54 #include "dsp.h"
  55
  56 #define SZ_1MB  0x100000
  57 #define SZ_64KB 0x10000
  58 #define SZ_4KB  0x1000
  59 #define SZ_1KB  0x400
  60 #define is_aligned(adr,align)   (!((adr)&((align)-1)))
  61 #define ORDER_1MB       (20 - PAGE_SHIFT)
  62 #define ORDER_64KB      (16 - PAGE_SHIFT)
  63 #define ORDER_4KB       (12 - PAGE_SHIFT)
  64
  65 #define PGDIR_MASK              (~(PGDIR_SIZE-1))
  66 #define PGDIR_ALIGN(addr)       (((addr)+PGDIR_SIZE-1)&(PGDIR_MASK))
  67
  68 #define dsp_mmu_enable() \
  69         do { \
  70                 omap_writew(DSPMMU_CNTL_MMU_EN | DSPMMU_CNTL_RESET_SW, \
  71                             DSPMMU_CNTL); \
  72         } while(0)
  73 #define dsp_mmu_disable() \
  74         do { omap_writew(0, DSPMMU_CNTL); } while(0)
  75 #define dsp_mmu_flush() \
  76         do { \
  77                 omap_writew(DSPMMU_FLUSH_ENTRY_FLUSH_ENTRY, \
  78                             DSPMMU_FLUSH_ENTRY); \
  79         } while(0)
  80 #define __dsp_mmu_gflush() \
  81         do { omap_writew(DSPMMU_GFLUSH_GFLUSH, DSPMMU_GFLUSH); } while(0)
  82 #define __dsp_mmu_itack() \
  83         do { omap_writew(DSPMMU_IT_ACK_IT_ACK, DSPMMU_IT_ACK); } while(0)
  84
  85 #define EMIF_PRIO_LB_MASK       0x0000f000
  86 #define EMIF_PRIO_LB_SHIFT      12
  87 #define EMIF_PRIO_DMA_MASK      0x00000f00
  88 #define EMIF_PRIO_DMA_SHIFT     8
  89 #define EMIF_PRIO_DSP_MASK      0x00000070
  90 #define EMIF_PRIO_DSP_SHIFT     4
  91 #define EMIF_PRIO_MPU_MASK      0x00000007
  92 #define EMIF_PRIO_MPU_SHIFT     0
  93 #define set_emiff_dma_prio(prio) \
  94         do { \
  95                 omap_writel((omap_readl(OMAP_TC_OCPT1_PRIOR) & \
  96         ~EMIF_PRIO_DMA_MASK) | \
  97                             ((prio) << EMIF_PRIO_DMA_SHIFT), \
  98                             OMAP_TC_OCPT1_PRIOR); \
  99         } while(0)
 100
 101 enum exmap_type {
 102         EXMAP_TYPE_MEM,
 103         EXMAP_TYPE_FB
 104 };
 105
 106 struct exmap_tbl {
 107         unsigned int valid:1;
 108         unsigned int cntnu:1;   /* grouping */
 109         int usecount;           /* reference count by mmap */
 110         enum exmap_type type;
 111         void *buf;              /* virtual address of the buffer,
 112                                  * i.e. 0xc0000000 - */
 113         void *vadr;             /* DSP shadow space,
 114                                  * i.e. 0xe0000000 - 0xe0ffffff */
 115         unsigned int order;
 116 };
 117 #define DSPMMU_TLB_LINES        32
 118 static struct exmap_tbl exmap_tbl[DSPMMU_TLB_LINES];
 119 static DECLARE_RWSEM(exmap_sem);
 120
 121 #ifdef CONFIG_FB_OMAP_LCDC_EXTERNAL
 122 static struct omapfb_notifier_block *omapfb_nb;
 123 static int omapfb_ready;
 124 #endif
 125
 126 static int dsp_exunmap(unsigned long dspadr);
 127
 128 static void *dspvect_page;
 129 static unsigned long dsp_fault_adr;
 130 static struct mem_sync_struct mem_sync;
 131
 132 static void *mempool_alloc_from_pool(mempool_t *pool,
 133                                      unsigned int __nocast gfp_mask)
 134 {
 135         spin_lock_irq(&pool->lock);
 136         if (likely(pool->curr_nr)) {
 137                 void *element = pool->elements[--pool->curr_nr];
 138                 spin_unlock_irq(&pool->lock);
 139                 return element;
 140         }
 141
 142         spin_unlock_irq(&pool->lock);
 143         return mempool_alloc(pool, gfp_mask);
 144 }
 145
 146 static __inline__ unsigned long lineup_offset(unsigned long adr,
 147                                               unsigned long ref,
 148                                               unsigned long mask)
 149 {
 150         unsigned long newadr;
 151
 152         newadr = (adr & ~mask) | (ref & mask);
 153         if (newadr < adr)
 154                 newadr += mask + 1;
 155         return newadr;
 156 }
 157
 158 int dsp_mem_sync_inc(void)
 159 {
 160         if (dsp_mem_enable((void *)dspmem_base) < 0)
 161                 return -1;
 162         if (mem_sync.DARAM)
 163                 mem_sync.DARAM->ad_arm++;
 164         if (mem_sync.SARAM)
 165                 mem_sync.SARAM->ad_arm++;
 166         if (mem_sync.SDRAM)
 167                 mem_sync.SDRAM->ad_arm++;
 168         dsp_mem_disable((void *)dspmem_base);
 169         return 0;
 170 }
 171
 172 /*
 173  * dsp_mem_sync_config() is called from mbx1 workqueue
 174  */
 175 int dsp_mem_sync_config(struct mem_sync_struct *sync)
 176 {
 177         size_t sync_seq_sz = sizeof(struct sync_seq);
 178
 179 #ifdef OLD_BINARY_SUPPORT
 180         if (sync == NULL) {
 181                 memset(&mem_sync, 0, sizeof(struct mem_sync_struct));
 182                 return 0;
 183         }
 184 #endif
 185         if ((dsp_mem_type(sync->DARAM, sync_seq_sz) != MEM_TYPE_DARAM) ||
 186             (dsp_mem_type(sync->SARAM, sync_seq_sz) != MEM_TYPE_SARAM) ||
 187             (dsp_mem_type(sync->SDRAM, sync_seq_sz) != MEM_TYPE_EXTERN)) {
 188                 printk(KERN_ERR
 189                        "omapdsp: mem_sync address validation failure!\n"
 190                        "  mem_sync.DARAM = 0x%p,\n"
 191                        "  mem_sync.SARAM = 0x%p,\n"
 192                        "  mem_sync.SDRAM = 0x%p,\n",
 193                        sync->DARAM, sync->SARAM, sync->SDRAM);
 194                 return -1;
 195         }
 196         memcpy(&mem_sync, sync, sizeof(struct mem_sync_struct));
 197         return 0;
 198 }
 199
 200 static mempool_t *kmem_pool_1M;
 201 static mempool_t *kmem_pool_64K;
 202
 203 static void *dsp_pool_alloc(unsigned int __nocast gfp, void *order)
 204 {
 205         return (void *)__get_dma_pages(gfp, (unsigned int)order);
 206 }
 207
 208 static void dsp_pool_free(void *buf, void *order)
 209 {
 210         free_pages((unsigned long)buf, (unsigned int)order);
 211 }
 212
 213 static void dsp_kmem_release(void)
 214 {
 215         if (kmem_pool_64K) {
 216                 mempool_destroy(kmem_pool_64K);
 217                 kmem_pool_64K = NULL;
 218         }
 219
 220         if (kmem_pool_1M) {
 221                 mempool_destroy(kmem_pool_1M);
 222                 kmem_pool_1M = NULL;
 223         }
 224 }
 225
 226 static int dsp_kmem_reserve(unsigned long size)
 227 {
 228         unsigned long len = size;
 229
 230         /* alignment check */
 231         if (!is_aligned(size, SZ_64KB)) {
 232                 printk(KERN_ERR
 233                        "omapdsp: size(0x%lx) is not multiple of 64KB.\n", size);
 234                 return -EINVAL;
 235         }
 236         if (size > DSPSPACE_SIZE) {
 237                 printk(KERN_ERR
 238                        "omapdsp: size(0x%lx) is larger than DSP memory space "
 239                        "size (0x%x.\n", size, DSPSPACE_SIZE);
 240                 return -EINVAL;
 241         }
 242
 243         if (size >= SZ_1MB) {
 244                 int nr = size >> 20;
 245
 246                 if (likely(!kmem_pool_1M))
 247                         kmem_pool_1M = mempool_create(nr,
 248                                                       dsp_pool_alloc,
 249                                                       dsp_pool_free,
 250                                                       (void *)ORDER_1MB);
 251                 else
 252                         mempool_resize(kmem_pool_1M, kmem_pool_1M->min_nr + nr,
 253                                        GFP_KERNEL);
 254
 255                 size &= ~(0xf << 20);
 256         }
 257
 258         if (size >= SZ_64KB) {
 259                 int nr = size >> 16;
 260
 261                 if (likely(!kmem_pool_64K))
 262                         kmem_pool_64K = mempool_create(nr,
 263                                                        dsp_pool_alloc,
 264                                                        dsp_pool_free,
 265                                                        (void *)ORDER_64KB);
 266                 else
 267                         mempool_resize(kmem_pool_64K,
 268                                        kmem_pool_64K->min_nr + nr, GFP_KERNEL);
 269
 270                 size &= ~(0xf << 16);
 271         }
 272
 273         if (size)
 274                 len -= size;
 275
 276         return len;
 277 }
 278
 279 static void dsp_mem_free_pages(unsigned long buf, unsigned int order)
 280 {
 281         struct page *page, *ps, *pe;
 282
 283         ps = virt_to_page(buf);
 284         pe = virt_to_page(buf + (1 << (PAGE_SHIFT + order)));
 285
 286         for (page = ps; page < pe; page++)
 287                 ClearPageReserved(page);
 288
 289         if (buf) {
 290                 if ((order == ORDER_64KB) && likely(kmem_pool_64K))
 291                         mempool_free((void *)buf, kmem_pool_64K);
 292                 else if ((order == ORDER_1MB) && likely(kmem_pool_1M))
 293                         mempool_free((void *)buf, kmem_pool_1M);
 294                 else
 295                         free_pages(buf, order);
 296         }
 297 }
 298
 299 static inline void
 300 exmap_alloc_pte(unsigned long virt, unsigned long phys, pgprot_t prot)
 301 {
 302         pgd_t *pgd;
 303         pud_t *pud;
 304         pmd_t *pmd;
 305         pte_t *pte;
 306
 307         pgd = pgd_offset_k(virt);
 308         pud = pud_offset(pgd, virt);
 309         pmd = pmd_offset(pud, virt);
 310
 311         if (pmd_none(*pmd)) {
 312                 pte = pte_alloc_one_kernel(&init_mm, 0);
 313                 if (!pte)
 314                         return;
 315
 316                 /* note: two PMDs will be set  */
 317                 pmd_populate_kernel(&init_mm, pmd, pte);
 318         }
 319
 320         pte = pte_offset_kernel(pmd, virt);
 321         set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, prot));
 322 }
 323
 324 static inline int
 325 exmap_alloc_sect(unsigned long virt, unsigned long phys, int prot)
 326 {
 327         pgd_t *pgd;
 328         pud_t *pud;
 329         pmd_t *pmd;
 330
 331         pgd = pgd_offset_k(virt);
 332         pud = pud_alloc(&init_mm, pgd, virt);
 333         pmd = pmd_alloc(&init_mm, pud, virt);
 334
 335         if (virt & (1 << 20))
 336                 pmd++;
 337
 338         if (!pmd_none(*pmd))
 339                 /* No good, fall back on smaller mappings. */
 340                 return -EINVAL;
 341
 342         *pmd = __pmd(phys | prot);
 343         flush_pmd_entry(pmd);
 344
 345         return 0;
 346 }
 347
 348 /*
 349  * ARM MMU operations
 350  */
 351 static int exmap_set_armmmu(unsigned long virt, unsigned long phys,
 352                             unsigned long size)
 353 {
 354         long off;
 355         pgprot_t prot_pte;
 356         int prot_sect;
 357
 358         printk(KERN_DEBUG
 359                "omapdsp: mapping in ARM MMU, v=0x%08lx, p=0x%08lx, sz=0x%lx\n",
 360                virt, phys, size);
 361
 362         prot_pte = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
 363                             L_PTE_DIRTY | L_PTE_WRITE);
 364
 365         prot_sect = PMD_TYPE_SECT | PMD_SECT_UNCACHED |
 366                     PMD_SECT_AP_WRITE | PMD_DOMAIN(DOMAIN_IO);
 367
 368         if (cpu_architecture() <= CPU_ARCH_ARMv5)
 369                 prot_sect |= PMD_BIT4;
 370
 371         off = phys - virt;
 372
 373         while ((virt & 0xfffff || (virt + off) & 0xfffff) && size >= PAGE_SIZE) {
 374                 exmap_alloc_pte(virt, virt + off, prot_pte);
 375
 376                 virt += PAGE_SIZE;
 377                 size -= PAGE_SIZE;
 378         }
 379
 380         /* XXX: Not yet.. confuses dspfb -- PFM. */
 381 #if 0
 382         while (size >= (PGDIR_SIZE / 2)) {
 383                 if (exmap_alloc_sect(virt, virt + off, prot_sect) < 0)
 384                         break;
 385
 386                 virt += (PGDIR_SIZE / 2);
 387                 size -= (PGDIR_SIZE / 2);
 388         }
 389 #endif
 390
 391         while (size >= PAGE_SIZE) {
 392                 exmap_alloc_pte(virt, virt + off, prot_pte);
 393
 394                 virt += PAGE_SIZE;
 395                 size -= PAGE_SIZE;
 396         }
 397
 398         BUG_ON(size);
 399
 400         return 0;
 401 }
 402
 403 static inline void
 404 exmap_clear_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
 405 {
 406         pte_t *pte;
 407
 408         pte = pte_offset_map(pmd, addr);
 409         do {
 410                 if (pte_none(*pte))
 411                         continue;
 412
 413                 pte_clear(&init_mm, addr, pte);
 414         } while (pte++, addr += PAGE_SIZE, addr != end);
 415
 416         pte_unmap(pte - 1);
 417 }
 418
 419 static inline void
 420 exmap_clear_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
 421 {
 422         pmd_t *pmd;
 423         unsigned long next;
 424
 425         pmd = pmd_offset(pud, addr);
 426         do {
 427                 next = pmd_addr_end(addr, end);
 428
 429                 if (addr & (1 << 20))
 430                         pmd++;
 431
 432                 if ((pmd_val(*pmd) & PMD_TYPE_MASK) == PMD_TYPE_SECT) {
 433                         *pmd = __pmd(0);
 434                         clean_pmd_entry(pmd);
 435                         continue;
 436                 }
 437
 438                 if (pmd_none_or_clear_bad(pmd))
 439                         continue;
 440
 441                 exmap_clear_pte_range(pmd, addr, next);
 442         } while (pmd++, addr = next, addr != end);
 443 }
 444
 445 static inline void
 446 exmap_clear_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
 447 {
 448         pud_t *pud;
 449         unsigned long next;
 450
 451         pud = pud_offset(pgd, addr);
 452         do {
 453                 next = pud_addr_end(addr, end);
 454                 if (pud_none_or_clear_bad(pud))
 455                         continue;
 456
 457                 exmap_clear_pmd_range(pud, addr, next);
 458         } while (pud++, addr = next, addr != end);
 459 }
 460
 461 static void exmap_clear_armmmu(unsigned long virt, unsigned long size)
 462 {
 463         unsigned long next, end;
 464         pgd_t *pgd;
 465
 466         printk(KERN_DEBUG
 467                "omapdsp: unmapping in ARM MMU, v=0x%08lx, sz=0x%lx\n",
 468                virt, size);
 469
 470         pgd = pgd_offset_k(virt);
 471         end = virt + size;
 472         do {
 473                 next = pgd_addr_end(virt, end);
 474                 if (pgd_none_or_clear_bad(pgd))
 475                         continue;
 476
 477                 exmap_clear_pud_range(pgd, virt, next);
 478         } while (pgd++, virt = next, virt != end);
 479 }
 480
 481 static int exmap_valid(void *vadr, size_t len)
 482 {
 483         /* exmap_sem should be held before calling this function */
 484         int i;
 485
 486 start:
 487         for (i = 0; i < DSPMMU_TLB_LINES; i++) {
 488                 void *mapadr;
 489                 unsigned long mapsize;
 490                 struct exmap_tbl *ent = &exmap_tbl[i];
 491
 492                 if (!ent->valid)
 493                         continue;
 494                 mapadr = (void *)ent->vadr;
 495                 mapsize = 1 << (ent->order + PAGE_SHIFT);
 496                 if ((vadr >= mapadr) && (vadr < mapadr + mapsize)) {
 497                         if (vadr + len <= mapadr + mapsize) {
 498                                 /* this map covers whole address. */
 499                                 return 1;
 500                         } else {
 501                                 /*
 502                                  * this map covers partially.
 503                                  * check rest portion.
 504                                  */
 505                                 len -= mapadr + mapsize - vadr;
 506                                 vadr = mapadr + mapsize;
 507                                 goto start;
 508                         }
 509                 }
 510         }
 511
 512         return 0;
 513 }
 514
 515 enum dsp_mem_type_e dsp_mem_type(void *vadr, size_t len)
 516 {
 517         void *ds = (void *)daram_base;
 518         void *de = (void *)daram_base + daram_size;
 519         void *ss = (void *)saram_base;
 520         void *se = (void *)saram_base + saram_size;
 521         int ret;
 522
 523         if ((vadr >= ds) && (vadr < de)) {
 524                 if (vadr + len > de)
 525                         return MEM_TYPE_CROSSING;
 526                 else
 527                         return MEM_TYPE_DARAM;
 528         } else if ((vadr >= ss) && (vadr < se)) {
 529                 if (vadr + len > se)
 530                         return MEM_TYPE_CROSSING;
 531                 else
 532                         return MEM_TYPE_SARAM;
 533         } else {
 534                 down_read(&exmap_sem);
 535                 if (exmap_valid(vadr, len))
 536                         ret = MEM_TYPE_EXTERN;
 537                 else
 538                         ret = MEM_TYPE_NONE;
 539                 up_read(&exmap_sem);
 540                 return ret;
 541         }
 542 }
 543
 544 int dsp_address_validate(void *p, size_t len, char *fmt, ...)
 545 {
 546         if (dsp_mem_type(p, len) <= 0) {
 547                 if (fmt != NULL) {
 548                         char s[64];
 549                         va_list args;
 550
 551                         va_start(args, fmt);
 552                         vsprintf(s, fmt, args);
 553                         va_end(args);
 554                         printk(KERN_ERR
 555                                "omapdsp: %s address(0x%p) and size(0x%x) is "
 556                                "not valid!\n"
 557                                "         (crossing different type of memories, or \n"
 558                                "          external memory space where no "
 559                                "actual memory is mapped)\n",
 560                                s, p, len);
 561                 }
 562                 return -1;
 563         }
 564
 565         return 0;
 566 }
 567
 568 /*
 569  * exmap_use(), unuse():
 570  * when the mapped area is exported to user space with mmap,
 571  * the usecount is incremented.
 572  * while the usecount > 0, that area can't be released.
 573  */
 574 void exmap_use(void *vadr, size_t len)
 575 {
 576         int i;
 577
 578         down_write(&exmap_sem);
 579         for (i = 0; i < DSPMMU_TLB_LINES; i++) {
 580                 void *mapadr;
 581                 unsigned long mapsize;
 582                 struct exmap_tbl *ent = &exmap_tbl[i];
 583
 584                 if (!ent->valid)
 585                         continue;
 586                 mapadr = (void *)ent->vadr;
 587                 mapsize = 1 << (ent->order + PAGE_SHIFT);
 588                 if ((vadr + len > mapadr) && (vadr < mapadr + mapsize)) {
 589                         ent->usecount++;
 590                 }
 591         }
 592         up_write(&exmap_sem);
 593 }
 594
 595 void exmap_unuse(void *vadr, size_t len)
 596 {
 597         int i;
 598
 599         down_write(&exmap_sem);
 600         for (i = 0; i < DSPMMU_TLB_LINES; i++) {
 601                 void *mapadr;
 602                 unsigned long mapsize;
 603                 struct exmap_tbl *ent = &exmap_tbl[i];
 604
 605                 if (!ent->valid)
 606                         continue;
 607                 mapadr = (void *)ent->vadr;
 608                 mapsize = 1 << (ent->order + PAGE_SHIFT);
 609                 if ((vadr + len > mapadr) && (vadr < mapadr + mapsize)) {
 610                         ent->usecount--;
 611                 }
 612         }
 613         up_write(&exmap_sem);
 614 }
 615
 616 /*
 617  * dsp_virt_to_phys()
 618  * returns physical address, and sets len to valid length
 619  */
 620 unsigned long dsp_virt_to_phys(void *vadr, size_t *len)
 621 {
 622         int i;
 623
 624         if (is_dsp_internal_mem(vadr)) {
 625                 /* DSRAM or SARAM */
 626                 *len = dspmem_base + dspmem_size - (unsigned long)vadr;
 627                 return (unsigned long)vadr;
 628         }
 629
 630         /* EXRAM */
 631         for (i = 0; i < DSPMMU_TLB_LINES; i++) {
 632                 void *mapadr;
 633                 unsigned long mapsize;
 634                 struct exmap_tbl *ent = &exmap_tbl[i];
 635
 636                 if (!ent->valid)
 637                         continue;
 638                 mapadr = (void *)ent->vadr;
 639                 mapsize = 1 << (ent->order + PAGE_SHIFT);
 640                 if ((vadr >= mapadr) && (vadr < mapadr + mapsize)) {
 641                         *len = mapadr + mapsize - vadr;
 642                         return __pa(ent->buf) + vadr - mapadr;
 643                 }
 644         }
 645
 646         /* valid mapping not found */
 647         return 0;
 648 }
 649
 650 /*
 651  * DSP MMU operations
 652  */
 653 static __inline__ unsigned short get_cam_l_va_mask(unsigned short slst)
 654 {
 655         switch (slst) {
 656         case DSPMMU_CAM_L_SLST_1MB:
 657                 return DSPMMU_CAM_L_VA_TAG_L1_MASK |
 658                        DSPMMU_CAM_L_VA_TAG_L2_MASK_1MB;
 659         case DSPMMU_CAM_L_SLST_64KB:
 660                 return DSPMMU_CAM_L_VA_TAG_L1_MASK |
 661                        DSPMMU_CAM_L_VA_TAG_L2_MASK_64KB;
 662         case DSPMMU_CAM_L_SLST_4KB:
 663                 return DSPMMU_CAM_L_VA_TAG_L1_MASK |
 664                        DSPMMU_CAM_L_VA_TAG_L2_MASK_4KB;
 665         case DSPMMU_CAM_L_SLST_1KB:
 666                 return DSPMMU_CAM_L_VA_TAG_L1_MASK |
 667                        DSPMMU_CAM_L_VA_TAG_L2_MASK_1KB;
 668         }
 669         return 0;
 670 }
 671
 672 static __inline__ void get_tlb_lock(int *base, int *victim)
 673 {
 674         unsigned short lock = omap_readw(DSPMMU_LOCK);
 675         if (base != NULL)
 676                 *base = (lock & DSPMMU_LOCK_BASE_MASK)
 677                         >> DSPMMU_LOCK_BASE_SHIFT;
 678         if (victim != NULL)
 679                 *victim = (lock & DSPMMU_LOCK_VICTIM_MASK)
 680                           >> DSPMMU_LOCK_VICTIM_SHIFT;
 681 }
 682
 683 static __inline__ void set_tlb_lock(int base, int victim)
 684 {
 685         omap_writew((base   << DSPMMU_LOCK_BASE_SHIFT) |
 686                     (victim << DSPMMU_LOCK_VICTIM_SHIFT), DSPMMU_LOCK);
 687 }
 688
 689 static __inline__ void __read_tlb(unsigned short lbase, unsigned short victim,
 690                                   unsigned short *cam_h, unsigned short *cam_l,
 691                                   unsigned short *ram_h, unsigned short *ram_l)
 692 {
 693         /* set victim */
 694         set_tlb_lock(lbase, victim);
 695
 696         /* read a TLB entry */
 697         omap_writew(DSPMMU_LD_TLB_RD, DSPMMU_LD_TLB);
 698
 699         if (cam_h != NULL)
 700                 *cam_h = omap_readw(DSPMMU_READ_CAM_H);
 701         if (cam_l != NULL)
 702                 *cam_l = omap_readw(DSPMMU_READ_CAM_L);
 703         if (ram_h != NULL)
 704                 *ram_h = omap_readw(DSPMMU_READ_RAM_H);
 705         if (ram_l != NULL)
 706                 *ram_l = omap_readw(DSPMMU_READ_RAM_L);
 707 }
 708
 709 static __inline__ void __load_tlb(unsigned short cam_h, unsigned short cam_l,
 710                                   unsigned short ram_h, unsigned short ram_l)
 711 {
 712         omap_writew(cam_h, DSPMMU_CAM_H);
 713         omap_writew(cam_l, DSPMMU_CAM_L);
 714         omap_writew(ram_h, DSPMMU_RAM_H);
 715         omap_writew(ram_l, DSPMMU_RAM_L);
 716
 717         /* flush the entry */
 718         dsp_mmu_flush();
 719
 720         /* load a TLB entry */
 721         omap_writew(DSPMMU_LD_TLB_LD, DSPMMU_LD_TLB);
 722 }
 723
 724 static int dsp_mmu_load_tlb(unsigned long vadr, unsigned long padr,
 725                             unsigned short slst, unsigned short prsvd,
 726                             unsigned short ap)
 727 {
 728         int lbase, victim;
 729         unsigned short cam_l_va_mask;
 730
 731         clk_enable(dsp_ck_handle);
 732
 733         get_tlb_lock(&lbase, NULL);
 734         for (victim = 0; victim < lbase; victim++) {
 735                 unsigned short cam_l;
 736
 737                 /* read a TLB entry */
 738                 __read_tlb(lbase, victim, NULL, &cam_l, NULL, NULL);
 739                 if (!(cam_l & DSPMMU_CAM_L_V))
 740                         goto found_victim;
 741         }
 742         set_tlb_lock(lbase, victim);
 743
 744 found_victim:
 745         /* The last (31st) entry cannot be locked? */
 746         if (victim == 31) {
 747                 printk(KERN_ERR "omapdsp: TLB is full.\n");
 748                 return -EBUSY;
 749         }
 750
 751         cam_l_va_mask = get_cam_l_va_mask(slst);
 752         if (vadr &
 753             ~(DSPMMU_CAM_H_VA_TAG_H_MASK << 22 |
 754               (unsigned long)cam_l_va_mask << 6)) {
 755                 printk(KERN_ERR
 756                        "omapdsp: mapping vadr (0x%06lx) is not "
 757                        "aligned boundary\n", vadr);
 758                 return -EINVAL;
 759         }
 760
 761         __load_tlb(vadr >> 22, (vadr >> 6 & cam_l_va_mask) | prsvd | slst,
 762                    padr >> 16, (padr & DSPMMU_RAM_L_RAM_LSB_MASK) | ap);
 763
 764         /* update lock base */
 765         if (victim == lbase)
 766                 lbase++;
 767         set_tlb_lock(lbase, lbase);
 768
 769         clk_disable(dsp_ck_handle);
 770         return 0;
 771 }
 772
 773 static int dsp_mmu_clear_tlb(unsigned long vadr)
 774 {
 775         int lbase;
 776         int i;
 777         int max_valid = 0;
 778
 779         clk_enable(dsp_ck_handle);
 780
 781         get_tlb_lock(&lbase, NULL);
 782         for (i = 0; i < lbase; i++) {
 783                 unsigned short cam_h, cam_l;
 784                 unsigned short cam_l_va_mask, cam_vld, slst;
 785                 unsigned long cam_va;
 786
 787                 /* read a TLB entry */
 788                 __read_tlb(lbase, i, &cam_h, &cam_l, NULL, NULL);
 789
 790                 cam_vld = cam_l & DSPMMU_CAM_L_V;
 791                 if (!cam_vld)
 792                         continue;
 793
 794                 slst = cam_l & DSPMMU_CAM_L_SLST_MASK;
 795                 cam_l_va_mask = get_cam_l_va_mask(slst);
 796                 cam_va = (unsigned long)(cam_h & DSPMMU_CAM_H_VA_TAG_H_MASK) << 22 |
 797                          (unsigned long)(cam_l & cam_l_va_mask) << 6;
 798
 799                 if (cam_va == vadr)
 800                         /* flush the entry */
 801                         dsp_mmu_flush();
 802                 else
 803                         max_valid = i;
 804         }
 805
 806         /* set new lock base */
 807         set_tlb_lock(max_valid+1, max_valid+1);
 808
 809         clk_disable(dsp_ck_handle);
 810         return 0;
 811 }
 812
 813 static void dsp_mmu_gflush(void)
 814 {
 815         clk_enable(dsp_ck_handle);
 816
 817         __dsp_mmu_gflush();
 818         set_tlb_lock(1, 1);
 819
 820         clk_disable(dsp_ck_handle);
 821 }
 822
 823 /*
 824  * dsp_exmap()
 825  *
 826  * OMAP_DSP_MEM_IOCTL_EXMAP ioctl calls this function with padr=0.
 827  * In this case, the buffer for DSP is allocated in this routine,
 828  * then it is mapped.
 829  * On the other hand, for example - frame buffer sharing, calls
 830  * this function with padr set. It means some known address space
 831  * pointed with padr is going to be shared with DSP.
 832  */
 833 static int dsp_exmap(unsigned long dspadr, unsigned long padr,
 834                      unsigned long size, enum exmap_type type)
 835 {
 836         unsigned short slst;
 837         void *buf;
 838         unsigned int order = 0;
 839         unsigned long unit;
 840         unsigned int cntnu = 0;
 841         unsigned long _dspadr = dspadr;
 842         unsigned long _padr = padr;
 843         void *_vadr = dspbyte_to_virt(dspadr);
 844         unsigned long _size = size;
 845         struct exmap_tbl *exmap_ent;
 846         int status;
 847         int i;
 848
 849 #define MINIMUM_PAGESZ  SZ_4KB
 850         /*
 851          * alignment check
 852          */
 853         if (!is_aligned(size, MINIMUM_PAGESZ)) {
 854                 printk(KERN_ERR
 855                        "omapdsp: size(0x%lx) is not multiple of 4KB.\n", size);
 856                 return -EINVAL;
 857         }
 858         if (!is_aligned(dspadr, MINIMUM_PAGESZ)) {
 859                 printk(KERN_ERR
 860                        "omapdsp: DSP address(0x%lx) is not aligned.\n", dspadr);
 861                 return -EINVAL;
 862         }
 863         if (!is_aligned(padr, MINIMUM_PAGESZ)) {
 864                 printk(KERN_ERR
 865                        "omapdsp: physical address(0x%lx) is not aligned.\n",
 866                        padr);
 867                 return -EINVAL;
 868         }
 869
 870         /* address validity check */
 871         if ((dspadr < dspmem_size) ||
 872             (dspadr >= DSPSPACE_SIZE) ||
 873             ((dspadr + size > DSP_INIT_PAGE) &&
 874              (dspadr < DSP_INIT_PAGE + PAGE_SIZE))) {
 875                 printk(KERN_ERR
 876                        "omapdsp: illegal address/size for dsp_exmap().\n");
 877                 return -EINVAL;
 878         }
 879
 880         down_write(&exmap_sem);
 881
 882         /* overlap check */
 883         for (i = 0; i < DSPMMU_TLB_LINES; i++) {
 884                 unsigned long mapsize;
 885                 struct exmap_tbl *tmp_ent = &exmap_tbl[i];
 886
 887                 if (!tmp_ent->valid)
 888                         continue;
 889                 mapsize = 1 << (tmp_ent->order + PAGE_SHIFT);
 890                 if ((_vadr + size > tmp_ent->vadr) &&
 891                     (_vadr < tmp_ent->vadr + mapsize)) {
 892                         printk(KERN_ERR "omapdsp: exmap page overlap!\n");
 893                         up_write(&exmap_sem);
 894                         return -EINVAL;
 895                 }
 896         }
 897
 898 start:
 899         buf = NULL;
 900         /* Are there any free TLB lines?  */
 901         for (i = 0; i < DSPMMU_TLB_LINES; i++) {
 902                 if (!exmap_tbl[i].valid)
 903                         goto found_free;
 904         }
 905         printk(KERN_ERR "omapdsp: DSP TLB is full.\n");
 906         status = -EBUSY;
 907         goto fail;
 908
 909 found_free:
 910         exmap_ent = &exmap_tbl[i];
 911
 912         if ((_size >= SZ_1MB) &&
 913             (is_aligned(_padr, SZ_1MB) || (padr == 0)) &&
 914             is_aligned(_dspadr, SZ_1MB)) {
 915                 unit = SZ_1MB;
 916                 slst = DSPMMU_CAM_L_SLST_1MB;
 917         } else if ((_size >= SZ_64KB) &&
 918                    (is_aligned(_padr, SZ_64KB) || (padr == 0)) &&
 919                    is_aligned(_dspadr, SZ_64KB)) {
 920                 unit = SZ_64KB;
 921                 slst = DSPMMU_CAM_L_SLST_64KB;
 922         } else {
 923                 unit = SZ_4KB;
 924                 slst = DSPMMU_CAM_L_SLST_4KB;
 925         }
 926
 927         order = get_order(unit);
 928
 929         /* buffer allocation */
 930         if (type == EXMAP_TYPE_MEM) {
 931                 struct page *page, *ps, *pe;
 932
 933                 if ((order == ORDER_1MB) && likely(kmem_pool_1M))
 934                         buf = mempool_alloc_from_pool(kmem_pool_1M, GFP_KERNEL);
 935                 else if ((order == ORDER_64KB) && likely(kmem_pool_64K))
 936                         buf = mempool_alloc_from_pool(kmem_pool_64K,GFP_KERNEL);
 937                 else {
 938                         buf = (void *)__get_dma_pages(GFP_KERNEL, order);
 939                         if (buf == NULL) {
 940                                 status = -ENOMEM;
 941                                 goto fail;
 942                         }
 943                 }
 944
 945                 /* mark the pages as reserved; this is needed for mmap */
 946                 ps = virt_to_page(buf);
 947                 pe = virt_to_page(buf + unit);
 948
 949                 for (page = ps; page < pe; page++)
 950                         SetPageReserved(page);
 951
 952                 _padr = __pa(buf);
 953         }
 954
 955         /*
 956          * mapping for ARM MMU:
 957          * we should not access to the allocated memory through 'buf'
 958          * since this area should not be cashed.
 959          */
 960         status = exmap_set_armmmu((unsigned long)_vadr, _padr, unit);
 961         if (status < 0)
 962                 goto fail;
 963
 964         /* loading DSP TLB entry */
 965         status = dsp_mmu_load_tlb(_dspadr, _padr, slst, 0, DSPMMU_RAM_L_AP_FA);
 966         if (status < 0) {
 967                 exmap_clear_armmmu((unsigned long)_vadr, unit);
 968                 goto fail;
 969         }
 970
 971         exmap_ent->buf      = buf;
 972         exmap_ent->vadr     = _vadr;
 973         exmap_ent->order    = order;
 974         exmap_ent->valid    = 1;
 975         exmap_ent->cntnu    = cntnu;
 976         exmap_ent->type     = type;
 977         exmap_ent->usecount = 0;
 978
 979         if ((_size -= unit) == 0) {     /* normal completion */
 980                 up_write(&exmap_sem);
 981                 return size;
 982         }
 983
 984         _dspadr += unit;
 985         _vadr   += unit;
 986         _padr = padr ? _padr + unit : 0;
 987         cntnu = 1;
 988         goto start;
 989
 990 fail:
 991         up_write(&exmap_sem);
 992         if (buf)
 993                 dsp_mem_free_pages((unsigned long)buf, order);
 994         dsp_exunmap(dspadr);
 995         return status;
 996 }
 997
 998 static unsigned long unmap_free_arm(struct exmap_tbl *ent)
 999 {
1000         unsigned long size;
1001
1002         /* clearing ARM MMU */
1003         size = 1 << (ent->order + PAGE_SHIFT);
1004         exmap_clear_armmmu((unsigned long)ent->vadr, size);
1005
1006         /* freeing allocated memory */
1007         if (ent->type == EXMAP_TYPE_MEM) {
1008                 dsp_mem_free_pages((unsigned long)ent->buf, ent->order);
1009                 printk(KERN_DEBUG
1010                        "omapdsp: freeing 0x%lx bytes @ adr 0x%8p\n",
1011                        size, ent->buf);
1012         }
1013 #ifdef CONFIG_FB_OMAP_LCDC_EXTERNAL
1014         else if (ent->type == EXMAP_TYPE_FB) {
1015                 int status;
1016                 if (omapfb_nb) {
1017                         status = omapfb_unregister_client(omapfb_nb);
1018                         if (!status)
1019                                 printk("omapfb_unregister_client(): "
1020                                        "success\n");
1021                         else
1022                                 printk("omapfb_runegister_client(): "
1023                                        "failure(%d)\n", status);
1024                         kfree(omapfb_nb);
1025                         omapfb_nb = NULL;
1026                         omapfb_ready = 0;
1027                 }
1028         }
1029 #endif
1030
1031         return size;
1032 }
1033
1034 static int dsp_exunmap(unsigned long dspadr)
1035 {
1036         void *vadr;
1037         unsigned long size;
1038         int total = 0;
1039         struct exmap_tbl *ent;
1040         int idx;
1041
1042         vadr = dspbyte_to_virt(dspadr);
1043         down_write(&exmap_sem);
1044         for (idx = 0; idx < DSPMMU_TLB_LINES; idx++) {
1045                 ent = &exmap_tbl[idx];
1046                 if (!ent->valid)
1047                         continue;
1048                 if (ent->vadr == vadr)
1049                         goto found_map;
1050         }
1051         up_write(&exmap_sem);
1052         printk(KERN_WARNING
1053                "omapdsp: address %06lx not found in exmap_tbl.\n", dspadr);
1054         return -EINVAL;
1055
1056 found_map:
1057         if (ent->usecount > 0) {
1058                 printk(KERN_ERR
1059                        "omapdsp: exmap reference count is not 0.\n"
1060                        "   idx=%d, vadr=%p, order=%d, usecount=%d\n",
1061                        idx, ent->vadr, ent->order, ent->usecount);
1062                 up_write(&exmap_sem);
1063                 return -EINVAL;
1064         }
1065         /* clearing DSP TLB entry */
1066         dsp_mmu_clear_tlb(dspadr);
1067
1068         /* clear ARM MMU and free buffer */
1069         size = unmap_free_arm(ent);
1070         ent->valid = 0;
1071         total += size;
1072
1073         /* we don't free PTEs */
1074
1075         /* flush TLB */
1076         flush_tlb_kernel_range((unsigned long)vadr, (unsigned long)vadr + size);
1077
1078         /* check if next mapping is in same group */
1079         if (++idx == DSPMMU_TLB_LINES)
1080                 goto up_out;    /* normal completion */
1081         ent = &exmap_tbl[idx];
1082         if (!ent->valid || !ent->cntnu)
1083                 goto up_out;    /* normal completion */
1084
1085         dspadr += size;
1086         vadr   += size;
1087         if (ent->vadr == vadr)
1088                 goto found_map; /* continue */
1089
1090         printk(KERN_ERR
1091                "omapdsp: illegal exmap_tbl grouping!\n"
1092                "expected vadr = %p, exmap_tbl[%d].vadr = %p\n",
1093                vadr, idx, ent->vadr);
1094         up_write(&exmap_sem);
1095         return -EINVAL;
1096
1097 up_out:
1098         up_write(&exmap_sem);
1099         return total;
1100 }
1101
1102 static void exmap_flush(void)
1103 {
1104         struct exmap_tbl *ent;
1105         int i;
1106
1107         down_write(&exmap_sem);
1108
1109         /* clearing DSP TLB entry */
1110         dsp_mmu_gflush();
1111
1112         /* exmap_tbl[0] should be preserved */
1113         for (i = 1; i < DSPMMU_TLB_LINES; i++) {
1114                 ent = &exmap_tbl[i];
1115                 if (ent->valid) {
1116                         unmap_free_arm(ent);
1117                         ent->valid = 0;
1118                 }
1119         }
1120
1121         /* flush TLB */
1122         flush_tlb_kernel_range(dspmem_base + dspmem_size,
1123                                dspmem_base + DSPSPACE_SIZE);
1124         up_write(&exmap_sem);
1125 }
1126
1127 #ifdef CONFIG_OMAP_DSP_FBEXPORT
1128 #ifndef CONFIG_FB
1129 #error You configured OMAP_DSP_FBEXPORT, but FB was not configured!
1130 #endif /* CONFIG_FB */
1131
1132 #ifdef CONFIG_FB_OMAP_LCDC_EXTERNAL
1133 static int omapfb_notifier_cb(struct omapfb_notifier_block *omapfb_nb,
1134                                unsigned long event, struct omapfb_device *fbdev)
1135 {
1136         /* XXX */
1137         printk("omapfb_notifier_cb(): event = %s\n",
1138                (event == OMAPFB_EVENT_READY)    ? "READY" :
1139                (event == OMAPFB_EVENT_DISABLED) ? "DISABLED" : "Unknown");
1140         if (event == OMAPFB_EVENT_READY)
1141                 omapfb_ready = 1;
1142         else if (event == OMAPFB_EVENT_DISABLED)
1143                 omapfb_ready = 0;
1144         return 0;
1145 }
1146 #endif
1147
1148 static int dsp_fbexport(unsigned long *dspadr)
1149 {
1150         unsigned long dspadr_actual;
1151         unsigned long padr_sys, padr, fbsz_sys, fbsz;
1152         int cnt;
1153 #ifdef CONFIG_FB_OMAP_LCDC_EXTERNAL
1154         int status;
1155 #endif
1156
1157         printk(KERN_DEBUG "omapdsp: frame buffer export\n");
1158
1159 #ifdef CONFIG_FB_OMAP_LCDC_EXTERNAL
1160         if (omapfb_nb) {
1161                 printk(KERN_WARNING
1162                        "omapdsp: frame buffer has been exported already!\n");
1163                 return -EBUSY;
1164         }
1165 #endif
1166
1167         if (num_registered_fb == 0) {
1168                 printk(KERN_INFO "omapdsp: frame buffer not registered.\n");
1169                 return -EINVAL;
1170         }
1171         if (num_registered_fb != 1) {
1172                 printk(KERN_INFO
1173                        "omapdsp: %d frame buffers found. we use first one.\n",
1174                        num_registered_fb);
1175         }
1176         padr_sys = registered_fb[0]->fix.smem_start;
1177         fbsz_sys = registered_fb[0]->fix.smem_len;
1178         if (fbsz_sys == 0) {
1179                 printk(KERN_ERR
1180                        "omapdsp: framebuffer doesn't seem to be configured "
1181                        "correctly! (size=0)\n");
1182                 return -EINVAL;
1183         }
1184
1185         /*
1186          * align padr and fbsz to 4kB boundary
1187          * (should be noted to the user afterwards!)
1188          */
1189         padr = padr_sys & ~(SZ_4KB-1);
1190         fbsz = (fbsz_sys + padr_sys - padr + SZ_4KB-1) & ~(SZ_4KB-1);
1191
1192         /* line up dspadr offset with padr */
1193         dspadr_actual =
1194                 (fbsz > SZ_1MB) ?  lineup_offset(*dspadr, padr, SZ_1MB-1) :
1195                 (fbsz > SZ_64KB) ? lineup_offset(*dspadr, padr, SZ_64KB-1) :
1196                 /* (fbsz > SZ_4KB) ? */ *dspadr;
1197         if (dspadr_actual != *dspadr)
1198                 printk(KERN_DEBUG
1199                        "omapdsp: actual dspadr for FBEXPORT = %08lx\n",
1200                        dspadr_actual);
1201         *dspadr = dspadr_actual;
1202
1203         cnt = dsp_exmap(dspadr_actual, padr, fbsz, EXMAP_TYPE_FB);
1204         if (cnt < 0) {
1205                 printk(KERN_ERR "omapdsp: exmap failure.\n");
1206                 return cnt;
1207         }
1208
1209         if ((padr != padr_sys) || (fbsz != fbsz_sys)) {
1210                 printk(KERN_WARNING
1211 "  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
1212 "  !!  screen base address or size is not aligned in 4kB:           !!\n"
1213 "  !!    actual screen  adr = %08lx, size = %08lx             !!\n"
1214 "  !!    exporting      adr = %08lx, size = %08lx             !!\n"
1215 "  !!  Make sure that the framebuffer is allocated with 4kB-order!  !!\n"
1216 "  !!  Otherwise DSP can corrupt the kernel memory.                 !!\n"
1217 "  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
1218                        padr_sys, fbsz_sys, padr, fbsz);
1219         }
1220
1221         /* increase the DMA priority */
1222         set_emiff_dma_prio(15);
1223
1224 #ifdef CONFIG_FB_OMAP_LCDC_EXTERNAL
1225         omapfb_nb = kmalloc(sizeof(struct omapfb_notifier_block), GFP_KERNEL);
1226         if (omapfb_nb == NULL) {
1227                 printk(KERN_ERR
1228                        "omapdsp: failed to allocate memory for omapfb_nb!\n");
1229                 dsp_exunmap(dspadr_actual);
1230                 return -ENOMEM;
1231         }
1232         status = omapfb_register_client(omapfb_nb, omapfb_notifier_cb, NULL);
1233         if (!status)
1234                 printk("omapfb_register_client(): success\n");
1235         else
1236                 printk("omapfb_register_client(): failure(%d)\n", status);
1237 #endif
1238
1239         return cnt;
1240 }
1241
1242 #else /* CONFIG_OMAP_DSP_FBEXPORT */
1243
1244 static int dsp_fbexport(unsigned long *dspadr)
1245 {
1246         printk(KERN_ERR "omapdsp: FBEXPORT function is not enabled.\n");
1247         return -EINVAL;
1248 }
1249
1250 #endif /* CONFIG_OMAP_DSP_FBEXPORT */
1251
1252 static int dsp_mmu_itack(void)
1253 {
1254         unsigned long dspadr;
1255
1256         printk(KERN_INFO "omapdsp: sending DSP MMU interrupt ack.\n");
1257         if (!dsp_err_mmu_isset()) {
1258                 printk(KERN_ERR "omapdsp: DSP MMU error has not been set.\n");
1259                 return -EINVAL;
1260         }
1261         dspadr = dsp_fault_adr & ~(SZ_4K-1);
1262         dsp_exmap(dspadr, 0, SZ_4K, EXMAP_TYPE_MEM);    /* FIXME: reserve TLB entry for this */
1263         printk(KERN_INFO "omapdsp: falling into recovery runlevel...\n");
1264         dsp_runlevel(OMAP_DSP_MBCMD_RUNLEVEL_RECOVERY);
1265         __dsp_mmu_itack();
1266         udelay(100);
1267         dsp_exunmap(dspadr);
1268         dsp_err_mmu_clear();
1269         return 0;
1270 }
1271
1272 static void dsp_mmu_init(void)
1273 {
1274         unsigned long phys;
1275         void *virt;
1276
1277         clk_enable(dsp_ck_handle);
1278         down_write(&exmap_sem);
1279
1280         dsp_mmu_disable();      /* clear all */
1281         udelay(100);
1282         dsp_mmu_enable();
1283
1284         /* mapping for ARM MMU */
1285         phys = __pa(dspvect_page);
1286         virt = dspbyte_to_virt(DSP_INIT_PAGE);  /* 0xe0fff000 */
1287         exmap_set_armmmu((unsigned long)virt, phys, PAGE_SIZE);
1288         exmap_tbl[0].buf      = dspvect_page;
1289         exmap_tbl[0].vadr     = virt;
1290         exmap_tbl[0].usecount = 0;
1291         exmap_tbl[0].order    = 0;
1292         exmap_tbl[0].valid    = 1;
1293         exmap_tbl[0].cntnu    = 0;
1294
1295         /* DSP TLB initialization */
1296         set_tlb_lock(0, 0);
1297         /* preserved, full access */
1298         dsp_mmu_load_tlb(DSP_INIT_PAGE, phys, DSPMMU_CAM_L_SLST_4KB,
1299                          DSPMMU_CAM_L_P, DSPMMU_RAM_L_AP_FA);
1300         up_write(&exmap_sem);
1301         clk_disable(dsp_ck_handle);
1302 }
1303
1304 static void dsp_mmu_shutdown(void)
1305 {
1306         exmap_flush();
1307         dsp_mmu_disable();      /* clear all */
1308 }
1309
1310 /*
1311  * intmem_enable() / disable():
1312  * if the address is in DSP internal memories,
1313  * we send PM mailbox commands so that DSP DMA domain won't go in idle
1314  * when ARM is accessing to those memories.
1315  */
1316 static int intmem_enable(void)
1317 {
1318         int ret = 0;
1319
1320         if (dsp_is_ready())
1321                 ret = dsp_mbsend(MBCMD(PM), OMAP_DSP_MBCMD_PM_ENABLE,
1322                                  DSPREG_ICR_DMA_IDLE_DOMAIN);
1323
1324         return ret;
1325 }
1326
1327 static void intmem_disable(void) {
1328         if (dsp_is_ready())
1329                 dsp_mbsend(MBCMD(PM), OMAP_DSP_MBCMD_PM_DISABLE,
1330                            DSPREG_ICR_DMA_IDLE_DOMAIN);
1331 }
1332
1333 /*
1334  * dsp_mem_enable() / disable()
1335  */
1336 int intmem_usecount;
1337
1338 int dsp_mem_enable(void *adr)
1339 {
1340         int ret = 0;
1341
1342         if (is_dsp_internal_mem(adr)) {
1343                 if (intmem_usecount++ == 0)
1344                         ret = omap_dsp_request_mem();
1345         } else
1346                 down_read(&exmap_sem);
1347
1348         return ret;
1349 }
1350
1351 void dsp_mem_disable(void *adr)
1352 {
1353         if (is_dsp_internal_mem(adr)) {
1354                 if (--intmem_usecount == 0)
1355                         omap_dsp_release_mem();
1356         } else
1357                 up_read(&exmap_sem);
1358 }
1359
1360 /* for safety */
1361 void dsp_mem_usecount_clear(void)
1362 {
1363         if (intmem_usecount != 0) {
1364                 printk(KERN_WARNING
1365                        "omapdsp: unbalanced memory request/release detected.\n"
1366                        "         intmem_usecount is not zero at where "
1367                        "it should be! ... fixed to be zero.\n");
1368                 intmem_usecount = 0;
1369                 omap_dsp_release_mem();
1370         }
1371 }
1372
1373 /*
1374  * dsp_mem file operations
1375  */
1376 static loff_t dsp_mem_lseek(struct file *file, loff_t offset, int orig)
1377 {
1378         loff_t ret;
1379
1380         mutex_lock(&file->f_dentry->d_inode->i_mutex);
1381         switch (orig) {
1382         case 0:
1383                 file->f_pos = offset;
1384                 ret = file->f_pos;
1385                 break;
1386         case 1:
1387                 file->f_pos += offset;
1388                 ret = file->f_pos;
1389                 break;
1390         default:
1391                 ret = -EINVAL;
1392         }
1393         mutex_unlock(&file->f_dentry->d_inode->i_mutex);
1394         return ret;
1395 }
1396
1397 static ssize_t intmem_read(struct file *file, char *buf, size_t count,
1398                            loff_t *ppos)
1399 {
1400         unsigned long p = *ppos;
1401         void *vadr = dspbyte_to_virt(p);
1402         ssize_t size = dspmem_size;
1403         ssize_t read;
1404
1405         if (p >= size)
1406                 return 0;
1407         clk_enable(api_ck_handle);
1408         read = count;
1409         if (count > size - p)
1410                 read = size - p;
1411         if (copy_to_user(buf, vadr, read)) {
1412                 read = -EFAULT;
1413                 goto out;
1414         }
1415         *ppos += read;
1416 out:
1417         clk_disable(api_ck_handle);
1418         return read;
1419 }
1420
1421 static ssize_t exmem_read(struct file *file, char *buf, size_t count,
1422                           loff_t *ppos)
1423 {
1424         unsigned long p = *ppos;
1425         void *vadr = dspbyte_to_virt(p);
1426
1427         if (!exmap_valid(vadr, count)) {
1428                 printk(KERN_ERR
1429                        "omapdsp: DSP address %08lx / size %08x "
1430                        "is not valid!\n", p, count);
1431                 return -EFAULT;
1432         }
1433         if (count > DSPSPACE_SIZE - p)
1434                 count = DSPSPACE_SIZE - p;
1435         if (copy_to_user(buf, vadr, count))
1436                 return -EFAULT;
1437         *ppos += count;
1438
1439         return count;
1440 }
1441
1442 static ssize_t dsp_mem_read(struct file *file, char *buf, size_t count,
1443                             loff_t *ppos)
1444 {
1445         int ret;
1446         void *vadr = dspbyte_to_virt(*(unsigned long *)ppos);
1447
1448         if (dsp_mem_enable(vadr) < 0)
1449                 return -EBUSY;
1450         if (is_dspbyte_internal_mem(*ppos))
1451                 ret = intmem_read(file, buf, count, ppos);
1452         else
1453                 ret = exmem_read(file, buf, count, ppos);
1454         dsp_mem_disable(vadr);
1455
1456         return ret;
1457 }
1458
1459 static ssize_t intmem_write(struct file *file, const char *buf, size_t count,
1460                             loff_t *ppos)
1461 {
1462         unsigned long p = *ppos;
1463         void *vadr = dspbyte_to_virt(p);
1464         ssize_t size = dspmem_size;
1465         ssize_t written;
1466
1467         if (p >= size)
1468                 return 0;
1469         clk_enable(api_ck_handle);
1470         written = count;
1471         if (count > size - p)
1472                 written = size - p;
1473         if (copy_from_user(vadr, buf, written)) {
1474                 written = -EFAULT;
1475                 goto out;
1476         }
1477         *ppos += written;
1478 out:
1479         clk_disable(api_ck_handle);
1480         return written;
1481 }
1482
1483 static ssize_t exmem_write(struct file *file, const char *buf, size_t count,
1484                            loff_t *ppos)
1485 {
1486         unsigned long p = *ppos;
1487         void *vadr = dspbyte_to_virt(p);
1488
1489         if (!exmap_valid(vadr, count)) {
1490                 printk(KERN_ERR
1491                        "omapdsp: DSP address %08lx / size %08x "
1492                        "is not valid!\n", p, count);
1493                 return -EFAULT;
1494         }
1495         if (count > DSPSPACE_SIZE - p)
1496                 count = DSPSPACE_SIZE - p;
1497         if (copy_from_user(vadr, buf, count))
1498                 return -EFAULT;
1499         *ppos += count;
1500
1501         return count;
1502 }
1503
1504 static ssize_t dsp_mem_write(struct file *file, const char *buf, size_t count,
1505                              loff_t *ppos)
1506 {
1507         int ret;
1508         void *vadr = dspbyte_to_virt(*(unsigned long *)ppos);
1509
1510         if (dsp_mem_enable(vadr) < 0)
1511                 return -EBUSY;
1512         if (is_dspbyte_internal_mem(*ppos))
1513                 ret = intmem_write(file, buf, count, ppos);
1514         else
1515                 ret = exmem_write(file, buf, count, ppos);
1516         dsp_mem_disable(vadr);
1517
1518         return ret;
1519 }
1520
1521 static int dsp_mem_ioctl(struct inode *inode, struct file *file,
1522                          unsigned int cmd, unsigned long arg)
1523 {
1524         switch (cmd) {
1525         case OMAP_DSP_MEM_IOCTL_MMUINIT:
1526                 dsp_mmu_init();
1527                 return 0;
1528
1529         case OMAP_DSP_MEM_IOCTL_EXMAP:
1530                 {
1531                         struct omap_dsp_mapinfo mapinfo;
1532                         if (copy_from_user(&mapinfo, (void *)arg,
1533                                            sizeof(mapinfo)))
1534                                 return -EFAULT;
1535                         return dsp_exmap(mapinfo.dspadr, 0, mapinfo.size,
1536                                          EXMAP_TYPE_MEM);
1537                 }
1538
1539         case OMAP_DSP_MEM_IOCTL_EXUNMAP:
1540                 return dsp_exunmap((unsigned long)arg);
1541
1542         case OMAP_DSP_MEM_IOCTL_EXMAP_FLUSH:
1543                 exmap_flush();
1544                 return 0;
1545
1546         case OMAP_DSP_MEM_IOCTL_FBEXPORT:
1547                 {
1548                         unsigned long dspadr;
1549                         int ret;
1550                         if (copy_from_user(&dspadr, (void *)arg, sizeof(long)))
1551                                 return -EFAULT;
1552                         ret = dsp_fbexport(&dspadr);
1553                         if (copy_to_user((void *)arg, &dspadr, sizeof(long)))
1554                                 return -EFAULT;
1555                         return ret;
1556                 }
1557
1558         case OMAP_DSP_MEM_IOCTL_MMUITACK:
1559                 return dsp_mmu_itack();
1560
1561         case OMAP_DSP_MEM_IOCTL_KMEM_RESERVE:
1562                 {
1563                         unsigned long size;
1564                         if (copy_from_user(&size, (void *)arg, sizeof(long)))
1565                                 return -EFAULT;
1566                         return dsp_kmem_reserve(size);
1567                 }
1568
1569         case OMAP_DSP_MEM_IOCTL_KMEM_RELEASE:
1570                 dsp_kmem_release();
1571                 return 0;
1572
1573         default:
1574                 return -ENOIOCTLCMD;
1575         }
1576 }
1577
1578 static int dsp_mem_mmap(struct file *file, struct vm_area_struct *vma)
1579 {
1580         /*
1581          * FIXME
1582          */
1583         return -ENOSYS;
1584 }
1585
1586 static int dsp_mem_open(struct inode *inode, struct file *file)
1587 {
1588         if (!capable(CAP_SYS_RAWIO))
1589                 return -EPERM;
1590
1591         return 0;
1592 }
1593
1594 static int dsp_mem_release(struct inode *inode, struct file *file)
1595 {
1596         return 0;
1597 }
1598
1599 #ifdef CONFIG_FB_OMAP_LCDC_EXTERNAL
1600 /*
1601  * fb update functions:
1602  * fbupd_response() is executed by the workqueue.
1603  * fbupd_cb() is called when fb update is done, in interrupt context.
1604  * mbx1_fbupd() is called when KFUNC:FBCTL:UPD is received from DSP.
1605  */
1606 static void fbupd_response(void *arg)
1607 {
1608         int status;
1609
1610         status = dsp_mbsend(MBCMD(KFUNC), OMAP_DSP_MBCMD_KFUNC_FBCTL,
1611                             OMAP_DSP_MBCMD_FBCTL_UPD);
1612         if (status < 0) {
1613                 /* FIXME: DSP is busy !! */
1614                 printk(KERN_ERR
1615                        "omapdsp: DSP is busy when trying to send FBCTL:UPD "
1616                        "response!\n");
1617         }
1618 }
1619
1620 static DECLARE_WORK(fbupd_response_work, (void (*)(void *))fbupd_response,
1621                     NULL);
1622
1623 static void fbupd_cb(void *arg)
1624 {
1625         schedule_work(&fbupd_response_work);
1626 }
1627
1628 void mbx1_fbctl_upd(void)
1629 {
1630         struct omapfb_update_window win;
1631         volatile unsigned short *buf = ipbuf_sys_da->d;
1632
1633         /* FIXME: try count sometimes exceeds 1000. */
1634         if (sync_with_dsp(&ipbuf_sys_da->s, OMAP_DSP_TID_ANON, 5000) < 0) {
1635                 printk(KERN_ERR "mbx: FBCTL:UPD - IPBUF sync failed!\n");
1636                 return;
1637         }
1638         win.x = buf[0];
1639         win.y = buf[1];
1640         win.width = buf[2];
1641         win.height = buf[3];
1642         win.format = buf[4];
1643         release_ipbuf_pvt(ipbuf_sys_da);
1644
1645         if (!omapfb_ready) {
1646                 printk(KERN_WARNING
1647                        "omapdsp: fbupd() called while HWA742 is not ready!\n");
1648                 return;
1649         }
1650         //printk("calling omapfb_update_window_async()\n");
1651         omapfb_update_window_async(&win, fbupd_cb, NULL);
1652 }
1653
1654 #else /* CONFIG_FB_OMAP_LCDC_EXTERNAL */
1655
1656 void mbx1_fbctl_upd(void)
1657 {
1658 }
1659 #endif /* CONFIG_FB_OMAP_LCDC_EXTERNAL */
1660
1661 /*
1662  * sysfs files
1663  */
1664 static ssize_t mmu_show(struct device *dev, struct device_attribute *attr,
1665                         char *buf)
1666 {
1667         int len;
1668         int lbase, victim;
1669         int i;
1670
1671         clk_enable(dsp_ck_handle);
1672         down_read(&exmap_sem);
1673
1674         get_tlb_lock(&lbase, &victim);
1675
1676         len = sprintf(buf, "p: preserved,  v: valid\n"
1677                            "ety       cam_va     ram_pa   sz ap\n");
1678                         /* 00: p v 0x300000 0x10171800 64KB FA */
1679         for (i = 0; i < 32; i++) {
1680                 unsigned short cam_h, cam_l, ram_h, ram_l;
1681                 unsigned short cam_l_va_mask, prsvd, cam_vld, slst;
1682                 unsigned long cam_va;
1683                 unsigned short ram_l_ap;
1684                 unsigned long ram_pa;
1685                 char *pgsz_str, *ap_str;
1686
1687                 /* read a TLB entry */
1688                 __read_tlb(lbase, i, &cam_h, &cam_l, &ram_h, &ram_l);
1689
1690                 slst = cam_l & DSPMMU_CAM_L_SLST_MASK;
1691                 cam_l_va_mask = get_cam_l_va_mask(slst);
1692                 pgsz_str = (slst == DSPMMU_CAM_L_SLST_1MB) ? " 1MB":
1693                            (slst == DSPMMU_CAM_L_SLST_64KB)? "64KB":
1694                            (slst == DSPMMU_CAM_L_SLST_4KB) ? " 4KB":
1695                                                              " 1KB";
1696                 prsvd    = cam_l & DSPMMU_CAM_L_P;
1697                 cam_vld  = cam_l & DSPMMU_CAM_L_V;
1698                 ram_l_ap = ram_l & DSPMMU_RAM_L_AP_MASK;
1699                 ap_str = (ram_l_ap == DSPMMU_RAM_L_AP_RO) ? "RO":
1700                          (ram_l_ap == DSPMMU_RAM_L_AP_FA) ? "FA":
1701                                                             "NA";
1702                 cam_va = (unsigned long)(cam_h & DSPMMU_CAM_H_VA_TAG_H_MASK) << 22 |
1703                          (unsigned long)(cam_l & cam_l_va_mask) << 6;
1704                 ram_pa = (unsigned long)ram_h << 16 |
1705                          (ram_l & DSPMMU_RAM_L_RAM_LSB_MASK);
1706
1707                 if (i == lbase)
1708                         len += sprintf(buf + len, "lock base = %d\n", lbase);
1709                 if (i == victim)
1710                         len += sprintf(buf + len, "victim    = %d\n", victim);
1711                 /* 00: p v 0x300000 0x10171800 64KB FA */
1712                 len += sprintf(buf + len,
1713                                "%02d: %c %c 0x%06lx 0x%08lx %s %s\n",
1714                                i,
1715                                prsvd   ? 'p' : ' ',
1716                                cam_vld ? 'v' : ' ',
1717                                cam_va, ram_pa, pgsz_str, ap_str);
1718         }
1719
1720         /* restore victim entry */
1721         set_tlb_lock(lbase, victim);
1722
1723         up_read(&exmap_sem);
1724         clk_disable(dsp_ck_handle);
1725         return len;
1726 }
1727
1728 static struct device_attribute dev_attr_mmu = __ATTR_RO(mmu);
1729
1730 static ssize_t exmap_show(struct device *dev, struct device_attribute *attr,
1731                           char *buf)
1732 {
1733         int len;
1734         int i;
1735
1736         down_read(&exmap_sem);
1737         len = sprintf(buf, "v: valid,  c: cntnu\n"
1738                            "ety           vadr        buf od uc\n");
1739                          /* 00: v c 0xe0300000 0xc0171800  0 */
1740         for (i = 0; i < DSPMMU_TLB_LINES; i++) {
1741                 struct exmap_tbl *ent = &exmap_tbl[i];
1742                 /* 00: v c 0xe0300000 0xc0171800  0 */
1743                 len += sprintf(buf + len, "%02d: %c %c 0x%8p 0x%8p %2d %2d\n",
1744                                i,
1745                                ent->valid ? 'v' : ' ',
1746                                ent->cntnu ? 'c' : ' ',
1747                                ent->vadr, ent->buf, ent->order, ent->usecount);
1748         }
1749
1750         up_read(&exmap_sem);
1751         return len;
1752 }
1753
1754 static struct device_attribute dev_attr_exmap = __ATTR_RO(exmap);
1755
1756 static ssize_t kmem_pool_show(struct device *dev,
1757                               struct device_attribute *attr, char *buf)
1758 {
1759         int nr_1M, nr_64K, total;
1760
1761         nr_1M = kmem_pool_1M->min_nr;
1762         nr_64K = kmem_pool_64K->min_nr;
1763         total = nr_1M * SZ_1MB + nr_64K * SZ_64KB;
1764
1765         return sprintf(buf, "0x%x %d %d\n", total, nr_1M, nr_64K);
1766 }
1767
1768 static struct device_attribute dev_attr_kmem_pool = __ATTR_RO(kmem_pool);
1769
1770 /*
1771  * DSP MMU interrupt handler
1772  */
1773
1774 /*
1775  * MMU fault mask:
1776  * We ignore prefetch err.
1777  */
1778 #define MMUFAULT_MASK \
1779         (DSPMMU_FAULT_ST_PERM |\
1780          DSPMMU_FAULT_ST_TLB_MISS |\
1781          DSPMMU_FAULT_ST_TRANS)
1782 irqreturn_t dsp_mmu_interrupt(int irq, void *dev_id, struct pt_regs *regs)
1783 {
1784         unsigned short status;
1785         unsigned short adh, adl;
1786         unsigned short dp;
1787
1788         status = omap_readw(DSPMMU_FAULT_ST);
1789         adh = omap_readw(DSPMMU_FAULT_AD_H);
1790         adl = omap_readw(DSPMMU_FAULT_AD_L);
1791         dp = adh & DSPMMU_FAULT_AD_H_DP;
1792         dsp_fault_adr = MKLONG(adh & DSPMMU_FAULT_AD_H_ADR_MASK, adl);
1793         /* if the fault is masked, nothing to do */
1794         if ((status & MMUFAULT_MASK) == 0) {
1795                 printk(KERN_DEBUG "DSP MMU interrupt, but ignoring.\n");
1796                 /*
1797                  * note: in OMAP1710,
1798                  * when CACHE + DMA domain gets out of idle in DSP,
1799                  * MMU interrupt occurs but DSPMMU_FAULT_ST is not set.
1800                  * in this case, we just ignore the interrupt.
1801                  */
1802                 if (status) {
1803                         printk(KERN_DEBUG "%s%s%s%s\n",
1804                                (status & DSPMMU_FAULT_ST_PREF)?
1805                                         "  (prefetch err)" : "",
1806                                (status & DSPMMU_FAULT_ST_PERM)?
1807                                         "  (permission fault)" : "",
1808                                (status & DSPMMU_FAULT_ST_TLB_MISS)?
1809                                         "  (TLB miss)" : "",
1810                                (status & DSPMMU_FAULT_ST_TRANS) ?
1811                                         "  (translation fault)": "");
1812                         printk(KERN_DEBUG
1813                                "fault address = %s: 0x%06lx\n",
1814                                dp ? "DATA" : "PROGRAM",
1815                                dsp_fault_adr);
1816                 }
1817                 return IRQ_HANDLED;
1818         }
1819
1820         printk(KERN_INFO "DSP MMU interrupt!\n");
1821         printk(KERN_INFO "%s%s%s%s\n",
1822                (status & DSPMMU_FAULT_ST_PREF)?
1823                         (MMUFAULT_MASK & DSPMMU_FAULT_ST_PREF)?
1824                                 "  prefetch err":
1825                                 "  (prefetch err)":
1826                                 "",
1827                (status & DSPMMU_FAULT_ST_PERM)?
1828                         (MMUFAULT_MASK & DSPMMU_FAULT_ST_PERM)?
1829                                 "  permission fault":
1830                                 "  (permission fault)":
1831                                 "",
1832                (status & DSPMMU_FAULT_ST_TLB_MISS)?
1833                         (MMUFAULT_MASK & DSPMMU_FAULT_ST_TLB_MISS)?
1834                                 "  TLB miss":
1835                                 "  (TLB miss)":
1836                                 "",
1837                (status & DSPMMU_FAULT_ST_TRANS)?
1838                         (MMUFAULT_MASK & DSPMMU_FAULT_ST_TRANS)?
1839                                 "  translation fault":
1840                                 "  (translation fault)":
1841                                 "");
1842         printk(KERN_INFO "fault address = %s: 0x%06lx\n",
1843                dp ? "DATA" : "PROGRAM",
1844                dsp_fault_adr);
1845
1846         if (dsp_is_ready()) {
1847                 /*
1848                  * If we call dsp_exmap() here,
1849                  * "kernel BUG at slab.c" occurs.
1850                  */
1851                 /* FIXME */
1852                 dsp_err_mmu_set(dsp_fault_adr);
1853         } else {
1854                 disable_irq(INT_DSP_MMU);
1855                 __dsp_mmu_itack();
1856                 printk(KERN_INFO "Resetting DSP...\n");
1857                 dsp_cpustat_request(CPUSTAT_RESET);
1858                 enable_irq(INT_DSP_MMU);
1859                 /*
1860                  * if we enable followings, semaphore lock should be avoided.
1861                  *
1862                 printk(KERN_INFO "Flushing DSP MMU...\n");
1863                 exmap_flush();
1864                 dsp_mmu_init();
1865                  */
1866         }
1867
1868         return IRQ_HANDLED;
1869 }
1870
1871 /*
1872  *
1873  */
1874 struct file_operations dsp_mem_fops = {
1875         .owner   = THIS_MODULE,
1876         .llseek  = dsp_mem_lseek,
1877         .read    = dsp_mem_read,
1878         .write   = dsp_mem_write,
1879         .ioctl   = dsp_mem_ioctl,
1880         .mmap    = dsp_mem_mmap,
1881         .open    = dsp_mem_open,
1882         .release = dsp_mem_release,
1883 };
1884
1885 void dsp_mem_start(void)
1886 {
1887         dsp_register_mem_cb(intmem_enable, intmem_disable);
1888 }
1889
1890 void dsp_mem_stop(void)
1891 {
1892         memset(&mem_sync, 0, sizeof(struct mem_sync_struct));
1893         dsp_unregister_mem_cb();
1894 }
1895
1896 int __init dsp_mem_init(void)
1897 {
1898         int i;
1899
1900         for (i = 0; i < DSPMMU_TLB_LINES; i++) {
1901                 exmap_tbl[i].valid = 0;
1902         }
1903
1904         dspvect_page = (void *)__get_dma_pages(GFP_KERNEL, 0);
1905         if (dspvect_page == NULL) {
1906                 printk(KERN_ERR
1907                        "omapdsp: failed to allocate memory "
1908                        "for dsp vector table\n");
1909                 return -ENOMEM;
1910         }
1911         dsp_mmu_init();
1912         dsp_set_idle_boot_base(IDLEPG_BASE, IDLEPG_SIZE);
1913
1914         device_create_file(&dsp_device.dev, &dev_attr_mmu);
1915         device_create_file(&dsp_device.dev, &dev_attr_exmap);
1916         device_create_file(&dsp_device.dev, &dev_attr_kmem_pool);
1917
1918         return 0;
1919 }
1920
1921 void dsp_mem_exit(void)
1922 {
1923         dsp_mmu_shutdown();
1924         dsp_kmem_release();
1925
1926         if (dspvect_page != NULL) {
1927                 unsigned long virt;
1928
1929                 down_read(&exmap_sem);
1930
1931                 virt = (unsigned long)dspbyte_to_virt(DSP_INIT_PAGE);
1932                 flush_tlb_kernel_range(virt, virt + PAGE_SIZE);
1933                 free_page((unsigned long)dspvect_page);
1934                 dspvect_page = NULL;
1935
1936                 up_read(&exmap_sem);
1937         }
1938
1939         device_remove_file(&dsp_device.dev, &dev_attr_mmu);
1940         device_remove_file(&dsp_device.dev, &dev_attr_exmap);
1941         device_remove_file(&dsp_device.dev, &dev_attr_kmem_pool);
1942 }