arch/blackfin/kernel/bfin_dma_5xx.c

   1 /*
   2  * File:         arch/blackfin/kernel/bfin_dma_5xx.c
   3  * Based on:
   4  * Author:
   5  *
   6  * Created:
   7  * Description:  This file contains the simple DMA Implementation for Blackfin
   8  *
   9  * Modified:
  10  *               Copyright 2004-2006 Analog Devices Inc.
  11  *
  12  * Bugs:         Enter bugs at http://blackfin.uclinux.org/
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  *
  24  * You should have received a copy of the GNU General Public License
  25  * along with this program; if not, see the file COPYING, or write
  26  * to the Free Software Foundation, Inc.,
  27  * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  28  */
  29
  30 #include <linux/errno.h>
  31 #include <linux/module.h>
  32 #include <linux/proc_fs.h>
  33 #include <linux/sched.h>
  34 #include <linux/seq_file.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/kernel.h>
  37 #include <linux/param.h>
  38
  39 #include <asm/blackfin.h>
  40 #include <asm/dma.h>
  41 #include <asm/cacheflush.h>
  42
  43 /**************************************************************************
  44  * Global Variables
  45 ***************************************************************************/
  46
  47 static struct dma_channel dma_ch[MAX_DMA_CHANNELS];
  48
  49 /*------------------------------------------------------------------------------
  50  *       Set the Buffer Clear bit in the Configuration register of specific DMA
  51  *       channel. This will stop the descriptor based DMA operation.
  52  *-----------------------------------------------------------------------------*/
  53 static void clear_dma_buffer(unsigned int channel)
  54 {
  55         dma_ch[channel].regs->cfg |= RESTART;
  56         SSYNC();
  57         dma_ch[channel].regs->cfg &= ~RESTART;
  58 }
  59
  60 static int __init blackfin_dma_init(void)
  61 {
  62         int i;
  63
  64         printk(KERN_INFO "Blackfin DMA Controller\n");
  65
  66         for (i = 0; i < MAX_DMA_CHANNELS; i++) {
  67                 dma_ch[i].chan_status = DMA_CHANNEL_FREE;
  68                 dma_ch[i].regs = dma_io_base_addr[i];
  69                 mutex_init(&(dma_ch[i].dmalock));
  70         }
  71         /* Mark MEMDMA Channel 0 as requested since we're using it internally */
  72         request_dma(CH_MEM_STREAM0_DEST, "Blackfin dma_memcpy");
  73         request_dma(CH_MEM_STREAM0_SRC, "Blackfin dma_memcpy");
  74
  75 #if defined(CONFIG_DEB_DMA_URGENT)
  76         bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
  77                          | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
  78 #endif
  79
  80         return 0;
  81 }
  82 arch_initcall(blackfin_dma_init);
  83
  84 #ifdef CONFIG_PROC_FS
  85
  86 static int proc_dma_show(struct seq_file *m, void *v)
  87 {
  88         int i;
  89
  90         for (i = 0 ; i < MAX_DMA_CHANNELS; ++i)
  91                 if (dma_ch[i].chan_status != DMA_CHANNEL_FREE)
  92                         seq_printf(m, "%2d: %s\n", i, dma_ch[i].device_id);
  93
  94         return 0;
  95 }
  96
  97 static int proc_dma_open(struct inode *inode, struct file *file)
  98 {
  99         return single_open(file, proc_dma_show, NULL);
 100 }
 101
 102 static const struct file_operations proc_dma_operations = {
 103         .open           = proc_dma_open,
 104         .read           = seq_read,
 105         .llseek         = seq_lseek,
 106         .release        = single_release,
 107 };
 108
 109 static int __init proc_dma_init(void)
 110 {
 111         return proc_create("dma", 0, NULL, &proc_dma_operations) != NULL;
 112 }
 113 late_initcall(proc_dma_init);
 114 #endif
 115
 116 /*------------------------------------------------------------------------------
 117  *      Request the specific DMA channel from the system.
 118  *-----------------------------------------------------------------------------*/
 119 int request_dma(unsigned int channel, const char *device_id)
 120 {
 121         pr_debug("request_dma() : BEGIN \n");
 122
 123         if (device_id == NULL)
 124                 printk(KERN_WARNING "request_dma(%u): no device_id given\n", channel);
 125
 126 #if defined(CONFIG_BF561) && ANOMALY_05000182
 127         if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
 128                 if (get_cclk() > 500000000) {
 129                         printk(KERN_WARNING
 130                                "Request IMDMA failed due to ANOMALY 05000182\n");
 131                         return -EFAULT;
 132                 }
 133         }
 134 #endif
 135
 136         mutex_lock(&(dma_ch[channel].dmalock));
 137
 138         if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED)
 139             || (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) {
 140                 mutex_unlock(&(dma_ch[channel].dmalock));
 141                 pr_debug("DMA CHANNEL IN USE  \n");
 142                 return -EBUSY;
 143         } else {
 144                 dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
 145                 pr_debug("DMA CHANNEL IS ALLOCATED  \n");
 146         }
 147
 148         mutex_unlock(&(dma_ch[channel].dmalock));
 149
 150 #ifdef CONFIG_BF54x
 151         if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) {
 152                 unsigned int per_map;
 153                 per_map = dma_ch[channel].regs->peripheral_map & 0xFFF;
 154                 if (strncmp(device_id, "BFIN_UART", 9) == 0)
 155                         dma_ch[channel].regs->peripheral_map = per_map |
 156                                 ((channel - CH_UART2_RX + 0xC)<<12);
 157                 else
 158                         dma_ch[channel].regs->peripheral_map = per_map |
 159                                 ((channel - CH_UART2_RX + 0x6)<<12);
 160         }
 161 #endif
 162
 163         dma_ch[channel].device_id = device_id;
 164         dma_ch[channel].irq_callback = NULL;
 165
 166         /* This is to be enabled by putting a restriction -
 167          * you have to request DMA, before doing any operations on
 168          * descriptor/channel
 169          */
 170         pr_debug("request_dma() : END  \n");
 171         return channel;
 172 }
 173 EXPORT_SYMBOL(request_dma);
 174
 175 int set_dma_callback(unsigned int channel, dma_interrupt_t callback, void *data)
 176 {
 177         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 178                && channel < MAX_DMA_CHANNELS));
 179
 180         if (callback != NULL) {
 181                 int ret_val;
 182                 dma_ch[channel].irq = channel2irq(channel);
 183                 dma_ch[channel].data = data;
 184
 185                 ret_val =
 186                     request_irq(dma_ch[channel].irq, callback, IRQF_DISABLED,
 187                                 dma_ch[channel].device_id, data);
 188                 if (ret_val) {
 189                         printk(KERN_NOTICE
 190                                "Request irq in DMA engine failed.\n");
 191                         return -EPERM;
 192                 }
 193                 dma_ch[channel].irq_callback = callback;
 194         }
 195         return 0;
 196 }
 197 EXPORT_SYMBOL(set_dma_callback);
 198
 199 void free_dma(unsigned int channel)
 200 {
 201         pr_debug("freedma() : BEGIN \n");
 202         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 203                && channel < MAX_DMA_CHANNELS));
 204
 205         /* Halt the DMA */
 206         disable_dma(channel);
 207         clear_dma_buffer(channel);
 208
 209         if (dma_ch[channel].irq_callback != NULL)
 210                 free_irq(dma_ch[channel].irq, dma_ch[channel].data);
 211
 212         /* Clear the DMA Variable in the Channel */
 213         mutex_lock(&(dma_ch[channel].dmalock));
 214         dma_ch[channel].chan_status = DMA_CHANNEL_FREE;
 215         mutex_unlock(&(dma_ch[channel].dmalock));
 216
 217         pr_debug("freedma() : END \n");
 218 }
 219 EXPORT_SYMBOL(free_dma);
 220
 221 void dma_enable_irq(unsigned int channel)
 222 {
 223         pr_debug("dma_enable_irq() : BEGIN \n");
 224         enable_irq(dma_ch[channel].irq);
 225 }
 226 EXPORT_SYMBOL(dma_enable_irq);
 227
 228 void dma_disable_irq(unsigned int channel)
 229 {
 230         pr_debug("dma_disable_irq() : BEGIN \n");
 231         disable_irq(dma_ch[channel].irq);
 232 }
 233 EXPORT_SYMBOL(dma_disable_irq);
 234
 235 int dma_channel_active(unsigned int channel)
 236 {
 237         if (dma_ch[channel].chan_status == DMA_CHANNEL_FREE) {
 238                 return 0;
 239         } else {
 240                 return 1;
 241         }
 242 }
 243 EXPORT_SYMBOL(dma_channel_active);
 244
 245 /*------------------------------------------------------------------------------
 246 *       stop the specific DMA channel.
 247 *-----------------------------------------------------------------------------*/
 248 void disable_dma(unsigned int channel)
 249 {
 250         pr_debug("stop_dma() : BEGIN \n");
 251         dma_ch[channel].regs->cfg &= ~DMAEN;    /* Clean the enable bit */
 252         SSYNC();
 253         dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
 254         /* Needs to be enabled Later */
 255         pr_debug("stop_dma() : END \n");
 256         return;
 257 }
 258 EXPORT_SYMBOL(disable_dma);
 259
 260 void enable_dma(unsigned int channel)
 261 {
 262         pr_debug("enable_dma() : BEGIN \n");
 263         dma_ch[channel].chan_status = DMA_CHANNEL_ENABLED;
 264         dma_ch[channel].regs->curr_x_count = 0;
 265         dma_ch[channel].regs->curr_y_count = 0;
 266
 267         dma_ch[channel].regs->cfg |= DMAEN;     /* Set the enable bit */
 268         pr_debug("enable_dma() : END \n");
 269         return;
 270 }
 271 EXPORT_SYMBOL(enable_dma);
 272
 273 /*------------------------------------------------------------------------------
 274 *               Set the Start Address register for the specific DMA channel
 275 *               This function can be used for register based DMA,
 276 *               to setup the start address
 277 *               addr:           Starting address of the DMA Data to be transferred.
 278 *-----------------------------------------------------------------------------*/
 279 void set_dma_start_addr(unsigned int channel, unsigned long addr)
 280 {
 281         pr_debug("set_dma_start_addr() : BEGIN \n");
 282         dma_ch[channel].regs->start_addr = addr;
 283         pr_debug("set_dma_start_addr() : END\n");
 284 }
 285 EXPORT_SYMBOL(set_dma_start_addr);
 286
 287 void set_dma_next_desc_addr(unsigned int channel, unsigned long addr)
 288 {
 289         pr_debug("set_dma_next_desc_addr() : BEGIN \n");
 290         dma_ch[channel].regs->next_desc_ptr = addr;
 291         pr_debug("set_dma_next_desc_addr() : END\n");
 292 }
 293 EXPORT_SYMBOL(set_dma_next_desc_addr);
 294
 295 void set_dma_curr_desc_addr(unsigned int channel, unsigned long addr)
 296 {
 297         pr_debug("set_dma_curr_desc_addr() : BEGIN \n");
 298         dma_ch[channel].regs->curr_desc_ptr = addr;
 299         pr_debug("set_dma_curr_desc_addr() : END\n");
 300 }
 301 EXPORT_SYMBOL(set_dma_curr_desc_addr);
 302
 303 void set_dma_x_count(unsigned int channel, unsigned short x_count)
 304 {
 305         dma_ch[channel].regs->x_count = x_count;
 306 }
 307 EXPORT_SYMBOL(set_dma_x_count);
 308
 309 void set_dma_y_count(unsigned int channel, unsigned short y_count)
 310 {
 311         dma_ch[channel].regs->y_count = y_count;
 312 }
 313 EXPORT_SYMBOL(set_dma_y_count);
 314
 315 void set_dma_x_modify(unsigned int channel, short x_modify)
 316 {
 317         dma_ch[channel].regs->x_modify = x_modify;
 318 }
 319 EXPORT_SYMBOL(set_dma_x_modify);
 320
 321 void set_dma_y_modify(unsigned int channel, short y_modify)
 322 {
 323         dma_ch[channel].regs->y_modify = y_modify;
 324 }
 325 EXPORT_SYMBOL(set_dma_y_modify);
 326
 327 void set_dma_config(unsigned int channel, unsigned short config)
 328 {
 329         dma_ch[channel].regs->cfg = config;
 330 }
 331 EXPORT_SYMBOL(set_dma_config);
 332
 333 unsigned short
 334 set_bfin_dma_config(char direction, char flow_mode,
 335                     char intr_mode, char dma_mode, char width, char syncmode)
 336 {
 337         unsigned short config;
 338
 339         config =
 340             ((direction << 1) | (width << 2) | (dma_mode << 4) |
 341              (intr_mode << 6) | (flow_mode << 12) | (syncmode << 5));
 342         return config;
 343 }
 344 EXPORT_SYMBOL(set_bfin_dma_config);
 345
 346 void set_dma_sg(unsigned int channel, struct dmasg *sg, int nr_sg)
 347 {
 348         dma_ch[channel].regs->cfg |= ((nr_sg & 0x0F) << 8);
 349         dma_ch[channel].regs->next_desc_ptr = (unsigned int)sg;
 350 }
 351 EXPORT_SYMBOL(set_dma_sg);
 352
 353 void set_dma_curr_addr(unsigned int channel, unsigned long addr)
 354 {
 355         dma_ch[channel].regs->curr_addr_ptr = addr;
 356 }
 357 EXPORT_SYMBOL(set_dma_curr_addr);
 358
 359 /*------------------------------------------------------------------------------
 360  *      Get the DMA status of a specific DMA channel from the system.
 361  *-----------------------------------------------------------------------------*/
 362 unsigned short get_dma_curr_irqstat(unsigned int channel)
 363 {
 364         return dma_ch[channel].regs->irq_status;
 365 }
 366 EXPORT_SYMBOL(get_dma_curr_irqstat);
 367
 368 /*------------------------------------------------------------------------------
 369  *      Clear the DMA_DONE bit in DMA status. Stop the DMA completion interrupt.
 370  *-----------------------------------------------------------------------------*/
 371 void clear_dma_irqstat(unsigned int channel)
 372 {
 373         dma_ch[channel].regs->irq_status |= 3;
 374 }
 375 EXPORT_SYMBOL(clear_dma_irqstat);
 376
 377 /*------------------------------------------------------------------------------
 378  *      Get current DMA xcount of a specific DMA channel from the system.
 379  *-----------------------------------------------------------------------------*/
 380 unsigned short get_dma_curr_xcount(unsigned int channel)
 381 {
 382         return dma_ch[channel].regs->curr_x_count;
 383 }
 384 EXPORT_SYMBOL(get_dma_curr_xcount);
 385
 386 /*------------------------------------------------------------------------------
 387  *      Get current DMA ycount of a specific DMA channel from the system.
 388  *-----------------------------------------------------------------------------*/
 389 unsigned short get_dma_curr_ycount(unsigned int channel)
 390 {
 391         return dma_ch[channel].regs->curr_y_count;
 392 }
 393 EXPORT_SYMBOL(get_dma_curr_ycount);
 394
 395 unsigned long get_dma_next_desc_ptr(unsigned int channel)
 396 {
 397         return dma_ch[channel].regs->next_desc_ptr;
 398 }
 399 EXPORT_SYMBOL(get_dma_next_desc_ptr);
 400
 401 unsigned long get_dma_curr_desc_ptr(unsigned int channel)
 402 {
 403         return dma_ch[channel].regs->curr_desc_ptr;
 404 }
 405 EXPORT_SYMBOL(get_dma_curr_desc_ptr);
 406
 407 unsigned long get_dma_curr_addr(unsigned int channel)
 408 {
 409         return dma_ch[channel].regs->curr_addr_ptr;
 410 }
 411 EXPORT_SYMBOL(get_dma_curr_addr);
 412
 413 #ifdef CONFIG_PM
 414 int blackfin_dma_suspend(void)
 415 {
 416         int i;
 417
 418 #ifdef CONFIG_BF561     /* IMDMA channels doesn't have a PERIPHERAL_MAP */
 419         for (i = 0; i <= CH_MEM_STREAM3_SRC; i++) {
 420 #else
 421         for (i = 0; i < MAX_DMA_CHANNELS; i++) {
 422 #endif
 423                 if (dma_ch[i].chan_status == DMA_CHANNEL_ENABLED) {
 424                         printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
 425                         return -EBUSY;
 426                 }
 427
 428                 dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
 429         }
 430
 431         return 0;
 432 }
 433
 434 void blackfin_dma_resume(void)
 435 {
 436         int i;
 437
 438 #ifdef CONFIG_BF561     /* IMDMA channels doesn't have a PERIPHERAL_MAP */
 439         for (i = 0; i <= CH_MEM_STREAM3_SRC; i++)
 440 #else
 441         for (i = 0; i < MAX_DMA_CHANNELS; i++)
 442 #endif
 443                 dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
 444 }
 445 #endif
 446
 447 static void *__dma_memcpy(void *dest, const void *src, size_t size)
 448 {
 449         int direction;  /* 1 - address decrease, 0 - address increase */
 450         int flag_align; /* 1 - address aligned,  0 - address unaligned */
 451         int flag_2D;    /* 1 - 2D DMA needed,    0 - 1D DMA needed */
 452         unsigned long flags;
 453
 454         if (size <= 0)
 455                 return NULL;
 456
 457         local_irq_save(flags);
 458
 459         if ((unsigned long)src < memory_end)
 460                 blackfin_dcache_flush_range((unsigned int)src,
 461                                             (unsigned int)(src + size));
 462
 463         if ((unsigned long)dest < memory_end)
 464                 blackfin_dcache_invalidate_range((unsigned int)dest,
 465                                                  (unsigned int)(dest + size));
 466
 467         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 468
 469         if ((unsigned long)src < (unsigned long)dest)
 470                 direction = 1;
 471         else
 472                 direction = 0;
 473
 474         if ((((unsigned long)dest % 2) == 0) && (((unsigned long)src % 2) == 0)
 475             && ((size % 2) == 0))
 476                 flag_align = 1;
 477         else
 478                 flag_align = 0;
 479
 480         if (size > 0x10000)     /* size > 64K */
 481                 flag_2D = 1;
 482         else
 483                 flag_2D = 0;
 484
 485         /* Setup destination and source start address */
 486         if (direction) {
 487                 if (flag_align) {
 488                         bfin_write_MDMA_D0_START_ADDR(dest + size - 2);
 489                         bfin_write_MDMA_S0_START_ADDR(src + size - 2);
 490                 } else {
 491                         bfin_write_MDMA_D0_START_ADDR(dest + size - 1);
 492                         bfin_write_MDMA_S0_START_ADDR(src + size - 1);
 493                 }
 494         } else {
 495                 bfin_write_MDMA_D0_START_ADDR(dest);
 496                 bfin_write_MDMA_S0_START_ADDR(src);
 497         }
 498
 499         /* Setup destination and source xcount */
 500         if (flag_2D) {
 501                 if (flag_align) {
 502                         bfin_write_MDMA_D0_X_COUNT(1024 / 2);
 503                         bfin_write_MDMA_S0_X_COUNT(1024 / 2);
 504                 } else {
 505                         bfin_write_MDMA_D0_X_COUNT(1024);
 506                         bfin_write_MDMA_S0_X_COUNT(1024);
 507                 }
 508                 bfin_write_MDMA_D0_Y_COUNT(size >> 10);
 509                 bfin_write_MDMA_S0_Y_COUNT(size >> 10);
 510         } else {
 511                 if (flag_align) {
 512                         bfin_write_MDMA_D0_X_COUNT(size / 2);
 513                         bfin_write_MDMA_S0_X_COUNT(size / 2);
 514                 } else {
 515                         bfin_write_MDMA_D0_X_COUNT(size);
 516                         bfin_write_MDMA_S0_X_COUNT(size);
 517                 }
 518         }
 519
 520         /* Setup destination and source xmodify and ymodify */
 521         if (direction) {
 522                 if (flag_align) {
 523                         bfin_write_MDMA_D0_X_MODIFY(-2);
 524                         bfin_write_MDMA_S0_X_MODIFY(-2);
 525                         if (flag_2D) {
 526                                 bfin_write_MDMA_D0_Y_MODIFY(-2);
 527                                 bfin_write_MDMA_S0_Y_MODIFY(-2);
 528                         }
 529                 } else {
 530                         bfin_write_MDMA_D0_X_MODIFY(-1);
 531                         bfin_write_MDMA_S0_X_MODIFY(-1);
 532                         if (flag_2D) {
 533                                 bfin_write_MDMA_D0_Y_MODIFY(-1);
 534                                 bfin_write_MDMA_S0_Y_MODIFY(-1);
 535                         }
 536                 }
 537         } else {
 538                 if (flag_align) {
 539                         bfin_write_MDMA_D0_X_MODIFY(2);
 540                         bfin_write_MDMA_S0_X_MODIFY(2);
 541                         if (flag_2D) {
 542                                 bfin_write_MDMA_D0_Y_MODIFY(2);
 543                                 bfin_write_MDMA_S0_Y_MODIFY(2);
 544                         }
 545                 } else {
 546                         bfin_write_MDMA_D0_X_MODIFY(1);
 547                         bfin_write_MDMA_S0_X_MODIFY(1);
 548                         if (flag_2D) {
 549                                 bfin_write_MDMA_D0_Y_MODIFY(1);
 550                                 bfin_write_MDMA_S0_Y_MODIFY(1);
 551                         }
 552                 }
 553         }
 554
 555         /* Enable source DMA */
 556         if (flag_2D) {
 557                 if (flag_align) {
 558                         bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D | WDSIZE_16);
 559                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D | WDSIZE_16);
 560                 } else {
 561                         bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D);
 562                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D);
 563                 }
 564         } else {
 565                 if (flag_align) {
 566                         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 567                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 568                 } else {
 569                         bfin_write_MDMA_S0_CONFIG(DMAEN);
 570                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN);
 571                 }
 572         }
 573
 574         SSYNC();
 575
 576         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
 577                 ;
 578
 579         bfin_write_MDMA_D0_IRQ_STATUS(bfin_read_MDMA_D0_IRQ_STATUS() |
 580                                       (DMA_DONE | DMA_ERR));
 581
 582         bfin_write_MDMA_S0_CONFIG(0);
 583         bfin_write_MDMA_D0_CONFIG(0);
 584
 585         local_irq_restore(flags);
 586
 587         return dest;
 588 }
 589
 590 void *dma_memcpy(void *dest, const void *src, size_t size)
 591 {
 592         size_t bulk;
 593         size_t rest;
 594         void * addr;
 595
 596         bulk = (size >> 16) << 16;
 597         rest = size - bulk;
 598         if (bulk)
 599                 __dma_memcpy(dest, src, bulk);
 600         __dma_memcpy(dest+bulk, src+bulk, rest);
 601         return dest;
 602 }
 603 EXPORT_SYMBOL(dma_memcpy);
 604
 605 void *safe_dma_memcpy(void *dest, const void *src, size_t size)
 606 {
 607         void *addr;
 608         addr = dma_memcpy(dest, src, size);
 609         return addr;
 610 }
 611 EXPORT_SYMBOL(safe_dma_memcpy);
 612
 613 void dma_outsb(unsigned long addr, const void *buf, unsigned short len)
 614 {
 615         unsigned long flags;
 616
 617         local_irq_save(flags);
 618
 619         blackfin_dcache_flush_range((unsigned int)buf,
 620                          (unsigned int)(buf) + len);
 621
 622         bfin_write_MDMA_D0_START_ADDR(addr);
 623         bfin_write_MDMA_D0_X_COUNT(len);
 624         bfin_write_MDMA_D0_X_MODIFY(0);
 625         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 626
 627         bfin_write_MDMA_S0_START_ADDR(buf);
 628         bfin_write_MDMA_S0_X_COUNT(len);
 629         bfin_write_MDMA_S0_X_MODIFY(1);
 630         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 631
 632         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
 633         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);
 634
 635         SSYNC();
 636
 637         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 638
 639         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 640
 641         bfin_write_MDMA_S0_CONFIG(0);
 642         bfin_write_MDMA_D0_CONFIG(0);
 643         local_irq_restore(flags);
 644
 645 }
 646 EXPORT_SYMBOL(dma_outsb);
 647
 648
 649 void dma_insb(unsigned long addr, void *buf, unsigned short len)
 650 {
 651         unsigned long flags;
 652
 653         blackfin_dcache_invalidate_range((unsigned int)buf,
 654                          (unsigned int)(buf) + len);
 655
 656         local_irq_save(flags);
 657         bfin_write_MDMA_D0_START_ADDR(buf);
 658         bfin_write_MDMA_D0_X_COUNT(len);
 659         bfin_write_MDMA_D0_X_MODIFY(1);
 660         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 661
 662         bfin_write_MDMA_S0_START_ADDR(addr);
 663         bfin_write_MDMA_S0_X_COUNT(len);
 664         bfin_write_MDMA_S0_X_MODIFY(0);
 665         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 666
 667         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
 668         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);
 669
 670         SSYNC();
 671
 672         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 673
 674         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 675
 676         bfin_write_MDMA_S0_CONFIG(0);
 677         bfin_write_MDMA_D0_CONFIG(0);
 678         local_irq_restore(flags);
 679
 680 }
 681 EXPORT_SYMBOL(dma_insb);
 682
 683 void dma_outsw(unsigned long addr, const void  *buf, unsigned short len)
 684 {
 685         unsigned long flags;
 686
 687         local_irq_save(flags);
 688
 689         blackfin_dcache_flush_range((unsigned int)buf,
 690                          (unsigned int)(buf) + len * sizeof(short));
 691
 692         bfin_write_MDMA_D0_START_ADDR(addr);
 693         bfin_write_MDMA_D0_X_COUNT(len);
 694         bfin_write_MDMA_D0_X_MODIFY(0);
 695         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 696
 697         bfin_write_MDMA_S0_START_ADDR(buf);
 698         bfin_write_MDMA_S0_X_COUNT(len);
 699         bfin_write_MDMA_S0_X_MODIFY(2);
 700         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 701
 702         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 703         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 704
 705         SSYNC();
 706
 707         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 708
 709         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 710
 711         bfin_write_MDMA_S0_CONFIG(0);
 712         bfin_write_MDMA_D0_CONFIG(0);
 713         local_irq_restore(flags);
 714
 715 }
 716 EXPORT_SYMBOL(dma_outsw);
 717
 718 void dma_insw(unsigned long addr, void *buf, unsigned short len)
 719 {
 720         unsigned long flags;
 721
 722         blackfin_dcache_invalidate_range((unsigned int)buf,
 723                          (unsigned int)(buf) + len * sizeof(short));
 724
 725         local_irq_save(flags);
 726
 727         bfin_write_MDMA_D0_START_ADDR(buf);
 728         bfin_write_MDMA_D0_X_COUNT(len);
 729         bfin_write_MDMA_D0_X_MODIFY(2);
 730         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 731
 732         bfin_write_MDMA_S0_START_ADDR(addr);
 733         bfin_write_MDMA_S0_X_COUNT(len);
 734         bfin_write_MDMA_S0_X_MODIFY(0);
 735         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 736
 737         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 738         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 739
 740         SSYNC();
 741
 742         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 743
 744         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 745
 746         bfin_write_MDMA_S0_CONFIG(0);
 747         bfin_write_MDMA_D0_CONFIG(0);
 748         local_irq_restore(flags);
 749
 750 }
 751 EXPORT_SYMBOL(dma_insw);
 752
 753 void dma_outsl(unsigned long addr, const void *buf, unsigned short len)
 754 {
 755         unsigned long flags;
 756
 757         local_irq_save(flags);
 758
 759         blackfin_dcache_flush_range((unsigned int)buf,
 760                          (unsigned int)(buf) + len * sizeof(long));
 761
 762         bfin_write_MDMA_D0_START_ADDR(addr);
 763         bfin_write_MDMA_D0_X_COUNT(len);
 764         bfin_write_MDMA_D0_X_MODIFY(0);
 765         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 766
 767         bfin_write_MDMA_S0_START_ADDR(buf);
 768         bfin_write_MDMA_S0_X_COUNT(len);
 769         bfin_write_MDMA_S0_X_MODIFY(4);
 770         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 771
 772         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
 773         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);
 774
 775         SSYNC();
 776
 777         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 778
 779         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 780
 781         bfin_write_MDMA_S0_CONFIG(0);
 782         bfin_write_MDMA_D0_CONFIG(0);
 783         local_irq_restore(flags);
 784
 785 }
 786 EXPORT_SYMBOL(dma_outsl);
 787
 788 void dma_insl(unsigned long addr, void *buf, unsigned short len)
 789 {
 790         unsigned long flags;
 791
 792         blackfin_dcache_invalidate_range((unsigned int)buf,
 793                          (unsigned int)(buf) + len * sizeof(long));
 794
 795         local_irq_save(flags);
 796
 797         bfin_write_MDMA_D0_START_ADDR(buf);
 798         bfin_write_MDMA_D0_X_COUNT(len);
 799         bfin_write_MDMA_D0_X_MODIFY(4);
 800         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 801
 802         bfin_write_MDMA_S0_START_ADDR(addr);
 803         bfin_write_MDMA_S0_X_COUNT(len);
 804         bfin_write_MDMA_S0_X_MODIFY(0);
 805         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 806
 807         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
 808         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);
 809
 810         SSYNC();
 811
 812         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 813
 814         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 815
 816         bfin_write_MDMA_S0_CONFIG(0);
 817         bfin_write_MDMA_D0_CONFIG(0);
 818         local_irq_restore(flags);
 819
 820 }
 821 EXPORT_SYMBOL(dma_insl);