drivers/net/ibm_newemac/core.c

   1 /*
   2  * drivers/net/ibm_newemac/core.c
   3  *
   4  * Driver for PowerPC 4xx on-chip ethernet controller.
   5  *
   6  * Copyright 2007 Benjamin Herrenschmidt, IBM Corp.
   7  *                <benh@kernel.crashing.org>
   8  *
   9  * Based on the arch/ppc version of the driver:
  10  *
  11  * Copyright (c) 2004, 2005 Zultys Technologies.
  12  * Eugene Surovegin <eugene.surovegin@zultys.com> or <ebs@ebshome.net>
  13  *
  14  * Based on original work by
  15  *      Matt Porter <mporter@kernel.crashing.org>
  16  *      (c) 2003 Benjamin Herrenschmidt <benh@kernel.crashing.org>
  17  *      Armin Kuster <akuster@mvista.com>
  18  *      Johnnie Peters <jpeters@mvista.com>
  19  *
  20  * This program is free software; you can redistribute  it and/or modify it
  21  * under  the terms of  the GNU General  Public License as published by the
  22  * Free Software Foundation;  either version 2 of the  License, or (at your
  23  * option) any later version.
  24  *
  25  */
  26
  27 #include <linux/sched.h>
  28 #include <linux/string.h>
  29 #include <linux/errno.h>
  30 #include <linux/delay.h>
  31 #include <linux/types.h>
  32 #include <linux/pci.h>
  33 #include <linux/etherdevice.h>
  34 #include <linux/skbuff.h>
  35 #include <linux/crc32.h>
  36 #include <linux/ethtool.h>
  37 #include <linux/mii.h>
  38 #include <linux/bitops.h>
  39 #include <linux/workqueue.h>
  40 #include <linux/of.h>
  41
  42 #include <asm/processor.h>
  43 #include <asm/io.h>
  44 #include <asm/dma.h>
  45 #include <asm/uaccess.h>
  46 #include <asm/dcr.h>
  47 #include <asm/dcr-regs.h>
  48
  49 #include "core.h"
  50
  51 /*
  52  * Lack of dma_unmap_???? calls is intentional.
  53  *
  54  * API-correct usage requires additional support state information to be
  55  * maintained for every RX and TX buffer descriptor (BD). Unfortunately, due to
  56  * EMAC design (e.g. TX buffer passed from network stack can be split into
  57  * several BDs, dma_map_single/dma_map_page can be used to map particular BD),
  58  * maintaining such information will add additional overhead.
  59  * Current DMA API implementation for 4xx processors only ensures cache coherency
  60  * and dma_unmap_???? routines are empty and are likely to stay this way.
  61  * I decided to omit dma_unmap_??? calls because I don't want to add additional
  62  * complexity just for the sake of following some abstract API, when it doesn't
  63  * add any real benefit to the driver. I understand that this decision maybe
  64  * controversial, but I really tried to make code API-correct and efficient
  65  * at the same time and didn't come up with code I liked :(.                --ebs
  66  */
  67
  68 #define DRV_NAME        "emac"
  69 #define DRV_VERSION     "3.54"
  70 #define DRV_DESC        "PPC 4xx OCP EMAC driver"
  71
  72 MODULE_DESCRIPTION(DRV_DESC);
  73 MODULE_AUTHOR
  74     ("Eugene Surovegin <eugene.surovegin@zultys.com> or <ebs@ebshome.net>");
  75 MODULE_LICENSE("GPL");
  76
  77 /*
  78  * PPC64 doesn't (yet) have a cacheable_memcpy
  79  */
  80 #ifdef CONFIG_PPC64
  81 #define cacheable_memcpy(d,s,n) memcpy((d),(s),(n))
  82 #endif
  83
  84 /* minimum number of free TX descriptors required to wake up TX process */
  85 #define EMAC_TX_WAKEUP_THRESH           (NUM_TX_BUFF / 4)
  86
  87 /* If packet size is less than this number, we allocate small skb and copy packet
  88  * contents into it instead of just sending original big skb up
  89  */
  90 #define EMAC_RX_COPY_THRESH             CONFIG_IBM_NEW_EMAC_RX_COPY_THRESHOLD
  91
  92 /* Since multiple EMACs share MDIO lines in various ways, we need
  93  * to avoid re-using the same PHY ID in cases where the arch didn't
  94  * setup precise phy_map entries
  95  *
  96  * XXX This is something that needs to be reworked as we can have multiple
  97  * EMAC "sets" (multiple ASICs containing several EMACs) though we can
  98  * probably require in that case to have explicit PHY IDs in the device-tree
  99  */
 100 static u32 busy_phy_map;
 101 static DEFINE_MUTEX(emac_phy_map_lock);
 102
 103 /* This is the wait queue used to wait on any event related to probe, that
 104  * is discovery of MALs, other EMACs, ZMII/RGMIIs, etc...
 105  */
 106 static DECLARE_WAIT_QUEUE_HEAD(emac_probe_wait);
 107
 108 /* Having stable interface names is a doomed idea. However, it would be nice
 109  * if we didn't have completely random interface names at boot too :-) It's
 110  * just a matter of making everybody's life easier. Since we are doing
 111  * threaded probing, it's a bit harder though. The base idea here is that
 112  * we make up a list of all emacs in the device-tree before we register the
 113  * driver. Every emac will then wait for the previous one in the list to
 114  * initialize before itself. We should also keep that list ordered by
 115  * cell_index.
 116  * That list is only 4 entries long, meaning that additional EMACs don't
 117  * get ordering guarantees unless EMAC_BOOT_LIST_SIZE is increased.
 118  */
 119
 120 #define EMAC_BOOT_LIST_SIZE     4
 121 static struct device_node *emac_boot_list[EMAC_BOOT_LIST_SIZE];
 122
 123 /* How long should I wait for dependent devices ? */
 124 #define EMAC_PROBE_DEP_TIMEOUT  (HZ * 5)
 125
 126 /* I don't want to litter system log with timeout errors
 127  * when we have brain-damaged PHY.
 128  */
 129 static inline void emac_report_timeout_error(struct emac_instance *dev,
 130                                              const char *error)
 131 {
 132         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX |
 133                                   EMAC_FTR_440EP_PHY_CLK_FIX))
 134                 DBG(dev, "%s" NL, error);
 135         else if (net_ratelimit())
 136                 printk(KERN_ERR "%s: %s\n", dev->ndev->name, error);
 137 }
 138
 139 /* EMAC PHY clock workaround:
 140  * 440EP/440GR has more sane SDR0_MFR register implementation than 440GX,
 141  * which allows controlling each EMAC clock
 142  */
 143 static inline void emac_rx_clk_tx(struct emac_instance *dev)
 144 {
 145 #ifdef CONFIG_PPC_DCR_NATIVE
 146         if (emac_has_feature(dev, EMAC_FTR_440EP_PHY_CLK_FIX))
 147                 dcri_clrset(SDR0, SDR0_MFR,
 148                             0, SDR0_MFR_ECS >> dev->cell_index);
 149 #endif
 150 }
 151
 152 static inline void emac_rx_clk_default(struct emac_instance *dev)
 153 {
 154 #ifdef CONFIG_PPC_DCR_NATIVE
 155         if (emac_has_feature(dev, EMAC_FTR_440EP_PHY_CLK_FIX))
 156                 dcri_clrset(SDR0, SDR0_MFR,
 157                             SDR0_MFR_ECS >> dev->cell_index, 0);
 158 #endif
 159 }
 160
 161 /* PHY polling intervals */
 162 #define PHY_POLL_LINK_ON        HZ
 163 #define PHY_POLL_LINK_OFF       (HZ / 5)
 164
 165 /* Graceful stop timeouts in us.
 166  * We should allow up to 1 frame time (full-duplex, ignoring collisions)
 167  */
 168 #define STOP_TIMEOUT_10         1230
 169 #define STOP_TIMEOUT_100        124
 170 #define STOP_TIMEOUT_1000       13
 171 #define STOP_TIMEOUT_1000_JUMBO 73
 172
 173 static unsigned char default_mcast_addr[] = {
 174         0x01, 0x80, 0xC2, 0x00, 0x00, 0x01
 175 };
 176
 177 /* Please, keep in sync with struct ibm_emac_stats/ibm_emac_error_stats */
 178 static const char emac_stats_keys[EMAC_ETHTOOL_STATS_COUNT][ETH_GSTRING_LEN] = {
 179         "rx_packets", "rx_bytes", "tx_packets", "tx_bytes", "rx_packets_csum",
 180         "tx_packets_csum", "tx_undo", "rx_dropped_stack", "rx_dropped_oom",
 181         "rx_dropped_error", "rx_dropped_resize", "rx_dropped_mtu",
 182         "rx_stopped", "rx_bd_errors", "rx_bd_overrun", "rx_bd_bad_packet",
 183         "rx_bd_runt_packet", "rx_bd_short_event", "rx_bd_alignment_error",
 184         "rx_bd_bad_fcs", "rx_bd_packet_too_long", "rx_bd_out_of_range",
 185         "rx_bd_in_range", "rx_parity", "rx_fifo_overrun", "rx_overrun",
 186         "rx_bad_packet", "rx_runt_packet", "rx_short_event",
 187         "rx_alignment_error", "rx_bad_fcs", "rx_packet_too_long",
 188         "rx_out_of_range", "rx_in_range", "tx_dropped", "tx_bd_errors",
 189         "tx_bd_bad_fcs", "tx_bd_carrier_loss", "tx_bd_excessive_deferral",
 190         "tx_bd_excessive_collisions", "tx_bd_late_collision",
 191         "tx_bd_multple_collisions", "tx_bd_single_collision",
 192         "tx_bd_underrun", "tx_bd_sqe", "tx_parity", "tx_underrun", "tx_sqe",
 193         "tx_errors"
 194 };
 195
 196 static irqreturn_t emac_irq(int irq, void *dev_instance);
 197 static void emac_clean_tx_ring(struct emac_instance *dev);
 198 static void __emac_set_multicast_list(struct emac_instance *dev);
 199
 200 static inline int emac_phy_supports_gige(int phy_mode)
 201 {
 202         return  phy_mode == PHY_MODE_GMII ||
 203                 phy_mode == PHY_MODE_RGMII ||
 204                 phy_mode == PHY_MODE_TBI ||
 205                 phy_mode == PHY_MODE_RTBI;
 206 }
 207
 208 static inline int emac_phy_gpcs(int phy_mode)
 209 {
 210         return  phy_mode == PHY_MODE_TBI ||
 211                 phy_mode == PHY_MODE_RTBI;
 212 }
 213
 214 static inline void emac_tx_enable(struct emac_instance *dev)
 215 {
 216         struct emac_regs __iomem *p = dev->emacp;
 217         u32 r;
 218
 219         DBG(dev, "tx_enable" NL);
 220
 221         r = in_be32(&p->mr0);
 222         if (!(r & EMAC_MR0_TXE))
 223                 out_be32(&p->mr0, r | EMAC_MR0_TXE);
 224 }
 225
 226 static void emac_tx_disable(struct emac_instance *dev)
 227 {
 228         struct emac_regs __iomem *p = dev->emacp;
 229         u32 r;
 230
 231         DBG(dev, "tx_disable" NL);
 232
 233         r = in_be32(&p->mr0);
 234         if (r & EMAC_MR0_TXE) {
 235                 int n = dev->stop_timeout;
 236                 out_be32(&p->mr0, r & ~EMAC_MR0_TXE);
 237                 while (!(in_be32(&p->mr0) & EMAC_MR0_TXI) && n) {
 238                         udelay(1);
 239                         --n;
 240                 }
 241                 if (unlikely(!n))
 242                         emac_report_timeout_error(dev, "TX disable timeout");
 243         }
 244 }
 245
 246 static void emac_rx_enable(struct emac_instance *dev)
 247 {
 248         struct emac_regs __iomem *p = dev->emacp;
 249         u32 r;
 250
 251         if (unlikely(test_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags)))
 252                 goto out;
 253
 254         DBG(dev, "rx_enable" NL);
 255
 256         r = in_be32(&p->mr0);
 257         if (!(r & EMAC_MR0_RXE)) {
 258                 if (unlikely(!(r & EMAC_MR0_RXI))) {
 259                         /* Wait if previous async disable is still in progress */
 260                         int n = dev->stop_timeout;
 261                         while (!(r = in_be32(&p->mr0) & EMAC_MR0_RXI) && n) {
 262                                 udelay(1);
 263                                 --n;
 264                         }
 265                         if (unlikely(!n))
 266                                 emac_report_timeout_error(dev,
 267                                                           "RX disable timeout");
 268                 }
 269                 out_be32(&p->mr0, r | EMAC_MR0_RXE);
 270         }
 271  out:
 272         ;
 273 }
 274
 275 static void emac_rx_disable(struct emac_instance *dev)
 276 {
 277         struct emac_regs __iomem *p = dev->emacp;
 278         u32 r;
 279
 280         DBG(dev, "rx_disable" NL);
 281
 282         r = in_be32(&p->mr0);
 283         if (r & EMAC_MR0_RXE) {
 284                 int n = dev->stop_timeout;
 285                 out_be32(&p->mr0, r & ~EMAC_MR0_RXE);
 286                 while (!(in_be32(&p->mr0) & EMAC_MR0_RXI) && n) {
 287                         udelay(1);
 288                         --n;
 289                 }
 290                 if (unlikely(!n))
 291                         emac_report_timeout_error(dev, "RX disable timeout");
 292         }
 293 }
 294
 295 static inline void emac_netif_stop(struct emac_instance *dev)
 296 {
 297         netif_tx_lock_bh(dev->ndev);
 298         dev->no_mcast = 1;
 299         netif_tx_unlock_bh(dev->ndev);
 300         dev->ndev->trans_start = jiffies;       /* prevent tx timeout */
 301         mal_poll_disable(dev->mal, &dev->commac);
 302         netif_tx_disable(dev->ndev);
 303 }
 304
 305 static inline void emac_netif_start(struct emac_instance *dev)
 306 {
 307         netif_tx_lock_bh(dev->ndev);
 308         dev->no_mcast = 0;
 309         if (dev->mcast_pending && netif_running(dev->ndev))
 310                 __emac_set_multicast_list(dev);
 311         netif_tx_unlock_bh(dev->ndev);
 312
 313         netif_wake_queue(dev->ndev);
 314
 315         /* NOTE: unconditional netif_wake_queue is only appropriate
 316          * so long as all callers are assured to have free tx slots
 317          * (taken from tg3... though the case where that is wrong is
 318          *  not terribly harmful)
 319          */
 320         mal_poll_enable(dev->mal, &dev->commac);
 321 }
 322
 323 static inline void emac_rx_disable_async(struct emac_instance *dev)
 324 {
 325         struct emac_regs __iomem *p = dev->emacp;
 326         u32 r;
 327
 328         DBG(dev, "rx_disable_async" NL);
 329
 330         r = in_be32(&p->mr0);
 331         if (r & EMAC_MR0_RXE)
 332                 out_be32(&p->mr0, r & ~EMAC_MR0_RXE);
 333 }
 334
 335 static int emac_reset(struct emac_instance *dev)
 336 {
 337         struct emac_regs __iomem *p = dev->emacp;
 338         int n = 20;
 339
 340         DBG(dev, "reset" NL);
 341
 342         if (!dev->reset_failed) {
 343                 /* 40x erratum suggests stopping RX channel before reset,
 344                  * we stop TX as well
 345                  */
 346                 emac_rx_disable(dev);
 347                 emac_tx_disable(dev);
 348         }
 349
 350         out_be32(&p->mr0, EMAC_MR0_SRST);
 351         while ((in_be32(&p->mr0) & EMAC_MR0_SRST) && n)
 352                 --n;
 353
 354         if (n) {
 355                 dev->reset_failed = 0;
 356                 return 0;
 357         } else {
 358                 emac_report_timeout_error(dev, "reset timeout");
 359                 dev->reset_failed = 1;
 360                 return -ETIMEDOUT;
 361         }
 362 }
 363
 364 static void emac_hash_mc(struct emac_instance *dev)
 365 {
 366         struct emac_regs __iomem *p = dev->emacp;
 367         u16 gaht[4] = { 0 };
 368         struct dev_mc_list *dmi;
 369
 370         DBG(dev, "hash_mc %d" NL, dev->ndev->mc_count);
 371
 372         for (dmi = dev->ndev->mc_list; dmi; dmi = dmi->next) {
 373                 int bit;
 374                 DBG2(dev, "mc %02x:%02x:%02x:%02x:%02x:%02x" NL,
 375                      dmi->dmi_addr[0], dmi->dmi_addr[1], dmi->dmi_addr[2],
 376                      dmi->dmi_addr[3], dmi->dmi_addr[4], dmi->dmi_addr[5]);
 377
 378                 bit = 63 - (ether_crc(ETH_ALEN, dmi->dmi_addr) >> 26);
 379                 gaht[bit >> 4] |= 0x8000 >> (bit & 0x0f);
 380         }
 381         out_be32(&p->gaht1, gaht[0]);
 382         out_be32(&p->gaht2, gaht[1]);
 383         out_be32(&p->gaht3, gaht[2]);
 384         out_be32(&p->gaht4, gaht[3]);
 385 }
 386
 387 static inline u32 emac_iff2rmr(struct net_device *ndev)
 388 {
 389         struct emac_instance *dev = netdev_priv(ndev);
 390         u32 r;
 391
 392         r = EMAC_RMR_SP | EMAC_RMR_SFCS | EMAC_RMR_IAE | EMAC_RMR_BAE;
 393
 394         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 395             r |= EMAC4_RMR_BASE;
 396         else
 397             r |= EMAC_RMR_BASE;
 398
 399         if (ndev->flags & IFF_PROMISC)
 400                 r |= EMAC_RMR_PME;
 401         else if (ndev->flags & IFF_ALLMULTI || ndev->mc_count > 32)
 402                 r |= EMAC_RMR_PMME;
 403         else if (ndev->mc_count > 0)
 404                 r |= EMAC_RMR_MAE;
 405
 406         return r;
 407 }
 408
 409 static u32 __emac_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 410 {
 411         u32 ret = EMAC_MR1_VLE | EMAC_MR1_IST | EMAC_MR1_TR0_MULT;
 412
 413         DBG2(dev, "__emac_calc_base_mr1" NL);
 414
 415         switch(tx_size) {
 416         case 2048:
 417                 ret |= EMAC_MR1_TFS_2K;
 418                 break;
 419         default:
 420                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 421                        dev->ndev->name, tx_size);
 422         }
 423
 424         switch(rx_size) {
 425         case 16384:
 426                 ret |= EMAC_MR1_RFS_16K;
 427                 break;
 428         case 4096:
 429                 ret |= EMAC_MR1_RFS_4K;
 430                 break;
 431         default:
 432                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 433                        dev->ndev->name, rx_size);
 434         }
 435
 436         return ret;
 437 }
 438
 439 static u32 __emac4_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 440 {
 441         u32 ret = EMAC_MR1_VLE | EMAC_MR1_IST | EMAC4_MR1_TR |
 442                 EMAC4_MR1_OBCI(dev->opb_bus_freq / 1000000);
 443
 444         DBG2(dev, "__emac4_calc_base_mr1" NL);
 445
 446         switch(tx_size) {
 447         case 4096:
 448                 ret |= EMAC4_MR1_TFS_4K;
 449                 break;
 450         case 2048:
 451                 ret |= EMAC4_MR1_TFS_2K;
 452                 break;
 453         default:
 454                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 455                        dev->ndev->name, tx_size);
 456         }
 457
 458         switch(rx_size) {
 459         case 16384:
 460                 ret |= EMAC4_MR1_RFS_16K;
 461                 break;
 462         case 4096:
 463                 ret |= EMAC4_MR1_RFS_4K;
 464                 break;
 465         case 2048:
 466                 ret |= EMAC4_MR1_RFS_2K;
 467                 break;
 468         default:
 469                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 470                        dev->ndev->name, rx_size);
 471         }
 472
 473         return ret;
 474 }
 475
 476 static u32 emac_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 477 {
 478         return emac_has_feature(dev, EMAC_FTR_EMAC4) ?
 479                 __emac4_calc_base_mr1(dev, tx_size, rx_size) :
 480                 __emac_calc_base_mr1(dev, tx_size, rx_size);
 481 }
 482
 483 static inline u32 emac_calc_trtr(struct emac_instance *dev, unsigned int size)
 484 {
 485         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 486                 return ((size >> 6) - 1) << EMAC_TRTR_SHIFT_EMAC4;
 487         else
 488                 return ((size >> 6) - 1) << EMAC_TRTR_SHIFT;
 489 }
 490
 491 static inline u32 emac_calc_rwmr(struct emac_instance *dev,
 492                                  unsigned int low, unsigned int high)
 493 {
 494         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 495                 return (low << 22) | ( (high & 0x3ff) << 6);
 496         else
 497                 return (low << 23) | ( (high & 0x1ff) << 7);
 498 }
 499
 500 static int emac_configure(struct emac_instance *dev)
 501 {
 502         struct emac_regs __iomem *p = dev->emacp;
 503         struct net_device *ndev = dev->ndev;
 504         int tx_size, rx_size, link = netif_carrier_ok(dev->ndev);
 505         u32 r, mr1 = 0;
 506
 507         DBG(dev, "configure" NL);
 508
 509         if (!link) {
 510                 out_be32(&p->mr1, in_be32(&p->mr1)
 511                          | EMAC_MR1_FDE | EMAC_MR1_ILE);
 512                 udelay(100);
 513         } else if (emac_reset(dev) < 0)
 514                 return -ETIMEDOUT;
 515
 516         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
 517                 tah_reset(dev->tah_dev);
 518
 519         DBG(dev, " link = %d duplex = %d, pause = %d, asym_pause = %d\n",
 520             link, dev->phy.duplex, dev->phy.pause, dev->phy.asym_pause);
 521
 522         /* Default fifo sizes */
 523         tx_size = dev->tx_fifo_size;
 524         rx_size = dev->rx_fifo_size;
 525
 526         /* No link, force loopback */
 527         if (!link)
 528                 mr1 = EMAC_MR1_FDE | EMAC_MR1_ILE;
 529
 530         /* Check for full duplex */
 531         else if (dev->phy.duplex == DUPLEX_FULL)
 532                 mr1 |= EMAC_MR1_FDE | EMAC_MR1_MWSW_001;
 533
 534         /* Adjust fifo sizes, mr1 and timeouts based on link speed */
 535         dev->stop_timeout = STOP_TIMEOUT_10;
 536         switch (dev->phy.speed) {
 537         case SPEED_1000:
 538                 if (emac_phy_gpcs(dev->phy.mode)) {
 539                         mr1 |= EMAC_MR1_MF_1000GPCS |
 540                                 EMAC_MR1_MF_IPPA(dev->phy.address);
 541
 542                         /* Put some arbitrary OUI, Manuf & Rev IDs so we can
 543                          * identify this GPCS PHY later.
 544                          */
 545                         out_be32(&p->ipcr, 0xdeadbeef);
 546                 } else
 547                         mr1 |= EMAC_MR1_MF_1000;
 548
 549                 /* Extended fifo sizes */
 550                 tx_size = dev->tx_fifo_size_gige;
 551                 rx_size = dev->rx_fifo_size_gige;
 552
 553                 if (dev->ndev->mtu > ETH_DATA_LEN) {
 554                         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 555                                 mr1 |= EMAC4_MR1_JPSM;
 556                         else
 557                                 mr1 |= EMAC_MR1_JPSM;
 558                         dev->stop_timeout = STOP_TIMEOUT_1000_JUMBO;
 559                 } else
 560                         dev->stop_timeout = STOP_TIMEOUT_1000;
 561                 break;
 562         case SPEED_100:
 563                 mr1 |= EMAC_MR1_MF_100;
 564                 dev->stop_timeout = STOP_TIMEOUT_100;
 565                 break;
 566         default: /* make gcc happy */
 567                 break;
 568         }
 569
 570         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 571                 rgmii_set_speed(dev->rgmii_dev, dev->rgmii_port,
 572                                 dev->phy.speed);
 573         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 574                 zmii_set_speed(dev->zmii_dev, dev->zmii_port, dev->phy.speed);
 575
 576         /* on 40x erratum forces us to NOT use integrated flow control,
 577          * let's hope it works on 44x ;)
 578          */
 579         if (!emac_has_feature(dev, EMAC_FTR_NO_FLOW_CONTROL_40x) &&
 580             dev->phy.duplex == DUPLEX_FULL) {
 581                 if (dev->phy.pause)
 582                         mr1 |= EMAC_MR1_EIFC | EMAC_MR1_APP;
 583                 else if (dev->phy.asym_pause)
 584                         mr1 |= EMAC_MR1_APP;
 585         }
 586
 587         /* Add base settings & fifo sizes & program MR1 */
 588         mr1 |= emac_calc_base_mr1(dev, tx_size, rx_size);
 589         out_be32(&p->mr1, mr1);
 590
 591         /* Set individual MAC address */
 592         out_be32(&p->iahr, (ndev->dev_addr[0] << 8) | ndev->dev_addr[1]);
 593         out_be32(&p->ialr, (ndev->dev_addr[2] << 24) |
 594                  (ndev->dev_addr[3] << 16) | (ndev->dev_addr[4] << 8) |
 595                  ndev->dev_addr[5]);
 596
 597         /* VLAN Tag Protocol ID */
 598         out_be32(&p->vtpid, 0x8100);
 599
 600         /* Receive mode register */
 601         r = emac_iff2rmr(ndev);
 602         if (r & EMAC_RMR_MAE)
 603                 emac_hash_mc(dev);
 604         out_be32(&p->rmr, r);
 605
 606         /* FIFOs thresholds */
 607         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 608                 r = EMAC4_TMR1((dev->mal_burst_size / dev->fifo_entry_size) + 1,
 609                                tx_size / 2 / dev->fifo_entry_size);
 610         else
 611                 r = EMAC_TMR1((dev->mal_burst_size / dev->fifo_entry_size) + 1,
 612                               tx_size / 2 / dev->fifo_entry_size);
 613         out_be32(&p->tmr1, r);
 614         out_be32(&p->trtr, emac_calc_trtr(dev, tx_size / 2));
 615
 616         /* PAUSE frame is sent when RX FIFO reaches its high-water mark,
 617            there should be still enough space in FIFO to allow the our link
 618            partner time to process this frame and also time to send PAUSE
 619            frame itself.
 620
 621            Here is the worst case scenario for the RX FIFO "headroom"
 622            (from "The Switch Book") (100Mbps, without preamble, inter-frame gap):
 623
 624            1) One maximum-length frame on TX                    1522 bytes
 625            2) One PAUSE frame time                                64 bytes
 626            3) PAUSE frame decode time allowance                   64 bytes
 627            4) One maximum-length frame on RX                    1522 bytes
 628            5) Round-trip propagation delay of the link (100Mb)    15 bytes
 629            ----------
 630            3187 bytes
 631
 632            I chose to set high-water mark to RX_FIFO_SIZE / 4 (1024 bytes)
 633            low-water mark  to RX_FIFO_SIZE / 8 (512 bytes)
 634          */
 635         r = emac_calc_rwmr(dev, rx_size / 8 / dev->fifo_entry_size,
 636                            rx_size / 4 / dev->fifo_entry_size);
 637         out_be32(&p->rwmr, r);
 638
 639         /* Set PAUSE timer to the maximum */
 640         out_be32(&p->ptr, 0xffff);
 641
 642         /* IRQ sources */
 643         r = EMAC_ISR_OVR | EMAC_ISR_BP | EMAC_ISR_SE |
 644                 EMAC_ISR_ALE | EMAC_ISR_BFCS | EMAC_ISR_PTLE | EMAC_ISR_ORE |
 645                 EMAC_ISR_IRE | EMAC_ISR_TE;
 646         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 647             r |= EMAC4_ISR_TXPE | EMAC4_ISR_RXPE /* | EMAC4_ISR_TXUE |
 648                                                   EMAC4_ISR_RXOE | */;
 649         out_be32(&p->iser,  r);
 650
 651         /* We need to take GPCS PHY out of isolate mode after EMAC reset */
 652         if (emac_phy_gpcs(dev->phy.mode))
 653                 emac_mii_reset_phy(&dev->phy);
 654
 655         /* Required for Pause packet support in EMAC */
 656         dev_mc_add(ndev, default_mcast_addr, sizeof(default_mcast_addr), 1);
 657
 658         return 0;
 659 }
 660
 661 static void emac_reinitialize(struct emac_instance *dev)
 662 {
 663         DBG(dev, "reinitialize" NL);
 664
 665         emac_netif_stop(dev);
 666         if (!emac_configure(dev)) {
 667                 emac_tx_enable(dev);
 668                 emac_rx_enable(dev);
 669         }
 670         emac_netif_start(dev);
 671 }
 672
 673 static void emac_full_tx_reset(struct emac_instance *dev)
 674 {
 675         DBG(dev, "full_tx_reset" NL);
 676
 677         emac_tx_disable(dev);
 678         mal_disable_tx_channel(dev->mal, dev->mal_tx_chan);
 679         emac_clean_tx_ring(dev);
 680         dev->tx_cnt = dev->tx_slot = dev->ack_slot = 0;
 681
 682         emac_configure(dev);
 683
 684         mal_enable_tx_channel(dev->mal, dev->mal_tx_chan);
 685         emac_tx_enable(dev);
 686         emac_rx_enable(dev);
 687 }
 688
 689 static void emac_reset_work(struct work_struct *work)
 690 {
 691         struct emac_instance *dev = container_of(work, struct emac_instance, reset_work);
 692
 693         DBG(dev, "reset_work" NL);
 694
 695         mutex_lock(&dev->link_lock);
 696         if (dev->opened) {
 697                 emac_netif_stop(dev);
 698                 emac_full_tx_reset(dev);
 699                 emac_netif_start(dev);
 700         }
 701         mutex_unlock(&dev->link_lock);
 702 }
 703
 704 static void emac_tx_timeout(struct net_device *ndev)
 705 {
 706         struct emac_instance *dev = netdev_priv(ndev);
 707
 708         DBG(dev, "tx_timeout" NL);
 709
 710         schedule_work(&dev->reset_work);
 711 }
 712
 713
 714 static inline int emac_phy_done(struct emac_instance *dev, u32 stacr)
 715 {
 716         int done = !!(stacr & EMAC_STACR_OC);
 717
 718         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 719                 done = !done;
 720
 721         return done;
 722 };
 723
 724 static int __emac_mdio_read(struct emac_instance *dev, u8 id, u8 reg)
 725 {
 726         struct emac_regs __iomem *p = dev->emacp;
 727         u32 r = 0;
 728         int n, err = -ETIMEDOUT;
 729
 730         mutex_lock(&dev->mdio_lock);
 731
 732         DBG2(dev, "mdio_read(%02x,%02x)" NL, id, reg);
 733
 734         /* Enable proper MDIO port */
 735         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 736                 zmii_get_mdio(dev->zmii_dev, dev->zmii_port);
 737         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 738                 rgmii_get_mdio(dev->rgmii_dev, dev->rgmii_port);
 739
 740         /* Wait for management interface to become idle */
 741         n = 10;
 742         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 743                 udelay(1);
 744                 if (!--n) {
 745                         DBG2(dev, " -> timeout wait idle\n");
 746                         goto bail;
 747                 }
 748         }
 749
 750         /* Issue read command */
 751         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 752                 r = EMAC4_STACR_BASE(dev->opb_bus_freq);
 753         else
 754                 r = EMAC_STACR_BASE(dev->opb_bus_freq);
 755         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 756                 r |= EMAC_STACR_OC;
 757         if (emac_has_feature(dev, EMAC_FTR_HAS_NEW_STACR))
 758                 r |= EMACX_STACR_STAC_READ;
 759         else
 760                 r |= EMAC_STACR_STAC_READ;
 761         r |= (reg & EMAC_STACR_PRA_MASK)
 762                 | ((id & EMAC_STACR_PCDA_MASK) << EMAC_STACR_PCDA_SHIFT);
 763         out_be32(&p->stacr, r);
 764
 765         /* Wait for read to complete */
 766         n = 100;
 767         while (!emac_phy_done(dev, (r = in_be32(&p->stacr)))) {
 768                 udelay(1);
 769                 if (!--n) {
 770                         DBG2(dev, " -> timeout wait complete\n");
 771                         goto bail;
 772                 }
 773         }
 774
 775         if (unlikely(r & EMAC_STACR_PHYE)) {
 776                 DBG(dev, "mdio_read(%02x, %02x) failed" NL, id, reg);
 777                 err = -EREMOTEIO;
 778                 goto bail;
 779         }
 780
 781         r = ((r >> EMAC_STACR_PHYD_SHIFT) & EMAC_STACR_PHYD_MASK);
 782
 783         DBG2(dev, "mdio_read -> %04x" NL, r);
 784         err = 0;
 785  bail:
 786         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 787                 rgmii_put_mdio(dev->rgmii_dev, dev->rgmii_port);
 788         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 789                 zmii_put_mdio(dev->zmii_dev, dev->zmii_port);
 790         mutex_unlock(&dev->mdio_lock);
 791
 792         return err == 0 ? r : err;
 793 }
 794
 795 static void __emac_mdio_write(struct emac_instance *dev, u8 id, u8 reg,
 796                               u16 val)
 797 {
 798         struct emac_regs __iomem *p = dev->emacp;
 799         u32 r = 0;
 800         int n, err = -ETIMEDOUT;
 801
 802         mutex_lock(&dev->mdio_lock);
 803
 804         DBG2(dev, "mdio_write(%02x,%02x,%04x)" NL, id, reg, val);
 805
 806         /* Enable proper MDIO port */
 807         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 808                 zmii_get_mdio(dev->zmii_dev, dev->zmii_port);
 809         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 810                 rgmii_get_mdio(dev->rgmii_dev, dev->rgmii_port);
 811
 812         /* Wait for management interface to be idle */
 813         n = 10;
 814         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 815                 udelay(1);
 816                 if (!--n) {
 817                         DBG2(dev, " -> timeout wait idle\n");
 818                         goto bail;
 819                 }
 820         }
 821
 822         /* Issue write command */
 823         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 824                 r = EMAC4_STACR_BASE(dev->opb_bus_freq);
 825         else
 826                 r = EMAC_STACR_BASE(dev->opb_bus_freq);
 827         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 828                 r |= EMAC_STACR_OC;
 829         if (emac_has_feature(dev, EMAC_FTR_HAS_NEW_STACR))
 830                 r |= EMACX_STACR_STAC_WRITE;
 831         else
 832                 r |= EMAC_STACR_STAC_WRITE;
 833         r |= (reg & EMAC_STACR_PRA_MASK) |
 834                 ((id & EMAC_STACR_PCDA_MASK) << EMAC_STACR_PCDA_SHIFT) |
 835                 (val << EMAC_STACR_PHYD_SHIFT);
 836         out_be32(&p->stacr, r);
 837
 838         /* Wait for write to complete */
 839         n = 100;
 840         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 841                 udelay(1);
 842                 if (!--n) {
 843                         DBG2(dev, " -> timeout wait complete\n");
 844                         goto bail;
 845                 }
 846         }
 847         err = 0;
 848  bail:
 849         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 850                 rgmii_put_mdio(dev->rgmii_dev, dev->rgmii_port);
 851         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 852                 zmii_put_mdio(dev->zmii_dev, dev->zmii_port);
 853         mutex_unlock(&dev->mdio_lock);
 854 }
 855
 856 static int emac_mdio_read(struct net_device *ndev, int id, int reg)
 857 {
 858         struct emac_instance *dev = netdev_priv(ndev);
 859         int res;
 860
 861         res = __emac_mdio_read(dev->mdio_instance ? dev->mdio_instance : dev,
 862                                (u8) id, (u8) reg);
 863         return res;
 864 }
 865
 866 static void emac_mdio_write(struct net_device *ndev, int id, int reg, int val)
 867 {
 868         struct emac_instance *dev = netdev_priv(ndev);
 869
 870         __emac_mdio_write(dev->mdio_instance ? dev->mdio_instance : dev,
 871                           (u8) id, (u8) reg, (u16) val);
 872 }
 873
 874 /* Tx lock BH */
 875 static void __emac_set_multicast_list(struct emac_instance *dev)
 876 {
 877         struct emac_regs __iomem *p = dev->emacp;
 878         u32 rmr = emac_iff2rmr(dev->ndev);
 879
 880         DBG(dev, "__multicast %08x" NL, rmr);
 881
 882         /* I decided to relax register access rules here to avoid
 883          * full EMAC reset.
 884          *
 885          * There is a real problem with EMAC4 core if we use MWSW_001 bit
 886          * in MR1 register and do a full EMAC reset.
 887          * One TX BD status update is delayed and, after EMAC reset, it
 888          * never happens, resulting in TX hung (it'll be recovered by TX
 889          * timeout handler eventually, but this is just gross).
 890          * So we either have to do full TX reset or try to cheat here :)
 891          *
 892          * The only required change is to RX mode register, so I *think* all
 893          * we need is just to stop RX channel. This seems to work on all
 894          * tested SoCs.                                                --ebs
 895          *
 896          * If we need the full reset, we might just trigger the workqueue
 897          * and do it async... a bit nasty but should work --BenH
 898          */
 899         dev->mcast_pending = 0;
 900         emac_rx_disable(dev);
 901         if (rmr & EMAC_RMR_MAE)
 902                 emac_hash_mc(dev);
 903         out_be32(&p->rmr, rmr);
 904         emac_rx_enable(dev);
 905 }
 906
 907 /* Tx lock BH */
 908 static void emac_set_multicast_list(struct net_device *ndev)
 909 {
 910         struct emac_instance *dev = netdev_priv(ndev);
 911
 912         DBG(dev, "multicast" NL);
 913
 914         BUG_ON(!netif_running(dev->ndev));
 915
 916         if (dev->no_mcast) {
 917                 dev->mcast_pending = 1;
 918                 return;
 919         }
 920         __emac_set_multicast_list(dev);
 921 }
 922
 923 static int emac_resize_rx_ring(struct emac_instance *dev, int new_mtu)
 924 {
 925         int rx_sync_size = emac_rx_sync_size(new_mtu);
 926         int rx_skb_size = emac_rx_skb_size(new_mtu);
 927         int i, ret = 0;
 928
 929         mutex_lock(&dev->link_lock);
 930         emac_netif_stop(dev);
 931         emac_rx_disable(dev);
 932         mal_disable_rx_channel(dev->mal, dev->mal_rx_chan);
 933
 934         if (dev->rx_sg_skb) {
 935                 ++dev->estats.rx_dropped_resize;
 936                 dev_kfree_skb(dev->rx_sg_skb);
 937                 dev->rx_sg_skb = NULL;
 938         }
 939
 940         /* Make a first pass over RX ring and mark BDs ready, dropping
 941          * non-processed packets on the way. We need this as a separate pass
 942          * to simplify error recovery in the case of allocation failure later.
 943          */
 944         for (i = 0; i < NUM_RX_BUFF; ++i) {
 945                 if (dev->rx_desc[i].ctrl & MAL_RX_CTRL_FIRST)
 946                         ++dev->estats.rx_dropped_resize;
 947
 948                 dev->rx_desc[i].data_len = 0;
 949                 dev->rx_desc[i].ctrl = MAL_RX_CTRL_EMPTY |
 950                     (i == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
 951         }
 952
 953         /* Reallocate RX ring only if bigger skb buffers are required */
 954         if (rx_skb_size <= dev->rx_skb_size)
 955                 goto skip;
 956
 957         /* Second pass, allocate new skbs */
 958         for (i = 0; i < NUM_RX_BUFF; ++i) {
 959                 struct sk_buff *skb = alloc_skb(rx_skb_size, GFP_ATOMIC);
 960                 if (!skb) {
 961                         ret = -ENOMEM;
 962                         goto oom;
 963                 }
 964
 965                 BUG_ON(!dev->rx_skb[i]);
 966                 dev_kfree_skb(dev->rx_skb[i]);
 967
 968                 skb_reserve(skb, EMAC_RX_SKB_HEADROOM + 2);
 969                 dev->rx_desc[i].data_ptr =
 970                     dma_map_single(&dev->ofdev->dev, skb->data - 2, rx_sync_size,
 971                                    DMA_FROM_DEVICE) + 2;
 972                 dev->rx_skb[i] = skb;
 973         }
 974  skip:
 975         /* Check if we need to change "Jumbo" bit in MR1 */
 976         if ((new_mtu > ETH_DATA_LEN) ^ (dev->ndev->mtu > ETH_DATA_LEN)) {
 977                 /* This is to prevent starting RX channel in emac_rx_enable() */
 978                 set_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
 979
 980                 dev->ndev->mtu = new_mtu;
 981                 emac_full_tx_reset(dev);
 982         }
 983
 984         mal_set_rcbs(dev->mal, dev->mal_rx_chan, emac_rx_size(new_mtu));
 985  oom:
 986         /* Restart RX */
 987         clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
 988         dev->rx_slot = 0;
 989         mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
 990         emac_rx_enable(dev);
 991         emac_netif_start(dev);
 992         mutex_unlock(&dev->link_lock);
 993
 994         return ret;
 995 }
 996
 997 /* Process ctx, rtnl_lock semaphore */
 998 static int emac_change_mtu(struct net_device *ndev, int new_mtu)
 999 {
1000         struct emac_instance *dev = netdev_priv(ndev);
1001         int ret = 0;
1002
1003         if (new_mtu < EMAC_MIN_MTU || new_mtu > dev->max_mtu)
1004                 return -EINVAL;
1005
1006         DBG(dev, "change_mtu(%d)" NL, new_mtu);
1007
1008         if (netif_running(ndev)) {
1009                 /* Check if we really need to reinitalize RX ring */
1010                 if (emac_rx_skb_size(ndev->mtu) != emac_rx_skb_size(new_mtu))
1011                         ret = emac_resize_rx_ring(dev, new_mtu);
1012         }
1013
1014         if (!ret) {
1015                 ndev->mtu = new_mtu;
1016                 dev->rx_skb_size = emac_rx_skb_size(new_mtu);
1017                 dev->rx_sync_size = emac_rx_sync_size(new_mtu);
1018         }
1019
1020         return ret;
1021 }
1022
1023 static void emac_clean_tx_ring(struct emac_instance *dev)
1024 {
1025         int i;
1026
1027         for (i = 0; i < NUM_TX_BUFF; ++i) {
1028                 if (dev->tx_skb[i]) {
1029                         dev_kfree_skb(dev->tx_skb[i]);
1030                         dev->tx_skb[i] = NULL;
1031                         if (dev->tx_desc[i].ctrl & MAL_TX_CTRL_READY)
1032                                 ++dev->estats.tx_dropped;
1033                 }
1034                 dev->tx_desc[i].ctrl = 0;
1035                 dev->tx_desc[i].data_ptr = 0;
1036         }
1037 }
1038
1039 static void emac_clean_rx_ring(struct emac_instance *dev)
1040 {
1041         int i;
1042
1043         for (i = 0; i < NUM_RX_BUFF; ++i)
1044                 if (dev->rx_skb[i]) {
1045                         dev->rx_desc[i].ctrl = 0;
1046                         dev_kfree_skb(dev->rx_skb[i]);
1047                         dev->rx_skb[i] = NULL;
1048                         dev->rx_desc[i].data_ptr = 0;
1049                 }
1050
1051         if (dev->rx_sg_skb) {
1052                 dev_kfree_skb(dev->rx_sg_skb);
1053                 dev->rx_sg_skb = NULL;
1054         }
1055 }
1056
1057 static inline int emac_alloc_rx_skb(struct emac_instance *dev, int slot,
1058                                     gfp_t flags)
1059 {
1060         struct sk_buff *skb = alloc_skb(dev->rx_skb_size, flags);
1061         if (unlikely(!skb))
1062                 return -ENOMEM;
1063
1064         dev->rx_skb[slot] = skb;
1065         dev->rx_desc[slot].data_len = 0;
1066
1067         skb_reserve(skb, EMAC_RX_SKB_HEADROOM + 2);
1068         dev->rx_desc[slot].data_ptr =
1069             dma_map_single(&dev->ofdev->dev, skb->data - 2, dev->rx_sync_size,
1070                            DMA_FROM_DEVICE) + 2;
1071         wmb();
1072         dev->rx_desc[slot].ctrl = MAL_RX_CTRL_EMPTY |
1073             (slot == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
1074
1075         return 0;
1076 }
1077
1078 static void emac_print_link_status(struct emac_instance *dev)
1079 {
1080         if (netif_carrier_ok(dev->ndev))
1081                 printk(KERN_INFO "%s: link is up, %d %s%s\n",
1082                        dev->ndev->name, dev->phy.speed,
1083                        dev->phy.duplex == DUPLEX_FULL ? "FDX" : "HDX",
1084                        dev->phy.pause ? ", pause enabled" :
1085                        dev->phy.asym_pause ? ", asymmetric pause enabled" : "");
1086         else
1087                 printk(KERN_INFO "%s: link is down\n", dev->ndev->name);
1088 }
1089
1090 /* Process ctx, rtnl_lock semaphore */
1091 static int emac_open(struct net_device *ndev)
1092 {
1093         struct emac_instance *dev = netdev_priv(ndev);
1094         int err, i;
1095
1096         DBG(dev, "open" NL);
1097
1098         /* Setup error IRQ handler */
1099         err = request_irq(dev->emac_irq, emac_irq, 0, "EMAC", dev);
1100         if (err) {
1101                 printk(KERN_ERR "%s: failed to request IRQ %d\n",
1102                        ndev->name, dev->emac_irq);
1103                 return err;
1104         }
1105
1106         /* Allocate RX ring */
1107         for (i = 0; i < NUM_RX_BUFF; ++i)
1108                 if (emac_alloc_rx_skb(dev, i, GFP_KERNEL)) {
1109                         printk(KERN_ERR "%s: failed to allocate RX ring\n",
1110                                ndev->name);
1111                         goto oom;
1112                 }
1113
1114         dev->tx_cnt = dev->tx_slot = dev->ack_slot = dev->rx_slot = 0;
1115         clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1116         dev->rx_sg_skb = NULL;
1117
1118         mutex_lock(&dev->link_lock);
1119         dev->opened = 1;
1120
1121         /* Start PHY polling now.
1122          */
1123         if (dev->phy.address >= 0) {
1124                 int link_poll_interval;
1125                 if (dev->phy.def->ops->poll_link(&dev->phy)) {
1126                         dev->phy.def->ops->read_link(&dev->phy);
1127                         emac_rx_clk_default(dev);
1128                         netif_carrier_on(dev->ndev);
1129                         link_poll_interval = PHY_POLL_LINK_ON;
1130                 } else {
1131                         emac_rx_clk_tx(dev);
1132                         netif_carrier_off(dev->ndev);
1133                         link_poll_interval = PHY_POLL_LINK_OFF;
1134                 }
1135                 dev->link_polling = 1;
1136                 wmb();
1137                 schedule_delayed_work(&dev->link_work, link_poll_interval);
1138                 emac_print_link_status(dev);
1139         } else
1140                 netif_carrier_on(dev->ndev);
1141
1142         emac_configure(dev);
1143         mal_poll_add(dev->mal, &dev->commac);
1144         mal_enable_tx_channel(dev->mal, dev->mal_tx_chan);
1145         mal_set_rcbs(dev->mal, dev->mal_rx_chan, emac_rx_size(ndev->mtu));
1146         mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
1147         emac_tx_enable(dev);
1148         emac_rx_enable(dev);
1149         emac_netif_start(dev);
1150
1151         mutex_unlock(&dev->link_lock);
1152
1153         return 0;
1154  oom:
1155         emac_clean_rx_ring(dev);
1156         free_irq(dev->emac_irq, dev);
1157
1158         return -ENOMEM;
1159 }
1160
1161 /* BHs disabled */
1162 #if 0
1163 static int emac_link_differs(struct emac_instance *dev)
1164 {
1165         u32 r = in_be32(&dev->emacp->mr1);
1166
1167         int duplex = r & EMAC_MR1_FDE ? DUPLEX_FULL : DUPLEX_HALF;
1168         int speed, pause, asym_pause;
1169
1170         if (r & EMAC_MR1_MF_1000)
1171                 speed = SPEED_1000;
1172         else if (r & EMAC_MR1_MF_100)
1173                 speed = SPEED_100;
1174         else
1175                 speed = SPEED_10;
1176
1177         switch (r & (EMAC_MR1_EIFC | EMAC_MR1_APP)) {
1178         case (EMAC_MR1_EIFC | EMAC_MR1_APP):
1179                 pause = 1;
1180                 asym_pause = 0;
1181                 break;
1182         case EMAC_MR1_APP:
1183                 pause = 0;
1184                 asym_pause = 1;
1185                 break;
1186         default:
1187                 pause = asym_pause = 0;
1188         }
1189         return speed != dev->phy.speed || duplex != dev->phy.duplex ||
1190             pause != dev->phy.pause || asym_pause != dev->phy.asym_pause;
1191 }
1192 #endif
1193
1194 static void emac_link_timer(struct work_struct *work)
1195 {
1196         struct emac_instance *dev =
1197                 container_of((struct delayed_work *)work,
1198                              struct emac_instance, link_work);
1199         int link_poll_interval;
1200
1201         mutex_lock(&dev->link_lock);
1202         DBG2(dev, "link timer" NL);
1203
1204         if (!dev->opened)
1205                 goto bail;
1206
1207         if (dev->phy.def->ops->poll_link(&dev->phy)) {
1208                 if (!netif_carrier_ok(dev->ndev)) {
1209                         emac_rx_clk_default(dev);
1210                         /* Get new link parameters */
1211                         dev->phy.def->ops->read_link(&dev->phy);
1212
1213                         netif_carrier_on(dev->ndev);
1214                         emac_netif_stop(dev);
1215                         emac_full_tx_reset(dev);
1216                         emac_netif_start(dev);
1217                         emac_print_link_status(dev);
1218                 }
1219                 link_poll_interval = PHY_POLL_LINK_ON;
1220         } else {
1221                 if (netif_carrier_ok(dev->ndev)) {
1222                         emac_rx_clk_tx(dev);
1223                         netif_carrier_off(dev->ndev);
1224                         netif_tx_disable(dev->ndev);
1225                         emac_reinitialize(dev);
1226                         emac_print_link_status(dev);
1227                 }
1228                 link_poll_interval = PHY_POLL_LINK_OFF;
1229         }
1230         schedule_delayed_work(&dev->link_work, link_poll_interval);
1231  bail:
1232         mutex_unlock(&dev->link_lock);
1233 }
1234
1235 static void emac_force_link_update(struct emac_instance *dev)
1236 {
1237         netif_carrier_off(dev->ndev);
1238         smp_rmb();
1239         if (dev->link_polling) {
1240                 cancel_rearming_delayed_work(&dev->link_work);
1241                 if (dev->link_polling)
1242                         schedule_delayed_work(&dev->link_work,  PHY_POLL_LINK_OFF);
1243         }
1244 }
1245
1246 /* Process ctx, rtnl_lock semaphore */
1247 static int emac_close(struct net_device *ndev)
1248 {
1249         struct emac_instance *dev = netdev_priv(ndev);
1250
1251         DBG(dev, "close" NL);
1252
1253         if (dev->phy.address >= 0) {
1254                 dev->link_polling = 0;
1255                 cancel_rearming_delayed_work(&dev->link_work);
1256         }
1257         mutex_lock(&dev->link_lock);
1258         emac_netif_stop(dev);
1259         dev->opened = 0;
1260         mutex_unlock(&dev->link_lock);
1261
1262         emac_rx_disable(dev);
1263         emac_tx_disable(dev);
1264         mal_disable_rx_channel(dev->mal, dev->mal_rx_chan);
1265         mal_disable_tx_channel(dev->mal, dev->mal_tx_chan);
1266         mal_poll_del(dev->mal, &dev->commac);
1267
1268         emac_clean_tx_ring(dev);
1269         emac_clean_rx_ring(dev);
1270
1271         free_irq(dev->emac_irq, dev);
1272
1273         return 0;
1274 }
1275
1276 static inline u16 emac_tx_csum(struct emac_instance *dev,
1277                                struct sk_buff *skb)
1278 {
1279         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH) &&
1280                 (skb->ip_summed == CHECKSUM_PARTIAL)) {
1281                 ++dev->stats.tx_packets_csum;
1282                 return EMAC_TX_CTRL_TAH_CSUM;
1283         }
1284         return 0;
1285 }
1286
1287 static inline int emac_xmit_finish(struct emac_instance *dev, int len)
1288 {
1289         struct emac_regs __iomem *p = dev->emacp;
1290         struct net_device *ndev = dev->ndev;
1291
1292         /* Send the packet out. If the if makes a significant perf
1293          * difference, then we can store the TMR0 value in "dev"
1294          * instead
1295          */
1296         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
1297                 out_be32(&p->tmr0, EMAC4_TMR0_XMIT);
1298         else
1299                 out_be32(&p->tmr0, EMAC_TMR0_XMIT);
1300
1301         if (unlikely(++dev->tx_cnt == NUM_TX_BUFF)) {
1302                 netif_stop_queue(ndev);
1303                 DBG2(dev, "stopped TX queue" NL);
1304         }
1305
1306         ndev->trans_start = jiffies;
1307         ++dev->stats.tx_packets;
1308         dev->stats.tx_bytes += len;
1309
1310         return 0;
1311 }
1312
1313 /* Tx lock BH */
1314 static int emac_start_xmit(struct sk_buff *skb, struct net_device *ndev)
1315 {
1316         struct emac_instance *dev = netdev_priv(ndev);
1317         unsigned int len = skb->len;
1318         int slot;
1319
1320         u16 ctrl = EMAC_TX_CTRL_GFCS | EMAC_TX_CTRL_GP | MAL_TX_CTRL_READY |
1321             MAL_TX_CTRL_LAST | emac_tx_csum(dev, skb);
1322
1323         slot = dev->tx_slot++;
1324         if (dev->tx_slot == NUM_TX_BUFF) {
1325                 dev->tx_slot = 0;
1326                 ctrl |= MAL_TX_CTRL_WRAP;
1327         }
1328
1329         DBG2(dev, "xmit(%u) %d" NL, len, slot);
1330
1331         dev->tx_skb[slot] = skb;
1332         dev->tx_desc[slot].data_ptr = dma_map_single(&dev->ofdev->dev,
1333                                                      skb->data, len,
1334                                                      DMA_TO_DEVICE);
1335         dev->tx_desc[slot].data_len = (u16) len;
1336         wmb();
1337         dev->tx_desc[slot].ctrl = ctrl;
1338
1339         return emac_xmit_finish(dev, len);
1340 }
1341
1342 static inline int emac_xmit_split(struct emac_instance *dev, int slot,
1343                                   u32 pd, int len, int last, u16 base_ctrl)
1344 {
1345         while (1) {
1346                 u16 ctrl = base_ctrl;
1347                 int chunk = min(len, MAL_MAX_TX_SIZE);
1348                 len -= chunk;
1349
1350                 slot = (slot + 1) % NUM_TX_BUFF;
1351
1352                 if (last && !len)
1353                         ctrl |= MAL_TX_CTRL_LAST;
1354                 if (slot == NUM_TX_BUFF - 1)
1355                         ctrl |= MAL_TX_CTRL_WRAP;
1356
1357                 dev->tx_skb[slot] = NULL;
1358                 dev->tx_desc[slot].data_ptr = pd;
1359                 dev->tx_desc[slot].data_len = (u16) chunk;
1360                 dev->tx_desc[slot].ctrl = ctrl;
1361                 ++dev->tx_cnt;
1362
1363                 if (!len)
1364                         break;
1365
1366                 pd += chunk;
1367         }
1368         return slot;
1369 }
1370
1371 /* Tx lock BH disabled (SG version for TAH equipped EMACs) */
1372 static int emac_start_xmit_sg(struct sk_buff *skb, struct net_device *ndev)
1373 {
1374         struct emac_instance *dev = netdev_priv(ndev);
1375         int nr_frags = skb_shinfo(skb)->nr_frags;
1376         int len = skb->len, chunk;
1377         int slot, i;
1378         u16 ctrl;
1379         u32 pd;
1380
1381         /* This is common "fast" path */
1382         if (likely(!nr_frags && len <= MAL_MAX_TX_SIZE))
1383                 return emac_start_xmit(skb, ndev);
1384
1385         len -= skb->data_len;
1386
1387         /* Note, this is only an *estimation*, we can still run out of empty
1388          * slots because of the additional fragmentation into
1389          * MAL_MAX_TX_SIZE-sized chunks
1390          */
1391         if (unlikely(dev->tx_cnt + nr_frags + mal_tx_chunks(len) > NUM_TX_BUFF))
1392                 goto stop_queue;
1393
1394         ctrl = EMAC_TX_CTRL_GFCS | EMAC_TX_CTRL_GP | MAL_TX_CTRL_READY |
1395             emac_tx_csum(dev, skb);
1396         slot = dev->tx_slot;
1397
1398         /* skb data */
1399         dev->tx_skb[slot] = NULL;
1400         chunk = min(len, MAL_MAX_TX_SIZE);
1401         dev->tx_desc[slot].data_ptr = pd =
1402             dma_map_single(&dev->ofdev->dev, skb->data, len, DMA_TO_DEVICE);
1403         dev->tx_desc[slot].data_len = (u16) chunk;
1404         len -= chunk;
1405         if (unlikely(len))
1406                 slot = emac_xmit_split(dev, slot, pd + chunk, len, !nr_frags,
1407                                        ctrl);
1408         /* skb fragments */
1409         for (i = 0; i < nr_frags; ++i) {
1410                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
1411                 len = frag->size;
1412
1413                 if (unlikely(dev->tx_cnt + mal_tx_chunks(len) >= NUM_TX_BUFF))
1414                         goto undo_frame;
1415
1416                 pd = dma_map_page(&dev->ofdev->dev, frag->page, frag->page_offset, len,
1417                                   DMA_TO_DEVICE);
1418
1419                 slot = emac_xmit_split(dev, slot, pd, len, i == nr_frags - 1,
1420                                        ctrl);
1421         }
1422
1423         DBG2(dev, "xmit_sg(%u) %d - %d" NL, skb->len, dev->tx_slot, slot);
1424
1425         /* Attach skb to the last slot so we don't release it too early */
1426         dev->tx_skb[slot] = skb;
1427
1428         /* Send the packet out */
1429         if (dev->tx_slot == NUM_TX_BUFF - 1)
1430                 ctrl |= MAL_TX_CTRL_WRAP;
1431         wmb();
1432         dev->tx_desc[dev->tx_slot].ctrl = ctrl;
1433         dev->tx_slot = (slot + 1) % NUM_TX_BUFF;
1434
1435         return emac_xmit_finish(dev, skb->len);
1436
1437  undo_frame:
1438         /* Well, too bad. Our previous estimation was overly optimistic.
1439          * Undo everything.
1440          */
1441         while (slot != dev->tx_slot) {
1442                 dev->tx_desc[slot].ctrl = 0;
1443                 --dev->tx_cnt;
1444                 if (--slot < 0)
1445                         slot = NUM_TX_BUFF - 1;
1446         }
1447         ++dev->estats.tx_undo;
1448
1449  stop_queue:
1450         netif_stop_queue(ndev);
1451         DBG2(dev, "stopped TX queue" NL);
1452         return 1;
1453 }
1454
1455 /* Tx lock BHs */
1456 static void emac_parse_tx_error(struct emac_instance *dev, u16 ctrl)
1457 {
1458         struct emac_error_stats *st = &dev->estats;
1459
1460         DBG(dev, "BD TX error %04x" NL, ctrl);
1461
1462         ++st->tx_bd_errors;
1463         if (ctrl & EMAC_TX_ST_BFCS)
1464                 ++st->tx_bd_bad_fcs;
1465         if (ctrl & EMAC_TX_ST_LCS)
1466                 ++st->tx_bd_carrier_loss;
1467         if (ctrl & EMAC_TX_ST_ED)
1468                 ++st->tx_bd_excessive_deferral;
1469         if (ctrl & EMAC_TX_ST_EC)
1470                 ++st->tx_bd_excessive_collisions;
1471         if (ctrl & EMAC_TX_ST_LC)
1472                 ++st->tx_bd_late_collision;
1473         if (ctrl & EMAC_TX_ST_MC)
1474                 ++st->tx_bd_multple_collisions;
1475         if (ctrl & EMAC_TX_ST_SC)
1476                 ++st->tx_bd_single_collision;
1477         if (ctrl & EMAC_TX_ST_UR)
1478                 ++st->tx_bd_underrun;
1479         if (ctrl & EMAC_TX_ST_SQE)
1480                 ++st->tx_bd_sqe;
1481 }
1482
1483 static void emac_poll_tx(void *param)
1484 {
1485         struct emac_instance *dev = param;
1486         u32 bad_mask;
1487
1488         DBG2(dev, "poll_tx, %d %d" NL, dev->tx_cnt, dev->ack_slot);
1489
1490         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
1491                 bad_mask = EMAC_IS_BAD_TX_TAH;
1492         else
1493                 bad_mask = EMAC_IS_BAD_TX;
1494
1495         netif_tx_lock_bh(dev->ndev);
1496         if (dev->tx_cnt) {
1497                 u16 ctrl;
1498                 int slot = dev->ack_slot, n = 0;
1499         again:
1500                 ctrl = dev->tx_desc[slot].ctrl;
1501                 if (!(ctrl & MAL_TX_CTRL_READY)) {
1502                         struct sk_buff *skb = dev->tx_skb[slot];
1503                         ++n;
1504
1505                         if (skb) {
1506                                 dev_kfree_skb(skb);
1507                                 dev->tx_skb[slot] = NULL;
1508                         }
1509                         slot = (slot + 1) % NUM_TX_BUFF;
1510
1511                         if (unlikely(ctrl & bad_mask))
1512                                 emac_parse_tx_error(dev, ctrl);
1513
1514                         if (--dev->tx_cnt)
1515                                 goto again;
1516                 }
1517                 if (n) {
1518                         dev->ack_slot = slot;
1519                         if (netif_queue_stopped(dev->ndev) &&
1520                             dev->tx_cnt < EMAC_TX_WAKEUP_THRESH)
1521                                 netif_wake_queue(dev->ndev);
1522
1523                         DBG2(dev, "tx %d pkts" NL, n);
1524                 }
1525         }
1526         netif_tx_unlock_bh(dev->ndev);
1527 }
1528
1529 static inline void emac_recycle_rx_skb(struct emac_instance *dev, int slot,
1530                                        int len)
1531 {
1532         struct sk_buff *skb = dev->rx_skb[slot];
1533
1534         DBG2(dev, "recycle %d %d" NL, slot, len);
1535
1536         if (len)
1537                 dma_map_single(&dev->ofdev->dev, skb->data - 2,
1538                                EMAC_DMA_ALIGN(len + 2), DMA_FROM_DEVICE);
1539
1540         dev->rx_desc[slot].data_len = 0;
1541         wmb();
1542         dev->rx_desc[slot].ctrl = MAL_RX_CTRL_EMPTY |
1543             (slot == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
1544 }
1545
1546 static void emac_parse_rx_error(struct emac_instance *dev, u16 ctrl)
1547 {
1548         struct emac_error_stats *st = &dev->estats;
1549
1550         DBG(dev, "BD RX error %04x" NL, ctrl);
1551
1552         ++st->rx_bd_errors;
1553         if (ctrl & EMAC_RX_ST_OE)
1554                 ++st->rx_bd_overrun;
1555         if (ctrl & EMAC_RX_ST_BP)
1556                 ++st->rx_bd_bad_packet;
1557         if (ctrl & EMAC_RX_ST_RP)
1558                 ++st->rx_bd_runt_packet;
1559         if (ctrl & EMAC_RX_ST_SE)
1560                 ++st->rx_bd_short_event;
1561         if (ctrl & EMAC_RX_ST_AE)
1562                 ++st->rx_bd_alignment_error;
1563         if (ctrl & EMAC_RX_ST_BFCS)
1564                 ++st->rx_bd_bad_fcs;
1565         if (ctrl & EMAC_RX_ST_PTL)
1566                 ++st->rx_bd_packet_too_long;
1567         if (ctrl & EMAC_RX_ST_ORE)
1568                 ++st->rx_bd_out_of_range;
1569         if (ctrl & EMAC_RX_ST_IRE)
1570                 ++st->rx_bd_in_range;
1571 }
1572
1573 static inline void emac_rx_csum(struct emac_instance *dev,
1574                                 struct sk_buff *skb, u16 ctrl)
1575 {
1576 #ifdef CONFIG_IBM_NEW_EMAC_TAH
1577         if (!ctrl && dev->tah_dev) {
1578                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1579                 ++dev->stats.rx_packets_csum;
1580         }
1581 #endif
1582 }
1583
1584 static inline int emac_rx_sg_append(struct emac_instance *dev, int slot)
1585 {
1586         if (likely(dev->rx_sg_skb != NULL)) {
1587                 int len = dev->rx_desc[slot].data_len;
1588                 int tot_len = dev->rx_sg_skb->len + len;
1589
1590                 if (unlikely(tot_len + 2 > dev->rx_skb_size)) {
1591                         ++dev->estats.rx_dropped_mtu;
1592                         dev_kfree_skb(dev->rx_sg_skb);
1593                         dev->rx_sg_skb = NULL;
1594                 } else {
1595                         cacheable_memcpy(skb_tail_pointer(dev->rx_sg_skb),
1596                                          dev->rx_skb[slot]->data, len);
1597                         skb_put(dev->rx_sg_skb, len);
1598                         emac_recycle_rx_skb(dev, slot, len);
1599                         return 0;
1600                 }
1601         }
1602         emac_recycle_rx_skb(dev, slot, 0);
1603         return -1;
1604 }
1605
1606 /* NAPI poll context */
1607 static int emac_poll_rx(void *param, int budget)
1608 {
1609         struct emac_instance *dev = param;
1610         int slot = dev->rx_slot, received = 0;
1611
1612         DBG2(dev, "poll_rx(%d)" NL, budget);
1613
1614  again:
1615         while (budget > 0) {
1616                 int len;
1617                 struct sk_buff *skb;
1618                 u16 ctrl = dev->rx_desc[slot].ctrl;
1619
1620                 if (ctrl & MAL_RX_CTRL_EMPTY)
1621                         break;
1622
1623                 skb = dev->rx_skb[slot];
1624                 mb();
1625                 len = dev->rx_desc[slot].data_len;
1626
1627                 if (unlikely(!MAL_IS_SINGLE_RX(ctrl)))
1628                         goto sg;
1629
1630                 ctrl &= EMAC_BAD_RX_MASK;
1631                 if (unlikely(ctrl && ctrl != EMAC_RX_TAH_BAD_CSUM)) {
1632                         emac_parse_rx_error(dev, ctrl);
1633                         ++dev->estats.rx_dropped_error;
1634                         emac_recycle_rx_skb(dev, slot, 0);
1635                         len = 0;
1636                         goto next;
1637                 }
1638
1639                 if (len && len < EMAC_RX_COPY_THRESH) {
1640                         struct sk_buff *copy_skb =
1641                             alloc_skb(len + EMAC_RX_SKB_HEADROOM + 2, GFP_ATOMIC);
1642                         if (unlikely(!copy_skb))
1643                                 goto oom;
1644
1645                         skb_reserve(copy_skb, EMAC_RX_SKB_HEADROOM + 2);
1646                         cacheable_memcpy(copy_skb->data - 2, skb->data - 2,
1647                                          len + 2);
1648                         emac_recycle_rx_skb(dev, slot, len);
1649                         skb = copy_skb;
1650                 } else if (unlikely(emac_alloc_rx_skb(dev, slot, GFP_ATOMIC)))
1651                         goto oom;
1652
1653                 skb_put(skb, len);
1654         push_packet:
1655                 skb->dev = dev->ndev;
1656                 skb->protocol = eth_type_trans(skb, dev->ndev);
1657                 emac_rx_csum(dev, skb, ctrl);
1658
1659                 if (unlikely(netif_receive_skb(skb) == NET_RX_DROP))
1660                         ++dev->estats.rx_dropped_stack;
1661         next:
1662                 ++dev->stats.rx_packets;
1663         skip:
1664                 dev->stats.rx_bytes += len;
1665                 slot = (slot + 1) % NUM_RX_BUFF;
1666                 --budget;
1667                 ++received;
1668                 continue;
1669         sg:
1670                 if (ctrl & MAL_RX_CTRL_FIRST) {
1671                         BUG_ON(dev->rx_sg_skb);
1672                         if (unlikely(emac_alloc_rx_skb(dev, slot, GFP_ATOMIC))) {
1673                                 DBG(dev, "rx OOM %d" NL, slot);
1674                                 ++dev->estats.rx_dropped_oom;
1675                                 emac_recycle_rx_skb(dev, slot, 0);
1676                         } else {
1677                                 dev->rx_sg_skb = skb;
1678                                 skb_put(skb, len);
1679                         }
1680                 } else if (!emac_rx_sg_append(dev, slot) &&
1681                            (ctrl & MAL_RX_CTRL_LAST)) {
1682
1683                         skb = dev->rx_sg_skb;
1684                         dev->rx_sg_skb = NULL;
1685
1686                         ctrl &= EMAC_BAD_RX_MASK;
1687                         if (unlikely(ctrl && ctrl != EMAC_RX_TAH_BAD_CSUM)) {
1688                                 emac_parse_rx_error(dev, ctrl);
1689                                 ++dev->estats.rx_dropped_error;
1690                                 dev_kfree_skb(skb);
1691                                 len = 0;
1692                         } else
1693                                 goto push_packet;
1694                 }
1695                 goto skip;
1696         oom:
1697                 DBG(dev, "rx OOM %d" NL, slot);
1698                 /* Drop the packet and recycle skb */
1699                 ++dev->estats.rx_dropped_oom;
1700                 emac_recycle_rx_skb(dev, slot, 0);
1701                 goto next;
1702         }
1703
1704         if (received) {
1705                 DBG2(dev, "rx %d BDs" NL, received);
1706                 dev->rx_slot = slot;
1707         }
1708
1709         if (unlikely(budget && test_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags))) {
1710                 mb();
1711                 if (!(dev->rx_desc[slot].ctrl & MAL_RX_CTRL_EMPTY)) {
1712                         DBG2(dev, "rx restart" NL);
1713                         received = 0;
1714                         goto again;
1715                 }
1716
1717                 if (dev->rx_sg_skb) {
1718                         DBG2(dev, "dropping partial rx packet" NL);
1719                         ++dev->estats.rx_dropped_error;
1720                         dev_kfree_skb(dev->rx_sg_skb);
1721                         dev->rx_sg_skb = NULL;
1722                 }
1723
1724                 clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1725                 mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
1726                 emac_rx_enable(dev);
1727                 dev->rx_slot = 0;
1728         }
1729         return received;
1730 }
1731
1732 /* NAPI poll context */
1733 static int emac_peek_rx(void *param)
1734 {
1735         struct emac_instance *dev = param;
1736
1737         return !(dev->rx_desc[dev->rx_slot].ctrl & MAL_RX_CTRL_EMPTY);
1738 }
1739
1740 /* NAPI poll context */
1741 static int emac_peek_rx_sg(void *param)
1742 {
1743         struct emac_instance *dev = param;
1744
1745         int slot = dev->rx_slot;
1746         while (1) {
1747                 u16 ctrl = dev->rx_desc[slot].ctrl;
1748                 if (ctrl & MAL_RX_CTRL_EMPTY)
1749                         return 0;
1750                 else if (ctrl & MAL_RX_CTRL_LAST)
1751                         return 1;
1752
1753                 slot = (slot + 1) % NUM_RX_BUFF;
1754
1755                 /* I'm just being paranoid here :) */
1756                 if (unlikely(slot == dev->rx_slot))
1757                         return 0;
1758         }
1759 }
1760
1761 /* Hard IRQ */
1762 static void emac_rxde(void *param)
1763 {
1764         struct emac_instance *dev = param;
1765
1766         ++dev->estats.rx_stopped;
1767         emac_rx_disable_async(dev);
1768 }
1769
1770 /* Hard IRQ */
1771 static irqreturn_t emac_irq(int irq, void *dev_instance)
1772 {
1773         struct emac_instance *dev = dev_instance;
1774         struct emac_regs __iomem *p = dev->emacp;
1775         struct emac_error_stats *st = &dev->estats;
1776         u32 isr;
1777
1778         spin_lock(&dev->lock);
1779
1780         isr = in_be32(&p->isr);
1781         out_be32(&p->isr, isr);
1782
1783         DBG(dev, "isr = %08x" NL, isr);
1784
1785         if (isr & EMAC4_ISR_TXPE)
1786                 ++st->tx_parity;
1787         if (isr & EMAC4_ISR_RXPE)
1788                 ++st->rx_parity;
1789         if (isr & EMAC4_ISR_TXUE)
1790                 ++st->tx_underrun;
1791         if (isr & EMAC4_ISR_RXOE)
1792                 ++st->rx_fifo_overrun;
1793         if (isr & EMAC_ISR_OVR)
1794                 ++st->rx_overrun;
1795         if (isr & EMAC_ISR_BP)
1796                 ++st->rx_bad_packet;
1797         if (isr & EMAC_ISR_RP)
1798                 ++st->rx_runt_packet;
1799         if (isr & EMAC_ISR_SE)
1800                 ++st->rx_short_event;
1801         if (isr & EMAC_ISR_ALE)
1802                 ++st->rx_alignment_error;
1803         if (isr & EMAC_ISR_BFCS)
1804                 ++st->rx_bad_fcs;
1805         if (isr & EMAC_ISR_PTLE)
1806                 ++st->rx_packet_too_long;
1807         if (isr & EMAC_ISR_ORE)
1808                 ++st->rx_out_of_range;
1809         if (isr & EMAC_ISR_IRE)
1810                 ++st->rx_in_range;
1811         if (isr & EMAC_ISR_SQE)
1812                 ++st->tx_sqe;
1813         if (isr & EMAC_ISR_TE)
1814                 ++st->tx_errors;
1815
1816         spin_unlock(&dev->lock);
1817
1818         return IRQ_HANDLED;
1819 }
1820
1821 static struct net_device_stats *emac_stats(struct net_device *ndev)
1822 {
1823         struct emac_instance *dev = netdev_priv(ndev);
1824         struct emac_stats *st = &dev->stats;
1825         struct emac_error_stats *est = &dev->estats;
1826         struct net_device_stats *nst = &dev->nstats;
1827         unsigned long flags;
1828
1829         DBG2(dev, "stats" NL);
1830
1831         /* Compute "legacy" statistics */
1832         spin_lock_irqsave(&dev->lock, flags);
1833         nst->rx_packets = (unsigned long)st->rx_packets;
1834         nst->rx_bytes = (unsigned long)st->rx_bytes;
1835         nst->tx_packets = (unsigned long)st->tx_packets;
1836         nst->tx_bytes = (unsigned long)st->tx_bytes;
1837         nst->rx_dropped = (unsigned long)(est->rx_dropped_oom +
1838                                           est->rx_dropped_error +
1839                                           est->rx_dropped_resize +
1840                                           est->rx_dropped_mtu);
1841         nst->tx_dropped = (unsigned long)est->tx_dropped;
1842
1843         nst->rx_errors = (unsigned long)est->rx_bd_errors;
1844         nst->rx_fifo_errors = (unsigned long)(est->rx_bd_overrun +
1845                                               est->rx_fifo_overrun +
1846                                               est->rx_overrun);
1847         nst->rx_frame_errors = (unsigned long)(est->rx_bd_alignment_error +
1848                                                est->rx_alignment_error);
1849         nst->rx_crc_errors = (unsigned long)(est->rx_bd_bad_fcs +
1850                                              est->rx_bad_fcs);
1851         nst->rx_length_errors = (unsigned long)(est->rx_bd_runt_packet +
1852                                                 est->rx_bd_short_event +
1853                                                 est->rx_bd_packet_too_long +
1854                                                 est->rx_bd_out_of_range +
1855                                                 est->rx_bd_in_range +
1856                                                 est->rx_runt_packet +
1857                                                 est->rx_short_event +
1858                                                 est->rx_packet_too_long +
1859                                                 est->rx_out_of_range +
1860                                                 est->rx_in_range);
1861
1862         nst->tx_errors = (unsigned long)(est->tx_bd_errors + est->tx_errors);
1863         nst->tx_fifo_errors = (unsigned long)(est->tx_bd_underrun +
1864                                               est->tx_underrun);
1865         nst->tx_carrier_errors = (unsigned long)est->tx_bd_carrier_loss;
1866         nst->collisions = (unsigned long)(est->tx_bd_excessive_deferral +
1867                                           est->tx_bd_excessive_collisions +
1868                                           est->tx_bd_late_collision +
1869                                           est->tx_bd_multple_collisions);
1870         spin_unlock_irqrestore(&dev->lock, flags);
1871         return nst;
1872 }
1873
1874 static struct mal_commac_ops emac_commac_ops = {
1875         .poll_tx = &emac_poll_tx,
1876         .poll_rx = &emac_poll_rx,
1877         .peek_rx = &emac_peek_rx,
1878         .rxde = &emac_rxde,
1879 };
1880
1881 static struct mal_commac_ops emac_commac_sg_ops = {
1882         .poll_tx = &emac_poll_tx,
1883         .poll_rx = &emac_poll_rx,
1884         .peek_rx = &emac_peek_rx_sg,
1885         .rxde = &emac_rxde,
1886 };
1887
1888 /* Ethtool support */
1889 static int emac_ethtool_get_settings(struct net_device *ndev,
1890                                      struct ethtool_cmd *cmd)
1891 {
1892         struct emac_instance *dev = netdev_priv(ndev);
1893
1894         cmd->supported = dev->phy.features;
1895         cmd->port = PORT_MII;
1896         cmd->phy_address = dev->phy.address;
1897         cmd->transceiver =
1898             dev->phy.address >= 0 ? XCVR_EXTERNAL : XCVR_INTERNAL;
1899
1900         mutex_lock(&dev->link_lock);
1901         cmd->advertising = dev->phy.advertising;
1902         cmd->autoneg = dev->phy.autoneg;
1903         cmd->speed = dev->phy.speed;
1904         cmd->duplex = dev->phy.duplex;
1905         mutex_unlock(&dev->link_lock);
1906
1907         return 0;
1908 }
1909
1910 static int emac_ethtool_set_settings(struct net_device *ndev,
1911                                      struct ethtool_cmd *cmd)
1912 {
1913         struct emac_instance *dev = netdev_priv(ndev);
1914         u32 f = dev->phy.features;
1915
1916         DBG(dev, "set_settings(%d, %d, %d, 0x%08x)" NL,
1917             cmd->autoneg, cmd->speed, cmd->duplex, cmd->advertising);
1918
1919         /* Basic sanity checks */
1920         if (dev->phy.address < 0)
1921                 return -EOPNOTSUPP;
1922         if (cmd->autoneg != AUTONEG_ENABLE && cmd->autoneg != AUTONEG_DISABLE)
1923                 return -EINVAL;
1924         if (cmd->autoneg == AUTONEG_ENABLE && cmd->advertising == 0)
1925                 return -EINVAL;
1926         if (cmd->duplex != DUPLEX_HALF && cmd->duplex != DUPLEX_FULL)
1927                 return -EINVAL;
1928
1929         if (cmd->autoneg == AUTONEG_DISABLE) {
1930                 switch (cmd->speed) {
1931                 case SPEED_10:
1932                         if (cmd->duplex == DUPLEX_HALF
1933                             && !(f & SUPPORTED_10baseT_Half))
1934                                 return -EINVAL;
1935                         if (cmd->duplex == DUPLEX_FULL
1936                             && !(f & SUPPORTED_10baseT_Full))
1937                                 return -EINVAL;
1938                         break;
1939                 case SPEED_100:
1940                         if (cmd->duplex == DUPLEX_HALF
1941                             && !(f & SUPPORTED_100baseT_Half))
1942                                 return -EINVAL;
1943                         if (cmd->duplex == DUPLEX_FULL
1944                             && !(f & SUPPORTED_100baseT_Full))
1945                                 return -EINVAL;
1946                         break;
1947                 case SPEED_1000:
1948                         if (cmd->duplex == DUPLEX_HALF
1949                             && !(f & SUPPORTED_1000baseT_Half))
1950                                 return -EINVAL;
1951                         if (cmd->duplex == DUPLEX_FULL
1952                             && !(f & SUPPORTED_1000baseT_Full))
1953                                 return -EINVAL;
1954                         break;
1955                 default:
1956                         return -EINVAL;
1957                 }
1958
1959                 mutex_lock(&dev->link_lock);
1960                 dev->phy.def->ops->setup_forced(&dev->phy, cmd->speed,
1961                                                 cmd->duplex);
1962                 mutex_unlock(&dev->link_lock);
1963
1964         } else {
1965                 if (!(f & SUPPORTED_Autoneg))
1966                         return -EINVAL;
1967
1968                 mutex_lock(&dev->link_lock);
1969                 dev->phy.def->ops->setup_aneg(&dev->phy,
1970                                               (cmd->advertising & f) |
1971                                               (dev->phy.advertising &
1972                                                (ADVERTISED_Pause |
1973                                                 ADVERTISED_Asym_Pause)));
1974                 mutex_unlock(&dev->link_lock);
1975         }
1976         emac_force_link_update(dev);
1977
1978         return 0;
1979 }
1980
1981 static void emac_ethtool_get_ringparam(struct net_device *ndev,
1982                                        struct ethtool_ringparam *rp)
1983 {
1984         rp->rx_max_pending = rp->rx_pending = NUM_RX_BUFF;
1985         rp->tx_max_pending = rp->tx_pending = NUM_TX_BUFF;
1986 }
1987
1988 static void emac_ethtool_get_pauseparam(struct net_device *ndev,
1989                                         struct ethtool_pauseparam *pp)
1990 {
1991         struct emac_instance *dev = netdev_priv(ndev);
1992
1993         mutex_lock(&dev->link_lock);
1994         if ((dev->phy.features & SUPPORTED_Autoneg) &&
1995             (dev->phy.advertising & (ADVERTISED_Pause | ADVERTISED_Asym_Pause)))
1996                 pp->autoneg = 1;
1997
1998         if (dev->phy.duplex == DUPLEX_FULL) {
1999                 if (dev->phy.pause)
2000                         pp->rx_pause = pp->tx_pause = 1;
2001                 else if (dev->phy.asym_pause)
2002                         pp->tx_pause = 1;
2003         }
2004         mutex_unlock(&dev->link_lock);
2005 }
2006
2007 static u32 emac_ethtool_get_rx_csum(struct net_device *ndev)
2008 {
2009         struct emac_instance *dev = netdev_priv(ndev);
2010
2011         return dev->tah_dev != NULL;
2012 }
2013
2014 static int emac_get_regs_len(struct emac_instance *dev)
2015 {
2016         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
2017                 return sizeof(struct emac_ethtool_regs_subhdr) +
2018                         EMAC4_ETHTOOL_REGS_SIZE;
2019         else
2020                 return sizeof(struct emac_ethtool_regs_subhdr) +
2021                         EMAC_ETHTOOL_REGS_SIZE;
2022 }
2023
2024 static int emac_ethtool_get_regs_len(struct net_device *ndev)
2025 {
2026         struct emac_instance *dev = netdev_priv(ndev);
2027         int size;
2028
2029         size = sizeof(struct emac_ethtool_regs_hdr) +
2030                 emac_get_regs_len(dev) + mal_get_regs_len(dev->mal);
2031         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
2032                 size += zmii_get_regs_len(dev->zmii_dev);
2033         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
2034                 size += rgmii_get_regs_len(dev->rgmii_dev);
2035         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
2036                 size += tah_get_regs_len(dev->tah_dev);
2037
2038         return size;
2039 }
2040
2041 static void *emac_dump_regs(struct emac_instance *dev, void *buf)
2042 {
2043         struct emac_ethtool_regs_subhdr *hdr = buf;
2044
2045         hdr->index = dev->cell_index;
2046         if (emac_has_feature(dev, EMAC_FTR_EMAC4)) {
2047                 hdr->version = EMAC4_ETHTOOL_REGS_VER;
2048                 memcpy_fromio(hdr + 1, dev->emacp, EMAC4_ETHTOOL_REGS_SIZE);
2049                 return ((void *)(hdr + 1) + EMAC4_ETHTOOL_REGS_SIZE);
2050         } else {
2051                 hdr->version = EMAC_ETHTOOL_REGS_VER;
2052                 memcpy_fromio(hdr + 1, dev->emacp, EMAC_ETHTOOL_REGS_SIZE);
2053                 return ((void *)(hdr + 1) + EMAC_ETHTOOL_REGS_SIZE);
2054         }
2055 }
2056
2057 static void emac_ethtool_get_regs(struct net_device *ndev,
2058                                   struct ethtool_regs *regs, void *buf)
2059 {
2060         struct emac_instance *dev = netdev_priv(ndev);
2061         struct emac_ethtool_regs_hdr *hdr = buf;
2062
2063         hdr->components = 0;
2064         buf = hdr + 1;
2065
2066         buf = mal_dump_regs(dev->mal, buf);
2067         buf = emac_dump_regs(dev, buf);
2068         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII)) {
2069                 hdr->components |= EMAC_ETHTOOL_REGS_ZMII;
2070                 buf = zmii_dump_regs(dev->zmii_dev, buf);
2071         }
2072         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII)) {
2073                 hdr->components |= EMAC_ETHTOOL_REGS_RGMII;
2074                 buf = rgmii_dump_regs(dev->rgmii_dev, buf);
2075         }
2076         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH)) {
2077                 hdr->components |= EMAC_ETHTOOL_REGS_TAH;
2078                 buf = tah_dump_regs(dev->tah_dev, buf);
2079         }
2080 }
2081
2082 static int emac_ethtool_nway_reset(struct net_device *ndev)
2083 {
2084         struct emac_instance *dev = netdev_priv(ndev);
2085         int res = 0;
2086
2087         DBG(dev, "nway_reset" NL);
2088
2089         if (dev->phy.address < 0)
2090                 return -EOPNOTSUPP;
2091
2092         mutex_lock(&dev->link_lock);
2093         if (!dev->phy.autoneg) {
2094                 res = -EINVAL;
2095                 goto out;
2096         }
2097
2098         dev->phy.def->ops->setup_aneg(&dev->phy, dev->phy.advertising);
2099  out:
2100         mutex_unlock(&dev->link_lock);
2101         emac_force_link_update(dev);
2102         return res;
2103 }
2104
2105 static int emac_ethtool_get_stats_count(struct net_device *ndev)
2106 {
2107         return EMAC_ETHTOOL_STATS_COUNT;
2108 }
2109
2110 static void emac_ethtool_get_strings(struct net_device *ndev, u32 stringset,
2111                                      u8 * buf)
2112 {
2113         if (stringset == ETH_SS_STATS)
2114                 memcpy(buf, &emac_stats_keys, sizeof(emac_stats_keys));
2115 }
2116
2117 static void emac_ethtool_get_ethtool_stats(struct net_device *ndev,
2118                                            struct ethtool_stats *estats,
2119                                            u64 * tmp_stats)
2120 {
2121         struct emac_instance *dev = netdev_priv(ndev);
2122
2123         memcpy(tmp_stats, &dev->stats, sizeof(dev->stats));
2124         tmp_stats += sizeof(dev->stats) / sizeof(u64);
2125         memcpy(tmp_stats, &dev->estats, sizeof(dev->estats));
2126 }
2127
2128 static void emac_ethtool_get_drvinfo(struct net_device *ndev,
2129                                      struct ethtool_drvinfo *info)
2130 {
2131         struct emac_instance *dev = netdev_priv(ndev);
2132
2133         strcpy(info->driver, "ibm_emac");
2134         strcpy(info->version, DRV_VERSION);
2135         info->fw_version[0] = '\0';
2136         sprintf(info->bus_info, "PPC 4xx EMAC-%d %s",
2137                 dev->cell_index, dev->ofdev->node->full_name);
2138         info->n_stats = emac_ethtool_get_stats_count(ndev);
2139         info->regdump_len = emac_ethtool_get_regs_len(ndev);
2140 }
2141
2142 static const struct ethtool_ops emac_ethtool_ops = {
2143         .get_settings = emac_ethtool_get_settings,
2144         .set_settings = emac_ethtool_set_settings,
2145         .get_drvinfo = emac_ethtool_get_drvinfo,
2146
2147         .get_regs_len = emac_ethtool_get_regs_len,
2148         .get_regs = emac_ethtool_get_regs,
2149
2150         .nway_reset = emac_ethtool_nway_reset,
2151
2152         .get_ringparam = emac_ethtool_get_ringparam,
2153         .get_pauseparam = emac_ethtool_get_pauseparam,
2154
2155         .get_rx_csum = emac_ethtool_get_rx_csum,
2156
2157         .get_strings = emac_ethtool_get_strings,
2158         .get_stats_count = emac_ethtool_get_stats_count,
2159         .get_ethtool_stats = emac_ethtool_get_ethtool_stats,
2160
2161         .get_link = ethtool_op_get_link,
2162         .get_tx_csum = ethtool_op_get_tx_csum,
2163         .get_sg = ethtool_op_get_sg,
2164 };
2165
2166 static int emac_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2167 {
2168         struct emac_instance *dev = netdev_priv(ndev);
2169         uint16_t *data = (uint16_t *) & rq->ifr_ifru;
2170
2171         DBG(dev, "ioctl %08x" NL, cmd);
2172
2173         if (dev->phy.address < 0)
2174                 return -EOPNOTSUPP;
2175
2176         switch (cmd) {
2177         case SIOCGMIIPHY:
2178         case SIOCDEVPRIVATE:
2179                 data[0] = dev->phy.address;
2180                 /* Fall through */
2181         case SIOCGMIIREG:
2182         case SIOCDEVPRIVATE + 1:
2183                 data[3] = emac_mdio_read(ndev, dev->phy.address, data[1]);
2184                 return 0;
2185
2186         case SIOCSMIIREG:
2187         case SIOCDEVPRIVATE + 2:
2188                 if (!capable(CAP_NET_ADMIN))
2189                         return -EPERM;
2190                 emac_mdio_write(ndev, dev->phy.address, data[1], data[2]);
2191                 return 0;
2192         default:
2193                 return -EOPNOTSUPP;
2194         }
2195 }
2196
2197 struct emac_depentry {
2198         u32                     phandle;
2199         struct device_node      *node;
2200         struct of_device        *ofdev;
2201         void                    *drvdata;
2202 };
2203
2204 #define EMAC_DEP_MAL_IDX        0
2205 #define EMAC_DEP_ZMII_IDX       1
2206 #define EMAC_DEP_RGMII_IDX      2
2207 #define EMAC_DEP_TAH_IDX        3
2208 #define EMAC_DEP_MDIO_IDX       4
2209 #define EMAC_DEP_PREV_IDX       5
2210 #define EMAC_DEP_COUNT          6
2211
2212 static int __devinit emac_check_deps(struct emac_instance *dev,
2213                                      struct emac_depentry *deps)
2214 {
2215         int i, there = 0;
2216         struct device_node *np;
2217
2218         for (i = 0; i < EMAC_DEP_COUNT; i++) {
2219                 /* no dependency on that item, allright */
2220                 if (deps[i].phandle == 0) {
2221                         there++;
2222                         continue;
2223                 }
2224                 /* special case for blist as the dependency might go away */
2225                 if (i == EMAC_DEP_PREV_IDX) {
2226                         np = *(dev->blist - 1);
2227                         if (np == NULL) {
2228                                 deps[i].phandle = 0;
2229                                 there++;
2230                                 continue;
2231                         }
2232                         if (deps[i].node == NULL)
2233                                 deps[i].node = of_node_get(np);
2234                 }
2235                 if (deps[i].node == NULL)
2236                         deps[i].node = of_find_node_by_phandle(deps[i].phandle);
2237                 if (deps[i].node == NULL)
2238                         continue;
2239                 if (deps[i].ofdev == NULL)
2240                         deps[i].ofdev = of_find_device_by_node(deps[i].node);
2241                 if (deps[i].ofdev == NULL)
2242                         continue;
2243                 if (deps[i].drvdata == NULL)
2244                         deps[i].drvdata = dev_get_drvdata(&deps[i].ofdev->dev);
2245                 if (deps[i].drvdata != NULL)
2246                         there++;
2247         }
2248         return (there == EMAC_DEP_COUNT);
2249 }
2250
2251 static void emac_put_deps(struct emac_instance *dev)
2252 {
2253         if (dev->mal_dev)
2254                 of_dev_put(dev->mal_dev);
2255         if (dev->zmii_dev)
2256                 of_dev_put(dev->zmii_dev);
2257         if (dev->rgmii_dev)
2258                 of_dev_put(dev->rgmii_dev);
2259         if (dev->mdio_dev)
2260                 of_dev_put(dev->mdio_dev);
2261         if (dev->tah_dev)
2262                 of_dev_put(dev->tah_dev);
2263 }
2264
2265 static int __devinit emac_of_bus_notify(struct notifier_block *nb,
2266                                         unsigned long action, void *data)
2267 {
2268         /* We are only intereted in device addition */
2269         if (action == BUS_NOTIFY_BOUND_DRIVER)
2270                 wake_up_all(&emac_probe_wait);
2271         return 0;
2272 }
2273
2274 static struct notifier_block emac_of_bus_notifier __devinitdata = {
2275         .notifier_call = emac_of_bus_notify
2276 };
2277
2278 static int __devinit emac_wait_deps(struct emac_instance *dev)
2279 {
2280         struct emac_depentry deps[EMAC_DEP_COUNT];
2281         int i, err;
2282
2283         memset(&deps, 0, sizeof(deps));
2284
2285         deps[EMAC_DEP_MAL_IDX].phandle = dev->mal_ph;
2286         deps[EMAC_DEP_ZMII_IDX].phandle = dev->zmii_ph;
2287         deps[EMAC_DEP_RGMII_IDX].phandle = dev->rgmii_ph;
2288         if (dev->tah_ph)
2289                 deps[EMAC_DEP_TAH_IDX].phandle = dev->tah_ph;
2290         if (dev->mdio_ph)
2291                 deps[EMAC_DEP_MDIO_IDX].phandle = dev->mdio_ph;
2292         if (dev->blist && dev->blist > emac_boot_list)
2293                 deps[EMAC_DEP_PREV_IDX].phandle = 0xffffffffu;
2294         bus_register_notifier(&of_platform_bus_type, &emac_of_bus_notifier);
2295         wait_event_timeout(emac_probe_wait,
2296                            emac_check_deps(dev, deps),
2297                            EMAC_PROBE_DEP_TIMEOUT);
2298         bus_unregister_notifier(&of_platform_bus_type, &emac_of_bus_notifier);
2299         err = emac_check_deps(dev, deps) ? 0 : -ENODEV;
2300         for (i = 0; i < EMAC_DEP_COUNT; i++) {
2301                 if (deps[i].node)
2302                         of_node_put(deps[i].node);
2303                 if (err && deps[i].ofdev)
2304                         of_dev_put(deps[i].ofdev);
2305         }
2306         if (err == 0) {
2307                 dev->mal_dev = deps[EMAC_DEP_MAL_IDX].ofdev;
2308                 dev->zmii_dev = deps[EMAC_DEP_ZMII_IDX].ofdev;
2309                 dev->rgmii_dev = deps[EMAC_DEP_RGMII_IDX].ofdev;
2310                 dev->tah_dev = deps[EMAC_DEP_TAH_IDX].ofdev;
2311                 dev->mdio_dev = deps[EMAC_DEP_MDIO_IDX].ofdev;
2312         }
2313         if (deps[EMAC_DEP_PREV_IDX].ofdev)
2314                 of_dev_put(deps[EMAC_DEP_PREV_IDX].ofdev);
2315         return err;
2316 }
2317
2318 static int __devinit emac_read_uint_prop(struct device_node *np, const char *name,
2319                                          u32 *val, int fatal)
2320 {
2321         int len;
2322         const u32 *prop = of_get_property(np, name, &len);
2323         if (prop == NULL || len < sizeof(u32)) {
2324                 if (fatal)
2325                         printk(KERN_ERR "%s: missing %s property\n",
2326                                np->full_name, name);
2327                 return -ENODEV;
2328         }
2329         *val = *prop;
2330         return 0;
2331 }
2332
2333 static int __devinit emac_init_phy(struct emac_instance *dev)
2334 {
2335         struct device_node *np = dev->ofdev->node;
2336         struct net_device *ndev = dev->ndev;
2337         u32 phy_map, adv;
2338         int i;
2339
2340         dev->phy.dev = ndev;
2341         dev->phy.mode = dev->phy_mode;
2342
2343         /* PHY-less configuration.
2344          * XXX I probably should move these settings to the dev tree
2345          */
2346         if (dev->phy_address == 0xffffffff && dev->phy_map == 0xffffffff) {
2347                 emac_reset(dev);
2348
2349                 /* PHY-less configuration.
2350                  * XXX I probably should move these settings to the dev tree
2351                  */
2352                 dev->phy.address = -1;
2353                 dev->phy.features = SUPPORTED_100baseT_Full | SUPPORTED_MII;
2354                 dev->phy.pause = 1;
2355
2356                 return 0;
2357         }
2358
2359         mutex_lock(&emac_phy_map_lock);
2360         phy_map = dev->phy_map | busy_phy_map;
2361
2362         DBG(dev, "PHY maps %08x %08x" NL, dev->phy_map, busy_phy_map);
2363
2364         dev->phy.mdio_read = emac_mdio_read;
2365         dev->phy.mdio_write = emac_mdio_write;
2366
2367         /* Enable internal clock source */
2368 #ifdef CONFIG_PPC_DCR_NATIVE
2369         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2370                 dcri_clrset(SDR0, SDR0_MFR, 0, SDR0_MFR_ECS);
2371 #endif
2372         /* PHY clock workaround */
2373         emac_rx_clk_tx(dev);
2374
2375         /* Enable internal clock source on 440GX*/
2376 #ifdef CONFIG_PPC_DCR_NATIVE
2377         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2378                 dcri_clrset(SDR0, SDR0_MFR, 0, SDR0_MFR_ECS);
2379 #endif
2380         /* Configure EMAC with defaults so we can at least use MDIO
2381          * This is needed mostly for 440GX
2382          */
2383         if (emac_phy_gpcs(dev->phy.mode)) {
2384                 /* XXX
2385                  * Make GPCS PHY address equal to EMAC index.
2386                  * We probably should take into account busy_phy_map
2387                  * and/or phy_map here.
2388                  *
2389                  * Note that the busy_phy_map is currently global
2390                  * while it should probably be per-ASIC...
2391                  */
2392                 dev->phy.address = dev->cell_index;
2393         }
2394
2395         emac_configure(dev);
2396
2397         if (dev->phy_address != 0xffffffff)
2398                 phy_map = ~(1 << dev->phy_address);
2399
2400         for (i = 0; i < 0x20; phy_map >>= 1, ++i)
2401                 if (!(phy_map & 1)) {
2402                         int r;
2403                         busy_phy_map |= 1 << i;
2404
2405                         /* Quick check if there is a PHY at the address */
2406                         r = emac_mdio_read(dev->ndev, i, MII_BMCR);
2407                         if (r == 0xffff || r < 0)
2408                                 continue;
2409                         if (!emac_mii_phy_probe(&dev->phy, i))
2410                                 break;
2411                 }
2412
2413         /* Enable external clock source */
2414 #ifdef CONFIG_PPC_DCR_NATIVE
2415         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2416                 dcri_clrset(SDR0, SDR0_MFR, SDR0_MFR_ECS, 0);
2417 #endif
2418         mutex_unlock(&emac_phy_map_lock);
2419         if (i == 0x20) {
2420                 printk(KERN_WARNING "%s: can't find PHY!\n", np->full_name);
2421                 return -ENXIO;
2422         }
2423
2424         /* Init PHY */
2425         if (dev->phy.def->ops->init)
2426                 dev->phy.def->ops->init(&dev->phy);
2427
2428         /* Disable any PHY features not supported by the platform */
2429         dev->phy.def->features &= ~dev->phy_feat_exc;
2430
2431         /* Setup initial link parameters */
2432         if (dev->phy.features & SUPPORTED_Autoneg) {
2433                 adv = dev->phy.features;
2434                 if (!emac_has_feature(dev, EMAC_FTR_NO_FLOW_CONTROL_40x))
2435                         adv |= ADVERTISED_Pause | ADVERTISED_Asym_Pause;
2436                 /* Restart autonegotiation */
2437                 dev->phy.def->ops->setup_aneg(&dev->phy, adv);
2438         } else {
2439                 u32 f = dev->phy.def->features;
2440                 int speed = SPEED_10, fd = DUPLEX_HALF;
2441
2442                 /* Select highest supported speed/duplex */
2443                 if (f & SUPPORTED_1000baseT_Full) {
2444                         speed = SPEED_1000;
2445                         fd = DUPLEX_FULL;
2446                 } else if (f & SUPPORTED_1000baseT_Half)
2447                         speed = SPEED_1000;
2448                 else if (f & SUPPORTED_100baseT_Full) {
2449                         speed = SPEED_100;
2450                         fd = DUPLEX_FULL;
2451                 } else if (f & SUPPORTED_100baseT_Half)
2452                         speed = SPEED_100;
2453                 else if (f & SUPPORTED_10baseT_Full)
2454                         fd = DUPLEX_FULL;
2455
2456                 /* Force link parameters */
2457                 dev->phy.def->ops->setup_forced(&dev->phy, speed, fd);
2458         }
2459         return 0;
2460 }
2461
2462 static int __devinit emac_init_config(struct emac_instance *dev)
2463 {
2464         struct device_node *np = dev->ofdev->node;
2465         const void *p;
2466         unsigned int plen;
2467         const char *pm, *phy_modes[] = {
2468                 [PHY_MODE_NA] = "",
2469                 [PHY_MODE_MII] = "mii",
2470                 [PHY_MODE_RMII] = "rmii",
2471                 [PHY_MODE_SMII] = "smii",
2472                 [PHY_MODE_RGMII] = "rgmii",
2473                 [PHY_MODE_TBI] = "tbi",
2474                 [PHY_MODE_GMII] = "gmii",
2475                 [PHY_MODE_RTBI] = "rtbi",
2476                 [PHY_MODE_SGMII] = "sgmii",
2477         };
2478
2479         /* Read config from device-tree */
2480         if (emac_read_uint_prop(np, "mal-device", &dev->mal_ph, 1))
2481                 return -ENXIO;
2482         if (emac_read_uint_prop(np, "mal-tx-channel", &dev->mal_tx_chan, 1))
2483                 return -ENXIO;
2484         if (emac_read_uint_prop(np, "mal-rx-channel", &dev->mal_rx_chan, 1))
2485                 return -ENXIO;
2486         if (emac_read_uint_prop(np, "cell-index", &dev->cell_index, 1))
2487                 return -ENXIO;
2488         if (emac_read_uint_prop(np, "max-frame-size", &dev->max_mtu, 0))
2489                 dev->max_mtu = 1500;
2490         if (emac_read_uint_prop(np, "rx-fifo-size", &dev->rx_fifo_size, 0))
2491                 dev->rx_fifo_size = 2048;
2492         if (emac_read_uint_prop(np, "tx-fifo-size", &dev->tx_fifo_size, 0))
2493                 dev->tx_fifo_size = 2048;
2494         if (emac_read_uint_prop(np, "rx-fifo-size-gige", &dev->rx_fifo_size_gige, 0))
2495                 dev->rx_fifo_size_gige = dev->rx_fifo_size;
2496         if (emac_read_uint_prop(np, "tx-fifo-size-gige", &dev->tx_fifo_size_gige, 0))
2497                 dev->tx_fifo_size_gige = dev->tx_fifo_size;
2498         if (emac_read_uint_prop(np, "phy-address", &dev->phy_address, 0))
2499                 dev->phy_address = 0xffffffff;
2500         if (emac_read_uint_prop(np, "phy-map", &dev->phy_map, 0))
2501                 dev->phy_map = 0xffffffff;
2502         if (emac_read_uint_prop(np->parent, "clock-frequency", &dev->opb_bus_freq, 1))
2503                 return -ENXIO;
2504         if (emac_read_uint_prop(np, "tah-device", &dev->tah_ph, 0))
2505                 dev->tah_ph = 0;
2506         if (emac_read_uint_prop(np, "tah-channel", &dev->tah_port, 0))
2507                 dev->tah_port = 0;
2508         if (emac_read_uint_prop(np, "mdio-device", &dev->mdio_ph, 0))
2509                 dev->mdio_ph = 0;
2510         if (emac_read_uint_prop(np, "zmii-device", &dev->zmii_ph, 0))
2511                 dev->zmii_ph = 0;;
2512         if (emac_read_uint_prop(np, "zmii-channel", &dev->zmii_port, 0))
2513                 dev->zmii_port = 0xffffffff;;
2514         if (emac_read_uint_prop(np, "rgmii-device", &dev->rgmii_ph, 0))
2515                 dev->rgmii_ph = 0;;
2516         if (emac_read_uint_prop(np, "rgmii-channel", &dev->rgmii_port, 0))
2517                 dev->rgmii_port = 0xffffffff;;
2518         if (emac_read_uint_prop(np, "fifo-entry-size", &dev->fifo_entry_size, 0))
2519                 dev->fifo_entry_size = 16;
2520         if (emac_read_uint_prop(np, "mal-burst-size", &dev->mal_burst_size, 0))
2521                 dev->mal_burst_size = 256;
2522
2523         /* PHY mode needs some decoding */
2524         dev->phy_mode = PHY_MODE_NA;
2525         pm = of_get_property(np, "phy-mode", &plen);
2526         if (pm != NULL) {
2527                 int i;
2528                 for (i = 0; i < ARRAY_SIZE(phy_modes); i++)
2529                         if (!strcasecmp(pm, phy_modes[i])) {
2530                                 dev->phy_mode = i;
2531                                 break;
2532                         }
2533         }
2534
2535         /* Backward compat with non-final DT */
2536         if (dev->phy_mode == PHY_MODE_NA && pm != NULL && plen == 4) {
2537                 u32 nmode = *(const u32 *)pm;
2538                 if (nmode > PHY_MODE_NA && nmode <= PHY_MODE_SGMII)
2539                         dev->phy_mode = nmode;
2540         }
2541
2542         /* Check EMAC version */
2543         if (of_device_is_compatible(np, "ibm,emac4")) {
2544                 dev->features |= EMAC_FTR_EMAC4;
2545                 if (of_device_is_compatible(np, "ibm,emac-440gx"))
2546                         dev->features |= EMAC_FTR_440GX_PHY_CLK_FIX;
2547         } else {
2548                 if (of_device_is_compatible(np, "ibm,emac-440ep") ||
2549                     of_device_is_compatible(np, "ibm,emac-440gr"))
2550                         dev->features |= EMAC_FTR_440EP_PHY_CLK_FIX;
2551         }
2552
2553         /* Fixup some feature bits based on the device tree */
2554         if (of_get_property(np, "has-inverted-stacr-oc", NULL))
2555                 dev->features |= EMAC_FTR_STACR_OC_INVERT;
2556         if (of_get_property(np, "has-new-stacr-staopc", NULL))
2557                 dev->features |= EMAC_FTR_HAS_NEW_STACR;
2558
2559         /* CAB lacks the appropriate properties */
2560         if (of_device_is_compatible(np, "ibm,emac-axon"))
2561                 dev->features |= EMAC_FTR_HAS_NEW_STACR |
2562                         EMAC_FTR_STACR_OC_INVERT;
2563
2564         /* Enable TAH/ZMII/RGMII features as found */
2565         if (dev->tah_ph != 0) {
2566 #ifdef CONFIG_IBM_NEW_EMAC_TAH
2567                 dev->features |= EMAC_FTR_HAS_TAH;
2568 #else
2569                 printk(KERN_ERR "%s: TAH support not enabled !\n",
2570                        np->full_name);
2571                 return -ENXIO;
2572 #endif
2573         }
2574
2575         if (dev->zmii_ph != 0) {
2576 #ifdef CONFIG_IBM_NEW_EMAC_ZMII
2577                 dev->features |= EMAC_FTR_HAS_ZMII;
2578 #else
2579                 printk(KERN_ERR "%s: ZMII support not enabled !\n",
2580                        np->full_name);
2581                 return -ENXIO;
2582 #endif
2583         }
2584
2585         if (dev->rgmii_ph != 0) {
2586 #ifdef CONFIG_IBM_NEW_EMAC_RGMII
2587                 dev->features |= EMAC_FTR_HAS_RGMII;
2588 #else
2589                 printk(KERN_ERR "%s: RGMII support not enabled !\n",
2590                        np->full_name);
2591                 return -ENXIO;
2592 #endif
2593         }
2594
2595         /* Read MAC-address */
2596         p = of_get_property(np, "local-mac-address", NULL);
2597         if (p == NULL) {
2598                 printk(KERN_ERR "%s: Can't find local-mac-address property\n",
2599                        np->full_name);
2600                 return -ENXIO;
2601         }
2602         memcpy(dev->ndev->dev_addr, p, 6);
2603
2604         DBG(dev, "features     : 0x%08x / 0x%08x\n", dev->features, EMAC_FTRS_POSSIBLE);
2605         DBG(dev, "tx_fifo_size : %d (%d gige)\n", dev->tx_fifo_size, dev->tx_fifo_size_gige);
2606         DBG(dev, "rx_fifo_size : %d (%d gige)\n", dev->rx_fifo_size, dev->rx_fifo_size_gige);
2607         DBG(dev, "max_mtu      : %d\n", dev->max_mtu);
2608         DBG(dev, "OPB freq     : %d\n", dev->opb_bus_freq);
2609
2610         return 0;
2611 }
2612
2613 static int __devinit emac_probe(struct of_device *ofdev,
2614                                 const struct of_device_id *match)
2615 {
2616         struct net_device *ndev;
2617         struct emac_instance *dev;
2618         struct device_node *np = ofdev->node;
2619         struct device_node **blist = NULL;
2620         int err, i;
2621
2622         /* Skip unused/unwired EMACS.  We leave the check for an unused
2623          * property here for now, but new flat device trees should set a
2624          * status property to "disabled" instead.
2625          */
2626         if (of_get_property(np, "unused", NULL) || !of_device_is_available(np))
2627                 return -ENODEV;
2628
2629         /* Find ourselves in the bootlist if we are there */
2630         for (i = 0; i < EMAC_BOOT_LIST_SIZE; i++)
2631                 if (emac_boot_list[i] == np)
2632                         blist = &emac_boot_list[i];
2633
2634         /* Allocate our net_device structure */
2635         err = -ENOMEM;
2636         ndev = alloc_etherdev(sizeof(struct emac_instance));
2637         if (!ndev) {
2638                 printk(KERN_ERR "%s: could not allocate ethernet device!\n",
2639                        np->full_name);
2640                 goto err_gone;
2641         }
2642         dev = netdev_priv(ndev);
2643         dev->ndev = ndev;
2644         dev->ofdev = ofdev;
2645         dev->blist = blist;
2646         SET_NETDEV_DEV(ndev, &ofdev->dev);
2647
2648         /* Initialize some embedded data structures */
2649         mutex_init(&dev->mdio_lock);
2650         mutex_init(&dev->link_lock);
2651         spin_lock_init(&dev->lock);
2652         INIT_WORK(&dev->reset_work, emac_reset_work);
2653
2654         /* Init various config data based on device-tree */
2655         err = emac_init_config(dev);
2656         if (err != 0)
2657                 goto err_free;
2658
2659         /* Get interrupts. EMAC irq is mandatory, WOL irq is optional */
2660         dev->emac_irq = irq_of_parse_and_map(np, 0);
2661         dev->wol_irq = irq_of_parse_and_map(np, 1);
2662         if (dev->emac_irq == NO_IRQ) {
2663                 printk(KERN_ERR "%s: Can't map main interrupt\n", np->full_name);
2664                 goto err_free;
2665         }
2666         ndev->irq = dev->emac_irq;
2667
2668         /* Map EMAC regs */
2669         if (of_address_to_resource(np, 0, &dev->rsrc_regs)) {
2670                 printk(KERN_ERR "%s: Can't get registers address\n",
2671                        np->full_name);
2672                 goto err_irq_unmap;
2673         }
2674         // TODO : request_mem_region
2675         dev->emacp = ioremap(dev->rsrc_regs.start, sizeof(struct emac_regs));
2676         if (dev->emacp == NULL) {
2677                 printk(KERN_ERR "%s: Can't map device registers!\n",
2678                        np->full_name);
2679                 err = -ENOMEM;
2680                 goto err_irq_unmap;
2681         }
2682
2683         /* Wait for dependent devices */
2684         err = emac_wait_deps(dev);
2685         if (err) {
2686                 printk(KERN_ERR
2687                        "%s: Timeout waiting for dependent devices\n",
2688                        np->full_name);
2689                 /*  display more info about what's missing ? */
2690                 goto err_reg_unmap;
2691         }
2692         dev->mal = dev_get_drvdata(&dev->mal_dev->dev);
2693         if (dev->mdio_dev != NULL)
2694                 dev->mdio_instance = dev_get_drvdata(&dev->mdio_dev->dev);
2695
2696         /* Register with MAL */
2697         dev->commac.ops = &emac_commac_ops;
2698         dev->commac.dev = dev;
2699         dev->commac.tx_chan_mask = MAL_CHAN_MASK(dev->mal_tx_chan);
2700         dev->commac.rx_chan_mask = MAL_CHAN_MASK(dev->mal_rx_chan);
2701         err = mal_register_commac(dev->mal, &dev->commac);
2702         if (err) {
2703                 printk(KERN_ERR "%s: failed to register with mal %s!\n",
2704                        np->full_name, dev->mal_dev->node->full_name);
2705                 goto err_rel_deps;
2706         }
2707         dev->rx_skb_size = emac_rx_skb_size(ndev->mtu);
2708         dev->rx_sync_size = emac_rx_sync_size(ndev->mtu);
2709
2710         /* Get pointers to BD rings */
2711         dev->tx_desc =
2712             dev->mal->bd_virt + mal_tx_bd_offset(dev->mal, dev->mal_tx_chan);
2713         dev->rx_desc =
2714             dev->mal->bd_virt + mal_rx_bd_offset(dev->mal, dev->mal_rx_chan);
2715
2716         DBG(dev, "tx_desc %p" NL, dev->tx_desc);
2717         DBG(dev, "rx_desc %p" NL, dev->rx_desc);
2718
2719         /* Clean rings */
2720         memset(dev->tx_desc, 0, NUM_TX_BUFF * sizeof(struct mal_descriptor));
2721         memset(dev->rx_desc, 0, NUM_RX_BUFF * sizeof(struct mal_descriptor));
2722
2723         /* Attach to ZMII, if needed */
2724         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII) &&
2725             (err = zmii_attach(dev->zmii_dev, dev->zmii_port, &dev->phy_mode)) != 0)
2726                 goto err_unreg_commac;
2727
2728         /* Attach to RGMII, if needed */
2729         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII) &&
2730             (err = rgmii_attach(dev->rgmii_dev, dev->rgmii_port, dev->phy_mode)) != 0)
2731                 goto err_detach_zmii;
2732
2733         /* Attach to TAH, if needed */
2734         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH) &&
2735             (err = tah_attach(dev->tah_dev, dev->tah_port)) != 0)
2736                 goto err_detach_rgmii;
2737
2738         /* Set some link defaults before we can find out real parameters */
2739         dev->phy.speed = SPEED_100;
2740         dev->phy.duplex = DUPLEX_FULL;
2741         dev->phy.autoneg = AUTONEG_DISABLE;
2742         dev->phy.pause = dev->phy.asym_pause = 0;
2743         dev->stop_timeout = STOP_TIMEOUT_100;
2744         INIT_DELAYED_WORK(&dev->link_work, emac_link_timer);
2745
2746         /* Find PHY if any */
2747         err = emac_init_phy(dev);
2748         if (err != 0)
2749                 goto err_detach_tah;
2750
2751         /* Fill in the driver function table */
2752         ndev->open = &emac_open;
2753         if (dev->tah_dev)
2754                 ndev->features |= NETIF_F_IP_CSUM | NETIF_F_SG;
2755         ndev->tx_timeout = &emac_tx_timeout;
2756         ndev->watchdog_timeo = 5 * HZ;
2757         ndev->stop = &emac_close;
2758         ndev->get_stats = &emac_stats;
2759         ndev->set_multicast_list = &emac_set_multicast_list;
2760         ndev->do_ioctl = &emac_ioctl;
2761         if (emac_phy_supports_gige(dev->phy_mode)) {
2762                 ndev->hard_start_xmit = &emac_start_xmit_sg;
2763                 ndev->change_mtu = &emac_change_mtu;
2764                 dev->commac.ops = &emac_commac_sg_ops;
2765         } else {
2766                 ndev->hard_start_xmit = &emac_start_xmit;
2767         }
2768         SET_ETHTOOL_OPS(ndev, &emac_ethtool_ops);
2769
2770         netif_carrier_off(ndev);
2771         netif_stop_queue(ndev);
2772
2773         err = register_netdev(ndev);
2774         if (err) {
2775                 printk(KERN_ERR "%s: failed to register net device (%d)!\n",
2776                        np->full_name, err);
2777                 goto err_detach_tah;
2778         }
2779
2780         /* Set our drvdata last as we don't want them visible until we are
2781          * fully initialized
2782          */
2783         wmb();
2784         dev_set_drvdata(&ofdev->dev, dev);
2785
2786         /* There's a new kid in town ! Let's tell everybody */
2787         wake_up_all(&emac_probe_wait);
2788
2789
2790         printk(KERN_INFO
2791                "%s: EMAC-%d %s, MAC %02x:%02x:%02x:%02x:%02x:%02x\n",
2792                ndev->name, dev->cell_index, np->full_name,
2793                ndev->dev_addr[0], ndev->dev_addr[1], ndev->dev_addr[2],
2794                ndev->dev_addr[3], ndev->dev_addr[4], ndev->dev_addr[5]);
2795
2796         if (dev->phy.address >= 0)
2797                 printk("%s: found %s PHY (0x%02x)\n", ndev->name,
2798                        dev->phy.def->name, dev->phy.address);
2799
2800         emac_dbg_register(dev);
2801
2802         /* Life is good */
2803         return 0;
2804
2805         /* I have a bad feeling about this ... */
2806
2807  err_detach_tah:
2808         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
2809                 tah_detach(dev->tah_dev, dev->tah_port);
2810  err_detach_rgmii:
2811         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
2812                 rgmii_detach(dev->rgmii_dev, dev->rgmii_port);
2813  err_detach_zmii:
2814         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
2815                 zmii_detach(dev->zmii_dev, dev->zmii_port);
2816  err_unreg_commac:
2817         mal_unregister_commac(dev->mal, &dev->commac);
2818  err_rel_deps:
2819         emac_put_deps(dev);
2820  err_reg_unmap:
2821         iounmap(dev->emacp);
2822  err_irq_unmap:
2823         if (dev->wol_irq != NO_IRQ)
2824                 irq_dispose_mapping(dev->wol_irq);
2825         if (dev->emac_irq != NO_IRQ)
2826                 irq_dispose_mapping(dev->emac_irq);
2827  err_free:
2828         kfree(ndev);
2829  err_gone:
2830         /* if we were on the bootlist, remove us as we won't show up and
2831          * wake up all waiters to notify them in case they were waiting
2832          * on us
2833          */
2834         if (blist) {
2835                 *blist = NULL;
2836                 wake_up_all(&emac_probe_wait);
2837         }
2838         return err;
2839 }
2840
2841 static int __devexit emac_remove(struct of_device *ofdev)
2842 {
2843         struct emac_instance *dev = dev_get_drvdata(&ofdev->dev);
2844
2845         DBG(dev, "remove" NL);
2846
2847         dev_set_drvdata(&ofdev->dev, NULL);
2848
2849         unregister_netdev(dev->ndev);
2850
2851         flush_scheduled_work();
2852
2853         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
2854                 tah_detach(dev->tah_dev, dev->tah_port);
2855         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
2856                 rgmii_detach(dev->rgmii_dev, dev->rgmii_port);
2857         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
2858                 zmii_detach(dev->zmii_dev, dev->zmii_port);
2859
2860         mal_unregister_commac(dev->mal, &dev->commac);
2861         emac_put_deps(dev);
2862
2863         emac_dbg_unregister(dev);
2864         iounmap(dev->emacp);
2865
2866         if (dev->wol_irq != NO_IRQ)
2867                 irq_dispose_mapping(dev->wol_irq);
2868         if (dev->emac_irq != NO_IRQ)
2869                 irq_dispose_mapping(dev->emac_irq);
2870
2871         kfree(dev->ndev);
2872
2873         return 0;
2874 }
2875
2876 /* XXX Features in here should be replaced by properties... */
2877 static struct of_device_id emac_match[] =
2878 {
2879         {
2880                 .type           = "network",
2881                 .compatible     = "ibm,emac",
2882         },
2883         {
2884                 .type           = "network",
2885                 .compatible     = "ibm,emac4",
2886         },
2887         {},
2888 };
2889
2890 static struct of_platform_driver emac_driver = {
2891         .name = "emac",
2892         .match_table = emac_match,
2893
2894         .probe = emac_probe,
2895         .remove = emac_remove,
2896 };
2897
2898 static void __init emac_make_bootlist(void)
2899 {
2900         struct device_node *np = NULL;
2901         int j, max, i = 0, k;
2902         int cell_indices[EMAC_BOOT_LIST_SIZE];
2903
2904         /* Collect EMACs */
2905         while((np = of_find_all_nodes(np)) != NULL) {
2906                 const u32 *idx;
2907
2908                 if (of_match_node(emac_match, np) == NULL)
2909                         continue;
2910                 if (of_get_property(np, "unused", NULL))
2911                         continue;
2912                 idx = of_get_property(np, "cell-index", NULL);
2913                 if (idx == NULL)
2914                         continue;
2915                 cell_indices[i] = *idx;
2916                 emac_boot_list[i++] = of_node_get(np);
2917                 if (i >= EMAC_BOOT_LIST_SIZE) {
2918                         of_node_put(np);
2919                         break;
2920                 }
2921         }
2922         max = i;
2923
2924         /* Bubble sort them (doh, what a creative algorithm :-) */
2925         for (i = 0; max > 1 && (i < (max - 1)); i++)
2926                 for (j = i; j < max; j++) {
2927                         if (cell_indices[i] > cell_indices[j]) {
2928                                 np = emac_boot_list[i];
2929                                 emac_boot_list[i] = emac_boot_list[j];
2930                                 emac_boot_list[j] = np;
2931                                 k = cell_indices[i];
2932                                 cell_indices[i] = cell_indices[j];
2933                                 cell_indices[j] = k;
2934                         }
2935                 }
2936 }
2937
2938 static int __init emac_init(void)
2939 {
2940         int rc;
2941
2942         printk(KERN_INFO DRV_DESC ", version " DRV_VERSION "\n");
2943
2944         /* Init debug stuff */
2945         emac_init_debug();
2946
2947         /* Build EMAC boot list */
2948         emac_make_bootlist();
2949
2950         /* Init submodules */
2951         rc = mal_init();
2952         if (rc)
2953                 goto err;
2954         rc = zmii_init();
2955         if (rc)
2956                 goto err_mal;
2957         rc = rgmii_init();
2958         if (rc)
2959                 goto err_zmii;
2960         rc = tah_init();
2961         if (rc)
2962                 goto err_rgmii;
2963         rc = of_register_platform_driver(&emac_driver);
2964         if (rc)
2965                 goto err_tah;
2966
2967         return 0;
2968
2969  err_tah:
2970         tah_exit();
2971  err_rgmii:
2972         rgmii_exit();
2973  err_zmii:
2974         zmii_exit();
2975  err_mal:
2976         mal_exit();
2977  err:
2978         return rc;
2979 }
2980
2981 static void __exit emac_exit(void)
2982 {
2983         int i;
2984
2985         of_unregister_platform_driver(&emac_driver);
2986
2987         tah_exit();
2988         rgmii_exit();
2989         zmii_exit();
2990         mal_exit();
2991         emac_fini_debug();
2992
2993         /* Destroy EMAC boot list */
2994         for (i = 0; i < EMAC_BOOT_LIST_SIZE; i++)
2995                 if (emac_boot_list[i])
2996                         of_node_put(emac_boot_list[i]);
2997 }
2998
2999 module_init(emac_init);
3000 module_exit(emac_exit);