drivers/net/tg3.c

   1 /*
   2  * tg3.c: Broadcom Tigon3 ethernet driver.
   3  *
   4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
   5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
   6  * Copyright (C) 2004 Sun Microsystems Inc.
   7  * Copyright (C) 2005-2007 Broadcom Corporation.
   8  *
   9  * Firmware is:
  10  *      Derived from proprietary unpublished source code,
  11  *      Copyright (C) 2000-2003 Broadcom Corporation.
  12  *
  13  *      Permission is hereby granted for the distribution of this firmware
  14  *      data in hexadecimal or equivalent format, provided this copyright
  15  *      notice is accompanying it.
  16  */
  17
  18
  19 #include <linux/module.h>
  20 #include <linux/moduleparam.h>
  21 #include <linux/kernel.h>
  22 #include <linux/types.h>
  23 #include <linux/compiler.h>
  24 #include <linux/slab.h>
  25 #include <linux/delay.h>
  26 #include <linux/in.h>
  27 #include <linux/init.h>
  28 #include <linux/ioport.h>
  29 #include <linux/pci.h>
  30 #include <linux/netdevice.h>
  31 #include <linux/etherdevice.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/mii.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/ip.h>
  37 #include <linux/tcp.h>
  38 #include <linux/workqueue.h>
  39 #include <linux/prefetch.h>
  40 #include <linux/dma-mapping.h>
  41
  42 #include <net/checksum.h>
  43 #include <net/ip.h>
  44
  45 #include <asm/system.h>
  46 #include <asm/io.h>
  47 #include <asm/byteorder.h>
  48 #include <asm/uaccess.h>
  49
  50 #ifdef CONFIG_SPARC
  51 #include <asm/idprom.h>
  52 #include <asm/prom.h>
  53 #endif
  54
  55 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
  56 #define TG3_VLAN_TAG_USED 1
  57 #else
  58 #define TG3_VLAN_TAG_USED 0
  59 #endif
  60
  61 #define TG3_TSO_SUPPORT 1
  62
  63 #include "tg3.h"
  64
  65 #define DRV_MODULE_NAME         "tg3"
  66 #define PFX DRV_MODULE_NAME     ": "
  67 #define DRV_MODULE_VERSION      "3.91"
  68 #define DRV_MODULE_RELDATE      "April 18, 2008"
  69
  70 #define TG3_DEF_MAC_MODE        0
  71 #define TG3_DEF_RX_MODE         0
  72 #define TG3_DEF_TX_MODE         0
  73 #define TG3_DEF_MSG_ENABLE        \
  74         (NETIF_MSG_DRV          | \
  75          NETIF_MSG_PROBE        | \
  76          NETIF_MSG_LINK         | \
  77          NETIF_MSG_TIMER        | \
  78          NETIF_MSG_IFDOWN       | \
  79          NETIF_MSG_IFUP         | \
  80          NETIF_MSG_RX_ERR       | \
  81          NETIF_MSG_TX_ERR)
  82
  83 /* length of time before we decide the hardware is borked,
  84  * and dev->tx_timeout() should be called to fix the problem
  85  */
  86 #define TG3_TX_TIMEOUT                  (5 * HZ)
  87
  88 /* hardware minimum and maximum for a single frame's data payload */
  89 #define TG3_MIN_MTU                     60
  90 #define TG3_MAX_MTU(tp) \
  91         ((tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) ? 9000 : 1500)
  92
  93 /* These numbers seem to be hard coded in the NIC firmware somehow.
  94  * You can't change the ring sizes, but you can change where you place
  95  * them in the NIC onboard memory.
  96  */
  97 #define TG3_RX_RING_SIZE                512
  98 #define TG3_DEF_RX_RING_PENDING         200
  99 #define TG3_RX_JUMBO_RING_SIZE          256
 100 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
 101
 102 /* Do not place this n-ring entries value into the tp struct itself,
 103  * we really want to expose these constants to GCC so that modulo et
 104  * al.  operations are done with shifts and masks instead of with
 105  * hw multiply/modulo instructions.  Another solution would be to
 106  * replace things like '% foo' with '& (foo - 1)'.
 107  */
 108 #define TG3_RX_RCB_RING_SIZE(tp)        \
 109         ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ?  512 : 1024)
 110
 111 #define TG3_TX_RING_SIZE                512
 112 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
 113
 114 #define TG3_RX_RING_BYTES       (sizeof(struct tg3_rx_buffer_desc) * \
 115                                  TG3_RX_RING_SIZE)
 116 #define TG3_RX_JUMBO_RING_BYTES (sizeof(struct tg3_rx_buffer_desc) * \
 117                                  TG3_RX_JUMBO_RING_SIZE)
 118 #define TG3_RX_RCB_RING_BYTES(tp) (sizeof(struct tg3_rx_buffer_desc) * \
 119                                    TG3_RX_RCB_RING_SIZE(tp))
 120 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
 121                                  TG3_TX_RING_SIZE)
 122 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
 123
 124 #define RX_PKT_BUF_SZ           (1536 + tp->rx_offset + 64)
 125 #define RX_JUMBO_PKT_BUF_SZ     (9046 + tp->rx_offset + 64)
 126
 127 /* minimum number of free TX descriptors required to wake up TX process */
 128 #define TG3_TX_WAKEUP_THRESH(tp)                ((tp)->tx_pending / 4)
 129
 130 /* number of ETHTOOL_GSTATS u64's */
 131 #define TG3_NUM_STATS           (sizeof(struct tg3_ethtool_stats)/sizeof(u64))
 132
 133 #define TG3_NUM_TEST            6
 134
 135 static char version[] __devinitdata =
 136         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
 137
 138 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
 139 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
 140 MODULE_LICENSE("GPL");
 141 MODULE_VERSION(DRV_MODULE_VERSION);
 142
 143 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
 144 module_param(tg3_debug, int, 0);
 145 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
 146
 147 static struct pci_device_id tg3_pci_tbl[] = {
 148         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
 149         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
 150         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
 151         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
 152         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
 153         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
 154         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
 155         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
 156         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
 157         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
 158         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
 159         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
 160         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
 161         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
 162         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
 163         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
 164         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
 165         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
 166         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901)},
 167         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2)},
 168         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
 169         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F)},
 170         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720)},
 171         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
 172         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
 173         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750)},
 174         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
 175         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M)},
 176         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
 177         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F)},
 178         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
 179         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
 180         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
 181         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
 182         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F)},
 183         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
 184         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
 185         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
 186         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
 187         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
 188         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
 189         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
 190         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
 191         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F)},
 192         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
 193         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
 194         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
 195         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
 196         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
 197         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
 198         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
 199         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
 200         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
 201         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5784)},
 202         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5764)},
 203         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5723)},
 204         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761)},
 205         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761E)},
 206         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
 207         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
 208         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
 209         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
 210         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
 211         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
 212         {PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
 213         {}
 214 };
 215
 216 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
 217
 218 static const struct {
 219         const char string[ETH_GSTRING_LEN];
 220 } ethtool_stats_keys[TG3_NUM_STATS] = {
 221         { "rx_octets" },
 222         { "rx_fragments" },
 223         { "rx_ucast_packets" },
 224         { "rx_mcast_packets" },
 225         { "rx_bcast_packets" },
 226         { "rx_fcs_errors" },
 227         { "rx_align_errors" },
 228         { "rx_xon_pause_rcvd" },
 229         { "rx_xoff_pause_rcvd" },
 230         { "rx_mac_ctrl_rcvd" },
 231         { "rx_xoff_entered" },
 232         { "rx_frame_too_long_errors" },
 233         { "rx_jabbers" },
 234         { "rx_undersize_packets" },
 235         { "rx_in_length_errors" },
 236         { "rx_out_length_errors" },
 237         { "rx_64_or_less_octet_packets" },
 238         { "rx_65_to_127_octet_packets" },
 239         { "rx_128_to_255_octet_packets" },
 240         { "rx_256_to_511_octet_packets" },
 241         { "rx_512_to_1023_octet_packets" },
 242         { "rx_1024_to_1522_octet_packets" },
 243         { "rx_1523_to_2047_octet_packets" },
 244         { "rx_2048_to_4095_octet_packets" },
 245         { "rx_4096_to_8191_octet_packets" },
 246         { "rx_8192_to_9022_octet_packets" },
 247
 248         { "tx_octets" },
 249         { "tx_collisions" },
 250
 251         { "tx_xon_sent" },
 252         { "tx_xoff_sent" },
 253         { "tx_flow_control" },
 254         { "tx_mac_errors" },
 255         { "tx_single_collisions" },
 256         { "tx_mult_collisions" },
 257         { "tx_deferred" },
 258         { "tx_excessive_collisions" },
 259         { "tx_late_collisions" },
 260         { "tx_collide_2times" },
 261         { "tx_collide_3times" },
 262         { "tx_collide_4times" },
 263         { "tx_collide_5times" },
 264         { "tx_collide_6times" },
 265         { "tx_collide_7times" },
 266         { "tx_collide_8times" },
 267         { "tx_collide_9times" },
 268         { "tx_collide_10times" },
 269         { "tx_collide_11times" },
 270         { "tx_collide_12times" },
 271         { "tx_collide_13times" },
 272         { "tx_collide_14times" },
 273         { "tx_collide_15times" },
 274         { "tx_ucast_packets" },
 275         { "tx_mcast_packets" },
 276         { "tx_bcast_packets" },
 277         { "tx_carrier_sense_errors" },
 278         { "tx_discards" },
 279         { "tx_errors" },
 280
 281         { "dma_writeq_full" },
 282         { "dma_write_prioq_full" },
 283         { "rxbds_empty" },
 284         { "rx_discards" },
 285         { "rx_errors" },
 286         { "rx_threshold_hit" },
 287
 288         { "dma_readq_full" },
 289         { "dma_read_prioq_full" },
 290         { "tx_comp_queue_full" },
 291
 292         { "ring_set_send_prod_index" },
 293         { "ring_status_update" },
 294         { "nic_irqs" },
 295         { "nic_avoided_irqs" },
 296         { "nic_tx_threshold_hit" }
 297 };
 298
 299 static const struct {
 300         const char string[ETH_GSTRING_LEN];
 301 } ethtool_test_keys[TG3_NUM_TEST] = {
 302         { "nvram test     (online) " },
 303         { "link test      (online) " },
 304         { "register test  (offline)" },
 305         { "memory test    (offline)" },
 306         { "loopback test  (offline)" },
 307         { "interrupt test (offline)" },
 308 };
 309
 310 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
 311 {
 312         writel(val, tp->regs + off);
 313 }
 314
 315 static u32 tg3_read32(struct tg3 *tp, u32 off)
 316 {
 317         return (readl(tp->regs + off));
 318 }
 319
 320 static void tg3_ape_write32(struct tg3 *tp, u32 off, u32 val)
 321 {
 322         writel(val, tp->aperegs + off);
 323 }
 324
 325 static u32 tg3_ape_read32(struct tg3 *tp, u32 off)
 326 {
 327         return (readl(tp->aperegs + off));
 328 }
 329
 330 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
 331 {
 332         unsigned long flags;
 333
 334         spin_lock_irqsave(&tp->indirect_lock, flags);
 335         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 336         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 337         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 338 }
 339
 340 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
 341 {
 342         writel(val, tp->regs + off);
 343         readl(tp->regs + off);
 344 }
 345
 346 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
 347 {
 348         unsigned long flags;
 349         u32 val;
 350
 351         spin_lock_irqsave(&tp->indirect_lock, flags);
 352         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 353         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 354         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 355         return val;
 356 }
 357
 358 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
 359 {
 360         unsigned long flags;
 361
 362         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
 363                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
 364                                        TG3_64BIT_REG_LOW, val);
 365                 return;
 366         }
 367         if (off == (MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW)) {
 368                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
 369                                        TG3_64BIT_REG_LOW, val);
 370                 return;
 371         }
 372
 373         spin_lock_irqsave(&tp->indirect_lock, flags);
 374         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 375         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 376         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 377
 378         /* In indirect mode when disabling interrupts, we also need
 379          * to clear the interrupt bit in the GRC local ctrl register.
 380          */
 381         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
 382             (val == 0x1)) {
 383                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
 384                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
 385         }
 386 }
 387
 388 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
 389 {
 390         unsigned long flags;
 391         u32 val;
 392
 393         spin_lock_irqsave(&tp->indirect_lock, flags);
 394         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 395         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 396         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 397         return val;
 398 }
 399
 400 /* usec_wait specifies the wait time in usec when writing to certain registers
 401  * where it is unsafe to read back the register without some delay.
 402  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
 403  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
 404  */
 405 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
 406 {
 407         if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) ||
 408             (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 409                 /* Non-posted methods */
 410                 tp->write32(tp, off, val);
 411         else {
 412                 /* Posted method */
 413                 tg3_write32(tp, off, val);
 414                 if (usec_wait)
 415                         udelay(usec_wait);
 416                 tp->read32(tp, off);
 417         }
 418         /* Wait again after the read for the posted method to guarantee that
 419          * the wait time is met.
 420          */
 421         if (usec_wait)
 422                 udelay(usec_wait);
 423 }
 424
 425 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
 426 {
 427         tp->write32_mbox(tp, off, val);
 428         if (!(tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) &&
 429             !(tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 430                 tp->read32_mbox(tp, off);
 431 }
 432
 433 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
 434 {
 435         void __iomem *mbox = tp->regs + off;
 436         writel(val, mbox);
 437         if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
 438                 writel(val, mbox);
 439         if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
 440                 readl(mbox);
 441 }
 442
 443 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
 444 {
 445         return (readl(tp->regs + off + GRCMBOX_BASE));
 446 }
 447
 448 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
 449 {
 450         writel(val, tp->regs + off + GRCMBOX_BASE);
 451 }
 452
 453 #define tw32_mailbox(reg, val)  tp->write32_mbox(tp, reg, val)
 454 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
 455 #define tw32_rx_mbox(reg, val)  tp->write32_rx_mbox(tp, reg, val)
 456 #define tw32_tx_mbox(reg, val)  tp->write32_tx_mbox(tp, reg, val)
 457 #define tr32_mailbox(reg)       tp->read32_mbox(tp, reg)
 458
 459 #define tw32(reg,val)           tp->write32(tp, reg, val)
 460 #define tw32_f(reg,val)         _tw32_flush(tp,(reg),(val), 0)
 461 #define tw32_wait_f(reg,val,us) _tw32_flush(tp,(reg),(val), (us))
 462 #define tr32(reg)               tp->read32(tp, reg)
 463
 464 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
 465 {
 466         unsigned long flags;
 467
 468         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 469             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
 470                 return;
 471
 472         spin_lock_irqsave(&tp->indirect_lock, flags);
 473         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 474                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 475                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 476
 477                 /* Always leave this as zero. */
 478                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 479         } else {
 480                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 481                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
 482
 483                 /* Always leave this as zero. */
 484                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 485         }
 486         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 487 }
 488
 489 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
 490 {
 491         unsigned long flags;
 492
 493         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 494             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
 495                 *val = 0;
 496                 return;
 497         }
 498
 499         spin_lock_irqsave(&tp->indirect_lock, flags);
 500         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 501                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 502                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 503
 504                 /* Always leave this as zero. */
 505                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 506         } else {
 507                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 508                 *val = tr32(TG3PCI_MEM_WIN_DATA);
 509
 510                 /* Always leave this as zero. */
 511                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 512         }
 513         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 514 }
 515
 516 static void tg3_ape_lock_init(struct tg3 *tp)
 517 {
 518         int i;
 519
 520         /* Make sure the driver hasn't any stale locks. */
 521         for (i = 0; i < 8; i++)
 522                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + 4 * i,
 523                                 APE_LOCK_GRANT_DRIVER);
 524 }
 525
 526 static int tg3_ape_lock(struct tg3 *tp, int locknum)
 527 {
 528         int i, off;
 529         int ret = 0;
 530         u32 status;
 531
 532         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
 533                 return 0;
 534
 535         switch (locknum) {
 536                 case TG3_APE_LOCK_MEM:
 537                         break;
 538                 default:
 539                         return -EINVAL;
 540         }
 541
 542         off = 4 * locknum;
 543
 544         tg3_ape_write32(tp, TG3_APE_LOCK_REQ + off, APE_LOCK_REQ_DRIVER);
 545
 546         /* Wait for up to 1 millisecond to acquire lock. */
 547         for (i = 0; i < 100; i++) {
 548                 status = tg3_ape_read32(tp, TG3_APE_LOCK_GRANT + off);
 549                 if (status == APE_LOCK_GRANT_DRIVER)
 550                         break;
 551                 udelay(10);
 552         }
 553
 554         if (status != APE_LOCK_GRANT_DRIVER) {
 555                 /* Revoke the lock request. */
 556                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off,
 557                                 APE_LOCK_GRANT_DRIVER);
 558
 559                 ret = -EBUSY;
 560         }
 561
 562         return ret;
 563 }
 564
 565 static void tg3_ape_unlock(struct tg3 *tp, int locknum)
 566 {
 567         int off;
 568
 569         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
 570                 return;
 571
 572         switch (locknum) {
 573                 case TG3_APE_LOCK_MEM:
 574                         break;
 575                 default:
 576                         return;
 577         }
 578
 579         off = 4 * locknum;
 580         tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off, APE_LOCK_GRANT_DRIVER);
 581 }
 582
 583 static void tg3_disable_ints(struct tg3 *tp)
 584 {
 585         tw32(TG3PCI_MISC_HOST_CTRL,
 586              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
 587         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
 588 }
 589
 590 static inline void tg3_cond_int(struct tg3 *tp)
 591 {
 592         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 593             (tp->hw_status->status & SD_STATUS_UPDATED))
 594                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
 595         else
 596                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 597                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
 598 }
 599
 600 static void tg3_enable_ints(struct tg3 *tp)
 601 {
 602         tp->irq_sync = 0;
 603         wmb();
 604
 605         tw32(TG3PCI_MISC_HOST_CTRL,
 606              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
 607         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 608                        (tp->last_tag << 24));
 609         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
 610                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 611                                (tp->last_tag << 24));
 612         tg3_cond_int(tp);
 613 }
 614
 615 static inline unsigned int tg3_has_work(struct tg3 *tp)
 616 {
 617         struct tg3_hw_status *sblk = tp->hw_status;
 618         unsigned int work_exists = 0;
 619
 620         /* check for phy events */
 621         if (!(tp->tg3_flags &
 622               (TG3_FLAG_USE_LINKCHG_REG |
 623                TG3_FLAG_POLL_SERDES))) {
 624                 if (sblk->status & SD_STATUS_LINK_CHG)
 625                         work_exists = 1;
 626         }
 627         /* check for RX/TX work to do */
 628         if (sblk->idx[0].tx_consumer != tp->tx_cons ||
 629             sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
 630                 work_exists = 1;
 631
 632         return work_exists;
 633 }
 634
 635 /* tg3_restart_ints
 636  *  similar to tg3_enable_ints, but it accurately determines whether there
 637  *  is new work pending and can return without flushing the PIO write
 638  *  which reenables interrupts
 639  */
 640 static void tg3_restart_ints(struct tg3 *tp)
 641 {
 642         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 643                      tp->last_tag << 24);
 644         mmiowb();
 645
 646         /* When doing tagged status, this work check is unnecessary.
 647          * The last_tag we write above tells the chip which piece of
 648          * work we've completed.
 649          */
 650         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 651             tg3_has_work(tp))
 652                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 653                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
 654 }
 655
 656 static inline void tg3_netif_stop(struct tg3 *tp)
 657 {
 658         tp->dev->trans_start = jiffies; /* prevent tx timeout */
 659         napi_disable(&tp->napi);
 660         netif_tx_disable(tp->dev);
 661 }
 662
 663 static inline void tg3_netif_start(struct tg3 *tp)
 664 {
 665         netif_wake_queue(tp->dev);
 666         /* NOTE: unconditional netif_wake_queue is only appropriate
 667          * so long as all callers are assured to have free tx slots
 668          * (such as after tg3_init_hw)
 669          */
 670         napi_enable(&tp->napi);
 671         tp->hw_status->status |= SD_STATUS_UPDATED;
 672         tg3_enable_ints(tp);
 673 }
 674
 675 static void tg3_switch_clocks(struct tg3 *tp)
 676 {
 677         u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
 678         u32 orig_clock_ctrl;
 679
 680         if ((tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
 681             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
 682                 return;
 683
 684         orig_clock_ctrl = clock_ctrl;
 685         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
 686                        CLOCK_CTRL_CLKRUN_OENABLE |
 687                        0x1f);
 688         tp->pci_clock_ctrl = clock_ctrl;
 689
 690         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
 691                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
 692                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
 693                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
 694                 }
 695         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
 696                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 697                             clock_ctrl |
 698                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
 699                             40);
 700                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 701                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
 702                             40);
 703         }
 704         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
 705 }
 706
 707 #define PHY_BUSY_LOOPS  5000
 708
 709 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
 710 {
 711         u32 frame_val;
 712         unsigned int loops;
 713         int ret;
 714
 715         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 716                 tw32_f(MAC_MI_MODE,
 717                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 718                 udelay(80);
 719         }
 720
 721         *val = 0x0;
 722
 723         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 724                       MI_COM_PHY_ADDR_MASK);
 725         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 726                       MI_COM_REG_ADDR_MASK);
 727         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
 728
 729         tw32_f(MAC_MI_COM, frame_val);
 730
 731         loops = PHY_BUSY_LOOPS;
 732         while (loops != 0) {
 733                 udelay(10);
 734                 frame_val = tr32(MAC_MI_COM);
 735
 736                 if ((frame_val & MI_COM_BUSY) == 0) {
 737                         udelay(5);
 738                         frame_val = tr32(MAC_MI_COM);
 739                         break;
 740                 }
 741                 loops -= 1;
 742         }
 743
 744         ret = -EBUSY;
 745         if (loops != 0) {
 746                 *val = frame_val & MI_COM_DATA_MASK;
 747                 ret = 0;
 748         }
 749
 750         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 751                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 752                 udelay(80);
 753         }
 754
 755         return ret;
 756 }
 757
 758 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
 759 {
 760         u32 frame_val;
 761         unsigned int loops;
 762         int ret;
 763
 764         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 &&
 765             (reg == MII_TG3_CTRL || reg == MII_TG3_AUX_CTRL))
 766                 return 0;
 767
 768         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 769                 tw32_f(MAC_MI_MODE,
 770                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 771                 udelay(80);
 772         }
 773
 774         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 775                       MI_COM_PHY_ADDR_MASK);
 776         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 777                       MI_COM_REG_ADDR_MASK);
 778         frame_val |= (val & MI_COM_DATA_MASK);
 779         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
 780
 781         tw32_f(MAC_MI_COM, frame_val);
 782
 783         loops = PHY_BUSY_LOOPS;
 784         while (loops != 0) {
 785                 udelay(10);
 786                 frame_val = tr32(MAC_MI_COM);
 787                 if ((frame_val & MI_COM_BUSY) == 0) {
 788                         udelay(5);
 789                         frame_val = tr32(MAC_MI_COM);
 790                         break;
 791                 }
 792                 loops -= 1;
 793         }
 794
 795         ret = -EBUSY;
 796         if (loops != 0)
 797                 ret = 0;
 798
 799         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 800                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 801                 udelay(80);
 802         }
 803
 804         return ret;
 805 }
 806
 807 static void tg3_phydsp_write(struct tg3 *tp, u32 reg, u32 val)
 808 {
 809         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
 810         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, val);
 811 }
 812
 813 static void tg3_phy_toggle_automdix(struct tg3 *tp, int enable)
 814 {
 815         u32 phy;
 816
 817         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
 818             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
 819                 return;
 820
 821         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
 822                 u32 ephy;
 823
 824                 if (!tg3_readphy(tp, MII_TG3_EPHY_TEST, &ephy)) {
 825                         tg3_writephy(tp, MII_TG3_EPHY_TEST,
 826                                      ephy | MII_TG3_EPHY_SHADOW_EN);
 827                         if (!tg3_readphy(tp, MII_TG3_EPHYTST_MISCCTRL, &phy)) {
 828                                 if (enable)
 829                                         phy |= MII_TG3_EPHYTST_MISCCTRL_MDIX;
 830                                 else
 831                                         phy &= ~MII_TG3_EPHYTST_MISCCTRL_MDIX;
 832                                 tg3_writephy(tp, MII_TG3_EPHYTST_MISCCTRL, phy);
 833                         }
 834                         tg3_writephy(tp, MII_TG3_EPHY_TEST, ephy);
 835                 }
 836         } else {
 837                 phy = MII_TG3_AUXCTL_MISC_RDSEL_MISC |
 838                       MII_TG3_AUXCTL_SHDWSEL_MISC;
 839                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, phy) &&
 840                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy)) {
 841                         if (enable)
 842                                 phy |= MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
 843                         else
 844                                 phy &= ~MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
 845                         phy |= MII_TG3_AUXCTL_MISC_WREN;
 846                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
 847                 }
 848         }
 849 }
 850
 851 static void tg3_phy_set_wirespeed(struct tg3 *tp)
 852 {
 853         u32 val;
 854
 855         if (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED)
 856                 return;
 857
 858         if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x7007) &&
 859             !tg3_readphy(tp, MII_TG3_AUX_CTRL, &val))
 860                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
 861                              (val | (1 << 15) | (1 << 4)));
 862 }
 863
 864 static int tg3_bmcr_reset(struct tg3 *tp)
 865 {
 866         u32 phy_control;
 867         int limit, err;
 868
 869         /* OK, reset it, and poll the BMCR_RESET bit until it
 870          * clears or we time out.
 871          */
 872         phy_control = BMCR_RESET;
 873         err = tg3_writephy(tp, MII_BMCR, phy_control);
 874         if (err != 0)
 875                 return -EBUSY;
 876
 877         limit = 5000;
 878         while (limit--) {
 879                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
 880                 if (err != 0)
 881                         return -EBUSY;
 882
 883                 if ((phy_control & BMCR_RESET) == 0) {
 884                         udelay(40);
 885                         break;
 886                 }
 887                 udelay(10);
 888         }
 889         if (limit <= 0)
 890                 return -EBUSY;
 891
 892         return 0;
 893 }
 894
 895 static void tg3_phy_apply_otp(struct tg3 *tp)
 896 {
 897         u32 otp, phy;
 898
 899         if (!tp->phy_otp)
 900                 return;
 901
 902         otp = tp->phy_otp;
 903
 904         /* Enable SM_DSP clock and tx 6dB coding. */
 905         phy = MII_TG3_AUXCTL_SHDWSEL_AUXCTL |
 906               MII_TG3_AUXCTL_ACTL_SMDSP_ENA |
 907               MII_TG3_AUXCTL_ACTL_TX_6DB;
 908         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
 909
 910         phy = ((otp & TG3_OTP_AGCTGT_MASK) >> TG3_OTP_AGCTGT_SHIFT);
 911         phy |= MII_TG3_DSP_TAP1_AGCTGT_DFLT;
 912         tg3_phydsp_write(tp, MII_TG3_DSP_TAP1, phy);
 913
 914         phy = ((otp & TG3_OTP_HPFFLTR_MASK) >> TG3_OTP_HPFFLTR_SHIFT) |
 915               ((otp & TG3_OTP_HPFOVER_MASK) >> TG3_OTP_HPFOVER_SHIFT);
 916         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH0, phy);
 917
 918         phy = ((otp & TG3_OTP_LPFDIS_MASK) >> TG3_OTP_LPFDIS_SHIFT);
 919         phy |= MII_TG3_DSP_AADJ1CH3_ADCCKADJ;
 920         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH3, phy);
 921
 922         phy = ((otp & TG3_OTP_VDAC_MASK) >> TG3_OTP_VDAC_SHIFT);
 923         tg3_phydsp_write(tp, MII_TG3_DSP_EXP75, phy);
 924
 925         phy = ((otp & TG3_OTP_10BTAMP_MASK) >> TG3_OTP_10BTAMP_SHIFT);
 926         tg3_phydsp_write(tp, MII_TG3_DSP_EXP96, phy);
 927
 928         phy = ((otp & TG3_OTP_ROFF_MASK) >> TG3_OTP_ROFF_SHIFT) |
 929               ((otp & TG3_OTP_RCOFF_MASK) >> TG3_OTP_RCOFF_SHIFT);
 930         tg3_phydsp_write(tp, MII_TG3_DSP_EXP97, phy);
 931
 932         /* Turn off SM_DSP clock. */
 933         phy = MII_TG3_AUXCTL_SHDWSEL_AUXCTL |
 934               MII_TG3_AUXCTL_ACTL_TX_6DB;
 935         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
 936 }
 937
 938 static int tg3_wait_macro_done(struct tg3 *tp)
 939 {
 940         int limit = 100;
 941
 942         while (limit--) {
 943                 u32 tmp32;
 944
 945                 if (!tg3_readphy(tp, 0x16, &tmp32)) {
 946                         if ((tmp32 & 0x1000) == 0)
 947                                 break;
 948                 }
 949         }
 950         if (limit <= 0)
 951                 return -EBUSY;
 952
 953         return 0;
 954 }
 955
 956 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
 957 {
 958         static const u32 test_pat[4][6] = {
 959         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
 960         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
 961         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
 962         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
 963         };
 964         int chan;
 965
 966         for (chan = 0; chan < 4; chan++) {
 967                 int i;
 968
 969                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 970                              (chan * 0x2000) | 0x0200);
 971                 tg3_writephy(tp, 0x16, 0x0002);
 972
 973                 for (i = 0; i < 6; i++)
 974                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
 975                                      test_pat[chan][i]);
 976
 977                 tg3_writephy(tp, 0x16, 0x0202);
 978                 if (tg3_wait_macro_done(tp)) {
 979                         *resetp = 1;
 980                         return -EBUSY;
 981                 }
 982
 983                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 984                              (chan * 0x2000) | 0x0200);
 985                 tg3_writephy(tp, 0x16, 0x0082);
 986                 if (tg3_wait_macro_done(tp)) {
 987                         *resetp = 1;
 988                         return -EBUSY;
 989                 }
 990
 991                 tg3_writephy(tp, 0x16, 0x0802);
 992                 if (tg3_wait_macro_done(tp)) {
 993                         *resetp = 1;
 994                         return -EBUSY;
 995                 }
 996
 997                 for (i = 0; i < 6; i += 2) {
 998                         u32 low, high;
 999
1000                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
1001                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
1002                             tg3_wait_macro_done(tp)) {
1003                                 *resetp = 1;
1004                                 return -EBUSY;
1005                         }
1006                         low &= 0x7fff;
1007                         high &= 0x000f;
1008                         if (low != test_pat[chan][i] ||
1009                             high != test_pat[chan][i+1]) {
1010                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
1011                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
1012                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
1013
1014                                 return -EBUSY;
1015                         }
1016                 }
1017         }
1018
1019         return 0;
1020 }
1021
1022 static int tg3_phy_reset_chanpat(struct tg3 *tp)
1023 {
1024         int chan;
1025
1026         for (chan = 0; chan < 4; chan++) {
1027                 int i;
1028
1029                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1030                              (chan * 0x2000) | 0x0200);
1031                 tg3_writephy(tp, 0x16, 0x0002);
1032                 for (i = 0; i < 6; i++)
1033                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
1034                 tg3_writephy(tp, 0x16, 0x0202);
1035                 if (tg3_wait_macro_done(tp))
1036                         return -EBUSY;
1037         }
1038
1039         return 0;
1040 }
1041
1042 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
1043 {
1044         u32 reg32, phy9_orig;
1045         int retries, do_phy_reset, err;
1046
1047         retries = 10;
1048         do_phy_reset = 1;
1049         do {
1050                 if (do_phy_reset) {
1051                         err = tg3_bmcr_reset(tp);
1052                         if (err)
1053                                 return err;
1054                         do_phy_reset = 0;
1055                 }
1056
1057                 /* Disable transmitter and interrupt.  */
1058                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
1059                         continue;
1060
1061                 reg32 |= 0x3000;
1062                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1063
1064                 /* Set full-duplex, 1000 mbps.  */
1065                 tg3_writephy(tp, MII_BMCR,
1066                              BMCR_FULLDPLX | TG3_BMCR_SPEED1000);
1067
1068                 /* Set to master mode.  */
1069                 if (tg3_readphy(tp, MII_TG3_CTRL, &phy9_orig))
1070                         continue;
1071
1072                 tg3_writephy(tp, MII_TG3_CTRL,
1073                              (MII_TG3_CTRL_AS_MASTER |
1074                               MII_TG3_CTRL_ENABLE_AS_MASTER));
1075
1076                 /* Enable SM_DSP_CLOCK and 6dB.  */
1077                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1078
1079                 /* Block the PHY control access.  */
1080                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1081                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0800);
1082
1083                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
1084                 if (!err)
1085                         break;
1086         } while (--retries);
1087
1088         err = tg3_phy_reset_chanpat(tp);
1089         if (err)
1090                 return err;
1091
1092         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1093         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0000);
1094
1095         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
1096         tg3_writephy(tp, 0x16, 0x0000);
1097
1098         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1099             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
1100                 /* Set Extended packet length bit for jumbo frames */
1101                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4400);
1102         }
1103         else {
1104                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1105         }
1106
1107         tg3_writephy(tp, MII_TG3_CTRL, phy9_orig);
1108
1109         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
1110                 reg32 &= ~0x3000;
1111                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1112         } else if (!err)
1113                 err = -EBUSY;
1114
1115         return err;
1116 }
1117
1118 static void tg3_link_report(struct tg3 *);
1119
1120 /* This will reset the tigon3 PHY if there is no valid
1121  * link unless the FORCE argument is non-zero.
1122  */
1123 static int tg3_phy_reset(struct tg3 *tp)
1124 {
1125         u32 cpmuctrl;
1126         u32 phy_status;
1127         int err;
1128
1129         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1130                 u32 val;
1131
1132                 val = tr32(GRC_MISC_CFG);
1133                 tw32_f(GRC_MISC_CFG, val & ~GRC_MISC_CFG_EPHY_IDDQ);
1134                 udelay(40);
1135         }
1136         err  = tg3_readphy(tp, MII_BMSR, &phy_status);
1137         err |= tg3_readphy(tp, MII_BMSR, &phy_status);
1138         if (err != 0)
1139                 return -EBUSY;
1140
1141         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
1142                 netif_carrier_off(tp->dev);
1143                 tg3_link_report(tp);
1144         }
1145
1146         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1147             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1148             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
1149                 err = tg3_phy_reset_5703_4_5(tp);
1150                 if (err)
1151                         return err;
1152                 goto out;
1153         }
1154
1155         cpmuctrl = 0;
1156         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
1157             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) {
1158                 cpmuctrl = tr32(TG3_CPMU_CTRL);
1159                 if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY)
1160                         tw32(TG3_CPMU_CTRL,
1161                              cpmuctrl & ~CPMU_CTRL_GPHY_10MB_RXONLY);
1162         }
1163
1164         err = tg3_bmcr_reset(tp);
1165         if (err)
1166                 return err;
1167
1168         if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY) {
1169                 u32 phy;
1170
1171                 phy = MII_TG3_DSP_EXP8_AEDW | MII_TG3_DSP_EXP8_REJ2MHz;
1172                 tg3_phydsp_write(tp, MII_TG3_DSP_EXP8, phy);
1173
1174                 tw32(TG3_CPMU_CTRL, cpmuctrl);
1175         }
1176
1177         if (tp->tg3_flags3 & TG3_FLG3_5761_5784_AX_FIXES) {
1178                 u32 val;
1179
1180                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
1181                 if ((val & CPMU_LSPD_1000MB_MACCLK_MASK) ==
1182                     CPMU_LSPD_1000MB_MACCLK_12_5) {
1183                         val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
1184                         udelay(40);
1185                         tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
1186                 }
1187
1188                 /* Disable GPHY autopowerdown. */
1189                 tg3_writephy(tp, MII_TG3_MISC_SHDW,
1190                              MII_TG3_MISC_SHDW_WREN |
1191                              MII_TG3_MISC_SHDW_APD_SEL |
1192                              MII_TG3_MISC_SHDW_APD_WKTM_84MS);
1193         }
1194
1195         tg3_phy_apply_otp(tp);
1196
1197 out:
1198         if (tp->tg3_flags2 & TG3_FLG2_PHY_ADC_BUG) {
1199                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1200                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1201                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x2aaa);
1202                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1203                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0323);
1204                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1205         }
1206         if (tp->tg3_flags2 & TG3_FLG2_PHY_5704_A0_BUG) {
1207                 tg3_writephy(tp, 0x1c, 0x8d68);
1208                 tg3_writephy(tp, 0x1c, 0x8d68);
1209         }
1210         if (tp->tg3_flags2 & TG3_FLG2_PHY_BER_BUG) {
1211                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1212                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1213                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x310b);
1214                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1215                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x9506);
1216                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x401f);
1217                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x14e2);
1218                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1219         }
1220         else if (tp->tg3_flags2 & TG3_FLG2_PHY_JITTER_BUG) {
1221                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1222                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1223                 if (tp->tg3_flags2 & TG3_FLG2_PHY_ADJUST_TRIM) {
1224                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x110b);
1225                         tg3_writephy(tp, MII_TG3_TEST1,
1226                                      MII_TG3_TEST1_TRIM_EN | 0x4);
1227                 } else
1228                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
1229                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1230         }
1231         /* Set Extended packet length bit (bit 14) on all chips that */
1232         /* support jumbo frames */
1233         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1234                 /* Cannot do read-modify-write on 5401 */
1235                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1236         } else if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1237                 u32 phy_reg;
1238
1239                 /* Set bit 14 with read-modify-write to preserve other bits */
1240                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0007) &&
1241                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy_reg))
1242                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy_reg | 0x4000);
1243         }
1244
1245         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
1246          * jumbo frames transmission.
1247          */
1248         if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1249                 u32 phy_reg;
1250
1251                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &phy_reg))
1252                     tg3_writephy(tp, MII_TG3_EXT_CTRL,
1253                                  phy_reg | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
1254         }
1255
1256         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1257                 /* adjust output voltage */
1258                 tg3_writephy(tp, MII_TG3_EPHY_PTEST, 0x12);
1259         }
1260
1261         tg3_phy_toggle_automdix(tp, 1);
1262         tg3_phy_set_wirespeed(tp);
1263         return 0;
1264 }
1265
1266 static void tg3_frob_aux_power(struct tg3 *tp)
1267 {
1268         struct tg3 *tp_peer = tp;
1269
1270         if ((tp->tg3_flags2 & TG3_FLG2_IS_NIC) == 0)
1271                 return;
1272
1273         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
1274             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
1275                 struct net_device *dev_peer;
1276
1277                 dev_peer = pci_get_drvdata(tp->pdev_peer);
1278                 /* remove_one() may have been run on the peer. */
1279                 if (!dev_peer)
1280                         tp_peer = tp;
1281                 else
1282                         tp_peer = netdev_priv(dev_peer);
1283         }
1284
1285         if ((tp->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1286             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0 ||
1287             (tp_peer->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1288             (tp_peer->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
1289                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1290                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1291                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1292                                     (GRC_LCLCTRL_GPIO_OE0 |
1293                                      GRC_LCLCTRL_GPIO_OE1 |
1294                                      GRC_LCLCTRL_GPIO_OE2 |
1295                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
1296                                      GRC_LCLCTRL_GPIO_OUTPUT1),
1297                                     100);
1298                 } else {
1299                         u32 no_gpio2;
1300                         u32 grc_local_ctrl = 0;
1301
1302                         if (tp_peer != tp &&
1303                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1304                                 return;
1305
1306                         /* Workaround to prevent overdrawing Amps. */
1307                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
1308                             ASIC_REV_5714) {
1309                                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
1310                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1311                                             grc_local_ctrl, 100);
1312                         }
1313
1314                         /* On 5753 and variants, GPIO2 cannot be used. */
1315                         no_gpio2 = tp->nic_sram_data_cfg &
1316                                     NIC_SRAM_DATA_CFG_NO_GPIO2;
1317
1318                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
1319                                          GRC_LCLCTRL_GPIO_OE1 |
1320                                          GRC_LCLCTRL_GPIO_OE2 |
1321                                          GRC_LCLCTRL_GPIO_OUTPUT1 |
1322                                          GRC_LCLCTRL_GPIO_OUTPUT2;
1323                         if (no_gpio2) {
1324                                 grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
1325                                                     GRC_LCLCTRL_GPIO_OUTPUT2);
1326                         }
1327                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1328                                                     grc_local_ctrl, 100);
1329
1330                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
1331
1332                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1333                                                     grc_local_ctrl, 100);
1334
1335                         if (!no_gpio2) {
1336                                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
1337                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1338                                             grc_local_ctrl, 100);
1339                         }
1340                 }
1341         } else {
1342                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
1343                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
1344                         if (tp_peer != tp &&
1345                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1346                                 return;
1347
1348                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1349                                     (GRC_LCLCTRL_GPIO_OE1 |
1350                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1351
1352                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1353                                     GRC_LCLCTRL_GPIO_OE1, 100);
1354
1355                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1356                                     (GRC_LCLCTRL_GPIO_OE1 |
1357                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1358                 }
1359         }
1360 }
1361
1362 static int tg3_5700_link_polarity(struct tg3 *tp, u32 speed)
1363 {
1364         if (tp->led_ctrl == LED_CTRL_MODE_PHY_2)
1365                 return 1;
1366         else if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411) {
1367                 if (speed != SPEED_10)
1368                         return 1;
1369         } else if (speed == SPEED_10)
1370                 return 1;
1371
1372         return 0;
1373 }
1374
1375 static int tg3_setup_phy(struct tg3 *, int);
1376
1377 #define RESET_KIND_SHUTDOWN     0
1378 #define RESET_KIND_INIT         1
1379 #define RESET_KIND_SUSPEND      2
1380
1381 static void tg3_write_sig_post_reset(struct tg3 *, int);
1382 static int tg3_halt_cpu(struct tg3 *, u32);
1383 static int tg3_nvram_lock(struct tg3 *);
1384 static void tg3_nvram_unlock(struct tg3 *);
1385
1386 static void tg3_power_down_phy(struct tg3 *tp)
1387 {
1388         u32 val;
1389
1390         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
1391                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
1392                         u32 sg_dig_ctrl = tr32(SG_DIG_CTRL);
1393                         u32 serdes_cfg = tr32(MAC_SERDES_CFG);
1394
1395                         sg_dig_ctrl |=
1396                                 SG_DIG_USING_HW_AUTONEG | SG_DIG_SOFT_RESET;
1397                         tw32(SG_DIG_CTRL, sg_dig_ctrl);
1398                         tw32(MAC_SERDES_CFG, serdes_cfg | (1 << 15));
1399                 }
1400                 return;
1401         }
1402
1403         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1404                 tg3_bmcr_reset(tp);
1405                 val = tr32(GRC_MISC_CFG);
1406                 tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ);
1407                 udelay(40);
1408                 return;
1409         } else {
1410                 tg3_writephy(tp, MII_TG3_EXT_CTRL,
1411                              MII_TG3_EXT_CTRL_FORCE_LED_OFF);
1412                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x01b2);
1413         }
1414
1415         /* The PHY should not be powered down on some chips because
1416          * of bugs.
1417          */
1418         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1419             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1420             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
1421              (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)))
1422                 return;
1423
1424         if (tp->tg3_flags3 & TG3_FLG3_5761_5784_AX_FIXES) {
1425                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
1426                 val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
1427                 val |= CPMU_LSPD_1000MB_MACCLK_12_5;
1428                 tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
1429         }
1430
1431         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
1432 }
1433
1434 static int tg3_set_power_state(struct tg3 *tp, pci_power_t state)
1435 {
1436         u32 misc_host_ctrl;
1437         u16 power_control, power_caps;
1438         int pm = tp->pm_cap;
1439
1440         /* Make sure register accesses (indirect or otherwise)
1441          * will function correctly.
1442          */
1443         pci_write_config_dword(tp->pdev,
1444                                TG3PCI_MISC_HOST_CTRL,
1445                                tp->misc_host_ctrl);
1446
1447         pci_read_config_word(tp->pdev,
1448                              pm + PCI_PM_CTRL,
1449                              &power_control);
1450         power_control |= PCI_PM_CTRL_PME_STATUS;
1451         power_control &= ~(PCI_PM_CTRL_STATE_MASK);
1452         switch (state) {
1453         case PCI_D0:
1454                 power_control |= 0;
1455                 pci_write_config_word(tp->pdev,
1456                                       pm + PCI_PM_CTRL,
1457                                       power_control);
1458                 udelay(100);    /* Delay after power state change */
1459
1460                 /* Switch out of Vaux if it is a NIC */
1461                 if (tp->tg3_flags2 & TG3_FLG2_IS_NIC)
1462                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl, 100);
1463
1464                 return 0;
1465
1466         case PCI_D1:
1467                 power_control |= 1;
1468                 break;
1469
1470         case PCI_D2:
1471                 power_control |= 2;
1472                 break;
1473
1474         case PCI_D3hot:
1475                 power_control |= 3;
1476                 break;
1477
1478         default:
1479                 printk(KERN_WARNING PFX "%s: Invalid power state (%d) "
1480                        "requested.\n",
1481                        tp->dev->name, state);
1482                 return -EINVAL;
1483         };
1484
1485         power_control |= PCI_PM_CTRL_PME_ENABLE;
1486
1487         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
1488         tw32(TG3PCI_MISC_HOST_CTRL,
1489              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
1490
1491         if (tp->link_config.phy_is_low_power == 0) {
1492                 tp->link_config.phy_is_low_power = 1;
1493                 tp->link_config.orig_speed = tp->link_config.speed;
1494                 tp->link_config.orig_duplex = tp->link_config.duplex;
1495                 tp->link_config.orig_autoneg = tp->link_config.autoneg;
1496         }
1497
1498         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
1499                 tp->link_config.speed = SPEED_10;
1500                 tp->link_config.duplex = DUPLEX_HALF;
1501                 tp->link_config.autoneg = AUTONEG_ENABLE;
1502                 tg3_setup_phy(tp, 0);
1503         }
1504
1505         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1506                 u32 val;
1507
1508                 val = tr32(GRC_VCPU_EXT_CTRL);
1509                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
1510         } else if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1511                 int i;
1512                 u32 val;
1513
1514                 for (i = 0; i < 200; i++) {
1515                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
1516                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
1517                                 break;
1518                         msleep(1);
1519                 }
1520         }
1521         if (tp->tg3_flags & TG3_FLAG_WOL_CAP)
1522                 tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
1523                                                      WOL_DRV_STATE_SHUTDOWN |
1524                                                      WOL_DRV_WOL |
1525                                                      WOL_SET_MAGIC_PKT);
1526
1527         pci_read_config_word(tp->pdev, pm + PCI_PM_PMC, &power_caps);
1528
1529         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE) {
1530                 u32 mac_mode;
1531
1532                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
1533                         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
1534                         udelay(40);
1535
1536                         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
1537                                 mac_mode = MAC_MODE_PORT_MODE_GMII;
1538                         else
1539                                 mac_mode = MAC_MODE_PORT_MODE_MII;
1540
1541                         mac_mode |= tp->mac_mode & MAC_MODE_LINK_POLARITY;
1542                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
1543                             ASIC_REV_5700) {
1544                                 u32 speed = (tp->tg3_flags &
1545                                              TG3_FLAG_WOL_SPEED_100MB) ?
1546                                              SPEED_100 : SPEED_10;
1547                                 if (tg3_5700_link_polarity(tp, speed))
1548                                         mac_mode |= MAC_MODE_LINK_POLARITY;
1549                                 else
1550                                         mac_mode &= ~MAC_MODE_LINK_POLARITY;
1551                         }
1552                 } else {
1553                         mac_mode = MAC_MODE_PORT_MODE_TBI;
1554                 }
1555
1556                 if (!(tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
1557                         tw32(MAC_LED_CTRL, tp->led_ctrl);
1558
1559                 if (((power_caps & PCI_PM_CAP_PME_D3cold) &&
1560                      (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)))
1561                         mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
1562
1563                 tw32_f(MAC_MODE, mac_mode);
1564                 udelay(100);
1565
1566                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
1567                 udelay(10);
1568         }
1569
1570         if (!(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB) &&
1571             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1572              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
1573                 u32 base_val;
1574
1575                 base_val = tp->pci_clock_ctrl;
1576                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
1577                              CLOCK_CTRL_TXCLK_DISABLE);
1578
1579                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
1580                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
1581         } else if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) ||
1582                    (tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
1583                    (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)) {
1584                 /* do nothing */
1585         } else if (!((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
1586                      (tp->tg3_flags & TG3_FLAG_ENABLE_ASF))) {
1587                 u32 newbits1, newbits2;
1588
1589                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1590                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1591                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
1592                                     CLOCK_CTRL_TXCLK_DISABLE |
1593                                     CLOCK_CTRL_ALTCLK);
1594                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1595                 } else if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
1596                         newbits1 = CLOCK_CTRL_625_CORE;
1597                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
1598                 } else {
1599                         newbits1 = CLOCK_CTRL_ALTCLK;
1600                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1601                 }
1602
1603                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
1604                             40);
1605
1606                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
1607                             40);
1608
1609                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
1610                         u32 newbits3;
1611
1612                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1613                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1614                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
1615                                             CLOCK_CTRL_TXCLK_DISABLE |
1616                                             CLOCK_CTRL_44MHZ_CORE);
1617                         } else {
1618                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
1619                         }
1620
1621                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
1622                                     tp->pci_clock_ctrl | newbits3, 40);
1623                 }
1624         }
1625
1626         if (!(tp->tg3_flags & TG3_FLAG_WOL_ENABLE) &&
1627             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF) &&
1628             !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
1629                 tg3_power_down_phy(tp);
1630
1631         tg3_frob_aux_power(tp);
1632
1633         /* Workaround for unstable PLL clock */
1634         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
1635             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
1636                 u32 val = tr32(0x7d00);
1637
1638                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
1639                 tw32(0x7d00, val);
1640                 if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1641                         int err;
1642
1643                         err = tg3_nvram_lock(tp);
1644                         tg3_halt_cpu(tp, RX_CPU_BASE);
1645                         if (!err)
1646                                 tg3_nvram_unlock(tp);
1647                 }
1648         }
1649
1650         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
1651
1652         /* Finally, set the new power state. */
1653         pci_write_config_word(tp->pdev, pm + PCI_PM_CTRL, power_control);
1654         udelay(100);    /* Delay after power state change */
1655
1656         return 0;
1657 }
1658
1659 static void tg3_link_report(struct tg3 *tp)
1660 {
1661         if (!netif_carrier_ok(tp->dev)) {
1662                 if (netif_msg_link(tp))
1663                         printk(KERN_INFO PFX "%s: Link is down.\n",
1664                                tp->dev->name);
1665         } else if (netif_msg_link(tp)) {
1666                 printk(KERN_INFO PFX "%s: Link is up at %d Mbps, %s duplex.\n",
1667                        tp->dev->name,
1668                        (tp->link_config.active_speed == SPEED_1000 ?
1669                         1000 :
1670                         (tp->link_config.active_speed == SPEED_100 ?
1671                          100 : 10)),
1672                        (tp->link_config.active_duplex == DUPLEX_FULL ?
1673                         "full" : "half"));
1674
1675                 printk(KERN_INFO PFX
1676                        "%s: Flow control is %s for TX and %s for RX.\n",
1677                        tp->dev->name,
1678                        (tp->link_config.active_flowctrl & TG3_FLOW_CTRL_TX) ?
1679                        "on" : "off",
1680                        (tp->link_config.active_flowctrl & TG3_FLOW_CTRL_RX) ?
1681                        "on" : "off");
1682         }
1683 }
1684
1685 static u16 tg3_advert_flowctrl_1000T(u8 flow_ctrl)
1686 {
1687         u16 miireg;
1688
1689         if ((flow_ctrl & TG3_FLOW_CTRL_TX) && (flow_ctrl & TG3_FLOW_CTRL_RX))
1690                 miireg = ADVERTISE_PAUSE_CAP;
1691         else if (flow_ctrl & TG3_FLOW_CTRL_TX)
1692                 miireg = ADVERTISE_PAUSE_ASYM;
1693         else if (flow_ctrl & TG3_FLOW_CTRL_RX)
1694                 miireg = ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
1695         else
1696                 miireg = 0;
1697
1698         return miireg;
1699 }
1700
1701 static u16 tg3_advert_flowctrl_1000X(u8 flow_ctrl)
1702 {
1703         u16 miireg;
1704
1705         if ((flow_ctrl & TG3_FLOW_CTRL_TX) && (flow_ctrl & TG3_FLOW_CTRL_RX))
1706                 miireg = ADVERTISE_1000XPAUSE;
1707         else if (flow_ctrl & TG3_FLOW_CTRL_TX)
1708                 miireg = ADVERTISE_1000XPSE_ASYM;
1709         else if (flow_ctrl & TG3_FLOW_CTRL_RX)
1710                 miireg = ADVERTISE_1000XPAUSE | ADVERTISE_1000XPSE_ASYM;
1711         else
1712                 miireg = 0;
1713
1714         return miireg;
1715 }
1716
1717 static u8 tg3_resolve_flowctrl_1000T(u16 lcladv, u16 rmtadv)
1718 {
1719         u8 cap = 0;
1720
1721         if (lcladv & ADVERTISE_PAUSE_CAP) {
1722                 if (lcladv & ADVERTISE_PAUSE_ASYM) {
1723                         if (rmtadv & LPA_PAUSE_CAP)
1724                                 cap = TG3_FLOW_CTRL_TX | TG3_FLOW_CTRL_RX;
1725                         else if (rmtadv & LPA_PAUSE_ASYM)
1726                                 cap = TG3_FLOW_CTRL_RX;
1727                 } else {
1728                         if (rmtadv & LPA_PAUSE_CAP)
1729                                 cap = TG3_FLOW_CTRL_TX | TG3_FLOW_CTRL_RX;
1730                 }
1731         } else if (lcladv & ADVERTISE_PAUSE_ASYM) {
1732                 if ((rmtadv & LPA_PAUSE_CAP) && (rmtadv & LPA_PAUSE_ASYM))
1733                         cap = TG3_FLOW_CTRL_TX;
1734         }
1735
1736         return cap;
1737 }
1738
1739 static u8 tg3_resolve_flowctrl_1000X(u16 lcladv, u16 rmtadv)
1740 {
1741         u8 cap = 0;
1742
1743         if (lcladv & ADVERTISE_1000XPAUSE) {
1744                 if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1745                         if (rmtadv & LPA_1000XPAUSE)
1746                                 cap = TG3_FLOW_CTRL_TX | TG3_FLOW_CTRL_RX;
1747                         else if (rmtadv & LPA_1000XPAUSE_ASYM)
1748                                 cap = TG3_FLOW_CTRL_RX;
1749                 } else {
1750                         if (rmtadv & LPA_1000XPAUSE)
1751                                 cap = TG3_FLOW_CTRL_TX | TG3_FLOW_CTRL_RX;
1752                 }
1753         } else if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1754                 if ((rmtadv & LPA_1000XPAUSE) && (rmtadv & LPA_1000XPAUSE_ASYM))
1755                         cap = TG3_FLOW_CTRL_TX;
1756         }
1757
1758         return cap;
1759 }
1760
1761 static void tg3_setup_flow_control(struct tg3 *tp, u32 local_adv, u32 remote_adv)
1762 {
1763         u8 new_tg3_flags = 0;
1764         u32 old_rx_mode = tp->rx_mode;
1765         u32 old_tx_mode = tp->tx_mode;
1766
1767         if (tp->link_config.autoneg == AUTONEG_ENABLE &&
1768             (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG)) {
1769                 if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
1770                         new_tg3_flags = tg3_resolve_flowctrl_1000X(local_adv,
1771                                                                    remote_adv);
1772                 else
1773                         new_tg3_flags = tg3_resolve_flowctrl_1000T(local_adv,
1774                                                                    remote_adv);
1775         } else {
1776                 new_tg3_flags = tp->link_config.flowctrl;
1777         }
1778
1779         tp->link_config.active_flowctrl = new_tg3_flags;
1780
1781         if (new_tg3_flags & TG3_FLOW_CTRL_RX)
1782                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1783         else
1784                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1785
1786         if (old_rx_mode != tp->rx_mode) {
1787                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1788         }
1789
1790         if (new_tg3_flags & TG3_FLOW_CTRL_TX)
1791                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1792         else
1793                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1794
1795         if (old_tx_mode != tp->tx_mode) {
1796                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1797         }
1798 }
1799
1800 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
1801 {
1802         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
1803         case MII_TG3_AUX_STAT_10HALF:
1804                 *speed = SPEED_10;
1805                 *duplex = DUPLEX_HALF;
1806                 break;
1807
1808         case MII_TG3_AUX_STAT_10FULL:
1809                 *speed = SPEED_10;
1810                 *duplex = DUPLEX_FULL;
1811                 break;
1812
1813         case MII_TG3_AUX_STAT_100HALF:
1814                 *speed = SPEED_100;
1815                 *duplex = DUPLEX_HALF;
1816                 break;
1817
1818         case MII_TG3_AUX_STAT_100FULL:
1819                 *speed = SPEED_100;
1820                 *duplex = DUPLEX_FULL;
1821                 break;
1822
1823         case MII_TG3_AUX_STAT_1000HALF:
1824                 *speed = SPEED_1000;
1825                 *duplex = DUPLEX_HALF;
1826                 break;
1827
1828         case MII_TG3_AUX_STAT_1000FULL:
1829                 *speed = SPEED_1000;
1830                 *duplex = DUPLEX_FULL;
1831                 break;
1832
1833         default:
1834                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1835                         *speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
1836                                  SPEED_10;
1837                         *duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
1838                                   DUPLEX_HALF;
1839                         break;
1840                 }
1841                 *speed = SPEED_INVALID;
1842                 *duplex = DUPLEX_INVALID;
1843                 break;
1844         };
1845 }
1846
1847 static void tg3_phy_copper_begin(struct tg3 *tp)
1848 {
1849         u32 new_adv;
1850         int i;
1851
1852         if (tp->link_config.phy_is_low_power) {
1853                 /* Entering low power mode.  Disable gigabit and
1854                  * 100baseT advertisements.
1855                  */
1856                 tg3_writephy(tp, MII_TG3_CTRL, 0);
1857
1858                 new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
1859                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1860                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
1861                         new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);
1862
1863                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1864         } else if (tp->link_config.speed == SPEED_INVALID) {
1865                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
1866                         tp->link_config.advertising &=
1867                                 ~(ADVERTISED_1000baseT_Half |
1868                                   ADVERTISED_1000baseT_Full);
1869
1870                 new_adv = ADVERTISE_CSMA;
1871                 if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
1872                         new_adv |= ADVERTISE_10HALF;
1873                 if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
1874                         new_adv |= ADVERTISE_10FULL;
1875                 if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
1876                         new_adv |= ADVERTISE_100HALF;
1877                 if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
1878                         new_adv |= ADVERTISE_100FULL;
1879
1880                 new_adv |= tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
1881
1882                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1883
1884                 if (tp->link_config.advertising &
1885                     (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
1886                         new_adv = 0;
1887                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
1888                                 new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
1889                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
1890                                 new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
1891                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY) &&
1892                             (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1893                              tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
1894                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1895                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1896                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1897                 } else {
1898                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1899                 }
1900         } else {
1901                 new_adv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
1902                 new_adv |= ADVERTISE_CSMA;
1903
1904                 /* Asking for a specific link mode. */
1905                 if (tp->link_config.speed == SPEED_1000) {
1906                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1907
1908                         if (tp->link_config.duplex == DUPLEX_FULL)
1909                                 new_adv = MII_TG3_CTRL_ADV_1000_FULL;
1910                         else
1911                                 new_adv = MII_TG3_CTRL_ADV_1000_HALF;
1912                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1913                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
1914                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1915                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1916                 } else {
1917                         if (tp->link_config.speed == SPEED_100) {
1918                                 if (tp->link_config.duplex == DUPLEX_FULL)
1919                                         new_adv |= ADVERTISE_100FULL;
1920                                 else
1921                                         new_adv |= ADVERTISE_100HALF;
1922                         } else {
1923                                 if (tp->link_config.duplex == DUPLEX_FULL)
1924                                         new_adv |= ADVERTISE_10FULL;
1925                                 else
1926                                         new_adv |= ADVERTISE_10HALF;
1927                         }
1928                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1929
1930                         new_adv = 0;
1931                 }
1932
1933                 tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1934         }
1935
1936         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
1937             tp->link_config.speed != SPEED_INVALID) {
1938                 u32 bmcr, orig_bmcr;
1939
1940                 tp->link_config.active_speed = tp->link_config.speed;
1941                 tp->link_config.active_duplex = tp->link_config.duplex;
1942
1943                 bmcr = 0;
1944                 switch (tp->link_config.speed) {
1945                 default:
1946                 case SPEED_10:
1947                         break;
1948
1949                 case SPEED_100:
1950                         bmcr |= BMCR_SPEED100;
1951                         break;
1952
1953                 case SPEED_1000:
1954                         bmcr |= TG3_BMCR_SPEED1000;
1955                         break;
1956                 };
1957
1958                 if (tp->link_config.duplex == DUPLEX_FULL)
1959                         bmcr |= BMCR_FULLDPLX;
1960
1961                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
1962                     (bmcr != orig_bmcr)) {
1963                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
1964                         for (i = 0; i < 1500; i++) {
1965                                 u32 tmp;
1966
1967                                 udelay(10);
1968                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
1969                                     tg3_readphy(tp, MII_BMSR, &tmp))
1970                                         continue;
1971                                 if (!(tmp & BMSR_LSTATUS)) {
1972                                         udelay(40);
1973                                         break;
1974                                 }
1975                         }
1976                         tg3_writephy(tp, MII_BMCR, bmcr);
1977                         udelay(40);
1978                 }
1979         } else {
1980                 tg3_writephy(tp, MII_BMCR,
1981                              BMCR_ANENABLE | BMCR_ANRESTART);
1982         }
1983 }
1984
1985 static int tg3_init_5401phy_dsp(struct tg3 *tp)
1986 {
1987         int err;
1988
1989         /* Turn off tap power management. */
1990         /* Set Extended packet length bit */
1991         err  = tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1992
1993         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0012);
1994         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1804);
1995
1996         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0013);
1997         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1204);
1998
1999         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
2000         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0132);
2001
2002         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
2003         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0232);
2004
2005         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
2006         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0a20);
2007
2008         udelay(40);
2009
2010         return err;
2011 }
2012
2013 static int tg3_copper_is_advertising_all(struct tg3 *tp, u32 mask)
2014 {
2015         u32 adv_reg, all_mask = 0;
2016
2017         if (mask & ADVERTISED_10baseT_Half)
2018                 all_mask |= ADVERTISE_10HALF;
2019         if (mask & ADVERTISED_10baseT_Full)
2020                 all_mask |= ADVERTISE_10FULL;
2021         if (mask & ADVERTISED_100baseT_Half)
2022                 all_mask |= ADVERTISE_100HALF;
2023         if (mask & ADVERTISED_100baseT_Full)
2024                 all_mask |= ADVERTISE_100FULL;
2025
2026         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
2027                 return 0;
2028
2029         if ((adv_reg & all_mask) != all_mask)
2030                 return 0;
2031         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
2032                 u32 tg3_ctrl;
2033
2034                 all_mask = 0;
2035                 if (mask & ADVERTISED_1000baseT_Half)
2036                         all_mask |= ADVERTISE_1000HALF;
2037                 if (mask & ADVERTISED_1000baseT_Full)
2038                         all_mask |= ADVERTISE_1000FULL;
2039
2040                 if (tg3_readphy(tp, MII_TG3_CTRL, &tg3_ctrl))
2041                         return 0;
2042
2043                 if ((tg3_ctrl & all_mask) != all_mask)
2044                         return 0;
2045         }
2046         return 1;
2047 }
2048
2049 static int tg3_adv_1000T_flowctrl_ok(struct tg3 *tp, u32 *lcladv, u32 *rmtadv)
2050 {
2051         u32 curadv, reqadv;
2052
2053         if (tg3_readphy(tp, MII_ADVERTISE, lcladv))
2054                 return 1;
2055
2056         curadv = *lcladv & (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
2057         reqadv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
2058
2059         if (tp->link_config.active_duplex == DUPLEX_FULL) {
2060                 if (curadv != reqadv)
2061                         return 0;
2062
2063                 if (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG)
2064                         tg3_readphy(tp, MII_LPA, rmtadv);
2065         } else {
2066                 /* Reprogram the advertisement register, even if it
2067                  * does not affect the current link.  If the link
2068                  * gets renegotiated in the future, we can save an
2069                  * additional renegotiation cycle by advertising
2070                  * it correctly in the first place.
2071                  */
2072                 if (curadv != reqadv) {
2073                         *lcladv &= ~(ADVERTISE_PAUSE_CAP |
2074                                      ADVERTISE_PAUSE_ASYM);
2075                         tg3_writephy(tp, MII_ADVERTISE, *lcladv | reqadv);
2076                 }
2077         }
2078
2079         return 1;
2080 }
2081
2082 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
2083 {
2084         int current_link_up;
2085         u32 bmsr, dummy;
2086         u32 lcl_adv, rmt_adv;
2087         u16 current_speed;
2088         u8 current_duplex;
2089         int i, err;
2090
2091         tw32(MAC_EVENT, 0);
2092
2093         tw32_f(MAC_STATUS,
2094              (MAC_STATUS_SYNC_CHANGED |
2095               MAC_STATUS_CFG_CHANGED |
2096               MAC_STATUS_MI_COMPLETION |
2097               MAC_STATUS_LNKSTATE_CHANGED));
2098         udelay(40);
2099
2100         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
2101                 tw32_f(MAC_MI_MODE,
2102                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
2103                 udelay(80);
2104         }
2105
2106         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x02);
2107
2108         /* Some third-party PHYs need to be reset on link going
2109          * down.
2110          */
2111         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
2112              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2113              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
2114             netif_carrier_ok(tp->dev)) {
2115                 tg3_readphy(tp, MII_BMSR, &bmsr);
2116                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
2117                     !(bmsr & BMSR_LSTATUS))
2118                         force_reset = 1;
2119         }
2120         if (force_reset)
2121                 tg3_phy_reset(tp);
2122
2123         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
2124                 tg3_readphy(tp, MII_BMSR, &bmsr);
2125                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
2126                     !(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE))
2127                         bmsr = 0;
2128
2129                 if (!(bmsr & BMSR_LSTATUS)) {
2130                         err = tg3_init_5401phy_dsp(tp);
2131                         if (err)
2132                                 return err;
2133
2134                         tg3_readphy(tp, MII_BMSR, &bmsr);
2135                         for (i = 0; i < 1000; i++) {
2136                                 udelay(10);
2137                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
2138                                     (bmsr & BMSR_LSTATUS)) {
2139                                         udelay(40);
2140                                         break;
2141                                 }
2142                         }
2143
2144                         if ((tp->phy_id & PHY_ID_REV_MASK) == PHY_REV_BCM5401_B0 &&
2145                             !(bmsr & BMSR_LSTATUS) &&
2146                             tp->link_config.active_speed == SPEED_1000) {
2147                                 err = tg3_phy_reset(tp);
2148                                 if (!err)
2149                                         err = tg3_init_5401phy_dsp(tp);
2150                                 if (err)
2151                                         return err;
2152                         }
2153                 }
2154         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
2155                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
2156                 /* 5701 {A0,B0} CRC bug workaround */
2157                 tg3_writephy(tp, 0x15, 0x0a75);
2158                 tg3_writephy(tp, 0x1c, 0x8c68);
2159                 tg3_writephy(tp, 0x1c, 0x8d68);
2160                 tg3_writephy(tp, 0x1c, 0x8c68);
2161         }
2162
2163         /* Clear pending interrupts... */
2164         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
2165         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
2166
2167         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT)
2168                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
2169         else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)
2170                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
2171
2172         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2173             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2174                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
2175                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
2176                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
2177                 else
2178                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
2179         }
2180
2181         current_link_up = 0;
2182         current_speed = SPEED_INVALID;
2183         current_duplex = DUPLEX_INVALID;
2184
2185         if (tp->tg3_flags2 & TG3_FLG2_CAPACITIVE_COUPLING) {
2186                 u32 val;
2187
2188                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4007);
2189                 tg3_readphy(tp, MII_TG3_AUX_CTRL, &val);
2190                 if (!(val & (1 << 10))) {
2191                         val |= (1 << 10);
2192                         tg3_writephy(tp, MII_TG3_AUX_CTRL, val);
2193                         goto relink;
2194                 }
2195         }
2196
2197         bmsr = 0;
2198         for (i = 0; i < 100; i++) {
2199                 tg3_readphy(tp, MII_BMSR, &bmsr);
2200                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
2201                     (bmsr & BMSR_LSTATUS))
2202                         break;
2203                 udelay(40);
2204         }
2205
2206         if (bmsr & BMSR_LSTATUS) {
2207                 u32 aux_stat, bmcr;
2208
2209                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
2210                 for (i = 0; i < 2000; i++) {
2211                         udelay(10);
2212                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
2213                             aux_stat)
2214                                 break;
2215                 }
2216
2217                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
2218                                              &current_speed,
2219                                              &current_duplex);
2220
2221                 bmcr = 0;
2222                 for (i = 0; i < 200; i++) {
2223                         tg3_readphy(tp, MII_BMCR, &bmcr);
2224                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
2225                                 continue;
2226                         if (bmcr && bmcr != 0x7fff)
2227                                 break;
2228                         udelay(10);
2229                 }
2230
2231                 lcl_adv = 0;
2232                 rmt_adv = 0;
2233
2234                 tp->link_config.active_speed = current_speed;
2235                 tp->link_config.active_duplex = current_duplex;
2236
2237                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2238                         if ((bmcr & BMCR_ANENABLE) &&
2239                             tg3_copper_is_advertising_all(tp,
2240                                                 tp->link_config.advertising)) {
2241                                 if (tg3_adv_1000T_flowctrl_ok(tp, &lcl_adv,
2242                                                                   &rmt_adv))
2243                                         current_link_up = 1;
2244                         }
2245                 } else {
2246                         if (!(bmcr & BMCR_ANENABLE) &&
2247                             tp->link_config.speed == current_speed &&
2248                             tp->link_config.duplex == current_duplex &&
2249                             tp->link_config.flowctrl ==
2250                             tp->link_config.active_flowctrl) {
2251                                 current_link_up = 1;
2252                         }
2253                 }
2254
2255                 if (current_link_up == 1 &&
2256                     tp->link_config.active_duplex == DUPLEX_FULL)
2257                         tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
2258         }
2259
2260 relink:
2261         if (current_link_up == 0 || tp->link_config.phy_is_low_power) {
2262                 u32 tmp;
2263
2264                 tg3_phy_copper_begin(tp);
2265
2266                 tg3_readphy(tp, MII_BMSR, &tmp);
2267                 if (!tg3_readphy(tp, MII_BMSR, &tmp) &&
2268                     (tmp & BMSR_LSTATUS))
2269                         current_link_up = 1;
2270         }
2271
2272         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
2273         if (current_link_up == 1) {
2274                 if (tp->link_config.active_speed == SPEED_100 ||
2275                     tp->link_config.active_speed == SPEED_10)
2276                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
2277                 else
2278                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
2279         } else
2280                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
2281
2282         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
2283         if (tp->link_config.active_duplex == DUPLEX_HALF)
2284                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
2285
2286         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
2287                 if (current_link_up == 1 &&
2288                     tg3_5700_link_polarity(tp, tp->link_config.active_speed))
2289                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
2290                 else
2291                         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
2292         }
2293
2294         /* ??? Without this setting Netgear GA302T PHY does not
2295          * ??? send/receive packets...
2296          */
2297         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411 &&
2298             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
2299                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
2300                 tw32_f(MAC_MI_MODE, tp->mi_mode);
2301                 udelay(80);
2302         }
2303
2304         tw32_f(MAC_MODE, tp->mac_mode);
2305         udelay(40);
2306
2307         if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
2308                 /* Polled via timer. */
2309                 tw32_f(MAC_EVENT, 0);
2310         } else {
2311                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2312         }
2313         udelay(40);
2314
2315         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
2316             current_link_up == 1 &&
2317             tp->link_config.active_speed == SPEED_1000 &&
2318             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) ||
2319              (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED))) {
2320                 udelay(120);
2321                 tw32_f(MAC_STATUS,
2322                      (MAC_STATUS_SYNC_CHANGED |
2323                       MAC_STATUS_CFG_CHANGED));
2324                 udelay(40);
2325                 tg3_write_mem(tp,
2326                               NIC_SRAM_FIRMWARE_MBOX,
2327                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
2328         }
2329
2330         if (current_link_up != netif_carrier_ok(tp->dev)) {
2331                 if (current_link_up)
2332                         netif_carrier_on(tp->dev);
2333                 else
2334                         netif_carrier_off(tp->dev);
2335                 tg3_link_report(tp);
2336         }
2337
2338         return 0;
2339 }
2340
2341 struct tg3_fiber_aneginfo {
2342         int state;
2343 #define ANEG_STATE_UNKNOWN              0
2344 #define ANEG_STATE_AN_ENABLE            1
2345 #define ANEG_STATE_RESTART_INIT         2
2346 #define ANEG_STATE_RESTART              3
2347 #define ANEG_STATE_DISABLE_LINK_OK      4
2348 #define ANEG_STATE_ABILITY_DETECT_INIT  5
2349 #define ANEG_STATE_ABILITY_DETECT       6
2350 #define ANEG_STATE_ACK_DETECT_INIT      7
2351 #define ANEG_STATE_ACK_DETECT           8
2352 #define ANEG_STATE_COMPLETE_ACK_INIT    9
2353 #define ANEG_STATE_COMPLETE_ACK         10
2354 #define ANEG_STATE_IDLE_DETECT_INIT     11
2355 #define ANEG_STATE_IDLE_DETECT          12
2356 #define ANEG_STATE_LINK_OK              13
2357 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
2358 #define ANEG_STATE_NEXT_PAGE_WAIT       15
2359
2360         u32 flags;
2361 #define MR_AN_ENABLE            0x00000001
2362 #define MR_RESTART_AN           0x00000002
2363 #define MR_AN_COMPLETE          0x00000004
2364 #define MR_PAGE_RX              0x00000008
2365 #define MR_NP_LOADED            0x00000010
2366 #define MR_TOGGLE_TX            0x00000020
2367 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
2368 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
2369 #define MR_LP_ADV_SYM_PAUSE     0x00000100
2370 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
2371 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
2372 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
2373 #define MR_LP_ADV_NEXT_PAGE     0x00001000
2374 #define MR_TOGGLE_RX            0x00002000
2375 #define MR_NP_RX                0x00004000
2376
2377 #define MR_LINK_OK              0x80000000
2378
2379         unsigned long link_time, cur_time;
2380
2381         u32 ability_match_cfg;
2382         int ability_match_count;
2383
2384         char ability_match, idle_match, ack_match;
2385
2386         u32 txconfig, rxconfig;
2387 #define ANEG_CFG_NP             0x00000080
2388 #define ANEG_CFG_ACK            0x00000040
2389 #define ANEG_CFG_RF2            0x00000020
2390 #define ANEG_CFG_RF1            0x00000010
2391 #define ANEG_CFG_PS2            0x00000001
2392 #define ANEG_CFG_PS1            0x00008000
2393 #define ANEG_CFG_HD             0x00004000
2394 #define ANEG_CFG_FD             0x00002000
2395 #define ANEG_CFG_INVAL          0x00001f06
2396
2397 };
2398 #define ANEG_OK         0
2399 #define ANEG_DONE       1
2400 #define ANEG_TIMER_ENAB 2
2401 #define ANEG_FAILED     -1
2402
2403 #define ANEG_STATE_SETTLE_TIME  10000
2404
2405 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
2406                                    struct tg3_fiber_aneginfo *ap)
2407 {
2408         u16 flowctrl;
2409         unsigned long delta;
2410         u32 rx_cfg_reg;
2411         int ret;
2412
2413         if (ap->state == ANEG_STATE_UNKNOWN) {
2414                 ap->rxconfig = 0;
2415                 ap->link_time = 0;
2416                 ap->cur_time = 0;
2417                 ap->ability_match_cfg = 0;
2418                 ap->ability_match_count = 0;
2419                 ap->ability_match = 0;
2420                 ap->idle_match = 0;
2421                 ap->ack_match = 0;
2422         }
2423         ap->cur_time++;
2424
2425         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
2426                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
2427
2428                 if (rx_cfg_reg != ap->ability_match_cfg) {
2429                         ap->ability_match_cfg = rx_cfg_reg;
2430                         ap->ability_match = 0;
2431                         ap->ability_match_count = 0;
2432                 } else {
2433                         if (++ap->ability_match_count > 1) {
2434                                 ap->ability_match = 1;
2435                                 ap->ability_match_cfg = rx_cfg_reg;
2436                         }
2437                 }
2438                 if (rx_cfg_reg & ANEG_CFG_ACK)
2439                         ap->ack_match = 1;
2440                 else
2441                         ap->ack_match = 0;
2442
2443                 ap->idle_match = 0;
2444         } else {
2445                 ap->idle_match = 1;
2446                 ap->ability_match_cfg = 0;
2447                 ap->ability_match_count = 0;
2448                 ap->ability_match = 0;
2449                 ap->ack_match = 0;
2450
2451                 rx_cfg_reg = 0;
2452         }
2453
2454         ap->rxconfig = rx_cfg_reg;
2455         ret = ANEG_OK;
2456
2457         switch(ap->state) {
2458         case ANEG_STATE_UNKNOWN:
2459                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
2460                         ap->state = ANEG_STATE_AN_ENABLE;
2461
2462                 /* fallthru */
2463         case ANEG_STATE_AN_ENABLE:
2464                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
2465                 if (ap->flags & MR_AN_ENABLE) {
2466                         ap->link_time = 0;
2467                         ap->cur_time = 0;
2468                         ap->ability_match_cfg = 0;
2469                         ap->ability_match_count = 0;
2470                         ap->ability_match = 0;
2471                         ap->idle_match = 0;
2472                         ap->ack_match = 0;
2473
2474                         ap->state = ANEG_STATE_RESTART_INIT;
2475                 } else {
2476                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
2477                 }
2478                 break;
2479
2480         case ANEG_STATE_RESTART_INIT:
2481                 ap->link_time = ap->cur_time;
2482                 ap->flags &= ~(MR_NP_LOADED);
2483                 ap->txconfig = 0;
2484                 tw32(MAC_TX_AUTO_NEG, 0);
2485                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2486                 tw32_f(MAC_MODE, tp->mac_mode);
2487                 udelay(40);
2488
2489                 ret = ANEG_TIMER_ENAB;
2490                 ap->state = ANEG_STATE_RESTART;
2491
2492                 /* fallthru */
2493         case ANEG_STATE_RESTART:
2494                 delta = ap->cur_time - ap->link_time;
2495                 if (delta > ANEG_STATE_SETTLE_TIME) {
2496                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
2497                 } else {
2498                         ret = ANEG_TIMER_ENAB;
2499                 }
2500                 break;
2501
2502         case ANEG_STATE_DISABLE_LINK_OK:
2503                 ret = ANEG_DONE;
2504                 break;
2505
2506         case ANEG_STATE_ABILITY_DETECT_INIT:
2507                 ap->flags &= ~(MR_TOGGLE_TX);
2508                 ap->txconfig = ANEG_CFG_FD;
2509                 flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
2510                 if (flowctrl & ADVERTISE_1000XPAUSE)
2511                         ap->txconfig |= ANEG_CFG_PS1;
2512                 if (flowctrl & ADVERTISE_1000XPSE_ASYM)
2513                         ap->txconfig |= ANEG_CFG_PS2;
2514                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2515                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2516                 tw32_f(MAC_MODE, tp->mac_mode);
2517                 udelay(40);
2518
2519                 ap->state = ANEG_STATE_ABILITY_DETECT;
2520                 break;
2521
2522         case ANEG_STATE_ABILITY_DETECT:
2523                 if (ap->ability_match != 0 && ap->rxconfig != 0) {
2524                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
2525                 }
2526                 break;
2527
2528         case ANEG_STATE_ACK_DETECT_INIT:
2529                 ap->txconfig |= ANEG_CFG_ACK;
2530                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2531                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2532                 tw32_f(MAC_MODE, tp->mac_mode);
2533                 udelay(40);
2534
2535                 ap->state = ANEG_STATE_ACK_DETECT;
2536
2537                 /* fallthru */
2538         case ANEG_STATE_ACK_DETECT:
2539                 if (ap->ack_match != 0) {
2540                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
2541                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
2542                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
2543                         } else {
2544                                 ap->state = ANEG_STATE_AN_ENABLE;
2545                         }
2546                 } else if (ap->ability_match != 0 &&
2547                            ap->rxconfig == 0) {
2548                         ap->state = ANEG_STATE_AN_ENABLE;
2549                 }
2550                 break;
2551
2552         case ANEG_STATE_COMPLETE_ACK_INIT:
2553                 if (ap->rxconfig & ANEG_CFG_INVAL) {
2554                         ret = ANEG_FAILED;
2555                         break;
2556                 }
2557                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
2558                                MR_LP_ADV_HALF_DUPLEX |
2559                                MR_LP_ADV_SYM_PAUSE |
2560                                MR_LP_ADV_ASYM_PAUSE |
2561                                MR_LP_ADV_REMOTE_FAULT1 |
2562                                MR_LP_ADV_REMOTE_FAULT2 |
2563                                MR_LP_ADV_NEXT_PAGE |
2564                                MR_TOGGLE_RX |
2565                                MR_NP_RX);
2566                 if (ap->rxconfig & ANEG_CFG_FD)
2567                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
2568                 if (ap->rxconfig & ANEG_CFG_HD)
2569                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
2570                 if (ap->rxconfig & ANEG_CFG_PS1)
2571                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
2572                 if (ap->rxconfig & ANEG_CFG_PS2)
2573                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
2574                 if (ap->rxconfig & ANEG_CFG_RF1)
2575                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
2576                 if (ap->rxconfig & ANEG_CFG_RF2)
2577                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
2578                 if (ap->rxconfig & ANEG_CFG_NP)
2579                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
2580
2581                 ap->link_time = ap->cur_time;
2582
2583                 ap->flags ^= (MR_TOGGLE_TX);
2584                 if (ap->rxconfig & 0x0008)
2585                         ap->flags |= MR_TOGGLE_RX;
2586                 if (ap->rxconfig & ANEG_CFG_NP)
2587                         ap->flags |= MR_NP_RX;
2588                 ap->flags |= MR_PAGE_RX;
2589
2590                 ap->state = ANEG_STATE_COMPLETE_ACK;
2591                 ret = ANEG_TIMER_ENAB;
2592                 break;
2593
2594         case ANEG_STATE_COMPLETE_ACK:
2595                 if (ap->ability_match != 0 &&
2596                     ap->rxconfig == 0) {
2597                         ap->state = ANEG_STATE_AN_ENABLE;
2598                         break;
2599                 }
2600                 delta = ap->cur_time - ap->link_time;
2601                 if (delta > ANEG_STATE_SETTLE_TIME) {
2602                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
2603                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2604                         } else {
2605                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
2606                                     !(ap->flags & MR_NP_RX)) {
2607                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2608                                 } else {
2609                                         ret = ANEG_FAILED;
2610                                 }
2611                         }
2612                 }
2613                 break;
2614
2615         case ANEG_STATE_IDLE_DETECT_INIT:
2616                 ap->link_time = ap->cur_time;
2617                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2618                 tw32_f(MAC_MODE, tp->mac_mode);
2619                 udelay(40);
2620
2621                 ap->state = ANEG_STATE_IDLE_DETECT;
2622                 ret = ANEG_TIMER_ENAB;
2623                 break;
2624
2625         case ANEG_STATE_IDLE_DETECT:
2626                 if (ap->ability_match != 0 &&
2627                     ap->rxconfig == 0) {
2628                         ap->state = ANEG_STATE_AN_ENABLE;
2629                         break;
2630                 }
2631                 delta = ap->cur_time - ap->link_time;
2632                 if (delta > ANEG_STATE_SETTLE_TIME) {
2633                         /* XXX another gem from the Broadcom driver :( */
2634                         ap->state = ANEG_STATE_LINK_OK;
2635                 }
2636                 break;
2637
2638         case ANEG_STATE_LINK_OK:
2639                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
2640                 ret = ANEG_DONE;
2641                 break;
2642
2643         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
2644                 /* ??? unimplemented */
2645                 break;
2646
2647         case ANEG_STATE_NEXT_PAGE_WAIT:
2648                 /* ??? unimplemented */
2649                 break;
2650
2651         default:
2652                 ret = ANEG_FAILED;
2653                 break;
2654         };
2655
2656         return ret;
2657 }
2658
2659 static int fiber_autoneg(struct tg3 *tp, u32 *txflags, u32 *rxflags)
2660 {
2661         int res = 0;
2662         struct tg3_fiber_aneginfo aninfo;
2663         int status = ANEG_FAILED;
2664         unsigned int tick;
2665         u32 tmp;
2666
2667         tw32_f(MAC_TX_AUTO_NEG, 0);
2668
2669         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
2670         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
2671         udelay(40);
2672
2673         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
2674         udelay(40);
2675
2676         memset(&aninfo, 0, sizeof(aninfo));
2677         aninfo.flags |= MR_AN_ENABLE;
2678         aninfo.state = ANEG_STATE_UNKNOWN;
2679         aninfo.cur_time = 0;
2680         tick = 0;
2681         while (++tick < 195000) {
2682                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
2683                 if (status == ANEG_DONE || status == ANEG_FAILED)
2684                         break;
2685
2686                 udelay(1);
2687         }
2688
2689         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2690         tw32_f(MAC_MODE, tp->mac_mode);
2691         udelay(40);
2692
2693         *txflags = aninfo.txconfig;
2694         *rxflags = aninfo.flags;
2695
2696         if (status == ANEG_DONE &&
2697             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
2698                              MR_LP_ADV_FULL_DUPLEX)))
2699                 res = 1;
2700
2701         return res;
2702 }
2703
2704 static void tg3_init_bcm8002(struct tg3 *tp)
2705 {
2706         u32 mac_status = tr32(MAC_STATUS);
2707         int i;
2708
2709         /* Reset when initting first time or we have a link. */
2710         if ((tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) &&
2711             !(mac_status & MAC_STATUS_PCS_SYNCED))
2712                 return;
2713
2714         /* Set PLL lock range. */
2715         tg3_writephy(tp, 0x16, 0x8007);
2716
2717         /* SW reset */
2718         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
2719
2720         /* Wait for reset to complete. */
2721         /* XXX schedule_timeout() ... */
2722         for (i = 0; i < 500; i++)
2723                 udelay(10);
2724
2725         /* Config mode; select PMA/Ch 1 regs. */
2726         tg3_writephy(tp, 0x10, 0x8411);
2727
2728         /* Enable auto-lock and comdet, select txclk for tx. */
2729         tg3_writephy(tp, 0x11, 0x0a10);
2730
2731         tg3_writephy(tp, 0x18, 0x00a0);
2732         tg3_writephy(tp, 0x16, 0x41ff);
2733
2734         /* Assert and deassert POR. */
2735         tg3_writephy(tp, 0x13, 0x0400);
2736         udelay(40);
2737         tg3_writephy(tp, 0x13, 0x0000);
2738
2739         tg3_writephy(tp, 0x11, 0x0a50);
2740         udelay(40);
2741         tg3_writephy(tp, 0x11, 0x0a10);
2742
2743         /* Wait for signal to stabilize */
2744         /* XXX schedule_timeout() ... */
2745         for (i = 0; i < 15000; i++)
2746                 udelay(10);
2747
2748         /* Deselect the channel register so we can read the PHYID
2749          * later.
2750          */
2751         tg3_writephy(tp, 0x10, 0x8011);
2752 }
2753
2754 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
2755 {
2756         u16 flowctrl;
2757         u32 sg_dig_ctrl, sg_dig_status;
2758         u32 serdes_cfg, expected_sg_dig_ctrl;
2759         int workaround, port_a;
2760         int current_link_up;
2761
2762         serdes_cfg = 0;
2763         expected_sg_dig_ctrl = 0;
2764         workaround = 0;
2765         port_a = 1;
2766         current_link_up = 0;
2767
2768         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
2769             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
2770                 workaround = 1;
2771                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
2772                         port_a = 0;
2773
2774                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
2775                 /* preserve bits 20-23 for voltage regulator */
2776                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
2777         }
2778
2779         sg_dig_ctrl = tr32(SG_DIG_CTRL);
2780
2781         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
2782                 if (sg_dig_ctrl & SG_DIG_USING_HW_AUTONEG) {
2783                         if (workaround) {
2784                                 u32 val = serdes_cfg;
2785
2786                                 if (port_a)
2787                                         val |= 0xc010000;
2788                                 else
2789                                         val |= 0x4010000;
2790                                 tw32_f(MAC_SERDES_CFG, val);
2791                         }
2792
2793                         tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
2794                 }
2795                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
2796                         tg3_setup_flow_control(tp, 0, 0);
2797                         current_link_up = 1;
2798                 }
2799                 goto out;
2800         }
2801
2802         /* Want auto-negotiation.  */
2803         expected_sg_dig_ctrl = SG_DIG_USING_HW_AUTONEG | SG_DIG_COMMON_SETUP;
2804
2805         flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
2806         if (flowctrl & ADVERTISE_1000XPAUSE)
2807                 expected_sg_dig_ctrl |= SG_DIG_PAUSE_CAP;
2808         if (flowctrl & ADVERTISE_1000XPSE_ASYM)
2809                 expected_sg_dig_ctrl |= SG_DIG_ASYM_PAUSE;
2810
2811         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
2812                 if ((tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT) &&
2813                     tp->serdes_counter &&
2814                     ((mac_status & (MAC_STATUS_PCS_SYNCED |
2815                                     MAC_STATUS_RCVD_CFG)) ==
2816                      MAC_STATUS_PCS_SYNCED)) {
2817                         tp->serdes_counter--;
2818                         current_link_up = 1;
2819                         goto out;
2820                 }
2821 restart_autoneg:
2822                 if (workaround)
2823                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
2824                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | SG_DIG_SOFT_RESET);
2825                 udelay(5);
2826                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
2827
2828                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
2829                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2830         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
2831                                  MAC_STATUS_SIGNAL_DET)) {
2832                 sg_dig_status = tr32(SG_DIG_STATUS);
2833                 mac_status = tr32(MAC_STATUS);
2834
2835                 if ((sg_dig_status & SG_DIG_AUTONEG_COMPLETE) &&
2836                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
2837                         u32 local_adv = 0, remote_adv = 0;
2838
2839                         if (sg_dig_ctrl & SG_DIG_PAUSE_CAP)
2840                                 local_adv |= ADVERTISE_1000XPAUSE;
2841                         if (sg_dig_ctrl & SG_DIG_ASYM_PAUSE)
2842                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
2843
2844                         if (sg_dig_status & SG_DIG_PARTNER_PAUSE_CAPABLE)
2845                                 remote_adv |= LPA_1000XPAUSE;
2846                         if (sg_dig_status & SG_DIG_PARTNER_ASYM_PAUSE)
2847                                 remote_adv |= LPA_1000XPAUSE_ASYM;
2848
2849                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2850                         current_link_up = 1;
2851                         tp->serdes_counter = 0;
2852                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2853                 } else if (!(sg_dig_status & SG_DIG_AUTONEG_COMPLETE)) {
2854                         if (tp->serdes_counter)
2855                                 tp->serdes_counter--;
2856                         else {
2857                                 if (workaround) {
2858                                         u32 val = serdes_cfg;
2859
2860                                         if (port_a)
2861                                                 val |= 0xc010000;
2862                                         else
2863                                                 val |= 0x4010000;
2864
2865                                         tw32_f(MAC_SERDES_CFG, val);
2866                                 }
2867
2868                                 tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
2869                                 udelay(40);
2870
2871                                 /* Link parallel detection - link is up */
2872                                 /* only if we have PCS_SYNC and not */
2873                                 /* receiving config code words */
2874                                 mac_status = tr32(MAC_STATUS);
2875                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
2876                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
2877                                         tg3_setup_flow_control(tp, 0, 0);
2878                                         current_link_up = 1;
2879                                         tp->tg3_flags2 |=
2880                                                 TG3_FLG2_PARALLEL_DETECT;
2881                                         tp->serdes_counter =
2882                                                 SERDES_PARALLEL_DET_TIMEOUT;
2883                                 } else
2884                                         goto restart_autoneg;
2885                         }
2886                 }
2887         } else {
2888                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
2889                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2890         }
2891
2892 out:
2893         return current_link_up;
2894 }
2895
2896 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
2897 {
2898         int current_link_up = 0;
2899
2900         if (!(mac_status & MAC_STATUS_PCS_SYNCED))
2901                 goto out;
2902
2903         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2904                 u32 txflags, rxflags;
2905                 int i;
2906
2907                 if (fiber_autoneg(tp, &txflags, &rxflags)) {
2908                         u32 local_adv = 0, remote_adv = 0;
2909
2910                         if (txflags & ANEG_CFG_PS1)
2911                                 local_adv |= ADVERTISE_1000XPAUSE;
2912                         if (txflags & ANEG_CFG_PS2)
2913                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
2914
2915                         if (rxflags & MR_LP_ADV_SYM_PAUSE)
2916                                 remote_adv |= LPA_1000XPAUSE;
2917                         if (rxflags & MR_LP_ADV_ASYM_PAUSE)
2918                                 remote_adv |= LPA_1000XPAUSE_ASYM;
2919
2920                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2921
2922                         current_link_up = 1;
2923                 }
2924                 for (i = 0; i < 30; i++) {
2925                         udelay(20);
2926                         tw32_f(MAC_STATUS,
2927                                (MAC_STATUS_SYNC_CHANGED |
2928                                 MAC_STATUS_CFG_CHANGED));
2929                         udelay(40);
2930                         if ((tr32(MAC_STATUS) &
2931                              (MAC_STATUS_SYNC_CHANGED |
2932                               MAC_STATUS_CFG_CHANGED)) == 0)
2933                                 break;
2934                 }
2935
2936                 mac_status = tr32(MAC_STATUS);
2937                 if (current_link_up == 0 &&
2938                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
2939                     !(mac_status & MAC_STATUS_RCVD_CFG))
2940                         current_link_up = 1;
2941         } else {
2942                 tg3_setup_flow_control(tp, 0, 0);
2943
2944                 /* Forcing 1000FD link up. */
2945                 current_link_up = 1;
2946
2947                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
2948                 udelay(40);
2949
2950                 tw32_f(MAC_MODE, tp->mac_mode);
2951                 udelay(40);
2952         }
2953
2954 out:
2955         return current_link_up;
2956 }
2957
2958 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
2959 {
2960         u32 orig_pause_cfg;
2961         u16 orig_active_speed;
2962         u8 orig_active_duplex;
2963         u32 mac_status;
2964         int current_link_up;
2965         int i;
2966
2967         orig_pause_cfg = tp->link_config.active_flowctrl;
2968         orig_active_speed = tp->link_config.active_speed;
2969         orig_active_duplex = tp->link_config.active_duplex;
2970
2971         if (!(tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) &&
2972             netif_carrier_ok(tp->dev) &&
2973             (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE)) {
2974                 mac_status = tr32(MAC_STATUS);
2975                 mac_status &= (MAC_STATUS_PCS_SYNCED |
2976                                MAC_STATUS_SIGNAL_DET |
2977                                MAC_STATUS_CFG_CHANGED |
2978                                MAC_STATUS_RCVD_CFG);
2979                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
2980                                    MAC_STATUS_SIGNAL_DET)) {
2981                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2982                                             MAC_STATUS_CFG_CHANGED));
2983                         return 0;
2984                 }
2985         }
2986
2987         tw32_f(MAC_TX_AUTO_NEG, 0);
2988
2989         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
2990         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
2991         tw32_f(MAC_MODE, tp->mac_mode);
2992         udelay(40);
2993
2994         if (tp->phy_id == PHY_ID_BCM8002)
2995                 tg3_init_bcm8002(tp);
2996
2997         /* Enable link change event even when serdes polling.  */
2998         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2999         udelay(40);
3000
3001         current_link_up = 0;
3002         mac_status = tr32(MAC_STATUS);
3003
3004         if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG)
3005                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
3006         else
3007                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
3008
3009         tp->hw_status->status =
3010                 (SD_STATUS_UPDATED |
3011                  (tp->hw_status->status & ~SD_STATUS_LINK_CHG));
3012
3013         for (i = 0; i < 100; i++) {
3014                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
3015                                     MAC_STATUS_CFG_CHANGED));
3016                 udelay(5);
3017                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
3018                                          MAC_STATUS_CFG_CHANGED |
3019                                          MAC_STATUS_LNKSTATE_CHANGED)) == 0)
3020                         break;
3021         }
3022
3023         mac_status = tr32(MAC_STATUS);
3024         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
3025                 current_link_up = 0;
3026                 if (tp->link_config.autoneg == AUTONEG_ENABLE &&
3027                     tp->serdes_counter == 0) {
3028                         tw32_f(MAC_MODE, (tp->mac_mode |
3029                                           MAC_MODE_SEND_CONFIGS));
3030                         udelay(1);
3031                         tw32_f(MAC_MODE, tp->mac_mode);
3032                 }
3033         }
3034
3035         if (current_link_up == 1) {
3036                 tp->link_config.active_speed = SPEED_1000;
3037                 tp->link_config.active_duplex = DUPLEX_FULL;
3038                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
3039                                     LED_CTRL_LNKLED_OVERRIDE |
3040                                     LED_CTRL_1000MBPS_ON));
3041         } else {
3042                 tp->link_config.active_speed = SPEED_INVALID;
3043                 tp->link_config.active_duplex = DUPLEX_INVALID;
3044                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
3045                                     LED_CTRL_LNKLED_OVERRIDE |
3046                                     LED_CTRL_TRAFFIC_OVERRIDE));
3047         }
3048
3049         if (current_link_up != netif_carrier_ok(tp->dev)) {
3050                 if (current_link_up)
3051                         netif_carrier_on(tp->dev);
3052                 else
3053                         netif_carrier_off(tp->dev);
3054                 tg3_link_report(tp);
3055         } else {
3056                 u32 now_pause_cfg = tp->link_config.active_flowctrl;
3057                 if (orig_pause_cfg != now_pause_cfg ||
3058                     orig_active_speed != tp->link_config.active_speed ||
3059                     orig_active_duplex != tp->link_config.active_duplex)
3060                         tg3_link_report(tp);
3061         }
3062
3063         return 0;
3064 }
3065
3066 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
3067 {
3068         int current_link_up, err = 0;
3069         u32 bmsr, bmcr;
3070         u16 current_speed;
3071         u8 current_duplex;
3072         u32 local_adv, remote_adv;
3073
3074         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3075         tw32_f(MAC_MODE, tp->mac_mode);
3076         udelay(40);
3077
3078         tw32(MAC_EVENT, 0);
3079
3080         tw32_f(MAC_STATUS,
3081              (MAC_STATUS_SYNC_CHANGED |
3082               MAC_STATUS_CFG_CHANGED |
3083               MAC_STATUS_MI_COMPLETION |
3084               MAC_STATUS_LNKSTATE_CHANGED));
3085         udelay(40);
3086
3087         if (force_reset)
3088                 tg3_phy_reset(tp);
3089
3090         current_link_up = 0;
3091         current_speed = SPEED_INVALID;
3092         current_duplex = DUPLEX_INVALID;
3093
3094         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3095         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3096         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
3097                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
3098                         bmsr |= BMSR_LSTATUS;
3099                 else
3100                         bmsr &= ~BMSR_LSTATUS;
3101         }
3102
3103         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
3104
3105         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
3106             (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT) &&
3107              tp->link_config.flowctrl == tp->link_config.active_flowctrl) {
3108                 /* do nothing, just check for link up at the end */
3109         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3110                 u32 adv, new_adv;
3111
3112                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
3113                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
3114                                   ADVERTISE_1000XPAUSE |
3115                                   ADVERTISE_1000XPSE_ASYM |
3116                                   ADVERTISE_SLCT);
3117
3118                 new_adv |= tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3119
3120                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
3121                         new_adv |= ADVERTISE_1000XHALF;
3122                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
3123                         new_adv |= ADVERTISE_1000XFULL;
3124
3125                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
3126                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
3127                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
3128                         tg3_writephy(tp, MII_BMCR, bmcr);
3129
3130                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3131                         tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
3132                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3133
3134                         return err;
3135                 }
3136         } else {
3137                 u32 new_bmcr;
3138
3139                 bmcr &= ~BMCR_SPEED1000;
3140                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
3141
3142                 if (tp->link_config.duplex == DUPLEX_FULL)
3143                         new_bmcr |= BMCR_FULLDPLX;
3144
3145                 if (new_bmcr != bmcr) {
3146                         /* BMCR_SPEED1000 is a reserved bit that needs
3147                          * to be set on write.
3148                          */
3149                         new_bmcr |= BMCR_SPEED1000;
3150
3151                         /* Force a linkdown */
3152                         if (netif_carrier_ok(tp->dev)) {
3153                                 u32 adv;
3154
3155                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
3156                                 adv &= ~(ADVERTISE_1000XFULL |
3157                                          ADVERTISE_1000XHALF |
3158                                          ADVERTISE_SLCT);
3159                                 tg3_writephy(tp, MII_ADVERTISE, adv);
3160                                 tg3_writephy(tp, MII_BMCR, bmcr |
3161                                                            BMCR_ANRESTART |
3162                                                            BMCR_ANENABLE);
3163                                 udelay(10);
3164                                 netif_carrier_off(tp->dev);
3165                         }
3166                         tg3_writephy(tp, MII_BMCR, new_bmcr);
3167                         bmcr = new_bmcr;
3168                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3169                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
3170                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
3171                             ASIC_REV_5714) {
3172                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
3173                                         bmsr |= BMSR_LSTATUS;
3174                                 else
3175                                         bmsr &= ~BMSR_LSTATUS;
3176                         }
3177                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3178                 }
3179         }
3180
3181         if (bmsr & BMSR_LSTATUS) {
3182                 current_speed = SPEED_1000;
3183                 current_link_up = 1;
3184                 if (bmcr & BMCR_FULLDPLX)
3185                         current_duplex = DUPLEX_FULL;
3186                 else
3187                         current_duplex = DUPLEX_HALF;
3188
3189                 local_adv = 0;
3190                 remote_adv = 0;
3191
3192                 if (bmcr & BMCR_ANENABLE) {
3193                         u32 common;
3194
3195                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
3196                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
3197                         common = local_adv & remote_adv;
3198                         if (common & (ADVERTISE_1000XHALF |
3199                                       ADVERTISE_1000XFULL)) {
3200                                 if (common & ADVERTISE_1000XFULL)
3201                                         current_duplex = DUPLEX_FULL;
3202                                 else
3203                                         current_duplex = DUPLEX_HALF;
3204                         }
3205                         else
3206                                 current_link_up = 0;
3207                 }
3208         }
3209
3210         if (current_link_up == 1 && current_duplex == DUPLEX_FULL)
3211                 tg3_setup_flow_control(tp, local_adv, remote_adv);
3212
3213         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
3214         if (tp->link_config.active_duplex == DUPLEX_HALF)
3215                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
3216
3217         tw32_f(MAC_MODE, tp->mac_mode);
3218         udelay(40);
3219
3220         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3221
3222         tp->link_config.active_speed = current_speed;
3223         tp->link_config.active_duplex = current_duplex;
3224
3225         if (current_link_up != netif_carrier_ok(tp->dev)) {
3226                 if (current_link_up)
3227                         netif_carrier_on(tp->dev);
3228                 else {
3229                         netif_carrier_off(tp->dev);
3230                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3231                 }
3232                 tg3_link_report(tp);
3233         }
3234         return err;
3235 }
3236
3237 static void tg3_serdes_parallel_detect(struct tg3 *tp)
3238 {
3239         if (tp->serdes_counter) {
3240                 /* Give autoneg time to complete. */
3241                 tp->serdes_counter--;
3242                 return;
3243         }
3244         if (!netif_carrier_ok(tp->dev) &&
3245             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
3246                 u32 bmcr;
3247
3248                 tg3_readphy(tp, MII_BMCR, &bmcr);
3249                 if (bmcr & BMCR_ANENABLE) {
3250                         u32 phy1, phy2;
3251
3252                         /* Select shadow register 0x1f */
3253                         tg3_writephy(tp, 0x1c, 0x7c00);
3254                         tg3_readphy(tp, 0x1c, &phy1);
3255
3256                         /* Select expansion interrupt status register */
3257                         tg3_writephy(tp, 0x17, 0x0f01);
3258                         tg3_readphy(tp, 0x15, &phy2);
3259                         tg3_readphy(tp, 0x15, &phy2);
3260
3261                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
3262                                 /* We have signal detect and not receiving
3263                                  * config code words, link is up by parallel
3264                                  * detection.
3265                                  */
3266
3267                                 bmcr &= ~BMCR_ANENABLE;
3268                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
3269                                 tg3_writephy(tp, MII_BMCR, bmcr);
3270                                 tp->tg3_flags2 |= TG3_FLG2_PARALLEL_DETECT;
3271                         }
3272                 }
3273         }
3274         else if (netif_carrier_ok(tp->dev) &&
3275                  (tp->link_config.autoneg == AUTONEG_ENABLE) &&
3276                  (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
3277                 u32 phy2;
3278
3279                 /* Select expansion interrupt status register */
3280                 tg3_writephy(tp, 0x17, 0x0f01);
3281                 tg3_readphy(tp, 0x15, &phy2);
3282                 if (phy2 & 0x20) {
3283                         u32 bmcr;
3284
3285                         /* Config code words received, turn on autoneg. */
3286                         tg3_readphy(tp, MII_BMCR, &bmcr);
3287                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
3288
3289                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3290
3291                 }
3292         }
3293 }
3294
3295 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
3296 {
3297         int err;
3298
3299         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
3300                 err = tg3_setup_fiber_phy(tp, force_reset);
3301         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
3302                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
3303         } else {
3304                 err = tg3_setup_copper_phy(tp, force_reset);
3305         }
3306
3307         if (tp->pci_chip_rev_id == CHIPREV_ID_5784_A0 ||
3308             tp->pci_chip_rev_id == CHIPREV_ID_5784_A1) {
3309                 u32 val, scale;
3310
3311                 val = tr32(TG3_CPMU_CLCK_STAT) & CPMU_CLCK_STAT_MAC_CLCK_MASK;
3312                 if (val == CPMU_CLCK_STAT_MAC_CLCK_62_5)
3313                         scale = 65;
3314                 else if (val == CPMU_CLCK_STAT_MAC_CLCK_6_25)
3315                         scale = 6;
3316                 else
3317                         scale = 12;
3318
3319                 val = tr32(GRC_MISC_CFG) & ~GRC_MISC_CFG_PRESCALAR_MASK;
3320                 val |= (scale << GRC_MISC_CFG_PRESCALAR_SHIFT);
3321                 tw32(GRC_MISC_CFG, val);
3322         }
3323
3324         if (tp->link_config.active_speed == SPEED_1000 &&
3325             tp->link_config.active_duplex == DUPLEX_HALF)
3326                 tw32(MAC_TX_LENGTHS,
3327                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
3328                       (6 << TX_LENGTHS_IPG_SHIFT) |
3329                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
3330         else
3331                 tw32(MAC_TX_LENGTHS,
3332                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
3333                       (6 << TX_LENGTHS_IPG_SHIFT) |
3334                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
3335
3336         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
3337                 if (netif_carrier_ok(tp->dev)) {
3338                         tw32(HOSTCC_STAT_COAL_TICKS,
3339                              tp->coal.stats_block_coalesce_usecs);
3340                 } else {
3341                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
3342                 }
3343         }
3344
3345         if (tp->tg3_flags & TG3_FLAG_ASPM_WORKAROUND) {
3346                 u32 val = tr32(PCIE_PWR_MGMT_THRESH);
3347                 if (!netif_carrier_ok(tp->dev))
3348                         val = (val & ~PCIE_PWR_MGMT_L1_THRESH_MSK) |
3349                               tp->pwrmgmt_thresh;
3350                 else
3351                         val |= PCIE_PWR_MGMT_L1_THRESH_MSK;
3352                 tw32(PCIE_PWR_MGMT_THRESH, val);
3353         }
3354
3355         return err;
3356 }
3357
3358 /* This is called whenever we suspect that the system chipset is re-
3359  * ordering the sequence of MMIO to the tx send mailbox. The symptom
3360  * is bogus tx completions. We try to recover by setting the
3361  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
3362  * in the workqueue.
3363  */
3364 static void tg3_tx_recover(struct tg3 *tp)
3365 {
3366         BUG_ON((tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) ||
3367                tp->write32_tx_mbox == tg3_write_indirect_mbox);
3368
3369         printk(KERN_WARNING PFX "%s: The system may be re-ordering memory-"
3370                "mapped I/O cycles to the network device, attempting to "
3371                "recover. Please report the problem to the driver maintainer "
3372                "and include system chipset information.\n", tp->dev->name);
3373
3374         spin_lock(&tp->lock);
3375         tp->tg3_flags |= TG3_FLAG_TX_RECOVERY_PENDING;
3376         spin_unlock(&tp->lock);
3377 }
3378
3379 static inline u32 tg3_tx_avail(struct tg3 *tp)
3380 {
3381         smp_mb();
3382         return (tp->tx_pending -
3383                 ((tp->tx_prod - tp->tx_cons) & (TG3_TX_RING_SIZE - 1)));
3384 }
3385
3386 /* Tigon3 never reports partial packet sends.  So we do not
3387  * need special logic to handle SKBs that have not had all
3388  * of their frags sent yet, like SunGEM does.
3389  */
3390 static void tg3_tx(struct tg3 *tp)
3391 {
3392         u32 hw_idx = tp->hw_status->idx[0].tx_consumer;
3393         u32 sw_idx = tp->tx_cons;
3394
3395         while (sw_idx != hw_idx) {
3396                 struct tx_ring_info *ri = &tp->tx_buffers[sw_idx];
3397                 struct sk_buff *skb = ri->skb;
3398                 int i, tx_bug = 0;
3399
3400                 if (unlikely(skb == NULL)) {
3401                         tg3_tx_recover(tp);
3402                         return;
3403                 }
3404
3405                 pci_unmap_single(tp->pdev,
3406                                  pci_unmap_addr(ri, mapping),
3407                                  skb_headlen(skb),
3408                                  PCI_DMA_TODEVICE);
3409
3410                 ri->skb = NULL;
3411
3412                 sw_idx = NEXT_TX(sw_idx);
3413
3414                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3415                         ri = &tp->tx_buffers[sw_idx];
3416                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
3417                                 tx_bug = 1;
3418
3419                         pci_unmap_page(tp->pdev,
3420                                        pci_unmap_addr(ri, mapping),
3421                                        skb_shinfo(skb)->frags[i].size,
3422                                        PCI_DMA_TODEVICE);
3423
3424                         sw_idx = NEXT_TX(sw_idx);
3425                 }
3426
3427                 dev_kfree_skb(skb);
3428
3429                 if (unlikely(tx_bug)) {
3430                         tg3_tx_recover(tp);
3431                         return;
3432                 }
3433         }
3434
3435         tp->tx_cons = sw_idx;
3436
3437         /* Need to make the tx_cons update visible to tg3_start_xmit()
3438          * before checking for netif_queue_stopped().  Without the
3439          * memory barrier, there is a small possibility that tg3_start_xmit()
3440          * will miss it and cause the queue to be stopped forever.
3441          */
3442         smp_mb();
3443
3444         if (unlikely(netif_queue_stopped(tp->dev) &&
3445                      (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp)))) {
3446                 netif_tx_lock(tp->dev);
3447                 if (netif_queue_stopped(tp->dev) &&
3448                     (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp)))
3449                         netif_wake_queue(tp->dev);
3450                 netif_tx_unlock(tp->dev);
3451         }
3452 }
3453
3454 /* Returns size of skb allocated or < 0 on error.
3455  *
3456  * We only need to fill in the address because the other members
3457  * of the RX descriptor are invariant, see tg3_init_rings.
3458  *
3459  * Note the purposeful assymetry of cpu vs. chip accesses.  For
3460  * posting buffers we only dirty the first cache line of the RX
3461  * descriptor (containing the address).  Whereas for the RX status
3462  * buffers the cpu only reads the last cacheline of the RX descriptor
3463  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
3464  */
3465 static int tg3_alloc_rx_skb(struct tg3 *tp, u32 opaque_key,
3466                             int src_idx, u32 dest_idx_unmasked)
3467 {
3468         struct tg3_rx_buffer_desc *desc;
3469         struct ring_info *map, *src_map;
3470         struct sk_buff *skb;
3471         dma_addr_t mapping;
3472         int skb_size, dest_idx;
3473
3474         src_map = NULL;
3475         switch (opaque_key) {
3476         case RXD_OPAQUE_RING_STD:
3477                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3478                 desc = &tp->rx_std[dest_idx];
3479                 map = &tp->rx_std_buffers[dest_idx];
3480                 if (src_idx >= 0)
3481                         src_map = &tp->rx_std_buffers[src_idx];
3482                 skb_size = tp->rx_pkt_buf_sz;
3483                 break;
3484
3485         case RXD_OPAQUE_RING_JUMBO:
3486                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3487                 desc = &tp->rx_jumbo[dest_idx];
3488                 map = &tp->rx_jumbo_buffers[dest_idx];
3489                 if (src_idx >= 0)
3490                         src_map = &tp->rx_jumbo_buffers[src_idx];
3491                 skb_size = RX_JUMBO_PKT_BUF_SZ;
3492                 break;
3493
3494         default:
3495                 return -EINVAL;
3496         };
3497
3498         /* Do not overwrite any of the map or rp information
3499          * until we are sure we can commit to a new buffer.
3500          *
3501          * Callers depend upon this behavior and assume that
3502          * we leave everything unchanged if we fail.
3503          */
3504         skb = netdev_alloc_skb(tp->dev, skb_size);
3505         if (skb == NULL)
3506                 return -ENOMEM;
3507
3508         skb_reserve(skb, tp->rx_offset);
3509
3510         mapping = pci_map_single(tp->pdev, skb->data,
3511                                  skb_size - tp->rx_offset,
3512                                  PCI_DMA_FROMDEVICE);
3513
3514         map->skb = skb;
3515         pci_unmap_addr_set(map, mapping, mapping);
3516
3517         if (src_map != NULL)
3518                 src_map->skb = NULL;
3519
3520         desc->addr_hi = ((u64)mapping >> 32);
3521         desc->addr_lo = ((u64)mapping & 0xffffffff);
3522
3523         return skb_size;
3524 }
3525
3526 /* We only need to move over in the address because the other
3527  * members of the RX descriptor are invariant.  See notes above
3528  * tg3_alloc_rx_skb for full details.
3529  */
3530 static void tg3_recycle_rx(struct tg3 *tp, u32 opaque_key,
3531                            int src_idx, u32 dest_idx_unmasked)
3532 {
3533         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
3534         struct ring_info *src_map, *dest_map;
3535         int dest_idx;
3536
3537         switch (opaque_key) {
3538         case RXD_OPAQUE_RING_STD:
3539                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3540                 dest_desc = &tp->rx_std[dest_idx];
3541                 dest_map = &tp->rx_std_buffers[dest_idx];
3542                 src_desc = &tp->rx_std[src_idx];
3543                 src_map = &tp->rx_std_buffers[src_idx];
3544                 break;
3545
3546         case RXD_OPAQUE_RING_JUMBO:
3547                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3548                 dest_desc = &tp->rx_jumbo[dest_idx];
3549                 dest_map = &tp->rx_jumbo_buffers[dest_idx];
3550                 src_desc = &tp->rx_jumbo[src_idx];
3551                 src_map = &tp->rx_jumbo_buffers[src_idx];
3552                 break;
3553
3554         default:
3555                 return;
3556         };
3557
3558         dest_map->skb = src_map->skb;
3559         pci_unmap_addr_set(dest_map, mapping,
3560                            pci_unmap_addr(src_map, mapping));
3561         dest_desc->addr_hi = src_desc->addr_hi;
3562         dest_desc->addr_lo = src_desc->addr_lo;
3563
3564         src_map->skb = NULL;
3565 }
3566
3567 #if TG3_VLAN_TAG_USED
3568 static int tg3_vlan_rx(struct tg3 *tp, struct sk_buff *skb, u16 vlan_tag)
3569 {
3570         return vlan_hwaccel_receive_skb(skb, tp->vlgrp, vlan_tag);
3571 }
3572 #endif
3573
3574 /* The RX ring scheme is composed of multiple rings which post fresh
3575  * buffers to the chip, and one special ring the chip uses to report
3576  * status back to the host.
3577  *
3578  * The special ring reports the status of received packets to the
3579  * host.  The chip does not write into the original descriptor the
3580  * RX buffer was obtained from.  The chip simply takes the original
3581  * descriptor as provided by the host, updates the status and length
3582  * field, then writes this into the next status ring entry.
3583  *
3584  * Each ring the host uses to post buffers to the chip is described
3585  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
3586  * it is first placed into the on-chip ram.  When the packet's length
3587  * is known, it walks down the TG3_BDINFO entries to select the ring.
3588  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
3589  * which is within the range of the new packet's length is chosen.
3590  *
3591  * The "separate ring for rx status" scheme may sound queer, but it makes
3592  * sense from a cache coherency perspective.  If only the host writes
3593  * to the buffer post rings, and only the chip writes to the rx status
3594  * rings, then cache lines never move beyond shared-modified state.
3595  * If both the host and chip were to write into the same ring, cache line
3596  * eviction could occur since both entities want it in an exclusive state.
3597  */
3598 static int tg3_rx(struct tg3 *tp, int budget)
3599 {
3600         u32 work_mask, rx_std_posted = 0;
3601         u32 sw_idx = tp->rx_rcb_ptr;
3602         u16 hw_idx;
3603         int received;
3604
3605         hw_idx = tp->hw_status->idx[0].rx_producer;
3606         /*
3607          * We need to order the read of hw_idx and the read of
3608          * the opaque cookie.
3609          */
3610         rmb();
3611         work_mask = 0;
3612         received = 0;
3613         while (sw_idx != hw_idx && budget > 0) {
3614                 struct tg3_rx_buffer_desc *desc = &tp->rx_rcb[sw_idx];
3615                 unsigned int len;
3616                 struct sk_buff *skb;
3617                 dma_addr_t dma_addr;
3618                 u32 opaque_key, desc_idx, *post_ptr;
3619
3620                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
3621                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
3622                 if (opaque_key == RXD_OPAQUE_RING_STD) {
3623                         dma_addr = pci_unmap_addr(&tp->rx_std_buffers[desc_idx],
3624                                                   mapping);
3625                         skb = tp->rx_std_buffers[desc_idx].skb;
3626                         post_ptr = &tp->rx_std_ptr;
3627                         rx_std_posted++;
3628                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
3629                         dma_addr = pci_unmap_addr(&tp->rx_jumbo_buffers[desc_idx],
3630                                                   mapping);
3631                         skb = tp->rx_jumbo_buffers[desc_idx].skb;
3632                         post_ptr = &tp->rx_jumbo_ptr;
3633                 }
3634                 else {
3635                         goto next_pkt_nopost;
3636                 }
3637
3638                 work_mask |= opaque_key;
3639
3640                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
3641                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
3642                 drop_it:
3643                         tg3_recycle_rx(tp, opaque_key,
3644                                        desc_idx, *post_ptr);
3645                 drop_it_no_recycle:
3646                         /* Other statistics kept track of by card. */
3647                         tp->net_stats.rx_dropped++;
3648                         goto next_pkt;
3649                 }
3650
3651                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4; /* omit crc */
3652
3653                 if (len > RX_COPY_THRESHOLD
3654                         && tp->rx_offset == 2
3655                         /* rx_offset != 2 iff this is a 5701 card running
3656                          * in PCI-X mode [see tg3_get_invariants()] */
3657                 ) {
3658                         int skb_size;
3659
3660                         skb_size = tg3_alloc_rx_skb(tp, opaque_key,
3661                                                     desc_idx, *post_ptr);
3662                         if (skb_size < 0)
3663                                 goto drop_it;
3664
3665                         pci_unmap_single(tp->pdev, dma_addr,
3666                                          skb_size - tp->rx_offset,
3667                                          PCI_DMA_FROMDEVICE);
3668
3669                         skb_put(skb, len);
3670                 } else {
3671                         struct sk_buff *copy_skb;
3672
3673                         tg3_recycle_rx(tp, opaque_key,
3674                                        desc_idx, *post_ptr);
3675
3676                         copy_skb = netdev_alloc_skb(tp->dev, len + 2);
3677                         if (copy_skb == NULL)
3678                                 goto drop_it_no_recycle;
3679
3680                         skb_reserve(copy_skb, 2);
3681                         skb_put(copy_skb, len);
3682                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3683                         skb_copy_from_linear_data(skb, copy_skb->data, len);
3684                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3685
3686                         /* We'll reuse the original ring buffer. */
3687                         skb = copy_skb;
3688                 }
3689
3690                 if ((tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) &&
3691                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
3692                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
3693                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
3694                         skb->ip_summed = CHECKSUM_UNNECESSARY;
3695                 else
3696                         skb->ip_summed = CHECKSUM_NONE;
3697
3698                 skb->protocol = eth_type_trans(skb, tp->dev);
3699 #if TG3_VLAN_TAG_USED
3700                 if (tp->vlgrp != NULL &&
3701                     desc->type_flags & RXD_FLAG_VLAN) {
3702                         tg3_vlan_rx(tp, skb,
3703                                     desc->err_vlan & RXD_VLAN_MASK);
3704                 } else
3705 #endif
3706                         netif_receive_skb(skb);
3707
3708                 tp->dev->last_rx = jiffies;
3709                 received++;
3710                 budget--;
3711
3712 next_pkt:
3713                 (*post_ptr)++;
3714
3715                 if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
3716                         u32 idx = *post_ptr % TG3_RX_RING_SIZE;
3717
3718                         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX +
3719                                      TG3_64BIT_REG_LOW, idx);
3720                         work_mask &= ~RXD_OPAQUE_RING_STD;
3721                         rx_std_posted = 0;
3722                 }
3723 next_pkt_nopost:
3724                 sw_idx++;
3725                 sw_idx &= (TG3_RX_RCB_RING_SIZE(tp) - 1);
3726
3727                 /* Refresh hw_idx to see if there is new work */
3728                 if (sw_idx == hw_idx) {
3729                         hw_idx = tp->hw_status->idx[0].rx_producer;
3730                         rmb();
3731                 }
3732         }
3733
3734         /* ACK the status ring. */
3735         tp->rx_rcb_ptr = sw_idx;
3736         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, sw_idx);
3737
3738         /* Refill RX ring(s). */
3739         if (work_mask & RXD_OPAQUE_RING_STD) {
3740                 sw_idx = tp->rx_std_ptr % TG3_RX_RING_SIZE;
3741                 tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
3742                              sw_idx);
3743         }
3744         if (work_mask & RXD_OPAQUE_RING_JUMBO) {
3745                 sw_idx = tp->rx_jumbo_ptr % TG3_RX_JUMBO_RING_SIZE;
3746                 tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
3747                              sw_idx);
3748         }
3749         mmiowb();
3750
3751         return received;
3752 }
3753
3754 static int tg3_poll_work(struct tg3 *tp, int work_done, int budget)
3755 {
3756         struct tg3_hw_status *sblk = tp->hw_status;
3757
3758         /* handle link change and other phy events */
3759         if (!(tp->tg3_flags &
3760               (TG3_FLAG_USE_LINKCHG_REG |
3761                TG3_FLAG_POLL_SERDES))) {
3762                 if (sblk->status & SD_STATUS_LINK_CHG) {
3763                         sblk->status = SD_STATUS_UPDATED |
3764                                 (sblk->status & ~SD_STATUS_LINK_CHG);
3765                         spin_lock(&tp->lock);
3766                         tg3_setup_phy(tp, 0);
3767                         spin_unlock(&tp->lock);
3768                 }
3769         }
3770
3771         /* run TX completion thread */
3772         if (sblk->idx[0].tx_consumer != tp->tx_cons) {
3773                 tg3_tx(tp);
3774                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
3775                         return work_done;
3776         }
3777
3778         /* run RX thread, within the bounds set by NAPI.
3779          * All RX "locking" is done by ensuring outside
3780          * code synchronizes with tg3->napi.poll()
3781          */
3782         if (sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
3783                 work_done += tg3_rx(tp, budget - work_done);
3784
3785         return work_done;
3786 }
3787
3788 static int tg3_poll(struct napi_struct *napi, int budget)
3789 {
3790         struct tg3 *tp = container_of(napi, struct tg3, napi);
3791         int work_done = 0;
3792         struct tg3_hw_status *sblk = tp->hw_status;
3793
3794         while (1) {
3795                 work_done = tg3_poll_work(tp, work_done, budget);
3796
3797                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
3798                         goto tx_recovery;
3799
3800                 if (unlikely(work_done >= budget))
3801                         break;
3802
3803                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
3804                         /* tp->last_tag is used in tg3_restart_ints() below
3805                          * to tell the hw how much work has been processed,
3806                          * so we must read it before checking for more work.
3807                          */
3808                         tp->last_tag = sblk->status_tag;
3809                         rmb();
3810                 } else
3811                         sblk->status &= ~SD_STATUS_UPDATED;
3812
3813                 if (likely(!tg3_has_work(tp))) {
3814                         netif_rx_complete(tp->dev, napi);
3815                         tg3_restart_ints(tp);
3816                         break;
3817                 }
3818         }
3819
3820         return work_done;
3821
3822 tx_recovery:
3823         /* work_done is guaranteed to be less than budget. */
3824         netif_rx_complete(tp->dev, napi);
3825         schedule_work(&tp->reset_task);
3826         return work_done;
3827 }
3828
3829 static void tg3_irq_quiesce(struct tg3 *tp)
3830 {
3831         BUG_ON(tp->irq_sync);
3832
3833         tp->irq_sync = 1;
3834         smp_mb();
3835
3836         synchronize_irq(tp->pdev->irq);
3837 }
3838
3839 static inline int tg3_irq_sync(struct tg3 *tp)
3840 {
3841         return tp->irq_sync;
3842 }
3843
3844 /* Fully shutdown all tg3 driver activity elsewhere in the system.
3845  * If irq_sync is non-zero, then the IRQ handler must be synchronized
3846  * with as well.  Most of the time, this is not necessary except when
3847  * shutting down the device.
3848  */
3849 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
3850 {
3851         spin_lock_bh(&tp->lock);
3852         if (irq_sync)
3853                 tg3_irq_quiesce(tp);
3854 }
3855
3856 static inline void tg3_full_unlock(struct tg3 *tp)
3857 {
3858         spin_unlock_bh(&tp->lock);
3859 }
3860
3861 /* One-shot MSI handler - Chip automatically disables interrupt
3862  * after sending MSI so driver doesn't have to do it.
3863  */
3864 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
3865 {
3866         struct net_device *dev = dev_id;
3867         struct tg3 *tp = netdev_priv(dev);
3868
3869         prefetch(tp->hw_status);
3870         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3871
3872         if (likely(!tg3_irq_sync(tp)))
3873                 netif_rx_schedule(dev, &tp->napi);
3874
3875         return IRQ_HANDLED;
3876 }
3877
3878 /* MSI ISR - No need to check for interrupt sharing and no need to
3879  * flush status block and interrupt mailbox. PCI ordering rules
3880  * guarantee that MSI will arrive after the status block.
3881  */
3882 static irqreturn_t tg3_msi(int irq, void *dev_id)
3883 {
3884         struct net_device *dev = dev_id;
3885         struct tg3 *tp = netdev_priv(dev);
3886
3887         prefetch(tp->hw_status);
3888         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3889         /*
3890          * Writing any value to intr-mbox-0 clears PCI INTA# and
3891          * chip-internal interrupt pending events.
3892          * Writing non-zero to intr-mbox-0 additional tells the
3893          * NIC to stop sending us irqs, engaging "in-intr-handler"
3894          * event coalescing.
3895          */
3896         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3897         if (likely(!tg3_irq_sync(tp)))
3898                 netif_rx_schedule(dev, &tp->napi);
3899
3900         return IRQ_RETVAL(1);
3901 }
3902
3903 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
3904 {
3905         struct net_device *dev = dev_id;
3906         struct tg3 *tp = netdev_priv(dev);
3907         struct tg3_hw_status *sblk = tp->hw_status;
3908         unsigned int handled = 1;
3909
3910         /* In INTx mode, it is possible for the interrupt to arrive at
3911          * the CPU before the status block posted prior to the interrupt.
3912          * Reading the PCI State register will confirm whether the
3913          * interrupt is ours and will flush the status block.
3914          */
3915         if (unlikely(!(sblk->status & SD_STATUS_UPDATED))) {
3916                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
3917                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3918                         handled = 0;
3919                         goto out;
3920                 }
3921         }
3922
3923         /*
3924          * Writing any value to intr-mbox-0 clears PCI INTA# and
3925          * chip-internal interrupt pending events.
3926          * Writing non-zero to intr-mbox-0 additional tells the
3927          * NIC to stop sending us irqs, engaging "in-intr-handler"
3928          * event coalescing.
3929          *
3930          * Flush the mailbox to de-assert the IRQ immediately to prevent
3931          * spurious interrupts.  The flush impacts performance but
3932          * excessive spurious interrupts can be worse in some cases.
3933          */
3934         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3935         if (tg3_irq_sync(tp))
3936                 goto out;
3937         sblk->status &= ~SD_STATUS_UPDATED;
3938         if (likely(tg3_has_work(tp))) {
3939                 prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3940                 netif_rx_schedule(dev, &tp->napi);
3941         } else {
3942                 /* No work, shared interrupt perhaps?  re-enable
3943                  * interrupts, and flush that PCI write
3944                  */
3945                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3946                                0x00000000);
3947         }
3948 out:
3949         return IRQ_RETVAL(handled);
3950 }
3951
3952 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
3953 {
3954         struct net_device *dev = dev_id;
3955         struct tg3 *tp = netdev_priv(dev);
3956         struct tg3_hw_status *sblk = tp->hw_status;
3957         unsigned int handled = 1;
3958
3959         /* In INTx mode, it is possible for the interrupt to arrive at
3960          * the CPU before the status block posted prior to the interrupt.
3961          * Reading the PCI State register will confirm whether the
3962          * interrupt is ours and will flush the status block.
3963          */
3964         if (unlikely(sblk->status_tag == tp->last_tag)) {
3965                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
3966                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3967                         handled = 0;
3968                         goto out;
3969                 }
3970         }
3971
3972         /*
3973          * writing any value to intr-mbox-0 clears PCI INTA# and
3974          * chip-internal interrupt pending events.
3975          * writing non-zero to intr-mbox-0 additional tells the
3976          * NIC to stop sending us irqs, engaging "in-intr-handler"
3977          * event coalescing.
3978          *
3979          * Flush the mailbox to de-assert the IRQ immediately to prevent
3980          * spurious interrupts.  The flush impacts performance but
3981          * excessive spurious interrupts can be worse in some cases.
3982          */
3983         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3984         if (tg3_irq_sync(tp))
3985                 goto out;
3986         if (netif_rx_schedule_prep(dev, &tp->napi)) {
3987                 prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3988                 /* Update last_tag to mark that this status has been
3989                  * seen. Because interrupt may be shared, we may be
3990                  * racing with tg3_poll(), so only update last_tag
3991                  * if tg3_poll() is not scheduled.
3992                  */
3993                 tp->last_tag = sblk->status_tag;
3994                 __netif_rx_schedule(dev, &tp->napi);
3995         }
3996 out:
3997         return IRQ_RETVAL(handled);
3998 }
3999
4000 /* ISR for interrupt test */
4001 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
4002 {
4003         struct net_device *dev = dev_id;
4004         struct tg3 *tp = netdev_priv(dev);
4005         struct tg3_hw_status *sblk = tp->hw_status;
4006
4007         if ((sblk->status & SD_STATUS_UPDATED) ||
4008             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
4009                 tg3_disable_ints(tp);
4010                 return IRQ_RETVAL(1);
4011         }
4012         return IRQ_RETVAL(0);
4013 }
4014
4015 static int tg3_init_hw(struct tg3 *, int);
4016 static int tg3_halt(struct tg3 *, int, int);
4017
4018 /* Restart hardware after configuration changes, self-test, etc.
4019  * Invoked with tp->lock held.
4020  */
4021 static int tg3_restart_hw(struct tg3 *tp, int reset_phy)
4022         __releases(tp->lock)
4023         __acquires(tp->lock)
4024 {
4025         int err;
4026
4027         err = tg3_init_hw(tp, reset_phy);
4028         if (err) {
4029                 printk(KERN_ERR PFX "%s: Failed to re-initialize device, "
4030                        "aborting.\n", tp->dev->name);
4031                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
4032                 tg3_full_unlock(tp);
4033                 del_timer_sync(&tp->timer);
4034                 tp->irq_sync = 0;
4035                 napi_enable(&tp->napi);
4036                 dev_close(tp->dev);
4037                 tg3_full_lock(tp, 0);
4038         }
4039         return err;
4040 }
4041
4042 #ifdef CONFIG_NET_POLL_CONTROLLER
4043 static void tg3_poll_controller(struct net_device *dev)
4044 {
4045         struct tg3 *tp = netdev_priv(dev);
4046
4047         tg3_interrupt(tp->pdev->irq, dev);
4048 }
4049 #endif
4050
4051 static void tg3_reset_task(struct work_struct *work)
4052 {
4053         struct tg3 *tp = container_of(work, struct tg3, reset_task);
4054         unsigned int restart_timer;
4055
4056         tg3_full_lock(tp, 0);
4057
4058         if (!netif_running(tp->dev)) {
4059                 tg3_full_unlock(tp);
4060                 return;
4061         }
4062
4063         tg3_full_unlock(tp);
4064
4065         tg3_netif_stop(tp);
4066
4067         tg3_full_lock(tp, 1);
4068
4069         restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER;
4070         tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER;
4071
4072         if (tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING) {
4073                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
4074                 tp->write32_rx_mbox = tg3_write_flush_reg32;
4075                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
4076                 tp->tg3_flags &= ~TG3_FLAG_TX_RECOVERY_PENDING;
4077         }
4078
4079         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
4080         if (tg3_init_hw(tp, 1))
4081                 goto out;
4082
4083         tg3_netif_start(tp);
4084
4085         if (restart_timer)
4086                 mod_timer(&tp->timer, jiffies + 1);
4087
4088 out:
4089         tg3_full_unlock(tp);
4090 }
4091
4092 static void tg3_dump_short_state(struct tg3 *tp)
4093 {
4094         printk(KERN_ERR PFX "DEBUG: MAC_TX_STATUS[%08x] MAC_RX_STATUS[%08x]\n",
4095                tr32(MAC_TX_STATUS), tr32(MAC_RX_STATUS));
4096         printk(KERN_ERR PFX "DEBUG: RDMAC_STATUS[%08x] WDMAC_STATUS[%08x]\n",
4097                tr32(RDMAC_STATUS), tr32(WDMAC_STATUS));
4098 }
4099
4100 static void tg3_tx_timeout(struct net_device *dev)
4101 {
4102         struct tg3 *tp = netdev_priv(dev);
4103
4104         if (netif_msg_tx_err(tp)) {
4105                 printk(KERN_ERR PFX "%s: transmit timed out, resetting\n",
4106                        dev->name);
4107                 tg3_dump_short_state(tp);
4108         }
4109
4110         schedule_work(&tp->reset_task);
4111 }
4112
4113 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
4114 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
4115 {
4116         u32 base = (u32) mapping & 0xffffffff;
4117
4118         return ((base > 0xffffdcc0) &&
4119                 (base + len + 8 < base));
4120 }
4121
4122 /* Test for DMA addresses > 40-bit */
4123 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
4124                                           int len)
4125 {
4126 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
4127         if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG)
4128                 return (((u64) mapping + len) > DMA_40BIT_MASK);
4129         return 0;
4130 #else
4131         return 0;
4132 #endif
4133 }
4134
4135 static void tg3_set_txd(struct tg3 *, int, dma_addr_t, int, u32, u32);
4136
4137 /* Workaround 4GB and 40-bit hardware DMA bugs. */
4138 static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb,
4139                                        u32 last_plus_one, u32 *start,
4140                                        u32 base_flags, u32 mss)
4141 {
4142         struct sk_buff *new_skb;
4143         dma_addr_t new_addr = 0;
4144         u32 entry = *start;
4145         int i, ret = 0;
4146
4147         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
4148                 new_skb = skb_copy(skb, GFP_ATOMIC);
4149         else {
4150                 int more_headroom = 4 - ((unsigned long)skb->data & 3);
4151
4152                 new_skb = skb_copy_expand(skb,
4153                                           skb_headroom(skb) + more_headroom,
4154                                           skb_tailroom(skb), GFP_ATOMIC);
4155         }
4156
4157         if (!new_skb) {
4158                 ret = -1;
4159         } else {
4160                 /* New SKB is guaranteed to be linear. */
4161                 entry = *start;
4162                 new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
4163                                           PCI_DMA_TODEVICE);
4164                 /* Make sure new skb does not cross any 4G boundaries.
4165                  * Drop the packet if it does.
4166                  */
4167                 if (tg3_4g_overflow_test(new_addr, new_skb->len)) {
4168                         ret = -1;
4169                         dev_kfree_skb(new_skb);
4170                         new_skb = NULL;
4171                 } else {
4172                         tg3_set_txd(tp, entry, new_addr, new_skb->len,
4173                                     base_flags, 1 | (mss << 1));
4174                         *start = NEXT_TX(entry);
4175                 }
4176         }
4177
4178         /* Now clean up the sw ring entries. */
4179         i = 0;
4180         while (entry != last_plus_one) {
4181                 int len;
4182
4183                 if (i == 0)
4184                         len = skb_headlen(skb);
4185                 else
4186                         len = skb_shinfo(skb)->frags[i-1].size;
4187                 pci_unmap_single(tp->pdev,
4188                                  pci_unmap_addr(&tp->tx_buffers[entry], mapping),
4189                                  len, PCI_DMA_TODEVICE);
4190                 if (i == 0) {
4191                         tp->tx_buffers[entry].skb = new_skb;
4192                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, new_addr);
4193                 } else {
4194                         tp->tx_buffers[entry].skb = NULL;
4195                 }
4196                 entry = NEXT_TX(entry);
4197                 i++;
4198         }
4199
4200         dev_kfree_skb(skb);
4201
4202         return ret;
4203 }
4204
4205 static void tg3_set_txd(struct tg3 *tp, int entry,
4206                         dma_addr_t mapping, int len, u32 flags,
4207                         u32 mss_and_is_end)
4208 {
4209         struct tg3_tx_buffer_desc *txd = &tp->tx_ring[entry];
4210         int is_end = (mss_and_is_end & 0x1);
4211         u32 mss = (mss_and_is_end >> 1);
4212         u32 vlan_tag = 0;
4213
4214         if (is_end)
4215                 flags |= TXD_FLAG_END;
4216         if (flags & TXD_FLAG_VLAN) {
4217                 vlan_tag = flags >> 16;
4218                 flags &= 0xffff;
4219         }
4220         vlan_tag |= (mss << TXD_MSS_SHIFT);
4221
4222         txd->addr_hi = ((u64) mapping >> 32);
4223         txd->addr_lo = ((u64) mapping & 0xffffffff);
4224         txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
4225         txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
4226 }
4227
4228 /* hard_start_xmit for devices that don't have any bugs and
4229  * support TG3_FLG2_HW_TSO_2 only.
4230  */
4231 static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
4232 {
4233         struct tg3 *tp = netdev_priv(dev);
4234         dma_addr_t mapping;
4235         u32 len, entry, base_flags, mss;
4236
4237         len = skb_headlen(skb);
4238
4239         /* We are running in BH disabled context with netif_tx_lock
4240          * and TX reclaim runs via tp->napi.poll inside of a software
4241          * interrupt.  Furthermore, IRQ processing runs lockless so we have
4242          * no IRQ context deadlocks to worry about either.  Rejoice!
4243          */
4244         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
4245                 if (!netif_queue_stopped(dev)) {
4246                         netif_stop_queue(dev);
4247
4248                         /* This is a hard error, log it. */
4249                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
4250                                "queue awake!\n", dev->name);
4251                 }
4252                 return NETDEV_TX_BUSY;
4253         }
4254
4255         entry = tp->tx_prod;
4256         base_flags = 0;
4257         mss = 0;
4258         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
4259                 int tcp_opt_len, ip_tcp_len;
4260
4261                 if (skb_header_cloned(skb) &&
4262                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
4263                         dev_kfree_skb(skb);
4264                         goto out_unlock;
4265                 }
4266
4267                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
4268                         mss |= (skb_headlen(skb) - ETH_HLEN) << 9;
4269                 else {
4270                         struct iphdr *iph = ip_hdr(skb);
4271
4272                         tcp_opt_len = tcp_optlen(skb);
4273                         ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
4274
4275                         iph->check = 0;
4276                         iph->tot_len = htons(mss + ip_tcp_len + tcp_opt_len);
4277                         mss |= (ip_tcp_len + tcp_opt_len) << 9;
4278                 }
4279
4280                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
4281                                TXD_FLAG_CPU_POST_DMA);
4282
4283                 tcp_hdr(skb)->check = 0;
4284
4285         }
4286         else if (skb->ip_summed == CHECKSUM_PARTIAL)
4287                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
4288 #if TG3_VLAN_TAG_USED
4289         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
4290                 base_flags |= (TXD_FLAG_VLAN |
4291                                (vlan_tx_tag_get(skb) << 16));
4292 #endif
4293
4294         /* Queue skb data, a.k.a. the main skb fragment. */
4295         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
4296
4297         tp->tx_buffers[entry].skb = skb;
4298         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4299
4300         tg3_set_txd(tp, entry, mapping, len, base_flags,
4301                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
4302
4303         entry = NEXT_TX(entry);
4304
4305         /* Now loop through additional data fragments, and queue them. */
4306         if (skb_shinfo(skb)->nr_frags > 0) {
4307                 unsigned int i, last;
4308
4309                 last = skb_shinfo(skb)->nr_frags - 1;
4310                 for (i = 0; i <= last; i++) {
4311                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4312
4313                         len = frag->size;
4314                         mapping = pci_map_page(tp->pdev,
4315                                                frag->page,
4316                                                frag->page_offset,
4317                                                len, PCI_DMA_TODEVICE);
4318
4319                         tp->tx_buffers[entry].skb = NULL;
4320                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4321
4322                         tg3_set_txd(tp, entry, mapping, len,
4323                                     base_flags, (i == last) | (mss << 1));
4324
4325                         entry = NEXT_TX(entry);
4326                 }
4327         }
4328
4329         /* Packets are ready, update Tx producer idx local and on card. */
4330         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
4331
4332         tp->tx_prod = entry;
4333         if (unlikely(tg3_tx_avail(tp) <= (MAX_SKB_FRAGS + 1))) {
4334                 netif_stop_queue(dev);
4335                 if (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp))
4336                         netif_wake_queue(tp->dev);
4337         }
4338
4339 out_unlock:
4340         mmiowb();
4341
4342         dev->trans_start = jiffies;
4343
4344         return NETDEV_TX_OK;
4345 }
4346
4347 static int tg3_start_xmit_dma_bug(struct sk_buff *, struct net_device *);
4348
4349 /* Use GSO to workaround a rare TSO bug that may be triggered when the
4350  * TSO header is greater than 80 bytes.
4351  */
4352 static int tg3_tso_bug(struct tg3 *tp, struct sk_buff *skb)
4353 {
4354         struct sk_buff *segs, *nskb;
4355
4356         /* Estimate the number of fragments in the worst case */
4357         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->gso_segs * 3))) {
4358                 netif_stop_queue(tp->dev);
4359                 if (tg3_tx_avail(tp) <= (skb_shinfo(skb)->gso_segs * 3))
4360                         return NETDEV_TX_BUSY;
4361
4362                 netif_wake_queue(tp->dev);
4363         }
4364
4365         segs = skb_gso_segment(skb, tp->dev->features & ~NETIF_F_TSO);
4366         if (IS_ERR(segs))
4367                 goto tg3_tso_bug_end;
4368
4369         do {
4370                 nskb = segs;
4371                 segs = segs->next;
4372                 nskb->next = NULL;
4373                 tg3_start_xmit_dma_bug(nskb, tp->dev);
4374         } while (segs);
4375
4376 tg3_tso_bug_end:
4377         dev_kfree_skb(skb);
4378
4379         return NETDEV_TX_OK;
4380 }
4381
4382 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
4383  * support TG3_FLG2_HW_TSO_1 or firmware TSO only.
4384  */
4385 static int tg3_start_xmit_dma_bug(struct sk_buff *skb, struct net_device *dev)
4386 {
4387         struct tg3 *tp = netdev_priv(dev);
4388         dma_addr_t mapping;
4389         u32 len, entry, base_flags, mss;
4390         int would_hit_hwbug;
4391
4392         len = skb_headlen(skb);
4393
4394         /* We are running in BH disabled context with netif_tx_lock
4395          * and TX reclaim runs via tp->napi.poll inside of a software
4396          * interrupt.  Furthermore, IRQ processing runs lockless so we have
4397          * no IRQ context deadlocks to worry about either.  Rejoice!
4398          */
4399         if (unlikely(tg3_tx_avail(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
4400                 if (!netif_queue_stopped(dev)) {
4401                         netif_stop_queue(dev);
4402
4403                         /* This is a hard error, log it. */
4404                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
4405                                "queue awake!\n", dev->name);
4406                 }
4407                 return NETDEV_TX_BUSY;
4408         }
4409
4410         entry = tp->tx_prod;
4411         base_flags = 0;
4412         if (skb->ip_summed == CHECKSUM_PARTIAL)
4413                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
4414         mss = 0;
4415         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
4416                 struct iphdr *iph;
4417                 int tcp_opt_len, ip_tcp_len, hdr_len;
4418
4419                 if (skb_header_cloned(skb) &&
4420                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
4421                         dev_kfree_skb(skb);
4422                         goto out_unlock;
4423                 }
4424
4425                 tcp_opt_len = tcp_optlen(skb);
4426                 ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
4427
4428                 hdr_len = ip_tcp_len + tcp_opt_len;
4429                 if (unlikely((ETH_HLEN + hdr_len) > 80) &&
4430                              (tp->tg3_flags2 & TG3_FLG2_TSO_BUG))
4431                         return (tg3_tso_bug(tp, skb));
4432
4433                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
4434                                TXD_FLAG_CPU_POST_DMA);
4435
4436                 iph = ip_hdr(skb);
4437                 iph->check = 0;
4438                 iph->tot_len = htons(mss + hdr_len);
4439                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
4440                         tcp_hdr(skb)->check = 0;
4441                         base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
4442                 } else
4443                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4444                                                                  iph->daddr, 0,
4445                                                                  IPPROTO_TCP,
4446                                                                  0);
4447
4448                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO) ||
4449                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)) {
4450                         if (tcp_opt_len || iph->ihl > 5) {
4451                                 int tsflags;
4452
4453                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
4454                                 mss |= (tsflags << 11);
4455                         }
4456                 } else {
4457                         if (tcp_opt_len || iph->ihl > 5) {
4458                                 int tsflags;
4459
4460                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
4461                                 base_flags |= tsflags << 12;
4462                         }
4463                 }
4464         }
4465 #if TG3_VLAN_TAG_USED
4466         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
4467                 base_flags |= (TXD_FLAG_VLAN |
4468                                (vlan_tx_tag_get(skb) << 16));
4469 #endif
4470
4471         /* Queue skb data, a.k.a. the main skb fragment. */
4472         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
4473
4474         tp->tx_buffers[entry].skb = skb;
4475         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4476
4477         would_hit_hwbug = 0;
4478
4479         if (tp->tg3_flags3 & TG3_FLG3_5701_DMA_BUG)
4480                 would_hit_hwbug = 1;
4481         else if (tg3_4g_overflow_test(mapping, len))
4482                 would_hit_hwbug = 1;
4483
4484         tg3_set_txd(tp, entry, mapping, len, base_flags,
4485                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
4486
4487         entry = NEXT_TX(entry);
4488
4489         /* Now loop through additional data fragments, and queue them. */
4490         if (skb_shinfo(skb)->nr_frags > 0) {
4491                 unsigned int i, last;
4492
4493                 last = skb_shinfo(skb)->nr_frags - 1;
4494                 for (i = 0; i <= last; i++) {
4495                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4496
4497                         len = frag->size;
4498                         mapping = pci_map_page(tp->pdev,
4499                                                frag->page,
4500                                                frag->page_offset,
4501                                                len, PCI_DMA_TODEVICE);
4502
4503                         tp->tx_buffers[entry].skb = NULL;
4504                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4505
4506                         if (tg3_4g_overflow_test(mapping, len))
4507                                 would_hit_hwbug = 1;
4508
4509                         if (tg3_40bit_overflow_test(tp, mapping, len))
4510                                 would_hit_hwbug = 1;
4511
4512                         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
4513                                 tg3_set_txd(tp, entry, mapping, len,
4514                                             base_flags, (i == last)|(mss << 1));
4515                         else
4516                                 tg3_set_txd(tp, entry, mapping, len,
4517                                             base_flags, (i == last));
4518
4519                         entry = NEXT_TX(entry);
4520                 }
4521         }
4522
4523         if (would_hit_hwbug) {
4524                 u32 last_plus_one = entry;
4525                 u32 start;
4526
4527                 start = entry - 1 - skb_shinfo(skb)->nr_frags;
4528                 start &= (TG3_TX_RING_SIZE - 1);
4529
4530                 /* If the workaround fails due to memory/mapping
4531                  * failure, silently drop this packet.
4532                  */
4533                 if (tigon3_dma_hwbug_workaround(tp, skb, last_plus_one,
4534                                                 &start, base_flags, mss))
4535                         goto out_unlock;
4536
4537                 entry = start;
4538         }
4539
4540         /* Packets are ready, update Tx producer idx local and on card. */
4541         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
4542
4543         tp->tx_prod = entry;
4544         if (unlikely(tg3_tx_avail(tp) <= (MAX_SKB_FRAGS + 1))) {
4545                 netif_stop_queue(dev);
4546                 if (tg3_tx_avail(tp) > TG3_TX_WAKEUP_THRESH(tp))
4547                         netif_wake_queue(tp->dev);
4548         }
4549
4550 out_unlock:
4551         mmiowb();
4552
4553         dev->trans_start = jiffies;
4554
4555         return NETDEV_TX_OK;
4556 }
4557
4558 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
4559                                int new_mtu)
4560 {
4561         dev->mtu = new_mtu;
4562
4563         if (new_mtu > ETH_DATA_LEN) {
4564                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
4565                         tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
4566                         ethtool_op_set_tso(dev, 0);
4567                 }
4568                 else
4569                         tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
4570         } else {
4571                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
4572                         tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
4573                 tp->tg3_flags &= ~TG3_FLAG_JUMBO_RING_ENABLE;
4574         }
4575 }
4576
4577 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
4578 {
4579         struct tg3 *tp = netdev_priv(dev);
4580         int err;
4581
4582         if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
4583                 return -EINVAL;
4584
4585         if (!netif_running(dev)) {
4586                 /* We'll just catch it later when the
4587                  * device is up'd.
4588                  */
4589                 tg3_set_mtu(dev, tp, new_mtu);
4590                 return 0;
4591         }
4592
4593         tg3_netif_stop(tp);
4594
4595         tg3_full_lock(tp, 1);
4596
4597         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
4598
4599         tg3_set_mtu(dev, tp, new_mtu);
4600
4601         err = tg3_restart_hw(tp, 0);
4602
4603         if (!err)
4604                 tg3_netif_start(tp);
4605
4606         tg3_full_unlock(tp);
4607
4608         return err;
4609 }
4610
4611 /* Free up pending packets in all rx/tx rings.
4612  *
4613  * The chip has been shut down and the driver detached from
4614  * the networking, so no interrupts or new tx packets will
4615  * end up in the driver.  tp->{tx,}lock is not held and we are not
4616  * in an interrupt context and thus may sleep.
4617  */
4618 static void tg3_free_rings(struct tg3 *tp)
4619 {
4620         struct ring_info *rxp;
4621         int i;
4622
4623         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4624                 rxp = &tp->rx_std_buffers[i];
4625
4626                 if (rxp->skb == NULL)
4627                         continue;
4628                 pci_unmap_single(tp->pdev,
4629                                  pci_unmap_addr(rxp, mapping),
4630                                  tp->rx_pkt_buf_sz - tp->rx_offset,
4631                                  PCI_DMA_FROMDEVICE);
4632                 dev_kfree_skb_any(rxp->skb);
4633                 rxp->skb = NULL;
4634         }
4635
4636         for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4637                 rxp = &tp->rx_jumbo_buffers[i];
4638
4639                 if (rxp->skb == NULL)
4640                         continue;
4641                 pci_unmap_single(tp->pdev,
4642                                  pci_unmap_addr(rxp, mapping),
4643                                  RX_JUMBO_PKT_BUF_SZ - tp->rx_offset,
4644                                  PCI_DMA_FROMDEVICE);
4645                 dev_kfree_skb_any(rxp->skb);
4646                 rxp->skb = NULL;
4647         }
4648
4649         for (i = 0; i < TG3_TX_RING_SIZE; ) {
4650                 struct tx_ring_info *txp;
4651                 struct sk_buff *skb;
4652                 int j;
4653
4654                 txp = &tp->tx_buffers[i];
4655                 skb = txp->skb;
4656
4657                 if (skb == NULL) {
4658                         i++;
4659                         continue;
4660                 }
4661
4662                 pci_unmap_single(tp->pdev,
4663                                  pci_unmap_addr(txp, mapping),
4664                                  skb_headlen(skb),
4665                                  PCI_DMA_TODEVICE);
4666                 txp->skb = NULL;
4667
4668                 i++;
4669
4670                 for (j = 0; j < skb_shinfo(skb)->nr_frags; j++) {
4671                         txp = &tp->tx_buffers[i & (TG3_TX_RING_SIZE - 1)];
4672                         pci_unmap_page(tp->pdev,
4673                                        pci_unmap_addr(txp, mapping),
4674                                        skb_shinfo(skb)->frags[j].size,
4675                                        PCI_DMA_TODEVICE);
4676                         i++;
4677                 }
4678
4679                 dev_kfree_skb_any(skb);
4680         }
4681 }
4682
4683 /* Initialize tx/rx rings for packet processing.
4684  *
4685  * The chip has been shut down and the driver detached from
4686  * the networking, so no interrupts or new tx packets will
4687  * end up in the driver.  tp->{tx,}lock are held and thus
4688  * we may not sleep.
4689  */
4690 static int tg3_init_rings(struct tg3 *tp)
4691 {
4692         u32 i;
4693
4694         /* Free up all the SKBs. */
4695         tg3_free_rings(tp);
4696
4697         /* Zero out all descriptors. */
4698         memset(tp->rx_std, 0, TG3_RX_RING_BYTES);
4699         memset(tp->rx_jumbo, 0, TG3_RX_JUMBO_RING_BYTES);
4700         memset(tp->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
4701         memset(tp->tx_ring, 0, TG3_TX_RING_BYTES);
4702
4703         tp->rx_pkt_buf_sz = RX_PKT_BUF_SZ;
4704         if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) &&
4705             (tp->dev->mtu > ETH_DATA_LEN))
4706                 tp->rx_pkt_buf_sz = RX_JUMBO_PKT_BUF_SZ;
4707
4708         /* Initialize invariants of the rings, we only set this
4709          * stuff once.  This works because the card does not
4710          * write into the rx buffer posting rings.
4711          */
4712         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4713                 struct tg3_rx_buffer_desc *rxd;
4714
4715                 rxd = &tp->rx_std[i];
4716                 rxd->idx_len = (tp->rx_pkt_buf_sz - tp->rx_offset - 64)
4717                         << RXD_LEN_SHIFT;
4718                 rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
4719                 rxd->opaque = (RXD_OPAQUE_RING_STD |
4720                                (i << RXD_OPAQUE_INDEX_SHIFT));
4721         }
4722
4723         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4724                 for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4725                         struct tg3_rx_buffer_desc *rxd;
4726
4727                         rxd = &tp->rx_jumbo[i];
4728                         rxd->idx_len = (RX_JUMBO_PKT_BUF_SZ - tp->rx_offset - 64)
4729                                 << RXD_LEN_SHIFT;
4730                         rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
4731                                 RXD_FLAG_JUMBO;
4732                         rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
4733                                (i << RXD_OPAQUE_INDEX_SHIFT));
4734                 }
4735         }
4736
4737         /* Now allocate fresh SKBs for each rx ring. */
4738         for (i = 0; i < tp->rx_pending; i++) {
4739                 if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_STD, -1, i) < 0) {
4740                         printk(KERN_WARNING PFX
4741                                "%s: Using a smaller RX standard ring, "
4742                                "only %d out of %d buffers were allocated "
4743                                "successfully.\n",
4744                                tp->dev->name, i, tp->rx_pending);
4745                         if (i == 0)
4746                                 return -ENOMEM;
4747                         tp->rx_pending = i;
4748                         break;
4749                 }
4750         }
4751
4752         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4753                 for (i = 0; i < tp->rx_jumbo_pending; i++) {
4754                         if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_JUMBO,
4755                                              -1, i) < 0) {
4756                                 printk(KERN_WARNING PFX
4757                                        "%s: Using a smaller RX jumbo ring, "
4758                                        "only %d out of %d buffers were "
4759                                        "allocated successfully.\n",
4760                                        tp->dev->name, i, tp->rx_jumbo_pending);
4761                                 if (i == 0) {
4762                                         tg3_free_rings(tp);
4763                                         return -ENOMEM;
4764                                 }
4765                                 tp->rx_jumbo_pending = i;
4766                                 break;
4767                         }
4768                 }
4769         }
4770         return 0;
4771 }
4772
4773 /*
4774  * Must not be invoked with interrupt sources disabled and
4775  * the hardware shutdown down.
4776  */
4777 static void tg3_free_consistent(struct tg3 *tp)
4778 {
4779         kfree(tp->rx_std_buffers);
4780         tp->rx_std_buffers = NULL;
4781         if (tp->rx_std) {
4782                 pci_free_consistent(tp->pdev, TG3_RX_RING_BYTES,
4783                                     tp->rx_std, tp->rx_std_mapping);
4784                 tp->rx_std = NULL;
4785         }
4786         if (tp->rx_jumbo) {
4787                 pci_free_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4788                                     tp->rx_jumbo, tp->rx_jumbo_mapping);
4789                 tp->rx_jumbo = NULL;
4790         }
4791         if (tp->rx_rcb) {
4792                 pci_free_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4793                                     tp->rx_rcb, tp->rx_rcb_mapping);
4794                 tp->rx_rcb = NULL;
4795         }
4796         if (tp->tx_ring) {
4797                 pci_free_consistent(tp->pdev, TG3_TX_RING_BYTES,
4798                         tp->tx_ring, tp->tx_desc_mapping);
4799                 tp->tx_ring = NULL;
4800         }
4801         if (tp->hw_status) {
4802                 pci_free_consistent(tp->pdev, TG3_HW_STATUS_SIZE,
4803                                     tp->hw_status, tp->status_mapping);
4804                 tp->hw_status = NULL;
4805         }
4806         if (tp->hw_stats) {
4807                 pci_free_consistent(tp->pdev, sizeof(struct tg3_hw_stats),
4808                                     tp->hw_stats, tp->stats_mapping);
4809                 tp->hw_stats = NULL;
4810         }
4811 }
4812
4813 /*
4814  * Must not be invoked with interrupt sources disabled and
4815  * the hardware shutdown down.  Can sleep.
4816  */
4817 static int tg3_alloc_consistent(struct tg3 *tp)
4818 {
4819         tp->rx_std_buffers = kzalloc((sizeof(struct ring_info) *
4820                                       (TG3_RX_RING_SIZE +
4821                                        TG3_RX_JUMBO_RING_SIZE)) +
4822                                      (sizeof(struct tx_ring_info) *
4823                                       TG3_TX_RING_SIZE),
4824                                      GFP_KERNEL);
4825         if (!tp->rx_std_buffers)
4826                 return -ENOMEM;
4827
4828         tp->rx_jumbo_buffers = &tp->rx_std_buffers[TG3_RX_RING_SIZE];
4829         tp->tx_buffers = (struct tx_ring_info *)
4830                 &tp->rx_jumbo_buffers[TG3_RX_JUMBO_RING_SIZE];
4831
4832         tp->rx_std = pci_alloc_consistent(tp->pdev, TG3_RX_RING_BYTES,
4833                                           &tp->rx_std_mapping);
4834         if (!tp->rx_std)
4835                 goto err_out;
4836
4837         tp->rx_jumbo = pci_alloc_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4838                                             &tp->rx_jumbo_mapping);
4839
4840         if (!tp->rx_jumbo)
4841                 goto err_out;
4842
4843         tp->rx_rcb = pci_alloc_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4844                                           &tp->rx_rcb_mapping);
4845         if (!tp->rx_rcb)
4846                 goto err_out;
4847
4848         tp->tx_ring = pci_alloc_consistent(tp->pdev, TG3_TX_RING_BYTES,
4849                                            &tp->tx_desc_mapping);
4850         if (!tp->tx_ring)
4851                 goto err_out;
4852
4853         tp->hw_status = pci_alloc_consistent(tp->pdev,
4854                                              TG3_HW_STATUS_SIZE,
4855                                              &tp->status_mapping);
4856         if (!tp->hw_status)
4857                 goto err_out;
4858
4859         tp->hw_stats = pci_alloc_consistent(tp->pdev,
4860                                             sizeof(struct tg3_hw_stats),
4861                                             &tp->stats_mapping);
4862         if (!tp->hw_stats)
4863                 goto err_out;
4864
4865         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4866         memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4867
4868         return 0;
4869
4870 err_out:
4871         tg3_free_consistent(tp);
4872         return -ENOMEM;
4873 }
4874
4875 #define MAX_WAIT_CNT 1000
4876
4877 /* To stop a block, clear the enable bit and poll till it
4878  * clears.  tp->lock is held.
4879  */
4880 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, int silent)
4881 {
4882         unsigned int i;
4883         u32 val;
4884
4885         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
4886                 switch (ofs) {
4887                 case RCVLSC_MODE:
4888                 case DMAC_MODE:
4889                 case MBFREE_MODE:
4890                 case BUFMGR_MODE:
4891                 case MEMARB_MODE:
4892                         /* We can't enable/disable these bits of the
4893                          * 5705/5750, just say success.
4894                          */
4895                         return 0;
4896
4897                 default:
4898                         break;
4899                 };
4900         }
4901
4902         val = tr32(ofs);
4903         val &= ~enable_bit;
4904         tw32_f(ofs, val);
4905
4906         for (i = 0; i < MAX_WAIT_CNT; i++) {
4907                 udelay(100);
4908                 val = tr32(ofs);
4909                 if ((val & enable_bit) == 0)
4910                         break;
4911         }
4912
4913         if (i == MAX_WAIT_CNT && !silent) {
4914                 printk(KERN_ERR PFX "tg3_stop_block timed out, "
4915                        "ofs=%lx enable_bit=%x\n",
4916                        ofs, enable_bit);
4917                 return -ENODEV;
4918         }
4919
4920         return 0;
4921 }
4922
4923 /* tp->lock is held. */
4924 static int tg3_abort_hw(struct tg3 *tp, int silent)
4925 {
4926         int i, err;
4927
4928         tg3_disable_ints(tp);
4929
4930         tp->rx_mode &= ~RX_MODE_ENABLE;
4931         tw32_f(MAC_RX_MODE, tp->rx_mode);
4932         udelay(10);
4933
4934         err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
4935         err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
4936         err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
4937         err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
4938         err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
4939         err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
4940
4941         err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
4942         err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
4943         err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
4944         err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
4945         err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
4946         err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
4947         err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
4948
4949         tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
4950         tw32_f(MAC_MODE, tp->mac_mode);
4951         udelay(40);
4952
4953         tp->tx_mode &= ~TX_MODE_ENABLE;
4954         tw32_f(MAC_TX_MODE, tp->tx_mode);
4955
4956         for (i = 0; i < MAX_WAIT_CNT; i++) {
4957                 udelay(100);
4958                 if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
4959                         break;
4960         }
4961         if (i >= MAX_WAIT_CNT) {
4962                 printk(KERN_ERR PFX "tg3_abort_hw timed out for %s, "
4963                        "TX_MODE_ENABLE will not clear MAC_TX_MODE=%08x\n",
4964                        tp->dev->name, tr32(MAC_TX_MODE));
4965                 err |= -ENODEV;
4966         }
4967
4968         err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
4969         err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
4970         err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
4971
4972         tw32(FTQ_RESET, 0xffffffff);
4973         tw32(FTQ_RESET, 0x00000000);
4974
4975         err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
4976         err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
4977
4978         if (tp->hw_status)
4979                 memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4980         if (tp->hw_stats)
4981                 memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4982
4983         return err;
4984 }
4985
4986 /* tp->lock is held. */
4987 static int tg3_nvram_lock(struct tg3 *tp)
4988 {
4989         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4990                 int i;
4991
4992                 if (tp->nvram_lock_cnt == 0) {
4993                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
4994                         for (i = 0; i < 8000; i++) {
4995                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
4996                                         break;
4997                                 udelay(20);
4998                         }
4999                         if (i == 8000) {
5000                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
5001                                 return -ENODEV;
5002                         }
5003                 }
5004                 tp->nvram_lock_cnt++;
5005         }
5006         return 0;
5007 }
5008
5009 /* tp->lock is held. */
5010 static void tg3_nvram_unlock(struct tg3 *tp)
5011 {
5012         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
5013                 if (tp->nvram_lock_cnt > 0)
5014                         tp->nvram_lock_cnt--;
5015                 if (tp->nvram_lock_cnt == 0)
5016                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
5017         }
5018 }
5019
5020 /* tp->lock is held. */
5021 static void tg3_enable_nvram_access(struct tg3 *tp)
5022 {
5023         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
5024             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
5025                 u32 nvaccess = tr32(NVRAM_ACCESS);
5026
5027                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
5028         }
5029 }
5030
5031 /* tp->lock is held. */
5032 static void tg3_disable_nvram_access(struct tg3 *tp)
5033 {
5034         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
5035             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
5036                 u32 nvaccess = tr32(NVRAM_ACCESS);
5037
5038                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
5039         }
5040 }
5041
5042 static void tg3_ape_send_event(struct tg3 *tp, u32 event)
5043 {
5044         int i;
5045         u32 apedata;
5046
5047         apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
5048         if (apedata != APE_SEG_SIG_MAGIC)
5049                 return;
5050
5051         apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
5052         if (apedata != APE_FW_STATUS_READY)
5053                 return;
5054
5055         /* Wait for up to 1 millisecond for APE to service previous event. */
5056         for (i = 0; i < 10; i++) {
5057                 if (tg3_ape_lock(tp, TG3_APE_LOCK_MEM))
5058                         return;
5059
5060                 apedata = tg3_ape_read32(tp, TG3_APE_EVENT_STATUS);
5061
5062                 if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
5063                         tg3_ape_write32(tp, TG3_APE_EVENT_STATUS,
5064                                         event | APE_EVENT_STATUS_EVENT_PENDING);
5065
5066                 tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
5067
5068                 if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
5069                         break;
5070
5071                 udelay(100);
5072         }
5073
5074         if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
5075                 tg3_ape_write32(tp, TG3_APE_EVENT, APE_EVENT_1);
5076 }
5077
5078 static void tg3_ape_driver_state_change(struct tg3 *tp, int kind)
5079 {
5080         u32 event;
5081         u32 apedata;
5082
5083         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
5084                 return;
5085
5086         switch (kind) {
5087                 case RESET_KIND_INIT:
5088                         tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG,
5089                                         APE_HOST_SEG_SIG_MAGIC);
5090                         tg3_ape_write32(tp, TG3_APE_HOST_SEG_LEN,
5091                                         APE_HOST_SEG_LEN_MAGIC);
5092                         apedata = tg3_ape_read32(tp, TG3_APE_HOST_INIT_COUNT);
5093                         tg3_ape_write32(tp, TG3_APE_HOST_INIT_COUNT, ++apedata);
5094                         tg3_ape_write32(tp, TG3_APE_HOST_DRIVER_ID,
5095                                         APE_HOST_DRIVER_ID_MAGIC);
5096                         tg3_ape_write32(tp, TG3_APE_HOST_BEHAVIOR,
5097                                         APE_HOST_BEHAV_NO_PHYLOCK);
5098
5099                         event = APE_EVENT_STATUS_STATE_START;
5100                         break;
5101                 case RESET_KIND_SHUTDOWN:
5102                         event = APE_EVENT_STATUS_STATE_UNLOAD;
5103                         break;
5104                 case RESET_KIND_SUSPEND:
5105                         event = APE_EVENT_STATUS_STATE_SUSPEND;
5106                         break;
5107                 default:
5108                         return;
5109         }
5110
5111         event |= APE_EVENT_STATUS_DRIVER_EVNT | APE_EVENT_STATUS_STATE_CHNGE;
5112
5113         tg3_ape_send_event(tp, event);
5114 }
5115
5116 /* tp->lock is held. */
5117 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
5118 {
5119         tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
5120                       NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
5121
5122         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
5123                 switch (kind) {
5124                 case RESET_KIND_INIT:
5125                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5126                                       DRV_STATE_START);
5127                         break;
5128
5129                 case RESET_KIND_SHUTDOWN:
5130                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5131                                       DRV_STATE_UNLOAD);
5132                         break;
5133
5134                 case RESET_KIND_SUSPEND:
5135                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5136                                       DRV_STATE_SUSPEND);
5137                         break;
5138
5139                 default:
5140                         break;
5141                 };
5142         }
5143
5144         if (kind == RESET_KIND_INIT ||
5145             kind == RESET_KIND_SUSPEND)
5146                 tg3_ape_driver_state_change(tp, kind);
5147 }
5148
5149 /* tp->lock is held. */
5150 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
5151 {
5152         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
5153                 switch (kind) {
5154                 case RESET_KIND_INIT:
5155                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5156                                       DRV_STATE_START_DONE);
5157                         break;
5158
5159                 case RESET_KIND_SHUTDOWN:
5160                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5161                                       DRV_STATE_UNLOAD_DONE);
5162                         break;
5163
5164                 default:
5165                         break;
5166                 };
5167         }
5168
5169         if (kind == RESET_KIND_SHUTDOWN)
5170                 tg3_ape_driver_state_change(tp, kind);
5171 }
5172
5173 /* tp->lock is held. */
5174 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
5175 {
5176         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
5177                 switch (kind) {
5178                 case RESET_KIND_INIT:
5179                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5180                                       DRV_STATE_START);
5181                         break;
5182
5183                 case RESET_KIND_SHUTDOWN:
5184                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5185                                       DRV_STATE_UNLOAD);
5186                         break;
5187
5188                 case RESET_KIND_SUSPEND:
5189                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
5190                                       DRV_STATE_SUSPEND);
5191                         break;
5192
5193                 default:
5194                         break;
5195                 };
5196         }
5197 }
5198
5199 static int tg3_poll_fw(struct tg3 *tp)
5200 {
5201         int i;
5202         u32 val;
5203
5204         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
5205                 /* Wait up to 20ms for init done. */
5206                 for (i = 0; i < 200; i++) {
5207                         if (tr32(VCPU_STATUS) & VCPU_STATUS_INIT_DONE)
5208                                 return 0;
5209                         udelay(100);
5210                 }
5211                 return -ENODEV;
5212         }
5213
5214         /* Wait for firmware initialization to complete. */
5215         for (i = 0; i < 100000; i++) {
5216                 tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
5217                 if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
5218                         break;
5219                 udelay(10);
5220         }
5221
5222         /* Chip might not be fitted with firmware.  Some Sun onboard
5223          * parts are configured like that.  So don't signal the timeout
5224          * of the above loop as an error, but do report the lack of
5225          * running firmware once.
5226          */
5227         if (i >= 100000 &&
5228             !(tp->tg3_flags2 & TG3_FLG2_NO_FWARE_REPORTED)) {
5229                 tp->tg3_flags2 |= TG3_FLG2_NO_FWARE_REPORTED;
5230
5231                 printk(KERN_INFO PFX "%s: No firmware running.\n",
5232                        tp->dev->name);
5233         }
5234
5235         return 0;
5236 }
5237
5238 /* Save PCI command register before chip reset */
5239 static void tg3_save_pci_state(struct tg3 *tp)
5240 {
5241         pci_read_config_word(tp->pdev, PCI_COMMAND, &tp->pci_cmd);
5242 }
5243
5244 /* Restore PCI state after chip reset */
5245 static void tg3_restore_pci_state(struct tg3 *tp)
5246 {
5247         u32 val;
5248
5249         /* Re-enable indirect register accesses. */
5250         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
5251                                tp->misc_host_ctrl);
5252
5253         /* Set MAX PCI retry to zero. */
5254         val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
5255         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
5256             (tp->tg3_flags & TG3_FLAG_PCIX_MODE))
5257                 val |= PCISTATE_RETRY_SAME_DMA;
5258         /* Allow reads and writes to the APE register and memory space. */
5259         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)
5260                 val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
5261                        PCISTATE_ALLOW_APE_SHMEM_WR;
5262         pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
5263
5264         pci_write_config_word(tp->pdev, PCI_COMMAND, tp->pci_cmd);
5265
5266         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)
5267                 pcie_set_readrq(tp->pdev, 4096);
5268         else {
5269                 pci_write_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
5270                                       tp->pci_cacheline_sz);
5271                 pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
5272                                       tp->pci_lat_timer);
5273         }
5274
5275         /* Make sure PCI-X relaxed ordering bit is clear. */
5276         if (tp->pcix_cap) {
5277                 u16 pcix_cmd;
5278
5279                 pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
5280                                      &pcix_cmd);
5281                 pcix_cmd &= ~PCI_X_CMD_ERO;
5282                 pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
5283                                       pcix_cmd);
5284         }
5285
5286         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
5287
5288                 /* Chip reset on 5780 will reset MSI enable bit,
5289                  * so need to restore it.
5290                  */
5291                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
5292                         u16 ctrl;
5293
5294                         pci_read_config_word(tp->pdev,
5295                                              tp->msi_cap + PCI_MSI_FLAGS,
5296                                              &ctrl);
5297                         pci_write_config_word(tp->pdev,
5298                                               tp->msi_cap + PCI_MSI_FLAGS,
5299                                               ctrl | PCI_MSI_FLAGS_ENABLE);
5300                         val = tr32(MSGINT_MODE);
5301                         tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
5302                 }
5303         }
5304 }
5305
5306 static void tg3_stop_fw(struct tg3 *);
5307
5308 /* tp->lock is held. */
5309 static int tg3_chip_reset(struct tg3 *tp)
5310 {
5311         u32 val;
5312         void (*write_op)(struct tg3 *, u32, u32);
5313         int err;
5314
5315         tg3_nvram_lock(tp);
5316
5317         /* No matching tg3_nvram_unlock() after this because
5318          * chip reset below will undo the nvram lock.
5319          */
5320         tp->nvram_lock_cnt = 0;
5321
5322         /* GRC_MISC_CFG core clock reset will clear the memory
5323          * enable bit in PCI register 4 and the MSI enable bit
5324          * on some chips, so we save relevant registers here.
5325          */
5326         tg3_save_pci_state(tp);
5327
5328         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
5329             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
5330             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
5331             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
5332             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
5333                 tw32(GRC_FASTBOOT_PC, 0);
5334
5335         /*
5336          * We must avoid the readl() that normally takes place.
5337          * It locks machines, causes machine checks, and other
5338          * fun things.  So, temporarily disable the 5701
5339          * hardware workaround, while we do the reset.
5340          */
5341         write_op = tp->write32;
5342         if (write_op == tg3_write_flush_reg32)
5343                 tp->write32 = tg3_write32;
5344
5345         /* Prevent the irq handler from reading or writing PCI registers
5346          * during chip reset when the memory enable bit in the PCI command
5347          * register may be cleared.  The chip does not generate interrupt
5348          * at this time, but the irq handler may still be called due to irq
5349          * sharing or irqpoll.
5350          */
5351         tp->tg3_flags |= TG3_FLAG_CHIP_RESETTING;
5352         if (tp->hw_status) {
5353                 tp->hw_status->status = 0;
5354                 tp->hw_status->status_tag = 0;
5355         }
5356         tp->last_tag = 0;
5357         smp_mb();
5358         synchronize_irq(tp->pdev->irq);
5359
5360         /* do the reset */
5361         val = GRC_MISC_CFG_CORECLK_RESET;
5362
5363         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
5364                 if (tr32(0x7e2c) == 0x60) {
5365                         tw32(0x7e2c, 0x20);
5366                 }
5367                 if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
5368                         tw32(GRC_MISC_CFG, (1 << 29));
5369                         val |= (1 << 29);
5370                 }
5371         }
5372
5373         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
5374                 tw32(VCPU_STATUS, tr32(VCPU_STATUS) | VCPU_STATUS_DRV_RESET);
5375                 tw32(GRC_VCPU_EXT_CTRL,
5376                      tr32(GRC_VCPU_EXT_CTRL) & ~GRC_VCPU_EXT_CTRL_HALT_CPU);
5377         }
5378
5379         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
5380                 val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
5381         tw32(GRC_MISC_CFG, val);
5382
5383         /* restore 5701 hardware bug workaround write method */
5384         tp->write32 = write_op;
5385
5386         /* Unfortunately, we have to delay before the PCI read back.
5387          * Some 575X chips even will not respond to a PCI cfg access
5388          * when the reset command is given to the chip.
5389          *
5390          * How do these hardware designers expect things to work
5391          * properly if the PCI write is posted for a long period
5392          * of time?  It is always necessary to have some method by
5393          * which a register read back can occur to push the write
5394          * out which does the reset.
5395          *
5396          * For most tg3 variants the trick below was working.
5397          * Ho hum...
5398          */
5399         udelay(120);
5400
5401         /* Flush PCI posted writes.  The normal MMIO registers
5402          * are inaccessible at this time so this is the only
5403          * way to make this reliably (actually, this is no longer
5404          * the case, see above).  I tried to use indirect
5405          * register read/write but this upset some 5701 variants.
5406          */
5407         pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
5408
5409         udelay(120);
5410
5411         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
5412                 if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A0) {
5413                         int i;
5414                         u32 cfg_val;
5415
5416                         /* Wait for link training to complete.  */
5417                         for (i = 0; i < 5000; i++)
5418                                 udelay(100);
5419
5420                         pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
5421                         pci_write_config_dword(tp->pdev, 0xc4,
5422                                                cfg_val | (1 << 15));
5423                 }
5424                 /* Set PCIE max payload size and clear error status.  */
5425                 pci_write_config_dword(tp->pdev, 0xd8, 0xf5000);
5426         }
5427
5428         tg3_restore_pci_state(tp);
5429
5430         tp->tg3_flags &= ~TG3_FLAG_CHIP_RESETTING;
5431
5432         val = 0;
5433         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
5434                 val = tr32(MEMARB_MODE);
5435         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
5436
5437         if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A3) {
5438                 tg3_stop_fw(tp);
5439                 tw32(0x5000, 0x400);
5440         }
5441
5442         tw32(GRC_MODE, tp->grc_mode);
5443
5444         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0) {
5445                 val = tr32(0xc4);
5446
5447                 tw32(0xc4, val | (1 << 15));
5448         }
5449
5450         if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
5451             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
5452                 tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
5453                 if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0)
5454                         tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
5455                 tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
5456         }
5457
5458         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
5459                 tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
5460                 tw32_f(MAC_MODE, tp->mac_mode);
5461         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
5462                 tp->mac_mode = MAC_MODE_PORT_MODE_GMII;
5463                 tw32_f(MAC_MODE, tp->mac_mode);
5464         } else
5465                 tw32_f(MAC_MODE, 0);
5466         udelay(40);
5467
5468         err = tg3_poll_fw(tp);
5469         if (err)
5470                 return err;
5471
5472         if ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
5473             tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
5474                 val = tr32(0x7c00);
5475
5476                 tw32(0x7c00, val | (1 << 25));
5477         }
5478
5479         /* Reprobe ASF enable state.  */
5480         tp->tg3_flags &= ~TG3_FLAG_ENABLE_ASF;
5481         tp->tg3_flags2 &= ~TG3_FLG2_ASF_NEW_HANDSHAKE;
5482         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
5483         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
5484                 u32 nic_cfg;
5485
5486                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
5487                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
5488                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
5489                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
5490                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
5491                 }
5492         }
5493
5494         return 0;
5495 }
5496
5497 /* tp->lock is held. */
5498 static void tg3_stop_fw(struct tg3 *tp)
5499 {
5500         if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) &&
5501            !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)) {
5502                 u32 val;
5503                 int i;
5504
5505                 tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
5506                 val = tr32(GRC_RX_CPU_EVENT);
5507                 val |= (1 << 14);
5508                 tw32(GRC_RX_CPU_EVENT, val);
5509
5510                 /* Wait for RX cpu to ACK the event.  */
5511                 for (i = 0; i < 100; i++) {
5512                         if (!(tr32(GRC_RX_CPU_EVENT) & (1 << 14)))
5513                                 break;
5514                         udelay(1);
5515                 }
5516         }
5517 }
5518
5519 /* tp->lock is held. */
5520 static int tg3_halt(struct tg3 *tp, int kind, int silent)
5521 {
5522         int err;
5523
5524         tg3_stop_fw(tp);
5525
5526         tg3_write_sig_pre_reset(tp, kind);
5527
5528         tg3_abort_hw(tp, silent);
5529         err = tg3_chip_reset(tp);
5530
5531         tg3_write_sig_legacy(tp, kind);
5532         tg3_write_sig_post_reset(tp, kind);
5533
5534         if (err)
5535                 return err;
5536
5537         return 0;
5538 }
5539
5540 #define TG3_FW_RELEASE_MAJOR    0x0
5541 #define TG3_FW_RELASE_MINOR     0x0
5542 #define TG3_FW_RELEASE_FIX      0x0
5543 #define TG3_FW_START_ADDR       0x08000000
5544 #define TG3_FW_TEXT_ADDR        0x08000000
5545 #define TG3_FW_TEXT_LEN         0x9c0
5546 #define TG3_FW_RODATA_ADDR      0x080009c0
5547 #define TG3_FW_RODATA_LEN       0x60
5548 #define TG3_FW_DATA_ADDR        0x08000a40
5549 #define TG3_FW_DATA_LEN         0x20
5550 #define TG3_FW_SBSS_ADDR        0x08000a60
5551 #define TG3_FW_SBSS_LEN         0xc
5552 #define TG3_FW_BSS_ADDR         0x08000a70
5553 #define TG3_FW_BSS_LEN          0x10
5554
5555 static const u32 tg3FwText[(TG3_FW_TEXT_LEN / sizeof(u32)) + 1] = {
5556         0x00000000, 0x10000003, 0x00000000, 0x0000000d, 0x0000000d, 0x3c1d0800,
5557         0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100000, 0x0e000018, 0x00000000,
5558         0x0000000d, 0x3c1d0800, 0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100034,
5559         0x0e00021c, 0x00000000, 0x0000000d, 0x00000000, 0x00000000, 0x00000000,
5560         0x27bdffe0, 0x3c1cc000, 0xafbf0018, 0xaf80680c, 0x0e00004c, 0x241b2105,
5561         0x97850000, 0x97870002, 0x9782002c, 0x9783002e, 0x3c040800, 0x248409c0,
5562         0xafa00014, 0x00021400, 0x00621825, 0x00052c00, 0xafa30010, 0x8f860010,
5563         0x00e52825, 0x0e000060, 0x24070102, 0x3c02ac00, 0x34420100, 0x3c03ac01,
5564         0x34630100, 0xaf820490, 0x3c02ffff, 0xaf820494, 0xaf830498, 0xaf82049c,
5565         0x24020001, 0xaf825ce0, 0x0e00003f, 0xaf825d00, 0x0e000140, 0x00000000,
5566         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x2402ffff, 0xaf825404, 0x8f835400,
5567         0x34630400, 0xaf835400, 0xaf825404, 0x3c020800, 0x24420034, 0xaf82541c,
5568         0x03e00008, 0xaf805400, 0x00000000, 0x00000000, 0x3c020800, 0x34423000,
5569         0x3c030800, 0x34633000, 0x3c040800, 0x348437ff, 0x3c010800, 0xac220a64,
5570         0x24020040, 0x3c010800, 0xac220a68, 0x3c010800, 0xac200a60, 0xac600000,
5571         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
5572         0x00804821, 0x8faa0010, 0x3c020800, 0x8c420a60, 0x3c040800, 0x8c840a68,
5573         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010800, 0xac230a60, 0x14400003,
5574         0x00004021, 0x3c010800, 0xac200a60, 0x3c020800, 0x8c420a60, 0x3c030800,
5575         0x8c630a64, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
5576         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020800, 0x8c420a60,
5577         0x3c030800, 0x8c630a64, 0x8f84680c, 0x00021140, 0x00431021, 0xac440008,
5578         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
5579         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5580         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5581         0, 0, 0, 0, 0, 0,
5582         0x02000008, 0x00000000, 0x0a0001e3, 0x3c0a0001, 0x0a0001e3, 0x3c0a0002,
5583         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5584         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5585         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5586         0x0a0001e3, 0x3c0a0007, 0x0a0001e3, 0x3c0a0008, 0x0a0001e3, 0x3c0a0009,
5587         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000b,
5588         0x0a0001e3, 0x3c0a000c, 0x0a0001e3, 0x3c0a000d, 0x0a0001e3, 0x00000000,
5589         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000e, 0x0a0001e3, 0x00000000,
5590         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5591         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
5592         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a0013, 0x0a0001e3, 0x3c0a0014,
5593         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5594         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5595         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5596         0x27bdffe0, 0x00001821, 0x00001021, 0xafbf0018, 0xafb10014, 0xafb00010,
5597         0x3c010800, 0x00220821, 0xac200a70, 0x3c010800, 0x00220821, 0xac200a74,
5598         0x3c010800, 0x00220821, 0xac200a78, 0x24630001, 0x1860fff5, 0x2442000c,
5599         0x24110001, 0x8f906810, 0x32020004, 0x14400005, 0x24040001, 0x3c020800,
5600         0x8c420a78, 0x18400003, 0x00002021, 0x0e000182, 0x00000000, 0x32020001,
5601         0x10400003, 0x00000000, 0x0e000169, 0x00000000, 0x0a000153, 0xaf915028,
5602         0x8fbf0018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020, 0x3c050800,
5603         0x8ca50a70, 0x3c060800, 0x8cc60a80, 0x3c070800, 0x8ce70a78, 0x27bdffe0,
5604         0x3c040800, 0x248409d0, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014,
5605         0x0e00017b, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x24020001,
5606         0x8f836810, 0x00821004, 0x00021027, 0x00621824, 0x03e00008, 0xaf836810,
5607         0x27bdffd8, 0xafbf0024, 0x1080002e, 0xafb00020, 0x8f825cec, 0xafa20018,
5608         0x8f825cec, 0x3c100800, 0x26100a78, 0xafa2001c, 0x34028000, 0xaf825cec,
5609         0x8e020000, 0x18400016, 0x00000000, 0x3c020800, 0x94420a74, 0x8fa3001c,
5610         0x000221c0, 0xac830004, 0x8fa2001c, 0x3c010800, 0x0e000201, 0xac220a74,
5611         0x10400005, 0x00000000, 0x8e020000, 0x24420001, 0x0a0001df, 0xae020000,
5612         0x3c020800, 0x8c420a70, 0x00021c02, 0x000321c0, 0x0a0001c5, 0xafa2001c,
5613         0x0e000201, 0x00000000, 0x1040001f, 0x00000000, 0x8e020000, 0x8fa3001c,
5614         0x24420001, 0x3c010800, 0xac230a70, 0x3c010800, 0xac230a74, 0x0a0001df,
5615         0xae020000, 0x3c100800, 0x26100a78, 0x8e020000, 0x18400028, 0x00000000,
5616         0x0e000201, 0x00000000, 0x14400024, 0x00000000, 0x8e020000, 0x3c030800,
5617         0x8c630a70, 0x2442ffff, 0xafa3001c, 0x18400006, 0xae020000, 0x00031402,
5618         0x000221c0, 0x8c820004, 0x3c010800, 0xac220a70, 0x97a2001e, 0x2442ff00,
5619         0x2c420300, 0x1440000b, 0x24024000, 0x3c040800, 0x248409dc, 0xafa00010,
5620         0xafa00014, 0x8fa6001c, 0x24050008, 0x0e000060, 0x00003821, 0x0a0001df,
5621         0x00000000, 0xaf825cf8, 0x3c020800, 0x8c420a40, 0x8fa3001c, 0x24420001,
5622         0xaf835cf8, 0x3c010800, 0xac220a40, 0x8fbf0024, 0x8fb00020, 0x03e00008,
5623         0x27bd0028, 0x27bdffe0, 0x3c040800, 0x248409e8, 0x00002821, 0x00003021,
5624         0x00003821, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x8fbf0018,
5625         0x03e00008, 0x27bd0020, 0x8f82680c, 0x8f85680c, 0x00021827, 0x0003182b,
5626         0x00031823, 0x00431024, 0x00441021, 0x00a2282b, 0x10a00006, 0x00000000,
5627         0x00401821, 0x8f82680c, 0x0043102b, 0x1440fffd, 0x00000000, 0x03e00008,
5628         0x00000000, 0x3c040800, 0x8c840000, 0x3c030800, 0x8c630a40, 0x0064102b,
5629         0x54400002, 0x00831023, 0x00641023, 0x2c420008, 0x03e00008, 0x38420001,
5630         0x27bdffe0, 0x00802821, 0x3c040800, 0x24840a00, 0x00003021, 0x00003821,
5631         0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x0a000216, 0x00000000,
5632         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000, 0x27bdffe0, 0x3c1cc000,
5633         0xafbf0018, 0x0e00004c, 0xaf80680c, 0x3c040800, 0x24840a10, 0x03802821,
5634         0x00003021, 0x00003821, 0xafa00010, 0x0e000060, 0xafa00014, 0x2402ffff,
5635         0xaf825404, 0x3c0200aa, 0x0e000234, 0xaf825434, 0x8fbf0018, 0x03e00008,
5636         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe8, 0xafb00010,
5637         0x24100001, 0xafbf0014, 0x3c01c003, 0xac200000, 0x8f826810, 0x30422000,
5638         0x10400003, 0x00000000, 0x0e000246, 0x00000000, 0x0a00023a, 0xaf905428,
5639         0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x27bdfff8, 0x8f845d0c,
5640         0x3c0200ff, 0x3c030800, 0x8c630a50, 0x3442fff8, 0x00821024, 0x1043001e,
5641         0x3c0500ff, 0x34a5fff8, 0x3c06c003, 0x3c074000, 0x00851824, 0x8c620010,
5642         0x3c010800, 0xac230a50, 0x30420008, 0x10400005, 0x00871025, 0x8cc20000,
5643         0x24420001, 0xacc20000, 0x00871025, 0xaf825d0c, 0x8fa20000, 0x24420001,
5644         0xafa20000, 0x8fa20000, 0x8fa20000, 0x24420001, 0xafa20000, 0x8fa20000,
5645         0x8f845d0c, 0x3c030800, 0x8c630a50, 0x00851024, 0x1443ffe8, 0x00851824,
5646         0x27bd0008, 0x03e00008, 0x00000000, 0x00000000, 0x00000000
5647 };
5648
5649 static const u32 tg3FwRodata[(TG3_FW_RODATA_LEN / sizeof(u32)) + 1] = {
5650         0x35373031, 0x726c7341, 0x00000000, 0x00000000, 0x53774576, 0x656e7430,
5651         0x00000000, 0x726c7045, 0x76656e74, 0x31000000, 0x556e6b6e, 0x45766e74,
5652         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
5653         0x00000000, 0x00000000, 0x4d61696e, 0x43707542, 0x00000000, 0x00000000,
5654         0x00000000
5655 };
5656
5657 #if 0 /* All zeros, don't eat up space with it. */
5658 u32 tg3FwData[(TG3_FW_DATA_LEN / sizeof(u32)) + 1] = {
5659         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
5660         0x00000000, 0x00000000, 0x00000000, 0x00000000
5661 };
5662 #endif
5663
5664 #define RX_CPU_SCRATCH_BASE     0x30000
5665 #define RX_CPU_SCRATCH_SIZE     0x04000
5666 #define TX_CPU_SCRATCH_BASE     0x34000
5667 #define TX_CPU_SCRATCH_SIZE     0x04000
5668
5669 /* tp->lock is held. */
5670 static int tg3_halt_cpu(struct tg3 *tp, u32 offset)
5671 {
5672         int i;
5673
5674         BUG_ON(offset == TX_CPU_BASE &&
5675             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS));
5676
5677         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
5678                 u32 val = tr32(GRC_VCPU_EXT_CTRL);
5679
5680                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_HALT_CPU);
5681                 return 0;
5682         }
5683         if (offset == RX_CPU_BASE) {
5684                 for (i = 0; i < 10000; i++) {
5685                         tw32(offset + CPU_STATE, 0xffffffff);
5686                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5687                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5688                                 break;
5689                 }
5690
5691                 tw32(offset + CPU_STATE, 0xffffffff);
5692                 tw32_f(offset + CPU_MODE,  CPU_MODE_HALT);
5693                 udelay(10);
5694         } else {
5695                 for (i = 0; i < 10000; i++) {
5696                         tw32(offset + CPU_STATE, 0xffffffff);
5697                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5698                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5699                                 break;
5700                 }
5701         }
5702
5703         if (i >= 10000) {
5704                 printk(KERN_ERR PFX "tg3_reset_cpu timed out for %s, "
5705                        "and %s CPU\n",
5706                        tp->dev->name,
5707                        (offset == RX_CPU_BASE ? "RX" : "TX"));
5708                 return -ENODEV;
5709         }
5710
5711         /* Clear firmware's nvram arbitration. */
5712         if (tp->tg3_flags & TG3_FLAG_NVRAM)
5713                 tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
5714         return 0;
5715 }
5716
5717 struct fw_info {
5718         unsigned int text_base;
5719         unsigned int text_len;
5720         const u32 *text_data;
5721         unsigned int rodata_base;
5722         unsigned int rodata_len;
5723         const u32 *rodata_data;
5724         unsigned int data_base;
5725         unsigned int data_len;
5726         const u32 *data_data;
5727 };
5728
5729 /* tp->lock is held. */
5730 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base, u32 cpu_scratch_base,
5731                                  int cpu_scratch_size, struct fw_info *info)
5732 {
5733         int err, lock_err, i;
5734         void (*write_op)(struct tg3 *, u32, u32);
5735
5736         if (cpu_base == TX_CPU_BASE &&
5737             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
5738                 printk(KERN_ERR PFX "tg3_load_firmware_cpu: Trying to load "
5739                        "TX cpu firmware on %s which is 5705.\n",
5740                        tp->dev->name);
5741                 return -EINVAL;
5742         }
5743
5744         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
5745                 write_op = tg3_write_mem;
5746         else
5747                 write_op = tg3_write_indirect_reg32;
5748
5749         /* It is possible that bootcode is still loading at this point.
5750          * Get the nvram lock first before halting the cpu.
5751          */
5752         lock_err = tg3_nvram_lock(tp);
5753         err = tg3_halt_cpu(tp, cpu_base);
5754         if (!lock_err)
5755                 tg3_nvram_unlock(tp);
5756         if (err)
5757                 goto out;
5758
5759         for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
5760                 write_op(tp, cpu_scratch_base + i, 0);
5761         tw32(cpu_base + CPU_STATE, 0xffffffff);
5762         tw32(cpu_base + CPU_MODE, tr32(cpu_base+CPU_MODE)|CPU_MODE_HALT);
5763         for (i = 0; i < (info->text_len / sizeof(u32)); i++)
5764                 write_op(tp, (cpu_scratch_base +
5765                               (info->text_base & 0xffff) +
5766                               (i * sizeof(u32))),
5767                          (info->text_data ?
5768                           info->text_data[i] : 0));
5769         for (i = 0; i < (info->rodata_len / sizeof(u32)); i++)
5770                 write_op(tp, (cpu_scratch_base +
5771                               (info->rodata_base & 0xffff) +
5772                               (i * sizeof(u32))),
5773                          (info->rodata_data ?
5774                           info->rodata_data[i] : 0));
5775         for (i = 0; i < (info->data_len / sizeof(u32)); i++)
5776                 write_op(tp, (cpu_scratch_base +
5777                               (info->data_base & 0xffff) +
5778                               (i * sizeof(u32))),
5779                          (info->data_data ?
5780                           info->data_data[i] : 0));
5781
5782         err = 0;
5783
5784 out:
5785         return err;
5786 }
5787
5788 /* tp->lock is held. */
5789 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
5790 {
5791         struct fw_info info;
5792         int err, i;
5793
5794         info.text_base = TG3_FW_TEXT_ADDR;
5795         info.text_len = TG3_FW_TEXT_LEN;
5796         info.text_data = &tg3FwText[0];
5797         info.rodata_base = TG3_FW_RODATA_ADDR;
5798         info.rodata_len = TG3_FW_RODATA_LEN;
5799         info.rodata_data = &tg3FwRodata[0];
5800         info.data_base = TG3_FW_DATA_ADDR;
5801         info.data_len = TG3_FW_DATA_LEN;
5802         info.data_data = NULL;
5803
5804         err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
5805                                     RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
5806                                     &info);
5807         if (err)
5808                 return err;
5809
5810         err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
5811                                     TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
5812                                     &info);
5813         if (err)
5814                 return err;
5815
5816         /* Now startup only the RX cpu. */
5817         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5818         tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5819
5820         for (i = 0; i < 5; i++) {
5821                 if (tr32(RX_CPU_BASE + CPU_PC) == TG3_FW_TEXT_ADDR)
5822                         break;
5823                 tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5824                 tw32(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
5825                 tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5826                 udelay(1000);
5827         }
5828         if (i >= 5) {
5829                 printk(KERN_ERR PFX "tg3_load_firmware fails for %s "
5830                        "to set RX CPU PC, is %08x should be %08x\n",
5831                        tp->dev->name, tr32(RX_CPU_BASE + CPU_PC),
5832                        TG3_FW_TEXT_ADDR);
5833                 return -ENODEV;
5834         }
5835         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5836         tw32_f(RX_CPU_BASE + CPU_MODE,  0x00000000);
5837
5838         return 0;
5839 }
5840
5841
5842 #define TG3_TSO_FW_RELEASE_MAJOR        0x1
5843 #define TG3_TSO_FW_RELASE_MINOR         0x6
5844 #define TG3_TSO_FW_RELEASE_FIX          0x0
5845 #define TG3_TSO_FW_START_ADDR           0x08000000
5846 #define TG3_TSO_FW_TEXT_ADDR            0x08000000
5847 #define TG3_TSO_FW_TEXT_LEN             0x1aa0
5848 #define TG3_TSO_FW_RODATA_ADDR          0x08001aa0
5849 #define TG3_TSO_FW_RODATA_LEN           0x60
5850 #define TG3_TSO_FW_DATA_ADDR            0x08001b20
5851 #define TG3_TSO_FW_DATA_LEN             0x30
5852 #define TG3_TSO_FW_SBSS_ADDR            0x08001b50
5853 #define TG3_TSO_FW_SBSS_LEN             0x2c
5854 #define TG3_TSO_FW_BSS_ADDR             0x08001b80
5855 #define TG3_TSO_FW_BSS_LEN              0x894
5856
5857 static const u32 tg3TsoFwText[(TG3_TSO_FW_TEXT_LEN / 4) + 1] = {
5858         0x0e000003, 0x00000000, 0x08001b24, 0x00000000, 0x10000003, 0x00000000,
5859         0x0000000d, 0x0000000d, 0x3c1d0800, 0x37bd4000, 0x03a0f021, 0x3c100800,
5860         0x26100000, 0x0e000010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
5861         0xafbf0018, 0x0e0005d8, 0x34840002, 0x0e000668, 0x00000000, 0x3c030800,
5862         0x90631b68, 0x24020002, 0x3c040800, 0x24841aac, 0x14620003, 0x24050001,
5863         0x3c040800, 0x24841aa0, 0x24060006, 0x00003821, 0xafa00010, 0x0e00067c,
5864         0xafa00014, 0x8f625c50, 0x34420001, 0xaf625c50, 0x8f625c90, 0x34420001,
5865         0xaf625c90, 0x2402ffff, 0x0e000034, 0xaf625404, 0x8fbf0018, 0x03e00008,
5866         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c,
5867         0xafb20018, 0xafb10014, 0x0e00005b, 0xafb00010, 0x24120002, 0x24110001,
5868         0x8f706820, 0x32020100, 0x10400003, 0x00000000, 0x0e0000bb, 0x00000000,
5869         0x8f706820, 0x32022000, 0x10400004, 0x32020001, 0x0e0001f0, 0x24040001,
5870         0x32020001, 0x10400003, 0x00000000, 0x0e0000a3, 0x00000000, 0x3c020800,
5871         0x90421b98, 0x14520003, 0x00000000, 0x0e0004c0, 0x00000000, 0x0a00003c,
5872         0xaf715028, 0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008,
5873         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ac0, 0x00002821, 0x00003021,
5874         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x3c040800,
5875         0x248423d8, 0xa4800000, 0x3c010800, 0xa0201b98, 0x3c010800, 0xac201b9c,
5876         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5877         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bbc, 0x8f624434, 0x3c010800,
5878         0xac221b88, 0x8f624438, 0x3c010800, 0xac221b8c, 0x8f624410, 0xac80f7a8,
5879         0x3c010800, 0xac201b84, 0x3c010800, 0xac2023e0, 0x3c010800, 0xac2023c8,
5880         0x3c010800, 0xac2023cc, 0x3c010800, 0xac202400, 0x3c010800, 0xac221b90,
5881         0x8f620068, 0x24030007, 0x00021702, 0x10430005, 0x00000000, 0x8f620068,
5882         0x00021702, 0x14400004, 0x24020001, 0x3c010800, 0x0a000097, 0xac20240c,
5883         0xac820034, 0x3c040800, 0x24841acc, 0x3c050800, 0x8ca5240c, 0x00003021,
5884         0x00003821, 0xafa00010, 0x0e00067c, 0xafa00014, 0x8fbf0018, 0x03e00008,
5885         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ad8, 0x00002821, 0x00003021,
5886         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x0e00005b,
5887         0x00000000, 0x0e0000b4, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5888         0x24020001, 0x8f636820, 0x00821004, 0x00021027, 0x00621824, 0x03e00008,
5889         0xaf636820, 0x27bdffd0, 0xafbf002c, 0xafb60028, 0xafb50024, 0xafb40020,
5890         0xafb3001c, 0xafb20018, 0xafb10014, 0xafb00010, 0x8f675c5c, 0x3c030800,
5891         0x24631bbc, 0x8c620000, 0x14470005, 0x3c0200ff, 0x3c020800, 0x90421b98,
5892         0x14400119, 0x3c0200ff, 0x3442fff8, 0x00e28824, 0xac670000, 0x00111902,
5893         0x306300ff, 0x30e20003, 0x000211c0, 0x00622825, 0x00a04021, 0x00071602,
5894         0x3c030800, 0x90631b98, 0x3044000f, 0x14600036, 0x00804821, 0x24020001,
5895         0x3c010800, 0xa0221b98, 0x00051100, 0x00821025, 0x3c010800, 0xac201b9c,
5896         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5897         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bb0, 0x3c010800, 0xac201bb4,
5898         0x3c010800, 0xa42223d8, 0x9622000c, 0x30437fff, 0x3c010800, 0xa4222410,
5899         0x30428000, 0x3c010800, 0xa4231bc6, 0x10400005, 0x24020001, 0x3c010800,
5900         0xac2223f4, 0x0a000102, 0x2406003e, 0x24060036, 0x3c010800, 0xac2023f4,
5901         0x9622000a, 0x3c030800, 0x94631bc6, 0x3c010800, 0xac2023f0, 0x3c010800,
5902         0xac2023f8, 0x00021302, 0x00021080, 0x00c21021, 0x00621821, 0x3c010800,
5903         0xa42223d0, 0x3c010800, 0x0a000115, 0xa4231b96, 0x9622000c, 0x3c010800,
5904         0xa42223ec, 0x3c040800, 0x24841b9c, 0x8c820000, 0x00021100, 0x3c010800,
5905         0x00220821, 0xac311bc8, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5906         0xac271bcc, 0x8c820000, 0x25030001, 0x306601ff, 0x00021100, 0x3c010800,
5907         0x00220821, 0xac261bd0, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5908         0xac291bd4, 0x96230008, 0x3c020800, 0x8c421bac, 0x00432821, 0x3c010800,
5909         0xac251bac, 0x9622000a, 0x30420004, 0x14400018, 0x00061100, 0x8f630c14,
5910         0x3063000f, 0x2c620002, 0x1440000b, 0x3c02c000, 0x8f630c14, 0x3c020800,
5911         0x8c421b40, 0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002,
5912         0x1040fff7, 0x3c02c000, 0x00e21825, 0xaf635c5c, 0x8f625c50, 0x30420002,
5913         0x10400014, 0x00000000, 0x0a000147, 0x00000000, 0x3c030800, 0x8c631b80,
5914         0x3c040800, 0x94841b94, 0x01221025, 0x3c010800, 0xa42223da, 0x24020001,
5915         0x3c010800, 0xac221bb8, 0x24630001, 0x0085202a, 0x3c010800, 0x10800003,
5916         0xac231b80, 0x3c010800, 0xa4251b94, 0x3c060800, 0x24c61b9c, 0x8cc20000,
5917         0x24420001, 0xacc20000, 0x28420080, 0x14400005, 0x00000000, 0x0e000656,
5918         0x24040002, 0x0a0001e6, 0x00000000, 0x3c020800, 0x8c421bb8, 0x10400078,
5919         0x24020001, 0x3c050800, 0x90a51b98, 0x14a20072, 0x00000000, 0x3c150800,
5920         0x96b51b96, 0x3c040800, 0x8c841bac, 0x32a3ffff, 0x0083102a, 0x1440006c,
5921         0x00000000, 0x14830003, 0x00000000, 0x3c010800, 0xac2523f0, 0x1060005c,
5922         0x00009021, 0x24d60004, 0x0060a021, 0x24d30014, 0x8ec20000, 0x00028100,
5923         0x3c110800, 0x02308821, 0x0e000625, 0x8e311bc8, 0x00402821, 0x10a00054,
5924         0x00000000, 0x9628000a, 0x31020040, 0x10400005, 0x2407180c, 0x8e22000c,
5925         0x2407188c, 0x00021400, 0xaca20018, 0x3c030800, 0x00701821, 0x8c631bd0,
5926         0x3c020800, 0x00501021, 0x8c421bd4, 0x00031d00, 0x00021400, 0x00621825,
5927         0xaca30014, 0x8ec30004, 0x96220008, 0x00432023, 0x3242ffff, 0x3083ffff,
5928         0x00431021, 0x0282102a, 0x14400002, 0x02b23023, 0x00803021, 0x8e620000,
5929         0x30c4ffff, 0x00441021, 0xae620000, 0x8e220000, 0xaca20000, 0x8e220004,
5930         0x8e63fff4, 0x00431021, 0xaca20004, 0xa4a6000e, 0x8e62fff4, 0x00441021,
5931         0xae62fff4, 0x96230008, 0x0043102a, 0x14400005, 0x02469021, 0x8e62fff0,
5932         0xae60fff4, 0x24420001, 0xae62fff0, 0xaca00008, 0x3242ffff, 0x14540008,
5933         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x24020905, 0xa4a2000c,
5934         0x0a0001cb, 0x34e70020, 0xa4a2000c, 0x3c020800, 0x8c4223f0, 0x10400003,
5935         0x3c024b65, 0x0a0001d3, 0x34427654, 0x3c02b49a, 0x344289ab, 0xaca2001c,
5936         0x30e2ffff, 0xaca20010, 0x0e0005a2, 0x00a02021, 0x3242ffff, 0x0054102b,
5937         0x1440ffa9, 0x00000000, 0x24020002, 0x3c010800, 0x0a0001e6, 0xa0221b98,
5938         0x8ec2083c, 0x24420001, 0x0a0001e6, 0xaec2083c, 0x0e0004c0, 0x00000000,
5939         0x8fbf002c, 0x8fb60028, 0x8fb50024, 0x8fb40020, 0x8fb3001c, 0x8fb20018,
5940         0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0030, 0x27bdffd0, 0xafbf0028,
5941         0xafb30024, 0xafb20020, 0xafb1001c, 0xafb00018, 0x8f725c9c, 0x3c0200ff,
5942         0x3442fff8, 0x3c070800, 0x24e71bb4, 0x02428824, 0x9623000e, 0x8ce20000,
5943         0x00431021, 0xace20000, 0x8e220010, 0x30420020, 0x14400011, 0x00809821,
5944         0x0e00063b, 0x02202021, 0x3c02c000, 0x02421825, 0xaf635c9c, 0x8f625c90,
5945         0x30420002, 0x1040011e, 0x00000000, 0xaf635c9c, 0x8f625c90, 0x30420002,
5946         0x10400119, 0x00000000, 0x0a00020d, 0x00000000, 0x8e240008, 0x8e230014,
5947         0x00041402, 0x000231c0, 0x00031502, 0x304201ff, 0x2442ffff, 0x3042007f,
5948         0x00031942, 0x30637800, 0x00021100, 0x24424000, 0x00624821, 0x9522000a,
5949         0x3084ffff, 0x30420008, 0x104000b0, 0x000429c0, 0x3c020800, 0x8c422400,
5950         0x14400024, 0x24c50008, 0x94c20014, 0x3c010800, 0xa42223d0, 0x8cc40010,
5951         0x00041402, 0x3c010800, 0xa42223d2, 0x3c010800, 0xa42423d4, 0x94c2000e,
5952         0x3083ffff, 0x00431023, 0x3c010800, 0xac222408, 0x94c2001a, 0x3c010800,
5953         0xac262400, 0x3c010800, 0xac322404, 0x3c010800, 0xac2223fc, 0x3c02c000,
5954         0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e5, 0x00000000,
5955         0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e0, 0x00000000, 0x0a000246,
5956         0x00000000, 0x94c2000e, 0x3c030800, 0x946323d4, 0x00434023, 0x3103ffff,
5957         0x2c620008, 0x1040001c, 0x00000000, 0x94c20014, 0x24420028, 0x00a22821,
5958         0x00031042, 0x1840000b, 0x00002021, 0x24e60848, 0x00403821, 0x94a30000,
5959         0x8cc20000, 0x24840001, 0x00431021, 0xacc20000, 0x0087102a, 0x1440fff9,
5960         0x24a50002, 0x31020001, 0x1040001f, 0x3c024000, 0x3c040800, 0x248423fc,
5961         0xa0a00001, 0x94a30000, 0x8c820000, 0x00431021, 0x0a000285, 0xac820000,
5962         0x8f626800, 0x3c030010, 0x00431024, 0x10400009, 0x00000000, 0x94c2001a,
5963         0x3c030800, 0x8c6323fc, 0x00431021, 0x3c010800, 0xac2223fc, 0x0a000286,
5964         0x3c024000, 0x94c2001a, 0x94c4001c, 0x3c030800, 0x8c6323fc, 0x00441023,
5965         0x00621821, 0x3c010800, 0xac2323fc, 0x3c024000, 0x02421825, 0xaf635c9c,
5966         0x8f625c90, 0x30420002, 0x1440fffc, 0x00000000, 0x9522000a, 0x30420010,
5967         0x1040009b, 0x00000000, 0x3c030800, 0x946323d4, 0x3c070800, 0x24e72400,
5968         0x8ce40000, 0x8f626800, 0x24630030, 0x00832821, 0x3c030010, 0x00431024,
5969         0x1440000a, 0x00000000, 0x94a20004, 0x3c040800, 0x8c842408, 0x3c030800,
5970         0x8c6323fc, 0x00441023, 0x00621821, 0x3c010800, 0xac2323fc, 0x3c040800,
5971         0x8c8423fc, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402, 0x00822021,
5972         0x00041027, 0xa4a20006, 0x3c030800, 0x8c632404, 0x3c0200ff, 0x3442fff8,
5973         0x00628824, 0x96220008, 0x24050001, 0x24034000, 0x000231c0, 0x00801021,
5974         0xa4c2001a, 0xa4c0001c, 0xace00000, 0x3c010800, 0xac251b60, 0xaf635cb8,
5975         0x8f625cb0, 0x30420002, 0x10400003, 0x00000000, 0x3c010800, 0xac201b60,
5976         0x8e220008, 0xaf625cb8, 0x8f625cb0, 0x30420002, 0x10400003, 0x00000000,
5977         0x3c010800, 0xac201b60, 0x3c020800, 0x8c421b60, 0x1040ffec, 0x00000000,
5978         0x3c040800, 0x0e00063b, 0x8c842404, 0x0a00032a, 0x00000000, 0x3c030800,
5979         0x90631b98, 0x24020002, 0x14620003, 0x3c034b65, 0x0a0002e1, 0x00008021,
5980         0x8e22001c, 0x34637654, 0x10430002, 0x24100002, 0x24100001, 0x00c02021,
5981         0x0e000350, 0x02003021, 0x24020003, 0x3c010800, 0xa0221b98, 0x24020002,
5982         0x1202000a, 0x24020001, 0x3c030800, 0x8c6323f0, 0x10620006, 0x00000000,
5983         0x3c020800, 0x944223d8, 0x00021400, 0x0a00031f, 0xae220014, 0x3c040800,
5984         0x248423da, 0x94820000, 0x00021400, 0xae220014, 0x3c020800, 0x8c421bbc,
5985         0x3c03c000, 0x3c010800, 0xa0201b98, 0x00431025, 0xaf625c5c, 0x8f625c50,
5986         0x30420002, 0x10400009, 0x00000000, 0x2484f7e2, 0x8c820000, 0x00431025,
5987         0xaf625c5c, 0x8f625c50, 0x30420002, 0x1440fffa, 0x00000000, 0x3c020800,
5988         0x24421b84, 0x8c430000, 0x24630001, 0xac430000, 0x8f630c14, 0x3063000f,
5989         0x2c620002, 0x1440000c, 0x3c024000, 0x8f630c14, 0x3c020800, 0x8c421b40,
5990         0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7,
5991         0x00000000, 0x3c024000, 0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002,
5992         0x1440fffc, 0x00000000, 0x12600003, 0x00000000, 0x0e0004c0, 0x00000000,
5993         0x8fbf0028, 0x8fb30024, 0x8fb20020, 0x8fb1001c, 0x8fb00018, 0x03e00008,
5994         0x27bd0030, 0x8f634450, 0x3c040800, 0x24841b88, 0x8c820000, 0x00031c02,
5995         0x0043102b, 0x14400007, 0x3c038000, 0x8c840004, 0x8f624450, 0x00021c02,
5996         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5997         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3c024000,
5998         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00000000,
5999         0x03e00008, 0x00000000, 0x27bdffe0, 0x00805821, 0x14c00011, 0x256e0008,
6000         0x3c020800, 0x8c4223f4, 0x10400007, 0x24020016, 0x3c010800, 0xa42223d2,
6001         0x2402002a, 0x3c010800, 0x0a000364, 0xa42223d4, 0x8d670010, 0x00071402,
6002         0x3c010800, 0xa42223d2, 0x3c010800, 0xa42723d4, 0x3c040800, 0x948423d4,
6003         0x3c030800, 0x946323d2, 0x95cf0006, 0x3c020800, 0x944223d0, 0x00832023,
6004         0x01e2c023, 0x3065ffff, 0x24a20028, 0x01c24821, 0x3082ffff, 0x14c0001a,
6005         0x01226021, 0x9582000c, 0x3042003f, 0x3c010800, 0xa42223d6, 0x95820004,
6006         0x95830006, 0x3c010800, 0xac2023e4, 0x3c010800, 0xac2023e8, 0x00021400,
6007         0x00431025, 0x3c010800, 0xac221bc0, 0x95220004, 0x3c010800, 0xa4221bc4,
6008         0x95230002, 0x01e51023, 0x0043102a, 0x10400010, 0x24020001, 0x3c010800,
6009         0x0a000398, 0xac2223f8, 0x3c030800, 0x8c6323e8, 0x3c020800, 0x94421bc4,
6010         0x00431021, 0xa5220004, 0x3c020800, 0x94421bc0, 0xa5820004, 0x3c020800,
6011         0x8c421bc0, 0xa5820006, 0x3c020800, 0x8c4223f0, 0x3c0d0800, 0x8dad23e4,
6012         0x3c0a0800, 0x144000e5, 0x8d4a23e8, 0x3c020800, 0x94421bc4, 0x004a1821,
6013         0x3063ffff, 0x0062182b, 0x24020002, 0x10c2000d, 0x01435023, 0x3c020800,
6014         0x944223d6, 0x30420009, 0x10400008, 0x00000000, 0x9582000c, 0x3042fff6,
6015         0xa582000c, 0x3c020800, 0x944223d6, 0x30420009, 0x01a26823, 0x3c020800,
6016         0x8c4223f8, 0x1040004a, 0x01203821, 0x3c020800, 0x944223d2, 0x00004021,
6017         0xa520000a, 0x01e21023, 0xa5220002, 0x3082ffff, 0x00021042, 0x18400008,
6018         0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021, 0x0103102a,
6019         0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061402,
6020         0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021, 0x2527000c,
6021         0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004, 0x1440fffb,
6022         0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023, 0x01803821,
6023         0x3082ffff, 0xa4e00010, 0x00621821, 0x00021042, 0x18400010, 0x00c33021,
6024         0x00404821, 0x94e20000, 0x24e70002, 0x00c23021, 0x30e2007f, 0x14400006,
6025         0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80, 0x00625824, 0x25670008,
6026         0x0109102a, 0x1440fff3, 0x00000000, 0x30820001, 0x10400005, 0x00061c02,
6027         0xa0e00001, 0x94e20000, 0x00c23021, 0x00061c02, 0x30c2ffff, 0x00623021,
6028         0x00061402, 0x00c23021, 0x0a00047d, 0x30c6ffff, 0x24020002, 0x14c20081,
6029         0x00000000, 0x3c020800, 0x8c42240c, 0x14400007, 0x00000000, 0x3c020800,
6030         0x944223d2, 0x95230002, 0x01e21023, 0x10620077, 0x00000000, 0x3c020800,
6031         0x944223d2, 0x01e21023, 0xa5220002, 0x3c020800, 0x8c42240c, 0x1040001a,
6032         0x31e3ffff, 0x8dc70010, 0x3c020800, 0x94421b96, 0x00e04021, 0x00072c02,
6033         0x00aa2021, 0x00431023, 0x00823823, 0x00072402, 0x30e2ffff, 0x00823821,
6034         0x00071027, 0xa522000a, 0x3102ffff, 0x3c040800, 0x948423d4, 0x00453023,
6035         0x00e02821, 0x00641823, 0x006d1821, 0x00c33021, 0x00061c02, 0x30c2ffff,
6036         0x0a00047d, 0x00623021, 0x01203821, 0x00004021, 0x3082ffff, 0x00021042,
6037         0x18400008, 0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021,
6038         0x0103102a, 0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021,
6039         0x00061402, 0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021,
6040         0x2527000c, 0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004,
6041         0x1440fffb, 0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023,
6042         0x01803821, 0x3082ffff, 0xa4e00010, 0x3c040800, 0x948423d4, 0x00621821,
6043         0x00c33021, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061c02, 0x3c020800,
6044         0x944223d0, 0x00c34821, 0x00441023, 0x00021fc2, 0x00431021, 0x00021043,
6045         0x18400010, 0x00003021, 0x00402021, 0x94e20000, 0x24e70002, 0x00c23021,
6046         0x30e2007f, 0x14400006, 0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80,
6047         0x00625824, 0x25670008, 0x0104102a, 0x1440fff3, 0x00000000, 0x3c020800,
6048         0x944223ec, 0x00c23021, 0x3122ffff, 0x00c23021, 0x00061c02, 0x30c2ffff,
6049         0x00623021, 0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010,
6050         0xadc00014, 0x0a00049d, 0xadc00000, 0x8dc70010, 0x00e04021, 0x11400007,
6051         0x00072c02, 0x00aa3021, 0x00061402, 0x30c3ffff, 0x00433021, 0x00061402,
6052         0x00c22821, 0x00051027, 0xa522000a, 0x3c030800, 0x946323d4, 0x3102ffff,
6053         0x01e21021, 0x00433023, 0x00cd3021, 0x00061c02, 0x30c2ffff, 0x00623021,
6054         0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010, 0x3102ffff,
6055         0x00051c00, 0x00431025, 0xadc20010, 0x3c020800, 0x8c4223f4, 0x10400005,
6056         0x2de205eb, 0x14400002, 0x25e2fff2, 0x34028870, 0xa5c20034, 0x3c030800,
6057         0x246323e8, 0x8c620000, 0x24420001, 0xac620000, 0x3c040800, 0x8c8423e4,
6058         0x3c020800, 0x8c421bc0, 0x3303ffff, 0x00832021, 0x00431821, 0x0062102b,
6059         0x3c010800, 0xac2423e4, 0x10400003, 0x2482ffff, 0x3c010800, 0xac2223e4,
6060         0x3c010800, 0xac231bc0, 0x03e00008, 0x27bd0020, 0x27bdffb8, 0x3c050800,
6061         0x24a51b96, 0xafbf0044, 0xafbe0040, 0xafb7003c, 0xafb60038, 0xafb50034,
6062         0xafb40030, 0xafb3002c, 0xafb20028, 0xafb10024, 0xafb00020, 0x94a90000,
6063         0x3c020800, 0x944223d0, 0x3c030800, 0x8c631bb0, 0x3c040800, 0x8c841bac,
6064         0x01221023, 0x0064182a, 0xa7a9001e, 0x106000be, 0xa7a20016, 0x24be0022,
6065         0x97b6001e, 0x24b3001a, 0x24b70016, 0x8fc20000, 0x14400008, 0x00000000,
6066         0x8fc2fff8, 0x97a30016, 0x8fc4fff4, 0x00431021, 0x0082202a, 0x148000b0,
6067         0x00000000, 0x97d50818, 0x32a2ffff, 0x104000a3, 0x00009021, 0x0040a021,
6068         0x00008821, 0x0e000625, 0x00000000, 0x00403021, 0x14c00007, 0x00000000,
6069         0x3c020800, 0x8c4223dc, 0x24420001, 0x3c010800, 0x0a000596, 0xac2223dc,
6070         0x3c100800, 0x02118021, 0x8e101bc8, 0x9608000a, 0x31020040, 0x10400005,
6071         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x31020080,
6072         0x54400001, 0x34e70010, 0x3c020800, 0x00511021, 0x8c421bd0, 0x3c030800,
6073         0x00711821, 0x8c631bd4, 0x00021500, 0x00031c00, 0x00431025, 0xacc20014,
6074         0x96040008, 0x3242ffff, 0x00821021, 0x0282102a, 0x14400002, 0x02b22823,
6075         0x00802821, 0x8e020000, 0x02459021, 0xacc20000, 0x8e020004, 0x00c02021,
6076         0x26310010, 0xac820004, 0x30e2ffff, 0xac800008, 0xa485000e, 0xac820010,
6077         0x24020305, 0x0e0005a2, 0xa482000c, 0x3242ffff, 0x0054102b, 0x1440ffc5,
6078         0x3242ffff, 0x0a00058e, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
6079         0x10400067, 0x00000000, 0x8e62fff0, 0x00028900, 0x3c100800, 0x02118021,
6080         0x0e000625, 0x8e101bc8, 0x00403021, 0x14c00005, 0x00000000, 0x8e62082c,
6081         0x24420001, 0x0a000596, 0xae62082c, 0x9608000a, 0x31020040, 0x10400005,
6082         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x3c020800,
6083         0x00511021, 0x8c421bd0, 0x3c030800, 0x00711821, 0x8c631bd4, 0x00021500,
6084         0x00031c00, 0x00431025, 0xacc20014, 0x8e63fff4, 0x96020008, 0x00432023,
6085         0x3242ffff, 0x3083ffff, 0x00431021, 0x02c2102a, 0x10400003, 0x00802821,
6086         0x97a9001e, 0x01322823, 0x8e620000, 0x30a4ffff, 0x00441021, 0xae620000,
6087         0xa4c5000e, 0x8e020000, 0xacc20000, 0x8e020004, 0x8e63fff4, 0x00431021,
6088         0xacc20004, 0x8e63fff4, 0x96020008, 0x00641821, 0x0062102a, 0x14400006,
6089         0x02459021, 0x8e62fff0, 0xae60fff4, 0x24420001, 0x0a000571, 0xae62fff0,
6090         0xae63fff4, 0xacc00008, 0x3242ffff, 0x10560003, 0x31020004, 0x10400006,
6091         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x34e70020, 0x24020905,
6092         0xa4c2000c, 0x8ee30000, 0x8ee20004, 0x14620007, 0x3c02b49a, 0x8ee20860,
6093         0x54400001, 0x34e70400, 0x3c024b65, 0x0a000588, 0x34427654, 0x344289ab,
6094         0xacc2001c, 0x30e2ffff, 0xacc20010, 0x0e0005a2, 0x00c02021, 0x3242ffff,
6095         0x0056102b, 0x1440ff9b, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
6096         0x1440ff48, 0x00000000, 0x8fbf0044, 0x8fbe0040, 0x8fb7003c, 0x8fb60038,
6097         0x8fb50034, 0x8fb40030, 0x8fb3002c, 0x8fb20028, 0x8fb10024, 0x8fb00020,
6098         0x03e00008, 0x27bd0048, 0x27bdffe8, 0xafbf0014, 0xafb00010, 0x8f624450,
6099         0x8f634410, 0x0a0005b1, 0x00808021, 0x8f626820, 0x30422000, 0x10400003,
6100         0x00000000, 0x0e0001f0, 0x00002021, 0x8f624450, 0x8f634410, 0x3042ffff,
6101         0x0043102b, 0x1440fff5, 0x00000000, 0x8f630c14, 0x3063000f, 0x2c620002,
6102         0x1440000b, 0x00000000, 0x8f630c14, 0x3c020800, 0x8c421b40, 0x3063000f,
6103         0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7, 0x00000000,
6104         0xaf705c18, 0x8f625c10, 0x30420002, 0x10400009, 0x00000000, 0x8f626820,
6105         0x30422000, 0x1040fff8, 0x00000000, 0x0e0001f0, 0x00002021, 0x0a0005c4,
6106         0x00000000, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000,
6107         0x00000000, 0x00000000, 0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010,
6108         0xaf60680c, 0x8f626804, 0x34420082, 0xaf626804, 0x8f634000, 0x24020b50,
6109         0x3c010800, 0xac221b54, 0x24020b78, 0x3c010800, 0xac221b64, 0x34630002,
6110         0xaf634000, 0x0e000605, 0x00808021, 0x3c010800, 0xa0221b68, 0x304200ff,
6111         0x24030002, 0x14430005, 0x00000000, 0x3c020800, 0x8c421b54, 0x0a0005f8,
6112         0xac5000c0, 0x3c020800, 0x8c421b54, 0xac5000bc, 0x8f624434, 0x8f634438,
6113         0x8f644410, 0x3c010800, 0xac221b5c, 0x3c010800, 0xac231b6c, 0x3c010800,
6114         0xac241b58, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x3c040800,
6115         0x8c870000, 0x3c03aa55, 0x3463aa55, 0x3c06c003, 0xac830000, 0x8cc20000,
6116         0x14430007, 0x24050002, 0x3c0355aa, 0x346355aa, 0xac830000, 0x8cc20000,
6117         0x50430001, 0x24050001, 0x3c020800, 0xac470000, 0x03e00008, 0x00a01021,
6118         0x27bdfff8, 0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe,
6119         0x00000000, 0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008,
6120         0x27bd0008, 0x8f634450, 0x3c020800, 0x8c421b5c, 0x00031c02, 0x0043102b,
6121         0x14400008, 0x3c038000, 0x3c040800, 0x8c841b6c, 0x8f624450, 0x00021c02,
6122         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
6123         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff,
6124         0x2442e000, 0x2c422001, 0x14400003, 0x3c024000, 0x0a000648, 0x2402ffff,
6125         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021,
6126         0x03e00008, 0x00000000, 0x8f624450, 0x3c030800, 0x8c631b58, 0x0a000651,
6127         0x3042ffff, 0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000,
6128         0x03e00008, 0x00000000, 0x27bdffe0, 0x00802821, 0x3c040800, 0x24841af0,
6129         0x00003021, 0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014,
6130         0x0a000660, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000,
6131         0x00000000, 0x00000000, 0x3c020800, 0x34423000, 0x3c030800, 0x34633000,
6132         0x3c040800, 0x348437ff, 0x3c010800, 0xac221b74, 0x24020040, 0x3c010800,
6133         0xac221b78, 0x3c010800, 0xac201b70, 0xac600000, 0x24630004, 0x0083102b,
6134         0x5040fffd, 0xac600000, 0x03e00008, 0x00000000, 0x00804821, 0x8faa0010,
6135         0x3c020800, 0x8c421b70, 0x3c040800, 0x8c841b78, 0x8fab0014, 0x24430001,
6136         0x0044102b, 0x3c010800, 0xac231b70, 0x14400003, 0x00004021, 0x3c010800,
6137         0xac201b70, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74, 0x91240000,
6138         0x00021140, 0x00431021, 0x00481021, 0x25080001, 0xa0440000, 0x29020008,
6139         0x1440fff4, 0x25290001, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74,
6140         0x8f64680c, 0x00021140, 0x00431021, 0xac440008, 0xac45000c, 0xac460010,
6141         0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c, 0x00000000, 0x00000000,
6142 };
6143
6144 static const u32 tg3TsoFwRodata[] = {
6145         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
6146         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x496e0000, 0x73746b6f,
6147         0x66662a2a, 0x00000000, 0x53774576, 0x656e7430, 0x00000000, 0x00000000,
6148         0x00000000, 0x00000000, 0x66617461, 0x6c457272, 0x00000000, 0x00000000,
6149         0x00000000,
6150 };
6151
6152 static const u32 tg3TsoFwData[] = {
6153         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x362e3000, 0x00000000,
6154         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
6155         0x00000000,
6156 };
6157
6158 /* 5705 needs a special version of the TSO firmware.  */
6159 #define TG3_TSO5_FW_RELEASE_MAJOR       0x1
6160 #define TG3_TSO5_FW_RELASE_MINOR        0x2
6161 #define TG3_TSO5_FW_RELEASE_FIX         0x0
6162 #define TG3_TSO5_FW_START_ADDR          0x00010000
6163 #define TG3_TSO5_FW_TEXT_ADDR           0x00010000
6164 #define TG3_TSO5_FW_TEXT_LEN            0xe90
6165 #define TG3_TSO5_FW_RODATA_ADDR         0x00010e90
6166 #define TG3_TSO5_FW_RODATA_LEN          0x50
6167 #define TG3_TSO5_FW_DATA_ADDR           0x00010f00
6168 #define TG3_TSO5_FW_DATA_LEN            0x20
6169 #define TG3_TSO5_FW_SBSS_ADDR           0x00010f20
6170 #define TG3_TSO5_FW_SBSS_LEN            0x28
6171 #define TG3_TSO5_FW_BSS_ADDR            0x00010f50
6172 #define TG3_TSO5_FW_BSS_LEN             0x88
6173
6174 static const u32 tg3Tso5FwText[(TG3_TSO5_FW_TEXT_LEN / 4) + 1] = {
6175         0x0c004003, 0x00000000, 0x00010f04, 0x00000000, 0x10000003, 0x00000000,
6176         0x0000000d, 0x0000000d, 0x3c1d0001, 0x37bde000, 0x03a0f021, 0x3c100001,
6177         0x26100000, 0x0c004010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
6178         0xafbf0018, 0x0c0042e8, 0x34840002, 0x0c004364, 0x00000000, 0x3c030001,
6179         0x90630f34, 0x24020002, 0x3c040001, 0x24840e9c, 0x14620003, 0x24050001,
6180         0x3c040001, 0x24840e90, 0x24060002, 0x00003821, 0xafa00010, 0x0c004378,
6181         0xafa00014, 0x0c00402c, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020,
6182         0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c, 0xafb20018, 0xafb10014,
6183         0x0c0042d4, 0xafb00010, 0x3c128000, 0x24110001, 0x8f706810, 0x32020400,
6184         0x10400007, 0x00000000, 0x8f641008, 0x00921024, 0x14400003, 0x00000000,
6185         0x0c004064, 0x00000000, 0x3c020001, 0x90420f56, 0x10510003, 0x32020200,
6186         0x1040fff1, 0x00000000, 0x0c0041b4, 0x00000000, 0x08004034, 0x00000000,
6187         0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020,
6188         0x27bdffe0, 0x3c040001, 0x24840eb0, 0x00002821, 0x00003021, 0x00003821,
6189         0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0000d021, 0x24020130,
6190         0xaf625000, 0x3c010001, 0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018,
6191         0x03e00008, 0x27bd0020, 0x00000000, 0x00000000, 0x3c030001, 0x24630f60,
6192         0x90620000, 0x27bdfff0, 0x14400003, 0x0080c021, 0x08004073, 0x00004821,
6193         0x3c022000, 0x03021024, 0x10400003, 0x24090002, 0x08004073, 0xa0600000,
6194         0x24090001, 0x00181040, 0x30431f80, 0x346f8008, 0x1520004b, 0x25eb0028,
6195         0x3c040001, 0x00832021, 0x8c848010, 0x3c050001, 0x24a50f7a, 0x00041402,
6196         0xa0a20000, 0x3c010001, 0xa0240f7b, 0x3c020001, 0x00431021, 0x94428014,
6197         0x3c010001, 0xa0220f7c, 0x3c0c0001, 0x01836021, 0x8d8c8018, 0x304200ff,
6198         0x24420008, 0x000220c3, 0x24020001, 0x3c010001, 0xa0220f60, 0x0124102b,
6199         0x1040000c, 0x00003821, 0x24a6000e, 0x01602821, 0x8ca20000, 0x8ca30004,
6200         0x24a50008, 0x24e70001, 0xacc20000, 0xacc30004, 0x00e4102b, 0x1440fff8,
6201         0x24c60008, 0x00003821, 0x3c080001, 0x25080f7b, 0x91060000, 0x3c020001,
6202         0x90420f7c, 0x2503000d, 0x00c32821, 0x00461023, 0x00021fc2, 0x00431021,
6203         0x00021043, 0x1840000c, 0x00002021, 0x91020001, 0x00461023, 0x00021fc2,
6204         0x00431021, 0x00021843, 0x94a20000, 0x24e70001, 0x00822021, 0x00e3102a,
6205         0x1440fffb, 0x24a50002, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
6206         0x00822021, 0x3c02ffff, 0x01821024, 0x3083ffff, 0x00431025, 0x3c010001,
6207         0x080040fa, 0xac220f80, 0x3c050001, 0x24a50f7c, 0x90a20000, 0x3c0c0001,
6208         0x01836021, 0x8d8c8018, 0x000220c2, 0x1080000e, 0x00003821, 0x01603021,
6209         0x24a5000c, 0x8ca20000, 0x8ca30004, 0x24a50008, 0x24e70001, 0xacc20000,
6210         0xacc30004, 0x00e4102b, 0x1440fff8, 0x24c60008, 0x3c050001, 0x24a50f7c,
6211         0x90a20000, 0x30430007, 0x24020004, 0x10620011, 0x28620005, 0x10400005,
6212         0x24020002, 0x10620008, 0x000710c0, 0x080040fa, 0x00000000, 0x24020006,
6213         0x1062000e, 0x000710c0, 0x080040fa, 0x00000000, 0x00a21821, 0x9463000c,
6214         0x004b1021, 0x080040fa, 0xa4430000, 0x000710c0, 0x00a21821, 0x8c63000c,
6215         0x004b1021, 0x080040fa, 0xac430000, 0x00a21821, 0x8c63000c, 0x004b2021,
6216         0x00a21021, 0xac830000, 0x94420010, 0xa4820004, 0x95e70006, 0x3c020001,
6217         0x90420f7c, 0x3c030001, 0x90630f7a, 0x00e2c823, 0x3c020001, 0x90420f7b,
6218         0x24630028, 0x01e34021, 0x24420028, 0x15200012, 0x01e23021, 0x94c2000c,
6219         0x3c010001, 0xa4220f78, 0x94c20004, 0x94c30006, 0x3c010001, 0xa4200f76,
6220         0x3c010001, 0xa4200f72, 0x00021400, 0x00431025, 0x3c010001, 0xac220f6c,
6221         0x95020004, 0x3c010001, 0x08004124, 0xa4220f70, 0x3c020001, 0x94420f70,
6222         0x3c030001, 0x94630f72, 0x00431021, 0xa5020004, 0x3c020001, 0x94420f6c,
6223         0xa4c20004, 0x3c020001, 0x8c420f6c, 0xa4c20006, 0x3c040001, 0x94840f72,
6224         0x3c020001, 0x94420f70, 0x3c0a0001, 0x954a0f76, 0x00441821, 0x3063ffff,
6225         0x0062182a, 0x24020002, 0x1122000b, 0x00832023, 0x3c030001, 0x94630f78,
6226         0x30620009, 0x10400006, 0x3062fff6, 0xa4c2000c, 0x3c020001, 0x94420f78,
6227         0x30420009, 0x01425023, 0x24020001, 0x1122001b, 0x29220002, 0x50400005,
6228         0x24020002, 0x11200007, 0x31a2ffff, 0x08004197, 0x00000000, 0x1122001d,
6229         0x24020016, 0x08004197, 0x31a2ffff, 0x3c0e0001, 0x95ce0f80, 0x10800005,
6230         0x01806821, 0x01c42021, 0x00041c02, 0x3082ffff, 0x00627021, 0x000e1027,
6231         0xa502000a, 0x3c030001, 0x90630f7b, 0x31a2ffff, 0x00e21021, 0x0800418d,
6232         0x00432023, 0x3c020001, 0x94420f80, 0x00442021, 0x00041c02, 0x3082ffff,
6233         0x00622021, 0x00807021, 0x00041027, 0x08004185, 0xa502000a, 0x3c050001,
6234         0x24a50f7a, 0x90a30000, 0x14620002, 0x24e2fff2, 0xa5e20034, 0x90a20000,
6235         0x00e21023, 0xa5020002, 0x3c030001, 0x94630f80, 0x3c020001, 0x94420f5a,
6236         0x30e5ffff, 0x00641821, 0x00451023, 0x00622023, 0x00041c02, 0x3082ffff,
6237         0x00622021, 0x00041027, 0xa502000a, 0x3c030001, 0x90630f7c, 0x24620001,
6238         0x14a20005, 0x00807021, 0x01631021, 0x90420000, 0x08004185, 0x00026200,
6239         0x24620002, 0x14a20003, 0x306200fe, 0x004b1021, 0x944c0000, 0x3c020001,
6240         0x94420f82, 0x3183ffff, 0x3c040001, 0x90840f7b, 0x00431021, 0x00e21021,
6241         0x00442023, 0x008a2021, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
6242         0x00822021, 0x00806821, 0x00041027, 0xa4c20010, 0x31a2ffff, 0x000e1c00,
6243         0x00431025, 0x3c040001, 0x24840f72, 0xade20010, 0x94820000, 0x3c050001,
6244         0x94a50f76, 0x3c030001, 0x8c630f6c, 0x24420001, 0x00b92821, 0xa4820000,
6245         0x3322ffff, 0x00622021, 0x0083182b, 0x3c010001, 0xa4250f76, 0x10600003,
6246         0x24a2ffff, 0x3c010001, 0xa4220f76, 0x3c024000, 0x03021025, 0x3c010001,
6247         0xac240f6c, 0xaf621008, 0x03e00008, 0x27bd0010, 0x3c030001, 0x90630f56,
6248         0x27bdffe8, 0x24020001, 0xafbf0014, 0x10620026, 0xafb00010, 0x8f620cf4,
6249         0x2442ffff, 0x3042007f, 0x00021100, 0x8c434000, 0x3c010001, 0xac230f64,
6250         0x8c434008, 0x24444000, 0x8c5c4004, 0x30620040, 0x14400002, 0x24020088,
6251         0x24020008, 0x3c010001, 0xa4220f68, 0x30620004, 0x10400005, 0x24020001,
6252         0x3c010001, 0xa0220f57, 0x080041d5, 0x00031402, 0x3c010001, 0xa0200f57,
6253         0x00031402, 0x3c010001, 0xa4220f54, 0x9483000c, 0x24020001, 0x3c010001,
6254         0xa4200f50, 0x3c010001, 0xa0220f56, 0x3c010001, 0xa4230f62, 0x24020001,
6255         0x1342001e, 0x00000000, 0x13400005, 0x24020003, 0x13420067, 0x00000000,
6256         0x080042cf, 0x00000000, 0x3c020001, 0x94420f62, 0x241a0001, 0x3c010001,
6257         0xa4200f5e, 0x3c010001, 0xa4200f52, 0x304407ff, 0x00021bc2, 0x00031823,
6258         0x3063003e, 0x34630036, 0x00021242, 0x3042003c, 0x00621821, 0x3c010001,
6259         0xa4240f58, 0x00832021, 0x24630030, 0x3c010001, 0xa4240f5a, 0x3c010001,
6260         0xa4230f5c, 0x3c060001, 0x24c60f52, 0x94c50000, 0x94c30002, 0x3c040001,
6261         0x94840f5a, 0x00651021, 0x0044102a, 0x10400013, 0x3c108000, 0x00a31021,
6262         0xa4c20000, 0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008,
6263         0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4,
6264         0x00501024, 0x104000b7, 0x00000000, 0x0800420f, 0x00000000, 0x3c030001,
6265         0x94630f50, 0x00851023, 0xa4c40000, 0x00621821, 0x3042ffff, 0x3c010001,
6266         0xa4230f50, 0xaf620ce8, 0x3c020001, 0x94420f68, 0x34420024, 0xaf620cec,
6267         0x94c30002, 0x3c020001, 0x94420f50, 0x14620012, 0x3c028000, 0x3c108000,
6268         0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008, 0x00901024,
6269         0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024,
6270         0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003, 0xaf620cf4, 0x3c108000,
6271         0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000,
6272         0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003,
6273         0x3c070001, 0x24e70f50, 0x94e20000, 0x03821021, 0xaf620ce0, 0x3c020001,
6274         0x8c420f64, 0xaf620ce4, 0x3c050001, 0x94a50f54, 0x94e30000, 0x3c040001,
6275         0x94840f58, 0x3c020001, 0x94420f5e, 0x00a32823, 0x00822023, 0x30a6ffff,
6276         0x3083ffff, 0x00c3102b, 0x14400043, 0x00000000, 0x3c020001, 0x94420f5c,
6277         0x00021400, 0x00621025, 0xaf620ce8, 0x94e20000, 0x3c030001, 0x94630f54,
6278         0x00441021, 0xa4e20000, 0x3042ffff, 0x14430021, 0x3c020008, 0x3c020001,
6279         0x90420f57, 0x10400006, 0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624,
6280         0x0800427c, 0x0000d021, 0x3c020001, 0x94420f68, 0x3c030008, 0x34630624,
6281         0x00431025, 0xaf620cec, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
6282         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
6283         0x00000000, 0x8f620cf4, 0x00501024, 0x10400015, 0x00000000, 0x08004283,
6284         0x00000000, 0x3c030001, 0x94630f68, 0x34420624, 0x3c108000, 0x00621825,
6285         0x3c028000, 0xaf630cec, 0xaf620cf4, 0x8f641008, 0x00901024, 0x14400003,
6286         0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7,
6287         0x00000000, 0x3c010001, 0x080042cf, 0xa4200f5e, 0x3c020001, 0x94420f5c,
6288         0x00021400, 0x00c21025, 0xaf620ce8, 0x3c020001, 0x90420f57, 0x10400009,
6289         0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624, 0x0000d021, 0x00431025,
6290         0xaf620cec, 0x080042c1, 0x3c108000, 0x3c020001, 0x94420f68, 0x3c030008,
6291         0x34630604, 0x00431025, 0xaf620cec, 0x3c020001, 0x94420f5e, 0x00451021,
6292         0x3c010001, 0xa4220f5e, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
6293         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
6294         0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x8fbf0014,
6295         0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000, 0x27bdffe0, 0x3c040001,
6296         0x24840ec0, 0x00002821, 0x00003021, 0x00003821, 0xafbf0018, 0xafa00010,
6297         0x0c004378, 0xafa00014, 0x0000d021, 0x24020130, 0xaf625000, 0x3c010001,
6298         0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018, 0x03e00008, 0x27bd0020,
6299         0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010, 0xaf60680c, 0x8f626804,
6300         0x34420082, 0xaf626804, 0x8f634000, 0x24020b50, 0x3c010001, 0xac220f20,
6301         0x24020b78, 0x3c010001, 0xac220f30, 0x34630002, 0xaf634000, 0x0c004315,
6302         0x00808021, 0x3c010001, 0xa0220f34, 0x304200ff, 0x24030002, 0x14430005,
6303         0x00000000, 0x3c020001, 0x8c420f20, 0x08004308, 0xac5000c0, 0x3c020001,
6304         0x8c420f20, 0xac5000bc, 0x8f624434, 0x8f634438, 0x8f644410, 0x3c010001,
6305         0xac220f28, 0x3c010001, 0xac230f38, 0x3c010001, 0xac240f24, 0x8fbf0014,
6306         0x8fb00010, 0x03e00008, 0x27bd0018, 0x03e00008, 0x24020001, 0x27bdfff8,
6307         0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe, 0x00000000,
6308         0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008, 0x27bd0008,
6309         0x8f634450, 0x3c020001, 0x8c420f28, 0x00031c02, 0x0043102b, 0x14400008,
6310         0x3c038000, 0x3c040001, 0x8c840f38, 0x8f624450, 0x00021c02, 0x0083102b,
6311         0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024, 0x1440fffd,
6312         0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff, 0x2442e000,
6313         0x2c422001, 0x14400003, 0x3c024000, 0x08004347, 0x2402ffff, 0x00822025,
6314         0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021, 0x03e00008,
6315         0x00000000, 0x8f624450, 0x3c030001, 0x8c630f24, 0x08004350, 0x3042ffff,
6316         0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000, 0x03e00008,
6317         0x00000000, 0x27bdffe0, 0x00802821, 0x3c040001, 0x24840ed0, 0x00003021,
6318         0x00003821, 0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0800435f,
6319         0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x3c020001, 0x3442d600,
6320         0x3c030001, 0x3463d600, 0x3c040001, 0x3484ddff, 0x3c010001, 0xac220f40,
6321         0x24020040, 0x3c010001, 0xac220f44, 0x3c010001, 0xac200f3c, 0xac600000,
6322         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
6323         0x00804821, 0x8faa0010, 0x3c020001, 0x8c420f3c, 0x3c040001, 0x8c840f44,
6324         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010001, 0xac230f3c, 0x14400003,
6325         0x00004021, 0x3c010001, 0xac200f3c, 0x3c020001, 0x8c420f3c, 0x3c030001,
6326         0x8c630f40, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
6327         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020001, 0x8c420f3c,
6328         0x3c030001, 0x8c630f40, 0x8f64680c, 0x00021140, 0x00431021, 0xac440008,
6329         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
6330         0x00000000, 0x00000000, 0x00000000,
6331 };
6332
6333 static const u32 tg3Tso5FwRodata[(TG3_TSO5_FW_RODATA_LEN / 4) + 1] = {
6334         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
6335         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x00000000, 0x00000000,
6336         0x73746b6f, 0x66666c64, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
6337         0x00000000, 0x00000000, 0x00000000,
6338 };
6339
6340 static const u32 tg3Tso5FwData[(TG3_TSO5_FW_DATA_LEN / 4) + 1] = {
6341         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x322e3000, 0x00000000,
6342         0x00000000, 0x00000000, 0x00000000,
6343 };
6344
6345 /* tp->lock is held. */
6346 static int tg3_load_tso_firmware(struct tg3 *tp)
6347 {
6348         struct fw_info info;
6349         unsigned long cpu_base, cpu_scratch_base, cpu_scratch_size;
6350         int err, i;
6351
6352         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6353                 return 0;
6354
6355         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
6356                 info.text_base = TG3_TSO5_FW_TEXT_ADDR;
6357                 info.text_len = TG3_TSO5_FW_TEXT_LEN;
6358                 info.text_data = &tg3Tso5FwText[0];
6359                 info.rodata_base = TG3_TSO5_FW_RODATA_ADDR;
6360                 info.rodata_len = TG3_TSO5_FW_RODATA_LEN;
6361                 info.rodata_data = &tg3Tso5FwRodata[0];
6362                 info.data_base = TG3_TSO5_FW_DATA_ADDR;
6363                 info.data_len = TG3_TSO5_FW_DATA_LEN;
6364                 info.data_data = &tg3Tso5FwData[0];
6365                 cpu_base = RX_CPU_BASE;
6366                 cpu_scratch_base = NIC_SRAM_MBUF_POOL_BASE5705;
6367                 cpu_scratch_size = (info.text_len +
6368                                     info.rodata_len +
6369                                     info.data_len +
6370                                     TG3_TSO5_FW_SBSS_LEN +
6371                                     TG3_TSO5_FW_BSS_LEN);
6372         } else {
6373                 info.text_base = TG3_TSO_FW_TEXT_ADDR;
6374                 info.text_len = TG3_TSO_FW_TEXT_LEN;
6375                 info.text_data = &tg3TsoFwText[0];
6376                 info.rodata_base = TG3_TSO_FW_RODATA_ADDR;
6377                 info.rodata_len = TG3_TSO_FW_RODATA_LEN;
6378                 info.rodata_data = &tg3TsoFwRodata[0];
6379                 info.data_base = TG3_TSO_FW_DATA_ADDR;
6380                 info.data_len = TG3_TSO_FW_DATA_LEN;
6381                 info.data_data = &tg3TsoFwData[0];
6382                 cpu_base = TX_CPU_BASE;
6383                 cpu_scratch_base = TX_CPU_SCRATCH_BASE;
6384                 cpu_scratch_size = TX_CPU_SCRATCH_SIZE;
6385         }
6386
6387         err = tg3_load_firmware_cpu(tp, cpu_base,
6388                                     cpu_scratch_base, cpu_scratch_size,
6389                                     &info);
6390         if (err)
6391                 return err;
6392
6393         /* Now startup the cpu. */
6394         tw32(cpu_base + CPU_STATE, 0xffffffff);
6395         tw32_f(cpu_base + CPU_PC,    info.text_base);
6396
6397         for (i = 0; i < 5; i++) {
6398                 if (tr32(cpu_base + CPU_PC) == info.text_base)
6399                         break;
6400                 tw32(cpu_base + CPU_STATE, 0xffffffff);
6401                 tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
6402                 tw32_f(cpu_base + CPU_PC,    info.text_base);
6403                 udelay(1000);
6404         }
6405         if (i >= 5) {
6406                 printk(KERN_ERR PFX "tg3_load_tso_firmware fails for %s "
6407                        "to set CPU PC, is %08x should be %08x\n",
6408                        tp->dev->name, tr32(cpu_base + CPU_PC),
6409                        info.text_base);
6410                 return -ENODEV;
6411         }
6412         tw32(cpu_base + CPU_STATE, 0xffffffff);
6413         tw32_f(cpu_base + CPU_MODE,  0x00000000);
6414         return 0;
6415 }
6416
6417
6418 /* tp->lock is held. */
6419 static void __tg3_set_mac_addr(struct tg3 *tp, int skip_mac_1)
6420 {
6421         u32 addr_high, addr_low;
6422         int i;
6423
6424         addr_high = ((tp->dev->dev_addr[0] << 8) |
6425                      tp->dev->dev_addr[1]);
6426         addr_low = ((tp->dev->dev_addr[2] << 24) |
6427                     (tp->dev->dev_addr[3] << 16) |
6428                     (tp->dev->dev_addr[4] <<  8) |
6429                     (tp->dev->dev_addr[5] <<  0));
6430         for (i = 0; i < 4; i++) {
6431                 if (i == 1 && skip_mac_1)
6432                         continue;
6433                 tw32(MAC_ADDR_0_HIGH + (i * 8), addr_high);
6434                 tw32(MAC_ADDR_0_LOW + (i * 8), addr_low);
6435         }
6436
6437         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
6438             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
6439                 for (i = 0; i < 12; i++) {
6440                         tw32(MAC_EXTADDR_0_HIGH + (i * 8), addr_high);
6441                         tw32(MAC_EXTADDR_0_LOW + (i * 8), addr_low);
6442                 }
6443         }
6444
6445         addr_high = (tp->dev->dev_addr[0] +
6446                      tp->dev->dev_addr[1] +
6447                      tp->dev->dev_addr[2] +
6448                      tp->dev->dev_addr[3] +
6449                      tp->dev->dev_addr[4] +
6450                      tp->dev->dev_addr[5]) &
6451                 TX_BACKOFF_SEED_MASK;
6452         tw32(MAC_TX_BACKOFF_SEED, addr_high);
6453 }
6454
6455 static int tg3_set_mac_addr(struct net_device *dev, void *p)
6456 {
6457         struct tg3 *tp = netdev_priv(dev);
6458         struct sockaddr *addr = p;
6459         int err = 0, skip_mac_1 = 0;
6460
6461         if (!is_valid_ether_addr(addr->sa_data))
6462                 return -EINVAL;
6463
6464         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
6465
6466         if (!netif_running(dev))
6467                 return 0;
6468
6469         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
6470                 u32 addr0_high, addr0_low, addr1_high, addr1_low;
6471
6472                 addr0_high = tr32(MAC_ADDR_0_HIGH);
6473                 addr0_low = tr32(MAC_ADDR_0_LOW);
6474                 addr1_high = tr32(MAC_ADDR_1_HIGH);
6475                 addr1_low = tr32(MAC_ADDR_1_LOW);
6476
6477                 /* Skip MAC addr 1 if ASF is using it. */
6478                 if ((addr0_high != addr1_high || addr0_low != addr1_low) &&
6479                     !(addr1_high == 0 && addr1_low == 0))
6480                         skip_mac_1 = 1;
6481         }
6482         spin_lock_bh(&tp->lock);
6483         __tg3_set_mac_addr(tp, skip_mac_1);
6484         spin_unlock_bh(&tp->lock);
6485
6486         return err;
6487 }
6488
6489 /* tp->lock is held. */
6490 static void tg3_set_bdinfo(struct tg3 *tp, u32 bdinfo_addr,
6491                            dma_addr_t mapping, u32 maxlen_flags,
6492                            u32 nic_addr)
6493 {
6494         tg3_write_mem(tp,
6495                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH),
6496                       ((u64) mapping >> 32));
6497         tg3_write_mem(tp,
6498                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW),
6499                       ((u64) mapping & 0xffffffff));
6500         tg3_write_mem(tp,
6501                       (bdinfo_addr + TG3_BDINFO_MAXLEN_FLAGS),
6502                        maxlen_flags);
6503
6504         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6505                 tg3_write_mem(tp,
6506                               (bdinfo_addr + TG3_BDINFO_NIC_ADDR),
6507                               nic_addr);
6508 }
6509
6510 static void __tg3_set_rx_mode(struct net_device *);
6511 static void __tg3_set_coalesce(struct tg3 *tp, struct ethtool_coalesce *ec)
6512 {
6513         tw32(HOSTCC_RXCOL_TICKS, ec->rx_coalesce_usecs);
6514         tw32(HOSTCC_TXCOL_TICKS, ec->tx_coalesce_usecs);
6515         tw32(HOSTCC_RXMAX_FRAMES, ec->rx_max_coalesced_frames);
6516         tw32(HOSTCC_TXMAX_FRAMES, ec->tx_max_coalesced_frames);
6517         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6518                 tw32(HOSTCC_RXCOAL_TICK_INT, ec->rx_coalesce_usecs_irq);
6519                 tw32(HOSTCC_TXCOAL_TICK_INT, ec->tx_coalesce_usecs_irq);
6520         }
6521         tw32(HOSTCC_RXCOAL_MAXF_INT, ec->rx_max_coalesced_frames_irq);
6522         tw32(HOSTCC_TXCOAL_MAXF_INT, ec->tx_max_coalesced_frames_irq);
6523         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6524                 u32 val = ec->stats_block_coalesce_usecs;
6525
6526                 if (!netif_carrier_ok(tp->dev))
6527                         val = 0;
6528
6529                 tw32(HOSTCC_STAT_COAL_TICKS, val);
6530         }
6531 }
6532
6533 /* tp->lock is held. */
6534 static int tg3_reset_hw(struct tg3 *tp, int reset_phy)
6535 {
6536         u32 val, rdmac_mode;
6537         int i, err, limit;
6538
6539         tg3_disable_ints(tp);
6540
6541         tg3_stop_fw(tp);
6542
6543         tg3_write_sig_pre_reset(tp, RESET_KIND_INIT);
6544
6545         if (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) {
6546                 tg3_abort_hw(tp, 1);
6547         }
6548
6549         if (reset_phy)
6550                 tg3_phy_reset(tp);
6551
6552         err = tg3_chip_reset(tp);
6553         if (err)
6554                 return err;
6555
6556         tg3_write_sig_legacy(tp, RESET_KIND_INIT);
6557
6558         if (tp->pci_chip_rev_id == CHIPREV_ID_5784_A0 ||
6559             tp->pci_chip_rev_id == CHIPREV_ID_5784_A1) {
6560                 val = tr32(TG3_CPMU_CTRL);
6561                 val &= ~(CPMU_CTRL_LINK_AWARE_MODE | CPMU_CTRL_LINK_IDLE_MODE);
6562                 tw32(TG3_CPMU_CTRL, val);
6563
6564                 val = tr32(TG3_CPMU_LSPD_10MB_CLK);
6565                 val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
6566                 val |= CPMU_LSPD_10MB_MACCLK_6_25;
6567                 tw32(TG3_CPMU_LSPD_10MB_CLK, val);
6568
6569                 val = tr32(TG3_CPMU_LNK_AWARE_PWRMD);
6570                 val &= ~CPMU_LNK_AWARE_MACCLK_MASK;
6571                 val |= CPMU_LNK_AWARE_MACCLK_6_25;
6572                 tw32(TG3_CPMU_LNK_AWARE_PWRMD, val);
6573
6574                 val = tr32(TG3_CPMU_HST_ACC);
6575                 val &= ~CPMU_HST_ACC_MACCLK_MASK;
6576                 val |= CPMU_HST_ACC_MACCLK_6_25;
6577                 tw32(TG3_CPMU_HST_ACC, val);
6578         }
6579
6580         /* This works around an issue with Athlon chipsets on
6581          * B3 tigon3 silicon.  This bit has no effect on any
6582          * other revision.  But do not set this on PCI Express
6583          * chips and don't even touch the clocks if the CPMU is present.
6584          */
6585         if (!(tp->tg3_flags & TG3_FLAG_CPMU_PRESENT)) {
6586                 if (!(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
6587                         tp->pci_clock_ctrl |= CLOCK_CTRL_DELAY_PCI_GRANT;
6588                 tw32_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
6589         }
6590
6591         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
6592             (tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
6593                 val = tr32(TG3PCI_PCISTATE);
6594                 val |= PCISTATE_RETRY_SAME_DMA;
6595                 tw32(TG3PCI_PCISTATE, val);
6596         }
6597
6598         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
6599                 /* Allow reads and writes to the
6600                  * APE register and memory space.
6601                  */
6602                 val = tr32(TG3PCI_PCISTATE);
6603                 val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
6604                        PCISTATE_ALLOW_APE_SHMEM_WR;
6605                 tw32(TG3PCI_PCISTATE, val);
6606         }
6607
6608         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_BX) {
6609                 /* Enable some hw fixes.  */
6610                 val = tr32(TG3PCI_MSI_DATA);
6611                 val |= (1 << 26) | (1 << 28) | (1 << 29);
6612                 tw32(TG3PCI_MSI_DATA, val);
6613         }
6614
6615         /* Descriptor ring init may make accesses to the
6616          * NIC SRAM area to setup the TX descriptors, so we
6617          * can only do this after the hardware has been
6618          * successfully reset.
6619          */
6620         err = tg3_init_rings(tp);
6621         if (err)
6622                 return err;
6623
6624         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 &&
6625             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5761) {
6626                 /* This value is determined during the probe time DMA
6627                  * engine test, tg3_test_dma.
6628                  */
6629                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
6630         }
6631
6632         tp->grc_mode &= ~(GRC_MODE_HOST_SENDBDS |
6633                           GRC_MODE_4X_NIC_SEND_RINGS |
6634                           GRC_MODE_NO_TX_PHDR_CSUM |
6635                           GRC_MODE_NO_RX_PHDR_CSUM);
6636         tp->grc_mode |= GRC_MODE_HOST_SENDBDS;
6637
6638         /* Pseudo-header checksum is done by hardware logic and not
6639          * the offload processers, so make the chip do the pseudo-
6640          * header checksums on receive.  For transmit it is more
6641          * convenient to do the pseudo-header checksum in software
6642          * as Linux does that on transmit for us in all cases.
6643          */
6644         tp->grc_mode |= GRC_MODE_NO_TX_PHDR_CSUM;
6645
6646         tw32(GRC_MODE,
6647              tp->grc_mode |
6648              (GRC_MODE_IRQ_ON_MAC_ATTN | GRC_MODE_HOST_STACKUP));
6649
6650         /* Setup the timer prescalar register.  Clock is always 66Mhz. */
6651         val = tr32(GRC_MISC_CFG);
6652         val &= ~0xff;
6653         val |= (65 << GRC_MISC_CFG_PRESCALAR_SHIFT);
6654         tw32(GRC_MISC_CFG, val);
6655
6656         /* Initialize MBUF/DESC pool. */
6657         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
6658                 /* Do nothing.  */
6659         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705) {
6660                 tw32(BUFMGR_MB_POOL_ADDR, NIC_SRAM_MBUF_POOL_BASE);
6661                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
6662                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE64);
6663                 else
6664                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE96);
6665                 tw32(BUFMGR_DMA_DESC_POOL_ADDR, NIC_SRAM_DMA_DESC_POOL_BASE);
6666                 tw32(BUFMGR_DMA_DESC_POOL_SIZE, NIC_SRAM_DMA_DESC_POOL_SIZE);
6667         }
6668         else if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
6669                 int fw_len;
6670
6671                 fw_len = (TG3_TSO5_FW_TEXT_LEN +
6672                           TG3_TSO5_FW_RODATA_LEN +
6673                           TG3_TSO5_FW_DATA_LEN +
6674                           TG3_TSO5_FW_SBSS_LEN +
6675                           TG3_TSO5_FW_BSS_LEN);
6676                 fw_len = (fw_len + (0x80 - 1)) & ~(0x80 - 1);
6677                 tw32(BUFMGR_MB_POOL_ADDR,
6678                      NIC_SRAM_MBUF_POOL_BASE5705 + fw_len);
6679                 tw32(BUFMGR_MB_POOL_SIZE,
6680                      NIC_SRAM_MBUF_POOL_SIZE5705 - fw_len - 0xa00);
6681         }
6682
6683         if (tp->dev->mtu <= ETH_DATA_LEN) {
6684                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
6685                      tp->bufmgr_config.mbuf_read_dma_low_water);
6686                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
6687                      tp->bufmgr_config.mbuf_mac_rx_low_water);
6688                 tw32(BUFMGR_MB_HIGH_WATER,
6689                      tp->bufmgr_config.mbuf_high_water);
6690         } else {
6691                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
6692                      tp->bufmgr_config.mbuf_read_dma_low_water_jumbo);
6693                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
6694                      tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo);
6695                 tw32(BUFMGR_MB_HIGH_WATER,
6696                      tp->bufmgr_config.mbuf_high_water_jumbo);
6697         }
6698         tw32(BUFMGR_DMA_LOW_WATER,
6699              tp->bufmgr_config.dma_low_water);
6700         tw32(BUFMGR_DMA_HIGH_WATER,
6701              tp->bufmgr_config.dma_high_water);
6702
6703         tw32(BUFMGR_MODE, BUFMGR_MODE_ENABLE | BUFMGR_MODE_ATTN_ENABLE);
6704         for (i = 0; i < 2000; i++) {
6705                 if (tr32(BUFMGR_MODE) & BUFMGR_MODE_ENABLE)
6706                         break;
6707                 udelay(10);
6708         }
6709         if (i >= 2000) {
6710                 printk(KERN_ERR PFX "tg3_reset_hw cannot enable BUFMGR for %s.\n",
6711                        tp->dev->name);
6712                 return -ENODEV;
6713         }
6714
6715         /* Setup replenish threshold. */
6716         val = tp->rx_pending / 8;
6717         if (val == 0)
6718                 val = 1;
6719         else if (val > tp->rx_std_max_post)
6720                 val = tp->rx_std_max_post;
6721         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
6722                 if (tp->pci_chip_rev_id == CHIPREV_ID_5906_A1)
6723                         tw32(ISO_PKT_TX, (tr32(ISO_PKT_TX) & ~0x3) | 0x2);
6724
6725                 if (val > (TG3_RX_INTERNAL_RING_SZ_5906 / 2))
6726                         val = TG3_RX_INTERNAL_RING_SZ_5906 / 2;
6727         }
6728
6729         tw32(RCVBDI_STD_THRESH, val);
6730
6731         /* Initialize TG3_BDINFO's at:
6732          *  RCVDBDI_STD_BD:     standard eth size rx ring
6733          *  RCVDBDI_JUMBO_BD:   jumbo frame rx ring
6734          *  RCVDBDI_MINI_BD:    small frame rx ring (??? does not work)
6735          *
6736          * like so:
6737          *  TG3_BDINFO_HOST_ADDR:       high/low parts of DMA address of ring
6738          *  TG3_BDINFO_MAXLEN_FLAGS:    (rx max buffer size << 16) |
6739          *                              ring attribute flags
6740          *  TG3_BDINFO_NIC_ADDR:        location of descriptors in nic SRAM
6741          *
6742          * Standard receive ring @ NIC_SRAM_RX_BUFFER_DESC, 512 entries.
6743          * Jumbo receive ring @ NIC_SRAM_RX_JUMBO_BUFFER_DESC, 256 entries.
6744          *
6745          * The size of each ring is fixed in the firmware, but the location is
6746          * configurable.
6747          */
6748         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
6749              ((u64) tp->rx_std_mapping >> 32));
6750         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
6751              ((u64) tp->rx_std_mapping & 0xffffffff));
6752         tw32(RCVDBDI_STD_BD + TG3_BDINFO_NIC_ADDR,
6753              NIC_SRAM_RX_BUFFER_DESC);
6754
6755         /* Don't even try to program the JUMBO/MINI buffer descriptor
6756          * configs on 5705.
6757          */
6758         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
6759                 tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS,
6760                      RX_STD_MAX_SIZE_5705 << BDINFO_FLAGS_MAXLEN_SHIFT);
6761         } else {
6762                 tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS,
6763                      RX_STD_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT);
6764
6765                 tw32(RCVDBDI_MINI_BD + TG3_BDINFO_MAXLEN_FLAGS,
6766                      BDINFO_FLAGS_DISABLED);
6767
6768                 /* Setup replenish threshold. */
6769                 tw32(RCVBDI_JUMBO_THRESH, tp->rx_jumbo_pending / 8);
6770
6771                 if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
6772                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
6773                              ((u64) tp->rx_jumbo_mapping >> 32));
6774                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
6775                              ((u64) tp->rx_jumbo_mapping & 0xffffffff));
6776                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
6777                              RX_JUMBO_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT);
6778                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_NIC_ADDR,
6779                              NIC_SRAM_RX_JUMBO_BUFFER_DESC);
6780                 } else {
6781                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
6782                              BDINFO_FLAGS_DISABLED);
6783                 }
6784
6785         }
6786
6787         /* There is only one send ring on 5705/5750, no need to explicitly
6788          * disable the others.
6789          */
6790         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6791                 /* Clear out send RCB ring in SRAM. */
6792                 for (i = NIC_SRAM_SEND_RCB; i < NIC_SRAM_RCV_RET_RCB; i += TG3_BDINFO_SIZE)
6793                         tg3_write_mem(tp, i + TG3_BDINFO_MAXLEN_FLAGS,
6794                                       BDINFO_FLAGS_DISABLED);
6795         }
6796
6797         tp->tx_prod = 0;
6798         tp->tx_cons = 0;
6799         tw32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW, 0);
6800         tw32_tx_mbox(MAILBOX_SNDNIC_PROD_IDX_0 + TG3_64BIT_REG_LOW, 0);
6801
6802         tg3_set_bdinfo(tp, NIC_SRAM_SEND_RCB,
6803                        tp->tx_desc_mapping,
6804                        (TG3_TX_RING_SIZE <<
6805                         BDINFO_FLAGS_MAXLEN_SHIFT),
6806                        NIC_SRAM_TX_BUFFER_DESC);
6807
6808         /* There is only one receive return ring on 5705/5750, no need
6809          * to explicitly disable the others.
6810          */
6811         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6812                 for (i = NIC_SRAM_RCV_RET_RCB; i < NIC_SRAM_STATS_BLK;
6813                      i += TG3_BDINFO_SIZE) {
6814                         tg3_write_mem(tp, i + TG3_BDINFO_MAXLEN_FLAGS,
6815                                       BDINFO_FLAGS_DISABLED);
6816                 }
6817         }
6818
6819         tp->rx_rcb_ptr = 0;
6820         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, 0);
6821
6822         tg3_set_bdinfo(tp, NIC_SRAM_RCV_RET_RCB,
6823                        tp->rx_rcb_mapping,
6824                        (TG3_RX_RCB_RING_SIZE(tp) <<
6825                         BDINFO_FLAGS_MAXLEN_SHIFT),
6826                        0);
6827
6828         tp->rx_std_ptr = tp->rx_pending;
6829         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
6830                      tp->rx_std_ptr);
6831
6832         tp->rx_jumbo_ptr = (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) ?
6833                                                 tp->rx_jumbo_pending : 0;
6834         tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
6835                      tp->rx_jumbo_ptr);
6836
6837         /* Initialize MAC address and backoff seed. */
6838         __tg3_set_mac_addr(tp, 0);
6839
6840         /* MTU + ethernet header + FCS + optional VLAN tag */
6841         tw32(MAC_RX_MTU_SIZE, tp->dev->mtu + ETH_HLEN + 8);
6842
6843         /* The slot time is changed by tg3_setup_phy if we
6844          * run at gigabit with half duplex.
6845          */
6846         tw32(MAC_TX_LENGTHS,
6847              (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
6848              (6 << TX_LENGTHS_IPG_SHIFT) |
6849              (32 << TX_LENGTHS_SLOT_TIME_SHIFT));
6850
6851         /* Receive rules. */
6852         tw32(MAC_RCV_RULE_CFG, RCV_RULE_CFG_DEFAULT_CLASS);
6853         tw32(RCVLPC_CONFIG, 0x0181);
6854
6855         /* Calculate RDMAC_MODE setting early, we need it to determine
6856          * the RCVLPC_STATE_ENABLE mask.
6857          */
6858         rdmac_mode = (RDMAC_MODE_ENABLE | RDMAC_MODE_TGTABORT_ENAB |
6859                       RDMAC_MODE_MSTABORT_ENAB | RDMAC_MODE_PARITYERR_ENAB |
6860                       RDMAC_MODE_ADDROFLOW_ENAB | RDMAC_MODE_FIFOOFLOW_ENAB |
6861                       RDMAC_MODE_FIFOURUN_ENAB | RDMAC_MODE_FIFOOREAD_ENAB |
6862                       RDMAC_MODE_LNGREAD_ENAB);
6863
6864         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784)
6865                 rdmac_mode |= RDMAC_MODE_BD_SBD_CRPT_ENAB |
6866                               RDMAC_MODE_MBUF_RBD_CRPT_ENAB |
6867                               RDMAC_MODE_MBUF_SBD_CRPT_ENAB;
6868
6869         /* If statement applies to 5705 and 5750 PCI devices only */
6870         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
6871              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
6872             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)) {
6873                 if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE &&
6874                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
6875                         rdmac_mode |= RDMAC_MODE_FIFO_SIZE_128;
6876                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
6877                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788)) {
6878                         rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
6879                 }
6880         }
6881
6882         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)
6883                 rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
6884
6885         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6886                 rdmac_mode |= (1 << 27);
6887
6888         /* Receive/send statistics. */
6889         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
6890                 val = tr32(RCVLPC_STATS_ENABLE);
6891                 val &= ~RCVLPC_STATSENAB_DACK_FIX;
6892                 tw32(RCVLPC_STATS_ENABLE, val);
6893         } else if ((rdmac_mode & RDMAC_MODE_FIFO_SIZE_128) &&
6894                    (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
6895                 val = tr32(RCVLPC_STATS_ENABLE);
6896                 val &= ~RCVLPC_STATSENAB_LNGBRST_RFIX;
6897                 tw32(RCVLPC_STATS_ENABLE, val);
6898         } else {
6899                 tw32(RCVLPC_STATS_ENABLE, 0xffffff);
6900         }
6901         tw32(RCVLPC_STATSCTRL, RCVLPC_STATSCTRL_ENABLE);
6902         tw32(SNDDATAI_STATSENAB, 0xffffff);
6903         tw32(SNDDATAI_STATSCTRL,
6904              (SNDDATAI_SCTRL_ENABLE |
6905               SNDDATAI_SCTRL_FASTUPD));
6906
6907         /* Setup host coalescing engine. */
6908         tw32(HOSTCC_MODE, 0);
6909         for (i = 0; i < 2000; i++) {
6910                 if (!(tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE))
6911                         break;
6912                 udelay(10);
6913         }
6914
6915         __tg3_set_coalesce(tp, &tp->coal);
6916
6917         /* set status block DMA address */
6918         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
6919              ((u64) tp->status_mapping >> 32));
6920         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
6921              ((u64) tp->status_mapping & 0xffffffff));
6922
6923         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6924                 /* Status/statistics block address.  See tg3_timer,
6925                  * the tg3_periodic_fetch_stats call there, and
6926                  * tg3_get_stats to see how this works for 5705/5750 chips.
6927                  */
6928                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
6929                      ((u64) tp->stats_mapping >> 32));
6930                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
6931                      ((u64) tp->stats_mapping & 0xffffffff));
6932                 tw32(HOSTCC_STATS_BLK_NIC_ADDR, NIC_SRAM_STATS_BLK);
6933                 tw32(HOSTCC_STATUS_BLK_NIC_ADDR, NIC_SRAM_STATUS_BLK);
6934         }
6935
6936         tw32(HOSTCC_MODE, HOSTCC_MODE_ENABLE | tp->coalesce_mode);
6937
6938         tw32(RCVCC_MODE, RCVCC_MODE_ENABLE | RCVCC_MODE_ATTN_ENABLE);
6939         tw32(RCVLPC_MODE, RCVLPC_MODE_ENABLE);
6940         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6941                 tw32(RCVLSC_MODE, RCVLSC_MODE_ENABLE | RCVLSC_MODE_ATTN_ENABLE);
6942
6943         /* Clear statistics/status block in chip, and status block in ram. */
6944         for (i = NIC_SRAM_STATS_BLK;
6945              i < NIC_SRAM_STATUS_BLK + TG3_HW_STATUS_SIZE;
6946              i += sizeof(u32)) {
6947                 tg3_write_mem(tp, i, 0);
6948                 udelay(40);
6949         }
6950         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
6951
6952         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
6953                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
6954                 /* reset to prevent losing 1st rx packet intermittently */
6955                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
6956                 udelay(10);
6957         }
6958
6959         tp->mac_mode = MAC_MODE_TXSTAT_ENABLE | MAC_MODE_RXSTAT_ENABLE |
6960                 MAC_MODE_TDE_ENABLE | MAC_MODE_RDE_ENABLE | MAC_MODE_FHDE_ENABLE;
6961         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
6962             !(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
6963             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700)
6964                 tp->mac_mode |= MAC_MODE_LINK_POLARITY;
6965         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_RXSTAT_CLEAR | MAC_MODE_TXSTAT_CLEAR);
6966         udelay(40);
6967
6968         /* tp->grc_local_ctrl is partially set up during tg3_get_invariants().
6969          * If TG3_FLG2_IS_NIC is zero, we should read the
6970          * register to preserve the GPIO settings for LOMs. The GPIOs,
6971          * whether used as inputs or outputs, are set by boot code after
6972          * reset.
6973          */
6974         if (!(tp->tg3_flags2 & TG3_FLG2_IS_NIC)) {
6975                 u32 gpio_mask;
6976
6977                 gpio_mask = GRC_LCLCTRL_GPIO_OE0 | GRC_LCLCTRL_GPIO_OE1 |
6978                             GRC_LCLCTRL_GPIO_OE2 | GRC_LCLCTRL_GPIO_OUTPUT0 |
6979                             GRC_LCLCTRL_GPIO_OUTPUT1 | GRC_LCLCTRL_GPIO_OUTPUT2;
6980
6981                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
6982                         gpio_mask |= GRC_LCLCTRL_GPIO_OE3 |
6983                                      GRC_LCLCTRL_GPIO_OUTPUT3;
6984
6985                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
6986                         gpio_mask |= GRC_LCLCTRL_GPIO_UART_SEL;
6987
6988                 tp->grc_local_ctrl &= ~gpio_mask;
6989                 tp->grc_local_ctrl |= tr32(GRC_LOCAL_CTRL) & gpio_mask;
6990
6991                 /* GPIO1 must be driven high for eeprom write protect */
6992                 if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT)
6993                         tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
6994                                                GRC_LCLCTRL_GPIO_OUTPUT1);
6995         }
6996         tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
6997         udelay(100);
6998
6999         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0);
7000         tp->last_tag = 0;
7001
7002         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
7003                 tw32_f(DMAC_MODE, DMAC_MODE_ENABLE);
7004                 udelay(40);
7005         }
7006
7007         val = (WDMAC_MODE_ENABLE | WDMAC_MODE_TGTABORT_ENAB |
7008                WDMAC_MODE_MSTABORT_ENAB | WDMAC_MODE_PARITYERR_ENAB |
7009                WDMAC_MODE_ADDROFLOW_ENAB | WDMAC_MODE_FIFOOFLOW_ENAB |
7010                WDMAC_MODE_FIFOURUN_ENAB | WDMAC_MODE_FIFOOREAD_ENAB |
7011                WDMAC_MODE_LNGREAD_ENAB);
7012
7013         /* If statement applies to 5705 and 5750 PCI devices only */
7014         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
7015              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
7016             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) {
7017                 if ((tp->tg3_flags & TG3_FLG2_TSO_CAPABLE) &&
7018                     (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 ||
7019                      tp->pci_chip_rev_id == CHIPREV_ID_5705_A2)) {
7020                         /* nothing */
7021                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
7022                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
7023                            !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
7024                         val |= WDMAC_MODE_RX_ACCEL;
7025                 }
7026         }
7027
7028         /* Enable host coalescing bug fix */
7029         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755) ||
7030             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787) ||
7031             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784) ||
7032             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761))
7033                 val |= (1 << 29);
7034
7035         tw32_f(WDMAC_MODE, val);
7036         udelay(40);
7037
7038         if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
7039                 u16 pcix_cmd;
7040
7041                 pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
7042                                      &pcix_cmd);
7043                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703) {
7044                         pcix_cmd &= ~PCI_X_CMD_MAX_READ;
7045                         pcix_cmd |= PCI_X_CMD_READ_2K;
7046                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
7047                         pcix_cmd &= ~(PCI_X_CMD_MAX_SPLIT | PCI_X_CMD_MAX_READ);
7048                         pcix_cmd |= PCI_X_CMD_READ_2K;
7049                 }
7050                 pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
7051                                       pcix_cmd);
7052         }
7053
7054         tw32_f(RDMAC_MODE, rdmac_mode);
7055         udelay(40);
7056
7057         tw32(RCVDCC_MODE, RCVDCC_MODE_ENABLE | RCVDCC_MODE_ATTN_ENABLE);
7058         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
7059                 tw32(MBFREE_MODE, MBFREE_MODE_ENABLE);
7060
7061         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
7062                 tw32(SNDDATAC_MODE,
7063                      SNDDATAC_MODE_ENABLE | SNDDATAC_MODE_CDELAY);
7064         else
7065                 tw32(SNDDATAC_MODE, SNDDATAC_MODE_ENABLE);
7066
7067         tw32(SNDBDC_MODE, SNDBDC_MODE_ENABLE | SNDBDC_MODE_ATTN_ENABLE);
7068         tw32(RCVBDI_MODE, RCVBDI_MODE_ENABLE | RCVBDI_MODE_RCB_ATTN_ENAB);
7069         tw32(RCVDBDI_MODE, RCVDBDI_MODE_ENABLE | RCVDBDI_MODE_INV_RING_SZ);
7070         tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE);
7071         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
7072                 tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE | 0x8);
7073         tw32(SNDBDI_MODE, SNDBDI_MODE_ENABLE | SNDBDI_MODE_ATTN_ENABLE);
7074         tw32(SNDBDS_MODE, SNDBDS_MODE_ENABLE | SNDBDS_MODE_ATTN_ENABLE);
7075
7076         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0) {
7077                 err = tg3_load_5701_a0_firmware_fix(tp);
7078                 if (err)
7079                         return err;
7080         }
7081
7082         if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
7083                 err = tg3_load_tso_firmware(tp);
7084                 if (err)
7085                         return err;
7086         }
7087
7088         tp->tx_mode = TX_MODE_ENABLE;
7089         tw32_f(MAC_TX_MODE, tp->tx_mode);
7090         udelay(100);
7091
7092         tp->rx_mode = RX_MODE_ENABLE;
7093         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
7094             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
7095                 tp->rx_mode |= RX_MODE_IPV6_CSUM_ENABLE;
7096
7097         tw32_f(MAC_RX_MODE, tp->rx_mode);
7098         udelay(10);
7099
7100         if (tp->link_config.phy_is_low_power) {
7101                 tp->link_config.phy_is_low_power = 0;
7102                 tp->link_config.speed = tp->link_config.orig_speed;
7103                 tp->link_config.duplex = tp->link_config.orig_duplex;
7104                 tp->link_config.autoneg = tp->link_config.orig_autoneg;
7105         }
7106
7107         tp->mi_mode &= ~MAC_MI_MODE_AUTO_POLL;
7108         tw32_f(MAC_MI_MODE, tp->mi_mode);
7109         udelay(80);
7110
7111         tw32(MAC_LED_CTRL, tp->led_ctrl);
7112
7113         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
7114         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
7115                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
7116                 udelay(10);
7117         }
7118         tw32_f(MAC_RX_MODE, tp->rx_mode);
7119         udelay(10);
7120
7121         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
7122                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) &&
7123                         !(tp->tg3_flags2 & TG3_FLG2_SERDES_PREEMPHASIS)) {
7124                         /* Set drive transmission level to 1.2V  */
7125                         /* only if the signal pre-emphasis bit is not set  */
7126                         val = tr32(MAC_SERDES_CFG);
7127                         val &= 0xfffff000;
7128                         val |= 0x880;
7129                         tw32(MAC_SERDES_CFG, val);
7130                 }
7131                 if (tp->pci_chip_rev_id == CHIPREV_ID_5703_A1)
7132                         tw32(MAC_SERDES_CFG, 0x616000);
7133         }
7134
7135         /* Prevent chip from dropping frames when flow control
7136          * is enabled.
7137          */
7138         tw32_f(MAC_LOW_WMARK_MAX_RX_FRAME, 2);
7139
7140         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 &&
7141             (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
7142                 /* Use hardware link auto-negotiation */
7143                 tp->tg3_flags2 |= TG3_FLG2_HW_AUTONEG;
7144         }
7145
7146         if ((tp->tg3_flags2 & TG3_FLG2_MII_SERDES) &&
7147             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
7148                 u32 tmp;
7149
7150                 tmp = tr32(SERDES_RX_CTRL);
7151                 tw32(SERDES_RX_CTRL, tmp | SERDES_RX_SIG_DETECT);
7152                 tp->grc_local_ctrl &= ~GRC_LCLCTRL_USE_EXT_SIG_DETECT;
7153                 tp->grc_local_ctrl |= GRC_LCLCTRL_USE_SIG_DETECT;
7154                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
7155         }
7156
7157         err = tg3_setup_phy(tp, 0);
7158         if (err)
7159                 return err;
7160
7161         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
7162             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906) {
7163                 u32 tmp;
7164
7165                 /* Clear CRC stats. */
7166                 if (!tg3_readphy(tp, MII_TG3_TEST1, &tmp)) {
7167                         tg3_writephy(tp, MII_TG3_TEST1,
7168                                      tmp | MII_TG3_TEST1_CRC_EN);
7169                         tg3_readphy(tp, 0x14, &tmp);
7170                 }
7171         }
7172
7173         __tg3_set_rx_mode(tp->dev);
7174
7175         /* Initialize receive rules. */
7176         tw32(MAC_RCV_RULE_0,  0xc2000000 & RCV_RULE_DISABLE_MASK);
7177         tw32(MAC_RCV_VALUE_0, 0xffffffff & RCV_RULE_DISABLE_MASK);
7178         tw32(MAC_RCV_RULE_1,  0x86000004 & RCV_RULE_DISABLE_MASK);
7179         tw32(MAC_RCV_VALUE_1, 0xffffffff & RCV_RULE_DISABLE_MASK);
7180
7181         if ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
7182             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
7183                 limit = 8;
7184         else
7185                 limit = 16;
7186         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF)
7187                 limit -= 4;
7188         switch (limit) {
7189         case 16:
7190                 tw32(MAC_RCV_RULE_15,  0); tw32(MAC_RCV_VALUE_15,  0);
7191         case 15:
7192                 tw32(MAC_RCV_RULE_14,  0); tw32(MAC_RCV_VALUE_14,  0);
7193         case 14:
7194                 tw32(MAC_RCV_RULE_13,  0); tw32(MAC_RCV_VALUE_13,  0);
7195         case 13:
7196                 tw32(MAC_RCV_RULE_12,  0); tw32(MAC_RCV_VALUE_12,  0);
7197         case 12:
7198                 tw32(MAC_RCV_RULE_11,  0); tw32(MAC_RCV_VALUE_11,  0);
7199         case 11:
7200                 tw32(MAC_RCV_RULE_10,  0); tw32(MAC_RCV_VALUE_10,  0);
7201         case 10:
7202                 tw32(MAC_RCV_RULE_9,  0); tw32(MAC_RCV_VALUE_9,  0);
7203         case 9:
7204                 tw32(MAC_RCV_RULE_8,  0); tw32(MAC_RCV_VALUE_8,  0);
7205         case 8:
7206                 tw32(MAC_RCV_RULE_7,  0); tw32(MAC_RCV_VALUE_7,  0);
7207         case 7:
7208                 tw32(MAC_RCV_RULE_6,  0); tw32(MAC_RCV_VALUE_6,  0);
7209         case 6:
7210                 tw32(MAC_RCV_RULE_5,  0); tw32(MAC_RCV_VALUE_5,  0);
7211         case 5:
7212                 tw32(MAC_RCV_RULE_4,  0); tw32(MAC_RCV_VALUE_4,  0);
7213         case 4:
7214                 /* tw32(MAC_RCV_RULE_3,  0); tw32(MAC_RCV_VALUE_3,  0); */
7215         case 3:
7216                 /* tw32(MAC_RCV_RULE_2,  0); tw32(MAC_RCV_VALUE_2,  0); */
7217         case 2:
7218         case 1:
7219
7220         default:
7221                 break;
7222         };
7223
7224         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)
7225                 /* Write our heartbeat update interval to APE. */
7226                 tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_INT_MS,
7227                                 APE_HOST_HEARTBEAT_INT_DISABLE);
7228
7229         tg3_write_sig_post_reset(tp, RESET_KIND_INIT);
7230
7231         return 0;
7232 }
7233
7234 /* Called at device open time to get the chip ready for
7235  * packet processing.  Invoked with tp->lock held.
7236  */
7237 static int tg3_init_hw(struct tg3 *tp, int reset_phy)
7238 {
7239         int err;
7240
7241         /* Force the chip into D0. */
7242         err = tg3_set_power_state(tp, PCI_D0);
7243         if (err)
7244                 goto out;
7245
7246         tg3_switch_clocks(tp);
7247
7248         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
7249
7250         err = tg3_reset_hw(tp, reset_phy);
7251
7252 out:
7253         return err;
7254 }
7255
7256 #define TG3_STAT_ADD32(PSTAT, REG) \
7257 do {    u32 __val = tr32(REG); \
7258         (PSTAT)->low += __val; \
7259         if ((PSTAT)->low < __val) \
7260                 (PSTAT)->high += 1; \
7261 } while (0)
7262
7263 static void tg3_periodic_fetch_stats(struct tg3 *tp)
7264 {
7265         struct tg3_hw_stats *sp = tp->hw_stats;
7266
7267         if (!netif_carrier_ok(tp->dev))
7268                 return;
7269
7270         TG3_STAT_ADD32(&sp->tx_octets, MAC_TX_STATS_OCTETS);
7271         TG3_STAT_ADD32(&sp->tx_collisions, MAC_TX_STATS_COLLISIONS);
7272         TG3_STAT_ADD32(&sp->tx_xon_sent, MAC_TX_STATS_XON_SENT);
7273         TG3_STAT_ADD32(&sp->tx_xoff_sent, MAC_TX_STATS_XOFF_SENT);
7274         TG3_STAT_ADD32(&sp->tx_mac_errors, MAC_TX_STATS_MAC_ERRORS);
7275         TG3_STAT_ADD32(&sp->tx_single_collisions, MAC_TX_STATS_SINGLE_COLLISIONS);
7276         TG3_STAT_ADD32(&sp->tx_mult_collisions, MAC_TX_STATS_MULT_COLLISIONS);
7277         TG3_STAT_ADD32(&sp->tx_deferred, MAC_TX_STATS_DEFERRED);
7278         TG3_STAT_ADD32(&sp->tx_excessive_collisions, MAC_TX_STATS_EXCESSIVE_COL);
7279         TG3_STAT_ADD32(&sp->tx_late_collisions, MAC_TX_STATS_LATE_COL);
7280         TG3_STAT_ADD32(&sp->tx_ucast_packets, MAC_TX_STATS_UCAST);
7281         TG3_STAT_ADD32(&sp->tx_mcast_packets, MAC_TX_STATS_MCAST);
7282         TG3_STAT_ADD32(&sp->tx_bcast_packets, MAC_TX_STATS_BCAST);
7283
7284         TG3_STAT_ADD32(&sp->rx_octets, MAC_RX_STATS_OCTETS);
7285         TG3_STAT_ADD32(&sp->rx_fragments, MAC_RX_STATS_FRAGMENTS);
7286         TG3_STAT_ADD32(&sp->rx_ucast_packets, MAC_RX_STATS_UCAST);
7287         TG3_STAT_ADD32(&sp->rx_mcast_packets, MAC_RX_STATS_MCAST);
7288         TG3_STAT_ADD32(&sp->rx_bcast_packets, MAC_RX_STATS_BCAST);
7289         TG3_STAT_ADD32(&sp->rx_fcs_errors, MAC_RX_STATS_FCS_ERRORS);
7290         TG3_STAT_ADD32(&sp->rx_align_errors, MAC_RX_STATS_ALIGN_ERRORS);
7291         TG3_STAT_ADD32(&sp->rx_xon_pause_rcvd, MAC_RX_STATS_XON_PAUSE_RECVD);
7292         TG3_STAT_ADD32(&sp->rx_xoff_pause_rcvd, MAC_RX_STATS_XOFF_PAUSE_RECVD);
7293         TG3_STAT_ADD32(&sp->rx_mac_ctrl_rcvd, MAC_RX_STATS_MAC_CTRL_RECVD);
7294         TG3_STAT_ADD32(&sp->rx_xoff_entered, MAC_RX_STATS_XOFF_ENTERED);
7295         TG3_STAT_ADD32(&sp->rx_frame_too_long_errors, MAC_RX_STATS_FRAME_TOO_LONG);
7296         TG3_STAT_ADD32(&sp->rx_jabbers, MAC_RX_STATS_JABBERS);
7297         TG3_STAT_ADD32(&sp->rx_undersize_packets, MAC_RX_STATS_UNDERSIZE);
7298
7299         TG3_STAT_ADD32(&sp->rxbds_empty, RCVLPC_NO_RCV_BD_CNT);
7300         TG3_STAT_ADD32(&sp->rx_discards, RCVLPC_IN_DISCARDS_CNT);
7301         TG3_STAT_ADD32(&sp->rx_errors, RCVLPC_IN_ERRORS_CNT);
7302 }
7303
7304 static void tg3_timer(unsigned long __opaque)
7305 {
7306         struct tg3 *tp = (struct tg3 *) __opaque;
7307
7308         if (tp->irq_sync)
7309                 goto restart_timer;
7310
7311         spin_lock(&tp->lock);
7312
7313         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
7314                 /* All of this garbage is because when using non-tagged
7315                  * IRQ status the mailbox/status_block protocol the chip
7316                  * uses with the cpu is race prone.
7317                  */
7318                 if (tp->hw_status->status & SD_STATUS_UPDATED) {
7319                         tw32(GRC_LOCAL_CTRL,
7320                              tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
7321                 } else {
7322                         tw32(HOSTCC_MODE, tp->coalesce_mode |
7323                              (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
7324                 }
7325
7326                 if (!(tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
7327                         tp->tg3_flags2 |= TG3_FLG2_RESTART_TIMER;
7328                         spin_unlock(&tp->lock);
7329                         schedule_work(&tp->reset_task);
7330                         return;
7331                 }
7332         }
7333
7334         /* This part only runs once per second. */
7335         if (!--tp->timer_counter) {
7336                 if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
7337                         tg3_periodic_fetch_stats(tp);
7338
7339                 if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
7340                         u32 mac_stat;
7341                         int phy_event;
7342
7343                         mac_stat = tr32(MAC_STATUS);
7344
7345                         phy_event = 0;
7346                         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) {
7347                                 if (mac_stat & MAC_STATUS_MI_INTERRUPT)
7348                                         phy_event = 1;
7349                         } else if (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)
7350                                 phy_event = 1;
7351
7352                         if (phy_event)
7353                                 tg3_setup_phy(tp, 0);
7354                 } else if (tp->tg3_flags & TG3_FLAG_POLL_SERDES) {
7355                         u32 mac_stat = tr32(MAC_STATUS);
7356                         int need_setup = 0;
7357
7358                         if (netif_carrier_ok(tp->dev) &&
7359                             (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)) {
7360                                 need_setup = 1;
7361                         }
7362                         if (! netif_carrier_ok(tp->dev) &&
7363                             (mac_stat & (MAC_STATUS_PCS_SYNCED |
7364                                          MAC_STATUS_SIGNAL_DET))) {
7365                                 need_setup = 1;
7366                         }
7367                         if (need_setup) {
7368                                 if (!tp->serdes_counter) {
7369                                         tw32_f(MAC_MODE,
7370                                              (tp->mac_mode &
7371                                               ~MAC_MODE_PORT_MODE_MASK));
7372                                         udelay(40);
7373                                         tw32_f(MAC_MODE, tp->mac_mode);
7374                                         udelay(40);
7375                                 }
7376                                 tg3_setup_phy(tp, 0);
7377                         }
7378                 } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
7379                         tg3_serdes_parallel_detect(tp);
7380
7381                 tp->timer_counter = tp->timer_multiplier;
7382         }
7383
7384         /* Heartbeat is only sent once every 2 seconds.
7385          *
7386          * The heartbeat is to tell the ASF firmware that the host
7387          * driver is still alive.  In the event that the OS crashes,
7388          * ASF needs to reset the hardware to free up the FIFO space
7389          * that may be filled with rx packets destined for the host.
7390          * If the FIFO is full, ASF will no longer function properly.
7391          *
7392          * Unintended resets have been reported on real time kernels
7393          * where the timer doesn't run on time.  Netpoll will also have
7394          * same problem.
7395          *
7396          * The new FWCMD_NICDRV_ALIVE3 command tells the ASF firmware
7397          * to check the ring condition when the heartbeat is expiring
7398          * before doing the reset.  This will prevent most unintended
7399          * resets.
7400          */
7401         if (!--tp->asf_counter) {
7402                 if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
7403                         u32 val;
7404
7405                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX,
7406                                       FWCMD_NICDRV_ALIVE3);
7407                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 4);
7408                         /* 5 seconds timeout */
7409                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX, 5);
7410                         val = tr32(GRC_RX_CPU_EVENT);
7411                         val |= (1 << 14);
7412                         tw32(GRC_RX_CPU_EVENT, val);
7413                 }
7414                 tp->asf_counter = tp->asf_multiplier;
7415         }
7416
7417         spin_unlock(&tp->lock);
7418
7419 restart_timer:
7420         tp->timer.expires = jiffies + tp->timer_offset;
7421         add_timer(&tp->timer);
7422 }
7423
7424 static int tg3_request_irq(struct tg3 *tp)
7425 {
7426         irq_handler_t fn;
7427         unsigned long flags;
7428         struct net_device *dev = tp->dev;
7429
7430         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7431                 fn = tg3_msi;
7432                 if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
7433                         fn = tg3_msi_1shot;
7434                 flags = IRQF_SAMPLE_RANDOM;
7435         } else {
7436                 fn = tg3_interrupt;
7437                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
7438                         fn = tg3_interrupt_tagged;
7439                 flags = IRQF_SHARED | IRQF_SAMPLE_RANDOM;
7440         }
7441         return (request_irq(tp->pdev->irq, fn, flags, dev->name, dev));
7442 }
7443
7444 static int tg3_test_interrupt(struct tg3 *tp)
7445 {
7446         struct net_device *dev = tp->dev;
7447         int err, i, intr_ok = 0;
7448
7449         if (!netif_running(dev))
7450                 return -ENODEV;
7451
7452         tg3_disable_ints(tp);
7453
7454         free_irq(tp->pdev->irq, dev);
7455
7456         err = request_irq(tp->pdev->irq, tg3_test_isr,
7457                           IRQF_SHARED | IRQF_SAMPLE_RANDOM, dev->name, dev);
7458         if (err)
7459                 return err;
7460
7461         tp->hw_status->status &= ~SD_STATUS_UPDATED;
7462         tg3_enable_ints(tp);
7463
7464         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
7465                HOSTCC_MODE_NOW);
7466
7467         for (i = 0; i < 5; i++) {
7468                 u32 int_mbox, misc_host_ctrl;
7469
7470                 int_mbox = tr32_mailbox(MAILBOX_INTERRUPT_0 +
7471                                         TG3_64BIT_REG_LOW);
7472                 misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
7473
7474                 if ((int_mbox != 0) ||
7475                     (misc_host_ctrl & MISC_HOST_CTRL_MASK_PCI_INT)) {
7476                         intr_ok = 1;
7477                         break;
7478                 }
7479
7480                 msleep(10);
7481         }
7482
7483         tg3_disable_ints(tp);
7484
7485         free_irq(tp->pdev->irq, dev);
7486
7487         err = tg3_request_irq(tp);
7488
7489         if (err)
7490                 return err;
7491
7492         if (intr_ok)
7493                 return 0;
7494
7495         return -EIO;
7496 }
7497
7498 /* Returns 0 if MSI test succeeds or MSI test fails and INTx mode is
7499  * successfully restored
7500  */
7501 static int tg3_test_msi(struct tg3 *tp)
7502 {
7503         struct net_device *dev = tp->dev;
7504         int err;
7505         u16 pci_cmd;
7506
7507         if (!(tp->tg3_flags2 & TG3_FLG2_USING_MSI))
7508                 return 0;
7509
7510         /* Turn off SERR reporting in case MSI terminates with Master
7511          * Abort.
7512          */
7513         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
7514         pci_write_config_word(tp->pdev, PCI_COMMAND,
7515                               pci_cmd & ~PCI_COMMAND_SERR);
7516
7517         err = tg3_test_interrupt(tp);
7518
7519         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
7520
7521         if (!err)
7522                 return 0;
7523
7524         /* other failures */
7525         if (err != -EIO)
7526                 return err;
7527
7528         /* MSI test failed, go back to INTx mode */
7529         printk(KERN_WARNING PFX "%s: No interrupt was generated using MSI, "
7530                "switching to INTx mode. Please report this failure to "
7531                "the PCI maintainer and include system chipset information.\n",
7532                        tp->dev->name);
7533
7534         free_irq(tp->pdev->irq, dev);
7535         pci_disable_msi(tp->pdev);
7536
7537         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7538
7539         err = tg3_request_irq(tp);
7540         if (err)
7541                 return err;
7542
7543         /* Need to reset the chip because the MSI cycle may have terminated
7544          * with Master Abort.
7545          */
7546         tg3_full_lock(tp, 1);
7547
7548         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7549         err = tg3_init_hw(tp, 1);
7550
7551         tg3_full_unlock(tp);
7552
7553         if (err)
7554                 free_irq(tp->pdev->irq, dev);
7555
7556         return err;
7557 }
7558
7559 static int tg3_open(struct net_device *dev)
7560 {
7561         struct tg3 *tp = netdev_priv(dev);
7562         int err;
7563
7564         netif_carrier_off(tp->dev);
7565
7566         tg3_full_lock(tp, 0);
7567
7568         err = tg3_set_power_state(tp, PCI_D0);
7569         if (err) {
7570                 tg3_full_unlock(tp);
7571                 return err;
7572         }
7573
7574         tg3_disable_ints(tp);
7575         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
7576
7577         tg3_full_unlock(tp);
7578
7579         /* The placement of this call is tied
7580          * to the setup and use of Host TX descriptors.
7581          */
7582         err = tg3_alloc_consistent(tp);
7583         if (err)
7584                 return err;
7585
7586         if (tp->tg3_flags & TG3_FLAG_SUPPORT_MSI) {
7587                 /* All MSI supporting chips should support tagged
7588                  * status.  Assert that this is the case.
7589                  */
7590                 if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
7591                         printk(KERN_WARNING PFX "%s: MSI without TAGGED? "
7592                                "Not using MSI.\n", tp->dev->name);
7593                 } else if (pci_enable_msi(tp->pdev) == 0) {
7594                         u32 msi_mode;
7595
7596                         msi_mode = tr32(MSGINT_MODE);
7597                         tw32(MSGINT_MODE, msi_mode | MSGINT_MODE_ENABLE);
7598                         tp->tg3_flags2 |= TG3_FLG2_USING_MSI;
7599                 }
7600         }
7601         err = tg3_request_irq(tp);
7602
7603         if (err) {
7604                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7605                         pci_disable_msi(tp->pdev);
7606                         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7607                 }
7608                 tg3_free_consistent(tp);
7609                 return err;
7610         }
7611
7612         napi_enable(&tp->napi);
7613
7614         tg3_full_lock(tp, 0);
7615
7616         err = tg3_init_hw(tp, 1);
7617         if (err) {
7618                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7619                 tg3_free_rings(tp);
7620         } else {
7621                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
7622                         tp->timer_offset = HZ;
7623                 else
7624                         tp->timer_offset = HZ / 10;
7625
7626                 BUG_ON(tp->timer_offset > HZ);
7627                 tp->timer_counter = tp->timer_multiplier =
7628                         (HZ / tp->timer_offset);
7629                 tp->asf_counter = tp->asf_multiplier =
7630                         ((HZ / tp->timer_offset) * 2);
7631
7632                 init_timer(&tp->timer);
7633                 tp->timer.expires = jiffies + tp->timer_offset;
7634                 tp->timer.data = (unsigned long) tp;
7635                 tp->timer.function = tg3_timer;
7636         }
7637
7638         tg3_full_unlock(tp);
7639
7640         if (err) {
7641                 napi_disable(&tp->napi);
7642                 free_irq(tp->pdev->irq, dev);
7643                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7644                         pci_disable_msi(tp->pdev);
7645                         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7646                 }
7647                 tg3_free_consistent(tp);
7648                 return err;
7649         }
7650
7651         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7652                 err = tg3_test_msi(tp);
7653
7654                 if (err) {
7655                         tg3_full_lock(tp, 0);
7656
7657                         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7658                                 pci_disable_msi(tp->pdev);
7659                                 tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7660                         }
7661                         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7662                         tg3_free_rings(tp);
7663                         tg3_free_consistent(tp);
7664
7665                         tg3_full_unlock(tp);
7666
7667                         napi_disable(&tp->napi);
7668
7669                         return err;
7670                 }
7671
7672                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7673                         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI) {
7674                                 u32 val = tr32(PCIE_TRANSACTION_CFG);
7675
7676                                 tw32(PCIE_TRANSACTION_CFG,
7677                                      val | PCIE_TRANS_CFG_1SHOT_MSI);
7678                         }
7679                 }
7680         }
7681
7682         tg3_full_lock(tp, 0);
7683
7684         add_timer(&tp->timer);
7685         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
7686         tg3_enable_ints(tp);
7687
7688         tg3_full_unlock(tp);
7689
7690         netif_start_queue(dev);
7691
7692         return 0;
7693 }
7694
7695 #if 0
7696 /*static*/ void tg3_dump_state(struct tg3 *tp)
7697 {
7698         u32 val32, val32_2, val32_3, val32_4, val32_5;
7699         u16 val16;
7700         int i;
7701
7702         pci_read_config_word(tp->pdev, PCI_STATUS, &val16);
7703         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE, &val32);
7704         printk("DEBUG: PCI status [%04x] TG3PCI state[%08x]\n",
7705                val16, val32);
7706
7707         /* MAC block */
7708         printk("DEBUG: MAC_MODE[%08x] MAC_STATUS[%08x]\n",
7709                tr32(MAC_MODE), tr32(MAC_STATUS));
7710         printk("       MAC_EVENT[%08x] MAC_LED_CTRL[%08x]\n",
7711                tr32(MAC_EVENT), tr32(MAC_LED_CTRL));
7712         printk("DEBUG: MAC_TX_MODE[%08x] MAC_TX_STATUS[%08x]\n",
7713                tr32(MAC_TX_MODE), tr32(MAC_TX_STATUS));
7714         printk("       MAC_RX_MODE[%08x] MAC_RX_STATUS[%08x]\n",
7715                tr32(MAC_RX_MODE), tr32(MAC_RX_STATUS));
7716
7717         /* Send data initiator control block */
7718         printk("DEBUG: SNDDATAI_MODE[%08x] SNDDATAI_STATUS[%08x]\n",
7719                tr32(SNDDATAI_MODE), tr32(SNDDATAI_STATUS));
7720         printk("       SNDDATAI_STATSCTRL[%08x]\n",
7721                tr32(SNDDATAI_STATSCTRL));
7722
7723         /* Send data completion control block */
7724         printk("DEBUG: SNDDATAC_MODE[%08x]\n", tr32(SNDDATAC_MODE));
7725
7726         /* Send BD ring selector block */
7727         printk("DEBUG: SNDBDS_MODE[%08x] SNDBDS_STATUS[%08x]\n",
7728                tr32(SNDBDS_MODE), tr32(SNDBDS_STATUS));
7729
7730         /* Send BD initiator control block */
7731         printk("DEBUG: SNDBDI_MODE[%08x] SNDBDI_STATUS[%08x]\n",
7732                tr32(SNDBDI_MODE), tr32(SNDBDI_STATUS));
7733
7734         /* Send BD completion control block */
7735         printk("DEBUG: SNDBDC_MODE[%08x]\n", tr32(SNDBDC_MODE));
7736
7737         /* Receive list placement control block */
7738         printk("DEBUG: RCVLPC_MODE[%08x] RCVLPC_STATUS[%08x]\n",
7739                tr32(RCVLPC_MODE), tr32(RCVLPC_STATUS));
7740         printk("       RCVLPC_STATSCTRL[%08x]\n",
7741                tr32(RCVLPC_STATSCTRL));
7742
7743         /* Receive data and receive BD initiator control block */
7744         printk("DEBUG: RCVDBDI_MODE[%08x] RCVDBDI_STATUS[%08x]\n",
7745                tr32(RCVDBDI_MODE), tr32(RCVDBDI_STATUS));
7746
7747         /* Receive data completion control block */
7748         printk("DEBUG: RCVDCC_MODE[%08x]\n",
7749                tr32(RCVDCC_MODE));
7750
7751         /* Receive BD initiator control block */
7752         printk("DEBUG: RCVBDI_MODE[%08x] RCVBDI_STATUS[%08x]\n",
7753                tr32(RCVBDI_MODE), tr32(RCVBDI_STATUS));
7754
7755         /* Receive BD completion control block */
7756         printk("DEBUG: RCVCC_MODE[%08x] RCVCC_STATUS[%08x]\n",
7757                tr32(RCVCC_MODE), tr32(RCVCC_STATUS));
7758
7759         /* Receive list selector control block */
7760         printk("DEBUG: RCVLSC_MODE[%08x] RCVLSC_STATUS[%08x]\n",
7761                tr32(RCVLSC_MODE), tr32(RCVLSC_STATUS));
7762
7763         /* Mbuf cluster free block */
7764         printk("DEBUG: MBFREE_MODE[%08x] MBFREE_STATUS[%08x]\n",
7765                tr32(MBFREE_MODE), tr32(MBFREE_STATUS));
7766
7767         /* Host coalescing control block */
7768         printk("DEBUG: HOSTCC_MODE[%08x] HOSTCC_STATUS[%08x]\n",
7769                tr32(HOSTCC_MODE), tr32(HOSTCC_STATUS));
7770         printk("DEBUG: HOSTCC_STATS_BLK_HOST_ADDR[%08x%08x]\n",
7771                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
7772                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
7773         printk("DEBUG: HOSTCC_STATUS_BLK_HOST_ADDR[%08x%08x]\n",
7774                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
7775                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
7776         printk("DEBUG: HOSTCC_STATS_BLK_NIC_ADDR[%08x]\n",
7777                tr32(HOSTCC_STATS_BLK_NIC_ADDR));
7778         printk("DEBUG: HOSTCC_STATUS_BLK_NIC_ADDR[%08x]\n",
7779                tr32(HOSTCC_STATUS_BLK_NIC_ADDR));
7780
7781         /* Memory arbiter control block */
7782         printk("DEBUG: MEMARB_MODE[%08x] MEMARB_STATUS[%08x]\n",
7783                tr32(MEMARB_MODE), tr32(MEMARB_STATUS));
7784
7785         /* Buffer manager control block */
7786         printk("DEBUG: BUFMGR_MODE[%08x] BUFMGR_STATUS[%08x]\n",
7787                tr32(BUFMGR_MODE), tr32(BUFMGR_STATUS));
7788         printk("DEBUG: BUFMGR_MB_POOL_ADDR[%08x] BUFMGR_MB_POOL_SIZE[%08x]\n",
7789                tr32(BUFMGR_MB_POOL_ADDR), tr32(BUFMGR_MB_POOL_SIZE));
7790         printk("DEBUG: BUFMGR_DMA_DESC_POOL_ADDR[%08x] "
7791                "BUFMGR_DMA_DESC_POOL_SIZE[%08x]\n",
7792                tr32(BUFMGR_DMA_DESC_POOL_ADDR),
7793                tr32(BUFMGR_DMA_DESC_POOL_SIZE));
7794
7795         /* Read DMA control block */
7796         printk("DEBUG: RDMAC_MODE[%08x] RDMAC_STATUS[%08x]\n",
7797                tr32(RDMAC_MODE), tr32(RDMAC_STATUS));
7798
7799         /* Write DMA control block */
7800         printk("DEBUG: WDMAC_MODE[%08x] WDMAC_STATUS[%08x]\n",
7801                tr32(WDMAC_MODE), tr32(WDMAC_STATUS));
7802
7803         /* DMA completion block */
7804         printk("DEBUG: DMAC_MODE[%08x]\n",
7805                tr32(DMAC_MODE));
7806
7807         /* GRC block */
7808         printk("DEBUG: GRC_MODE[%08x] GRC_MISC_CFG[%08x]\n",
7809                tr32(GRC_MODE), tr32(GRC_MISC_CFG));
7810         printk("DEBUG: GRC_LOCAL_CTRL[%08x]\n",
7811                tr32(GRC_LOCAL_CTRL));
7812
7813         /* TG3_BDINFOs */
7814         printk("DEBUG: RCVDBDI_JUMBO_BD[%08x%08x:%08x:%08x]\n",
7815                tr32(RCVDBDI_JUMBO_BD + 0x0),
7816                tr32(RCVDBDI_JUMBO_BD + 0x4),
7817                tr32(RCVDBDI_JUMBO_BD + 0x8),
7818                tr32(RCVDBDI_JUMBO_BD + 0xc));
7819         printk("DEBUG: RCVDBDI_STD_BD[%08x%08x:%08x:%08x]\n",
7820                tr32(RCVDBDI_STD_BD + 0x0),
7821                tr32(RCVDBDI_STD_BD + 0x4),
7822                tr32(RCVDBDI_STD_BD + 0x8),
7823                tr32(RCVDBDI_STD_BD + 0xc));
7824         printk("DEBUG: RCVDBDI_MINI_BD[%08x%08x:%08x:%08x]\n",
7825                tr32(RCVDBDI_MINI_BD + 0x0),
7826                tr32(RCVDBDI_MINI_BD + 0x4),
7827                tr32(RCVDBDI_MINI_BD + 0x8),
7828                tr32(RCVDBDI_MINI_BD + 0xc));
7829
7830         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x0, &val32);
7831         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x4, &val32_2);
7832         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x8, &val32_3);
7833         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0xc, &val32_4);
7834         printk("DEBUG: SRAM_SEND_RCB_0[%08x%08x:%08x:%08x]\n",
7835                val32, val32_2, val32_3, val32_4);
7836
7837         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x0, &val32);
7838         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x4, &val32_2);
7839         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x8, &val32_3);
7840         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0xc, &val32_4);
7841         printk("DEBUG: SRAM_RCV_RET_RCB_0[%08x%08x:%08x:%08x]\n",
7842                val32, val32_2, val32_3, val32_4);
7843
7844         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x0, &val32);
7845         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x4, &val32_2);
7846         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x8, &val32_3);
7847         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0xc, &val32_4);
7848         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x10, &val32_5);
7849         printk("DEBUG: SRAM_STATUS_BLK[%08x:%08x:%08x:%08x:%08x]\n",
7850                val32, val32_2, val32_3, val32_4, val32_5);
7851
7852         /* SW status block */
7853         printk("DEBUG: Host status block [%08x:%08x:(%04x:%04x:%04x):(%04x:%04x)]\n",
7854                tp->hw_status->status,
7855                tp->hw_status->status_tag,
7856                tp->hw_status->rx_jumbo_consumer,
7857                tp->hw_status->rx_consumer,
7858                tp->hw_status->rx_mini_consumer,
7859                tp->hw_status->idx[0].rx_producer,
7860                tp->hw_status->idx[0].tx_consumer);
7861
7862         /* SW statistics block */
7863         printk("DEBUG: Host statistics block [%08x:%08x:%08x:%08x]\n",
7864                ((u32 *)tp->hw_stats)[0],
7865                ((u32 *)tp->hw_stats)[1],
7866                ((u32 *)tp->hw_stats)[2],
7867                ((u32 *)tp->hw_stats)[3]);
7868
7869         /* Mailboxes */
7870         printk("DEBUG: SNDHOST_PROD[%08x%08x] SNDNIC_PROD[%08x%08x]\n",
7871                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x0),
7872                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x4),
7873                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x0),
7874                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x4));
7875
7876         /* NIC side send descriptors. */
7877         for (i = 0; i < 6; i++) {
7878                 unsigned long txd;
7879
7880                 txd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_TX_BUFFER_DESC
7881                         + (i * sizeof(struct tg3_tx_buffer_desc));
7882                 printk("DEBUG: NIC TXD(%d)[%08x:%08x:%08x:%08x]\n",
7883                        i,
7884                        readl(txd + 0x0), readl(txd + 0x4),
7885                        readl(txd + 0x8), readl(txd + 0xc));
7886         }
7887
7888         /* NIC side RX descriptors. */
7889         for (i = 0; i < 6; i++) {
7890                 unsigned long rxd;
7891
7892                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_BUFFER_DESC
7893                         + (i * sizeof(struct tg3_rx_buffer_desc));
7894                 printk("DEBUG: NIC RXD_STD(%d)[0][%08x:%08x:%08x:%08x]\n",
7895                        i,
7896                        readl(rxd + 0x0), readl(rxd + 0x4),
7897                        readl(rxd + 0x8), readl(rxd + 0xc));
7898                 rxd += (4 * sizeof(u32));
7899                 printk("DEBUG: NIC RXD_STD(%d)[1][%08x:%08x:%08x:%08x]\n",
7900                        i,
7901                        readl(rxd + 0x0), readl(rxd + 0x4),
7902                        readl(rxd + 0x8), readl(rxd + 0xc));
7903         }
7904
7905         for (i = 0; i < 6; i++) {
7906                 unsigned long rxd;
7907
7908                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_JUMBO_BUFFER_DESC
7909                         + (i * sizeof(struct tg3_rx_buffer_desc));
7910                 printk("DEBUG: NIC RXD_JUMBO(%d)[0][%08x:%08x:%08x:%08x]\n",
7911                        i,
7912                        readl(rxd + 0x0), readl(rxd + 0x4),
7913                        readl(rxd + 0x8), readl(rxd + 0xc));
7914                 rxd += (4 * sizeof(u32));
7915                 printk("DEBUG: NIC RXD_JUMBO(%d)[1][%08x:%08x:%08x:%08x]\n",
7916                        i,
7917                        readl(rxd + 0x0), readl(rxd + 0x4),
7918                        readl(rxd + 0x8), readl(rxd + 0xc));
7919         }
7920 }
7921 #endif
7922
7923 static struct net_device_stats *tg3_get_stats(struct net_device *);
7924 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *);
7925
7926 static int tg3_close(struct net_device *dev)
7927 {
7928         struct tg3 *tp = netdev_priv(dev);
7929
7930         napi_disable(&tp->napi);
7931         cancel_work_sync(&tp->reset_task);
7932
7933         netif_stop_queue(dev);
7934
7935         del_timer_sync(&tp->timer);
7936
7937         tg3_full_lock(tp, 1);
7938 #if 0
7939         tg3_dump_state(tp);
7940 #endif
7941
7942         tg3_disable_ints(tp);
7943
7944         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7945         tg3_free_rings(tp);
7946         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
7947
7948         tg3_full_unlock(tp);
7949
7950         free_irq(tp->pdev->irq, dev);
7951         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7952                 pci_disable_msi(tp->pdev);
7953                 tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7954         }
7955
7956         memcpy(&tp->net_stats_prev, tg3_get_stats(tp->dev),
7957                sizeof(tp->net_stats_prev));
7958         memcpy(&tp->estats_prev, tg3_get_estats(tp),
7959                sizeof(tp->estats_prev));
7960
7961         tg3_free_consistent(tp);
7962
7963         tg3_set_power_state(tp, PCI_D3hot);
7964
7965         netif_carrier_off(tp->dev);
7966
7967         return 0;
7968 }
7969
7970 static inline unsigned long get_stat64(tg3_stat64_t *val)
7971 {
7972         unsigned long ret;
7973
7974 #if (BITS_PER_LONG == 32)
7975         ret = val->low;
7976 #else
7977         ret = ((u64)val->high << 32) | ((u64)val->low);
7978 #endif
7979         return ret;
7980 }
7981
7982 static unsigned long calc_crc_errors(struct tg3 *tp)
7983 {
7984         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7985
7986         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
7987             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
7988              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
7989                 u32 val;
7990
7991                 spin_lock_bh(&tp->lock);
7992                 if (!tg3_readphy(tp, MII_TG3_TEST1, &val)) {
7993                         tg3_writephy(tp, MII_TG3_TEST1,
7994                                      val | MII_TG3_TEST1_CRC_EN);
7995                         tg3_readphy(tp, 0x14, &val);
7996                 } else
7997                         val = 0;
7998                 spin_unlock_bh(&tp->lock);
7999
8000                 tp->phy_crc_errors += val;
8001
8002                 return tp->phy_crc_errors;
8003         }
8004
8005         return get_stat64(&hw_stats->rx_fcs_errors);
8006 }
8007
8008 #define ESTAT_ADD(member) \
8009         estats->member =        old_estats->member + \
8010                                 get_stat64(&hw_stats->member)
8011
8012 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *tp)
8013 {
8014         struct tg3_ethtool_stats *estats = &tp->estats;
8015         struct tg3_ethtool_stats *old_estats = &tp->estats_prev;
8016         struct tg3_hw_stats *hw_stats = tp->hw_stats;
8017
8018         if (!hw_stats)
8019                 return old_estats;
8020
8021         ESTAT_ADD(rx_octets);
8022         ESTAT_ADD(rx_fragments);
8023         ESTAT_ADD(rx_ucast_packets);
8024         ESTAT_ADD(rx_mcast_packets);
8025         ESTAT_ADD(rx_bcast_packets);
8026         ESTAT_ADD(rx_fcs_errors);
8027         ESTAT_ADD(rx_align_errors);
8028         ESTAT_ADD(rx_xon_pause_rcvd);
8029         ESTAT_ADD(rx_xoff_pause_rcvd);
8030         ESTAT_ADD(rx_mac_ctrl_rcvd);
8031         ESTAT_ADD(rx_xoff_entered);
8032         ESTAT_ADD(rx_frame_too_long_errors);
8033         ESTAT_ADD(rx_jabbers);
8034         ESTAT_ADD(rx_undersize_packets);
8035         ESTAT_ADD(rx_in_length_errors);
8036         ESTAT_ADD(rx_out_length_errors);
8037         ESTAT_ADD(rx_64_or_less_octet_packets);
8038         ESTAT_ADD(rx_65_to_127_octet_packets);
8039         ESTAT_ADD(rx_128_to_255_octet_packets);
8040         ESTAT_ADD(rx_256_to_511_octet_packets);
8041         ESTAT_ADD(rx_512_to_1023_octet_packets);
8042         ESTAT_ADD(rx_1024_to_1522_octet_packets);
8043         ESTAT_ADD(rx_1523_to_2047_octet_packets);
8044         ESTAT_ADD(rx_2048_to_4095_octet_packets);
8045         ESTAT_ADD(rx_4096_to_8191_octet_packets);
8046         ESTAT_ADD(rx_8192_to_9022_octet_packets);
8047
8048         ESTAT_ADD(tx_octets);
8049         ESTAT_ADD(tx_collisions);
8050         ESTAT_ADD(tx_xon_sent);
8051         ESTAT_ADD(tx_xoff_sent);
8052         ESTAT_ADD(tx_flow_control);
8053         ESTAT_ADD(tx_mac_errors);
8054         ESTAT_ADD(tx_single_collisions);
8055         ESTAT_ADD(tx_mult_collisions);
8056         ESTAT_ADD(tx_deferred);
8057         ESTAT_ADD(tx_excessive_collisions);
8058         ESTAT_ADD(tx_late_collisions);
8059         ESTAT_ADD(tx_collide_2times);
8060         ESTAT_ADD(tx_collide_3times);
8061         ESTAT_ADD(tx_collide_4times);
8062         ESTAT_ADD(tx_collide_5times);
8063         ESTAT_ADD(tx_collide_6times);
8064         ESTAT_ADD(tx_collide_7times);
8065         ESTAT_ADD(tx_collide_8times);
8066         ESTAT_ADD(tx_collide_9times);
8067         ESTAT_ADD(tx_collide_10times);
8068         ESTAT_ADD(tx_collide_11times);
8069         ESTAT_ADD(tx_collide_12times);
8070         ESTAT_ADD(tx_collide_13times);
8071         ESTAT_ADD(tx_collide_14times);
8072         ESTAT_ADD(tx_collide_15times);
8073         ESTAT_ADD(tx_ucast_packets);
8074         ESTAT_ADD(tx_mcast_packets);
8075         ESTAT_ADD(tx_bcast_packets);
8076         ESTAT_ADD(tx_carrier_sense_errors);
8077         ESTAT_ADD(tx_discards);
8078         ESTAT_ADD(tx_errors);
8079
8080         ESTAT_ADD(dma_writeq_full);
8081         ESTAT_ADD(dma_write_prioq_full);
8082         ESTAT_ADD(rxbds_empty);
8083         ESTAT_ADD(rx_discards);
8084         ESTAT_ADD(rx_errors);
8085         ESTAT_ADD(rx_threshold_hit);
8086
8087         ESTAT_ADD(dma_readq_full);
8088         ESTAT_ADD(dma_read_prioq_full);
8089         ESTAT_ADD(tx_comp_queue_full);
8090
8091         ESTAT_ADD(ring_set_send_prod_index);
8092         ESTAT_ADD(ring_status_update);
8093         ESTAT_ADD(nic_irqs);
8094         ESTAT_ADD(nic_avoided_irqs);
8095         ESTAT_ADD(nic_tx_threshold_hit);
8096
8097         return estats;
8098 }
8099
8100 static struct net_device_stats *tg3_get_stats(struct net_device *dev)
8101 {
8102         struct tg3 *tp = netdev_priv(dev);
8103         struct net_device_stats *stats = &tp->net_stats;
8104         struct net_device_stats *old_stats = &tp->net_stats_prev;
8105         struct tg3_hw_stats *hw_stats = tp->hw_stats;
8106
8107         if (!hw_stats)
8108                 return old_stats;
8109
8110         stats->rx_packets = old_stats->rx_packets +
8111                 get_stat64(&hw_stats->rx_ucast_packets) +
8112                 get_stat64(&hw_stats->rx_mcast_packets) +
8113                 get_stat64(&hw_stats->rx_bcast_packets);
8114
8115         stats->tx_packets = old_stats->tx_packets +
8116                 get_stat64(&hw_stats->tx_ucast_packets) +
8117                 get_stat64(&hw_stats->tx_mcast_packets) +
8118                 get_stat64(&hw_stats->tx_bcast_packets);
8119
8120         stats->rx_bytes = old_stats->rx_bytes +
8121                 get_stat64(&hw_stats->rx_octets);
8122         stats->tx_bytes = old_stats->tx_bytes +
8123                 get_stat64(&hw_stats->tx_octets);
8124
8125         stats->rx_errors = old_stats->rx_errors +
8126                 get_stat64(&hw_stats->rx_errors);
8127         stats->tx_errors = old_stats->tx_errors +
8128                 get_stat64(&hw_stats->tx_errors) +
8129                 get_stat64(&hw_stats->tx_mac_errors) +
8130                 get_stat64(&hw_stats->tx_carrier_sense_errors) +
8131                 get_stat64(&hw_stats->tx_discards);
8132
8133         stats->multicast = old_stats->multicast +
8134                 get_stat64(&hw_stats->rx_mcast_packets);
8135         stats->collisions = old_stats->collisions +
8136                 get_stat64(&hw_stats->tx_collisions);
8137
8138         stats->rx_length_errors = old_stats->rx_length_errors +
8139                 get_stat64(&hw_stats->rx_frame_too_long_errors) +
8140                 get_stat64(&hw_stats->rx_undersize_packets);
8141
8142         stats->rx_over_errors = old_stats->rx_over_errors +
8143                 get_stat64(&hw_stats->rxbds_empty);
8144         stats->rx_frame_errors = old_stats->rx_frame_errors +
8145                 get_stat64(&hw_stats->rx_align_errors);
8146         stats->tx_aborted_errors = old_stats->tx_aborted_errors +
8147                 get_stat64(&hw_stats->tx_discards);
8148         stats->tx_carrier_errors = old_stats->tx_carrier_errors +
8149                 get_stat64(&hw_stats->tx_carrier_sense_errors);
8150
8151         stats->rx_crc_errors = old_stats->rx_crc_errors +
8152                 calc_crc_errors(tp);
8153
8154         stats->rx_missed_errors = old_stats->rx_missed_errors +
8155                 get_stat64(&hw_stats->rx_discards);
8156
8157         return stats;
8158 }
8159
8160 static inline u32 calc_crc(unsigned char *buf, int len)
8161 {
8162         u32 reg;
8163         u32 tmp;
8164         int j, k;
8165
8166         reg = 0xffffffff;
8167
8168         for (j = 0; j < len; j++) {
8169                 reg ^= buf[j];
8170
8171                 for (k = 0; k < 8; k++) {
8172                         tmp = reg & 0x01;
8173
8174                         reg >>= 1;
8175
8176                         if (tmp) {
8177                                 reg ^= 0xedb88320;
8178                         }
8179                 }
8180         }
8181
8182         return ~reg;
8183 }
8184
8185 static void tg3_set_multi(struct tg3 *tp, unsigned int accept_all)
8186 {
8187         /* accept or reject all multicast frames */
8188         tw32(MAC_HASH_REG_0, accept_all ? 0xffffffff : 0);
8189         tw32(MAC_HASH_REG_1, accept_all ? 0xffffffff : 0);
8190         tw32(MAC_HASH_REG_2, accept_all ? 0xffffffff : 0);
8191         tw32(MAC_HASH_REG_3, accept_all ? 0xffffffff : 0);
8192 }
8193
8194 static void __tg3_set_rx_mode(struct net_device *dev)
8195 {
8196         struct tg3 *tp = netdev_priv(dev);
8197         u32 rx_mode;
8198
8199         rx_mode = tp->rx_mode & ~(RX_MODE_PROMISC |
8200                                   RX_MODE_KEEP_VLAN_TAG);
8201
8202         /* When ASF is in use, we always keep the RX_MODE_KEEP_VLAN_TAG
8203          * flag clear.
8204          */
8205 #if TG3_VLAN_TAG_USED
8206         if (!tp->vlgrp &&
8207             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
8208                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
8209 #else
8210         /* By definition, VLAN is disabled always in this
8211          * case.
8212          */
8213         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
8214                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
8215 #endif
8216
8217         if (dev->flags & IFF_PROMISC) {
8218                 /* Promiscuous mode. */
8219                 rx_mode |= RX_MODE_PROMISC;
8220         } else if (dev->flags & IFF_ALLMULTI) {
8221                 /* Accept all multicast. */
8222                 tg3_set_multi (tp, 1);
8223         } else if (dev->mc_count < 1) {
8224                 /* Reject all multicast. */
8225                 tg3_set_multi (tp, 0);
8226         } else {
8227                 /* Accept one or more multicast(s). */
8228                 struct dev_mc_list *mclist;
8229                 unsigned int i;
8230                 u32 mc_filter[4] = { 0, };
8231                 u32 regidx;
8232                 u32 bit;
8233                 u32 crc;
8234
8235                 for (i = 0, mclist = dev->mc_list; mclist && i < dev->mc_count;
8236                      i++, mclist = mclist->next) {
8237
8238                         crc = calc_crc (mclist->dmi_addr, ETH_ALEN);
8239                         bit = ~crc & 0x7f;
8240                         regidx = (bit & 0x60) >> 5;
8241                         bit &= 0x1f;
8242                         mc_filter[regidx] |= (1 << bit);
8243                 }
8244
8245                 tw32(MAC_HASH_REG_0, mc_filter[0]);
8246                 tw32(MAC_HASH_REG_1, mc_filter[1]);
8247                 tw32(MAC_HASH_REG_2, mc_filter[2]);
8248                 tw32(MAC_HASH_REG_3, mc_filter[3]);
8249         }
8250
8251         if (rx_mode != tp->rx_mode) {
8252                 tp->rx_mode = rx_mode;
8253                 tw32_f(MAC_RX_MODE, rx_mode);
8254                 udelay(10);
8255         }
8256 }
8257
8258 static void tg3_set_rx_mode(struct net_device *dev)
8259 {
8260         struct tg3 *tp = netdev_priv(dev);
8261
8262         if (!netif_running(dev))
8263                 return;
8264
8265         tg3_full_lock(tp, 0);
8266         __tg3_set_rx_mode(dev);
8267         tg3_full_unlock(tp);
8268 }
8269
8270 #define TG3_REGDUMP_LEN         (32 * 1024)
8271
8272 static int tg3_get_regs_len(struct net_device *dev)
8273 {
8274         return TG3_REGDUMP_LEN;
8275 }
8276
8277 static void tg3_get_regs(struct net_device *dev,
8278                 struct ethtool_regs *regs, void *_p)
8279 {
8280         u32 *p = _p;
8281         struct tg3 *tp = netdev_priv(dev);
8282         u8 *orig_p = _p;
8283         int i;
8284
8285         regs->version = 0;
8286
8287         memset(p, 0, TG3_REGDUMP_LEN);
8288
8289         if (tp->link_config.phy_is_low_power)
8290                 return;
8291
8292         tg3_full_lock(tp, 0);
8293
8294 #define __GET_REG32(reg)        (*(p)++ = tr32(reg))
8295 #define GET_REG32_LOOP(base,len)                \
8296 do {    p = (u32 *)(orig_p + (base));           \
8297         for (i = 0; i < len; i += 4)            \
8298                 __GET_REG32((base) + i);        \
8299 } while (0)
8300 #define GET_REG32_1(reg)                        \
8301 do {    p = (u32 *)(orig_p + (reg));            \
8302         __GET_REG32((reg));                     \
8303 } while (0)
8304
8305         GET_REG32_LOOP(TG3PCI_VENDOR, 0xb0);
8306         GET_REG32_LOOP(MAILBOX_INTERRUPT_0, 0x200);
8307         GET_REG32_LOOP(MAC_MODE, 0x4f0);
8308         GET_REG32_LOOP(SNDDATAI_MODE, 0xe0);
8309         GET_REG32_1(SNDDATAC_MODE);
8310         GET_REG32_LOOP(SNDBDS_MODE, 0x80);
8311         GET_REG32_LOOP(SNDBDI_MODE, 0x48);
8312         GET_REG32_1(SNDBDC_MODE);
8313         GET_REG32_LOOP(RCVLPC_MODE, 0x20);
8314         GET_REG32_LOOP(RCVLPC_SELLST_BASE, 0x15c);
8315         GET_REG32_LOOP(RCVDBDI_MODE, 0x0c);
8316         GET_REG32_LOOP(RCVDBDI_JUMBO_BD, 0x3c);
8317         GET_REG32_LOOP(RCVDBDI_BD_PROD_IDX_0, 0x44);
8318         GET_REG32_1(RCVDCC_MODE);
8319         GET_REG32_LOOP(RCVBDI_MODE, 0x20);
8320         GET_REG32_LOOP(RCVCC_MODE, 0x14);
8321         GET_REG32_LOOP(RCVLSC_MODE, 0x08);
8322         GET_REG32_1(MBFREE_MODE);
8323         GET_REG32_LOOP(HOSTCC_MODE, 0x100);
8324         GET_REG32_LOOP(MEMARB_MODE, 0x10);
8325         GET_REG32_LOOP(BUFMGR_MODE, 0x58);
8326         GET_REG32_LOOP(RDMAC_MODE, 0x08);
8327         GET_REG32_LOOP(WDMAC_MODE, 0x08);
8328         GET_REG32_1(RX_CPU_MODE);
8329         GET_REG32_1(RX_CPU_STATE);
8330         GET_REG32_1(RX_CPU_PGMCTR);
8331         GET_REG32_1(RX_CPU_HWBKPT);
8332         GET_REG32_1(TX_CPU_MODE);
8333         GET_REG32_1(TX_CPU_STATE);
8334         GET_REG32_1(TX_CPU_PGMCTR);
8335         GET_REG32_LOOP(GRCMBOX_INTERRUPT_0, 0x110);
8336         GET_REG32_LOOP(FTQ_RESET, 0x120);
8337         GET_REG32_LOOP(MSGINT_MODE, 0x0c);
8338         GET_REG32_1(DMAC_MODE);
8339         GET_REG32_LOOP(GRC_MODE, 0x4c);
8340         if (tp->tg3_flags & TG3_FLAG_NVRAM)
8341                 GET_REG32_LOOP(NVRAM_CMD, 0x24);
8342
8343 #undef __GET_REG32
8344 #undef GET_REG32_LOOP
8345 #undef GET_REG32_1
8346
8347         tg3_full_unlock(tp);
8348 }
8349
8350 static int tg3_get_eeprom_len(struct net_device *dev)
8351 {
8352         struct tg3 *tp = netdev_priv(dev);
8353
8354         return tp->nvram_size;
8355 }
8356
8357 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val);
8358 static int tg3_nvram_read_le(struct tg3 *tp, u32 offset, __le32 *val);
8359 static int tg3_nvram_read_swab(struct tg3 *tp, u32 offset, u32 *val);
8360
8361 static int tg3_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
8362 {
8363         struct tg3 *tp = netdev_priv(dev);
8364         int ret;
8365         u8  *pd;
8366         u32 i, offset, len, b_offset, b_count;
8367         __le32 val;
8368
8369         if (tp->link_config.phy_is_low_power)
8370                 return -EAGAIN;
8371
8372         offset = eeprom->offset;
8373         len = eeprom->len;
8374         eeprom->len = 0;
8375
8376         eeprom->magic = TG3_EEPROM_MAGIC;
8377
8378         if (offset & 3) {
8379                 /* adjustments to start on required 4 byte boundary */
8380                 b_offset = offset & 3;
8381                 b_count = 4 - b_offset;
8382                 if (b_count > len) {
8383                         /* i.e. offset=1 len=2 */
8384                         b_count = len;
8385                 }
8386                 ret = tg3_nvram_read_le(tp, offset-b_offset, &val);
8387                 if (ret)
8388                         return ret;
8389                 memcpy(data, ((char*)&val) + b_offset, b_count);
8390                 len -= b_count;
8391                 offset += b_count;
8392                 eeprom->len += b_count;
8393         }
8394
8395         /* read bytes upto the last 4 byte boundary */
8396         pd = &data[eeprom->len];
8397         for (i = 0; i < (len - (len & 3)); i += 4) {
8398                 ret = tg3_nvram_read_le(tp, offset + i, &val);
8399                 if (ret) {
8400                         eeprom->len += i;
8401                         return ret;
8402                 }
8403                 memcpy(pd + i, &val, 4);
8404         }
8405         eeprom->len += i;
8406
8407         if (len & 3) {
8408                 /* read last bytes not ending on 4 byte boundary */
8409                 pd = &data[eeprom->len];
8410                 b_count = len & 3;
8411                 b_offset = offset + len - b_count;
8412                 ret = tg3_nvram_read_le(tp, b_offset, &val);
8413                 if (ret)
8414                         return ret;
8415                 memcpy(pd, &val, b_count);
8416                 eeprom->len += b_count;
8417         }
8418         return 0;
8419 }
8420
8421 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf);
8422
8423 static int tg3_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
8424 {
8425         struct tg3 *tp = netdev_priv(dev);
8426         int ret;
8427         u32 offset, len, b_offset, odd_len;
8428         u8 *buf;
8429         __le32 start, end;
8430
8431         if (tp->link_config.phy_is_low_power)
8432                 return -EAGAIN;
8433
8434         if (eeprom->magic != TG3_EEPROM_MAGIC)
8435                 return -EINVAL;
8436
8437         offset = eeprom->offset;
8438         len = eeprom->len;
8439
8440         if ((b_offset = (offset & 3))) {
8441                 /* adjustments to start on required 4 byte boundary */
8442                 ret = tg3_nvram_read_le(tp, offset-b_offset, &start);
8443                 if (ret)
8444                         return ret;
8445                 len += b_offset;
8446                 offset &= ~3;
8447                 if (len < 4)
8448                         len = 4;
8449         }
8450
8451         odd_len = 0;
8452         if (len & 3) {
8453                 /* adjustments to end on required 4 byte boundary */
8454                 odd_len = 1;
8455                 len = (len + 3) & ~3;
8456                 ret = tg3_nvram_read_le(tp, offset+len-4, &end);
8457                 if (ret)
8458                         return ret;
8459         }
8460
8461         buf = data;
8462         if (b_offset || odd_len) {
8463                 buf = kmalloc(len, GFP_KERNEL);
8464                 if (!buf)
8465                         return -ENOMEM;
8466                 if (b_offset)
8467                         memcpy(buf, &start, 4);
8468                 if (odd_len)
8469                         memcpy(buf+len-4, &end, 4);
8470                 memcpy(buf + b_offset, data, eeprom->len);
8471         }
8472
8473         ret = tg3_nvram_write_block(tp, offset, len, buf);
8474
8475         if (buf != data)
8476                 kfree(buf);
8477
8478         return ret;
8479 }
8480
8481 static int tg3_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
8482 {
8483         struct tg3 *tp = netdev_priv(dev);
8484
8485         cmd->supported = (SUPPORTED_Autoneg);
8486
8487         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
8488                 cmd->supported |= (SUPPORTED_1000baseT_Half |
8489                                    SUPPORTED_1000baseT_Full);
8490
8491         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
8492                 cmd->supported |= (SUPPORTED_100baseT_Half |
8493                                   SUPPORTED_100baseT_Full |
8494                                   SUPPORTED_10baseT_Half |
8495                                   SUPPORTED_10baseT_Full |
8496                                   SUPPORTED_TP);
8497                 cmd->port = PORT_TP;
8498         } else {
8499                 cmd->supported |= SUPPORTED_FIBRE;
8500                 cmd->port = PORT_FIBRE;
8501         }
8502
8503         cmd->advertising = tp->link_config.advertising;
8504         if (netif_running(dev)) {
8505                 cmd->speed = tp->link_config.active_speed;
8506                 cmd->duplex = tp->link_config.active_duplex;
8507         }
8508         cmd->phy_address = PHY_ADDR;
8509         cmd->transceiver = 0;
8510         cmd->autoneg = tp->link_config.autoneg;
8511         cmd->maxtxpkt = 0;
8512         cmd->maxrxpkt = 0;
8513         return 0;
8514 }
8515
8516 static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
8517 {
8518         struct tg3 *tp = netdev_priv(dev);
8519
8520         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) {
8521                 /* These are the only valid advertisement bits allowed.  */
8522                 if (cmd->autoneg == AUTONEG_ENABLE &&
8523                     (cmd->advertising & ~(ADVERTISED_1000baseT_Half |
8524                                           ADVERTISED_1000baseT_Full |
8525                                           ADVERTISED_Autoneg |
8526                                           ADVERTISED_FIBRE)))
8527                         return -EINVAL;
8528                 /* Fiber can only do SPEED_1000.  */
8529                 else if ((cmd->autoneg != AUTONEG_ENABLE) &&
8530                          (cmd->speed != SPEED_1000))
8531                         return -EINVAL;
8532         /* Copper cannot force SPEED_1000.  */
8533         } else if ((cmd->autoneg != AUTONEG_ENABLE) &&
8534                    (cmd->speed == SPEED_1000))
8535                 return -EINVAL;
8536         else if ((cmd->speed == SPEED_1000) &&
8537                  (tp->tg3_flags2 & TG3_FLAG_10_100_ONLY))
8538                 return -EINVAL;
8539
8540         tg3_full_lock(tp, 0);
8541
8542         tp->link_config.autoneg = cmd->autoneg;
8543         if (cmd->autoneg == AUTONEG_ENABLE) {
8544                 tp->link_config.advertising = (cmd->advertising |
8545                                               ADVERTISED_Autoneg);
8546                 tp->link_config.speed = SPEED_INVALID;
8547                 tp->link_config.duplex = DUPLEX_INVALID;
8548         } else {
8549                 tp->link_config.advertising = 0;
8550                 tp->link_config.speed = cmd->speed;
8551                 tp->link_config.duplex = cmd->duplex;
8552         }
8553
8554         tp->link_config.orig_speed = tp->link_config.speed;
8555         tp->link_config.orig_duplex = tp->link_config.duplex;
8556         tp->link_config.orig_autoneg = tp->link_config.autoneg;
8557
8558         if (netif_running(dev))
8559                 tg3_setup_phy(tp, 1);
8560
8561         tg3_full_unlock(tp);
8562
8563         return 0;
8564 }
8565
8566 static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
8567 {
8568         struct tg3 *tp = netdev_priv(dev);
8569
8570         strcpy(info->driver, DRV_MODULE_NAME);
8571         strcpy(info->version, DRV_MODULE_VERSION);
8572         strcpy(info->fw_version, tp->fw_ver);
8573         strcpy(info->bus_info, pci_name(tp->pdev));
8574 }
8575
8576 static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
8577 {
8578         struct tg3 *tp = netdev_priv(dev);
8579
8580         if (tp->tg3_flags & TG3_FLAG_WOL_CAP)
8581                 wol->supported = WAKE_MAGIC;
8582         else
8583                 wol->supported = 0;
8584         wol->wolopts = 0;
8585         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)
8586                 wol->wolopts = WAKE_MAGIC;
8587         memset(&wol->sopass, 0, sizeof(wol->sopass));
8588 }
8589
8590 static int tg3_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
8591 {
8592         struct tg3 *tp = netdev_priv(dev);
8593
8594         if (wol->wolopts & ~WAKE_MAGIC)
8595                 return -EINVAL;
8596         if ((wol->wolopts & WAKE_MAGIC) &&
8597             !(tp->tg3_flags & TG3_FLAG_WOL_CAP))
8598                 return -EINVAL;
8599
8600         spin_lock_bh(&tp->lock);
8601         if (wol->wolopts & WAKE_MAGIC)
8602                 tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
8603         else
8604                 tp->tg3_flags &= ~TG3_FLAG_WOL_ENABLE;
8605         spin_unlock_bh(&tp->lock);
8606
8607         return 0;
8608 }
8609
8610 static u32 tg3_get_msglevel(struct net_device *dev)
8611 {
8612         struct tg3 *tp = netdev_priv(dev);
8613         return tp->msg_enable;
8614 }
8615
8616 static void tg3_set_msglevel(struct net_device *dev, u32 value)
8617 {
8618         struct tg3 *tp = netdev_priv(dev);
8619         tp->msg_enable = value;
8620 }
8621
8622 static int tg3_set_tso(struct net_device *dev, u32 value)
8623 {
8624         struct tg3 *tp = netdev_priv(dev);
8625
8626         if (!(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
8627                 if (value)
8628                         return -EINVAL;
8629                 return 0;
8630         }
8631         if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO_2) &&
8632             (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)) {
8633                 if (value) {
8634                         dev->features |= NETIF_F_TSO6;
8635                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
8636                                 dev->features |= NETIF_F_TSO_ECN;
8637                 } else
8638                         dev->features &= ~(NETIF_F_TSO6 | NETIF_F_TSO_ECN);
8639         }
8640         return ethtool_op_set_tso(dev, value);
8641 }
8642
8643 static int tg3_nway_reset(struct net_device *dev)
8644 {
8645         struct tg3 *tp = netdev_priv(dev);
8646         u32 bmcr;
8647         int r;
8648
8649         if (!netif_running(dev))
8650                 return -EAGAIN;
8651
8652         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
8653                 return -EINVAL;
8654
8655         spin_lock_bh(&tp->lock);
8656         r = -EINVAL;
8657         tg3_readphy(tp, MII_BMCR, &bmcr);
8658         if (!tg3_readphy(tp, MII_BMCR, &bmcr) &&
8659             ((bmcr & BMCR_ANENABLE) ||
8660              (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT))) {
8661                 tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANRESTART |
8662                                            BMCR_ANENABLE);
8663                 r = 0;
8664         }
8665         spin_unlock_bh(&tp->lock);
8666
8667         return r;
8668 }
8669
8670 static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
8671 {
8672         struct tg3 *tp = netdev_priv(dev);
8673
8674         ering->rx_max_pending = TG3_RX_RING_SIZE - 1;
8675         ering->rx_mini_max_pending = 0;
8676         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
8677                 ering->rx_jumbo_max_pending = TG3_RX_JUMBO_RING_SIZE - 1;
8678         else
8679                 ering->rx_jumbo_max_pending = 0;
8680
8681         ering->tx_max_pending = TG3_TX_RING_SIZE - 1;
8682
8683         ering->rx_pending = tp->rx_pending;
8684         ering->rx_mini_pending = 0;
8685         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
8686                 ering->rx_jumbo_pending = tp->rx_jumbo_pending;
8687         else
8688                 ering->rx_jumbo_pending = 0;
8689
8690         ering->tx_pending = tp->tx_pending;
8691 }
8692
8693 static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
8694 {
8695         struct tg3 *tp = netdev_priv(dev);
8696         int irq_sync = 0, err = 0;
8697
8698         if ((ering->rx_pending > TG3_RX_RING_SIZE - 1) ||
8699             (ering->rx_jumbo_pending > TG3_RX_JUMBO_RING_SIZE - 1) ||
8700             (ering->tx_pending > TG3_TX_RING_SIZE - 1) ||
8701             (ering->tx_pending <= MAX_SKB_FRAGS) ||
8702             ((tp->tg3_flags2 & TG3_FLG2_TSO_BUG) &&
8703              (ering->tx_pending <= (MAX_SKB_FRAGS * 3))))
8704                 return -EINVAL;
8705
8706         if (netif_running(dev)) {
8707                 tg3_netif_stop(tp);
8708                 irq_sync = 1;
8709         }
8710
8711         tg3_full_lock(tp, irq_sync);
8712
8713         tp->rx_pending = ering->rx_pending;
8714
8715         if ((tp->tg3_flags2 & TG3_FLG2_MAX_RXPEND_64) &&
8716             tp->rx_pending > 63)
8717                 tp->rx_pending = 63;
8718         tp->rx_jumbo_pending = ering->rx_jumbo_pending;
8719         tp->tx_pending = ering->tx_pending;
8720
8721         if (netif_running(dev)) {
8722                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8723                 err = tg3_restart_hw(tp, 1);
8724                 if (!err)
8725                         tg3_netif_start(tp);
8726         }
8727
8728         tg3_full_unlock(tp);
8729
8730         return err;
8731 }
8732
8733 static void tg3_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
8734 {
8735         struct tg3 *tp = netdev_priv(dev);
8736
8737         epause->autoneg = (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG) != 0;
8738
8739         if (tp->link_config.active_flowctrl & TG3_FLOW_CTRL_RX)
8740                 epause->rx_pause = 1;
8741         else
8742                 epause->rx_pause = 0;
8743
8744         if (tp->link_config.active_flowctrl & TG3_FLOW_CTRL_TX)
8745                 epause->tx_pause = 1;
8746         else
8747                 epause->tx_pause = 0;
8748 }
8749
8750 static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
8751 {
8752         struct tg3 *tp = netdev_priv(dev);
8753         int irq_sync = 0, err = 0;
8754
8755         if (netif_running(dev)) {
8756                 tg3_netif_stop(tp);
8757                 irq_sync = 1;
8758         }
8759
8760         tg3_full_lock(tp, irq_sync);
8761
8762         if (epause->autoneg)
8763                 tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
8764         else
8765                 tp->tg3_flags &= ~TG3_FLAG_PAUSE_AUTONEG;
8766         if (epause->rx_pause)
8767                 tp->link_config.flowctrl |= TG3_FLOW_CTRL_RX;
8768         else
8769                 tp->link_config.flowctrl &= ~TG3_FLOW_CTRL_RX;
8770         if (epause->tx_pause)
8771                 tp->link_config.flowctrl |= TG3_FLOW_CTRL_TX;
8772         else
8773                 tp->link_config.flowctrl &= ~TG3_FLOW_CTRL_TX;
8774
8775         if (netif_running(dev)) {
8776                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8777                 err = tg3_restart_hw(tp, 1);
8778                 if (!err)
8779                         tg3_netif_start(tp);
8780         }
8781
8782         tg3_full_unlock(tp);
8783
8784         return err;
8785 }
8786
8787 static u32 tg3_get_rx_csum(struct net_device *dev)
8788 {
8789         struct tg3 *tp = netdev_priv(dev);
8790         return (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0;
8791 }
8792
8793 static int tg3_set_rx_csum(struct net_device *dev, u32 data)
8794 {
8795         struct tg3 *tp = netdev_priv(dev);
8796
8797         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
8798                 if (data != 0)
8799                         return -EINVAL;
8800                 return 0;
8801         }
8802
8803         spin_lock_bh(&tp->lock);
8804         if (data)
8805                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
8806         else
8807                 tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
8808         spin_unlock_bh(&tp->lock);
8809
8810         return 0;
8811 }
8812
8813 static int tg3_set_tx_csum(struct net_device *dev, u32 data)
8814 {
8815         struct tg3 *tp = netdev_priv(dev);
8816
8817         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
8818                 if (data != 0)
8819                         return -EINVAL;
8820                 return 0;
8821         }
8822
8823         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
8824             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
8825             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
8826             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
8827                 ethtool_op_set_tx_ipv6_csum(dev, data);
8828         else
8829                 ethtool_op_set_tx_csum(dev, data);
8830
8831         return 0;
8832 }
8833
8834 static int tg3_get_sset_count (struct net_device *dev, int sset)
8835 {
8836         switch (sset) {
8837         case ETH_SS_TEST:
8838                 return TG3_NUM_TEST;
8839         case ETH_SS_STATS:
8840                 return TG3_NUM_STATS;
8841         default:
8842                 return -EOPNOTSUPP;
8843         }
8844 }
8845
8846 static void tg3_get_strings (struct net_device *dev, u32 stringset, u8 *buf)
8847 {
8848         switch (stringset) {
8849         case ETH_SS_STATS:
8850                 memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
8851                 break;
8852         case ETH_SS_TEST:
8853                 memcpy(buf, &ethtool_test_keys, sizeof(ethtool_test_keys));
8854                 break;
8855         default:
8856                 WARN_ON(1);     /* we need a WARN() */
8857                 break;
8858         }
8859 }
8860
8861 static int tg3_phys_id(struct net_device *dev, u32 data)
8862 {
8863         struct tg3 *tp = netdev_priv(dev);
8864         int i;
8865
8866         if (!netif_running(tp->dev))
8867                 return -EAGAIN;
8868
8869         if (data == 0)
8870                 data = UINT_MAX / 2;
8871
8872         for (i = 0; i < (data * 2); i++) {
8873                 if ((i % 2) == 0)
8874                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
8875                                            LED_CTRL_1000MBPS_ON |
8876                                            LED_CTRL_100MBPS_ON |
8877                                            LED_CTRL_10MBPS_ON |
8878                                            LED_CTRL_TRAFFIC_OVERRIDE |
8879                                            LED_CTRL_TRAFFIC_BLINK |
8880                                            LED_CTRL_TRAFFIC_LED);
8881
8882                 else
8883                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
8884                                            LED_CTRL_TRAFFIC_OVERRIDE);
8885
8886                 if (msleep_interruptible(500))
8887                         break;
8888         }
8889         tw32(MAC_LED_CTRL, tp->led_ctrl);
8890         return 0;
8891 }
8892
8893 static void tg3_get_ethtool_stats (struct net_device *dev,
8894                                    struct ethtool_stats *estats, u64 *tmp_stats)
8895 {
8896         struct tg3 *tp = netdev_priv(dev);
8897         memcpy(tmp_stats, tg3_get_estats(tp), sizeof(tp->estats));
8898 }
8899
8900 #define NVRAM_TEST_SIZE 0x100
8901 #define NVRAM_SELFBOOT_FORMAT1_0_SIZE   0x14
8902 #define NVRAM_SELFBOOT_FORMAT1_2_SIZE   0x18
8903 #define NVRAM_SELFBOOT_FORMAT1_3_SIZE   0x1c
8904 #define NVRAM_SELFBOOT_HW_SIZE 0x20
8905 #define NVRAM_SELFBOOT_DATA_SIZE 0x1c
8906
8907 static int tg3_test_nvram(struct tg3 *tp)
8908 {
8909         u32 csum, magic;
8910         __le32 *buf;
8911         int i, j, k, err = 0, size;
8912
8913         if (tg3_nvram_read_swab(tp, 0, &magic) != 0)
8914                 return -EIO;
8915
8916         if (magic == TG3_EEPROM_MAGIC)
8917                 size = NVRAM_TEST_SIZE;
8918         else if ((magic & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW) {
8919                 if ((magic & TG3_EEPROM_SB_FORMAT_MASK) ==
8920                     TG3_EEPROM_SB_FORMAT_1) {
8921                         switch (magic & TG3_EEPROM_SB_REVISION_MASK) {
8922                         case TG3_EEPROM_SB_REVISION_0:
8923                                 size = NVRAM_SELFBOOT_FORMAT1_0_SIZE;
8924                                 break;
8925                         case TG3_EEPROM_SB_REVISION_2:
8926                                 size = NVRAM_SELFBOOT_FORMAT1_2_SIZE;
8927                                 break;
8928                         case TG3_EEPROM_SB_REVISION_3:
8929                                 size = NVRAM_SELFBOOT_FORMAT1_3_SIZE;
8930                                 break;
8931                         default:
8932                                 return 0;
8933                         }
8934                 } else
8935                         return 0;
8936         } else if ((magic & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
8937                 size = NVRAM_SELFBOOT_HW_SIZE;
8938         else
8939                 return -EIO;
8940
8941         buf = kmalloc(size, GFP_KERNEL);
8942         if (buf == NULL)
8943                 return -ENOMEM;
8944
8945         err = -EIO;
8946         for (i = 0, j = 0; i < size; i += 4, j++) {
8947                 if ((err = tg3_nvram_read_le(tp, i, &buf[j])) != 0)
8948                         break;
8949         }
8950         if (i < size)
8951                 goto out;
8952
8953         /* Selfboot format */
8954         magic = swab32(le32_to_cpu(buf[0]));
8955         if ((magic & TG3_EEPROM_MAGIC_FW_MSK) ==
8956             TG3_EEPROM_MAGIC_FW) {
8957                 u8 *buf8 = (u8 *) buf, csum8 = 0;
8958
8959                 if ((magic & TG3_EEPROM_SB_REVISION_MASK) ==
8960                     TG3_EEPROM_SB_REVISION_2) {
8961                         /* For rev 2, the csum doesn't include the MBA. */
8962                         for (i = 0; i < TG3_EEPROM_SB_F1R2_MBA_OFF; i++)
8963                                 csum8 += buf8[i];
8964                         for (i = TG3_EEPROM_SB_F1R2_MBA_OFF + 4; i < size; i++)
8965                                 csum8 += buf8[i];
8966                 } else {
8967                         for (i = 0; i < size; i++)
8968                                 csum8 += buf8[i];
8969                 }
8970
8971                 if (csum8 == 0) {
8972                         err = 0;
8973                         goto out;
8974                 }
8975
8976                 err = -EIO;
8977                 goto out;
8978         }
8979
8980         if ((magic & TG3_EEPROM_MAGIC_HW_MSK) ==
8981             TG3_EEPROM_MAGIC_HW) {
8982                 u8 data[NVRAM_SELFBOOT_DATA_SIZE];
8983                 u8 parity[NVRAM_SELFBOOT_DATA_SIZE];
8984                 u8 *buf8 = (u8 *) buf;
8985
8986                 /* Separate the parity bits and the data bytes.  */
8987                 for (i = 0, j = 0, k = 0; i < NVRAM_SELFBOOT_HW_SIZE; i++) {
8988                         if ((i == 0) || (i == 8)) {
8989                                 int l;
8990                                 u8 msk;
8991
8992                                 for (l = 0, msk = 0x80; l < 7; l++, msk >>= 1)
8993                                         parity[k++] = buf8[i] & msk;
8994                                 i++;
8995                         }
8996                         else if (i == 16) {
8997                                 int l;
8998                                 u8 msk;
8999
9000                                 for (l = 0, msk = 0x20; l < 6; l++, msk >>= 1)
9001                                         parity[k++] = buf8[i] & msk;
9002                                 i++;
9003
9004                                 for (l = 0, msk = 0x80; l < 8; l++, msk >>= 1)
9005                                         parity[k++] = buf8[i] & msk;
9006                                 i++;
9007                         }
9008                         data[j++] = buf8[i];
9009                 }
9010
9011                 err = -EIO;
9012                 for (i = 0; i < NVRAM_SELFBOOT_DATA_SIZE; i++) {
9013                         u8 hw8 = hweight8(data[i]);
9014
9015                         if ((hw8 & 0x1) && parity[i])
9016                                 goto out;
9017                         else if (!(hw8 & 0x1) && !parity[i])
9018                                 goto out;
9019                 }
9020                 err = 0;
9021                 goto out;
9022         }
9023
9024         /* Bootstrap checksum at offset 0x10 */
9025         csum = calc_crc((unsigned char *) buf, 0x10);
9026         if(csum != le32_to_cpu(buf[0x10/4]))
9027                 goto out;
9028
9029         /* Manufacturing block starts at offset 0x74, checksum at 0xfc */
9030         csum = calc_crc((unsigned char *) &buf[0x74/4], 0x88);
9031         if (csum != le32_to_cpu(buf[0xfc/4]))
9032                  goto out;
9033
9034         err = 0;
9035
9036 out:
9037         kfree(buf);
9038         return err;
9039 }
9040
9041 #define TG3_SERDES_TIMEOUT_SEC  2
9042 #define TG3_COPPER_TIMEOUT_SEC  6
9043
9044 static int tg3_test_link(struct tg3 *tp)
9045 {
9046         int i, max;
9047
9048         if (!netif_running(tp->dev))
9049                 return -ENODEV;
9050
9051         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
9052                 max = TG3_SERDES_TIMEOUT_SEC;
9053         else
9054                 max = TG3_COPPER_TIMEOUT_SEC;
9055
9056         for (i = 0; i < max; i++) {
9057                 if (netif_carrier_ok(tp->dev))
9058                         return 0;
9059
9060                 if (msleep_interruptible(1000))
9061                         break;
9062         }
9063
9064         return -EIO;
9065 }
9066
9067 /* Only test the commonly used registers */
9068 static int tg3_test_registers(struct tg3 *tp)
9069 {
9070         int i, is_5705, is_5750;
9071         u32 offset, read_mask, write_mask, val, save_val, read_val;
9072         static struct {
9073                 u16 offset;
9074                 u16 flags;
9075 #define TG3_FL_5705     0x1
9076 #define TG3_FL_NOT_5705 0x2
9077 #define TG3_FL_NOT_5788 0x4
9078 #define TG3_FL_NOT_5750 0x8
9079                 u32 read_mask;
9080                 u32 write_mask;
9081         } reg_tbl[] = {
9082                 /* MAC Control Registers */
9083                 { MAC_MODE, TG3_FL_NOT_5705,
9084                         0x00000000, 0x00ef6f8c },
9085                 { MAC_MODE, TG3_FL_5705,
9086                         0x00000000, 0x01ef6b8c },
9087                 { MAC_STATUS, TG3_FL_NOT_5705,
9088                         0x03800107, 0x00000000 },
9089                 { MAC_STATUS, TG3_FL_5705,
9090                         0x03800100, 0x00000000 },
9091                 { MAC_ADDR_0_HIGH, 0x0000,
9092                         0x00000000, 0x0000ffff },
9093                 { MAC_ADDR_0_LOW, 0x0000,
9094                         0x00000000, 0xffffffff },
9095                 { MAC_RX_MTU_SIZE, 0x0000,
9096                         0x00000000, 0x0000ffff },
9097                 { MAC_TX_MODE, 0x0000,
9098                         0x00000000, 0x00000070 },
9099                 { MAC_TX_LENGTHS, 0x0000,
9100                         0x00000000, 0x00003fff },
9101                 { MAC_RX_MODE, TG3_FL_NOT_5705,
9102                         0x00000000, 0x000007fc },
9103                 { MAC_RX_MODE, TG3_FL_5705,
9104                         0x00000000, 0x000007dc },
9105                 { MAC_HASH_REG_0, 0x0000,
9106                         0x00000000, 0xffffffff },
9107                 { MAC_HASH_REG_1, 0x0000,
9108                         0x00000000, 0xffffffff },
9109                 { MAC_HASH_REG_2, 0x0000,
9110                         0x00000000, 0xffffffff },
9111                 { MAC_HASH_REG_3, 0x0000,
9112                         0x00000000, 0xffffffff },
9113
9114                 /* Receive Data and Receive BD Initiator Control Registers. */
9115                 { RCVDBDI_JUMBO_BD+0, TG3_FL_NOT_5705,
9116                         0x00000000, 0xffffffff },
9117                 { RCVDBDI_JUMBO_BD+4, TG3_FL_NOT_5705,
9118                         0x00000000, 0xffffffff },
9119                 { RCVDBDI_JUMBO_BD+8, TG3_FL_NOT_5705,
9120                         0x00000000, 0x00000003 },
9121                 { RCVDBDI_JUMBO_BD+0xc, TG3_FL_NOT_5705,
9122                         0x00000000, 0xffffffff },
9123                 { RCVDBDI_STD_BD+0, 0x0000,
9124                         0x00000000, 0xffffffff },
9125                 { RCVDBDI_STD_BD+4, 0x0000,
9126                         0x00000000, 0xffffffff },
9127                 { RCVDBDI_STD_BD+8, 0x0000,
9128                         0x00000000, 0xffff0002 },
9129                 { RCVDBDI_STD_BD+0xc, 0x0000,
9130                         0x00000000, 0xffffffff },
9131
9132                 /* Receive BD Initiator Control Registers. */
9133                 { RCVBDI_STD_THRESH, TG3_FL_NOT_5705,
9134                         0x00000000, 0xffffffff },
9135                 { RCVBDI_STD_THRESH, TG3_FL_5705,
9136                         0x00000000, 0x000003ff },
9137                 { RCVBDI_JUMBO_THRESH, TG3_FL_NOT_5705,
9138                         0x00000000, 0xffffffff },
9139
9140                 /* Host Coalescing Control Registers. */
9141                 { HOSTCC_MODE, TG3_FL_NOT_5705,
9142                         0x00000000, 0x00000004 },
9143                 { HOSTCC_MODE, TG3_FL_5705,
9144                         0x00000000, 0x000000f6 },
9145                 { HOSTCC_RXCOL_TICKS, TG3_FL_NOT_5705,
9146                         0x00000000, 0xffffffff },
9147                 { HOSTCC_RXCOL_TICKS, TG3_FL_5705,
9148                         0x00000000, 0x000003ff },
9149                 { HOSTCC_TXCOL_TICKS, TG3_FL_NOT_5705,
9150                         0x00000000, 0xffffffff },
9151                 { HOSTCC_TXCOL_TICKS, TG3_FL_5705,
9152                         0x00000000, 0x000003ff },
9153                 { HOSTCC_RXMAX_FRAMES, TG3_FL_NOT_5705,
9154                         0x00000000, 0xffffffff },
9155                 { HOSTCC_RXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
9156                         0x00000000, 0x000000ff },
9157                 { HOSTCC_TXMAX_FRAMES, TG3_FL_NOT_5705,
9158                         0x00000000, 0xffffffff },
9159                 { HOSTCC_TXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
9160                         0x00000000, 0x000000ff },
9161                 { HOSTCC_RXCOAL_TICK_INT, TG3_FL_NOT_5705,
9162                         0x00000000, 0xffffffff },
9163                 { HOSTCC_TXCOAL_TICK_INT, TG3_FL_NOT_5705,
9164                         0x00000000, 0xffffffff },
9165                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_NOT_5705,
9166                         0x00000000, 0xffffffff },
9167                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
9168                         0x00000000, 0x000000ff },
9169                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_NOT_5705,
9170                         0x00000000, 0xffffffff },
9171                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
9172                         0x00000000, 0x000000ff },
9173                 { HOSTCC_STAT_COAL_TICKS, TG3_FL_NOT_5705,
9174                         0x00000000, 0xffffffff },
9175                 { HOSTCC_STATS_BLK_HOST_ADDR, TG3_FL_NOT_5705,
9176                         0x00000000, 0xffffffff },
9177                 { HOSTCC_STATS_BLK_HOST_ADDR+4, TG3_FL_NOT_5705,
9178                         0x00000000, 0xffffffff },
9179                 { HOSTCC_STATUS_BLK_HOST_ADDR, 0x0000,
9180                         0x00000000, 0xffffffff },
9181                 { HOSTCC_STATUS_BLK_HOST_ADDR+4, 0x0000,
9182                         0x00000000, 0xffffffff },
9183                 { HOSTCC_STATS_BLK_NIC_ADDR, 0x0000,
9184                         0xffffffff, 0x00000000 },
9185                 { HOSTCC_STATUS_BLK_NIC_ADDR, 0x0000,
9186                         0xffffffff, 0x00000000 },
9187
9188                 /* Buffer Manager Control Registers. */
9189                 { BUFMGR_MB_POOL_ADDR, TG3_FL_NOT_5750,
9190                         0x00000000, 0x007fff80 },
9191                 { BUFMGR_MB_POOL_SIZE, TG3_FL_NOT_5750,
9192                         0x00000000, 0x007fffff },
9193                 { BUFMGR_MB_RDMA_LOW_WATER, 0x0000,
9194                         0x00000000, 0x0000003f },
9195                 { BUFMGR_MB_MACRX_LOW_WATER, 0x0000,
9196                         0x00000000, 0x000001ff },
9197                 { BUFMGR_MB_HIGH_WATER, 0x0000,
9198                         0x00000000, 0x000001ff },
9199                 { BUFMGR_DMA_DESC_POOL_ADDR, TG3_FL_NOT_5705,
9200                         0xffffffff, 0x00000000 },
9201                 { BUFMGR_DMA_DESC_POOL_SIZE, TG3_FL_NOT_5705,
9202                         0xffffffff, 0x00000000 },
9203
9204                 /* Mailbox Registers */
9205                 { GRCMBOX_RCVSTD_PROD_IDX+4, 0x0000,
9206                         0x00000000, 0x000001ff },
9207                 { GRCMBOX_RCVJUMBO_PROD_IDX+4, TG3_FL_NOT_5705,
9208                         0x00000000, 0x000001ff },
9209                 { GRCMBOX_RCVRET_CON_IDX_0+4, 0x0000,
9210                         0x00000000, 0x000007ff },
9211                 { GRCMBOX_SNDHOST_PROD_IDX_0+4, 0x0000,
9212                         0x00000000, 0x000001ff },
9213
9214                 { 0xffff, 0x0000, 0x00000000, 0x00000000 },
9215         };
9216
9217         is_5705 = is_5750 = 0;
9218         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
9219                 is_5705 = 1;
9220                 if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
9221                         is_5750 = 1;
9222         }
9223
9224         for (i = 0; reg_tbl[i].offset != 0xffff; i++) {
9225                 if (is_5705 && (reg_tbl[i].flags & TG3_FL_NOT_5705))
9226                         continue;
9227
9228                 if (!is_5705 && (reg_tbl[i].flags & TG3_FL_5705))
9229                         continue;
9230
9231                 if ((tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
9232                     (reg_tbl[i].flags & TG3_FL_NOT_5788))
9233                         continue;
9234
9235                 if (is_5750 && (reg_tbl[i].flags & TG3_FL_NOT_5750))
9236                         continue;
9237
9238                 offset = (u32) reg_tbl[i].offset;
9239                 read_mask = reg_tbl[i].read_mask;
9240                 write_mask = reg_tbl[i].write_mask;
9241
9242                 /* Save the original register content */
9243                 save_val = tr32(offset);
9244
9245                 /* Determine the read-only value. */
9246                 read_val = save_val & read_mask;
9247
9248                 /* Write zero to the register, then make sure the read-only bits
9249                  * are not changed and the read/write bits are all zeros.
9250                  */
9251                 tw32(offset, 0);
9252
9253                 val = tr32(offset);
9254
9255                 /* Test the read-only and read/write bits. */
9256                 if (((val & read_mask) != read_val) || (val & write_mask))
9257                         goto out;
9258
9259                 /* Write ones to all the bits defined by RdMask and WrMask, then
9260                  * make sure the read-only bits are not changed and the
9261                  * read/write bits are all ones.
9262                  */
9263                 tw32(offset, read_mask | write_mask);
9264
9265                 val = tr32(offset);
9266
9267                 /* Test the read-only bits. */
9268                 if ((val & read_mask) != read_val)
9269                         goto out;
9270
9271                 /* Test the read/write bits. */
9272                 if ((val & write_mask) != write_mask)
9273                         goto out;
9274
9275                 tw32(offset, save_val);
9276         }
9277
9278         return 0;
9279
9280 out:
9281         if (netif_msg_hw(tp))
9282                 printk(KERN_ERR PFX "Register test failed at offset %x\n",
9283                        offset);
9284         tw32(offset, save_val);
9285         return -EIO;
9286 }
9287
9288 static int tg3_do_mem_test(struct tg3 *tp, u32 offset, u32 len)
9289 {
9290         static const u32 test_pattern[] = { 0x00000000, 0xffffffff, 0xaa55a55a };
9291         int i;
9292         u32 j;
9293
9294         for (i = 0; i < ARRAY_SIZE(test_pattern); i++) {
9295                 for (j = 0; j < len; j += 4) {
9296                         u32 val;
9297
9298                         tg3_write_mem(tp, offset + j, test_pattern[i]);
9299                         tg3_read_mem(tp, offset + j, &val);
9300                         if (val != test_pattern[i])
9301                                 return -EIO;
9302                 }
9303         }
9304         return 0;
9305 }
9306
9307 static int tg3_test_memory(struct tg3 *tp)
9308 {
9309         static struct mem_entry {
9310                 u32 offset;
9311                 u32 len;
9312         } mem_tbl_570x[] = {
9313                 { 0x00000000, 0x00b50},
9314                 { 0x00002000, 0x1c000},
9315                 { 0xffffffff, 0x00000}
9316         }, mem_tbl_5705[] = {
9317                 { 0x00000100, 0x0000c},
9318                 { 0x00000200, 0x00008},
9319                 { 0x00004000, 0x00800},
9320                 { 0x00006000, 0x01000},
9321                 { 0x00008000, 0x02000},
9322                 { 0x00010000, 0x0e000},
9323                 { 0xffffffff, 0x00000}
9324         }, mem_tbl_5755[] = {
9325                 { 0x00000200, 0x00008},
9326                 { 0x00004000, 0x00800},
9327                 { 0x00006000, 0x00800},
9328                 { 0x00008000, 0x02000},
9329                 { 0x00010000, 0x0c000},
9330                 { 0xffffffff, 0x00000}
9331         }, mem_tbl_5906[] = {
9332                 { 0x00000200, 0x00008},
9333                 { 0x00004000, 0x00400},
9334                 { 0x00006000, 0x00400},
9335                 { 0x00008000, 0x01000},
9336                 { 0x00010000, 0x01000},
9337                 { 0xffffffff, 0x00000}
9338         };
9339         struct mem_entry *mem_tbl;
9340         int err = 0;
9341         int i;
9342
9343         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
9344                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
9345                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
9346                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
9347                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
9348                         mem_tbl = mem_tbl_5755;
9349                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
9350                         mem_tbl = mem_tbl_5906;
9351                 else
9352                         mem_tbl = mem_tbl_5705;
9353         } else
9354                 mem_tbl = mem_tbl_570x;
9355
9356         for (i = 0; mem_tbl[i].offset != 0xffffffff; i++) {
9357                 if ((err = tg3_do_mem_test(tp, mem_tbl[i].offset,
9358                     mem_tbl[i].len)) != 0)
9359                         break;
9360         }
9361
9362         return err;
9363 }
9364
9365 #define TG3_MAC_LOOPBACK        0
9366 #define TG3_PHY_LOOPBACK        1
9367
9368 static int tg3_run_loopback(struct tg3 *tp, int loopback_mode)
9369 {
9370         u32 mac_mode, rx_start_idx, rx_idx, tx_idx, opaque_key;
9371         u32 desc_idx;
9372         struct sk_buff *skb, *rx_skb;
9373         u8 *tx_data;
9374         dma_addr_t map;
9375         int num_pkts, tx_len, rx_len, i, err;
9376         struct tg3_rx_buffer_desc *desc;
9377
9378         if (loopback_mode == TG3_MAC_LOOPBACK) {
9379                 /* HW errata - mac loopback fails in some cases on 5780.
9380                  * Normal traffic and PHY loopback are not affected by
9381                  * errata.
9382                  */
9383                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780)
9384                         return 0;
9385
9386                 mac_mode = (tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK) |
9387                            MAC_MODE_PORT_INT_LPBACK;
9388                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
9389                         mac_mode |= MAC_MODE_LINK_POLARITY;
9390                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
9391                         mac_mode |= MAC_MODE_PORT_MODE_MII;
9392                 else
9393                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
9394                 tw32(MAC_MODE, mac_mode);
9395         } else if (loopback_mode == TG3_PHY_LOOPBACK) {
9396                 u32 val;
9397
9398                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
9399                         u32 phytest;
9400
9401                         if (!tg3_readphy(tp, MII_TG3_EPHY_TEST, &phytest)) {
9402                                 u32 phy;
9403
9404                                 tg3_writephy(tp, MII_TG3_EPHY_TEST,
9405                                              phytest | MII_TG3_EPHY_SHADOW_EN);
9406                                 if (!tg3_readphy(tp, 0x1b, &phy))
9407                                         tg3_writephy(tp, 0x1b, phy & ~0x20);
9408                                 tg3_writephy(tp, MII_TG3_EPHY_TEST, phytest);
9409                         }
9410                         val = BMCR_LOOPBACK | BMCR_FULLDPLX | BMCR_SPEED100;
9411                 } else
9412                         val = BMCR_LOOPBACK | BMCR_FULLDPLX | BMCR_SPEED1000;
9413
9414                 tg3_phy_toggle_automdix(tp, 0);
9415
9416                 tg3_writephy(tp, MII_BMCR, val);
9417                 udelay(40);
9418
9419                 mac_mode = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
9420                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
9421                         tg3_writephy(tp, MII_TG3_EPHY_PTEST, 0x1800);
9422                         mac_mode |= MAC_MODE_PORT_MODE_MII;
9423                 } else
9424                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
9425
9426                 /* reset to prevent losing 1st rx packet intermittently */
9427                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
9428                         tw32_f(MAC_RX_MODE, RX_MODE_RESET);
9429                         udelay(10);
9430                         tw32_f(MAC_RX_MODE, tp->rx_mode);
9431                 }
9432                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
9433                         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401)
9434                                 mac_mode &= ~MAC_MODE_LINK_POLARITY;
9435                         else if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411)
9436                                 mac_mode |= MAC_MODE_LINK_POLARITY;
9437                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
9438                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
9439                 }
9440                 tw32(MAC_MODE, mac_mode);
9441         }
9442         else
9443                 return -EINVAL;
9444
9445         err = -EIO;
9446
9447         tx_len = 1514;
9448         skb = netdev_alloc_skb(tp->dev, tx_len);
9449         if (!skb)
9450                 return -ENOMEM;
9451
9452         tx_data = skb_put(skb, tx_len);
9453         memcpy(tx_data, tp->dev->dev_addr, 6);
9454         memset(tx_data + 6, 0x0, 8);
9455
9456         tw32(MAC_RX_MTU_SIZE, tx_len + 4);
9457
9458         for (i = 14; i < tx_len; i++)
9459                 tx_data[i] = (u8) (i & 0xff);
9460
9461         map = pci_map_single(tp->pdev, skb->data, tx_len, PCI_DMA_TODEVICE);
9462
9463         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
9464              HOSTCC_MODE_NOW);
9465
9466         udelay(10);
9467
9468         rx_start_idx = tp->hw_status->idx[0].rx_producer;
9469
9470         num_pkts = 0;
9471
9472         tg3_set_txd(tp, tp->tx_prod, map, tx_len, 0, 1);
9473
9474         tp->tx_prod++;
9475         num_pkts++;
9476
9477         tw32_tx_mbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW,
9478                      tp->tx_prod);
9479         tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW);
9480
9481         udelay(10);
9482
9483         /* 250 usec to allow enough time on some 10/100 Mbps devices.  */
9484         for (i = 0; i < 25; i++) {
9485                 tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
9486                        HOSTCC_MODE_NOW);
9487
9488                 udelay(10);
9489
9490                 tx_idx = tp->hw_status->idx[0].tx_consumer;
9491                 rx_idx = tp->hw_status->idx[0].rx_producer;
9492                 if ((tx_idx == tp->tx_prod) &&
9493                     (rx_idx == (rx_start_idx + num_pkts)))
9494                         break;
9495         }
9496
9497         pci_unmap_single(tp->pdev, map, tx_len, PCI_DMA_TODEVICE);
9498         dev_kfree_skb(skb);
9499
9500         if (tx_idx != tp->tx_prod)
9501                 goto out;
9502
9503         if (rx_idx != rx_start_idx + num_pkts)
9504                 goto out;
9505
9506         desc = &tp->rx_rcb[rx_start_idx];
9507         desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
9508         opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
9509         if (opaque_key != RXD_OPAQUE_RING_STD)
9510                 goto out;
9511
9512         if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
9513             (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII))
9514                 goto out;
9515
9516         rx_len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4;
9517         if (rx_len != tx_len)
9518                 goto out;
9519
9520         rx_skb = tp->rx_std_buffers[desc_idx].skb;
9521
9522         map = pci_unmap_addr(&tp->rx_std_buffers[desc_idx], mapping);
9523         pci_dma_sync_single_for_cpu(tp->pdev, map, rx_len, PCI_DMA_FROMDEVICE);
9524
9525         for (i = 14; i < tx_len; i++) {
9526                 if (*(rx_skb->data + i) != (u8) (i & 0xff))
9527                         goto out;
9528         }
9529         err = 0;
9530
9531         /* tg3_free_rings will unmap and free the rx_skb */
9532 out:
9533         return err;
9534 }
9535
9536 #define TG3_MAC_LOOPBACK_FAILED         1
9537 #define TG3_PHY_LOOPBACK_FAILED         2
9538 #define TG3_LOOPBACK_FAILED             (TG3_MAC_LOOPBACK_FAILED |      \
9539                                          TG3_PHY_LOOPBACK_FAILED)
9540
9541 static int tg3_test_loopback(struct tg3 *tp)
9542 {
9543         int err = 0;
9544         u32 cpmuctrl = 0;
9545
9546         if (!netif_running(tp->dev))
9547                 return TG3_LOOPBACK_FAILED;
9548
9549         err = tg3_reset_hw(tp, 1);
9550         if (err)
9551                 return TG3_LOOPBACK_FAILED;
9552
9553         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
9554             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761) {
9555                 int i;
9556                 u32 status;
9557
9558                 tw32(TG3_CPMU_MUTEX_REQ, CPMU_MUTEX_REQ_DRIVER);
9559
9560                 /* Wait for up to 40 microseconds to acquire lock. */
9561                 for (i = 0; i < 4; i++) {
9562                         status = tr32(TG3_CPMU_MUTEX_GNT);
9563                         if (status == CPMU_MUTEX_GNT_DRIVER)
9564                                 break;
9565                         udelay(10);
9566                 }
9567
9568                 if (status != CPMU_MUTEX_GNT_DRIVER)
9569                         return TG3_LOOPBACK_FAILED;
9570
9571                 /* Turn off link-based power management. */
9572                 cpmuctrl = tr32(TG3_CPMU_CTRL);
9573                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
9574                     GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5761_AX)
9575                         tw32(TG3_CPMU_CTRL,
9576                              cpmuctrl & ~(CPMU_CTRL_LINK_SPEED_MODE |
9577                                           CPMU_CTRL_LINK_AWARE_MODE));
9578                 else
9579                         tw32(TG3_CPMU_CTRL,
9580                              cpmuctrl & ~CPMU_CTRL_LINK_AWARE_MODE);
9581         }
9582
9583         if (tg3_run_loopback(tp, TG3_MAC_LOOPBACK))
9584                 err |= TG3_MAC_LOOPBACK_FAILED;
9585
9586         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
9587             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761) {
9588                 tw32(TG3_CPMU_CTRL, cpmuctrl);
9589
9590                 /* Release the mutex */
9591                 tw32(TG3_CPMU_MUTEX_GNT, CPMU_MUTEX_GNT_DRIVER);
9592         }
9593
9594         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
9595                 if (tg3_run_loopback(tp, TG3_PHY_LOOPBACK))
9596                         err |= TG3_PHY_LOOPBACK_FAILED;
9597         }
9598
9599         return err;
9600 }
9601
9602 static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest,
9603                           u64 *data)
9604 {
9605         struct tg3 *tp = netdev_priv(dev);
9606
9607         if (tp->link_config.phy_is_low_power)
9608                 tg3_set_power_state(tp, PCI_D0);
9609
9610         memset(data, 0, sizeof(u64) * TG3_NUM_TEST);
9611
9612         if (tg3_test_nvram(tp) != 0) {
9613                 etest->flags |= ETH_TEST_FL_FAILED;
9614                 data[0] = 1;
9615         }
9616         if (tg3_test_link(tp) != 0) {
9617                 etest->flags |= ETH_TEST_FL_FAILED;
9618                 data[1] = 1;
9619         }
9620         if (etest->flags & ETH_TEST_FL_OFFLINE) {
9621                 int err, irq_sync = 0;
9622
9623                 if (netif_running(dev)) {
9624                         tg3_netif_stop(tp);
9625                         irq_sync = 1;
9626                 }
9627
9628                 tg3_full_lock(tp, irq_sync);
9629
9630                 tg3_halt(tp, RESET_KIND_SUSPEND, 1);
9631                 err = tg3_nvram_lock(tp);
9632                 tg3_halt_cpu(tp, RX_CPU_BASE);
9633                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
9634                         tg3_halt_cpu(tp, TX_CPU_BASE);
9635                 if (!err)
9636                         tg3_nvram_unlock(tp);
9637
9638                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
9639                         tg3_phy_reset(tp);
9640
9641                 if (tg3_test_registers(tp) != 0) {
9642                         etest->flags |= ETH_TEST_FL_FAILED;
9643                         data[2] = 1;
9644                 }
9645                 if (tg3_test_memory(tp) != 0) {
9646                         etest->flags |= ETH_TEST_FL_FAILED;
9647                         data[3] = 1;
9648                 }
9649                 if ((data[4] = tg3_test_loopback(tp)) != 0)
9650                         etest->flags |= ETH_TEST_FL_FAILED;
9651
9652                 tg3_full_unlock(tp);
9653
9654                 if (tg3_test_interrupt(tp) != 0) {
9655                         etest->flags |= ETH_TEST_FL_FAILED;
9656                         data[5] = 1;
9657                 }
9658
9659                 tg3_full_lock(tp, 0);
9660
9661                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
9662                 if (netif_running(dev)) {
9663                         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
9664                         if (!tg3_restart_hw(tp, 1))
9665                                 tg3_netif_start(tp);
9666                 }
9667
9668                 tg3_full_unlock(tp);
9669         }
9670         if (tp->link_config.phy_is_low_power)
9671                 tg3_set_power_state(tp, PCI_D3hot);
9672
9673 }
9674
9675 static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
9676 {
9677         struct mii_ioctl_data *data = if_mii(ifr);
9678         struct tg3 *tp = netdev_priv(dev);
9679         int err;
9680
9681         switch(cmd) {
9682         case SIOCGMIIPHY:
9683                 data->phy_id = PHY_ADDR;
9684
9685                 /* fallthru */
9686         case SIOCGMIIREG: {
9687                 u32 mii_regval;
9688
9689                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
9690                         break;                  /* We have no PHY */
9691
9692                 if (tp->link_config.phy_is_low_power)
9693                         return -EAGAIN;
9694
9695                 spin_lock_bh(&tp->lock);
9696                 err = tg3_readphy(tp, data->reg_num & 0x1f, &mii_regval);
9697                 spin_unlock_bh(&tp->lock);
9698
9699                 data->val_out = mii_regval;
9700
9701                 return err;
9702         }
9703
9704         case SIOCSMIIREG:
9705                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
9706                         break;                  /* We have no PHY */
9707
9708                 if (!capable(CAP_NET_ADMIN))
9709                         return -EPERM;
9710
9711                 if (tp->link_config.phy_is_low_power)
9712                         return -EAGAIN;
9713
9714                 spin_lock_bh(&tp->lock);
9715                 err = tg3_writephy(tp, data->reg_num & 0x1f, data->val_in);
9716                 spin_unlock_bh(&tp->lock);
9717
9718                 return err;
9719
9720         default:
9721                 /* do nothing */
9722                 break;
9723         }
9724         return -EOPNOTSUPP;
9725 }
9726
9727 #if TG3_VLAN_TAG_USED
9728 static void tg3_vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
9729 {
9730         struct tg3 *tp = netdev_priv(dev);
9731
9732         if (netif_running(dev))
9733                 tg3_netif_stop(tp);
9734
9735         tg3_full_lock(tp, 0);
9736
9737         tp->vlgrp = grp;
9738
9739         /* Update RX_MODE_KEEP_VLAN_TAG bit in RX_MODE register. */
9740         __tg3_set_rx_mode(dev);
9741
9742         if (netif_running(dev))
9743                 tg3_netif_start(tp);
9744
9745         tg3_full_unlock(tp);
9746 }
9747 #endif
9748
9749 static int tg3_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
9750 {
9751         struct tg3 *tp = netdev_priv(dev);
9752
9753         memcpy(ec, &tp->coal, sizeof(*ec));
9754         return 0;
9755 }
9756
9757 static int tg3_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
9758 {
9759         struct tg3 *tp = netdev_priv(dev);
9760         u32 max_rxcoal_tick_int = 0, max_txcoal_tick_int = 0;
9761         u32 max_stat_coal_ticks = 0, min_stat_coal_ticks = 0;
9762
9763         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
9764                 max_rxcoal_tick_int = MAX_RXCOAL_TICK_INT;
9765                 max_txcoal_tick_int = MAX_TXCOAL_TICK_INT;
9766                 max_stat_coal_ticks = MAX_STAT_COAL_TICKS;
9767                 min_stat_coal_ticks = MIN_STAT_COAL_TICKS;
9768         }
9769
9770         if ((ec->rx_coalesce_usecs > MAX_RXCOL_TICKS) ||
9771             (ec->tx_coalesce_usecs > MAX_TXCOL_TICKS) ||
9772             (ec->rx_max_coalesced_frames > MAX_RXMAX_FRAMES) ||
9773             (ec->tx_max_coalesced_frames > MAX_TXMAX_FRAMES) ||
9774             (ec->rx_coalesce_usecs_irq > max_rxcoal_tick_int) ||
9775             (ec->tx_coalesce_usecs_irq > max_txcoal_tick_int) ||
9776             (ec->rx_max_coalesced_frames_irq > MAX_RXCOAL_MAXF_INT) ||
9777             (ec->tx_max_coalesced_frames_irq > MAX_TXCOAL_MAXF_INT) ||
9778             (ec->stats_block_coalesce_usecs > max_stat_coal_ticks) ||
9779             (ec->stats_block_coalesce_usecs < min_stat_coal_ticks))
9780                 return -EINVAL;
9781
9782         /* No rx interrupts will be generated if both are zero */
9783         if ((ec->rx_coalesce_usecs == 0) &&
9784             (ec->rx_max_coalesced_frames == 0))
9785                 return -EINVAL;
9786
9787         /* No tx interrupts will be generated if both are zero */
9788         if ((ec->tx_coalesce_usecs == 0) &&
9789             (ec->tx_max_coalesced_frames == 0))
9790                 return -EINVAL;
9791
9792         /* Only copy relevant parameters, ignore all others. */
9793         tp->coal.rx_coalesce_usecs = ec->rx_coalesce_usecs;
9794         tp->coal.tx_coalesce_usecs = ec->tx_coalesce_usecs;
9795         tp->coal.rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
9796         tp->coal.tx_max_coalesced_frames = ec->tx_max_coalesced_frames;
9797         tp->coal.rx_coalesce_usecs_irq = ec->rx_coalesce_usecs_irq;
9798         tp->coal.tx_coalesce_usecs_irq = ec->tx_coalesce_usecs_irq;
9799         tp->coal.rx_max_coalesced_frames_irq = ec->rx_max_coalesced_frames_irq;
9800         tp->coal.tx_max_coalesced_frames_irq = ec->tx_max_coalesced_frames_irq;
9801         tp->coal.stats_block_coalesce_usecs = ec->stats_block_coalesce_usecs;
9802
9803         if (netif_running(dev)) {
9804                 tg3_full_lock(tp, 0);
9805                 __tg3_set_coalesce(tp, &tp->coal);
9806                 tg3_full_unlock(tp);
9807         }
9808         return 0;
9809 }
9810
9811 static const struct ethtool_ops tg3_ethtool_ops = {
9812         .get_settings           = tg3_get_settings,
9813         .set_settings           = tg3_set_settings,
9814         .get_drvinfo            = tg3_get_drvinfo,
9815         .get_regs_len           = tg3_get_regs_len,
9816         .get_regs               = tg3_get_regs,
9817         .get_wol                = tg3_get_wol,
9818         .set_wol                = tg3_set_wol,
9819         .get_msglevel           = tg3_get_msglevel,
9820         .set_msglevel           = tg3_set_msglevel,
9821         .nway_reset             = tg3_nway_reset,
9822         .get_link               = ethtool_op_get_link,
9823         .get_eeprom_len         = tg3_get_eeprom_len,
9824         .get_eeprom             = tg3_get_eeprom,
9825         .set_eeprom             = tg3_set_eeprom,
9826         .get_ringparam          = tg3_get_ringparam,
9827         .set_ringparam          = tg3_set_ringparam,
9828         .get_pauseparam         = tg3_get_pauseparam,
9829         .set_pauseparam         = tg3_set_pauseparam,
9830         .get_rx_csum            = tg3_get_rx_csum,
9831         .set_rx_csum            = tg3_set_rx_csum,
9832         .set_tx_csum            = tg3_set_tx_csum,
9833         .set_sg                 = ethtool_op_set_sg,
9834         .set_tso                = tg3_set_tso,
9835         .self_test              = tg3_self_test,
9836         .get_strings            = tg3_get_strings,
9837         .phys_id                = tg3_phys_id,
9838         .get_ethtool_stats      = tg3_get_ethtool_stats,
9839         .get_coalesce           = tg3_get_coalesce,
9840         .set_coalesce           = tg3_set_coalesce,
9841         .get_sset_count         = tg3_get_sset_count,
9842 };
9843
9844 static void __devinit tg3_get_eeprom_size(struct tg3 *tp)
9845 {
9846         u32 cursize, val, magic;
9847
9848         tp->nvram_size = EEPROM_CHIP_SIZE;
9849
9850         if (tg3_nvram_read_swab(tp, 0, &magic) != 0)
9851                 return;
9852
9853         if ((magic != TG3_EEPROM_MAGIC) &&
9854             ((magic & TG3_EEPROM_MAGIC_FW_MSK) != TG3_EEPROM_MAGIC_FW) &&
9855             ((magic & TG3_EEPROM_MAGIC_HW_MSK) != TG3_EEPROM_MAGIC_HW))
9856                 return;
9857
9858         /*
9859          * Size the chip by reading offsets at increasing powers of two.
9860          * When we encounter our validation signature, we know the addressing
9861          * has wrapped around, and thus have our chip size.
9862          */
9863         cursize = 0x10;
9864
9865         while (cursize < tp->nvram_size) {
9866                 if (tg3_nvram_read_swab(tp, cursize, &val) != 0)
9867                         return;
9868
9869                 if (val == magic)
9870                         break;
9871
9872                 cursize <<= 1;
9873         }
9874
9875         tp->nvram_size = cursize;
9876 }
9877
9878 static void __devinit tg3_get_nvram_size(struct tg3 *tp)
9879 {
9880         u32 val;
9881
9882         if (tg3_nvram_read_swab(tp, 0, &val) != 0)
9883                 return;
9884
9885         /* Selfboot format */
9886         if (val != TG3_EEPROM_MAGIC) {
9887                 tg3_get_eeprom_size(tp);
9888                 return;
9889         }
9890
9891         if (tg3_nvram_read(tp, 0xf0, &val) == 0) {
9892                 if (val != 0) {
9893                         tp->nvram_size = (val >> 16) * 1024;
9894                         return;
9895                 }
9896         }
9897         tp->nvram_size = 0x80000;
9898 }
9899
9900 static void __devinit tg3_get_nvram_info(struct tg3 *tp)
9901 {
9902         u32 nvcfg1;
9903
9904         nvcfg1 = tr32(NVRAM_CFG1);
9905         if (nvcfg1 & NVRAM_CFG1_FLASHIF_ENAB) {
9906                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
9907         }
9908         else {
9909                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9910                 tw32(NVRAM_CFG1, nvcfg1);
9911         }
9912
9913         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) ||
9914             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
9915                 switch (nvcfg1 & NVRAM_CFG1_VENDOR_MASK) {
9916                         case FLASH_VENDOR_ATMEL_FLASH_BUFFERED:
9917                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9918                                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
9919                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9920                                 break;
9921                         case FLASH_VENDOR_ATMEL_FLASH_UNBUFFERED:
9922                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9923                                 tp->nvram_pagesize = ATMEL_AT25F512_PAGE_SIZE;
9924                                 break;
9925                         case FLASH_VENDOR_ATMEL_EEPROM:
9926                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9927                                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9928                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9929                                 break;
9930                         case FLASH_VENDOR_ST:
9931                                 tp->nvram_jedecnum = JEDEC_ST;
9932                                 tp->nvram_pagesize = ST_M45PEX0_PAGE_SIZE;
9933                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9934                                 break;
9935                         case FLASH_VENDOR_SAIFUN:
9936                                 tp->nvram_jedecnum = JEDEC_SAIFUN;
9937                                 tp->nvram_pagesize = SAIFUN_SA25F0XX_PAGE_SIZE;
9938                                 break;
9939                         case FLASH_VENDOR_SST_SMALL:
9940                         case FLASH_VENDOR_SST_LARGE:
9941                                 tp->nvram_jedecnum = JEDEC_SST;
9942                                 tp->nvram_pagesize = SST_25VF0X0_PAGE_SIZE;
9943                                 break;
9944                 }
9945         }
9946         else {
9947                 tp->nvram_jedecnum = JEDEC_ATMEL;
9948                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
9949                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9950         }
9951 }
9952
9953 static void __devinit tg3_get_5752_nvram_info(struct tg3 *tp)
9954 {
9955         u32 nvcfg1;
9956
9957         nvcfg1 = tr32(NVRAM_CFG1);
9958
9959         /* NVRAM protection for TPM */
9960         if (nvcfg1 & (1 << 27))
9961                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
9962
9963         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
9964                 case FLASH_5752VENDOR_ATMEL_EEPROM_64KHZ:
9965                 case FLASH_5752VENDOR_ATMEL_EEPROM_376KHZ:
9966                         tp->nvram_jedecnum = JEDEC_ATMEL;
9967                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9968                         break;
9969                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
9970                         tp->nvram_jedecnum = JEDEC_ATMEL;
9971                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9972                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9973                         break;
9974                 case FLASH_5752VENDOR_ST_M45PE10:
9975                 case FLASH_5752VENDOR_ST_M45PE20:
9976                 case FLASH_5752VENDOR_ST_M45PE40:
9977                         tp->nvram_jedecnum = JEDEC_ST;
9978                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9979                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9980                         break;
9981         }
9982
9983         if (tp->tg3_flags2 & TG3_FLG2_FLASH) {
9984                 switch (nvcfg1 & NVRAM_CFG1_5752PAGE_SIZE_MASK) {
9985                         case FLASH_5752PAGE_SIZE_256:
9986                                 tp->nvram_pagesize = 256;
9987                                 break;
9988                         case FLASH_5752PAGE_SIZE_512:
9989                                 tp->nvram_pagesize = 512;
9990                                 break;
9991                         case FLASH_5752PAGE_SIZE_1K:
9992                                 tp->nvram_pagesize = 1024;
9993                                 break;
9994                         case FLASH_5752PAGE_SIZE_2K:
9995                                 tp->nvram_pagesize = 2048;
9996                                 break;
9997                         case FLASH_5752PAGE_SIZE_4K:
9998                                 tp->nvram_pagesize = 4096;
9999                                 break;
10000                         case FLASH_5752PAGE_SIZE_264:
10001                                 tp->nvram_pagesize = 264;
10002                                 break;
10003                 }
10004         }
10005         else {
10006                 /* For eeprom, set pagesize to maximum eeprom size */
10007                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
10008
10009                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
10010                 tw32(NVRAM_CFG1, nvcfg1);
10011         }
10012 }
10013
10014 static void __devinit tg3_get_5755_nvram_info(struct tg3 *tp)
10015 {
10016         u32 nvcfg1, protect = 0;
10017
10018         nvcfg1 = tr32(NVRAM_CFG1);
10019
10020         /* NVRAM protection for TPM */
10021         if (nvcfg1 & (1 << 27)) {
10022                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
10023                 protect = 1;
10024         }
10025
10026         nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
10027         switch (nvcfg1) {
10028                 case FLASH_5755VENDOR_ATMEL_FLASH_1:
10029                 case FLASH_5755VENDOR_ATMEL_FLASH_2:
10030                 case FLASH_5755VENDOR_ATMEL_FLASH_3:
10031                 case FLASH_5755VENDOR_ATMEL_FLASH_5:
10032                         tp->nvram_jedecnum = JEDEC_ATMEL;
10033                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10034                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
10035                         tp->nvram_pagesize = 264;
10036                         if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_1 ||
10037                             nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_5)
10038                                 tp->nvram_size = (protect ? 0x3e200 : 0x80000);
10039                         else if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_2)
10040                                 tp->nvram_size = (protect ? 0x1f200 : 0x40000);
10041                         else
10042                                 tp->nvram_size = (protect ? 0x1f200 : 0x20000);
10043                         break;
10044                 case FLASH_5752VENDOR_ST_M45PE10:
10045                 case FLASH_5752VENDOR_ST_M45PE20:
10046                 case FLASH_5752VENDOR_ST_M45PE40:
10047                         tp->nvram_jedecnum = JEDEC_ST;
10048                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10049                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
10050                         tp->nvram_pagesize = 256;
10051                         if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE10)
10052                                 tp->nvram_size = (protect ? 0x10000 : 0x20000);
10053                         else if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE20)
10054                                 tp->nvram_size = (protect ? 0x10000 : 0x40000);
10055                         else
10056                                 tp->nvram_size = (protect ? 0x20000 : 0x80000);
10057                         break;
10058         }
10059 }
10060
10061 static void __devinit tg3_get_5787_nvram_info(struct tg3 *tp)
10062 {
10063         u32 nvcfg1;
10064
10065         nvcfg1 = tr32(NVRAM_CFG1);
10066
10067         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
10068                 case FLASH_5787VENDOR_ATMEL_EEPROM_64KHZ:
10069                 case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
10070                 case FLASH_5787VENDOR_MICRO_EEPROM_64KHZ:
10071                 case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
10072                         tp->nvram_jedecnum = JEDEC_ATMEL;
10073                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10074                         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
10075
10076                         nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
10077                         tw32(NVRAM_CFG1, nvcfg1);
10078                         break;
10079                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
10080                 case FLASH_5755VENDOR_ATMEL_FLASH_1:
10081                 case FLASH_5755VENDOR_ATMEL_FLASH_2:
10082                 case FLASH_5755VENDOR_ATMEL_FLASH_3:
10083                         tp->nvram_jedecnum = JEDEC_ATMEL;
10084                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10085                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
10086                         tp->nvram_pagesize = 264;
10087                         break;
10088                 case FLASH_5752VENDOR_ST_M45PE10:
10089                 case FLASH_5752VENDOR_ST_M45PE20:
10090                 case FLASH_5752VENDOR_ST_M45PE40:
10091                         tp->nvram_jedecnum = JEDEC_ST;
10092                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10093                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
10094                         tp->nvram_pagesize = 256;
10095                         break;
10096         }
10097 }
10098
10099 static void __devinit tg3_get_5761_nvram_info(struct tg3 *tp)
10100 {
10101         u32 nvcfg1, protect = 0;
10102
10103         nvcfg1 = tr32(NVRAM_CFG1);
10104
10105         /* NVRAM protection for TPM */
10106         if (nvcfg1 & (1 << 27)) {
10107                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
10108                 protect = 1;
10109         }
10110
10111         nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
10112         switch (nvcfg1) {
10113                 case FLASH_5761VENDOR_ATMEL_ADB021D:
10114                 case FLASH_5761VENDOR_ATMEL_ADB041D:
10115                 case FLASH_5761VENDOR_ATMEL_ADB081D:
10116                 case FLASH_5761VENDOR_ATMEL_ADB161D:
10117                 case FLASH_5761VENDOR_ATMEL_MDB021D:
10118                 case FLASH_5761VENDOR_ATMEL_MDB041D:
10119                 case FLASH_5761VENDOR_ATMEL_MDB081D:
10120                 case FLASH_5761VENDOR_ATMEL_MDB161D:
10121                         tp->nvram_jedecnum = JEDEC_ATMEL;
10122                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10123                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
10124                         tp->tg3_flags3 |= TG3_FLG3_NO_NVRAM_ADDR_TRANS;
10125                         tp->nvram_pagesize = 256;
10126                         break;
10127                 case FLASH_5761VENDOR_ST_A_M45PE20:
10128                 case FLASH_5761VENDOR_ST_A_M45PE40:
10129                 case FLASH_5761VENDOR_ST_A_M45PE80:
10130                 case FLASH_5761VENDOR_ST_A_M45PE16:
10131                 case FLASH_5761VENDOR_ST_M_M45PE20:
10132                 case FLASH_5761VENDOR_ST_M_M45PE40:
10133                 case FLASH_5761VENDOR_ST_M_M45PE80:
10134                 case FLASH_5761VENDOR_ST_M_M45PE16:
10135                         tp->nvram_jedecnum = JEDEC_ST;
10136                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10137                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
10138                         tp->nvram_pagesize = 256;
10139                         break;
10140         }
10141
10142         if (protect) {
10143                 tp->nvram_size = tr32(NVRAM_ADDR_LOCKOUT);
10144         } else {
10145                 switch (nvcfg1) {
10146                         case FLASH_5761VENDOR_ATMEL_ADB161D:
10147                         case FLASH_5761VENDOR_ATMEL_MDB161D:
10148                         case FLASH_5761VENDOR_ST_A_M45PE16:
10149                         case FLASH_5761VENDOR_ST_M_M45PE16:
10150                                 tp->nvram_size = 0x100000;
10151                                 break;
10152                         case FLASH_5761VENDOR_ATMEL_ADB081D:
10153                         case FLASH_5761VENDOR_ATMEL_MDB081D:
10154                         case FLASH_5761VENDOR_ST_A_M45PE80:
10155                         case FLASH_5761VENDOR_ST_M_M45PE80:
10156                                 tp->nvram_size = 0x80000;
10157                                 break;
10158                         case FLASH_5761VENDOR_ATMEL_ADB041D:
10159                         case FLASH_5761VENDOR_ATMEL_MDB041D:
10160                         case FLASH_5761VENDOR_ST_A_M45PE40:
10161                         case FLASH_5761VENDOR_ST_M_M45PE40:
10162                                 tp->nvram_size = 0x40000;
10163                                 break;
10164                         case FLASH_5761VENDOR_ATMEL_ADB021D:
10165                         case FLASH_5761VENDOR_ATMEL_MDB021D:
10166                         case FLASH_5761VENDOR_ST_A_M45PE20:
10167                         case FLASH_5761VENDOR_ST_M_M45PE20:
10168                                 tp->nvram_size = 0x20000;
10169                                 break;
10170                 }
10171         }
10172 }
10173
10174 static void __devinit tg3_get_5906_nvram_info(struct tg3 *tp)
10175 {
10176         tp->nvram_jedecnum = JEDEC_ATMEL;
10177         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10178         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
10179 }
10180
10181 /* Chips other than 5700/5701 use the NVRAM for fetching info. */
10182 static void __devinit tg3_nvram_init(struct tg3 *tp)
10183 {
10184         tw32_f(GRC_EEPROM_ADDR,
10185              (EEPROM_ADDR_FSM_RESET |
10186               (EEPROM_DEFAULT_CLOCK_PERIOD <<
10187                EEPROM_ADDR_CLKPERD_SHIFT)));
10188
10189         msleep(1);
10190
10191         /* Enable seeprom accesses. */
10192         tw32_f(GRC_LOCAL_CTRL,
10193              tr32(GRC_LOCAL_CTRL) | GRC_LCLCTRL_AUTO_SEEPROM);
10194         udelay(100);
10195
10196         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
10197             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
10198                 tp->tg3_flags |= TG3_FLAG_NVRAM;
10199
10200                 if (tg3_nvram_lock(tp)) {
10201                         printk(KERN_WARNING PFX "%s: Cannot get nvarm lock, "
10202                                "tg3_nvram_init failed.\n", tp->dev->name);
10203                         return;
10204                 }
10205                 tg3_enable_nvram_access(tp);
10206
10207                 tp->nvram_size = 0;
10208
10209                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
10210                         tg3_get_5752_nvram_info(tp);
10211                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
10212                         tg3_get_5755_nvram_info(tp);
10213                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
10214                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784)
10215                         tg3_get_5787_nvram_info(tp);
10216                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
10217                         tg3_get_5761_nvram_info(tp);
10218                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
10219                         tg3_get_5906_nvram_info(tp);
10220                 else
10221                         tg3_get_nvram_info(tp);
10222
10223                 if (tp->nvram_size == 0)
10224                         tg3_get_nvram_size(tp);
10225
10226                 tg3_disable_nvram_access(tp);
10227                 tg3_nvram_unlock(tp);
10228
10229         } else {
10230                 tp->tg3_flags &= ~(TG3_FLAG_NVRAM | TG3_FLAG_NVRAM_BUFFERED);
10231
10232                 tg3_get_eeprom_size(tp);
10233         }
10234 }
10235
10236 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
10237                                         u32 offset, u32 *val)
10238 {
10239         u32 tmp;
10240         int i;
10241
10242         if (offset > EEPROM_ADDR_ADDR_MASK ||
10243             (offset % 4) != 0)
10244                 return -EINVAL;
10245
10246         tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
10247                                         EEPROM_ADDR_DEVID_MASK |
10248                                         EEPROM_ADDR_READ);
10249         tw32(GRC_EEPROM_ADDR,
10250              tmp |
10251              (0 << EEPROM_ADDR_DEVID_SHIFT) |
10252              ((offset << EEPROM_ADDR_ADDR_SHIFT) &
10253               EEPROM_ADDR_ADDR_MASK) |
10254              EEPROM_ADDR_READ | EEPROM_ADDR_START);
10255
10256         for (i = 0; i < 1000; i++) {
10257                 tmp = tr32(GRC_EEPROM_ADDR);
10258
10259                 if (tmp & EEPROM_ADDR_COMPLETE)
10260                         break;
10261                 msleep(1);
10262         }
10263         if (!(tmp & EEPROM_ADDR_COMPLETE))
10264                 return -EBUSY;
10265
10266         *val = tr32(GRC_EEPROM_DATA);
10267         return 0;
10268 }
10269
10270 #define NVRAM_CMD_TIMEOUT 10000
10271
10272 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
10273 {
10274         int i;
10275
10276         tw32(NVRAM_CMD, nvram_cmd);
10277         for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
10278                 udelay(10);
10279                 if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
10280                         udelay(10);
10281                         break;
10282                 }
10283         }
10284         if (i == NVRAM_CMD_TIMEOUT) {
10285                 return -EBUSY;
10286         }
10287         return 0;
10288 }
10289
10290 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
10291 {
10292         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
10293             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
10294             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
10295            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
10296             (tp->nvram_jedecnum == JEDEC_ATMEL))
10297
10298                 addr = ((addr / tp->nvram_pagesize) <<
10299                         ATMEL_AT45DB0X1B_PAGE_POS) +
10300                        (addr % tp->nvram_pagesize);
10301
10302         return addr;
10303 }
10304
10305 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
10306 {
10307         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
10308             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
10309             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
10310            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
10311             (tp->nvram_jedecnum == JEDEC_ATMEL))
10312
10313                 addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
10314                         tp->nvram_pagesize) +
10315                        (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
10316
10317         return addr;
10318 }
10319
10320 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
10321 {
10322         int ret;
10323
10324         if (!(tp->tg3_flags & TG3_FLAG_NVRAM))
10325                 return tg3_nvram_read_using_eeprom(tp, offset, val);
10326
10327         offset = tg3_nvram_phys_addr(tp, offset);
10328
10329         if (offset > NVRAM_ADDR_MSK)
10330                 return -EINVAL;
10331
10332         ret = tg3_nvram_lock(tp);
10333         if (ret)
10334                 return ret;
10335
10336         tg3_enable_nvram_access(tp);
10337
10338         tw32(NVRAM_ADDR, offset);
10339         ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
10340                 NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
10341
10342         if (ret == 0)
10343                 *val = swab32(tr32(NVRAM_RDDATA));
10344
10345         tg3_disable_nvram_access(tp);
10346
10347         tg3_nvram_unlock(tp);
10348
10349         return ret;
10350 }
10351
10352 static int tg3_nvram_read_le(struct tg3 *tp, u32 offset, __le32 *val)
10353 {
10354         u32 v;
10355         int res = tg3_nvram_read(tp, offset, &v);
10356         if (!res)
10357                 *val = cpu_to_le32(v);
10358         return res;
10359 }
10360
10361 static int tg3_nvram_read_swab(struct tg3 *tp, u32 offset, u32 *val)
10362 {
10363         int err;
10364         u32 tmp;
10365
10366         err = tg3_nvram_read(tp, offset, &tmp);
10367         *val = swab32(tmp);
10368         return err;
10369 }
10370
10371 static int tg3_nvram_write_block_using_eeprom(struct tg3 *tp,
10372                                     u32 offset, u32 len, u8 *buf)
10373 {
10374         int i, j, rc = 0;
10375         u32 val;
10376
10377         for (i = 0; i < len; i += 4) {
10378                 u32 addr;
10379                 __le32 data;
10380
10381                 addr = offset + i;
10382
10383                 memcpy(&data, buf + i, 4);
10384
10385                 tw32(GRC_EEPROM_DATA, le32_to_cpu(data));
10386
10387                 val = tr32(GRC_EEPROM_ADDR);
10388                 tw32(GRC_EEPROM_ADDR, val | EEPROM_ADDR_COMPLETE);
10389
10390                 val &= ~(EEPROM_ADDR_ADDR_MASK | EEPROM_ADDR_DEVID_MASK |
10391                         EEPROM_ADDR_READ);
10392                 tw32(GRC_EEPROM_ADDR, val |
10393                         (0 << EEPROM_ADDR_DEVID_SHIFT) |
10394                         (addr & EEPROM_ADDR_ADDR_MASK) |
10395                         EEPROM_ADDR_START |
10396                         EEPROM_ADDR_WRITE);
10397
10398                 for (j = 0; j < 1000; j++) {
10399                         val = tr32(GRC_EEPROM_ADDR);
10400
10401                         if (val & EEPROM_ADDR_COMPLETE)
10402                                 break;
10403                         msleep(1);
10404                 }
10405                 if (!(val & EEPROM_ADDR_COMPLETE)) {
10406                         rc = -EBUSY;
10407                         break;
10408                 }
10409         }
10410
10411         return rc;
10412 }
10413
10414 /* offset and length are dword aligned */
10415 static int tg3_nvram_write_block_unbuffered(struct tg3 *tp, u32 offset, u32 len,
10416                 u8 *buf)
10417 {
10418         int ret = 0;
10419         u32 pagesize = tp->nvram_pagesize;
10420         u32 pagemask = pagesize - 1;
10421         u32 nvram_cmd;
10422         u8 *tmp;
10423
10424         tmp = kmalloc(pagesize, GFP_KERNEL);
10425         if (tmp == NULL)
10426                 return -ENOMEM;
10427
10428         while (len) {
10429                 int j;
10430                 u32 phy_addr, page_off, size;
10431
10432                 phy_addr = offset & ~pagemask;
10433
10434                 for (j = 0; j < pagesize; j += 4) {
10435                         if ((ret = tg3_nvram_read_le(tp, phy_addr + j,
10436                                                 (__le32 *) (tmp + j))))
10437                                 break;
10438                 }
10439                 if (ret)
10440                         break;
10441
10442                 page_off = offset & pagemask;
10443                 size = pagesize;
10444                 if (len < size)
10445                         size = len;
10446
10447                 len -= size;
10448
10449                 memcpy(tmp + page_off, buf, size);
10450
10451                 offset = offset + (pagesize - page_off);
10452
10453                 tg3_enable_nvram_access(tp);
10454
10455                 /*
10456                  * Before we can erase the flash page, we need
10457                  * to issue a special "write enable" command.
10458                  */
10459                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
10460
10461                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
10462                         break;
10463
10464                 /* Erase the target page */
10465                 tw32(NVRAM_ADDR, phy_addr);
10466
10467                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR |
10468                         NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_ERASE;
10469
10470                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
10471                         break;
10472
10473                 /* Issue another write enable to start the write. */
10474                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
10475
10476                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
10477                         break;
10478
10479                 for (j = 0; j < pagesize; j += 4) {
10480                         __be32 data;
10481
10482                         data = *((__be32 *) (tmp + j));
10483                         /* swab32(le32_to_cpu(data)), actually */
10484                         tw32(NVRAM_WRDATA, be32_to_cpu(data));
10485
10486                         tw32(NVRAM_ADDR, phy_addr + j);
10487
10488                         nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE |
10489                                 NVRAM_CMD_WR;
10490
10491                         if (j == 0)
10492                                 nvram_cmd |= NVRAM_CMD_FIRST;
10493                         else if (j == (pagesize - 4))
10494                                 nvram_cmd |= NVRAM_CMD_LAST;
10495
10496                         if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
10497                                 break;
10498                 }
10499                 if (ret)
10500                         break;
10501         }
10502
10503         nvram_cmd = NVRAM_CMD_WRDI | NVRAM_CMD_GO | NVRAM_CMD_DONE;
10504         tg3_nvram_exec_cmd(tp, nvram_cmd);
10505
10506         kfree(tmp);
10507
10508         return ret;
10509 }
10510
10511 /* offset and length are dword aligned */
10512 static int tg3_nvram_write_block_buffered(struct tg3 *tp, u32 offset, u32 len,
10513                 u8 *buf)
10514 {
10515         int i, ret = 0;
10516
10517         for (i = 0; i < len; i += 4, offset += 4) {
10518                 u32 page_off, phy_addr, nvram_cmd;
10519                 __be32 data;
10520
10521                 memcpy(&data, buf + i, 4);
10522                 tw32(NVRAM_WRDATA, be32_to_cpu(data));
10523
10524                 page_off = offset % tp->nvram_pagesize;
10525
10526                 phy_addr = tg3_nvram_phys_addr(tp, offset);
10527
10528                 tw32(NVRAM_ADDR, phy_addr);
10529
10530                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR;
10531
10532                 if ((page_off == 0) || (i == 0))
10533                         nvram_cmd |= NVRAM_CMD_FIRST;
10534                 if (page_off == (tp->nvram_pagesize - 4))
10535                         nvram_cmd |= NVRAM_CMD_LAST;
10536
10537                 if (i == (len - 4))
10538                         nvram_cmd |= NVRAM_CMD_LAST;
10539
10540                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752) &&
10541                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5755) &&
10542                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5787) &&
10543                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784) &&
10544                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5761) &&
10545                     (tp->nvram_jedecnum == JEDEC_ST) &&
10546                     (nvram_cmd & NVRAM_CMD_FIRST)) {
10547
10548                         if ((ret = tg3_nvram_exec_cmd(tp,
10549                                 NVRAM_CMD_WREN | NVRAM_CMD_GO |
10550                                 NVRAM_CMD_DONE)))
10551
10552                                 break;
10553                 }
10554                 if (!(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
10555                         /* We always do complete word writes to eeprom. */
10556                         nvram_cmd |= (NVRAM_CMD_FIRST | NVRAM_CMD_LAST);
10557                 }
10558
10559                 if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
10560                         break;
10561         }
10562         return ret;
10563 }
10564
10565 /* offset and length are dword aligned */
10566 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf)
10567 {
10568         int ret;
10569
10570         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
10571                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl &
10572                        ~GRC_LCLCTRL_GPIO_OUTPUT1);
10573                 udelay(40);
10574         }
10575
10576         if (!(tp->tg3_flags & TG3_FLAG_NVRAM)) {
10577                 ret = tg3_nvram_write_block_using_eeprom(tp, offset, len, buf);
10578         }
10579         else {
10580                 u32 grc_mode;
10581
10582                 ret = tg3_nvram_lock(tp);
10583                 if (ret)
10584                         return ret;
10585
10586                 tg3_enable_nvram_access(tp);
10587                 if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
10588                     !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM))
10589                         tw32(NVRAM_WRITE1, 0x406);
10590
10591                 grc_mode = tr32(GRC_MODE);
10592                 tw32(GRC_MODE, grc_mode | GRC_MODE_NVRAM_WR_ENABLE);
10593
10594                 if ((tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) ||
10595                         !(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
10596
10597                         ret = tg3_nvram_write_block_buffered(tp, offset, len,
10598                                 buf);
10599                 }
10600                 else {
10601                         ret = tg3_nvram_write_block_unbuffered(tp, offset, len,
10602                                 buf);
10603                 }
10604
10605                 grc_mode = tr32(GRC_MODE);
10606                 tw32(GRC_MODE, grc_mode & ~GRC_MODE_NVRAM_WR_ENABLE);
10607
10608                 tg3_disable_nvram_access(tp);
10609                 tg3_nvram_unlock(tp);
10610         }
10611
10612         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
10613                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
10614                 udelay(40);
10615         }
10616
10617         return ret;
10618 }
10619
10620 struct subsys_tbl_ent {
10621         u16 subsys_vendor, subsys_devid;
10622         u32 phy_id;
10623 };
10624
10625 static struct subsys_tbl_ent subsys_id_to_phy_id[] = {
10626         /* Broadcom boards. */
10627         { PCI_VENDOR_ID_BROADCOM, 0x1644, PHY_ID_BCM5401 }, /* BCM95700A6 */
10628         { PCI_VENDOR_ID_BROADCOM, 0x0001, PHY_ID_BCM5701 }, /* BCM95701A5 */
10629         { PCI_VENDOR_ID_BROADCOM, 0x0002, PHY_ID_BCM8002 }, /* BCM95700T6 */
10630         { PCI_VENDOR_ID_BROADCOM, 0x0003, 0 },              /* BCM95700A9 */
10631         { PCI_VENDOR_ID_BROADCOM, 0x0005, PHY_ID_BCM5701 }, /* BCM95701T1 */
10632         { PCI_VENDOR_ID_BROADCOM, 0x0006, PHY_ID_BCM5701 }, /* BCM95701T8 */
10633         { PCI_VENDOR_ID_BROADCOM, 0x0007, 0 },              /* BCM95701A7 */
10634         { PCI_VENDOR_ID_BROADCOM, 0x0008, PHY_ID_BCM5701 }, /* BCM95701A10 */
10635         { PCI_VENDOR_ID_BROADCOM, 0x8008, PHY_ID_BCM5701 }, /* BCM95701A12 */
10636         { PCI_VENDOR_ID_BROADCOM, 0x0009, PHY_ID_BCM5703 }, /* BCM95703Ax1 */
10637         { PCI_VENDOR_ID_BROADCOM, 0x8009, PHY_ID_BCM5703 }, /* BCM95703Ax2 */
10638
10639         /* 3com boards. */
10640         { PCI_VENDOR_ID_3COM, 0x1000, PHY_ID_BCM5401 }, /* 3C996T */
10641         { PCI_VENDOR_ID_3COM, 0x1006, PHY_ID_BCM5701 }, /* 3C996BT */
10642         { PCI_VENDOR_ID_3COM, 0x1004, 0 },              /* 3C996SX */
10643         { PCI_VENDOR_ID_3COM, 0x1007, PHY_ID_BCM5701 }, /* 3C1000T */
10644         { PCI_VENDOR_ID_3COM, 0x1008, PHY_ID_BCM5701 }, /* 3C940BR01 */
10645
10646         /* DELL boards. */
10647         { PCI_VENDOR_ID_DELL, 0x00d1, PHY_ID_BCM5401 }, /* VIPER */
10648         { PCI_VENDOR_ID_DELL, 0x0106, PHY_ID_BCM5401 }, /* JAGUAR */
10649         { PCI_VENDOR_ID_DELL, 0x0109, PHY_ID_BCM5411 }, /* MERLOT */
10650         { PCI_VENDOR_ID_DELL, 0x010a, PHY_ID_BCM5411 }, /* SLIM_MERLOT */
10651
10652         /* Compaq boards. */
10653         { PCI_VENDOR_ID_COMPAQ, 0x007c, PHY_ID_BCM5701 }, /* BANSHEE */
10654         { PCI_VENDOR_ID_COMPAQ, 0x009a, PHY_ID_BCM5701 }, /* BANSHEE_2 */
10655         { PCI_VENDOR_ID_COMPAQ, 0x007d, 0 },              /* CHANGELING */
10656         { PCI_VENDOR_ID_COMPAQ, 0x0085, PHY_ID_BCM5701 }, /* NC7780 */
10657         { PCI_VENDOR_ID_COMPAQ, 0x0099, PHY_ID_BCM5701 }, /* NC7780_2 */
10658
10659         /* IBM boards. */
10660         { PCI_VENDOR_ID_IBM, 0x0281, 0 } /* IBM??? */
10661 };
10662
10663 static inline struct subsys_tbl_ent *lookup_by_subsys(struct tg3 *tp)
10664 {
10665         int i;
10666
10667         for (i = 0; i < ARRAY_SIZE(subsys_id_to_phy_id); i++) {
10668                 if ((subsys_id_to_phy_id[i].subsys_vendor ==
10669                      tp->pdev->subsystem_vendor) &&
10670                     (subsys_id_to_phy_id[i].subsys_devid ==
10671                      tp->pdev->subsystem_device))
10672                         return &subsys_id_to_phy_id[i];
10673         }
10674         return NULL;
10675 }
10676
10677 static void __devinit tg3_get_eeprom_hw_cfg(struct tg3 *tp)
10678 {
10679         u32 val;
10680         u16 pmcsr;
10681
10682         /* On some early chips the SRAM cannot be accessed in D3hot state,
10683          * so need make sure we're in D0.
10684          */
10685         pci_read_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, &pmcsr);
10686         pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
10687         pci_write_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, pmcsr);
10688         msleep(1);
10689
10690         /* Make sure register accesses (indirect or otherwise)
10691          * will function correctly.
10692          */
10693         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
10694                                tp->misc_host_ctrl);
10695
10696         /* The memory arbiter has to be enabled in order for SRAM accesses
10697          * to succeed.  Normally on powerup the tg3 chip firmware will make
10698          * sure it is enabled, but other entities such as system netboot
10699          * code might disable it.
10700          */
10701         val = tr32(MEMARB_MODE);
10702         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
10703
10704         tp->phy_id = PHY_ID_INVALID;
10705         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
10706
10707         /* Assume an onboard device and WOL capable by default.  */
10708         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT | TG3_FLAG_WOL_CAP;
10709
10710         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
10711                 if (!(tr32(PCIE_TRANSACTION_CFG) & PCIE_TRANS_CFG_LOM)) {
10712                         tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
10713                         tp->tg3_flags2 |= TG3_FLG2_IS_NIC;
10714                 }
10715                 val = tr32(VCPU_CFGSHDW);
10716                 if (val & VCPU_CFGSHDW_ASPM_DBNC)
10717                         tp->tg3_flags |= TG3_FLAG_ASPM_WORKAROUND;
10718                 if ((val & VCPU_CFGSHDW_WOL_ENABLE) &&
10719                     (val & VCPU_CFGSHDW_WOL_MAGPKT))
10720                         tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
10721                 return;
10722         }
10723
10724         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
10725         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
10726                 u32 nic_cfg, led_cfg;
10727                 u32 nic_phy_id, ver, cfg2 = 0, eeprom_phy_id;
10728                 int eeprom_phy_serdes = 0;
10729
10730                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
10731                 tp->nic_sram_data_cfg = nic_cfg;
10732
10733                 tg3_read_mem(tp, NIC_SRAM_DATA_VER, &ver);
10734                 ver >>= NIC_SRAM_DATA_VER_SHIFT;
10735                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700) &&
10736                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) &&
10737                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5703) &&
10738                     (ver > 0) && (ver < 0x100))
10739                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_2, &cfg2);
10740
10741                 if ((nic_cfg & NIC_SRAM_DATA_CFG_PHY_TYPE_MASK) ==
10742                     NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER)
10743                         eeprom_phy_serdes = 1;
10744
10745                 tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
10746                 if (nic_phy_id != 0) {
10747                         u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
10748                         u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
10749
10750                         eeprom_phy_id  = (id1 >> 16) << 10;
10751                         eeprom_phy_id |= (id2 & 0xfc00) << 16;
10752                         eeprom_phy_id |= (id2 & 0x03ff) <<  0;
10753                 } else
10754                         eeprom_phy_id = 0;
10755
10756                 tp->phy_id = eeprom_phy_id;
10757                 if (eeprom_phy_serdes) {
10758                         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
10759                                 tp->tg3_flags2 |= TG3_FLG2_MII_SERDES;
10760                         else
10761                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
10762                 }
10763
10764                 if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
10765                         led_cfg = cfg2 & (NIC_SRAM_DATA_CFG_LED_MODE_MASK |
10766                                     SHASTA_EXT_LED_MODE_MASK);
10767                 else
10768                         led_cfg = nic_cfg & NIC_SRAM_DATA_CFG_LED_MODE_MASK;
10769
10770                 switch (led_cfg) {
10771                 default:
10772                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_1:
10773                         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
10774                         break;
10775
10776                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_2:
10777                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
10778                         break;
10779
10780                 case NIC_SRAM_DATA_CFG_LED_MODE_MAC:
10781                         tp->led_ctrl = LED_CTRL_MODE_MAC;
10782
10783                         /* Default to PHY_1_MODE if 0 (MAC_MODE) is
10784                          * read on some older 5700/5701 bootcode.
10785                          */
10786                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
10787                             ASIC_REV_5700 ||
10788                             GET_ASIC_REV(tp->pci_chip_rev_id) ==
10789                             ASIC_REV_5701)
10790                                 tp->led_ctrl = LED_CTRL_MODE_PHY_1;
10791
10792                         break;
10793
10794                 case SHASTA_EXT_LED_SHARED:
10795                         tp->led_ctrl = LED_CTRL_MODE_SHARED;
10796                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0 &&
10797                             tp->pci_chip_rev_id != CHIPREV_ID_5750_A1)
10798                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
10799                                                  LED_CTRL_MODE_PHY_2);
10800                         break;
10801
10802                 case SHASTA_EXT_LED_MAC:
10803                         tp->led_ctrl = LED_CTRL_MODE_SHASTA_MAC;
10804                         break;
10805
10806                 case SHASTA_EXT_LED_COMBO:
10807                         tp->led_ctrl = LED_CTRL_MODE_COMBO;
10808                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0)
10809                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
10810                                                  LED_CTRL_MODE_PHY_2);
10811                         break;
10812
10813                 };
10814
10815                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
10816                      GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) &&
10817                     tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
10818                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
10819
10820                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX)
10821                         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
10822
10823                 if (nic_cfg & NIC_SRAM_DATA_CFG_EEPROM_WP) {
10824                         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT;
10825                         if ((tp->pdev->subsystem_vendor ==
10826                              PCI_VENDOR_ID_ARIMA) &&
10827                             (tp->pdev->subsystem_device == 0x205a ||
10828                              tp->pdev->subsystem_device == 0x2063))
10829                                 tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
10830                 } else {
10831                         tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
10832                         tp->tg3_flags2 |= TG3_FLG2_IS_NIC;
10833                 }
10834
10835                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
10836                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
10837                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
10838                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
10839                 }
10840                 if (nic_cfg & NIC_SRAM_DATA_CFG_APE_ENABLE)
10841                         tp->tg3_flags3 |= TG3_FLG3_ENABLE_APE;
10842                 if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES &&
10843                     !(nic_cfg & NIC_SRAM_DATA_CFG_FIBER_WOL))
10844                         tp->tg3_flags &= ~TG3_FLAG_WOL_CAP;
10845
10846                 if (tp->tg3_flags & TG3_FLAG_WOL_CAP &&
10847                     nic_cfg & NIC_SRAM_DATA_CFG_WOL_ENABLE)
10848                         tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
10849
10850                 if (cfg2 & (1 << 17))
10851                         tp->tg3_flags2 |= TG3_FLG2_CAPACITIVE_COUPLING;
10852
10853                 /* serdes signal pre-emphasis in register 0x590 set by */
10854                 /* bootcode if bit 18 is set */
10855                 if (cfg2 & (1 << 18))
10856                         tp->tg3_flags2 |= TG3_FLG2_SERDES_PREEMPHASIS;
10857
10858                 if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
10859                         u32 cfg3;
10860
10861                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_3, &cfg3);
10862                         if (cfg3 & NIC_SRAM_ASPM_DEBOUNCE)
10863                                 tp->tg3_flags |= TG3_FLAG_ASPM_WORKAROUND;
10864                 }
10865         }
10866 }
10867
10868 static int __devinit tg3_issue_otp_command(struct tg3 *tp, u32 cmd)
10869 {
10870         int i;
10871         u32 val;
10872
10873         tw32(OTP_CTRL, cmd | OTP_CTRL_OTP_CMD_START);
10874         tw32(OTP_CTRL, cmd);
10875
10876         /* Wait for up to 1 ms for command to execute. */
10877         for (i = 0; i < 100; i++) {
10878                 val = tr32(OTP_STATUS);
10879                 if (val & OTP_STATUS_CMD_DONE)
10880                         break;
10881                 udelay(10);
10882         }
10883
10884         return (val & OTP_STATUS_CMD_DONE) ? 0 : -EBUSY;
10885 }
10886
10887 /* Read the gphy configuration from the OTP region of the chip.  The gphy
10888  * configuration is a 32-bit value that straddles the alignment boundary.
10889  * We do two 32-bit reads and then shift and merge the results.
10890  */
10891 static u32 __devinit tg3_read_otp_phycfg(struct tg3 *tp)
10892 {
10893         u32 bhalf_otp, thalf_otp;
10894
10895         tw32(OTP_MODE, OTP_MODE_OTP_THRU_GRC);
10896
10897         if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_INIT))
10898                 return 0;
10899
10900         tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC1);
10901
10902         if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
10903                 return 0;
10904
10905         thalf_otp = tr32(OTP_READ_DATA);
10906
10907         tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC2);
10908
10909         if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
10910                 return 0;
10911
10912         bhalf_otp = tr32(OTP_READ_DATA);
10913
10914         return ((thalf_otp & 0x0000ffff) << 16) | (bhalf_otp >> 16);
10915 }
10916
10917 static int __devinit tg3_phy_probe(struct tg3 *tp)
10918 {
10919         u32 hw_phy_id_1, hw_phy_id_2;
10920         u32 hw_phy_id, hw_phy_id_masked;
10921         int err;
10922
10923         /* Reading the PHY ID register can conflict with ASF
10924          * firwmare access to the PHY hardware.
10925          */
10926         err = 0;
10927         if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
10928             (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)) {
10929                 hw_phy_id = hw_phy_id_masked = PHY_ID_INVALID;
10930         } else {
10931                 /* Now read the physical PHY_ID from the chip and verify
10932                  * that it is sane.  If it doesn't look good, we fall back
10933                  * to either the hard-coded table based PHY_ID and failing
10934                  * that the value found in the eeprom area.
10935                  */
10936                 err |= tg3_readphy(tp, MII_PHYSID1, &hw_phy_id_1);
10937                 err |= tg3_readphy(tp, MII_PHYSID2, &hw_phy_id_2);
10938
10939                 hw_phy_id  = (hw_phy_id_1 & 0xffff) << 10;
10940                 hw_phy_id |= (hw_phy_id_2 & 0xfc00) << 16;
10941                 hw_phy_id |= (hw_phy_id_2 & 0x03ff) <<  0;
10942
10943                 hw_phy_id_masked = hw_phy_id & PHY_ID_MASK;
10944         }
10945
10946         if (!err && KNOWN_PHY_ID(hw_phy_id_masked)) {
10947                 tp->phy_id = hw_phy_id;
10948                 if (hw_phy_id_masked == PHY_ID_BCM8002)
10949                         tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
10950                 else
10951                         tp->tg3_flags2 &= ~TG3_FLG2_PHY_SERDES;
10952         } else {
10953                 if (tp->phy_id != PHY_ID_INVALID) {
10954                         /* Do nothing, phy ID already set up in
10955                          * tg3_get_eeprom_hw_cfg().
10956                          */
10957                 } else {
10958                         struct subsys_tbl_ent *p;
10959
10960                         /* No eeprom signature?  Try the hardcoded
10961                          * subsys device table.
10962                          */
10963                         p = lookup_by_subsys(tp);
10964                         if (!p)
10965                                 return -ENODEV;
10966
10967                         tp->phy_id = p->phy_id;
10968                         if (!tp->phy_id ||
10969                             tp->phy_id == PHY_ID_BCM8002)
10970                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
10971                 }
10972         }
10973
10974         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) &&
10975             !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) &&
10976             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
10977                 u32 bmsr, adv_reg, tg3_ctrl, mask;
10978
10979                 tg3_readphy(tp, MII_BMSR, &bmsr);
10980                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
10981                     (bmsr & BMSR_LSTATUS))
10982                         goto skip_phy_reset;
10983
10984                 err = tg3_phy_reset(tp);
10985                 if (err)
10986                         return err;
10987
10988                 adv_reg = (ADVERTISE_10HALF | ADVERTISE_10FULL |
10989                            ADVERTISE_100HALF | ADVERTISE_100FULL |
10990                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
10991                 tg3_ctrl = 0;
10992                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
10993                         tg3_ctrl = (MII_TG3_CTRL_ADV_1000_HALF |
10994                                     MII_TG3_CTRL_ADV_1000_FULL);
10995                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
10996                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
10997                                 tg3_ctrl |= (MII_TG3_CTRL_AS_MASTER |
10998                                              MII_TG3_CTRL_ENABLE_AS_MASTER);
10999                 }
11000
11001                 mask = (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
11002                         ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
11003                         ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full);
11004                 if (!tg3_copper_is_advertising_all(tp, mask)) {
11005                         tg3_writephy(tp, MII_ADVERTISE, adv_reg);
11006
11007                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
11008                                 tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
11009
11010                         tg3_writephy(tp, MII_BMCR,
11011                                      BMCR_ANENABLE | BMCR_ANRESTART);
11012                 }
11013                 tg3_phy_set_wirespeed(tp);
11014
11015                 tg3_writephy(tp, MII_ADVERTISE, adv_reg);
11016                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
11017                         tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
11018         }
11019
11020 skip_phy_reset:
11021         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
11022                 err = tg3_init_5401phy_dsp(tp);
11023                 if (err)
11024                         return err;
11025         }
11026
11027         if (!err && ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401)) {
11028                 err = tg3_init_5401phy_dsp(tp);
11029         }
11030
11031         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
11032                 tp->link_config.advertising =
11033                         (ADVERTISED_1000baseT_Half |
11034                          ADVERTISED_1000baseT_Full |
11035                          ADVERTISED_Autoneg |
11036                          ADVERTISED_FIBRE);
11037         if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
11038                 tp->link_config.advertising &=
11039                         ~(ADVERTISED_1000baseT_Half |
11040                           ADVERTISED_1000baseT_Full);
11041
11042         return err;
11043 }
11044
11045 static void __devinit tg3_read_partno(struct tg3 *tp)
11046 {
11047         unsigned char vpd_data[256];
11048         unsigned int i;
11049         u32 magic;
11050
11051         if (tg3_nvram_read_swab(tp, 0x0, &magic))
11052                 goto out_not_found;
11053
11054         if (magic == TG3_EEPROM_MAGIC) {
11055                 for (i = 0; i < 256; i += 4) {
11056                         u32 tmp;
11057
11058                         if (tg3_nvram_read(tp, 0x100 + i, &tmp))
11059                                 goto out_not_found;
11060
11061                         vpd_data[i + 0] = ((tmp >>  0) & 0xff);
11062                         vpd_data[i + 1] = ((tmp >>  8) & 0xff);
11063                         vpd_data[i + 2] = ((tmp >> 16) & 0xff);
11064                         vpd_data[i + 3] = ((tmp >> 24) & 0xff);
11065                 }
11066         } else {
11067                 int vpd_cap;
11068
11069                 vpd_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_VPD);
11070                 for (i = 0; i < 256; i += 4) {
11071                         u32 tmp, j = 0;
11072                         __le32 v;
11073                         u16 tmp16;
11074
11075                         pci_write_config_word(tp->pdev, vpd_cap + PCI_VPD_ADDR,
11076                                               i);
11077                         while (j++ < 100) {
11078                                 pci_read_config_word(tp->pdev, vpd_cap +
11079                                                      PCI_VPD_ADDR, &tmp16);
11080                                 if (tmp16 & 0x8000)
11081                                         break;
11082                                 msleep(1);
11083                         }
11084                         if (!(tmp16 & 0x8000))
11085                                 goto out_not_found;
11086
11087                         pci_read_config_dword(tp->pdev, vpd_cap + PCI_VPD_DATA,
11088                                               &tmp);
11089                         v = cpu_to_le32(tmp);
11090                         memcpy(&vpd_data[i], &v, 4);
11091                 }
11092         }
11093
11094         /* Now parse and find the part number. */
11095         for (i = 0; i < 254; ) {
11096                 unsigned char val = vpd_data[i];
11097                 unsigned int block_end;
11098
11099                 if (val == 0x82 || val == 0x91) {
11100                         i = (i + 3 +
11101                              (vpd_data[i + 1] +
11102                               (vpd_data[i + 2] << 8)));
11103                         continue;
11104                 }
11105
11106                 if (val != 0x90)
11107                         goto out_not_found;
11108
11109                 block_end = (i + 3 +
11110                              (vpd_data[i + 1] +
11111                               (vpd_data[i + 2] << 8)));
11112                 i += 3;
11113
11114                 if (block_end > 256)
11115                         goto out_not_found;
11116
11117                 while (i < (block_end - 2)) {
11118                         if (vpd_data[i + 0] == 'P' &&
11119                             vpd_data[i + 1] == 'N') {
11120                                 int partno_len = vpd_data[i + 2];
11121
11122                                 i += 3;
11123                                 if (partno_len > 24 || (partno_len + i) > 256)
11124                                         goto out_not_found;
11125
11126                                 memcpy(tp->board_part_number,
11127                                        &vpd_data[i], partno_len);
11128
11129                                 /* Success. */
11130                                 return;
11131                         }
11132                         i += 3 + vpd_data[i + 2];
11133                 }
11134
11135                 /* Part number not found. */
11136                 goto out_not_found;
11137         }
11138
11139 out_not_found:
11140         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11141                 strcpy(tp->board_part_number, "BCM95906");
11142         else
11143                 strcpy(tp->board_part_number, "none");
11144 }
11145
11146 static int __devinit tg3_fw_img_is_valid(struct tg3 *tp, u32 offset)
11147 {
11148         u32 val;
11149
11150         if (tg3_nvram_read_swab(tp, offset, &val) ||
11151             (val & 0xfc000000) != 0x0c000000 ||
11152             tg3_nvram_read_swab(tp, offset + 4, &val) ||
11153             val != 0)
11154                 return 0;
11155
11156         return 1;
11157 }
11158
11159 static void __devinit tg3_read_fw_ver(struct tg3 *tp)
11160 {
11161         u32 val, offset, start;
11162         u32 ver_offset;
11163         int i, bcnt;
11164
11165         if (tg3_nvram_read_swab(tp, 0, &val))
11166                 return;
11167
11168         if (val != TG3_EEPROM_MAGIC)
11169                 return;
11170
11171         if (tg3_nvram_read_swab(tp, 0xc, &offset) ||
11172             tg3_nvram_read_swab(tp, 0x4, &start))
11173                 return;
11174
11175         offset = tg3_nvram_logical_addr(tp, offset);
11176
11177         if (!tg3_fw_img_is_valid(tp, offset) ||
11178             tg3_nvram_read_swab(tp, offset + 8, &ver_offset))
11179                 return;
11180
11181         offset = offset + ver_offset - start;
11182         for (i = 0; i < 16; i += 4) {
11183                 __le32 v;
11184                 if (tg3_nvram_read_le(tp, offset + i, &v))
11185                         return;
11186
11187                 memcpy(tp->fw_ver + i, &v, 4);
11188         }
11189
11190         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
11191              (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
11192                 return;
11193
11194         for (offset = TG3_NVM_DIR_START;
11195              offset < TG3_NVM_DIR_END;
11196              offset += TG3_NVM_DIRENT_SIZE) {
11197                 if (tg3_nvram_read_swab(tp, offset, &val))
11198                         return;
11199
11200                 if ((val >> TG3_NVM_DIRTYPE_SHIFT) == TG3_NVM_DIRTYPE_ASFINI)
11201                         break;
11202         }
11203
11204         if (offset == TG3_NVM_DIR_END)
11205                 return;
11206
11207         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
11208                 start = 0x08000000;
11209         else if (tg3_nvram_read_swab(tp, offset - 4, &start))
11210                 return;
11211
11212         if (tg3_nvram_read_swab(tp, offset + 4, &offset) ||
11213             !tg3_fw_img_is_valid(tp, offset) ||
11214             tg3_nvram_read_swab(tp, offset + 8, &val))
11215                 return;
11216
11217         offset += val - start;
11218
11219         bcnt = strlen(tp->fw_ver);
11220
11221         tp->fw_ver[bcnt++] = ',';
11222         tp->fw_ver[bcnt++] = ' ';
11223
11224         for (i = 0; i < 4; i++) {
11225                 __le32 v;
11226                 if (tg3_nvram_read_le(tp, offset, &v))
11227                         return;
11228
11229                 offset += sizeof(v);
11230
11231                 if (bcnt > TG3_VER_SIZE - sizeof(v)) {
11232                         memcpy(&tp->fw_ver[bcnt], &v, TG3_VER_SIZE - bcnt);
11233                         break;
11234                 }
11235
11236                 memcpy(&tp->fw_ver[bcnt], &v, sizeof(v));
11237                 bcnt += sizeof(v);
11238         }
11239
11240         tp->fw_ver[TG3_VER_SIZE - 1] = 0;
11241 }
11242
11243 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *);
11244
11245 static int __devinit tg3_get_invariants(struct tg3 *tp)
11246 {
11247         static struct pci_device_id write_reorder_chipsets[] = {
11248                 { PCI_DEVICE(PCI_VENDOR_ID_AMD,
11249                              PCI_DEVICE_ID_AMD_FE_GATE_700C) },
11250                 { PCI_DEVICE(PCI_VENDOR_ID_AMD,
11251                              PCI_DEVICE_ID_AMD_8131_BRIDGE) },
11252                 { PCI_DEVICE(PCI_VENDOR_ID_VIA,
11253                              PCI_DEVICE_ID_VIA_8385_0) },
11254                 { },
11255         };
11256         u32 misc_ctrl_reg;
11257         u32 cacheline_sz_reg;
11258         u32 pci_state_reg, grc_misc_cfg;
11259         u32 val;
11260         u16 pci_cmd;
11261         int err, pcie_cap;
11262
11263         /* Force memory write invalidate off.  If we leave it on,
11264          * then on 5700_BX chips we have to enable a workaround.
11265          * The workaround is to set the TG3PCI_DMA_RW_CTRL boundary
11266          * to match the cacheline size.  The Broadcom driver have this
11267          * workaround but turns MWI off all the times so never uses
11268          * it.  This seems to suggest that the workaround is insufficient.
11269          */
11270         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11271         pci_cmd &= ~PCI_COMMAND_INVALIDATE;
11272         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11273
11274         /* It is absolutely critical that TG3PCI_MISC_HOST_CTRL
11275          * has the register indirect write enable bit set before
11276          * we try to access any of the MMIO registers.  It is also
11277          * critical that the PCI-X hw workaround situation is decided
11278          * before that as well.
11279          */
11280         pci_read_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
11281                               &misc_ctrl_reg);
11282
11283         tp->pci_chip_rev_id = (misc_ctrl_reg >>
11284                                MISC_HOST_CTRL_CHIPREV_SHIFT);
11285         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_USE_PROD_ID_REG) {
11286                 u32 prod_id_asic_rev;
11287
11288                 pci_read_config_dword(tp->pdev, TG3PCI_PRODID_ASICREV,
11289                                       &prod_id_asic_rev);
11290                 tp->pci_chip_rev_id = prod_id_asic_rev & PROD_ID_ASIC_REV_MASK;
11291         }
11292
11293         /* Wrong chip ID in 5752 A0. This code can be removed later
11294          * as A0 is not in production.
11295          */
11296         if (tp->pci_chip_rev_id == CHIPREV_ID_5752_A0_HW)
11297                 tp->pci_chip_rev_id = CHIPREV_ID_5752_A0;
11298
11299         /* If we have 5702/03 A1 or A2 on certain ICH chipsets,
11300          * we need to disable memory and use config. cycles
11301          * only to access all registers. The 5702/03 chips
11302          * can mistakenly decode the special cycles from the
11303          * ICH chipsets as memory write cycles, causing corruption
11304          * of register and memory space. Only certain ICH bridges
11305          * will drive special cycles with non-zero data during the
11306          * address phase which can fall within the 5703's address
11307          * range. This is not an ICH bug as the PCI spec allows
11308          * non-zero address during special cycles. However, only
11309          * these ICH bridges are known to drive non-zero addresses
11310          * during special cycles.
11311          *
11312          * Since special cycles do not cross PCI bridges, we only
11313          * enable this workaround if the 5703 is on the secondary
11314          * bus of these ICH bridges.
11315          */
11316         if ((tp->pci_chip_rev_id == CHIPREV_ID_5703_A1) ||
11317             (tp->pci_chip_rev_id == CHIPREV_ID_5703_A2)) {
11318                 static struct tg3_dev_id {
11319                         u32     vendor;
11320                         u32     device;
11321                         u32     rev;
11322                 } ich_chipsets[] = {
11323                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AA_8,
11324                           PCI_ANY_ID },
11325                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AB_8,
11326                           PCI_ANY_ID },
11327                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_11,
11328                           0xa },
11329                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_6,
11330                           PCI_ANY_ID },
11331                         { },
11332                 };
11333                 struct tg3_dev_id *pci_id = &ich_chipsets[0];
11334                 struct pci_dev *bridge = NULL;
11335
11336                 while (pci_id->vendor != 0) {
11337                         bridge = pci_get_device(pci_id->vendor, pci_id->device,
11338                                                 bridge);
11339                         if (!bridge) {
11340                                 pci_id++;
11341                                 continue;
11342                         }
11343                         if (pci_id->rev != PCI_ANY_ID) {
11344                                 if (bridge->revision > pci_id->rev)
11345                                         continue;
11346                         }
11347                         if (bridge->subordinate &&
11348                             (bridge->subordinate->number ==
11349                              tp->pdev->bus->number)) {
11350
11351                                 tp->tg3_flags2 |= TG3_FLG2_ICH_WORKAROUND;
11352                                 pci_dev_put(bridge);
11353                                 break;
11354                         }
11355                 }
11356         }
11357
11358         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
11359                 static struct tg3_dev_id {
11360                         u32     vendor;
11361                         u32     device;
11362                 } bridge_chipsets[] = {
11363                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_0 },
11364                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_1 },
11365                         { },
11366                 };
11367                 struct tg3_dev_id *pci_id = &bridge_chipsets[0];
11368                 struct pci_dev *bridge = NULL;
11369
11370                 while (pci_id->vendor != 0) {
11371                         bridge = pci_get_device(pci_id->vendor,
11372                                                 pci_id->device,
11373                                                 bridge);
11374                         if (!bridge) {
11375                                 pci_id++;
11376                                 continue;
11377                         }
11378                         if (bridge->subordinate &&
11379                             (bridge->subordinate->number <=
11380                              tp->pdev->bus->number) &&
11381                             (bridge->subordinate->subordinate >=
11382                              tp->pdev->bus->number)) {
11383                                 tp->tg3_flags3 |= TG3_FLG3_5701_DMA_BUG;
11384                                 pci_dev_put(bridge);
11385                                 break;
11386                         }
11387                 }
11388         }
11389
11390         /* The EPB bridge inside 5714, 5715, and 5780 cannot support
11391          * DMA addresses > 40-bit. This bridge may have other additional
11392          * 57xx devices behind it in some 4-port NIC designs for example.
11393          * Any tg3 device found behind the bridge will also need the 40-bit
11394          * DMA workaround.
11395          */
11396         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 ||
11397             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
11398                 tp->tg3_flags2 |= TG3_FLG2_5780_CLASS;
11399                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
11400                 tp->msi_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_MSI);
11401         }
11402         else {
11403                 struct pci_dev *bridge = NULL;
11404
11405                 do {
11406                         bridge = pci_get_device(PCI_VENDOR_ID_SERVERWORKS,
11407                                                 PCI_DEVICE_ID_SERVERWORKS_EPB,
11408                                                 bridge);
11409                         if (bridge && bridge->subordinate &&
11410                             (bridge->subordinate->number <=
11411                              tp->pdev->bus->number) &&
11412                             (bridge->subordinate->subordinate >=
11413                              tp->pdev->bus->number)) {
11414                                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
11415                                 pci_dev_put(bridge);
11416                                 break;
11417                         }
11418                 } while (bridge);
11419         }
11420
11421         /* Initialize misc host control in PCI block. */
11422         tp->misc_host_ctrl |= (misc_ctrl_reg &
11423                                MISC_HOST_CTRL_CHIPREV);
11424         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
11425                                tp->misc_host_ctrl);
11426
11427         pci_read_config_dword(tp->pdev, TG3PCI_CACHELINESZ,
11428                               &cacheline_sz_reg);
11429
11430         tp->pci_cacheline_sz = (cacheline_sz_reg >>  0) & 0xff;
11431         tp->pci_lat_timer    = (cacheline_sz_reg >>  8) & 0xff;
11432         tp->pci_hdr_type     = (cacheline_sz_reg >> 16) & 0xff;
11433         tp->pci_bist         = (cacheline_sz_reg >> 24) & 0xff;
11434
11435         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
11436             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714))
11437                 tp->pdev_peer = tg3_find_peer(tp);
11438
11439         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
11440             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
11441             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11442             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11443             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11444             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
11445             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 ||
11446             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
11447                 tp->tg3_flags2 |= TG3_FLG2_5750_PLUS;
11448
11449         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) ||
11450             (tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
11451                 tp->tg3_flags2 |= TG3_FLG2_5705_PLUS;
11452
11453         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
11454                 tp->tg3_flags |= TG3_FLAG_SUPPORT_MSI;
11455                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX ||
11456                     GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX ||
11457                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714 &&
11458                      tp->pci_chip_rev_id <= CHIPREV_ID_5714_A2 &&
11459                      tp->pdev_peer == tp->pdev))
11460                         tp->tg3_flags &= ~TG3_FLAG_SUPPORT_MSI;
11461
11462                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11463                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11464                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11465                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
11466                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
11467                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_2;
11468                         tp->tg3_flags2 |= TG3_FLG2_1SHOT_MSI;
11469                 } else {
11470                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_1 | TG3_FLG2_TSO_BUG;
11471                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
11472                                 ASIC_REV_5750 &&
11473                             tp->pci_chip_rev_id >= CHIPREV_ID_5750_C2)
11474                                 tp->tg3_flags2 &= ~TG3_FLG2_TSO_BUG;
11475                 }
11476         }
11477
11478         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705 &&
11479             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5750 &&
11480             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752 &&
11481             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5755 &&
11482             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5787 &&
11483             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 &&
11484             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5761 &&
11485             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)
11486                 tp->tg3_flags2 |= TG3_FLG2_JUMBO_CAPABLE;
11487
11488         pcie_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_EXP);
11489         if (pcie_cap != 0) {
11490                 tp->tg3_flags2 |= TG3_FLG2_PCI_EXPRESS;
11491
11492                 pcie_set_readrq(tp->pdev, 4096);
11493
11494                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
11495                         u16 lnkctl;
11496
11497                         pci_read_config_word(tp->pdev,
11498                                              pcie_cap + PCI_EXP_LNKCTL,
11499                                              &lnkctl);
11500                         if (lnkctl & PCI_EXP_LNKCTL_CLKREQ_EN)
11501                                 tp->tg3_flags2 &= ~TG3_FLG2_HW_TSO_2;
11502                 }
11503         }
11504
11505         /* If we have an AMD 762 or VIA K8T800 chipset, write
11506          * reordering to the mailbox registers done by the host
11507          * controller can cause major troubles.  We read back from
11508          * every mailbox register write to force the writes to be
11509          * posted to the chip in order.
11510          */
11511         if (pci_dev_present(write_reorder_chipsets) &&
11512             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
11513                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
11514
11515         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
11516             tp->pci_lat_timer < 64) {
11517                 tp->pci_lat_timer = 64;
11518
11519                 cacheline_sz_reg  = ((tp->pci_cacheline_sz & 0xff) <<  0);
11520                 cacheline_sz_reg |= ((tp->pci_lat_timer    & 0xff) <<  8);
11521                 cacheline_sz_reg |= ((tp->pci_hdr_type     & 0xff) << 16);
11522                 cacheline_sz_reg |= ((tp->pci_bist         & 0xff) << 24);
11523
11524                 pci_write_config_dword(tp->pdev, TG3PCI_CACHELINESZ,
11525                                        cacheline_sz_reg);
11526         }
11527
11528         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
11529             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
11530                 tp->pcix_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_PCIX);
11531                 if (!tp->pcix_cap) {
11532                         printk(KERN_ERR PFX "Cannot find PCI-X "
11533                                             "capability, aborting.\n");
11534                         return -EIO;
11535                 }
11536         }
11537
11538         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
11539                               &pci_state_reg);
11540
11541         if (tp->pcix_cap && (pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0) {
11542                 tp->tg3_flags |= TG3_FLAG_PCIX_MODE;
11543
11544                 /* If this is a 5700 BX chipset, and we are in PCI-X
11545                  * mode, enable register write workaround.
11546                  *
11547                  * The workaround is to use indirect register accesses
11548                  * for all chip writes not to mailbox registers.
11549                  */
11550                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX) {
11551                         u32 pm_reg;
11552
11553                         tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
11554
11555                         /* The chip can have it's power management PCI config
11556                          * space registers clobbered due to this bug.
11557                          * So explicitly force the chip into D0 here.
11558                          */
11559                         pci_read_config_dword(tp->pdev,
11560                                               tp->pm_cap + PCI_PM_CTRL,
11561                                               &pm_reg);
11562                         pm_reg &= ~PCI_PM_CTRL_STATE_MASK;
11563                         pm_reg |= PCI_PM_CTRL_PME_ENABLE | 0 /* D0 */;
11564                         pci_write_config_dword(tp->pdev,
11565                                                tp->pm_cap + PCI_PM_CTRL,
11566                                                pm_reg);
11567
11568                         /* Also, force SERR#/PERR# in PCI command. */
11569                         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11570                         pci_cmd |= PCI_COMMAND_PARITY | PCI_COMMAND_SERR;
11571                         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11572                 }
11573         }
11574
11575         /* 5700 BX chips need to have their TX producer index mailboxes
11576          * written twice to workaround a bug.
11577          */
11578         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX)
11579                 tp->tg3_flags |= TG3_FLAG_TXD_MBOX_HWBUG;
11580
11581         if ((pci_state_reg & PCISTATE_BUS_SPEED_HIGH) != 0)
11582                 tp->tg3_flags |= TG3_FLAG_PCI_HIGH_SPEED;
11583         if ((pci_state_reg & PCISTATE_BUS_32BIT) != 0)
11584                 tp->tg3_flags |= TG3_FLAG_PCI_32BIT;
11585
11586         /* Chip-specific fixup from Broadcom driver */
11587         if ((tp->pci_chip_rev_id == CHIPREV_ID_5704_A0) &&
11588             (!(pci_state_reg & PCISTATE_RETRY_SAME_DMA))) {
11589                 pci_state_reg |= PCISTATE_RETRY_SAME_DMA;
11590                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, pci_state_reg);
11591         }
11592
11593         /* Default fast path register access methods */
11594         tp->read32 = tg3_read32;
11595         tp->write32 = tg3_write32;
11596         tp->read32_mbox = tg3_read32;
11597         tp->write32_mbox = tg3_write32;
11598         tp->write32_tx_mbox = tg3_write32;
11599         tp->write32_rx_mbox = tg3_write32;
11600
11601         /* Various workaround register access methods */
11602         if (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG)
11603                 tp->write32 = tg3_write_indirect_reg32;
11604         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
11605                  ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
11606                   tp->pci_chip_rev_id == CHIPREV_ID_5750_A0)) {
11607                 /*
11608                  * Back to back register writes can cause problems on these
11609                  * chips, the workaround is to read back all reg writes
11610                  * except those to mailbox regs.
11611                  *
11612                  * See tg3_write_indirect_reg32().
11613                  */
11614                 tp->write32 = tg3_write_flush_reg32;
11615         }
11616
11617
11618         if ((tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG) ||
11619             (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)) {
11620                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
11621                 if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
11622                         tp->write32_rx_mbox = tg3_write_flush_reg32;
11623         }
11624
11625         if (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND) {
11626                 tp->read32 = tg3_read_indirect_reg32;
11627                 tp->write32 = tg3_write_indirect_reg32;
11628                 tp->read32_mbox = tg3_read_indirect_mbox;
11629                 tp->write32_mbox = tg3_write_indirect_mbox;
11630                 tp->write32_tx_mbox = tg3_write_indirect_mbox;
11631                 tp->write32_rx_mbox = tg3_write_indirect_mbox;
11632
11633                 iounmap(tp->regs);
11634                 tp->regs = NULL;
11635
11636                 pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11637                 pci_cmd &= ~PCI_COMMAND_MEMORY;
11638                 pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11639         }
11640         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
11641                 tp->read32_mbox = tg3_read32_mbox_5906;
11642                 tp->write32_mbox = tg3_write32_mbox_5906;
11643                 tp->write32_tx_mbox = tg3_write32_mbox_5906;
11644                 tp->write32_rx_mbox = tg3_write32_mbox_5906;
11645         }
11646
11647         if (tp->write32 == tg3_write_indirect_reg32 ||
11648             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
11649              (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
11650               GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)))
11651                 tp->tg3_flags |= TG3_FLAG_SRAM_USE_CONFIG;
11652
11653         /* Get eeprom hw config before calling tg3_set_power_state().
11654          * In particular, the TG3_FLG2_IS_NIC flag must be
11655          * determined before calling tg3_set_power_state() so that
11656          * we know whether or not to switch out of Vaux power.
11657          * When the flag is set, it means that GPIO1 is used for eeprom
11658          * write protect and also implies that it is a LOM where GPIOs
11659          * are not used to switch power.
11660          */
11661         tg3_get_eeprom_hw_cfg(tp);
11662
11663         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
11664                 /* Allow reads and writes to the
11665                  * APE register and memory space.
11666                  */
11667                 pci_state_reg |= PCISTATE_ALLOW_APE_CTLSPC_WR |
11668                                  PCISTATE_ALLOW_APE_SHMEM_WR;
11669                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE,
11670                                        pci_state_reg);
11671         }
11672
11673         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11674             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761) {
11675                 tp->tg3_flags |= TG3_FLAG_CPMU_PRESENT;
11676
11677                 if (tp->pci_chip_rev_id == CHIPREV_ID_5784_A0 ||
11678                     tp->pci_chip_rev_id == CHIPREV_ID_5784_A1 ||
11679                     tp->pci_chip_rev_id == CHIPREV_ID_5761_A0 ||
11680                     tp->pci_chip_rev_id == CHIPREV_ID_5761_A1)
11681                         tp->tg3_flags3 |= TG3_FLG3_5761_5784_AX_FIXES;
11682         }
11683
11684         /* Set up tp->grc_local_ctrl before calling tg3_set_power_state().
11685          * GPIO1 driven high will bring 5700's external PHY out of reset.
11686          * It is also used as eeprom write protect on LOMs.
11687          */
11688         tp->grc_local_ctrl = GRC_LCLCTRL_INT_ON_ATTN | GRC_LCLCTRL_AUTO_SEEPROM;
11689         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
11690             (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT))
11691                 tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
11692                                        GRC_LCLCTRL_GPIO_OUTPUT1);
11693         /* Unused GPIO3 must be driven as output on 5752 because there
11694          * are no pull-up resistors on unused GPIO pins.
11695          */
11696         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
11697                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
11698
11699         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
11700                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
11701
11702         /* Force the chip into D0. */
11703         err = tg3_set_power_state(tp, PCI_D0);
11704         if (err) {
11705                 printk(KERN_ERR PFX "(%s) transition to D0 failed\n",
11706                        pci_name(tp->pdev));
11707                 return err;
11708         }
11709
11710         /* 5700 B0 chips do not support checksumming correctly due
11711          * to hardware bugs.
11712          */
11713         if (tp->pci_chip_rev_id == CHIPREV_ID_5700_B0)
11714                 tp->tg3_flags |= TG3_FLAG_BROKEN_CHECKSUMS;
11715
11716         /* Derive initial jumbo mode from MTU assigned in
11717          * ether_setup() via the alloc_etherdev() call
11718          */
11719         if (tp->dev->mtu > ETH_DATA_LEN &&
11720             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
11721                 tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
11722
11723         /* Determine WakeOnLan speed to use. */
11724         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
11725             tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
11726             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0 ||
11727             tp->pci_chip_rev_id == CHIPREV_ID_5701_B2) {
11728                 tp->tg3_flags &= ~(TG3_FLAG_WOL_SPEED_100MB);
11729         } else {
11730                 tp->tg3_flags |= TG3_FLAG_WOL_SPEED_100MB;
11731         }
11732
11733         /* A few boards don't want Ethernet@WireSpeed phy feature */
11734         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
11735             ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
11736              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) &&
11737              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A1)) ||
11738             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) ||
11739             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
11740                 tp->tg3_flags2 |= TG3_FLG2_NO_ETH_WIRE_SPEED;
11741
11742         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5703_AX ||
11743             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_AX)
11744                 tp->tg3_flags2 |= TG3_FLG2_PHY_ADC_BUG;
11745         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0)
11746                 tp->tg3_flags2 |= TG3_FLG2_PHY_5704_A0_BUG;
11747
11748         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
11749                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11750                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11751                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11752                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761) {
11753                         if (tp->pdev->device != PCI_DEVICE_ID_TIGON3_5756 &&
11754                             tp->pdev->device != PCI_DEVICE_ID_TIGON3_5722)
11755                                 tp->tg3_flags2 |= TG3_FLG2_PHY_JITTER_BUG;
11756                         if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5755M)
11757                                 tp->tg3_flags2 |= TG3_FLG2_PHY_ADJUST_TRIM;
11758                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906)
11759                         tp->tg3_flags2 |= TG3_FLG2_PHY_BER_BUG;
11760         }
11761
11762         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
11763             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) {
11764                 tp->phy_otp = tg3_read_otp_phycfg(tp);
11765                 if (tp->phy_otp == 0)
11766                         tp->phy_otp = TG3_OTP_DEFAULT;
11767         }
11768
11769         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11770             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
11771                 tp->mi_mode = MAC_MI_MODE_500KHZ_CONST;
11772         else
11773                 tp->mi_mode = MAC_MI_MODE_BASE;
11774
11775         tp->coalesce_mode = 0;
11776         if (GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_AX &&
11777             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_BX)
11778                 tp->coalesce_mode |= HOSTCC_MODE_32BYTE;
11779
11780         /* Initialize MAC MI mode, polling disabled. */
11781         tw32_f(MAC_MI_MODE, tp->mi_mode);
11782         udelay(80);
11783
11784         /* Initialize data/descriptor byte/word swapping. */
11785         val = tr32(GRC_MODE);
11786         val &= GRC_MODE_HOST_STACKUP;
11787         tw32(GRC_MODE, val | tp->grc_mode);
11788
11789         tg3_switch_clocks(tp);
11790
11791         /* Clear this out for sanity. */
11792         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
11793
11794         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
11795                               &pci_state_reg);
11796         if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0 &&
11797             (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) == 0) {
11798                 u32 chiprevid = GET_CHIP_REV_ID(tp->misc_host_ctrl);
11799
11800                 if (chiprevid == CHIPREV_ID_5701_A0 ||
11801                     chiprevid == CHIPREV_ID_5701_B0 ||
11802                     chiprevid == CHIPREV_ID_5701_B2 ||
11803                     chiprevid == CHIPREV_ID_5701_B5) {
11804                         void __iomem *sram_base;
11805
11806                         /* Write some dummy words into the SRAM status block
11807                          * area, see if it reads back correctly.  If the return
11808                          * value is bad, force enable the PCIX workaround.
11809                          */
11810                         sram_base = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_STATS_BLK;
11811
11812                         writel(0x00000000, sram_base);
11813                         writel(0x00000000, sram_base + 4);
11814                         writel(0xffffffff, sram_base + 4);
11815                         if (readl(sram_base) != 0x00000000)
11816                                 tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
11817                 }
11818         }
11819
11820         udelay(50);
11821         tg3_nvram_init(tp);
11822
11823         grc_misc_cfg = tr32(GRC_MISC_CFG);
11824         grc_misc_cfg &= GRC_MISC_CFG_BOARD_ID_MASK;
11825
11826         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
11827             (grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788 ||
11828              grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788M))
11829                 tp->tg3_flags2 |= TG3_FLG2_IS_5788;
11830
11831         if (!(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
11832             (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700))
11833                 tp->tg3_flags |= TG3_FLAG_TAGGED_STATUS;
11834         if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
11835                 tp->coalesce_mode |= (HOSTCC_MODE_CLRTICK_RXBD |
11836                                       HOSTCC_MODE_CLRTICK_TXBD);
11837
11838                 tp->misc_host_ctrl |= MISC_HOST_CTRL_TAGGED_STATUS;
11839                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
11840                                        tp->misc_host_ctrl);
11841         }
11842
11843         /* these are limited to 10/100 only */
11844         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
11845              (grc_misc_cfg == 0x8000 || grc_misc_cfg == 0x4000)) ||
11846             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
11847              tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
11848              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901 ||
11849               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901_2 ||
11850               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5705F)) ||
11851             (tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
11852              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5751F ||
11853               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5753F ||
11854               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5787F)) ||
11855             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11856                 tp->tg3_flags |= TG3_FLAG_10_100_ONLY;
11857
11858         err = tg3_phy_probe(tp);
11859         if (err) {
11860                 printk(KERN_ERR PFX "(%s) phy probe failed, err %d\n",
11861                        pci_name(tp->pdev), err);
11862                 /* ... but do not return immediately ... */
11863         }
11864
11865         tg3_read_partno(tp);
11866         tg3_read_fw_ver(tp);
11867
11868         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
11869                 tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
11870         } else {
11871                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
11872                         tp->tg3_flags |= TG3_FLAG_USE_MI_INTERRUPT;
11873                 else
11874                         tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
11875         }
11876
11877         /* 5700 {AX,BX} chips have a broken status block link
11878          * change bit implementation, so we must use the
11879          * status register in those cases.
11880          */
11881         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
11882                 tp->tg3_flags |= TG3_FLAG_USE_LINKCHG_REG;
11883         else
11884                 tp->tg3_flags &= ~TG3_FLAG_USE_LINKCHG_REG;
11885
11886         /* The led_ctrl is set during tg3_phy_probe, here we might
11887          * have to force the link status polling mechanism based
11888          * upon subsystem IDs.
11889          */
11890         if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
11891             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
11892             !(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
11893                 tp->tg3_flags |= (TG3_FLAG_USE_MI_INTERRUPT |
11894                                   TG3_FLAG_USE_LINKCHG_REG);
11895         }
11896
11897         /* For all SERDES we poll the MAC status register. */
11898         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
11899                 tp->tg3_flags |= TG3_FLAG_POLL_SERDES;
11900         else
11901                 tp->tg3_flags &= ~TG3_FLAG_POLL_SERDES;
11902
11903         /* All chips before 5787 can get confused if TX buffers
11904          * straddle the 4GB address boundary in some cases.
11905          */
11906         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11907             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11908             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11909             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
11910             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11911                 tp->dev->hard_start_xmit = tg3_start_xmit;
11912         else
11913                 tp->dev->hard_start_xmit = tg3_start_xmit_dma_bug;
11914
11915         tp->rx_offset = 2;
11916         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
11917             (tp->tg3_flags & TG3_FLAG_PCIX_MODE) != 0)
11918                 tp->rx_offset = 0;
11919
11920         tp->rx_std_max_post = TG3_RX_RING_SIZE;
11921
11922         /* Increment the rx prod index on the rx std ring by at most
11923          * 8 for these chips to workaround hw errata.
11924          */
11925         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
11926             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
11927             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
11928                 tp->rx_std_max_post = 8;
11929
11930         if (tp->tg3_flags & TG3_FLAG_ASPM_WORKAROUND)
11931                 tp->pwrmgmt_thresh = tr32(PCIE_PWR_MGMT_THRESH) &
11932                                      PCIE_PWR_MGMT_L1_THRESH_MSK;
11933
11934         return err;
11935 }
11936
11937 #ifdef CONFIG_SPARC
11938 static int __devinit tg3_get_macaddr_sparc(struct tg3 *tp)
11939 {
11940         struct net_device *dev = tp->dev;
11941         struct pci_dev *pdev = tp->pdev;
11942         struct device_node *dp = pci_device_to_OF_node(pdev);
11943         const unsigned char *addr;
11944         int len;
11945
11946         addr = of_get_property(dp, "local-mac-address", &len);
11947         if (addr && len == 6) {
11948                 memcpy(dev->dev_addr, addr, 6);
11949                 memcpy(dev->perm_addr, dev->dev_addr, 6);
11950                 return 0;
11951         }
11952         return -ENODEV;
11953 }
11954
11955 static int __devinit tg3_get_default_macaddr_sparc(struct tg3 *tp)
11956 {
11957         struct net_device *dev = tp->dev;
11958
11959         memcpy(dev->dev_addr, idprom->id_ethaddr, 6);
11960         memcpy(dev->perm_addr, idprom->id_ethaddr, 6);
11961         return 0;
11962 }
11963 #endif
11964
11965 static int __devinit tg3_get_device_address(struct tg3 *tp)
11966 {
11967         struct net_device *dev = tp->dev;
11968         u32 hi, lo, mac_offset;
11969         int addr_ok = 0;
11970
11971 #ifdef CONFIG_SPARC
11972         if (!tg3_get_macaddr_sparc(tp))
11973                 return 0;
11974 #endif
11975
11976         mac_offset = 0x7c;
11977         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
11978             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
11979                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
11980                         mac_offset = 0xcc;
11981                 if (tg3_nvram_lock(tp))
11982                         tw32_f(NVRAM_CMD, NVRAM_CMD_RESET);
11983                 else
11984                         tg3_nvram_unlock(tp);
11985         }
11986         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11987                 mac_offset = 0x10;
11988
11989         /* First try to get it from MAC address mailbox. */
11990         tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_HIGH_MBOX, &hi);
11991         if ((hi >> 16) == 0x484b) {
11992                 dev->dev_addr[0] = (hi >>  8) & 0xff;
11993                 dev->dev_addr[1] = (hi >>  0) & 0xff;
11994
11995                 tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_LOW_MBOX, &lo);
11996                 dev->dev_addr[2] = (lo >> 24) & 0xff;
11997                 dev->dev_addr[3] = (lo >> 16) & 0xff;
11998                 dev->dev_addr[4] = (lo >>  8) & 0xff;
11999                 dev->dev_addr[5] = (lo >>  0) & 0xff;
12000
12001                 /* Some old bootcode may report a 0 MAC address in SRAM */
12002                 addr_ok = is_valid_ether_addr(&dev->dev_addr[0]);
12003         }
12004         if (!addr_ok) {
12005                 /* Next, try NVRAM. */
12006                 if (!tg3_nvram_read(tp, mac_offset + 0, &hi) &&
12007                     !tg3_nvram_read(tp, mac_offset + 4, &lo)) {
12008                         dev->dev_addr[0] = ((hi >> 16) & 0xff);
12009                         dev->dev_addr[1] = ((hi >> 24) & 0xff);
12010                         dev->dev_addr[2] = ((lo >>  0) & 0xff);
12011                         dev->dev_addr[3] = ((lo >>  8) & 0xff);
12012                         dev->dev_addr[4] = ((lo >> 16) & 0xff);
12013                         dev->dev_addr[5] = ((lo >> 24) & 0xff);
12014                 }
12015                 /* Finally just fetch it out of the MAC control regs. */
12016                 else {
12017                         hi = tr32(MAC_ADDR_0_HIGH);
12018                         lo = tr32(MAC_ADDR_0_LOW);
12019
12020                         dev->dev_addr[5] = lo & 0xff;
12021                         dev->dev_addr[4] = (lo >> 8) & 0xff;
12022                         dev->dev_addr[3] = (lo >> 16) & 0xff;
12023                         dev->dev_addr[2] = (lo >> 24) & 0xff;
12024                         dev->dev_addr[1] = hi & 0xff;
12025                         dev->dev_addr[0] = (hi >> 8) & 0xff;
12026                 }
12027         }
12028
12029         if (!is_valid_ether_addr(&dev->dev_addr[0])) {
12030 #ifdef CONFIG_SPARC
12031                 if (!tg3_get_default_macaddr_sparc(tp))
12032                         return 0;
12033 #endif
12034                 return -EINVAL;
12035         }
12036         memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
12037         return 0;
12038 }
12039
12040 #define BOUNDARY_SINGLE_CACHELINE       1
12041 #define BOUNDARY_MULTI_CACHELINE        2
12042
12043 static u32 __devinit tg3_calc_dma_bndry(struct tg3 *tp, u32 val)
12044 {
12045         int cacheline_size;
12046         u8 byte;
12047         int goal;
12048
12049         pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE, &byte);
12050         if (byte == 0)
12051                 cacheline_size = 1024;
12052         else
12053                 cacheline_size = (int) byte * 4;
12054
12055         /* On 5703 and later chips, the boundary bits have no
12056          * effect.
12057          */
12058         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
12059             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701 &&
12060             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
12061                 goto out;
12062
12063 #if defined(CONFIG_PPC64) || defined(CONFIG_IA64) || defined(CONFIG_PARISC)
12064         goal = BOUNDARY_MULTI_CACHELINE;
12065 #else
12066 #if defined(CONFIG_SPARC64) || defined(CONFIG_ALPHA)
12067         goal = BOUNDARY_SINGLE_CACHELINE;
12068 #else
12069         goal = 0;
12070 #endif
12071 #endif
12072
12073         if (!goal)
12074                 goto out;
12075
12076         /* PCI controllers on most RISC systems tend to disconnect
12077          * when a device tries to burst across a cache-line boundary.
12078          * Therefore, letting tg3 do so just wastes PCI bandwidth.
12079          *
12080          * Unfortunately, for PCI-E there are only limited
12081          * write-side controls for this, and thus for reads
12082          * we will still get the disconnects.  We'll also waste
12083          * these PCI cycles for both read and write for chips
12084          * other than 5700 and 5701 which do not implement the
12085          * boundary bits.
12086          */
12087         if ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
12088             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
12089                 switch (cacheline_size) {
12090                 case 16:
12091                 case 32:
12092                 case 64:
12093                 case 128:
12094                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
12095                                 val |= (DMA_RWCTRL_READ_BNDRY_128_PCIX |
12096                                         DMA_RWCTRL_WRITE_BNDRY_128_PCIX);
12097                         } else {
12098                                 val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
12099                                         DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
12100                         }
12101                         break;
12102
12103                 case 256:
12104                         val |= (DMA_RWCTRL_READ_BNDRY_256_PCIX |
12105                                 DMA_RWCTRL_WRITE_BNDRY_256_PCIX);
12106                         break;
12107
12108                 default:
12109                         val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
12110                                 DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
12111                         break;
12112                 };
12113         } else if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
12114                 switch (cacheline_size) {
12115                 case 16:
12116                 case 32:
12117                 case 64:
12118                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
12119                                 val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
12120                                 val |= DMA_RWCTRL_WRITE_BNDRY_64_PCIE;
12121                                 break;
12122                         }
12123                         /* fallthrough */
12124                 case 128:
12125                 default:
12126                         val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
12127                         val |= DMA_RWCTRL_WRITE_BNDRY_128_PCIE;
12128                         break;
12129                 };
12130         } else {
12131                 switch (cacheline_size) {
12132                 case 16:
12133                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
12134                                 val |= (DMA_RWCTRL_READ_BNDRY_16 |
12135                                         DMA_RWCTRL_WRITE_BNDRY_16);
12136                                 break;
12137                         }
12138                         /* fallthrough */
12139                 case 32:
12140                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
12141                                 val |= (DMA_RWCTRL_READ_BNDRY_32 |
12142                                         DMA_RWCTRL_WRITE_BNDRY_32);
12143                                 break;
12144                         }
12145                         /* fallthrough */
12146                 case 64:
12147                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
12148                                 val |= (DMA_RWCTRL_READ_BNDRY_64 |
12149                                         DMA_RWCTRL_WRITE_BNDRY_64);
12150                                 break;
12151                         }
12152                         /* fallthrough */
12153                 case 128:
12154                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
12155                                 val |= (DMA_RWCTRL_READ_BNDRY_128 |
12156                                         DMA_RWCTRL_WRITE_BNDRY_128);
12157                                 break;
12158                         }
12159                         /* fallthrough */
12160                 case 256:
12161                         val |= (DMA_RWCTRL_READ_BNDRY_256 |
12162                                 DMA_RWCTRL_WRITE_BNDRY_256);
12163                         break;
12164                 case 512:
12165                         val |= (DMA_RWCTRL_READ_BNDRY_512 |
12166                                 DMA_RWCTRL_WRITE_BNDRY_512);
12167                         break;
12168                 case 1024:
12169                 default:
12170                         val |= (DMA_RWCTRL_READ_BNDRY_1024 |
12171                                 DMA_RWCTRL_WRITE_BNDRY_1024);
12172                         break;
12173                 };
12174         }
12175
12176 out:
12177         return val;
12178 }
12179
12180 static int __devinit tg3_do_test_dma(struct tg3 *tp, u32 *buf, dma_addr_t buf_dma, int size, int to_device)
12181 {
12182         struct tg3_internal_buffer_desc test_desc;
12183         u32 sram_dma_descs;
12184         int i, ret;
12185
12186         sram_dma_descs = NIC_SRAM_DMA_DESC_POOL_BASE;
12187
12188         tw32(FTQ_RCVBD_COMP_FIFO_ENQDEQ, 0);
12189         tw32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ, 0);
12190         tw32(RDMAC_STATUS, 0);
12191         tw32(WDMAC_STATUS, 0);
12192
12193         tw32(BUFMGR_MODE, 0);
12194         tw32(FTQ_RESET, 0);
12195
12196         test_desc.addr_hi = ((u64) buf_dma) >> 32;
12197         test_desc.addr_lo = buf_dma & 0xffffffff;
12198         test_desc.nic_mbuf = 0x00002100;
12199         test_desc.len = size;
12200
12201         /*
12202          * HP ZX1 was seeing test failures for 5701 cards running at 33Mhz
12203          * the *second* time the tg3 driver was getting loaded after an
12204          * initial scan.
12205          *
12206          * Broadcom tells me:
12207          *   ...the DMA engine is connected to the GRC block and a DMA
12208          *   reset may affect the GRC block in some unpredictable way...
12209          *   The behavior of resets to individual blocks has not been tested.
12210          *
12211          * Broadcom noted the GRC reset will also reset all sub-components.
12212          */
12213         if (to_device) {
12214                 test_desc.cqid_sqid = (13 << 8) | 2;
12215
12216                 tw32_f(RDMAC_MODE, RDMAC_MODE_ENABLE);
12217                 udelay(40);
12218         } else {
12219                 test_desc.cqid_sqid = (16 << 8) | 7;
12220
12221                 tw32_f(WDMAC_MODE, WDMAC_MODE_ENABLE);
12222                 udelay(40);
12223         }
12224         test_desc.flags = 0x00000005;
12225
12226         for (i = 0; i < (sizeof(test_desc) / sizeof(u32)); i++) {
12227                 u32 val;
12228
12229                 val = *(((u32 *)&test_desc) + i);
12230                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR,
12231                                        sram_dma_descs + (i * sizeof(u32)));
12232                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
12233         }
12234         pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
12235
12236         if (to_device) {
12237                 tw32(FTQ_DMA_HIGH_READ_FIFO_ENQDEQ, sram_dma_descs);
12238         } else {
12239                 tw32(FTQ_DMA_HIGH_WRITE_FIFO_ENQDEQ, sram_dma_descs);
12240         }
12241
12242         ret = -ENODEV;
12243         for (i = 0; i < 40; i++) {
12244                 u32 val;
12245
12246                 if (to_device)
12247                         val = tr32(FTQ_RCVBD_COMP_FIFO_ENQDEQ);
12248                 else
12249                         val = tr32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ);
12250                 if ((val & 0xffff) == sram_dma_descs) {
12251                         ret = 0;
12252                         break;
12253                 }
12254
12255                 udelay(100);
12256         }
12257
12258         return ret;
12259 }
12260
12261 #define TEST_BUFFER_SIZE        0x2000
12262
12263 static int __devinit tg3_test_dma(struct tg3 *tp)
12264 {
12265         dma_addr_t buf_dma;
12266         u32 *buf, saved_dma_rwctrl;
12267         int ret;
12268
12269         buf = pci_alloc_consistent(tp->pdev, TEST_BUFFER_SIZE, &buf_dma);
12270         if (!buf) {
12271                 ret = -ENOMEM;
12272                 goto out_nofree;
12273         }
12274
12275         tp->dma_rwctrl = ((0x7 << DMA_RWCTRL_PCI_WRITE_CMD_SHIFT) |
12276                           (0x6 << DMA_RWCTRL_PCI_READ_CMD_SHIFT));
12277
12278         tp->dma_rwctrl = tg3_calc_dma_bndry(tp, tp->dma_rwctrl);
12279
12280         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
12281                 /* DMA read watermark not used on PCIE */
12282                 tp->dma_rwctrl |= 0x00180000;
12283         } else if (!(tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
12284                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 ||
12285                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)
12286                         tp->dma_rwctrl |= 0x003f0000;
12287                 else
12288                         tp->dma_rwctrl |= 0x003f000f;
12289         } else {
12290                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
12291                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
12292                         u32 ccval = (tr32(TG3PCI_CLOCK_CTRL) & 0x1f);
12293                         u32 read_water = 0x7;
12294
12295                         /* If the 5704 is behind the EPB bridge, we can
12296                          * do the less restrictive ONE_DMA workaround for
12297                          * better performance.
12298                          */
12299                         if ((tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) &&
12300                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
12301                                 tp->dma_rwctrl |= 0x8000;
12302                         else if (ccval == 0x6 || ccval == 0x7)
12303                                 tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
12304
12305                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703)
12306                                 read_water = 4;
12307                         /* Set bit 23 to enable PCIX hw bug fix */
12308                         tp->dma_rwctrl |=
12309                                 (read_water << DMA_RWCTRL_READ_WATER_SHIFT) |
12310                                 (0x3 << DMA_RWCTRL_WRITE_WATER_SHIFT) |
12311                                 (1 << 23);
12312                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780) {
12313                         /* 5780 always in PCIX mode */
12314                         tp->dma_rwctrl |= 0x00144000;
12315                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
12316                         /* 5714 always in PCIX mode */
12317                         tp->dma_rwctrl |= 0x00148000;
12318                 } else {
12319                         tp->dma_rwctrl |= 0x001b000f;
12320                 }
12321         }
12322
12323         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
12324             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
12325                 tp->dma_rwctrl &= 0xfffffff0;
12326
12327         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
12328             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
12329                 /* Remove this if it causes problems for some boards. */
12330                 tp->dma_rwctrl |= DMA_RWCTRL_USE_MEM_READ_MULT;
12331
12332                 /* On 5700/5701 chips, we need to set this bit.
12333                  * Otherwise the chip will issue cacheline transactions
12334                  * to streamable DMA memory with not all the byte
12335                  * enables turned on.  This is an error on several
12336                  * RISC PCI controllers, in particular sparc64.
12337                  *
12338                  * On 5703/5704 chips, this bit has been reassigned
12339                  * a different meaning.  In particular, it is used
12340                  * on those chips to enable a PCI-X workaround.
12341                  */
12342                 tp->dma_rwctrl |= DMA_RWCTRL_ASSERT_ALL_BE;
12343         }
12344
12345         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12346
12347 #if 0
12348         /* Unneeded, already done by tg3_get_invariants.  */
12349         tg3_switch_clocks(tp);
12350 #endif
12351
12352         ret = 0;
12353         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
12354             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
12355                 goto out;
12356
12357         /* It is best to perform DMA test with maximum write burst size
12358          * to expose the 5700/5701 write DMA bug.
12359          */
12360         saved_dma_rwctrl = tp->dma_rwctrl;
12361         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
12362         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12363
12364         while (1) {
12365                 u32 *p = buf, i;
12366
12367                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++)
12368                         p[i] = i;
12369
12370                 /* Send the buffer to the chip. */
12371                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 1);
12372                 if (ret) {
12373                         printk(KERN_ERR "tg3_test_dma() Write the buffer failed %d\n", ret);
12374                         break;
12375                 }
12376
12377 #if 0
12378                 /* validate data reached card RAM correctly. */
12379                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
12380                         u32 val;
12381                         tg3_read_mem(tp, 0x2100 + (i*4), &val);
12382                         if (le32_to_cpu(val) != p[i]) {
12383                                 printk(KERN_ERR "  tg3_test_dma()  Card buffer corrupted on write! (%d != %d)\n", val, i);
12384                                 /* ret = -ENODEV here? */
12385                         }
12386                         p[i] = 0;
12387                 }
12388 #endif
12389                 /* Now read it back. */
12390                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 0);
12391                 if (ret) {
12392                         printk(KERN_ERR "tg3_test_dma() Read the buffer failed %d\n", ret);
12393
12394                         break;
12395                 }
12396
12397                 /* Verify it. */
12398                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
12399                         if (p[i] == i)
12400                                 continue;
12401
12402                         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
12403                             DMA_RWCTRL_WRITE_BNDRY_16) {
12404                                 tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
12405                                 tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
12406                                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12407                                 break;
12408                         } else {
12409                                 printk(KERN_ERR "tg3_test_dma() buffer corrupted on read back! (%d != %d)\n", p[i], i);
12410                                 ret = -ENODEV;
12411                                 goto out;
12412                         }
12413                 }
12414
12415                 if (i == (TEST_BUFFER_SIZE / sizeof(u32))) {
12416                         /* Success. */
12417                         ret = 0;
12418                         break;
12419                 }
12420         }
12421         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
12422             DMA_RWCTRL_WRITE_BNDRY_16) {
12423                 static struct pci_device_id dma_wait_state_chipsets[] = {
12424                         { PCI_DEVICE(PCI_VENDOR_ID_APPLE,
12425                                      PCI_DEVICE_ID_APPLE_UNI_N_PCI15) },
12426                         { },
12427                 };
12428
12429                 /* DMA test passed without adjusting DMA boundary,
12430                  * now look for chipsets that are known to expose the
12431                  * DMA bug without failing the test.
12432                  */
12433                 if (pci_dev_present(dma_wait_state_chipsets)) {
12434                         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
12435                         tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
12436                 }
12437                 else
12438                         /* Safe to use the calculated DMA boundary. */
12439                         tp->dma_rwctrl = saved_dma_rwctrl;
12440
12441                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
12442         }
12443
12444 out:
12445         pci_free_consistent(tp->pdev, TEST_BUFFER_SIZE, buf, buf_dma);
12446 out_nofree:
12447         return ret;
12448 }
12449
12450 static void __devinit tg3_init_link_config(struct tg3 *tp)
12451 {
12452         tp->link_config.advertising =
12453                 (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
12454                  ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
12455                  ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full |
12456                  ADVERTISED_Autoneg | ADVERTISED_MII);
12457         tp->link_config.speed = SPEED_INVALID;
12458         tp->link_config.duplex = DUPLEX_INVALID;
12459         tp->link_config.autoneg = AUTONEG_ENABLE;
12460         tp->link_config.active_speed = SPEED_INVALID;
12461         tp->link_config.active_duplex = DUPLEX_INVALID;
12462         tp->link_config.phy_is_low_power = 0;
12463         tp->link_config.orig_speed = SPEED_INVALID;
12464         tp->link_config.orig_duplex = DUPLEX_INVALID;
12465         tp->link_config.orig_autoneg = AUTONEG_INVALID;
12466 }
12467
12468 static void __devinit tg3_init_bufmgr_config(struct tg3 *tp)
12469 {
12470         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
12471                 tp->bufmgr_config.mbuf_read_dma_low_water =
12472                         DEFAULT_MB_RDMA_LOW_WATER_5705;
12473                 tp->bufmgr_config.mbuf_mac_rx_low_water =
12474                         DEFAULT_MB_MACRX_LOW_WATER_5705;
12475                 tp->bufmgr_config.mbuf_high_water =
12476                         DEFAULT_MB_HIGH_WATER_5705;
12477                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
12478                         tp->bufmgr_config.mbuf_mac_rx_low_water =
12479                                 DEFAULT_MB_MACRX_LOW_WATER_5906;
12480                         tp->bufmgr_config.mbuf_high_water =
12481                                 DEFAULT_MB_HIGH_WATER_5906;
12482                 }
12483
12484                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
12485                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO_5780;
12486                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
12487                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO_5780;
12488                 tp->bufmgr_config.mbuf_high_water_jumbo =
12489                         DEFAULT_MB_HIGH_WATER_JUMBO_5780;
12490         } else {
12491                 tp->bufmgr_config.mbuf_read_dma_low_water =
12492                         DEFAULT_MB_RDMA_LOW_WATER;
12493                 tp->bufmgr_config.mbuf_mac_rx_low_water =
12494                         DEFAULT_MB_MACRX_LOW_WATER;
12495                 tp->bufmgr_config.mbuf_high_water =
12496                         DEFAULT_MB_HIGH_WATER;
12497
12498                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
12499                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO;
12500                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
12501                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO;
12502                 tp->bufmgr_config.mbuf_high_water_jumbo =
12503                         DEFAULT_MB_HIGH_WATER_JUMBO;
12504         }
12505
12506         tp->bufmgr_config.dma_low_water = DEFAULT_DMA_LOW_WATER;
12507         tp->bufmgr_config.dma_high_water = DEFAULT_DMA_HIGH_WATER;
12508 }
12509
12510 static char * __devinit tg3_phy_string(struct tg3 *tp)
12511 {
12512         switch (tp->phy_id & PHY_ID_MASK) {
12513         case PHY_ID_BCM5400:    return "5400";
12514         case PHY_ID_BCM5401:    return "5401";
12515         case PHY_ID_BCM5411:    return "5411";
12516         case PHY_ID_BCM5701:    return "5701";
12517         case PHY_ID_BCM5703:    return "5703";
12518         case PHY_ID_BCM5704:    return "5704";
12519         case PHY_ID_BCM5705:    return "5705";
12520         case PHY_ID_BCM5750:    return "5750";
12521         case PHY_ID_BCM5752:    return "5752";
12522         case PHY_ID_BCM5714:    return "5714";
12523         case PHY_ID_BCM5780:    return "5780";
12524         case PHY_ID_BCM5755:    return "5755";
12525         case PHY_ID_BCM5787:    return "5787";
12526         case PHY_ID_BCM5784:    return "5784";
12527         case PHY_ID_BCM5756:    return "5722/5756";
12528         case PHY_ID_BCM5906:    return "5906";
12529         case PHY_ID_BCM5761:    return "5761";
12530         case PHY_ID_BCM8002:    return "8002/serdes";
12531         case 0:                 return "serdes";
12532         default:                return "unknown";
12533         };
12534 }
12535
12536 static char * __devinit tg3_bus_string(struct tg3 *tp, char *str)
12537 {
12538         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
12539                 strcpy(str, "PCI Express");
12540                 return str;
12541         } else if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
12542                 u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL) & 0x1f;
12543
12544                 strcpy(str, "PCIX:");
12545
12546                 if ((clock_ctrl == 7) ||
12547                     ((tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK) ==
12548                      GRC_MISC_CFG_BOARD_ID_5704CIOBE))
12549                         strcat(str, "133MHz");
12550                 else if (clock_ctrl == 0)
12551                         strcat(str, "33MHz");
12552                 else if (clock_ctrl == 2)
12553                         strcat(str, "50MHz");
12554                 else if (clock_ctrl == 4)
12555                         strcat(str, "66MHz");
12556                 else if (clock_ctrl == 6)
12557                         strcat(str, "100MHz");
12558         } else {
12559                 strcpy(str, "PCI:");
12560                 if (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED)
12561                         strcat(str, "66MHz");
12562                 else
12563                         strcat(str, "33MHz");
12564         }
12565         if (tp->tg3_flags & TG3_FLAG_PCI_32BIT)
12566                 strcat(str, ":32-bit");
12567         else
12568                 strcat(str, ":64-bit");
12569         return str;
12570 }
12571
12572 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *tp)
12573 {
12574         struct pci_dev *peer;
12575         unsigned int func, devnr = tp->pdev->devfn & ~7;
12576
12577         for (func = 0; func < 8; func++) {
12578                 peer = pci_get_slot(tp->pdev->bus, devnr | func);
12579                 if (peer && peer != tp->pdev)
12580                         break;
12581                 pci_dev_put(peer);
12582         }
12583         /* 5704 can be configured in single-port mode, set peer to
12584          * tp->pdev in that case.
12585          */
12586         if (!peer) {
12587                 peer = tp->pdev;
12588                 return peer;
12589         }
12590
12591         /*
12592          * We don't need to keep the refcount elevated; there's no way
12593          * to remove one half of this device without removing the other
12594          */
12595         pci_dev_put(peer);
12596
12597         return peer;
12598 }
12599
12600 static void __devinit tg3_init_coal(struct tg3 *tp)
12601 {
12602         struct ethtool_coalesce *ec = &tp->coal;
12603
12604         memset(ec, 0, sizeof(*ec));
12605         ec->cmd = ETHTOOL_GCOALESCE;
12606         ec->rx_coalesce_usecs = LOW_RXCOL_TICKS;
12607         ec->tx_coalesce_usecs = LOW_TXCOL_TICKS;
12608         ec->rx_max_coalesced_frames = LOW_RXMAX_FRAMES;
12609         ec->tx_max_coalesced_frames = LOW_TXMAX_FRAMES;
12610         ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT;
12611         ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT;
12612         ec->rx_max_coalesced_frames_irq = DEFAULT_RXCOAL_MAXF_INT;
12613         ec->tx_max_coalesced_frames_irq = DEFAULT_TXCOAL_MAXF_INT;
12614         ec->stats_block_coalesce_usecs = DEFAULT_STAT_COAL_TICKS;
12615
12616         if (tp->coalesce_mode & (HOSTCC_MODE_CLRTICK_RXBD |
12617                                  HOSTCC_MODE_CLRTICK_TXBD)) {
12618                 ec->rx_coalesce_usecs = LOW_RXCOL_TICKS_CLRTCKS;
12619                 ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT_CLRTCKS;
12620                 ec->tx_coalesce_usecs = LOW_TXCOL_TICKS_CLRTCKS;
12621                 ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT_CLRTCKS;
12622         }
12623
12624         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
12625                 ec->rx_coalesce_usecs_irq = 0;
12626                 ec->tx_coalesce_usecs_irq = 0;
12627                 ec->stats_block_coalesce_usecs = 0;
12628         }
12629 }
12630
12631 static int __devinit tg3_init_one(struct pci_dev *pdev,
12632                                   const struct pci_device_id *ent)
12633 {
12634         static int tg3_version_printed = 0;
12635         resource_size_t tg3reg_base;
12636         unsigned long tg3reg_len;
12637         struct net_device *dev;
12638         struct tg3 *tp;
12639         int err, pm_cap;
12640         char str[40];
12641         u64 dma_mask, persist_dma_mask;
12642         DECLARE_MAC_BUF(mac);
12643
12644         if (tg3_version_printed++ == 0)
12645                 printk(KERN_INFO "%s", version);
12646
12647         err = pci_enable_device(pdev);
12648         if (err) {
12649                 printk(KERN_ERR PFX "Cannot enable PCI device, "
12650                        "aborting.\n");
12651                 return err;
12652         }
12653
12654         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
12655                 printk(KERN_ERR PFX "Cannot find proper PCI device "
12656                        "base address, aborting.\n");
12657                 err = -ENODEV;
12658                 goto err_out_disable_pdev;
12659         }
12660
12661         err = pci_request_regions(pdev, DRV_MODULE_NAME);
12662         if (err) {
12663                 printk(KERN_ERR PFX "Cannot obtain PCI resources, "
12664                        "aborting.\n");
12665                 goto err_out_disable_pdev;
12666         }
12667
12668         pci_set_master(pdev);
12669
12670         /* Find power-management capability. */
12671         pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM);
12672         if (pm_cap == 0) {
12673                 printk(KERN_ERR PFX "Cannot find PowerManagement capability, "
12674                        "aborting.\n");
12675                 err = -EIO;
12676                 goto err_out_free_res;
12677         }
12678
12679         tg3reg_base = pci_resource_start(pdev, 0);
12680         tg3reg_len = pci_resource_len(pdev, 0);
12681
12682         dev = alloc_etherdev(sizeof(*tp));
12683         if (!dev) {
12684                 printk(KERN_ERR PFX "Etherdev alloc failed, aborting.\n");
12685                 err = -ENOMEM;
12686                 goto err_out_free_res;
12687         }
12688
12689         SET_NETDEV_DEV(dev, &pdev->dev);
12690
12691 #if TG3_VLAN_TAG_USED
12692         dev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
12693         dev->vlan_rx_register = tg3_vlan_rx_register;
12694 #endif
12695
12696         tp = netdev_priv(dev);
12697         tp->pdev = pdev;
12698         tp->dev = dev;
12699         tp->pm_cap = pm_cap;
12700         tp->mac_mode = TG3_DEF_MAC_MODE;
12701         tp->rx_mode = TG3_DEF_RX_MODE;
12702         tp->tx_mode = TG3_DEF_TX_MODE;
12703
12704         if (tg3_debug > 0)
12705                 tp->msg_enable = tg3_debug;
12706         else
12707                 tp->msg_enable = TG3_DEF_MSG_ENABLE;
12708
12709         /* The word/byte swap controls here control register access byte
12710          * swapping.  DMA data byte swapping is controlled in the GRC_MODE
12711          * setting below.
12712          */
12713         tp->misc_host_ctrl =
12714                 MISC_HOST_CTRL_MASK_PCI_INT |
12715                 MISC_HOST_CTRL_WORD_SWAP |
12716                 MISC_HOST_CTRL_INDIR_ACCESS |
12717                 MISC_HOST_CTRL_PCISTATE_RW;
12718
12719         /* The NONFRM (non-frame) byte/word swap controls take effect
12720          * on descriptor entries, anything which isn't packet data.
12721          *
12722          * The StrongARM chips on the board (one for tx, one for rx)
12723          * are running in big-endian mode.
12724          */
12725         tp->grc_mode = (GRC_MODE_WSWAP_DATA | GRC_MODE_BSWAP_DATA |
12726                         GRC_MODE_WSWAP_NONFRM_DATA);
12727 #ifdef __BIG_ENDIAN
12728         tp->grc_mode |= GRC_MODE_BSWAP_NONFRM_DATA;
12729 #endif
12730         spin_lock_init(&tp->lock);
12731         spin_lock_init(&tp->indirect_lock);
12732         INIT_WORK(&tp->reset_task, tg3_reset_task);
12733
12734         tp->regs = ioremap_nocache(tg3reg_base, tg3reg_len);
12735         if (!tp->regs) {
12736                 printk(KERN_ERR PFX "Cannot map device registers, "
12737                        "aborting.\n");
12738                 err = -ENOMEM;
12739                 goto err_out_free_dev;
12740         }
12741
12742         tg3_init_link_config(tp);
12743
12744         tp->rx_pending = TG3_DEF_RX_RING_PENDING;
12745         tp->rx_jumbo_pending = TG3_DEF_RX_JUMBO_RING_PENDING;
12746         tp->tx_pending = TG3_DEF_TX_RING_PENDING;
12747
12748         dev->open = tg3_open;
12749         dev->stop = tg3_close;
12750         dev->get_stats = tg3_get_stats;
12751         dev->set_multicast_list = tg3_set_rx_mode;
12752         dev->set_mac_address = tg3_set_mac_addr;
12753         dev->do_ioctl = tg3_ioctl;
12754         dev->tx_timeout = tg3_tx_timeout;
12755         netif_napi_add(dev, &tp->napi, tg3_poll, 64);
12756         dev->ethtool_ops = &tg3_ethtool_ops;
12757         dev->watchdog_timeo = TG3_TX_TIMEOUT;
12758         dev->change_mtu = tg3_change_mtu;
12759         dev->irq = pdev->irq;
12760 #ifdef CONFIG_NET_POLL_CONTROLLER
12761         dev->poll_controller = tg3_poll_controller;
12762 #endif
12763
12764         err = tg3_get_invariants(tp);
12765         if (err) {
12766                 printk(KERN_ERR PFX "Problem fetching invariants of chip, "
12767                        "aborting.\n");
12768                 goto err_out_iounmap;
12769         }
12770
12771         /* The EPB bridge inside 5714, 5715, and 5780 and any
12772          * device behind the EPB cannot support DMA addresses > 40-bit.
12773          * On 64-bit systems with IOMMU, use 40-bit dma_mask.
12774          * On 64-bit systems without IOMMU, use 64-bit dma_mask and
12775          * do DMA address check in tg3_start_xmit().
12776          */
12777         if (tp->tg3_flags2 & TG3_FLG2_IS_5788)
12778                 persist_dma_mask = dma_mask = DMA_32BIT_MASK;
12779         else if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) {
12780                 persist_dma_mask = dma_mask = DMA_40BIT_MASK;
12781 #ifdef CONFIG_HIGHMEM
12782                 dma_mask = DMA_64BIT_MASK;
12783 #endif
12784         } else
12785                 persist_dma_mask = dma_mask = DMA_64BIT_MASK;
12786
12787         /* Configure DMA attributes. */
12788         if (dma_mask > DMA_32BIT_MASK) {
12789                 err = pci_set_dma_mask(pdev, dma_mask);
12790                 if (!err) {
12791                         dev->features |= NETIF_F_HIGHDMA;
12792                         err = pci_set_consistent_dma_mask(pdev,
12793                                                           persist_dma_mask);
12794                         if (err < 0) {
12795                                 printk(KERN_ERR PFX "Unable to obtain 64 bit "
12796                                        "DMA for consistent allocations\n");
12797                                 goto err_out_iounmap;
12798                         }
12799                 }
12800         }
12801         if (err || dma_mask == DMA_32BIT_MASK) {
12802                 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
12803                 if (err) {
12804                         printk(KERN_ERR PFX "No usable DMA configuration, "
12805                                "aborting.\n");
12806                         goto err_out_iounmap;
12807                 }
12808         }
12809
12810         tg3_init_bufmgr_config(tp);
12811
12812         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
12813                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
12814         }
12815         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
12816             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
12817             tp->pci_chip_rev_id == CHIPREV_ID_5705_A0 ||
12818             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 ||
12819             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
12820                 tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
12821         } else {
12822                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE | TG3_FLG2_TSO_BUG;
12823         }
12824
12825         /* TSO is on by default on chips that support hardware TSO.
12826          * Firmware TSO on older chips gives lower performance, so it
12827          * is off by default, but can be enabled using ethtool.
12828          */
12829         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
12830                 dev->features |= NETIF_F_TSO;
12831                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO_2) &&
12832                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5906))
12833                         dev->features |= NETIF_F_TSO6;
12834                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
12835                         dev->features |= NETIF_F_TSO_ECN;
12836         }
12837
12838
12839         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 &&
12840             !(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) &&
12841             !(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH)) {
12842                 tp->tg3_flags2 |= TG3_FLG2_MAX_RXPEND_64;
12843                 tp->rx_pending = 63;
12844         }
12845
12846         err = tg3_get_device_address(tp);
12847         if (err) {
12848                 printk(KERN_ERR PFX "Could not obtain valid ethernet address, "
12849                        "aborting.\n");
12850                 goto err_out_iounmap;
12851         }
12852
12853         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
12854                 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
12855                         printk(KERN_ERR PFX "Cannot find proper PCI device "
12856                                "base address for APE, aborting.\n");
12857                         err = -ENODEV;
12858                         goto err_out_iounmap;
12859                 }
12860
12861                 tg3reg_base = pci_resource_start(pdev, 2);
12862                 tg3reg_len = pci_resource_len(pdev, 2);
12863
12864                 tp->aperegs = ioremap_nocache(tg3reg_base, tg3reg_len);
12865                 if (!tp->aperegs) {
12866                         printk(KERN_ERR PFX "Cannot map APE registers, "
12867                                "aborting.\n");
12868                         err = -ENOMEM;
12869                         goto err_out_iounmap;
12870                 }
12871
12872                 tg3_ape_lock_init(tp);
12873         }
12874
12875         /*
12876          * Reset chip in case UNDI or EFI driver did not shutdown
12877          * DMA self test will enable WDMAC and we'll see (spurious)
12878          * pending DMA on the PCI bus at that point.
12879          */
12880         if ((tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE) ||
12881             (tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
12882                 tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
12883                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
12884         }
12885
12886         err = tg3_test_dma(tp);
12887         if (err) {
12888                 printk(KERN_ERR PFX "DMA engine test failed, aborting.\n");
12889                 goto err_out_apeunmap;
12890         }
12891
12892         /* Tigon3 can do ipv4 only... and some chips have buggy
12893          * checksumming.
12894          */
12895         if ((tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) == 0) {
12896                 dev->features |= NETIF_F_IP_CSUM | NETIF_F_SG;
12897                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
12898                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
12899                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
12900                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
12901                         dev->features |= NETIF_F_IPV6_CSUM;
12902
12903                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
12904         } else
12905                 tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
12906
12907         /* flow control autonegotiation is default behavior */
12908         tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
12909         tp->link_config.flowctrl = TG3_FLOW_CTRL_TX | TG3_FLOW_CTRL_RX;
12910
12911         tg3_init_coal(tp);
12912
12913         pci_set_drvdata(pdev, dev);
12914
12915         err = register_netdev(dev);
12916         if (err) {
12917                 printk(KERN_ERR PFX "Cannot register net device, "
12918                        "aborting.\n");
12919                 goto err_out_apeunmap;
12920         }
12921
12922         printk(KERN_INFO "%s: Tigon3 [partno(%s) rev %04x PHY(%s)] "
12923                "(%s) %s Ethernet %s\n",
12924                dev->name,
12925                tp->board_part_number,
12926                tp->pci_chip_rev_id,
12927                tg3_phy_string(tp),
12928                tg3_bus_string(tp, str),
12929                ((tp->tg3_flags & TG3_FLAG_10_100_ONLY) ? "10/100Base-TX" :
12930                 ((tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) ? "1000Base-SX" :
12931                  "10/100/1000Base-T")),
12932                print_mac(mac, dev->dev_addr));
12933
12934         printk(KERN_INFO "%s: RXcsums[%d] LinkChgREG[%d] "
12935                "MIirq[%d] ASF[%d] WireSpeed[%d] TSOcap[%d]\n",
12936                dev->name,
12937                (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0,
12938                (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) != 0,
12939                (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) != 0,
12940                (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0,
12941                (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED) == 0,
12942                (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) != 0);
12943         printk(KERN_INFO "%s: dma_rwctrl[%08x] dma_mask[%d-bit]\n",
12944                dev->name, tp->dma_rwctrl,
12945                (pdev->dma_mask == DMA_32BIT_MASK) ? 32 :
12946                 (((u64) pdev->dma_mask == DMA_40BIT_MASK) ? 40 : 64));
12947
12948         return 0;
12949
12950 err_out_apeunmap:
12951         if (tp->aperegs) {
12952                 iounmap(tp->aperegs);
12953                 tp->aperegs = NULL;
12954         }
12955
12956 err_out_iounmap:
12957         if (tp->regs) {
12958                 iounmap(tp->regs);
12959                 tp->regs = NULL;
12960         }
12961
12962 err_out_free_dev:
12963         free_netdev(dev);
12964
12965 err_out_free_res:
12966         pci_release_regions(pdev);
12967
12968 err_out_disable_pdev:
12969         pci_disable_device(pdev);
12970         pci_set_drvdata(pdev, NULL);
12971         return err;
12972 }
12973
12974 static void __devexit tg3_remove_one(struct pci_dev *pdev)
12975 {
12976         struct net_device *dev = pci_get_drvdata(pdev);
12977
12978         if (dev) {
12979                 struct tg3 *tp = netdev_priv(dev);
12980
12981                 flush_scheduled_work();
12982                 unregister_netdev(dev);
12983                 if (tp->aperegs) {
12984                         iounmap(tp->aperegs);
12985                         tp->aperegs = NULL;
12986                 }
12987                 if (tp->regs) {
12988                         iounmap(tp->regs);
12989                         tp->regs = NULL;
12990                 }
12991                 free_netdev(dev);
12992                 pci_release_regions(pdev);
12993                 pci_disable_device(pdev);
12994                 pci_set_drvdata(pdev, NULL);
12995         }
12996 }
12997
12998 static int tg3_suspend(struct pci_dev *pdev, pm_message_t state)
12999 {
13000         struct net_device *dev = pci_get_drvdata(pdev);
13001         struct tg3 *tp = netdev_priv(dev);
13002         int err;
13003
13004         /* PCI register 4 needs to be saved whether netif_running() or not.
13005          * MSI address and data need to be saved if using MSI and
13006          * netif_running().
13007          */
13008         pci_save_state(pdev);
13009
13010         if (!netif_running(dev))
13011                 return 0;
13012
13013         flush_scheduled_work();
13014         tg3_netif_stop(tp);
13015
13016         del_timer_sync(&tp->timer);
13017
13018         tg3_full_lock(tp, 1);
13019         tg3_disable_ints(tp);
13020         tg3_full_unlock(tp);
13021
13022         netif_device_detach(dev);
13023
13024         tg3_full_lock(tp, 0);
13025         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
13026         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
13027         tg3_full_unlock(tp);
13028
13029         err = tg3_set_power_state(tp, pci_choose_state(pdev, state));
13030         if (err) {
13031                 tg3_full_lock(tp, 0);
13032
13033                 tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
13034                 if (tg3_restart_hw(tp, 1))
13035                         goto out;
13036
13037                 tp->timer.expires = jiffies + tp->timer_offset;
13038                 add_timer(&tp->timer);
13039
13040                 netif_device_attach(dev);
13041                 tg3_netif_start(tp);
13042
13043 out:
13044                 tg3_full_unlock(tp);
13045         }
13046
13047         return err;
13048 }
13049
13050 static int tg3_resume(struct pci_dev *pdev)
13051 {
13052         struct net_device *dev = pci_get_drvdata(pdev);
13053         struct tg3 *tp = netdev_priv(dev);
13054         int err;
13055
13056         pci_restore_state(tp->pdev);
13057
13058         if (!netif_running(dev))
13059                 return 0;
13060
13061         err = tg3_set_power_state(tp, PCI_D0);
13062         if (err)
13063                 return err;
13064
13065         netif_device_attach(dev);
13066
13067         tg3_full_lock(tp, 0);
13068
13069         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
13070         err = tg3_restart_hw(tp, 1);
13071         if (err)
13072                 goto out;
13073
13074         tp->timer.expires = jiffies + tp->timer_offset;
13075         add_timer(&tp->timer);
13076
13077         tg3_netif_start(tp);
13078
13079 out:
13080         tg3_full_unlock(tp);
13081
13082         return err;
13083 }
13084
13085 static struct pci_driver tg3_driver = {
13086         .name           = DRV_MODULE_NAME,
13087         .id_table       = tg3_pci_tbl,
13088         .probe          = tg3_init_one,
13089         .remove         = __devexit_p(tg3_remove_one),
13090         .suspend        = tg3_suspend,
13091         .resume         = tg3_resume
13092 };
13093
13094 static int __init tg3_init(void)
13095 {
13096         return pci_register_driver(&tg3_driver);
13097 }
13098
13099 static void __exit tg3_cleanup(void)
13100 {
13101         pci_unregister_driver(&tg3_driver);
13102 }
13103
13104 module_init(tg3_init);
13105 module_exit(tg3_cleanup);