drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static u32 crit_err_count;
  50 u32 int_mod_timer_init;
  51 u32 int_mod_cq_depth_256;
  52 u32 int_mod_cq_depth_128;
  53 u32 int_mod_cq_depth_32;
  54 u32 int_mod_cq_depth_24;
  55 u32 int_mod_cq_depth_16;
  56 u32 int_mod_cq_depth_4;
  57 u32 int_mod_cq_depth_1;
  58
  59 #include "nes_cm.h"
  60
  61 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  62 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  63 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  64                            u8 OneG_Mode);
  65 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  66 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  67 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  68 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  69                                    struct nes_hw_aeqe *aeqe);
  70 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  71 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  72
  73 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  74 static unsigned char *nes_iwarp_state_str[] = {
  75         "Non-Existant",
  76         "Idle",
  77         "RTS",
  78         "Closing",
  79         "RSVD1",
  80         "Terminate",
  81         "Error",
  82         "RSVD2",
  83 };
  84
  85 static unsigned char *nes_tcp_state_str[] = {
  86         "Non-Existant",
  87         "Closed",
  88         "Listen",
  89         "SYN Sent",
  90         "SYN Rcvd",
  91         "Established",
  92         "Close Wait",
  93         "FIN Wait 1",
  94         "Closing",
  95         "Last Ack",
  96         "FIN Wait 2",
  97         "Time Wait",
  98         "RSVD1",
  99         "RSVD2",
 100         "RSVD3",
 101         "RSVD4",
 102 };
 103 #endif
 104
 105
 106 /**
 107  * nes_nic_init_timer_defaults
 108  */
 109 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 110 {
 111         unsigned long flags;
 112         struct nes_adapter *nesadapter = nesdev->nesadapter;
 113         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 114
 115         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 116
 117         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 118         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 119         if (jumbomode) {
 120                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 121                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 122                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 123         } else {
 124                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 125                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 126                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 127         }
 128
 129         /* todo use netdev->mtu to set thresholds */
 130         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 131 }
 132
 133
 134 /**
 135  * nes_nic_init_timer
 136  */
 137 static void  nes_nic_init_timer(struct nes_device *nesdev)
 138 {
 139         unsigned long flags;
 140         struct nes_adapter *nesadapter = nesdev->nesadapter;
 141         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 142
 143         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 144
 145         if (shared_timer->timer_in_use_old == 0) {
 146                 nesdev->deepcq_count = 0;
 147                 shared_timer->timer_direction_upward = 0;
 148                 shared_timer->timer_direction_downward = 0;
 149                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 150                 shared_timer->timer_in_use_old = 0;
 151
 152         }
 153         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 154                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 155                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 156                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 157         }
 158         /* todo use netdev->mtu to set thresholds */
 159         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 160 }
 161
 162
 163 /**
 164  * nes_nic_tune_timer
 165  */
 166 static void nes_nic_tune_timer(struct nes_device *nesdev)
 167 {
 168         unsigned long flags;
 169         struct nes_adapter *nesadapter = nesdev->nesadapter;
 170         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 171         u16 cq_count = nesdev->currcq_count;
 172
 173         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 174
 175         if (shared_timer->cq_count_old <= cq_count)
 176                 shared_timer->cq_direction_downward = 0;
 177         else
 178                 shared_timer->cq_direction_downward++;
 179         shared_timer->cq_count_old = cq_count;
 180         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 181                 if (cq_count <= shared_timer->threshold_low &&
 182                     shared_timer->threshold_low > 4) {
 183                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 184                         shared_timer->cq_direction_downward=0;
 185                         nesdev->currcq_count = 0;
 186                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 187                         return;
 188                 }
 189         }
 190
 191         if (cq_count > 1) {
 192                 nesdev->deepcq_count += cq_count;
 193                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 194                         shared_timer->timer_direction_upward++;
 195                         shared_timer->timer_direction_downward = 0;
 196                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 197                         shared_timer->timer_direction_upward = 0;
 198                         shared_timer->timer_direction_downward = 0;
 199                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 200                         shared_timer->timer_direction_downward++;
 201                         shared_timer->timer_direction_upward = 0;
 202                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 203                         shared_timer->timer_in_use -= 2;
 204                         shared_timer->timer_direction_upward = 0;
 205                         shared_timer->timer_direction_downward++;
 206                 } else {
 207                         shared_timer->timer_in_use -= 4;
 208                         shared_timer->timer_direction_upward = 0;
 209                         shared_timer->timer_direction_downward++;
 210                 }
 211
 212                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 213                         shared_timer->timer_in_use += 3;
 214                         shared_timer->timer_direction_upward = 0;
 215                         shared_timer->timer_direction_downward = 0;
 216                 }
 217                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 218                         shared_timer->timer_in_use -= 4 ;
 219                         shared_timer->timer_direction_downward = 0;
 220                         shared_timer->timer_direction_upward = 0;
 221                 }
 222         }
 223
 224         /* boundary checking */
 225         if (shared_timer->timer_in_use > NES_NIC_FAST_TIMER_HIGH)
 226                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_HIGH;
 227         else if (shared_timer->timer_in_use < NES_NIC_FAST_TIMER_LOW) {
 228                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_LOW;
 229         }
 230
 231         nesdev->currcq_count = 0;
 232
 233         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 234 }
 235
 236
 237 /**
 238  * nes_init_adapter - initialize adapter
 239  */
 240 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 241         struct nes_adapter *nesadapter = NULL;
 242         unsigned long num_pds;
 243         u32 u32temp;
 244         u32 port_count;
 245         u16 max_rq_wrs;
 246         u16 max_sq_wrs;
 247         u32 max_mr;
 248         u32 max_256pbl;
 249         u32 max_4kpbl;
 250         u32 max_qp;
 251         u32 max_irrq;
 252         u32 max_cq;
 253         u32 hte_index_mask;
 254         u32 adapter_size;
 255         u32 arp_table_size;
 256         u16 vendor_id;
 257         u8  OneG_Mode;
 258         u8  func_index;
 259
 260         /* search the list of existing adapters */
 261         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 262                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 263                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 264                                 nesdev->pcidev->devfn,
 265                                 PCI_SLOT(nesadapter->devfn),
 266                                 nesadapter->bus_number,
 267                                 PCI_SLOT(nesdev->pcidev->devfn),
 268                                 nesdev->pcidev->bus->number );
 269                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 270                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 271                         nesadapter->ref_count++;
 272                         return nesadapter;
 273                 }
 274         }
 275
 276         /* no adapter found */
 277         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 278         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 279                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 280                                 hw_rev);
 281                 return NULL;
 282         }
 283
 284         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 285                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 286                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 287                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 288                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 289
 290         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 291
 292
 293         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 294                 return NULL;
 295         if (nes_init_serdes(nesdev, hw_rev, port_count, OneG_Mode))
 296                 return NULL;
 297         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 298
 299         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 300         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 301
 302         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 303         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 304                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 305                                 max_qp, u32temp);
 306                 max_qp = (u32)1 << (u32temp & 0x001f);
 307         }
 308
 309         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 310         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 311                         max_qp, hte_index_mask);
 312
 313         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 314
 315         max_irrq = 1 << (u32temp & 0x001f);
 316
 317         if (max_qp > max_irrq) {
 318                 max_qp = max_irrq;
 319                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 320                                 max_qp);
 321         }
 322
 323         /* there should be no reason to allocate more pds than qps */
 324         if (num_pds > max_qp)
 325                 num_pds = max_qp;
 326
 327         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 328         max_mr = (u32)8192 << (u32temp & 0x7);
 329
 330         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 331         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 332         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 333         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 334
 335         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 336         arp_table_size = 1 << u32temp;
 337
 338         adapter_size = (sizeof(struct nes_adapter) +
 339                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 340         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 341         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 342         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 343         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 345         adapter_size += sizeof(struct nes_qp **) * max_qp;
 346
 347         /* allocate a new adapter struct */
 348         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 349         if (nesadapter == NULL) {
 350                 return NULL;
 351         }
 352
 353         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 354                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 355
 356         /* populate the new nesadapter */
 357         nesadapter->devfn = nesdev->pcidev->devfn;
 358         nesadapter->bus_number = nesdev->pcidev->bus->number;
 359         nesadapter->ref_count = 1;
 360         nesadapter->timer_int_req = 0xffff0000;
 361         nesadapter->OneG_Mode = OneG_Mode;
 362         nesadapter->doorbell_start = nesdev->doorbell_region;
 363
 364         /* nesadapter->tick_delta = clk_divisor; */
 365         nesadapter->hw_rev = hw_rev;
 366         nesadapter->port_count = port_count;
 367
 368         nesadapter->max_qp = max_qp;
 369         nesadapter->hte_index_mask = hte_index_mask;
 370         nesadapter->max_irrq = max_irrq;
 371         nesadapter->max_mr = max_mr;
 372         nesadapter->max_256pbl = max_256pbl - 1;
 373         nesadapter->max_4kpbl = max_4kpbl - 1;
 374         nesadapter->max_cq = max_cq;
 375         nesadapter->free_256pbl = max_256pbl - 1;
 376         nesadapter->free_4kpbl = max_4kpbl - 1;
 377         nesadapter->max_pd = num_pds;
 378         nesadapter->arp_table_size = arp_table_size;
 379
 380         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 381         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 382                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 383                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 384                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 385         } else {
 386                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 387                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 388                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 389                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 390         }
 391         /* Setup and enable the periodic timer */
 392         if (nesadapter->et_rx_coalesce_usecs_irq)
 393                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 394                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 395         else
 396                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 397
 398         nesadapter->base_pd = 1;
 399
 400         nesadapter->device_cap_flags =
 401                 IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW;
 402
 403         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 404                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 405         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 406         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 407         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 408         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 409         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 410
 411
 412         /* mark the usual suspect QPs and CQs as in use */
 413         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 414                 set_bit(u32temp, nesadapter->allocated_qps);
 415                 set_bit(u32temp, nesadapter->allocated_cqs);
 416         }
 417
 418         for (u32temp = 0; u32temp < 20; u32temp++)
 419                 set_bit(u32temp, nesadapter->allocated_pds);
 420         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 421
 422         max_rq_wrs = ((u32temp >> 8) & 3);
 423         switch (max_rq_wrs) {
 424                 case 0:
 425                         max_rq_wrs = 4;
 426                         break;
 427                 case 1:
 428                         max_rq_wrs = 16;
 429                         break;
 430                 case 2:
 431                         max_rq_wrs = 32;
 432                         break;
 433                 case 3:
 434                         max_rq_wrs = 512;
 435                         break;
 436         }
 437
 438         max_sq_wrs = (u32temp & 3);
 439         switch (max_sq_wrs) {
 440                 case 0:
 441                         max_sq_wrs = 4;
 442                         break;
 443                 case 1:
 444                         max_sq_wrs = 16;
 445                         break;
 446                 case 2:
 447                         max_sq_wrs = 32;
 448                         break;
 449                 case 3:
 450                         max_sq_wrs = 512;
 451                         break;
 452         }
 453         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 454         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 455
 456         nesadapter->max_sge = 4;
 457         nesadapter->max_cqe = 32767;
 458
 459         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 460                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 461                 kfree(nesadapter);
 462                 return NULL;
 463         }
 464
 465         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 466         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 467                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 468
 469         /* setup port configuration */
 470         if (nesadapter->port_count == 1) {
 471                 u32temp = 0x00000000;
 472                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 473                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 474                 else
 475                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 476         } else {
 477                 if (nesadapter->port_count == 2)
 478                         u32temp = 0x00000044;
 479                 else
 480                         u32temp = 0x000000e4;
 481                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 482         }
 483
 484         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT, u32temp);
 485         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 486                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 487
 488         spin_lock_init(&nesadapter->resource_lock);
 489         spin_lock_init(&nesadapter->phy_lock);
 490         spin_lock_init(&nesadapter->pbl_lock);
 491         spin_lock_init(&nesadapter->periodic_timer_lock);
 492
 493         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 494         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 495         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 496         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 497
 498         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 499                 u32 pcs_control_status0, pcs_control_status1;
 500                 u32 reset_value;
 501                 u32 i = 0;
 502                 u32 int_cnt = 0;
 503                 u32 ext_cnt = 0;
 504                 unsigned long flags;
 505                 u32 j = 0;
 506
 507                 pcs_control_status0 = nes_read_indexed(nesdev,
 508                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 509                 pcs_control_status1 = nes_read_indexed(nesdev,
 510                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 511
 512                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 513                         pcs_control_status0 = nes_read_indexed(nesdev,
 514                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 515                         pcs_control_status1 = nes_read_indexed(nesdev,
 516                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 517                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 518                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 519                                 int_cnt++;
 520                         msleep(1);
 521                 }
 522                 if (int_cnt > 1) {
 523                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 524                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 525                         mh_detected++;
 526                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 527                         reset_value |= 0x0000003d;
 528                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 529
 530                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 531                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 532                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 533
 534                         pcs_control_status0 = nes_read_indexed(nesdev,
 535                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 536                         pcs_control_status1 = nes_read_indexed(nesdev,
 537                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 538
 539                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 540                                 pcs_control_status0 = nes_read_indexed(nesdev,
 541                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 542                                 pcs_control_status1 = nes_read_indexed(nesdev,
 543                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 544                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 545                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 546                                         if (++ext_cnt > int_cnt) {
 547                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 548                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 549                                                                 0x0000F0C8);
 550                                                 mh_detected++;
 551                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 552                                                 reset_value |= 0x0000003d;
 553                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 554
 555                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 556                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 557                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 558                                                 break;
 559                                         }
 560                                 }
 561                                 msleep(1);
 562                         }
 563                 }
 564         }
 565
 566         if (nesadapter->hw_rev == NE020_REV) {
 567                 init_timer(&nesadapter->mh_timer);
 568                 nesadapter->mh_timer.function = nes_mh_fix;
 569                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 570                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 571                 add_timer(&nesadapter->mh_timer);
 572         } else {
 573                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 574         }
 575
 576         init_timer(&nesadapter->lc_timer);
 577         nesadapter->lc_timer.function = nes_clc;
 578         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 579         nesadapter->lc_timer.data = (unsigned long)nesdev;
 580         add_timer(&nesadapter->lc_timer);
 581
 582         list_add_tail(&nesadapter->list, &nes_adapter_list);
 583
 584         for (func_index = 0; func_index < 8; func_index++) {
 585                 pci_bus_read_config_word(nesdev->pcidev->bus,
 586                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 587                                         func_index), 0, &vendor_id);
 588                 if (vendor_id == 0xffff)
 589                         break;
 590         }
 591         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 592                 func_index, pci_name(nesdev->pcidev));
 593         nesadapter->adapter_fcn_count = func_index;
 594
 595         return nesadapter;
 596 }
 597
 598
 599 /**
 600  * nes_reset_adapter_ne020
 601  */
 602 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 603 {
 604         u32 port_count;
 605         u32 u32temp;
 606         u32 i;
 607
 608         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 609         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 610         /* TODO: assuming that both SERDES are set the same for now */
 611         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 612         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 613                         u32temp, port_count);
 614         if (*OneG_Mode)
 615                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 616         u32temp &= 0xff00ffc0;
 617         switch (port_count) {
 618                 case 1:
 619                         u32temp |= 0x00ee0000;
 620                         break;
 621                 case 2:
 622                         u32temp |= 0x00cc0000;
 623                         break;
 624                 case 4:
 625                         u32temp |= 0x00000000;
 626                         break;
 627                 default:
 628                         return 0;
 629                         break;
 630         }
 631
 632         /* check and do full reset if needed */
 633         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 634                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 635                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 636
 637                 i = 0;
 638                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 639                         mdelay(1);
 640                 if (i >= 10000) {
 641                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 642                         return 0;
 643                 }
 644
 645                 i = 0;
 646                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 647                         mdelay(1);
 648                 if (i >= 10000) {
 649                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 650                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 651                         return 0;
 652                 }
 653         }
 654
 655         /* port reset */
 656         switch (port_count) {
 657                 case 1:
 658                         u32temp |= 0x00ee0010;
 659                         break;
 660                 case 2:
 661                         u32temp |= 0x00cc0030;
 662                         break;
 663                 case 4:
 664                         u32temp |= 0x00000030;
 665                         break;
 666         }
 667
 668         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 669         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 670
 671         i = 0;
 672         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 673                 mdelay(1);
 674         if (i >= 10000) {
 675                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 676                 return 0;
 677         }
 678
 679         /* serdes 0 */
 680         i = 0;
 681         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 682                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 683                 mdelay(1);
 684         if (i >= 5000) {
 685                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 686                 return 0;
 687         }
 688
 689         /* serdes 1 */
 690         if (port_count > 1) {
 691                 i = 0;
 692                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 693                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 694                         mdelay(1);
 695                 if (i >= 5000) {
 696                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 697                         return 0;
 698                 }
 699         }
 700
 701         return port_count;
 702 }
 703
 704
 705 /**
 706  * nes_init_serdes
 707  */
 708 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 709                            u8 OneG_Mode)
 710 {
 711         int i;
 712         u32 u32temp;
 713
 714         if (hw_rev != NE020_REV) {
 715                 /* init serdes 0 */
 716
 717                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 718                 if (!OneG_Mode)
 719                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 720                 if (port_count > 1) {
 721                         /* init serdes 1 */
 722                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 723                         if (!OneG_Mode)
 724                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 725                         }
 726         } else {
 727                 /* init serdes 0 */
 728                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 729                 i = 0;
 730                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 731                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 732                         mdelay(1);
 733                 if (i >= 5000) {
 734                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 735                         return 1;
 736                 }
 737                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 738                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 739                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 740                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 741                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 742                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 743                 if (OneG_Mode)
 744                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 745                 else
 746                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 747
 748                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 749                 if (port_count > 1) {
 750                         /* init serdes 1 */
 751                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 752                         i = 0;
 753                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 754                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 755                                 mdelay(1);
 756                         if (i >= 5000) {
 757                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 758                                 /* return 1; */
 759                         }
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 762                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 763                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 764                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 765                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 766                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 767                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 768                 }
 769         }
 770         return 0;
 771 }
 772
 773
 774 /**
 775  * nes_init_csr_ne020
 776  * Initialize registers for ne020 hardware
 777  */
 778 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 779 {
 780         u32 u32temp;
 781
 782         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 783
 784         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 785         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 786         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 787         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 788         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 789         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 790         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 791         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 792
 793         /* TODO: move these MAC register settings to NIC bringup */
 794         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 795         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 796         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 797         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 798         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 799         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 800         if (port_count > 1) {
 801                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 802                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 803                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 804                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 805                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 806                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 807                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 808         }
 809         if (port_count > 2) {
 810                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 811                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 812                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 813                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 814                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 815                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 816                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 817
 818                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 819                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 820                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 821                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 822                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 823                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 824                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 825         }
 826
 827         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 828         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 829         nes_write_indexed(nesdev, 0x00005004, 0x00020001);
 830         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 831         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 832         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 833         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 834         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 835
 836         /* TODO: move this to code, get from EEPROM */
 837         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 838         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 839         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 840
 841         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 842         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 843
 844         if (hw_rev != NE020_REV) {
 845                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 846                 u32temp |= 0x80000000;
 847                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 848                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 849                 u32temp &= 0x7fffffff;
 850                 u32temp |= 0x7fff0010;
 851                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 852         }
 853 }
 854
 855
 856 /**
 857  * nes_destroy_adapter - destroy the adapter structure
 858  */
 859 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 860 {
 861         struct nes_adapter *tmp_adapter;
 862
 863         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 864                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 865                                 tmp_adapter);
 866         }
 867
 868         nesadapter->ref_count--;
 869         if (!nesadapter->ref_count) {
 870                 if (nesadapter->hw_rev == NE020_REV) {
 871                         del_timer(&nesadapter->mh_timer);
 872                 }
 873                 del_timer(&nesadapter->lc_timer);
 874
 875                 list_del(&nesadapter->list);
 876                 kfree(nesadapter);
 877         }
 878 }
 879
 880
 881 /**
 882  * nes_init_cqp
 883  */
 884 int nes_init_cqp(struct nes_device *nesdev)
 885 {
 886         struct nes_adapter *nesadapter = nesdev->nesadapter;
 887         struct nes_hw_cqp_qp_context *cqp_qp_context;
 888         struct nes_hw_cqp_wqe *cqp_wqe;
 889         struct nes_hw_ceq *ceq;
 890         struct nes_hw_ceq *nic_ceq;
 891         struct nes_hw_aeq *aeq;
 892         void *vmem;
 893         dma_addr_t pmem;
 894         u32 count=0;
 895         u32 cqp_head;
 896         u64 u64temp;
 897         u32 u32temp;
 898
 899         /* allocate CQP memory */
 900         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 901         /* SQ is 512 byte aligned, others are 256 byte aligned */
 902         nesdev->cqp_mem_size = 512 +
 903                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 904                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 905                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 906                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 907                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 908                         sizeof(struct nes_hw_cqp_qp_context);
 909
 910         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 911                         &nesdev->cqp_pbase);
 912         if (!nesdev->cqp_vbase) {
 913                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 914                 return -ENOMEM;
 915         }
 916         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 917
 918         /* Allocate a twice the number of CQP requests as the SQ size */
 919         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 920                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 921         if (nesdev->nes_cqp_requests == NULL) {
 922                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 923                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 924                                 nesdev->cqp.sq_pbase);
 925                 return -ENOMEM;
 926         }
 927
 928         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 929                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 930
 931         spin_lock_init(&nesdev->cqp.lock);
 932         init_waitqueue_head(&nesdev->cqp.waitq);
 933
 934         /* Setup Various Structures */
 935         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 936                         ~(unsigned long)(512 - 1));
 937         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 938                         ~(unsigned long long)(512 - 1));
 939
 940         nesdev->cqp.sq_vbase = vmem;
 941         nesdev->cqp.sq_pbase = pmem;
 942         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 943         nesdev->cqp.sq_head = 0;
 944         nesdev->cqp.sq_tail = 0;
 945         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 946
 947         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 948         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 949
 950         nesdev->ccq.cq_vbase = vmem;
 951         nesdev->ccq.cq_pbase = pmem;
 952         nesdev->ccq.cq_size = NES_CCQ_SIZE;
 953         nesdev->ccq.cq_head = 0;
 954         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
 955         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
 956
 957         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 958         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 959
 960         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
 961         ceq = &nesadapter->ceq[nesdev->ceq_index];
 962         ceq->ceq_vbase = vmem;
 963         ceq->ceq_pbase = pmem;
 964         ceq->ceq_size = NES_CCEQ_SIZE;
 965         ceq->ceq_head = 0;
 966
 967         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 968         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 969
 970         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
 971         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
 972         nic_ceq->ceq_vbase = vmem;
 973         nic_ceq->ceq_pbase = pmem;
 974         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
 975         nic_ceq->ceq_head = 0;
 976
 977         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 978         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 979
 980         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
 981         aeq->aeq_vbase = vmem;
 982         aeq->aeq_pbase = pmem;
 983         aeq->aeq_size = nesadapter->max_qp;
 984         aeq->aeq_head = 0;
 985
 986         /* Setup QP Context */
 987         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 988         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 989
 990         cqp_qp_context = vmem;
 991         cqp_qp_context->context_words[0] =
 992                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
 993         cqp_qp_context->context_words[1] = 0;
 994         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
 995         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
 996
 997
 998         /* Write the address to Create CQP */
 999         if ((sizeof(dma_addr_t) > 4)) {
1000                 nes_write_indexed(nesdev,
1001                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1002                                 ((u64)pmem) >> 32);
1003         } else {
1004                 nes_write_indexed(nesdev,
1005                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1006         }
1007         nes_write_indexed(nesdev,
1008                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1009                         (u32)pmem);
1010
1011         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1012         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1013
1014         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1015                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1016                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1017         }
1018
1019         /* Write Create CCQ WQE */
1020         cqp_head = nesdev->cqp.sq_head++;
1021         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1022         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1023         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1024                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1025                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1026         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1027                             (nesdev->ccq.cq_number |
1028                              ((u32)nesdev->ceq_index << 16)));
1029         u64temp = (u64)nesdev->ccq.cq_pbase;
1030         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1031         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1032         u64temp = (unsigned long)&nesdev->ccq;
1033         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1034                         cpu_to_le32((u32)(u64temp >> 1));
1035         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1036                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1037         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1038
1039         /* Write Create CEQ WQE */
1040         cqp_head = nesdev->cqp.sq_head++;
1041         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1042         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1043         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1044                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1045         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1046         u64temp = (u64)ceq->ceq_pbase;
1047         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1048
1049         /* Write Create AEQ WQE */
1050         cqp_head = nesdev->cqp.sq_head++;
1051         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1052         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1053         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1054                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1055         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1056         u64temp = (u64)aeq->aeq_pbase;
1057         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1058
1059         /* Write Create NIC CEQ WQE */
1060         cqp_head = nesdev->cqp.sq_head++;
1061         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1062         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1063         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1064                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1065         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1066         u64temp = (u64)nic_ceq->ceq_pbase;
1067         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1068
1069         /* Poll until CCQP done */
1070         count = 0;
1071         do {
1072                 if (count++ > 1000) {
1073                         printk(KERN_ERR PFX "Error creating CQP\n");
1074                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1075                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1076                         return -1;
1077                 }
1078                 udelay(10);
1079         } while (!(nes_read_indexed(nesdev,
1080                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1081
1082         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1083                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1084
1085         u32temp = 0x04800000;
1086         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1087
1088         /* wait for the CCQ, CEQ, and AEQ to get created */
1089         count = 0;
1090         do {
1091                 if (count++ > 1000) {
1092                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1093                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1094                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1095                         return -1;
1096                 }
1097                 udelay(10);
1098         } while (((nes_read_indexed(nesdev,
1099                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1100
1101         /* dump the QP status value */
1102         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1103                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1104
1105         nesdev->cqp.sq_tail++;
1106
1107         return 0;
1108 }
1109
1110
1111 /**
1112  * nes_destroy_cqp
1113  */
1114 int nes_destroy_cqp(struct nes_device *nesdev)
1115 {
1116         struct nes_hw_cqp_wqe *cqp_wqe;
1117         u32 count = 0;
1118         u32 cqp_head;
1119         unsigned long flags;
1120
1121         do {
1122                 if (count++ > 1000)
1123                         break;
1124                 udelay(10);
1125         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1126
1127         /* Reset CCQ */
1128         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1129                         nesdev->ccq.cq_number);
1130
1131         /* Disable device interrupts */
1132         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1133
1134         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1135
1136         /* Destroy the AEQ */
1137         cqp_head = nesdev->cqp.sq_head++;
1138         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1139         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1140         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1141                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1142         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1143
1144         /* Destroy the NIC CEQ */
1145         cqp_head = nesdev->cqp.sq_head++;
1146         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1147         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1148         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1149                         ((u32)nesdev->nic_ceq_index << 8));
1150
1151         /* Destroy the CEQ */
1152         cqp_head = nesdev->cqp.sq_head++;
1153         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1154         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1155         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1156                         (nesdev->ceq_index << 8));
1157
1158         /* Destroy the CCQ */
1159         cqp_head = nesdev->cqp.sq_head++;
1160         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1161         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1162         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1163         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1164                         ((u32)nesdev->ceq_index << 16));
1165
1166         /* Destroy CQP */
1167         cqp_head = nesdev->cqp.sq_head++;
1168         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1169         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1170         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1171                         NES_CQP_QP_TYPE_CQP);
1172         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1173
1174         barrier();
1175         /* Ring doorbell (5 WQEs) */
1176         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1177
1178         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1179
1180         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1181         count = 0;
1182         do {
1183                 if (count++ > 1000) {
1184                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1185                                         PCI_FUNC(nesdev->pcidev->devfn));
1186                         break;
1187                 }
1188                 udelay(10);
1189         } while (((nes_read_indexed(nesdev,
1190                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1191
1192         /* dump the QP status value */
1193         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1194                         PCI_FUNC(nesdev->pcidev->devfn),
1195                         nes_read_indexed(nesdev,
1196                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1197
1198         kfree(nesdev->nes_cqp_requests);
1199
1200         /* Free the control structures */
1201         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1202                         nesdev->cqp.sq_pbase);
1203
1204         return 0;
1205 }
1206
1207
1208 /**
1209  * nes_init_phy
1210  */
1211 int nes_init_phy(struct nes_device *nesdev)
1212 {
1213         struct nes_adapter *nesadapter = nesdev->nesadapter;
1214         u32 counter = 0;
1215         u32 sds_common_control0;
1216         u32 mac_index = nesdev->mac_index;
1217         u32 tx_config = 0;
1218         u16 phy_data;
1219         u32 temp_phy_data = 0;
1220         u32 temp_phy_data2 = 0;
1221         u32 i = 0;
1222
1223         if ((nesadapter->OneG_Mode) &&
1224             (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
1225                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1226                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1227                         printk(PFX "%s: Programming mdc config for 1G\n", __func__);
1228                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1229                         tx_config |= 0x04;
1230                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1231                 }
1232
1233                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1234                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1235                                 nesadapter->phy_index[mac_index], phy_data);
1236                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index], 0xb000);
1237
1238                 /* Reset the PHY */
1239                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1240                 udelay(100);
1241                 counter = 0;
1242                 do {
1243                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1244                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1245                         if (counter++ > 100) break;
1246                 } while (phy_data & 0x8000);
1247
1248                 /* Setting no phy loopback */
1249                 phy_data &= 0xbfff;
1250                 phy_data |= 0x1140;
1251                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1252                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1253                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1254
1255                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1256                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1257
1258                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1259                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1260
1261                 /* Setting the interrupt mask */
1262                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1263                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1264                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1265
1266                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1267                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1268
1269                 /* turning on flow control */
1270                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1271                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1272                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1273                                 (phy_data & ~(0x03E0)) | 0xc00);
1274                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1275                                 phy_data | 0xc00); */
1276                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1277                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1278
1279                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1280                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1281                 /* Clear Half duplex */
1282                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1283                                 phy_data & ~(0x0100));
1284                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1285                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1286
1287                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1288                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1289         } else {
1290                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) ||
1291                     (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1292                         /* setup 10G MDIO operation */
1293                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1294                         tx_config |= 0x14;
1295                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1296                 }
1297                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1298                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1299
1300                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1301                         mdelay(10);
1302                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1303                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1304
1305                         /*
1306                          * if firmware is already running (like from a
1307                          * driver un-load/load, don't do anything.
1308                          */
1309                         if (temp_phy_data == temp_phy_data2) {
1310                                 /* configure QT2505 AMCC PHY */
1311                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0x0000, 0x8000);
1312                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0000);
1313                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc302, 0x0044);
1314                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc318, 0x0052);
1315                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc319, 0x0008);
1316                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc31a, 0x0098);
1317                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0026, 0x0E00);
1318                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0027, 0x0000);
1319                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0028, 0xA528);
1320
1321                                 /*
1322                                  * remove micro from reset; chip boots from ROM,
1323                                  * uploads EEPROM f/w image, uC executes f/w
1324                                  */
1325                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0002);
1326
1327                                 /*
1328                                  * wait for heart beat to start to
1329                                  * know loading is done
1330                                  */
1331                                 counter = 0;
1332                                 do {
1333                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1334                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1335                                         if (counter++ > 1000) {
1336                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from heartbeat check <this is bad!!!> \n");
1337                                                 break;
1338                                         }
1339                                         mdelay(100);
1340                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1341                                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1342                                 } while ((temp_phy_data2 == temp_phy_data));
1343
1344                                 /*
1345                                  * wait for tracking to start to know
1346                                  * f/w is good to go
1347                                  */
1348                                 counter = 0;
1349                                 do {
1350                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7fd);
1351                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1352                                         if (counter++ > 1000) {
1353                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from status check <this is bad!!!> \n");
1354                                                 break;
1355                                         }
1356                                         mdelay(1000);
1357                                         /*
1358                                          * nes_debug(NES_DBG_PHY, "AMCC PHY- phy_status not ready yet = 0x%02X\n",
1359                                          *                      temp_phy_data);
1360                                          */
1361                                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1362
1363                                 /* set LOS Control invert RXLOSB_I_PADINV */
1364                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd003, 0x0000);
1365                                 /* set LOS Control to mask of RXLOSB_I */
1366                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc314, 0x0042);
1367                                 /* set LED1 to input mode (LED1 and LED2 share same LED) */
1368                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd006, 0x0007);
1369                                 /* set LED2 to RX link_status and activity */
1370                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd007, 0x000A);
1371                                 /* set LED3 to RX link_status */
1372                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd008, 0x0009);
1373
1374                                 /*
1375                                  * reset the res-calibration on t2
1376                                  * serdes; ensures it is stable after
1377                                  * the amcc phy is stable
1378                                  */
1379
1380                                 sds_common_control0  = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
1381                                 sds_common_control0 |= 0x1;
1382                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1383
1384                                 /* release the res-calibration reset */
1385                                 sds_common_control0 &= 0xfffffffe;
1386                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1387
1388                                 i = 0;
1389                                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1390                                                 && (i++ < 5000)) {
1391                                         /* mdelay(1); */
1392                                 }
1393
1394                                 /*
1395                                  * wait for link train done before moving on,
1396                                  * or will get an interupt storm
1397                                  */
1398                                 counter = 0;
1399                                 do {
1400                                         temp_phy_data = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
1401                                                                 (0x200 * (nesdev->mac_index & 1)));
1402                                         if (counter++ > 1000) {
1403                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from link train wait <this is bad, link didnt train!!!>\n");
1404                                                 break;
1405                                         }
1406                                         mdelay(1);
1407                                 } while (((temp_phy_data & 0x0f1f0000) != 0x0f0f0000));
1408                         }
1409                 }
1410         }
1411         return 0;
1412 }
1413
1414
1415 /**
1416  * nes_replenish_nic_rq
1417  */
1418 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1419 {
1420         unsigned long flags;
1421         dma_addr_t bus_address;
1422         struct sk_buff *skb;
1423         struct nes_hw_nic_rq_wqe *nic_rqe;
1424         struct nes_hw_nic *nesnic;
1425         struct nes_device *nesdev;
1426         u32 rx_wqes_posted = 0;
1427
1428         nesnic = &nesvnic->nic;
1429         nesdev = nesvnic->nesdev;
1430         spin_lock_irqsave(&nesnic->rq_lock, flags);
1431         if (nesnic->replenishing_rq !=0) {
1432                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1433                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1434                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1435                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1436                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1437                         add_timer(&nesvnic->rq_wqes_timer);
1438                 } else
1439                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1440                 return;
1441         }
1442         nesnic->replenishing_rq = 1;
1443         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1444         do {
1445                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1446                 if (skb) {
1447                         skb->dev = nesvnic->netdev;
1448
1449                         bus_address = pci_map_single(nesdev->pcidev,
1450                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1451
1452                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1453                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1454                                         cpu_to_le32(nesvnic->max_frame_size);
1455                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1456                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1457                                         cpu_to_le32((u32)bus_address);
1458                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1459                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1460                         nesnic->rx_skb[nesnic->rq_head] = skb;
1461                         nesnic->rq_head++;
1462                         nesnic->rq_head &= nesnic->rq_size - 1;
1463                         atomic_dec(&nesvnic->rx_skbs_needed);
1464                         barrier();
1465                         if (++rx_wqes_posted == 255) {
1466                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1467                                 rx_wqes_posted = 0;
1468                         }
1469                 } else {
1470                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1471                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1472                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1473                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1474                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1475                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1476                                 add_timer(&nesvnic->rq_wqes_timer);
1477                         } else
1478                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1479                         break;
1480                 }
1481         } while (atomic_read(&nesvnic->rx_skbs_needed));
1482         barrier();
1483         if (rx_wqes_posted)
1484                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1485         nesnic->replenishing_rq = 0;
1486 }
1487
1488
1489 /**
1490  * nes_rq_wqes_timeout
1491  */
1492 static void nes_rq_wqes_timeout(unsigned long parm)
1493 {
1494         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1495         printk("%s: Timer fired.\n", __func__);
1496         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1497         if (atomic_read(&nesvnic->rx_skbs_needed))
1498                 nes_replenish_nic_rq(nesvnic);
1499 }
1500
1501
1502 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1503                                void **tcph, u64 *hdr_flags, void *priv)
1504 {
1505         unsigned int ip_len;
1506         struct iphdr *iph;
1507         skb_reset_network_header(skb);
1508         iph = ip_hdr(skb);
1509         if (iph->protocol != IPPROTO_TCP)
1510                 return -1;
1511         ip_len = ip_hdrlen(skb);
1512         skb_set_transport_header(skb, ip_len);
1513         *tcph = tcp_hdr(skb);
1514
1515         *hdr_flags = LRO_IPV4 | LRO_TCP;
1516         *iphdr = iph;
1517         return 0;
1518 }
1519
1520
1521 /**
1522  * nes_init_nic_qp
1523  */
1524 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1525 {
1526         struct nes_hw_cqp_wqe *cqp_wqe;
1527         struct nes_hw_nic_sq_wqe *nic_sqe;
1528         struct nes_hw_nic_qp_context *nic_context;
1529         struct sk_buff *skb;
1530         struct nes_hw_nic_rq_wqe *nic_rqe;
1531         struct nes_vnic *nesvnic = netdev_priv(netdev);
1532         unsigned long flags;
1533         void *vmem;
1534         dma_addr_t pmem;
1535         u64 u64temp;
1536         int ret;
1537         u32 cqp_head;
1538         u32 counter;
1539         u32 wqe_count;
1540         u8 jumbomode=0;
1541
1542         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1543         nesvnic->nic_mem_size = 256 +
1544                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1545                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1546                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1547                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1548                         sizeof(struct nes_hw_nic_qp_context);
1549
1550         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1551                         &nesvnic->nic_pbase);
1552         if (!nesvnic->nic_vbase) {
1553                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1554                 return -ENOMEM;
1555         }
1556         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1557         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1558                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1559
1560         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1561                         ~(unsigned long)(256 - 1));
1562         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1563                         ~(unsigned long long)(256 - 1));
1564
1565         /* Setup the first Fragment buffers */
1566         nesvnic->nic.first_frag_vbase = vmem;
1567
1568         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1569                 nesvnic->nic.frag_paddr[counter] = pmem;
1570                 pmem += sizeof(struct nes_first_frag);
1571         }
1572
1573         /* setup the SQ */
1574         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1575
1576         nesvnic->nic.sq_vbase = (void *)vmem;
1577         nesvnic->nic.sq_pbase = pmem;
1578         nesvnic->nic.sq_head = 0;
1579         nesvnic->nic.sq_tail = 0;
1580         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1581         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1582                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1583                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1584                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1585                                 NES_NIC_SQ_WQE_COMPLETION);
1586                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1587                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1588                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1589                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1590                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1591                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1592         }
1593
1594         nesvnic->get_cqp_request = nes_get_cqp_request;
1595         nesvnic->post_cqp_request = nes_post_cqp_request;
1596         nesvnic->mcrq_mcast_filter = NULL;
1597
1598         spin_lock_init(&nesvnic->nic.sq_lock);
1599         spin_lock_init(&nesvnic->nic.rq_lock);
1600
1601         /* setup the RQ */
1602         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1603         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1604
1605
1606         nesvnic->nic.rq_vbase = vmem;
1607         nesvnic->nic.rq_pbase = pmem;
1608         nesvnic->nic.rq_head = 0;
1609         nesvnic->nic.rq_tail = 0;
1610         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1611
1612         /* setup the CQ */
1613         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1614         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1615
1616         if (nesdev->nesadapter->netdev_count > 2)
1617                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1618         else
1619                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1620
1621         nesvnic->nic_cq.cq_vbase = vmem;
1622         nesvnic->nic_cq.cq_pbase = pmem;
1623         nesvnic->nic_cq.cq_head = 0;
1624         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1625
1626         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1627
1628         /* Send CreateCQ request to CQP */
1629         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1630         cqp_head = nesdev->cqp.sq_head;
1631
1632         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1633         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1634
1635         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1636                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1637                         ((u32)nesvnic->nic_cq.cq_size << 16));
1638         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1639                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1640         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1641         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1642         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1643         u64temp = (unsigned long)&nesvnic->nic_cq;
1644         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1645         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1646                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1647         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1648         if (++cqp_head >= nesdev->cqp.sq_size)
1649                 cqp_head = 0;
1650         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1651         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1652
1653         /* Send CreateQP request to CQP */
1654         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1655         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1656                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1657                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1658         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1659                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1660                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1661         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1662                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1663         }
1664
1665         u64temp = (u64)nesvnic->nic.sq_pbase;
1666         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1667         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1668         u64temp = (u64)nesvnic->nic.rq_pbase;
1669         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1670         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1671
1672         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1673                         NES_CQP_QP_TYPE_NIC);
1674         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1675         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1676                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1677         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1678
1679         if (++cqp_head >= nesdev->cqp.sq_size)
1680                 cqp_head = 0;
1681         nesdev->cqp.sq_head = cqp_head;
1682
1683         barrier();
1684
1685         /* Ring doorbell (2 WQEs) */
1686         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1687
1688         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1689         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1690                         nesvnic->nic.qp_id);
1691
1692         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1693                         NES_EVENT_TIMEOUT);
1694         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1695                         nesvnic->nic.qp_id, ret);
1696         if (!ret) {
1697                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1698                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1699                                 nesvnic->nic_pbase);
1700                 return -EIO;
1701         }
1702
1703         /* Populate the RQ */
1704         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1705                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1706                 if (!skb) {
1707                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1708
1709                         nes_destroy_nic_qp(nesvnic);
1710                         return -ENOMEM;
1711                 }
1712
1713                 skb->dev = netdev;
1714
1715                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1716                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1717
1718                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1719                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1720                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1721                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1722                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1723                 nesvnic->nic.rx_skb[counter] = skb;
1724         }
1725
1726         wqe_count = NES_NIC_WQ_SIZE - 1;
1727         nesvnic->nic.rq_head = wqe_count;
1728         barrier();
1729         do {
1730                 counter = min(wqe_count, ((u32)255));
1731                 wqe_count -= counter;
1732                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1733         } while (wqe_count);
1734         init_timer(&nesvnic->rq_wqes_timer);
1735         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1736         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1737         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1738         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1739         {
1740                 nes_nic_init_timer(nesdev);
1741                 if (netdev->mtu > 1500)
1742                         jumbomode = 1;
1743                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1744         }
1745         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1746         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1747         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1748         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1749         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1750         nesvnic->lro_mgr.dev            = netdev;
1751         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1752         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1753         return 0;
1754 }
1755
1756
1757 /**
1758  * nes_destroy_nic_qp
1759  */
1760 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1761 {
1762         struct nes_device *nesdev = nesvnic->nesdev;
1763         struct nes_hw_cqp_wqe *cqp_wqe;
1764         struct nes_hw_nic_rq_wqe *nic_rqe;
1765         u64 wqe_frag;
1766         u32 cqp_head;
1767         unsigned long flags;
1768         int ret;
1769
1770         /* Free remaining NIC receive buffers */
1771         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1772                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1773                 wqe_frag  = (u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1774                 wqe_frag |= ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
1775                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1776                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1777                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1778                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1779         }
1780
1781         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1782
1783         /* Destroy NIC QP */
1784         cqp_head = nesdev->cqp.sq_head;
1785         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1786         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1787
1788         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1789                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1790         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1791                 nesvnic->nic.qp_id);
1792
1793         if (++cqp_head >= nesdev->cqp.sq_size)
1794                 cqp_head = 0;
1795
1796         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1797
1798         /* Destroy NIC CQ */
1799         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1800         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1801                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1802         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1803                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1804
1805         if (++cqp_head >= nesdev->cqp.sq_size)
1806                 cqp_head = 0;
1807
1808         nesdev->cqp.sq_head = cqp_head;
1809         barrier();
1810
1811         /* Ring doorbell (2 WQEs) */
1812         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1813
1814         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1815         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1816                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1817                         cqp_head, nesdev->cqp.sq_head,
1818                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1819
1820         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1821                         NES_EVENT_TIMEOUT);
1822
1823         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1824                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1825                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1826         if (!ret) {
1827                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1828                                 nesvnic->nic.qp_id);
1829         }
1830
1831         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1832                         nesvnic->nic_pbase);
1833 }
1834
1835 /**
1836  * nes_napi_isr
1837  */
1838 int nes_napi_isr(struct nes_device *nesdev)
1839 {
1840         struct nes_adapter *nesadapter = nesdev->nesadapter;
1841         u32 int_stat;
1842
1843         if (nesdev->napi_isr_ran) {
1844                 /* interrupt status has already been read in ISR */
1845                 int_stat = nesdev->int_stat;
1846         } else {
1847                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1848                 nesdev->int_stat = int_stat;
1849                 nesdev->napi_isr_ran = 1;
1850         }
1851
1852         int_stat &= nesdev->int_req;
1853         /* iff NIC, process here, else wait for DPC */
1854         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1855                 nesdev->napi_isr_ran = 0;
1856                 nes_write32(nesdev->regs + NES_INT_STAT,
1857                         (int_stat &
1858                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1859
1860                 /* Process the CEQs */
1861                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1862
1863                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1864                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1865                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1866                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1867                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1868                                 /* Enable Periodic timer interrupts */
1869                                 nesdev->int_req |= NES_INT_TIMER;
1870                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1871                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1872                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1873                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1874                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1875                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1876                         }
1877
1878                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1879                         {
1880                                 nes_nic_init_timer(nesdev);
1881                         }
1882                         /* Enable interrupts, except CEQs */
1883                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1884                 } else {
1885                         /* Enable interrupts, make sure timer is off */
1886                         nesdev->int_req &= ~NES_INT_TIMER;
1887                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1888                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1889                 }
1890                 nesdev->deepcq_count = 0;
1891                 return 1;
1892         } else {
1893                 return 0;
1894         }
1895 }
1896
1897
1898 /**
1899  * nes_dpc
1900  */
1901 void nes_dpc(unsigned long param)
1902 {
1903         struct nes_device *nesdev = (struct nes_device *)param;
1904         struct nes_adapter *nesadapter = nesdev->nesadapter;
1905         u32 counter;
1906         u32 loop_counter = 0;
1907         u32 int_status_bit;
1908         u32 int_stat;
1909         u32 timer_stat;
1910         u32 temp_int_stat;
1911         u32 intf_int_stat;
1912         u32 debug_error;
1913         u32 processed_intf_int = 0;
1914         u16 processed_timer_int = 0;
1915         u16 completion_ints = 0;
1916         u16 timer_ints = 0;
1917
1918         /* nes_debug(NES_DBG_ISR, "\n"); */
1919
1920         do {
1921                 timer_stat = 0;
1922                 if (nesdev->napi_isr_ran) {
1923                         nesdev->napi_isr_ran = 0;
1924                         int_stat = nesdev->int_stat;
1925                 } else
1926                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
1927                 if (processed_intf_int != 0)
1928                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
1929                 else
1930                         int_stat &= nesdev->int_req;
1931                 if (processed_timer_int == 0) {
1932                         processed_timer_int = 1;
1933                         if (int_stat & NES_INT_TIMER) {
1934                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
1935                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
1936                                         int_stat &= ~NES_INT_TIMER;
1937                                 }
1938                         }
1939                 } else {
1940                         int_stat &= ~NES_INT_TIMER;
1941                 }
1942
1943                 if (int_stat) {
1944                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
1945                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
1946                                 /* Ack the interrupts */
1947                                 nes_write32(nesdev->regs+NES_INT_STAT,
1948                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
1949                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1950                         }
1951
1952                         temp_int_stat = int_stat;
1953                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
1954                                 if (int_stat & int_status_bit) {
1955                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
1956                                         temp_int_stat &= ~int_status_bit;
1957                                         completion_ints = 1;
1958                                 }
1959                                 if (!(temp_int_stat & 0x0000ffff))
1960                                         break;
1961                                 int_status_bit <<= 1;
1962                         }
1963
1964                         /* Process the AEQ for this pci function */
1965                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
1966                         if (int_stat & int_status_bit) {
1967                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
1968                         }
1969
1970                         /* Process the MAC interrupt for this pci function */
1971                         int_status_bit = 1 << (24 + nesdev->mac_index);
1972                         if (int_stat & int_status_bit) {
1973                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
1974                         }
1975
1976                         if (int_stat & NES_INT_TIMER) {
1977                                 if (timer_stat & nesdev->timer_int_req) {
1978                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
1979                                                         (timer_stat & nesdev->timer_int_req) |
1980                                                         ~(nesdev->nesadapter->timer_int_req));
1981                                         timer_ints = 1;
1982                                 }
1983                         }
1984
1985                         if (int_stat & NES_INT_INTF) {
1986                                 processed_intf_int = 1;
1987                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
1988                                 intf_int_stat &= nesdev->intf_int_req;
1989                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
1990                                         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1991                                         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1992                                                         (u16)debug_error);
1993                                         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1994                                                         0x01010000 | (debug_error & 0x0000ffff));
1995                                         /* BUG(); */
1996                                         if (crit_err_count++ > 10)
1997                                                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1998                                 }
1999                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2000                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2001                                         BUG();
2002                                 }
2003                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2004                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2005                                         BUG();
2006                                 }
2007                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2008                         }
2009
2010                         if (int_stat & NES_INT_TSW) {
2011                         }
2012                 }
2013                 /* Don't use the interface interrupt bit stay in loop */
2014                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2015                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2016         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2017
2018         if (timer_ints == 1) {
2019                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2020                         if (completion_ints == 0) {
2021                                 nesdev->timer_only_int_count++;
2022                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2023                                         nesdev->timer_only_int_count = 0;
2024                                         nesdev->int_req &= ~NES_INT_TIMER;
2025                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2026                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2027                                 } else {
2028                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2029                                 }
2030                         } else {
2031                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2032                                 {
2033                                         nes_nic_init_timer(nesdev);
2034                                 }
2035                                 nesdev->timer_only_int_count = 0;
2036                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2037                         }
2038                 } else {
2039                         nesdev->timer_only_int_count = 0;
2040                         nesdev->int_req &= ~NES_INT_TIMER;
2041                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2042                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2043                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2044                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2045                 }
2046         } else {
2047                 if ( (completion_ints == 1) &&
2048                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2049                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2050                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2051                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2052                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2053                         nesdev->timer_only_int_count = 0;
2054                         nesdev->int_req |= NES_INT_TIMER;
2055                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2056                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2057                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2058                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2059                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2060                 } else {
2061                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2062                 }
2063         }
2064         nesdev->deepcq_count = 0;
2065 }
2066
2067
2068 /**
2069  * nes_process_ceq
2070  */
2071 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2072 {
2073         u64 u64temp;
2074         struct nes_hw_cq *cq;
2075         u32 head;
2076         u32 ceq_size;
2077
2078         /* nes_debug(NES_DBG_CQ, "\n"); */
2079         head = ceq->ceq_head;
2080         ceq_size = ceq->ceq_size;
2081
2082         do {
2083                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2084                                 NES_CEQE_VALID) {
2085                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2086                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2087                         u64temp <<= 1;
2088                         cq = *((struct nes_hw_cq **)&u64temp);
2089                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2090                         barrier();
2091                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2092
2093                         /* call the event handler */
2094                         cq->ce_handler(nesdev, cq);
2095
2096                         if (++head >= ceq_size)
2097                                 head = 0;
2098                 } else {
2099                         break;
2100                 }
2101
2102         } while (1);
2103
2104         ceq->ceq_head = head;
2105 }
2106
2107
2108 /**
2109  * nes_process_aeq
2110  */
2111 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2112 {
2113         /* u64 u64temp; */
2114         u32 head;
2115         u32 aeq_size;
2116         u32 aeqe_misc;
2117         u32 aeqe_cq_id;
2118         struct nes_hw_aeqe volatile *aeqe;
2119
2120         head = aeq->aeq_head;
2121         aeq_size = aeq->aeq_size;
2122
2123         do {
2124                 aeqe = &aeq->aeq_vbase[head];
2125                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2126                         break;
2127                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2128                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2129                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2130                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2131                                 /* dealing with an accelerated QP related AE */
2132                                 /*
2133                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2134                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2135                                  */
2136                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2137                         } else {
2138                                 /* TODO: dealing with a CQP related AE */
2139                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2140                                                 (u16)(aeqe_misc >> 16));
2141                         }
2142                 }
2143
2144                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2145
2146                 if (++head >= aeq_size)
2147                         head = 0;
2148         }
2149         while (1);
2150         aeq->aeq_head = head;
2151 }
2152
2153 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2154 {
2155         struct nes_adapter *nesadapter = nesdev->nesadapter;
2156         u32 reset_value;
2157         u32 i=0;
2158         u32 u32temp;
2159
2160         if (nesadapter->hw_rev == NE020_REV) {
2161                 return;
2162         }
2163         mh_detected++;
2164
2165         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2166
2167         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2168                 reset_value |= 0x0000001d;
2169         else
2170                 reset_value |= 0x0000002d;
2171
2172         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2173                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2174                         nesadapter->link_interrupt_count[0] = 0;
2175                         nesadapter->link_interrupt_count[1] = 0;
2176                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2177                         if (0x00000040 & u32temp)
2178                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2179                         else
2180                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2181
2182                         reset_value |= 0x0000003d;
2183                 }
2184                 nesadapter->link_interrupt_count[mac_index] = 0;
2185         }
2186
2187         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2188
2189         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2190                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2191
2192         if (0x0000003d == (reset_value & 0x0000003d)) {
2193                 u32 pcs_control_status0, pcs_control_status1;
2194
2195                 for (i = 0; i < 10; i++) {
2196                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2197                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2198                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2199                              && (pcs_control_status0 & 0x00100000))
2200                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2201                                 && (pcs_control_status1 & 0x00100000)))
2202                                 continue;
2203                         else
2204                                 break;
2205                 }
2206                 if (10 == i) {
2207                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2208                         if (0x00000040 & u32temp)
2209                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2210                         else
2211                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2212
2213                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2214
2215                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2216                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2217                 }
2218         }
2219 }
2220
2221 /**
2222  * nes_process_mac_intr
2223  */
2224 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2225 {
2226         unsigned long flags;
2227         u32 pcs_control_status;
2228         struct nes_adapter *nesadapter = nesdev->nesadapter;
2229         struct nes_vnic *nesvnic;
2230         u32 mac_status;
2231         u32 mac_index = nesdev->mac_index;
2232         u32 u32temp;
2233         u16 phy_data;
2234         u16 temp_phy_data;
2235         u32 pcs_val  = 0x0f0f0000;
2236         u32 pcs_mask = 0x0f1f0000;
2237
2238         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2239         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2240                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2241                 return;
2242         }
2243         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2244         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2245
2246         /* ack the MAC interrupt */
2247         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2248         /* Clear the interrupt */
2249         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2250
2251         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2252
2253         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2254                 nesdev->link_status_interrupts++;
2255                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2256                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2257                         nes_reset_link(nesdev, mac_index);
2258                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2259                 }
2260                 /* read the PHY interrupt status register */
2261                 if (nesadapter->OneG_Mode) {
2262                         do {
2263                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2264                                                 nesadapter->phy_index[mac_index], &phy_data);
2265                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2266                                                 nesadapter->phy_index[mac_index], phy_data);
2267                         } while (phy_data&0x8000);
2268
2269                         temp_phy_data = 0;
2270                         do {
2271                                 nes_read_1G_phy_reg(nesdev, 0x11,
2272                                                 nesadapter->phy_index[mac_index], &phy_data);
2273                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2274                                                 nesadapter->phy_index[mac_index], phy_data);
2275                                 if (temp_phy_data == phy_data)
2276                                         break;
2277                                 temp_phy_data = phy_data;
2278                         } while (1);
2279
2280                         nes_read_1G_phy_reg(nesdev, 0x1e,
2281                                         nesadapter->phy_index[mac_index], &phy_data);
2282                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2283                                         nesadapter->phy_index[mac_index], phy_data);
2284
2285                         nes_read_1G_phy_reg(nesdev, 1,
2286                                         nesadapter->phy_index[mac_index], &phy_data);
2287                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2288                                         nesadapter->phy_index[mac_index], phy_data);
2289
2290                         if (temp_phy_data & 0x1000) {
2291                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2292                                 phy_data = 4;
2293                         } else {
2294                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2295                         }
2296                 }
2297                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2298                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2299                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2300
2301                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2302                         switch (mac_index) {
2303                         case 1:
2304                         case 3:
2305                                 pcs_control_status = nes_read_indexed(nesdev,
2306                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2307                                 break;
2308                         default:
2309                                 pcs_control_status = nes_read_indexed(nesdev,
2310                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2311                                 break;
2312                         }
2313                 } else {
2314                         pcs_control_status = nes_read_indexed(nesdev,
2315                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2316                         pcs_control_status = nes_read_indexed(nesdev,
2317                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2318                 }
2319
2320                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2321                                 mac_index, pcs_control_status);
2322                 if ((nesadapter->OneG_Mode) &&
2323                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2324                         u32temp = 0x01010000;
2325                         if (nesadapter->port_count > 2) {
2326                                 u32temp |= 0x02020000;
2327                         }
2328                         if ((pcs_control_status & u32temp)!= u32temp) {
2329                                 phy_data = 0;
2330                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2331                         }
2332                 } else {
2333                         switch (nesadapter->phy_type[mac_index]) {
2334                         case NES_PHY_TYPE_IRIS:
2335                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2336                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2337                                 u32temp = 20;
2338                                 do {
2339                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2340                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2341                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2342                                                 break;
2343                                         temp_phy_data = phy_data;
2344                                 } while (1);
2345                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2346                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2347                                 break;
2348
2349                         case NES_PHY_TYPE_ARGUS:
2350                                 /* clear the alarms */
2351                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2352                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2353                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2354                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2355                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2356                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2357                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2358                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2359                                 /* check link status */
2360                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2361                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2362                                 u32temp = 100;
2363                                 do {
2364                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2365
2366                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2367                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2368                                                 break;
2369                                         temp_phy_data = phy_data;
2370                                 } while (1);
2371                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2372                                         __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2373                                 break;
2374
2375                         case NES_PHY_TYPE_PUMA_1G:
2376                                 if (mac_index < 2)
2377                                         pcs_val = pcs_mask = 0x01010000;
2378                                 else
2379                                         pcs_val = pcs_mask = 0x02020000;
2380                                 /* fall through */
2381                         default:
2382                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2383                                 break;
2384                         }
2385                 }
2386
2387                 if (phy_data & 0x0004) {
2388                         nesadapter->mac_link_down[mac_index] = 0;
2389                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2390                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2391                                                 nesvnic->linkup);
2392                                 if (nesvnic->linkup == 0) {
2393                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2394                                                         nesvnic->netdev->name, nesvnic->netdev);
2395                                         if (netif_queue_stopped(nesvnic->netdev))
2396                                                 netif_start_queue(nesvnic->netdev);
2397                                         nesvnic->linkup = 1;
2398                                         netif_carrier_on(nesvnic->netdev);
2399                                 }
2400                         }
2401                 } else {
2402                         nesadapter->mac_link_down[mac_index] = 1;
2403                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2404                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2405                                                 nesvnic->linkup);
2406                                 if (nesvnic->linkup == 1) {
2407                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2408                                                         nesvnic->netdev->name, nesvnic->netdev);
2409                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2410                                                 netif_stop_queue(nesvnic->netdev);
2411                                         nesvnic->linkup = 0;
2412                                         netif_carrier_off(nesvnic->netdev);
2413                                 }
2414                         }
2415                 }
2416         }
2417
2418         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2419 }
2420
2421
2422
2423 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2424 {
2425         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2426
2427         netif_rx_schedule(nesdev->netdev[nesvnic->netdev_index], &nesvnic->napi);
2428 }
2429
2430
2431 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2432 * getting out of nic_ce_handler
2433 */
2434 #define MAX_RQES_TO_PROCESS     384
2435
2436 /**
2437  * nes_nic_ce_handler
2438  */
2439 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2440 {
2441         u64 u64temp;
2442         dma_addr_t bus_address;
2443         struct nes_hw_nic *nesnic;
2444         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2445         struct nes_adapter *nesadapter = nesdev->nesadapter;
2446         struct nes_hw_nic_rq_wqe *nic_rqe;
2447         struct nes_hw_nic_sq_wqe *nic_sqe;
2448         struct sk_buff *skb;
2449         struct sk_buff *rx_skb;
2450         __le16 *wqe_fragment_length;
2451         u32 head;
2452         u32 cq_size;
2453         u32 rx_pkt_size;
2454         u32 cqe_count=0;
2455         u32 cqe_errv;
2456         u32 cqe_misc;
2457         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2458         u16 vlan_tag;
2459         u16 pkt_type;
2460         u16 rqes_processed = 0;
2461         u8 sq_cqes = 0;
2462         u8 nes_use_lro = 0;
2463
2464         head = cq->cq_head;
2465         cq_size = cq->cq_size;
2466         cq->cqes_pending = 1;
2467         if (nesvnic->netdev->features & NETIF_F_LRO)
2468                 nes_use_lro = 1;
2469         do {
2470                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2471                                 NES_NIC_CQE_VALID) {
2472                         nesnic = &nesvnic->nic;
2473                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2474                         if (cqe_misc & NES_NIC_CQE_SQ) {
2475                                 sq_cqes++;
2476                                 wqe_fragment_index = 1;
2477                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2478                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2479                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2480                                 /* bump past the vlan tag */
2481                                 wqe_fragment_length++;
2482                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2483                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2484                                                         wqe_fragment_index * 2]);
2485                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2486                                                         wqe_fragment_index * 2])) << 32;
2487                                         bus_address = (dma_addr_t)u64temp;
2488                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2489                                                 pci_unmap_single(nesdev->pcidev,
2490                                                                 bus_address,
2491                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2492                                                                 PCI_DMA_TODEVICE);
2493                                         }
2494                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2495                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2496                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2497                                                                                 wqe_fragment_index * 2]);
2498                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2499                                                                                 + wqe_fragment_index * 2])) <<32;
2500                                                         bus_address = (dma_addr_t)u64temp;
2501                                                         pci_unmap_page(nesdev->pcidev,
2502                                                                         bus_address,
2503                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2504                                                                         PCI_DMA_TODEVICE);
2505                                                 } else
2506                                                         break;
2507                                         }
2508                                         if (skb)
2509                                                 dev_kfree_skb_any(skb);
2510                                 }
2511                                 nesnic->sq_tail++;
2512                                 nesnic->sq_tail &= nesnic->sq_size-1;
2513                                 if (sq_cqes > 128) {
2514                                         barrier();
2515                                 /* restart the queue if it had been stopped */
2516                                 if (netif_queue_stopped(nesvnic->netdev))
2517                                         netif_wake_queue(nesvnic->netdev);
2518                                         sq_cqes = 0;
2519                                 }
2520                         } else {
2521                                 rqes_processed ++;
2522
2523                                 cq->rx_cqes_completed++;
2524                                 cq->rx_pkts_indicated++;
2525                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2526                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2527                                 /* Get the skb */
2528                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2529                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2530                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2531                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2532                                 pci_unmap_single(nesdev->pcidev, bus_address,
2533                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2534                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2535                                 /* rx_skb->len = rx_pkt_size; */
2536                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2537                                 skb_put(rx_skb, rx_pkt_size);
2538                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2539                                 nesnic->rq_tail++;
2540                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2541
2542                                 atomic_inc(&nesvnic->rx_skbs_needed);
2543                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2544                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2545                                                         cq->cq_number | (cqe_count << 16));
2546                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2547                                         nesdev->currcq_count += cqe_count;
2548                                         cqe_count = 0;
2549                                         nes_replenish_nic_rq(nesvnic);
2550                                 }
2551                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2552                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2553                                 rx_skb->ip_summed = CHECKSUM_NONE;
2554
2555                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2556                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2557                                         if ((cqe_errv &
2558                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2559                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2560                                                 if (nesvnic->rx_checksum_disabled == 0) {
2561                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2562                                                 }
2563                                         } else
2564                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2565                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2566                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2567
2568                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2569                                         if ((cqe_errv &
2570                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2571                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2572                                                 if (nesvnic->rx_checksum_disabled == 0) {
2573                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2574                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2575                                                                   nesvnic->netdev->name); */
2576                                                 }
2577                                         } else
2578                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2579                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2580                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2581                                         }
2582                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2583                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2584
2585                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2586                                         nes_cm_recv(rx_skb, nesvnic->netdev);
2587                                 } else {
2588                                         if ((cqe_misc & NES_NIC_CQE_TAG_VALID) && (nesvnic->vlan_grp != NULL)) {
2589                                                 vlan_tag = (u16)(le32_to_cpu(
2590                                                                 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2591                                                                 >> 16);
2592                                                 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2593                                                                 nesvnic->netdev->name, vlan_tag);
2594                                                 if (nes_use_lro)
2595                                                         lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2596                                                                         nesvnic->vlan_grp, vlan_tag, NULL);
2597                                                 else
2598                                                         nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2599                                         } else {
2600                                                 if (nes_use_lro)
2601                                                         lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2602                                                 else
2603                                                         nes_netif_rx(rx_skb);
2604                                         }
2605                                 }
2606
2607                                 nesvnic->netdev->last_rx = jiffies;
2608                                 /* nesvnic->netstats.rx_packets++; */
2609                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2610                         }
2611
2612                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2613                         /* Accounting... */
2614                         cqe_count++;
2615                         if (++head >= cq_size)
2616                                 head = 0;
2617                         if (cqe_count == 255) {
2618                                 /* Replenish Nic CQ */
2619                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2620                                                 cq->cq_number | (cqe_count << 16));
2621                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2622                                 nesdev->currcq_count += cqe_count;
2623                                 cqe_count = 0;
2624                         }
2625
2626                         if (cq->rx_cqes_completed >= nesvnic->budget)
2627                                 break;
2628                 } else {
2629                         cq->cqes_pending = 0;
2630                         break;
2631                 }
2632
2633         } while (1);
2634
2635         if (nes_use_lro)
2636                 lro_flush_all(&nesvnic->lro_mgr);
2637         if (sq_cqes) {
2638                 barrier();
2639                 /* restart the queue if it had been stopped */
2640                 if (netif_queue_stopped(nesvnic->netdev))
2641                         netif_wake_queue(nesvnic->netdev);
2642         }
2643         cq->cq_head = head;
2644         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2645                         cq->cq_number, cqe_count, cq->cq_head); */
2646         cq->cqe_allocs_pending = cqe_count;
2647         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2648         {
2649                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2650                 nesdev->currcq_count += cqe_count;
2651                 nes_nic_tune_timer(nesdev);
2652         }
2653         if (atomic_read(&nesvnic->rx_skbs_needed))
2654                 nes_replenish_nic_rq(nesvnic);
2655 }
2656
2657
2658 /**
2659  * nes_cqp_ce_handler
2660  */
2661 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2662 {
2663         u64 u64temp;
2664         unsigned long flags;
2665         struct nes_hw_cqp *cqp = NULL;
2666         struct nes_cqp_request *cqp_request;
2667         struct nes_hw_cqp_wqe *cqp_wqe;
2668         u32 head;
2669         u32 cq_size;
2670         u32 cqe_count=0;
2671         u32 error_code;
2672         /* u32 counter; */
2673
2674         head = cq->cq_head;
2675         cq_size = cq->cq_size;
2676
2677         do {
2678                 /* process the CQE */
2679                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2680                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2681
2682                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2683                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2684                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2685                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2686                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2687                         cqp = *((struct nes_hw_cqp **)&u64temp);
2688
2689                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2690                         if (error_code) {
2691                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2692                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2693                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2694                                                 (u16)(error_code >> 16),
2695                                                 (u16)error_code);
2696                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2697                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2698                         }
2699
2700                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2701                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2702                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2703                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2704                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2705                         if (cqp_request) {
2706                                 if (cqp_request->waiting) {
2707                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2708                                         cqp_request->major_code = (u16)(error_code >> 16);
2709                                         cqp_request->minor_code = (u16)error_code;
2710                                         barrier();
2711                                         cqp_request->request_done = 1;
2712                                         wake_up(&cqp_request->waitq);
2713                                         if (atomic_dec_and_test(&cqp_request->refcount)) {
2714                                                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2715                                                                 cqp_request,
2716                                                                 le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2717                                                 if (cqp_request->dynamic) {
2718                                                         kfree(cqp_request);
2719                                                 } else {
2720                                                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2721                                                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2722                                                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2723                                                 }
2724                                         }
2725                                 } else if (cqp_request->callback) {
2726                                         /* Envoke the callback routine */
2727                                         cqp_request->cqp_callback(nesdev, cqp_request);
2728                                         if (cqp_request->dynamic) {
2729                                                 kfree(cqp_request);
2730                                         } else {
2731                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2732                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2733                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2734                                         }
2735                                 } else {
2736                                         nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2737                                                         cqp_request,
2738                                                         le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX]) & 0x3f);
2739                                         if (cqp_request->dynamic) {
2740                                                 kfree(cqp_request);
2741                                         } else {
2742                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2743                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2744                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2745                                         }
2746                                 }
2747                         } else {
2748                                 wake_up(&nesdev->cqp.waitq);
2749                         }
2750
2751                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2752                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2753                         if (++cqp->sq_tail >= cqp->sq_size)
2754                                 cqp->sq_tail = 0;
2755
2756                         /* Accounting... */
2757                         cqe_count++;
2758                         if (++head >= cq_size)
2759                                 head = 0;
2760                 } else {
2761                         break;
2762                 }
2763         } while (1);
2764         cq->cq_head = head;
2765
2766         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2767         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2768                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2769                         (nesdev->cqp.sq_size - 1)) != 1)) {
2770                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2771                                 struct nes_cqp_request, list);
2772                 list_del_init(&cqp_request->list);
2773                 head = nesdev->cqp.sq_head++;
2774                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2775                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2776                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2777                 barrier();
2778                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2779                         cpu_to_le32((u32)((unsigned long)cqp_request));
2780                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2781                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2782                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2783                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2784                 /* Ring doorbell (1 WQEs) */
2785                 barrier();
2786                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2787         }
2788         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2789
2790         /* Arm the CCQ */
2791         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2792                         cq->cq_number);
2793         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2794 }
2795
2796
2797 /**
2798  * nes_process_iwarp_aeqe
2799  */
2800 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2801                                    struct nes_hw_aeqe *aeqe)
2802 {
2803         u64 context;
2804         u64 aeqe_context = 0;
2805         unsigned long flags;
2806         struct nes_qp *nesqp;
2807         int resource_allocated;
2808         /* struct iw_cm_id *cm_id; */
2809         struct nes_adapter *nesadapter = nesdev->nesadapter;
2810         struct ib_event ibevent;
2811         /* struct iw_cm_event cm_event; */
2812         u32 aeq_info;
2813         u32 next_iwarp_state = 0;
2814         u16 async_event_id;
2815         u8 tcp_state;
2816         u8 iwarp_state;
2817
2818         nes_debug(NES_DBG_AEQ, "\n");
2819         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2820         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2821                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2822                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2823         } else {
2824                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2825                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2826                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2827                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
2828                 BUG_ON(!context);
2829         }
2830
2831         async_event_id = (u16)aeq_info;
2832         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2833         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2834         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2835                         " Tcp state = %s, iWARP state = %s\n",
2836                         async_event_id,
2837                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2838                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2839
2840         switch (async_event_id) {
2841                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2842                         nesqp = *((struct nes_qp **)&context);
2843                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2844                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2845                                 nes_add_ref(&nesqp->ibqp);
2846                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2847                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2848                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2849                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2850                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2851                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2852                                                 async_event_id, nesqp->last_aeq, tcp_state);
2853                         }
2854                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2855                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2856                                 /* FIN Received but tcp state or IB state moved on,
2857                                                 should expect a close complete */
2858                                 return;
2859                         }
2860                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2861                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2862                 case NES_AEQE_AEID_TERMINATE_SENT:
2863                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2864                 case NES_AEQE_AEID_RESET_SENT:
2865                         nesqp = *((struct nes_qp **)&context);
2866                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2867                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2868                         }
2869                         nes_add_ref(&nesqp->ibqp);
2870                         spin_lock_irqsave(&nesqp->lock, flags);
2871                         nesqp->hw_iwarp_state = iwarp_state;
2872                         nesqp->hw_tcp_state = tcp_state;
2873                         nesqp->last_aeq = async_event_id;
2874
2875                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2876                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2877                                 nesqp->hte_added = 0;
2878                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2879                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2880                                                 nesqp->hwqp.qp_id);
2881                                 nes_hw_modify_qp(nesdev, nesqp,
2882                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2883                                 spin_lock_irqsave(&nesqp->lock, flags);
2884                         }
2885
2886                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2887                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2888                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2889                                 switch (nesqp->hw_iwarp_state) {
2890                                         case NES_AEQE_IWARP_STATE_RTS:
2891                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2892                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2893                                                 break;
2894                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2895                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2896                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2897                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2898                                                         next_iwarp_state |= 0x02000000;
2899                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2900                                                 }
2901                                                 break;
2902                                         default:
2903                                                 next_iwarp_state = 0;
2904                                 }
2905                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2906                                 if (next_iwarp_state) {
2907                                         nes_add_ref(&nesqp->ibqp);
2908                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2909                                                         " also added another reference\n",
2910                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2911                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2912                                 }
2913                                 nes_cm_disconn(nesqp);
2914                         } else {
2915                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
2916                                         /* FIN Received but ib state not RTS,
2917                                                         close complete will be on its way */
2918                                         spin_unlock_irqrestore(&nesqp->lock, flags);
2919                                         nes_rem_ref(&nesqp->ibqp);
2920                                         return;
2921                                 }
2922                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2923                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2924                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
2925                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2926                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2927                                                         " also added another reference\n",
2928                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2929                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2930                                 }
2931                                 nes_cm_disconn(nesqp);
2932                         }
2933                         break;
2934                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
2935                         nesqp = *((struct nes_qp **)&context);
2936                         spin_lock_irqsave(&nesqp->lock, flags);
2937                         nesqp->hw_iwarp_state = iwarp_state;
2938                         nesqp->hw_tcp_state = tcp_state;
2939                         nesqp->last_aeq = async_event_id;
2940                         spin_unlock_irqrestore(&nesqp->lock, flags);
2941                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
2942                                         " event on QP%u \n  Q2 Data:\n",
2943                                         nesqp->hwqp.qp_id);
2944                         if (nesqp->ibqp.event_handler) {
2945                                 ibevent.device = nesqp->ibqp.device;
2946                                 ibevent.element.qp = &nesqp->ibqp;
2947                                 ibevent.event = IB_EVENT_QP_FATAL;
2948                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2949                         }
2950                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2951                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
2952                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2953                                 nes_add_ref(&nesqp->ibqp);
2954                                 nes_cm_disconn(nesqp);
2955                         } else {
2956                                 nesqp->in_disconnect = 0;
2957                                 wake_up(&nesqp->kick_waitq);
2958                         }
2959                         break;
2960                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
2961                         nesqp = *((struct nes_qp **)&context);
2962                         nes_add_ref(&nesqp->ibqp);
2963                         spin_lock_irqsave(&nesqp->lock, flags);
2964                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
2965                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2966                         nesqp->last_aeq = async_event_id;
2967                         if (nesqp->cm_id) {
2968                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2969                                                 " event on QP%u, remote IP = 0x%08X \n",
2970                                                 nesqp->hwqp.qp_id,
2971                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
2972                         } else {
2973                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2974                                                 " event on QP%u \n",
2975                                                 nesqp->hwqp.qp_id);
2976                         }
2977                         spin_unlock_irqrestore(&nesqp->lock, flags);
2978                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
2979                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2980                         if (nesqp->ibqp.event_handler) {
2981                                 ibevent.device = nesqp->ibqp.device;
2982                                 ibevent.element.qp = &nesqp->ibqp;
2983                                 ibevent.event = IB_EVENT_QP_FATAL;
2984                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2985                         }
2986                         break;
2987                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
2988                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
2989                                 nesqp = nesadapter->qp_table[le32_to_cpu(
2990                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2991                         } else {
2992                                 /* TODO: get the actual WQE and mask off wqe index */
2993                                 context &= ~((u64)511);
2994                                 nesqp = *((struct nes_qp **)&context);
2995                         }
2996                         spin_lock_irqsave(&nesqp->lock, flags);
2997                         nesqp->hw_iwarp_state = iwarp_state;
2998                         nesqp->hw_tcp_state = tcp_state;
2999                         nesqp->last_aeq = async_event_id;
3000                         spin_unlock_irqrestore(&nesqp->lock, flags);
3001                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
3002                                         nesqp->hwqp.qp_id);
3003                         if (nesqp->ibqp.event_handler) {
3004                                 ibevent.device = nesqp->ibqp.device;
3005                                 ibevent.element.qp = &nesqp->ibqp;
3006                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3007                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3008                         }
3009                         break;
3010                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3011                         nesqp = *((struct nes_qp **)&context);
3012                         spin_lock_irqsave(&nesqp->lock, flags);
3013                         nesqp->hw_iwarp_state = iwarp_state;
3014                         nesqp->hw_tcp_state = tcp_state;
3015                         nesqp->last_aeq = async_event_id;
3016                         spin_unlock_irqrestore(&nesqp->lock, flags);
3017                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
3018                                         nesqp->hwqp.qp_id);
3019                         if (nesqp->ibqp.event_handler) {
3020                                 ibevent.device = nesqp->ibqp.device;
3021                                 ibevent.element.qp = &nesqp->ibqp;
3022                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3023                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3024                         }
3025                         break;
3026                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3027                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
3028                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3029                         spin_lock_irqsave(&nesqp->lock, flags);
3030                         nesqp->hw_iwarp_state = iwarp_state;
3031                         nesqp->hw_tcp_state = tcp_state;
3032                         nesqp->last_aeq = async_event_id;
3033                         spin_unlock_irqrestore(&nesqp->lock, flags);
3034                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
3035                                         " nesqp = %p, AE reported %p\n",
3036                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
3037                         if (nesqp->ibqp.event_handler) {
3038                                 ibevent.device = nesqp->ibqp.device;
3039                                 ibevent.element.qp = &nesqp->ibqp;
3040                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3041                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3042                         }
3043                         break;
3044                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3045                         context <<= 1;
3046                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3047                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3048                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3049                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3050                         if (resource_allocated) {
3051                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3052                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3053                         }
3054                         break;
3055                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3056                         nesqp = nesadapter->qp_table[le32_to_cpu(
3057                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3058                         spin_lock_irqsave(&nesqp->lock, flags);
3059                         nesqp->hw_iwarp_state = iwarp_state;
3060                         nesqp->hw_tcp_state = tcp_state;
3061                         nesqp->last_aeq = async_event_id;
3062                         spin_unlock_irqrestore(&nesqp->lock, flags);
3063                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
3064                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
3065                                         nesqp->hwqp.qp_id);
3066                         if (nesqp->ibqp.event_handler) {
3067                                 ibevent.device = nesqp->ibqp.device;
3068                                 ibevent.element.qp = &nesqp->ibqp;
3069                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3070                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3071                         }
3072                         /* tell cm to disconnect, cm will queue work to thread */
3073                         nes_add_ref(&nesqp->ibqp);
3074                         nes_cm_disconn(nesqp);
3075                         break;
3076                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3077                         nesqp = *((struct nes_qp **)&context);
3078                         spin_lock_irqsave(&nesqp->lock, flags);
3079                         nesqp->hw_iwarp_state = iwarp_state;
3080                         nesqp->hw_tcp_state = tcp_state;
3081                         nesqp->last_aeq = async_event_id;
3082                         spin_unlock_irqrestore(&nesqp->lock, flags);
3083                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
3084                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
3085                                         nesqp->hwqp.qp_id);
3086                         if (nesqp->ibqp.event_handler) {
3087                                 ibevent.device = nesqp->ibqp.device;
3088                                 ibevent.element.qp = &nesqp->ibqp;
3089                                 ibevent.event = IB_EVENT_QP_FATAL;
3090                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3091                         }
3092                         /* tell cm to disconnect, cm will queue work to thread */
3093                         nes_add_ref(&nesqp->ibqp);
3094                         nes_cm_disconn(nesqp);
3095                         break;
3096                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3097                         nesqp = *((struct nes_qp **)&context);
3098                         spin_lock_irqsave(&nesqp->lock, flags);
3099                         nesqp->hw_iwarp_state = iwarp_state;
3100                         nesqp->hw_tcp_state = tcp_state;
3101                         nesqp->last_aeq = async_event_id;
3102                         spin_unlock_irqrestore(&nesqp->lock, flags);
3103                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
3104                                         " event on QP%u \n  Q2 Data:\n",
3105                                         nesqp->hwqp.qp_id);
3106                         if (nesqp->ibqp.event_handler) {
3107                                 ibevent.device = nesqp->ibqp.device;
3108                                 ibevent.element.qp = &nesqp->ibqp;
3109                                 ibevent.event = IB_EVENT_QP_FATAL;
3110                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3111                         }
3112                         /* tell cm to disconnect, cm will queue work to thread */
3113                         nes_add_ref(&nesqp->ibqp);
3114                         nes_cm_disconn(nesqp);
3115                         break;
3116                         /* TODO: additional AEs need to be here */
3117                 default:
3118                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3119                                         async_event_id);
3120                         break;
3121         }
3122
3123 }
3124
3125
3126 /**
3127  * nes_iwarp_ce_handler
3128  */
3129 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3130 {
3131         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3132
3133         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3134                         nescq->hw_cq.cq_number); */
3135         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3136
3137         if (nescq->ibcq.comp_handler)
3138                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3139
3140         return;
3141 }
3142
3143
3144 /**
3145  * nes_manage_apbvt()
3146  */
3147 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3148                 u32 nic_index, u32 add_port)
3149 {
3150         struct nes_device *nesdev = nesvnic->nesdev;
3151         struct nes_hw_cqp_wqe *cqp_wqe;
3152         unsigned long flags;
3153         struct nes_cqp_request *cqp_request;
3154         int ret = 0;
3155         u16 major_code;
3156
3157         /* Send manage APBVT request to CQP */
3158         cqp_request = nes_get_cqp_request(nesdev);
3159         if (cqp_request == NULL) {
3160                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3161                 return -ENOMEM;
3162         }
3163         cqp_request->waiting = 1;
3164         cqp_wqe = &cqp_request->cqp_wqe;
3165
3166         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3167                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3168                         accel_local_port, accel_local_port, nic_index);
3169
3170         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3171         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3172                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3173         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3174                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3175
3176         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3177
3178         atomic_set(&cqp_request->refcount, 2);
3179         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3180
3181         if (add_port == NES_MANAGE_APBVT_ADD)
3182                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3183                                 NES_EVENT_TIMEOUT);
3184         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3185                         ret, cqp_request->major_code, cqp_request->minor_code);
3186         major_code = cqp_request->major_code;
3187         if (atomic_dec_and_test(&cqp_request->refcount)) {
3188                 if (cqp_request->dynamic) {
3189                         kfree(cqp_request);
3190                 } else {
3191                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3192                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3193                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3194                 }
3195         }
3196         if (!ret)
3197                 return -ETIME;
3198         else if (major_code)
3199                 return -EIO;
3200         else
3201                 return 0;
3202 }
3203
3204
3205 /**
3206  * nes_manage_arp_cache
3207  */
3208 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3209                 u32 ip_addr, u32 action)
3210 {
3211         struct nes_hw_cqp_wqe *cqp_wqe;
3212         struct nes_vnic *nesvnic = netdev_priv(netdev);
3213         struct nes_device *nesdev;
3214         struct nes_cqp_request *cqp_request;
3215         int arp_index;
3216
3217         nesdev = nesvnic->nesdev;
3218         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3219         if (arp_index == -1) {
3220                 return;
3221         }
3222
3223         /* update the ARP entry */
3224         cqp_request = nes_get_cqp_request(nesdev);
3225         if (cqp_request == NULL) {
3226                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3227                 return;
3228         }
3229         cqp_request->waiting = 0;
3230         cqp_wqe = &cqp_request->cqp_wqe;
3231         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3232
3233         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3234                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3235         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3236                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3237         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3238
3239         if (action == NES_ARP_ADD) {
3240                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3241                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3242                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3243                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3244                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3245                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3246         } else {
3247                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3248                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3249         }
3250
3251         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3252                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3253
3254         atomic_set(&cqp_request->refcount, 1);
3255         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3256 }
3257
3258
3259 /**
3260  * flush_wqes
3261  */
3262 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3263                 u32 which_wq, u32 wait_completion)
3264 {
3265         unsigned long flags;
3266         struct nes_cqp_request *cqp_request;
3267         struct nes_hw_cqp_wqe *cqp_wqe;
3268         int ret;
3269
3270         cqp_request = nes_get_cqp_request(nesdev);
3271         if (cqp_request == NULL) {
3272                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3273                 return;
3274         }
3275         if (wait_completion) {
3276                 cqp_request->waiting = 1;
3277                 atomic_set(&cqp_request->refcount, 2);
3278         } else {
3279                 cqp_request->waiting = 0;
3280         }
3281         cqp_wqe = &cqp_request->cqp_wqe;
3282         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3283
3284         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3285                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3286         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3287
3288         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3289
3290         if (wait_completion) {
3291                 /* Wait for CQP */
3292                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3293                                 NES_EVENT_TIMEOUT);
3294                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3295                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3296                                 ret, cqp_request->major_code, cqp_request->minor_code);
3297                 if (atomic_dec_and_test(&cqp_request->refcount)) {
3298                         if (cqp_request->dynamic) {
3299                                 kfree(cqp_request);
3300                         } else {
3301                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3302                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3303                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3304                         }
3305                 }
3306         }
3307 }