drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel-NE, Inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static u32 crit_err_count;
  50 u32 int_mod_timer_init;
  51 u32 int_mod_cq_depth_256;
  52 u32 int_mod_cq_depth_128;
  53 u32 int_mod_cq_depth_32;
  54 u32 int_mod_cq_depth_24;
  55 u32 int_mod_cq_depth_16;
  56 u32 int_mod_cq_depth_4;
  57 u32 int_mod_cq_depth_1;
  58 static const u8 nes_max_critical_error_count = 100;
  59 #include "nes_cm.h"
  60
  61 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  62 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  63 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  64                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  65 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  66 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  67 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  68 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  69                                    struct nes_hw_aeqe *aeqe);
  70 static void process_critical_error(struct nes_device *nesdev);
  71 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  72 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  73
  74 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  75 static unsigned char *nes_iwarp_state_str[] = {
  76         "Non-Existant",
  77         "Idle",
  78         "RTS",
  79         "Closing",
  80         "RSVD1",
  81         "Terminate",
  82         "Error",
  83         "RSVD2",
  84 };
  85
  86 static unsigned char *nes_tcp_state_str[] = {
  87         "Non-Existant",
  88         "Closed",
  89         "Listen",
  90         "SYN Sent",
  91         "SYN Rcvd",
  92         "Established",
  93         "Close Wait",
  94         "FIN Wait 1",
  95         "Closing",
  96         "Last Ack",
  97         "FIN Wait 2",
  98         "Time Wait",
  99         "RSVD1",
 100         "RSVD2",
 101         "RSVD3",
 102         "RSVD4",
 103 };
 104 #endif
 105
 106
 107 /**
 108  * nes_nic_init_timer_defaults
 109  */
 110 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 111 {
 112         unsigned long flags;
 113         struct nes_adapter *nesadapter = nesdev->nesadapter;
 114         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 115
 116         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 117
 118         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 119         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 120         if (jumbomode) {
 121                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 122                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 123                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 124         } else {
 125                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 126                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 127                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 128         }
 129
 130         /* todo use netdev->mtu to set thresholds */
 131         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 132 }
 133
 134
 135 /**
 136  * nes_nic_init_timer
 137  */
 138 static void  nes_nic_init_timer(struct nes_device *nesdev)
 139 {
 140         unsigned long flags;
 141         struct nes_adapter *nesadapter = nesdev->nesadapter;
 142         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 143
 144         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 145
 146         if (shared_timer->timer_in_use_old == 0) {
 147                 nesdev->deepcq_count = 0;
 148                 shared_timer->timer_direction_upward = 0;
 149                 shared_timer->timer_direction_downward = 0;
 150                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 151                 shared_timer->timer_in_use_old = 0;
 152
 153         }
 154         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 155                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 156                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 157                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 158         }
 159         /* todo use netdev->mtu to set thresholds */
 160         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 161 }
 162
 163
 164 /**
 165  * nes_nic_tune_timer
 166  */
 167 static void nes_nic_tune_timer(struct nes_device *nesdev)
 168 {
 169         unsigned long flags;
 170         struct nes_adapter *nesadapter = nesdev->nesadapter;
 171         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 172         u16 cq_count = nesdev->currcq_count;
 173
 174         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 175
 176         if (shared_timer->cq_count_old <= cq_count)
 177                 shared_timer->cq_direction_downward = 0;
 178         else
 179                 shared_timer->cq_direction_downward++;
 180         shared_timer->cq_count_old = cq_count;
 181         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 182                 if (cq_count <= shared_timer->threshold_low &&
 183                     shared_timer->threshold_low > 4) {
 184                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 185                         shared_timer->cq_direction_downward=0;
 186                         nesdev->currcq_count = 0;
 187                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 188                         return;
 189                 }
 190         }
 191
 192         if (cq_count > 1) {
 193                 nesdev->deepcq_count += cq_count;
 194                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 195                         shared_timer->timer_direction_upward++;
 196                         shared_timer->timer_direction_downward = 0;
 197                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 198                         shared_timer->timer_direction_upward = 0;
 199                         shared_timer->timer_direction_downward = 0;
 200                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 201                         shared_timer->timer_direction_downward++;
 202                         shared_timer->timer_direction_upward = 0;
 203                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 204                         shared_timer->timer_in_use -= 2;
 205                         shared_timer->timer_direction_upward = 0;
 206                         shared_timer->timer_direction_downward++;
 207                 } else {
 208                         shared_timer->timer_in_use -= 4;
 209                         shared_timer->timer_direction_upward = 0;
 210                         shared_timer->timer_direction_downward++;
 211                 }
 212
 213                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 214                         shared_timer->timer_in_use += 3;
 215                         shared_timer->timer_direction_upward = 0;
 216                         shared_timer->timer_direction_downward = 0;
 217                 }
 218                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 219                         shared_timer->timer_in_use -= 4 ;
 220                         shared_timer->timer_direction_downward = 0;
 221                         shared_timer->timer_direction_upward = 0;
 222                 }
 223         }
 224
 225         /* boundary checking */
 226         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 227                 shared_timer->timer_in_use = shared_timer->threshold_high;
 228         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 229                 shared_timer->timer_in_use = shared_timer->threshold_low;
 230
 231         nesdev->currcq_count = 0;
 232
 233         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 234 }
 235
 236
 237 /**
 238  * nes_init_adapter - initialize adapter
 239  */
 240 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 241         struct nes_adapter *nesadapter = NULL;
 242         unsigned long num_pds;
 243         u32 u32temp;
 244         u32 port_count;
 245         u16 max_rq_wrs;
 246         u16 max_sq_wrs;
 247         u32 max_mr;
 248         u32 max_256pbl;
 249         u32 max_4kpbl;
 250         u32 max_qp;
 251         u32 max_irrq;
 252         u32 max_cq;
 253         u32 hte_index_mask;
 254         u32 adapter_size;
 255         u32 arp_table_size;
 256         u16 vendor_id;
 257         u16 device_id;
 258         u8  OneG_Mode;
 259         u8  func_index;
 260
 261         /* search the list of existing adapters */
 262         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 263                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 264                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 265                                 nesdev->pcidev->devfn,
 266                                 PCI_SLOT(nesadapter->devfn),
 267                                 nesadapter->bus_number,
 268                                 PCI_SLOT(nesdev->pcidev->devfn),
 269                                 nesdev->pcidev->bus->number );
 270                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 271                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 272                         nesadapter->ref_count++;
 273                         return nesadapter;
 274                 }
 275         }
 276
 277         /* no adapter found */
 278         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 279         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 280                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 281                                 hw_rev);
 282                 return NULL;
 283         }
 284
 285         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 286                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 287                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 288                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 289                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 290
 291         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 292
 293
 294         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 295                 return NULL;
 296
 297         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 298         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 299
 300         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 301         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 302                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 303                                 max_qp, u32temp);
 304                 max_qp = (u32)1 << (u32temp & 0x001f);
 305         }
 306
 307         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 308         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 309                         max_qp, hte_index_mask);
 310
 311         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 312
 313         max_irrq = 1 << (u32temp & 0x001f);
 314
 315         if (max_qp > max_irrq) {
 316                 max_qp = max_irrq;
 317                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 318                                 max_qp);
 319         }
 320
 321         /* there should be no reason to allocate more pds than qps */
 322         if (num_pds > max_qp)
 323                 num_pds = max_qp;
 324
 325         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 326         max_mr = (u32)8192 << (u32temp & 0x7);
 327
 328         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 329         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 330         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 331         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 332
 333         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 334         arp_table_size = 1 << u32temp;
 335
 336         adapter_size = (sizeof(struct nes_adapter) +
 337                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 338         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 339         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 340         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 341         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 342         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 343         adapter_size += sizeof(struct nes_qp **) * max_qp;
 344
 345         /* allocate a new adapter struct */
 346         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 347         if (nesadapter == NULL) {
 348                 return NULL;
 349         }
 350
 351         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 352                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 353
 354         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 355                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 356                 kfree(nesadapter);
 357                 return NULL;
 358         }
 359
 360         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 361                                 (nesadapter->mac_addr_low >> 24);
 362
 363         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 364                                  PCI_DEVICE_ID, &device_id);
 365         nesadapter->vendor_part_id = device_id;
 366
 367         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 368                                                         OneG_Mode)) {
 369                 kfree(nesadapter);
 370                 return NULL;
 371         }
 372         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 373
 374         memset(nesadapter->pft_mcast_map, 255,
 375                sizeof nesadapter->pft_mcast_map);
 376
 377         /* populate the new nesadapter */
 378         nesadapter->devfn = nesdev->pcidev->devfn;
 379         nesadapter->bus_number = nesdev->pcidev->bus->number;
 380         nesadapter->ref_count = 1;
 381         nesadapter->timer_int_req = 0xffff0000;
 382         nesadapter->OneG_Mode = OneG_Mode;
 383         nesadapter->doorbell_start = nesdev->doorbell_region;
 384
 385         /* nesadapter->tick_delta = clk_divisor; */
 386         nesadapter->hw_rev = hw_rev;
 387         nesadapter->port_count = port_count;
 388
 389         nesadapter->max_qp = max_qp;
 390         nesadapter->hte_index_mask = hte_index_mask;
 391         nesadapter->max_irrq = max_irrq;
 392         nesadapter->max_mr = max_mr;
 393         nesadapter->max_256pbl = max_256pbl - 1;
 394         nesadapter->max_4kpbl = max_4kpbl - 1;
 395         nesadapter->max_cq = max_cq;
 396         nesadapter->free_256pbl = max_256pbl - 1;
 397         nesadapter->free_4kpbl = max_4kpbl - 1;
 398         nesadapter->max_pd = num_pds;
 399         nesadapter->arp_table_size = arp_table_size;
 400
 401         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 402         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 403                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 404                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 405                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 406         } else {
 407                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 408                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 409                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 410                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 411         }
 412         /* Setup and enable the periodic timer */
 413         if (nesadapter->et_rx_coalesce_usecs_irq)
 414                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 415                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 416         else
 417                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 418
 419         nesadapter->base_pd = 1;
 420
 421         nesadapter->device_cap_flags =
 422                 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
 423
 424         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 425                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 426         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 427         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 428         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 429         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 430         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 431
 432
 433         /* mark the usual suspect QPs and CQs as in use */
 434         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 435                 set_bit(u32temp, nesadapter->allocated_qps);
 436                 set_bit(u32temp, nesadapter->allocated_cqs);
 437         }
 438
 439         for (u32temp = 0; u32temp < 20; u32temp++)
 440                 set_bit(u32temp, nesadapter->allocated_pds);
 441         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 442
 443         max_rq_wrs = ((u32temp >> 8) & 3);
 444         switch (max_rq_wrs) {
 445                 case 0:
 446                         max_rq_wrs = 4;
 447                         break;
 448                 case 1:
 449                         max_rq_wrs = 16;
 450                         break;
 451                 case 2:
 452                         max_rq_wrs = 32;
 453                         break;
 454                 case 3:
 455                         max_rq_wrs = 512;
 456                         break;
 457         }
 458
 459         max_sq_wrs = (u32temp & 3);
 460         switch (max_sq_wrs) {
 461                 case 0:
 462                         max_sq_wrs = 4;
 463                         break;
 464                 case 1:
 465                         max_sq_wrs = 16;
 466                         break;
 467                 case 2:
 468                         max_sq_wrs = 32;
 469                         break;
 470                 case 3:
 471                         max_sq_wrs = 512;
 472                         break;
 473         }
 474         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 475         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 476
 477         nesadapter->max_sge = 4;
 478         nesadapter->max_cqe = 32767;
 479
 480         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 481                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 482                 kfree(nesadapter);
 483                 return NULL;
 484         }
 485
 486         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 487         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 488                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 489
 490         /* setup port configuration */
 491         if (nesadapter->port_count == 1) {
 492                 nesadapter->log_port = 0x00000000;
 493                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 494                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 495                 else
 496                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 497         } else {
 498                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 499                         nesadapter->log_port = 0x000000D8;
 500                 } else {
 501                         if (nesadapter->port_count == 2)
 502                                 nesadapter->log_port = 0x00000044;
 503                         else
 504                                 nesadapter->log_port = 0x000000e4;
 505                 }
 506                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 507         }
 508
 509         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 510                                                 nesadapter->log_port);
 511         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 512                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 513
 514         spin_lock_init(&nesadapter->resource_lock);
 515         spin_lock_init(&nesadapter->phy_lock);
 516         spin_lock_init(&nesadapter->pbl_lock);
 517         spin_lock_init(&nesadapter->periodic_timer_lock);
 518
 519         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 520         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 521         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 522         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 523
 524         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 525                 u32 pcs_control_status0, pcs_control_status1;
 526                 u32 reset_value;
 527                 u32 i = 0;
 528                 u32 int_cnt = 0;
 529                 u32 ext_cnt = 0;
 530                 unsigned long flags;
 531                 u32 j = 0;
 532
 533                 pcs_control_status0 = nes_read_indexed(nesdev,
 534                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 535                 pcs_control_status1 = nes_read_indexed(nesdev,
 536                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 537
 538                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 539                         pcs_control_status0 = nes_read_indexed(nesdev,
 540                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 541                         pcs_control_status1 = nes_read_indexed(nesdev,
 542                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 543                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 544                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 545                                 int_cnt++;
 546                         msleep(1);
 547                 }
 548                 if (int_cnt > 1) {
 549                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 550                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 551                         mh_detected++;
 552                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 553                         reset_value |= 0x0000003d;
 554                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 555
 556                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 557                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 558                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 559
 560                         pcs_control_status0 = nes_read_indexed(nesdev,
 561                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 562                         pcs_control_status1 = nes_read_indexed(nesdev,
 563                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 564
 565                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 566                                 pcs_control_status0 = nes_read_indexed(nesdev,
 567                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 568                                 pcs_control_status1 = nes_read_indexed(nesdev,
 569                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 570                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 571                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 572                                         if (++ext_cnt > int_cnt) {
 573                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 574                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 575                                                                 0x0000F0C8);
 576                                                 mh_detected++;
 577                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 578                                                 reset_value |= 0x0000003d;
 579                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 580
 581                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 582                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 583                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 584                                                 break;
 585                                         }
 586                                 }
 587                                 msleep(1);
 588                         }
 589                 }
 590         }
 591
 592         if (nesadapter->hw_rev == NE020_REV) {
 593                 init_timer(&nesadapter->mh_timer);
 594                 nesadapter->mh_timer.function = nes_mh_fix;
 595                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 596                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 597                 add_timer(&nesadapter->mh_timer);
 598         } else {
 599                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 600         }
 601
 602         init_timer(&nesadapter->lc_timer);
 603         nesadapter->lc_timer.function = nes_clc;
 604         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 605         nesadapter->lc_timer.data = (unsigned long)nesdev;
 606         add_timer(&nesadapter->lc_timer);
 607
 608         list_add_tail(&nesadapter->list, &nes_adapter_list);
 609
 610         for (func_index = 0; func_index < 8; func_index++) {
 611                 pci_bus_read_config_word(nesdev->pcidev->bus,
 612                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 613                                         func_index), 0, &vendor_id);
 614                 if (vendor_id == 0xffff)
 615                         break;
 616         }
 617         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 618                 func_index, pci_name(nesdev->pcidev));
 619         nesadapter->adapter_fcn_count = func_index;
 620
 621         return nesadapter;
 622 }
 623
 624
 625 /**
 626  * nes_reset_adapter_ne020
 627  */
 628 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 629 {
 630         u32 port_count;
 631         u32 u32temp;
 632         u32 i;
 633
 634         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 635         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 636         /* TODO: assuming that both SERDES are set the same for now */
 637         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 638         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 639                         u32temp, port_count);
 640         if (*OneG_Mode)
 641                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 642         u32temp &= 0xff00ffc0;
 643         switch (port_count) {
 644                 case 1:
 645                         u32temp |= 0x00ee0000;
 646                         break;
 647                 case 2:
 648                         u32temp |= 0x00cc0000;
 649                         break;
 650                 case 4:
 651                         u32temp |= 0x00000000;
 652                         break;
 653                 default:
 654                         return 0;
 655                         break;
 656         }
 657
 658         /* check and do full reset if needed */
 659         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 660                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 661                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 662
 663                 i = 0;
 664                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 665                         mdelay(1);
 666                 if (i >= 10000) {
 667                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 668                         return 0;
 669                 }
 670
 671                 i = 0;
 672                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 673                         mdelay(1);
 674                 if (i >= 10000) {
 675                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 676                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 677                         return 0;
 678                 }
 679         }
 680
 681         /* port reset */
 682         switch (port_count) {
 683                 case 1:
 684                         u32temp |= 0x00ee0010;
 685                         break;
 686                 case 2:
 687                         u32temp |= 0x00cc0030;
 688                         break;
 689                 case 4:
 690                         u32temp |= 0x00000030;
 691                         break;
 692         }
 693
 694         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 695         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 696
 697         i = 0;
 698         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 699                 mdelay(1);
 700         if (i >= 10000) {
 701                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 702                 return 0;
 703         }
 704
 705         /* serdes 0 */
 706         i = 0;
 707         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 708                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 709                 mdelay(1);
 710         if (i >= 5000) {
 711                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 712                 return 0;
 713         }
 714
 715         /* serdes 1 */
 716         if (port_count > 1) {
 717                 i = 0;
 718                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 719                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 720                         mdelay(1);
 721                 if (i >= 5000) {
 722                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 723                         return 0;
 724                 }
 725         }
 726
 727         return port_count;
 728 }
 729
 730
 731 /**
 732  * nes_init_serdes
 733  */
 734 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 735                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 736 {
 737         int i;
 738         u32 u32temp;
 739         u32 serdes_common_control;
 740
 741         if (hw_rev != NE020_REV) {
 742                 /* init serdes 0 */
 743
 744                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 745                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 746                         serdes_common_control = nes_read_indexed(nesdev,
 747                                         NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 748                         serdes_common_control |= 0x000000100;
 749                         nes_write_indexed(nesdev,
 750                                         NES_IDX_ETH_SERDES_COMMON_CONTROL0,
 751                                         serdes_common_control);
 752                 } else if (!OneG_Mode) {
 753                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 754                 }
 755                 if (((port_count > 1) &&
 756                         (nesadapter->phy_type[0] != NES_PHY_TYPE_PUMA_1G)) ||
 757                         ((port_count > 2) &&
 758                         (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G))) {
 759                         /* init serdes 1 */
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 761                         if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 762                                 serdes_common_control = nes_read_indexed(nesdev,
 763                                         NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 764                                 serdes_common_control |= 0x000000100;
 765                                 nes_write_indexed(nesdev,
 766                                         NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 767                                         serdes_common_control);
 768                         } else if (!OneG_Mode) {
 769                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 770                         }
 771                 }
 772         } else {
 773                 /* init serdes 0 */
 774                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 775                 i = 0;
 776                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 777                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 778                         mdelay(1);
 779                 if (i >= 5000) {
 780                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 781                         return 1;
 782                 }
 783                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 784                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 785                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 786                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 787                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 788                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 789                 if (OneG_Mode)
 790                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 791                 else
 792                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 793
 794                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 795                 if (port_count > 1) {
 796                         /* init serdes 1 */
 797                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 798                         i = 0;
 799                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 800                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 801                                 mdelay(1);
 802                         if (i >= 5000) {
 803                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 804                                 /* return 1; */
 805                         }
 806                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 807                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 808                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 809                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 810                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 811                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 812                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 813                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 814                 }
 815         }
 816         return 0;
 817 }
 818
 819
 820 /**
 821  * nes_init_csr_ne020
 822  * Initialize registers for ne020 hardware
 823  */
 824 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 825 {
 826         u32 u32temp;
 827
 828         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 829
 830         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 831         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 832         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 833         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 834         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 835         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 836         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 837         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 838
 839         /* TODO: move these MAC register settings to NIC bringup */
 840         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 841         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 842         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 843         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 844         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 845         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 846         if (port_count > 1) {
 847                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 848                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 849                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 850                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 851                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 852                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 853                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 854         }
 855         if (port_count > 2) {
 856                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 857                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 858                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 859                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 860                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 861                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 862                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 863
 864                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 865                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 866                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 867                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 868                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 869                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 870                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 871         }
 872
 873         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 874         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 875         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 876                                                          0x00000001);
 877         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 878         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 879         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 880         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 881         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 882
 883         /* TODO: move this to code, get from EEPROM */
 884         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 885         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 886         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 887
 888         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 889         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 890
 891         if (hw_rev != NE020_REV) {
 892                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 893                 u32temp |= 0x80000000;
 894                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 895                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 896                 u32temp &= 0x7fffffff;
 897                 u32temp |= 0x7fff0010;
 898                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 899         }
 900 }
 901
 902
 903 /**
 904  * nes_destroy_adapter - destroy the adapter structure
 905  */
 906 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 907 {
 908         struct nes_adapter *tmp_adapter;
 909
 910         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 911                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 912                                 tmp_adapter);
 913         }
 914
 915         nesadapter->ref_count--;
 916         if (!nesadapter->ref_count) {
 917                 if (nesadapter->hw_rev == NE020_REV) {
 918                         del_timer(&nesadapter->mh_timer);
 919                 }
 920                 del_timer(&nesadapter->lc_timer);
 921
 922                 list_del(&nesadapter->list);
 923                 kfree(nesadapter);
 924         }
 925 }
 926
 927
 928 /**
 929  * nes_init_cqp
 930  */
 931 int nes_init_cqp(struct nes_device *nesdev)
 932 {
 933         struct nes_adapter *nesadapter = nesdev->nesadapter;
 934         struct nes_hw_cqp_qp_context *cqp_qp_context;
 935         struct nes_hw_cqp_wqe *cqp_wqe;
 936         struct nes_hw_ceq *ceq;
 937         struct nes_hw_ceq *nic_ceq;
 938         struct nes_hw_aeq *aeq;
 939         void *vmem;
 940         dma_addr_t pmem;
 941         u32 count=0;
 942         u32 cqp_head;
 943         u64 u64temp;
 944         u32 u32temp;
 945
 946         /* allocate CQP memory */
 947         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 948         /* SQ is 512 byte aligned, others are 256 byte aligned */
 949         nesdev->cqp_mem_size = 512 +
 950                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 951                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 952                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 953                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 954                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 955                         sizeof(struct nes_hw_cqp_qp_context);
 956
 957         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 958                         &nesdev->cqp_pbase);
 959         if (!nesdev->cqp_vbase) {
 960                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 961                 return -ENOMEM;
 962         }
 963         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 964
 965         /* Allocate a twice the number of CQP requests as the SQ size */
 966         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 967                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 968         if (nesdev->nes_cqp_requests == NULL) {
 969                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 970                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 971                                 nesdev->cqp.sq_pbase);
 972                 return -ENOMEM;
 973         }
 974
 975         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 976                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 977
 978         spin_lock_init(&nesdev->cqp.lock);
 979         init_waitqueue_head(&nesdev->cqp.waitq);
 980
 981         /* Setup Various Structures */
 982         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 983                         ~(unsigned long)(512 - 1));
 984         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 985                         ~(unsigned long long)(512 - 1));
 986
 987         nesdev->cqp.sq_vbase = vmem;
 988         nesdev->cqp.sq_pbase = pmem;
 989         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 990         nesdev->cqp.sq_head = 0;
 991         nesdev->cqp.sq_tail = 0;
 992         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 993
 994         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 995         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 996
 997         nesdev->ccq.cq_vbase = vmem;
 998         nesdev->ccq.cq_pbase = pmem;
 999         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1000         nesdev->ccq.cq_head = 0;
1001         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1002         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1003
1004         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1005         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1006
1007         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1008         ceq = &nesadapter->ceq[nesdev->ceq_index];
1009         ceq->ceq_vbase = vmem;
1010         ceq->ceq_pbase = pmem;
1011         ceq->ceq_size = NES_CCEQ_SIZE;
1012         ceq->ceq_head = 0;
1013
1014         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1015         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1016
1017         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1018         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1019         nic_ceq->ceq_vbase = vmem;
1020         nic_ceq->ceq_pbase = pmem;
1021         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1022         nic_ceq->ceq_head = 0;
1023
1024         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1025         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1026
1027         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1028         aeq->aeq_vbase = vmem;
1029         aeq->aeq_pbase = pmem;
1030         aeq->aeq_size = nesadapter->max_qp;
1031         aeq->aeq_head = 0;
1032
1033         /* Setup QP Context */
1034         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1035         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1036
1037         cqp_qp_context = vmem;
1038         cqp_qp_context->context_words[0] =
1039                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1040         cqp_qp_context->context_words[1] = 0;
1041         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1042         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1043
1044
1045         /* Write the address to Create CQP */
1046         if ((sizeof(dma_addr_t) > 4)) {
1047                 nes_write_indexed(nesdev,
1048                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1049                                 ((u64)pmem) >> 32);
1050         } else {
1051                 nes_write_indexed(nesdev,
1052                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1053         }
1054         nes_write_indexed(nesdev,
1055                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1056                         (u32)pmem);
1057
1058         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1059         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1060
1061         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1062                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1063                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1064         }
1065
1066         /* Write Create CCQ WQE */
1067         cqp_head = nesdev->cqp.sq_head++;
1068         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1069         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1070         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1071                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1072                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1073         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1074                             (nesdev->ccq.cq_number |
1075                              ((u32)nesdev->ceq_index << 16)));
1076         u64temp = (u64)nesdev->ccq.cq_pbase;
1077         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1078         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1079         u64temp = (unsigned long)&nesdev->ccq;
1080         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1081                         cpu_to_le32((u32)(u64temp >> 1));
1082         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1083                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1084         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1085
1086         /* Write Create CEQ WQE */
1087         cqp_head = nesdev->cqp.sq_head++;
1088         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1089         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1090         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1091                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1092         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1093         u64temp = (u64)ceq->ceq_pbase;
1094         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1095
1096         /* Write Create AEQ WQE */
1097         cqp_head = nesdev->cqp.sq_head++;
1098         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1099         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1100         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1101                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1102         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1103         u64temp = (u64)aeq->aeq_pbase;
1104         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1105
1106         /* Write Create NIC CEQ WQE */
1107         cqp_head = nesdev->cqp.sq_head++;
1108         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1109         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1110         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1111                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1112         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1113         u64temp = (u64)nic_ceq->ceq_pbase;
1114         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1115
1116         /* Poll until CCQP done */
1117         count = 0;
1118         do {
1119                 if (count++ > 1000) {
1120                         printk(KERN_ERR PFX "Error creating CQP\n");
1121                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1122                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1123                         return -1;
1124                 }
1125                 udelay(10);
1126         } while (!(nes_read_indexed(nesdev,
1127                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1128
1129         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1130                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1131
1132         u32temp = 0x04800000;
1133         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1134
1135         /* wait for the CCQ, CEQ, and AEQ to get created */
1136         count = 0;
1137         do {
1138                 if (count++ > 1000) {
1139                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1140                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1141                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1142                         return -1;
1143                 }
1144                 udelay(10);
1145         } while (((nes_read_indexed(nesdev,
1146                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1147
1148         /* dump the QP status value */
1149         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1150                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1151
1152         nesdev->cqp.sq_tail++;
1153
1154         return 0;
1155 }
1156
1157
1158 /**
1159  * nes_destroy_cqp
1160  */
1161 int nes_destroy_cqp(struct nes_device *nesdev)
1162 {
1163         struct nes_hw_cqp_wqe *cqp_wqe;
1164         u32 count = 0;
1165         u32 cqp_head;
1166         unsigned long flags;
1167
1168         do {
1169                 if (count++ > 1000)
1170                         break;
1171                 udelay(10);
1172         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1173
1174         /* Reset CCQ */
1175         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1176                         nesdev->ccq.cq_number);
1177
1178         /* Disable device interrupts */
1179         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1180
1181         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1182
1183         /* Destroy the AEQ */
1184         cqp_head = nesdev->cqp.sq_head++;
1185         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1186         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1187         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1188                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1189         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1190
1191         /* Destroy the NIC CEQ */
1192         cqp_head = nesdev->cqp.sq_head++;
1193         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1194         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1195         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1196                         ((u32)nesdev->nic_ceq_index << 8));
1197
1198         /* Destroy the CEQ */
1199         cqp_head = nesdev->cqp.sq_head++;
1200         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1201         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1202         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1203                         (nesdev->ceq_index << 8));
1204
1205         /* Destroy the CCQ */
1206         cqp_head = nesdev->cqp.sq_head++;
1207         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1208         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1209         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1210         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1211                         ((u32)nesdev->ceq_index << 16));
1212
1213         /* Destroy CQP */
1214         cqp_head = nesdev->cqp.sq_head++;
1215         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1216         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1217         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1218                         NES_CQP_QP_TYPE_CQP);
1219         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1220
1221         barrier();
1222         /* Ring doorbell (5 WQEs) */
1223         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1224
1225         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1226
1227         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1228         count = 0;
1229         do {
1230                 if (count++ > 1000) {
1231                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1232                                         PCI_FUNC(nesdev->pcidev->devfn));
1233                         break;
1234                 }
1235                 udelay(10);
1236         } while (((nes_read_indexed(nesdev,
1237                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1238
1239         /* dump the QP status value */
1240         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1241                         PCI_FUNC(nesdev->pcidev->devfn),
1242                         nes_read_indexed(nesdev,
1243                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1244
1245         kfree(nesdev->nes_cqp_requests);
1246
1247         /* Free the control structures */
1248         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1249                         nesdev->cqp.sq_pbase);
1250
1251         return 0;
1252 }
1253
1254
1255 /**
1256  * nes_init_phy
1257  */
1258 int nes_init_phy(struct nes_device *nesdev)
1259 {
1260         struct nes_adapter *nesadapter = nesdev->nesadapter;
1261         u32 counter = 0;
1262         u32 sds_common_control0;
1263         u32 mac_index = nesdev->mac_index;
1264         u32 tx_config = 0;
1265         u16 phy_data;
1266         u32 temp_phy_data = 0;
1267         u32 temp_phy_data2 = 0;
1268         u32 i = 0;
1269
1270         if ((nesadapter->OneG_Mode) &&
1271             (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
1272                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1273                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1274                         printk(PFX "%s: Programming mdc config for 1G\n", __func__);
1275                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1276                         tx_config &= 0xFFFFFFE3;
1277                         tx_config |= 0x04;
1278                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1279                 }
1280
1281                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1282                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1283                                 nesadapter->phy_index[mac_index], phy_data);
1284                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index], 0xb000);
1285
1286                 /* Reset the PHY */
1287                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1288                 udelay(100);
1289                 counter = 0;
1290                 do {
1291                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1292                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1293                         if (counter++ > 100) break;
1294                 } while (phy_data & 0x8000);
1295
1296                 /* Setting no phy loopback */
1297                 phy_data &= 0xbfff;
1298                 phy_data |= 0x1140;
1299                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1300                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1301                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1302
1303                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1304                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1305
1306                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1307                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1308
1309                 /* Setting the interrupt mask */
1310                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1311                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1312                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1313
1314                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1315                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1316
1317                 /* turning on flow control */
1318                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1319                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1320                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1321                                 (phy_data & ~(0x03E0)) | 0xc00);
1322                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1323                                 phy_data | 0xc00); */
1324                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1325                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1326
1327                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1328                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1329                 /* Clear Half duplex */
1330                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1331                                 phy_data & ~(0x0100));
1332                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1333                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1334
1335                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1336                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1337         } else {
1338                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) ||
1339                     (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1340                         /* setup 10G MDIO operation */
1341                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1342                         tx_config &= 0xFFFFFFE3;
1343                         tx_config |= 0x15;
1344                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1345                 }
1346                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1347                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1348
1349                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1350                         mdelay(10);
1351                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1352                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1353
1354                         /*
1355                          * if firmware is already running (like from a
1356                          * driver un-load/load, don't do anything.
1357                          */
1358                         if (temp_phy_data == temp_phy_data2) {
1359                                 /* configure QT2505 AMCC PHY */
1360                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0x0000, 0x8000);
1361                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0000);
1362                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc302, 0x0044);
1363                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc318, 0x0052);
1364                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc319, 0x0008);
1365                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc31a, 0x0098);
1366                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0026, 0x0E00);
1367                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0027, 0x0001);
1368                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0028, 0xA528);
1369
1370                                 /*
1371                                  * remove micro from reset; chip boots from ROM,
1372                                  * uploads EEPROM f/w image, uC executes f/w
1373                                  */
1374                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0002);
1375
1376                                 /*
1377                                  * wait for heart beat to start to
1378                                  * know loading is done
1379                                  */
1380                                 counter = 0;
1381                                 do {
1382                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1383                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1384                                         if (counter++ > 1000) {
1385                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from heartbeat check <this is bad!!!> \n");
1386                                                 break;
1387                                         }
1388                                         mdelay(100);
1389                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1390                                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1391                                 } while ((temp_phy_data2 == temp_phy_data));
1392
1393                                 /*
1394                                  * wait for tracking to start to know
1395                                  * f/w is good to go
1396                                  */
1397                                 counter = 0;
1398                                 do {
1399                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7fd);
1400                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1401                                         if (counter++ > 1000) {
1402                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from status check <this is bad!!!> \n");
1403                                                 break;
1404                                         }
1405                                         mdelay(1000);
1406                                         /*
1407                                          * nes_debug(NES_DBG_PHY, "AMCC PHY- phy_status not ready yet = 0x%02X\n",
1408                                          *                      temp_phy_data);
1409                                          */
1410                                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1411
1412                                 /* set LOS Control invert RXLOSB_I_PADINV */
1413                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd003, 0x0000);
1414                                 /* set LOS Control to mask of RXLOSB_I */
1415                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc314, 0x0042);
1416                                 /* set LED1 to input mode (LED1 and LED2 share same LED) */
1417                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd006, 0x0007);
1418                                 /* set LED2 to RX link_status and activity */
1419                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd007, 0x000A);
1420                                 /* set LED3 to RX link_status */
1421                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd008, 0x0009);
1422
1423                                 /*
1424                                  * reset the res-calibration on t2
1425                                  * serdes; ensures it is stable after
1426                                  * the amcc phy is stable
1427                                  */
1428
1429                                 sds_common_control0  = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
1430                                 sds_common_control0 |= 0x1;
1431                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1432
1433                                 /* release the res-calibration reset */
1434                                 sds_common_control0 &= 0xfffffffe;
1435                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1436
1437                                 i = 0;
1438                                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1439                                                 && (i++ < 5000)) {
1440                                         /* mdelay(1); */
1441                                 }
1442
1443                                 /*
1444                                  * wait for link train done before moving on,
1445                                  * or will get an interupt storm
1446                                  */
1447                                 counter = 0;
1448                                 do {
1449                                         temp_phy_data = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
1450                                                                 (0x200 * (nesdev->mac_index & 1)));
1451                                         if (counter++ > 1000) {
1452                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from link train wait <this is bad, link didnt train!!!>\n");
1453                                                 break;
1454                                         }
1455                                         mdelay(1);
1456                                 } while (((temp_phy_data & 0x0f1f0000) != 0x0f0f0000));
1457                         }
1458                 }
1459         }
1460         return 0;
1461 }
1462
1463
1464 /**
1465  * nes_replenish_nic_rq
1466  */
1467 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1468 {
1469         unsigned long flags;
1470         dma_addr_t bus_address;
1471         struct sk_buff *skb;
1472         struct nes_hw_nic_rq_wqe *nic_rqe;
1473         struct nes_hw_nic *nesnic;
1474         struct nes_device *nesdev;
1475         u32 rx_wqes_posted = 0;
1476
1477         nesnic = &nesvnic->nic;
1478         nesdev = nesvnic->nesdev;
1479         spin_lock_irqsave(&nesnic->rq_lock, flags);
1480         if (nesnic->replenishing_rq !=0) {
1481                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1482                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1483                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1484                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1485                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1486                         add_timer(&nesvnic->rq_wqes_timer);
1487                 } else
1488                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1489                 return;
1490         }
1491         nesnic->replenishing_rq = 1;
1492         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1493         do {
1494                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1495                 if (skb) {
1496                         skb->dev = nesvnic->netdev;
1497
1498                         bus_address = pci_map_single(nesdev->pcidev,
1499                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1500
1501                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1502                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1503                                         cpu_to_le32(nesvnic->max_frame_size);
1504                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1505                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1506                                         cpu_to_le32((u32)bus_address);
1507                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1508                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1509                         nesnic->rx_skb[nesnic->rq_head] = skb;
1510                         nesnic->rq_head++;
1511                         nesnic->rq_head &= nesnic->rq_size - 1;
1512                         atomic_dec(&nesvnic->rx_skbs_needed);
1513                         barrier();
1514                         if (++rx_wqes_posted == 255) {
1515                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1516                                 rx_wqes_posted = 0;
1517                         }
1518                 } else {
1519                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1520                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1521                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1522                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1523                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1524                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1525                                 add_timer(&nesvnic->rq_wqes_timer);
1526                         } else
1527                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1528                         break;
1529                 }
1530         } while (atomic_read(&nesvnic->rx_skbs_needed));
1531         barrier();
1532         if (rx_wqes_posted)
1533                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1534         nesnic->replenishing_rq = 0;
1535 }
1536
1537
1538 /**
1539  * nes_rq_wqes_timeout
1540  */
1541 static void nes_rq_wqes_timeout(unsigned long parm)
1542 {
1543         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1544         printk("%s: Timer fired.\n", __func__);
1545         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1546         if (atomic_read(&nesvnic->rx_skbs_needed))
1547                 nes_replenish_nic_rq(nesvnic);
1548 }
1549
1550
1551 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1552                                void **tcph, u64 *hdr_flags, void *priv)
1553 {
1554         unsigned int ip_len;
1555         struct iphdr *iph;
1556         skb_reset_network_header(skb);
1557         iph = ip_hdr(skb);
1558         if (iph->protocol != IPPROTO_TCP)
1559                 return -1;
1560         ip_len = ip_hdrlen(skb);
1561         skb_set_transport_header(skb, ip_len);
1562         *tcph = tcp_hdr(skb);
1563
1564         *hdr_flags = LRO_IPV4 | LRO_TCP;
1565         *iphdr = iph;
1566         return 0;
1567 }
1568
1569
1570 /**
1571  * nes_init_nic_qp
1572  */
1573 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1574 {
1575         struct nes_hw_cqp_wqe *cqp_wqe;
1576         struct nes_hw_nic_sq_wqe *nic_sqe;
1577         struct nes_hw_nic_qp_context *nic_context;
1578         struct sk_buff *skb;
1579         struct nes_hw_nic_rq_wqe *nic_rqe;
1580         struct nes_vnic *nesvnic = netdev_priv(netdev);
1581         unsigned long flags;
1582         void *vmem;
1583         dma_addr_t pmem;
1584         u64 u64temp;
1585         int ret;
1586         u32 cqp_head;
1587         u32 counter;
1588         u32 wqe_count;
1589         u8 jumbomode=0;
1590
1591         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1592         nesvnic->nic_mem_size = 256 +
1593                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1594                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1595                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1596                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1597                         sizeof(struct nes_hw_nic_qp_context);
1598
1599         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1600                         &nesvnic->nic_pbase);
1601         if (!nesvnic->nic_vbase) {
1602                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1603                 return -ENOMEM;
1604         }
1605         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1606         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1607                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1608
1609         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1610                         ~(unsigned long)(256 - 1));
1611         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1612                         ~(unsigned long long)(256 - 1));
1613
1614         /* Setup the first Fragment buffers */
1615         nesvnic->nic.first_frag_vbase = vmem;
1616
1617         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1618                 nesvnic->nic.frag_paddr[counter] = pmem;
1619                 pmem += sizeof(struct nes_first_frag);
1620         }
1621
1622         /* setup the SQ */
1623         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1624
1625         nesvnic->nic.sq_vbase = (void *)vmem;
1626         nesvnic->nic.sq_pbase = pmem;
1627         nesvnic->nic.sq_head = 0;
1628         nesvnic->nic.sq_tail = 0;
1629         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1630         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1631                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1632                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1633                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1634                                 NES_NIC_SQ_WQE_COMPLETION);
1635                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1636                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1637                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1638                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1639                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1640                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1641         }
1642
1643         nesvnic->get_cqp_request = nes_get_cqp_request;
1644         nesvnic->post_cqp_request = nes_post_cqp_request;
1645         nesvnic->mcrq_mcast_filter = NULL;
1646
1647         spin_lock_init(&nesvnic->nic.rq_lock);
1648
1649         /* setup the RQ */
1650         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1651         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1652
1653
1654         nesvnic->nic.rq_vbase = vmem;
1655         nesvnic->nic.rq_pbase = pmem;
1656         nesvnic->nic.rq_head = 0;
1657         nesvnic->nic.rq_tail = 0;
1658         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1659
1660         /* setup the CQ */
1661         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1662         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1663
1664         if (nesdev->nesadapter->netdev_count > 2)
1665                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1666         else
1667                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1668
1669         nesvnic->nic_cq.cq_vbase = vmem;
1670         nesvnic->nic_cq.cq_pbase = pmem;
1671         nesvnic->nic_cq.cq_head = 0;
1672         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1673
1674         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1675
1676         /* Send CreateCQ request to CQP */
1677         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1678         cqp_head = nesdev->cqp.sq_head;
1679
1680         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1681         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1682
1683         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1684                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1685                         ((u32)nesvnic->nic_cq.cq_size << 16));
1686         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1687                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1688         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1689         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1690         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1691         u64temp = (unsigned long)&nesvnic->nic_cq;
1692         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1693         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1694                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1695         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1696         if (++cqp_head >= nesdev->cqp.sq_size)
1697                 cqp_head = 0;
1698         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1699         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1700
1701         /* Send CreateQP request to CQP */
1702         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1703         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1704                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1705                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1706         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1707                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1708                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1709         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1710                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1711         }
1712
1713         u64temp = (u64)nesvnic->nic.sq_pbase;
1714         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1715         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1716         u64temp = (u64)nesvnic->nic.rq_pbase;
1717         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1718         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1719
1720         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1721                         NES_CQP_QP_TYPE_NIC);
1722         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1723         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1724                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1725         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1726
1727         if (++cqp_head >= nesdev->cqp.sq_size)
1728                 cqp_head = 0;
1729         nesdev->cqp.sq_head = cqp_head;
1730
1731         barrier();
1732
1733         /* Ring doorbell (2 WQEs) */
1734         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1735
1736         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1737         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1738                         nesvnic->nic.qp_id);
1739
1740         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1741                         NES_EVENT_TIMEOUT);
1742         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1743                         nesvnic->nic.qp_id, ret);
1744         if (!ret) {
1745                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1746                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1747                                 nesvnic->nic_pbase);
1748                 return -EIO;
1749         }
1750
1751         /* Populate the RQ */
1752         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1753                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1754                 if (!skb) {
1755                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1756
1757                         nes_destroy_nic_qp(nesvnic);
1758                         return -ENOMEM;
1759                 }
1760
1761                 skb->dev = netdev;
1762
1763                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1764                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1765
1766                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1767                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1768                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1769                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1770                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1771                 nesvnic->nic.rx_skb[counter] = skb;
1772         }
1773
1774         wqe_count = NES_NIC_WQ_SIZE - 1;
1775         nesvnic->nic.rq_head = wqe_count;
1776         barrier();
1777         do {
1778                 counter = min(wqe_count, ((u32)255));
1779                 wqe_count -= counter;
1780                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1781         } while (wqe_count);
1782         init_timer(&nesvnic->rq_wqes_timer);
1783         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1784         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1785         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1786         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1787         {
1788                 nes_nic_init_timer(nesdev);
1789                 if (netdev->mtu > 1500)
1790                         jumbomode = 1;
1791                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1792         }
1793         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1794         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1795         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1796         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1797         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1798         nesvnic->lro_mgr.dev            = netdev;
1799         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1800         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1801         return 0;
1802 }
1803
1804
1805 /**
1806  * nes_destroy_nic_qp
1807  */
1808 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1809 {
1810         u64 u64temp;
1811         dma_addr_t bus_address;
1812         struct nes_device *nesdev = nesvnic->nesdev;
1813         struct nes_hw_cqp_wqe *cqp_wqe;
1814         struct nes_hw_nic_sq_wqe *nic_sqe;
1815         struct nes_hw_nic_rq_wqe *nic_rqe;
1816         __le16 *wqe_fragment_length;
1817         u16  wqe_fragment_index;
1818         u64 wqe_frag;
1819         u32 cqp_head;
1820         unsigned long flags;
1821         int ret;
1822
1823         /* Free remaining NIC receive buffers */
1824         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1825                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1826                 wqe_frag  = (u64)le32_to_cpu(
1827                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1828                 wqe_frag |= ((u64)le32_to_cpu(
1829                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1830                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1831                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1832                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1833                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1834         }
1835
1836         /* Free remaining NIC transmit buffers */
1837         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1838                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1839                 wqe_fragment_index = 1;
1840                 wqe_fragment_length = (__le16 *)
1841                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1842                 /* bump past the vlan tag */
1843                 wqe_fragment_length++;
1844                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1845                         u64temp = (u64)le32_to_cpu(
1846                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1847                                 wqe_fragment_index*2]);
1848                         u64temp += ((u64)le32_to_cpu(
1849                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1850                                 + wqe_fragment_index*2]))<<32;
1851                         bus_address = (dma_addr_t)u64temp;
1852                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1853                                         nesvnic->nic.first_frag_overflow)) {
1854                                 pci_unmap_single(nesdev->pcidev,
1855                                                 bus_address,
1856                                                 le16_to_cpu(wqe_fragment_length[
1857                                                         wqe_fragment_index++]),
1858                                                 PCI_DMA_TODEVICE);
1859                         }
1860                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1861                                 if (wqe_fragment_length[wqe_fragment_index]) {
1862                                         u64temp = le32_to_cpu(
1863                                                 nic_sqe->wqe_words[
1864                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1865                                                 wqe_fragment_index*2]);
1866                                         u64temp += ((u64)le32_to_cpu(
1867                                                 nic_sqe->wqe_words[
1868                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1869                                                 wqe_fragment_index*2]))<<32;
1870                                         bus_address = (dma_addr_t)u64temp;
1871                                         pci_unmap_page(nesdev->pcidev,
1872                                                         bus_address,
1873                                                         le16_to_cpu(
1874                                                         wqe_fragment_length[
1875                                                         wqe_fragment_index]),
1876                                                         PCI_DMA_TODEVICE);
1877                                 } else
1878                                         break;
1879                         }
1880                 }
1881                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1882                         dev_kfree_skb(
1883                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1884
1885                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1886                                         & (nesvnic->nic.sq_size - 1);
1887         }
1888
1889         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1890
1891         /* Destroy NIC QP */
1892         cqp_head = nesdev->cqp.sq_head;
1893         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1894         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1895
1896         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1897                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1898         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1899                 nesvnic->nic.qp_id);
1900
1901         if (++cqp_head >= nesdev->cqp.sq_size)
1902                 cqp_head = 0;
1903
1904         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1905
1906         /* Destroy NIC CQ */
1907         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1908         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1909                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1910         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1911                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1912
1913         if (++cqp_head >= nesdev->cqp.sq_size)
1914                 cqp_head = 0;
1915
1916         nesdev->cqp.sq_head = cqp_head;
1917         barrier();
1918
1919         /* Ring doorbell (2 WQEs) */
1920         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1921
1922         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1923         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1924                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1925                         cqp_head, nesdev->cqp.sq_head,
1926                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1927
1928         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1929                         NES_EVENT_TIMEOUT);
1930
1931         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1932                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1933                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1934         if (!ret) {
1935                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1936                                 nesvnic->nic.qp_id);
1937         }
1938
1939         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1940                         nesvnic->nic_pbase);
1941 }
1942
1943 /**
1944  * nes_napi_isr
1945  */
1946 int nes_napi_isr(struct nes_device *nesdev)
1947 {
1948         struct nes_adapter *nesadapter = nesdev->nesadapter;
1949         u32 int_stat;
1950
1951         if (nesdev->napi_isr_ran) {
1952                 /* interrupt status has already been read in ISR */
1953                 int_stat = nesdev->int_stat;
1954         } else {
1955                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1956                 nesdev->int_stat = int_stat;
1957                 nesdev->napi_isr_ran = 1;
1958         }
1959
1960         int_stat &= nesdev->int_req;
1961         /* iff NIC, process here, else wait for DPC */
1962         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1963                 nesdev->napi_isr_ran = 0;
1964                 nes_write32(nesdev->regs + NES_INT_STAT,
1965                         (int_stat &
1966                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1967
1968                 /* Process the CEQs */
1969                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1970
1971                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1972                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1973                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1974                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1975                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1976                                 /* Enable Periodic timer interrupts */
1977                                 nesdev->int_req |= NES_INT_TIMER;
1978                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1979                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1980                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1981                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1982                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1983                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1984                         }
1985
1986                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1987                         {
1988                                 nes_nic_init_timer(nesdev);
1989                         }
1990                         /* Enable interrupts, except CEQs */
1991                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1992                 } else {
1993                         /* Enable interrupts, make sure timer is off */
1994                         nesdev->int_req &= ~NES_INT_TIMER;
1995                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1996                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1997                 }
1998                 nesdev->deepcq_count = 0;
1999                 return 1;
2000         } else {
2001                 return 0;
2002         }
2003 }
2004
2005 static void process_critical_error(struct nes_device *nesdev)
2006 {
2007         u32 debug_error;
2008         u32 nes_idx_debug_error_masks0 = 0;
2009         u16 error_module = 0;
2010
2011         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2012         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2013                         (u16)debug_error);
2014         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2015                         0x01010000 | (debug_error & 0x0000ffff));
2016         if (crit_err_count++ > 10)
2017                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2018         error_module = (u16) (debug_error & 0x1F00) >> 8;
2019         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2020                         nes_max_critical_error_count) {
2021                 printk(KERN_ERR PFX "Masking off critical error for module "
2022                         "0x%02X\n", (u16)error_module);
2023                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2024                         NES_IDX_DEBUG_ERROR_MASKS0);
2025                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2026                         nes_idx_debug_error_masks0 | (1 << error_module));
2027         }
2028 }
2029 /**
2030  * nes_dpc
2031  */
2032 void nes_dpc(unsigned long param)
2033 {
2034         struct nes_device *nesdev = (struct nes_device *)param;
2035         struct nes_adapter *nesadapter = nesdev->nesadapter;
2036         u32 counter;
2037         u32 loop_counter = 0;
2038         u32 int_status_bit;
2039         u32 int_stat;
2040         u32 timer_stat;
2041         u32 temp_int_stat;
2042         u32 intf_int_stat;
2043         u32 processed_intf_int = 0;
2044         u16 processed_timer_int = 0;
2045         u16 completion_ints = 0;
2046         u16 timer_ints = 0;
2047
2048         /* nes_debug(NES_DBG_ISR, "\n"); */
2049
2050         do {
2051                 timer_stat = 0;
2052                 if (nesdev->napi_isr_ran) {
2053                         nesdev->napi_isr_ran = 0;
2054                         int_stat = nesdev->int_stat;
2055                 } else
2056                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2057                 if (processed_intf_int != 0)
2058                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2059                 else
2060                         int_stat &= nesdev->int_req;
2061                 if (processed_timer_int == 0) {
2062                         processed_timer_int = 1;
2063                         if (int_stat & NES_INT_TIMER) {
2064                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2065                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2066                                         int_stat &= ~NES_INT_TIMER;
2067                                 }
2068                         }
2069                 } else {
2070                         int_stat &= ~NES_INT_TIMER;
2071                 }
2072
2073                 if (int_stat) {
2074                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2075                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2076                                 /* Ack the interrupts */
2077                                 nes_write32(nesdev->regs+NES_INT_STAT,
2078                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2079                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2080                         }
2081
2082                         temp_int_stat = int_stat;
2083                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2084                                 if (int_stat & int_status_bit) {
2085                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2086                                         temp_int_stat &= ~int_status_bit;
2087                                         completion_ints = 1;
2088                                 }
2089                                 if (!(temp_int_stat & 0x0000ffff))
2090                                         break;
2091                                 int_status_bit <<= 1;
2092                         }
2093
2094                         /* Process the AEQ for this pci function */
2095                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2096                         if (int_stat & int_status_bit) {
2097                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2098                         }
2099
2100                         /* Process the MAC interrupt for this pci function */
2101                         int_status_bit = 1 << (24 + nesdev->mac_index);
2102                         if (int_stat & int_status_bit) {
2103                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2104                         }
2105
2106                         if (int_stat & NES_INT_TIMER) {
2107                                 if (timer_stat & nesdev->timer_int_req) {
2108                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2109                                                         (timer_stat & nesdev->timer_int_req) |
2110                                                         ~(nesdev->nesadapter->timer_int_req));
2111                                         timer_ints = 1;
2112                                 }
2113                         }
2114
2115                         if (int_stat & NES_INT_INTF) {
2116                                 processed_intf_int = 1;
2117                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2118                                 intf_int_stat &= nesdev->intf_int_req;
2119                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2120                                         process_critical_error(nesdev);
2121                                 }
2122                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2123                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2124                                         BUG();
2125                                 }
2126                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2127                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2128                                         BUG();
2129                                 }
2130                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2131                         }
2132
2133                         if (int_stat & NES_INT_TSW) {
2134                         }
2135                 }
2136                 /* Don't use the interface interrupt bit stay in loop */
2137                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2138                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2139         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2140
2141         if (timer_ints == 1) {
2142                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2143                         if (completion_ints == 0) {
2144                                 nesdev->timer_only_int_count++;
2145                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2146                                         nesdev->timer_only_int_count = 0;
2147                                         nesdev->int_req &= ~NES_INT_TIMER;
2148                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2149                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2150                                 } else {
2151                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2152                                 }
2153                         } else {
2154                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2155                                 {
2156                                         nes_nic_init_timer(nesdev);
2157                                 }
2158                                 nesdev->timer_only_int_count = 0;
2159                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2160                         }
2161                 } else {
2162                         nesdev->timer_only_int_count = 0;
2163                         nesdev->int_req &= ~NES_INT_TIMER;
2164                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2165                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2166                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2167                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2168                 }
2169         } else {
2170                 if ( (completion_ints == 1) &&
2171                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2172                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2173                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2174                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2175                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2176                         nesdev->timer_only_int_count = 0;
2177                         nesdev->int_req |= NES_INT_TIMER;
2178                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2179                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2180                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2181                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2182                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2183                 } else {
2184                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2185                 }
2186         }
2187         nesdev->deepcq_count = 0;
2188 }
2189
2190
2191 /**
2192  * nes_process_ceq
2193  */
2194 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2195 {
2196         u64 u64temp;
2197         struct nes_hw_cq *cq;
2198         u32 head;
2199         u32 ceq_size;
2200
2201         /* nes_debug(NES_DBG_CQ, "\n"); */
2202         head = ceq->ceq_head;
2203         ceq_size = ceq->ceq_size;
2204
2205         do {
2206                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2207                                 NES_CEQE_VALID) {
2208                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2209                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2210                         u64temp <<= 1;
2211                         cq = *((struct nes_hw_cq **)&u64temp);
2212                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2213                         barrier();
2214                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2215
2216                         /* call the event handler */
2217                         cq->ce_handler(nesdev, cq);
2218
2219                         if (++head >= ceq_size)
2220                                 head = 0;
2221                 } else {
2222                         break;
2223                 }
2224
2225         } while (1);
2226
2227         ceq->ceq_head = head;
2228 }
2229
2230
2231 /**
2232  * nes_process_aeq
2233  */
2234 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2235 {
2236         /* u64 u64temp; */
2237         u32 head;
2238         u32 aeq_size;
2239         u32 aeqe_misc;
2240         u32 aeqe_cq_id;
2241         struct nes_hw_aeqe volatile *aeqe;
2242
2243         head = aeq->aeq_head;
2244         aeq_size = aeq->aeq_size;
2245
2246         do {
2247                 aeqe = &aeq->aeq_vbase[head];
2248                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2249                         break;
2250                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2251                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2252                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2253                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2254                                 /* dealing with an accelerated QP related AE */
2255                                 /*
2256                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2257                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2258                                  */
2259                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2260                         } else {
2261                                 /* TODO: dealing with a CQP related AE */
2262                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2263                                                 (u16)(aeqe_misc >> 16));
2264                         }
2265                 }
2266
2267                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2268
2269                 if (++head >= aeq_size)
2270                         head = 0;
2271
2272                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2273         }
2274         while (1);
2275         aeq->aeq_head = head;
2276 }
2277
2278 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2279 {
2280         struct nes_adapter *nesadapter = nesdev->nesadapter;
2281         u32 reset_value;
2282         u32 i=0;
2283         u32 u32temp;
2284
2285         if (nesadapter->hw_rev == NE020_REV) {
2286                 return;
2287         }
2288         mh_detected++;
2289
2290         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2291
2292         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2293                 reset_value |= 0x0000001d;
2294         else
2295                 reset_value |= 0x0000002d;
2296
2297         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2298                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2299                         nesadapter->link_interrupt_count[0] = 0;
2300                         nesadapter->link_interrupt_count[1] = 0;
2301                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2302                         if (0x00000040 & u32temp)
2303                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2304                         else
2305                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2306
2307                         reset_value |= 0x0000003d;
2308                 }
2309                 nesadapter->link_interrupt_count[mac_index] = 0;
2310         }
2311
2312         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2313
2314         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2315                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2316
2317         if (0x0000003d == (reset_value & 0x0000003d)) {
2318                 u32 pcs_control_status0, pcs_control_status1;
2319
2320                 for (i = 0; i < 10; i++) {
2321                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2322                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2323                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2324                              && (pcs_control_status0 & 0x00100000))
2325                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2326                                 && (pcs_control_status1 & 0x00100000)))
2327                                 continue;
2328                         else
2329                                 break;
2330                 }
2331                 if (10 == i) {
2332                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2333                         if (0x00000040 & u32temp)
2334                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2335                         else
2336                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2337
2338                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2339
2340                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2341                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2342                 }
2343         }
2344 }
2345
2346 /**
2347  * nes_process_mac_intr
2348  */
2349 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2350 {
2351         unsigned long flags;
2352         u32 pcs_control_status;
2353         struct nes_adapter *nesadapter = nesdev->nesadapter;
2354         struct nes_vnic *nesvnic;
2355         u32 mac_status;
2356         u32 mac_index = nesdev->mac_index;
2357         u32 u32temp;
2358         u16 phy_data;
2359         u16 temp_phy_data;
2360         u32 pcs_val  = 0x0f0f0000;
2361         u32 pcs_mask = 0x0f1f0000;
2362
2363         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2364         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2365                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2366                 return;
2367         }
2368         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2369         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2370
2371         /* ack the MAC interrupt */
2372         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2373         /* Clear the interrupt */
2374         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2375
2376         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2377
2378         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2379                 nesdev->link_status_interrupts++;
2380                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2381                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2382                         nes_reset_link(nesdev, mac_index);
2383                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2384                 }
2385                 /* read the PHY interrupt status register */
2386                 if ((nesadapter->OneG_Mode) &&
2387                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2388                         do {
2389                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2390                                                 nesadapter->phy_index[mac_index], &phy_data);
2391                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2392                                                 nesadapter->phy_index[mac_index], phy_data);
2393                         } while (phy_data&0x8000);
2394
2395                         temp_phy_data = 0;
2396                         do {
2397                                 nes_read_1G_phy_reg(nesdev, 0x11,
2398                                                 nesadapter->phy_index[mac_index], &phy_data);
2399                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2400                                                 nesadapter->phy_index[mac_index], phy_data);
2401                                 if (temp_phy_data == phy_data)
2402                                         break;
2403                                 temp_phy_data = phy_data;
2404                         } while (1);
2405
2406                         nes_read_1G_phy_reg(nesdev, 0x1e,
2407                                         nesadapter->phy_index[mac_index], &phy_data);
2408                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2409                                         nesadapter->phy_index[mac_index], phy_data);
2410
2411                         nes_read_1G_phy_reg(nesdev, 1,
2412                                         nesadapter->phy_index[mac_index], &phy_data);
2413                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2414                                         nesadapter->phy_index[mac_index], phy_data);
2415
2416                         if (temp_phy_data & 0x1000) {
2417                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2418                                 phy_data = 4;
2419                         } else {
2420                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2421                         }
2422                 }
2423                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2424                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2425                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2426
2427                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2428                         switch (mac_index) {
2429                         case 1:
2430                         case 3:
2431                                 pcs_control_status = nes_read_indexed(nesdev,
2432                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2433                                 break;
2434                         default:
2435                                 pcs_control_status = nes_read_indexed(nesdev,
2436                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2437                                 break;
2438                         }
2439                 } else {
2440                         pcs_control_status = nes_read_indexed(nesdev,
2441                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2442                         pcs_control_status = nes_read_indexed(nesdev,
2443                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2444                 }
2445
2446                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2447                                 mac_index, pcs_control_status);
2448                 if ((nesadapter->OneG_Mode) &&
2449                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2450                         u32temp = 0x01010000;
2451                         if (nesadapter->port_count > 2) {
2452                                 u32temp |= 0x02020000;
2453                         }
2454                         if ((pcs_control_status & u32temp)!= u32temp) {
2455                                 phy_data = 0;
2456                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2457                         }
2458                 } else {
2459                         switch (nesadapter->phy_type[mac_index]) {
2460                         case NES_PHY_TYPE_IRIS:
2461                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2462                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2463                                 u32temp = 20;
2464                                 do {
2465                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2466                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2467                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2468                                                 break;
2469                                         temp_phy_data = phy_data;
2470                                 } while (1);
2471                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2472                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2473                                 break;
2474
2475                         case NES_PHY_TYPE_ARGUS:
2476                                 /* clear the alarms */
2477                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2478                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2479                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2480                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2481                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2482                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2483                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2484                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2485                                 /* check link status */
2486                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2487                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2488                                 u32temp = 100;
2489                                 do {
2490                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2491
2492                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2493                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2494                                                 break;
2495                                         temp_phy_data = phy_data;
2496                                 } while (1);
2497                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2498                                         __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2499                                 break;
2500
2501                         case NES_PHY_TYPE_PUMA_1G:
2502                                 if (mac_index < 2)
2503                                         pcs_val = pcs_mask = 0x01010000;
2504                                 else
2505                                         pcs_val = pcs_mask = 0x02020000;
2506                                 /* fall through */
2507                         default:
2508                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2509                                 break;
2510                         }
2511                 }
2512
2513                 if (phy_data & 0x0004) {
2514                         nesadapter->mac_link_down[mac_index] = 0;
2515                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2516                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2517                                                 nesvnic->linkup);
2518                                 if (nesvnic->linkup == 0) {
2519                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2520                                                         nesvnic->netdev->name, nesvnic->netdev);
2521                                         if (netif_queue_stopped(nesvnic->netdev))
2522                                                 netif_start_queue(nesvnic->netdev);
2523                                         nesvnic->linkup = 1;
2524                                         netif_carrier_on(nesvnic->netdev);
2525                                 }
2526                         }
2527                 } else {
2528                         nesadapter->mac_link_down[mac_index] = 1;
2529                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2530                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2531                                                 nesvnic->linkup);
2532                                 if (nesvnic->linkup == 1) {
2533                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2534                                                         nesvnic->netdev->name, nesvnic->netdev);
2535                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2536                                                 netif_stop_queue(nesvnic->netdev);
2537                                         nesvnic->linkup = 0;
2538                                         netif_carrier_off(nesvnic->netdev);
2539                                 }
2540                         }
2541                 }
2542         }
2543
2544         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2545 }
2546
2547
2548
2549 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2550 {
2551         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2552
2553         napi_schedule(&nesvnic->napi);
2554 }
2555
2556
2557 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2558 * getting out of nic_ce_handler
2559 */
2560 #define MAX_RQES_TO_PROCESS     384
2561
2562 /**
2563  * nes_nic_ce_handler
2564  */
2565 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2566 {
2567         u64 u64temp;
2568         dma_addr_t bus_address;
2569         struct nes_hw_nic *nesnic;
2570         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2571         struct nes_adapter *nesadapter = nesdev->nesadapter;
2572         struct nes_hw_nic_rq_wqe *nic_rqe;
2573         struct nes_hw_nic_sq_wqe *nic_sqe;
2574         struct sk_buff *skb;
2575         struct sk_buff *rx_skb;
2576         __le16 *wqe_fragment_length;
2577         u32 head;
2578         u32 cq_size;
2579         u32 rx_pkt_size;
2580         u32 cqe_count=0;
2581         u32 cqe_errv;
2582         u32 cqe_misc;
2583         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2584         u16 vlan_tag;
2585         u16 pkt_type;
2586         u16 rqes_processed = 0;
2587         u8 sq_cqes = 0;
2588         u8 nes_use_lro = 0;
2589
2590         head = cq->cq_head;
2591         cq_size = cq->cq_size;
2592         cq->cqes_pending = 1;
2593         if (nesvnic->netdev->features & NETIF_F_LRO)
2594                 nes_use_lro = 1;
2595         do {
2596                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2597                                 NES_NIC_CQE_VALID) {
2598                         nesnic = &nesvnic->nic;
2599                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2600                         if (cqe_misc & NES_NIC_CQE_SQ) {
2601                                 sq_cqes++;
2602                                 wqe_fragment_index = 1;
2603                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2604                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2605                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2606                                 /* bump past the vlan tag */
2607                                 wqe_fragment_length++;
2608                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2609                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2610                                                         wqe_fragment_index * 2]);
2611                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2612                                                         wqe_fragment_index * 2])) << 32;
2613                                         bus_address = (dma_addr_t)u64temp;
2614                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2615                                                 pci_unmap_single(nesdev->pcidev,
2616                                                                 bus_address,
2617                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2618                                                                 PCI_DMA_TODEVICE);
2619                                         }
2620                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2621                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2622                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2623                                                                                 wqe_fragment_index * 2]);
2624                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2625                                                                                 + wqe_fragment_index * 2])) <<32;
2626                                                         bus_address = (dma_addr_t)u64temp;
2627                                                         pci_unmap_page(nesdev->pcidev,
2628                                                                         bus_address,
2629                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2630                                                                         PCI_DMA_TODEVICE);
2631                                                 } else
2632                                                         break;
2633                                         }
2634                                 }
2635                                 if (skb)
2636                                         dev_kfree_skb_any(skb);
2637                                 nesnic->sq_tail++;
2638                                 nesnic->sq_tail &= nesnic->sq_size-1;
2639                                 if (sq_cqes > 128) {
2640                                         barrier();
2641                                 /* restart the queue if it had been stopped */
2642                                 if (netif_queue_stopped(nesvnic->netdev))
2643                                         netif_wake_queue(nesvnic->netdev);
2644                                         sq_cqes = 0;
2645                                 }
2646                         } else {
2647                                 rqes_processed ++;
2648
2649                                 cq->rx_cqes_completed++;
2650                                 cq->rx_pkts_indicated++;
2651                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2652                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2653                                 /* Get the skb */
2654                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2655                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2656                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2657                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2658                                 pci_unmap_single(nesdev->pcidev, bus_address,
2659                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2660                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2661                                 /* rx_skb->len = rx_pkt_size; */
2662                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2663                                 skb_put(rx_skb, rx_pkt_size);
2664                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2665                                 nesnic->rq_tail++;
2666                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2667
2668                                 atomic_inc(&nesvnic->rx_skbs_needed);
2669                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2670                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2671                                                         cq->cq_number | (cqe_count << 16));
2672                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2673                                         nesdev->currcq_count += cqe_count;
2674                                         cqe_count = 0;
2675                                         nes_replenish_nic_rq(nesvnic);
2676                                 }
2677                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2678                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2679                                 rx_skb->ip_summed = CHECKSUM_NONE;
2680
2681                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2682                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2683                                         if ((cqe_errv &
2684                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2685                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2686                                                 if (nesvnic->rx_checksum_disabled == 0) {
2687                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2688                                                 }
2689                                         } else
2690                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2691                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2692                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2693
2694                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2695                                         if ((cqe_errv &
2696                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2697                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2698                                                 if (nesvnic->rx_checksum_disabled == 0) {
2699                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2700                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2701                                                                   nesvnic->netdev->name); */
2702                                                 }
2703                                         } else
2704                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2705                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2706                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2707                                         }
2708                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2709                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2710
2711                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2712                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2713                                                 rx_skb = NULL;
2714                                 }
2715                                 if (rx_skb == NULL)
2716                                         goto skip_rx_indicate0;
2717
2718
2719                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2720                                     (nesvnic->vlan_grp != NULL)) {
2721                                         vlan_tag = (u16)(le32_to_cpu(
2722                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2723                                                         >> 16);
2724                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2725                                                         nesvnic->netdev->name, vlan_tag);
2726                                         if (nes_use_lro)
2727                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2728                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2729                                         else
2730                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2731                                 } else {
2732                                         if (nes_use_lro)
2733                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2734                                         else
2735                                                 nes_netif_rx(rx_skb);
2736                                 }
2737
2738 skip_rx_indicate0:
2739                                 nesvnic->netdev->last_rx = jiffies;
2740                                 /* nesvnic->netstats.rx_packets++; */
2741                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2742                         }
2743
2744                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2745                         /* Accounting... */
2746                         cqe_count++;
2747                         if (++head >= cq_size)
2748                                 head = 0;
2749                         if (cqe_count == 255) {
2750                                 /* Replenish Nic CQ */
2751                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2752                                                 cq->cq_number | (cqe_count << 16));
2753                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2754                                 nesdev->currcq_count += cqe_count;
2755                                 cqe_count = 0;
2756                         }
2757
2758                         if (cq->rx_cqes_completed >= nesvnic->budget)
2759                                 break;
2760                 } else {
2761                         cq->cqes_pending = 0;
2762                         break;
2763                 }
2764
2765         } while (1);
2766
2767         if (nes_use_lro)
2768                 lro_flush_all(&nesvnic->lro_mgr);
2769         if (sq_cqes) {
2770                 barrier();
2771                 /* restart the queue if it had been stopped */
2772                 if (netif_queue_stopped(nesvnic->netdev))
2773                         netif_wake_queue(nesvnic->netdev);
2774         }
2775         cq->cq_head = head;
2776         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2777                         cq->cq_number, cqe_count, cq->cq_head); */
2778         cq->cqe_allocs_pending = cqe_count;
2779         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2780         {
2781                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2782                 nesdev->currcq_count += cqe_count;
2783                 nes_nic_tune_timer(nesdev);
2784         }
2785         if (atomic_read(&nesvnic->rx_skbs_needed))
2786                 nes_replenish_nic_rq(nesvnic);
2787 }
2788
2789
2790 /**
2791  * nes_cqp_ce_handler
2792  */
2793 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2794 {
2795         u64 u64temp;
2796         unsigned long flags;
2797         struct nes_hw_cqp *cqp = NULL;
2798         struct nes_cqp_request *cqp_request;
2799         struct nes_hw_cqp_wqe *cqp_wqe;
2800         u32 head;
2801         u32 cq_size;
2802         u32 cqe_count=0;
2803         u32 error_code;
2804         /* u32 counter; */
2805
2806         head = cq->cq_head;
2807         cq_size = cq->cq_size;
2808
2809         do {
2810                 /* process the CQE */
2811                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2812                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2813
2814                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2815                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2816                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2817                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2818                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2819                         cqp = *((struct nes_hw_cqp **)&u64temp);
2820
2821                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2822                         if (error_code) {
2823                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2824                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2825                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2826                                                 (u16)(error_code >> 16),
2827                                                 (u16)error_code);
2828                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2829                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2830                         }
2831
2832                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2833                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2834                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2835                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2836                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2837                         if (cqp_request) {
2838                                 if (cqp_request->waiting) {
2839                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2840                                         cqp_request->major_code = (u16)(error_code >> 16);
2841                                         cqp_request->minor_code = (u16)error_code;
2842                                         barrier();
2843                                         cqp_request->request_done = 1;
2844                                         wake_up(&cqp_request->waitq);
2845                                         nes_put_cqp_request(nesdev, cqp_request);
2846                                 } else {
2847                                         if (cqp_request->callback)
2848                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2849                                         nes_free_cqp_request(nesdev, cqp_request);
2850                                 }
2851                         } else {
2852                                 wake_up(&nesdev->cqp.waitq);
2853                         }
2854
2855                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2856                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2857                         if (++cqp->sq_tail >= cqp->sq_size)
2858                                 cqp->sq_tail = 0;
2859
2860                         /* Accounting... */
2861                         cqe_count++;
2862                         if (++head >= cq_size)
2863                                 head = 0;
2864                 } else {
2865                         break;
2866                 }
2867         } while (1);
2868         cq->cq_head = head;
2869
2870         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2871         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2872                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2873                         (nesdev->cqp.sq_size - 1)) != 1)) {
2874                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2875                                 struct nes_cqp_request, list);
2876                 list_del_init(&cqp_request->list);
2877                 head = nesdev->cqp.sq_head++;
2878                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2879                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2880                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2881                 barrier();
2882                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2883                         cpu_to_le32((u32)((unsigned long)cqp_request));
2884                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2885                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2886                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2887                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2888                 /* Ring doorbell (1 WQEs) */
2889                 barrier();
2890                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2891         }
2892         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2893
2894         /* Arm the CCQ */
2895         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2896                         cq->cq_number);
2897         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2898 }
2899
2900
2901 /**
2902  * nes_process_iwarp_aeqe
2903  */
2904 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2905                                    struct nes_hw_aeqe *aeqe)
2906 {
2907         u64 context;
2908         u64 aeqe_context = 0;
2909         unsigned long flags;
2910         struct nes_qp *nesqp;
2911         int resource_allocated;
2912         /* struct iw_cm_id *cm_id; */
2913         struct nes_adapter *nesadapter = nesdev->nesadapter;
2914         struct ib_event ibevent;
2915         /* struct iw_cm_event cm_event; */
2916         u32 aeq_info;
2917         u32 next_iwarp_state = 0;
2918         u16 async_event_id;
2919         u8 tcp_state;
2920         u8 iwarp_state;
2921
2922         nes_debug(NES_DBG_AEQ, "\n");
2923         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2924         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2925                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2926                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2927         } else {
2928                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2929                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2930                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2931                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
2932                 BUG_ON(!context);
2933         }
2934
2935         async_event_id = (u16)aeq_info;
2936         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2937         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2938         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2939                         " Tcp state = %s, iWARP state = %s\n",
2940                         async_event_id,
2941                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2942                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2943
2944         switch (async_event_id) {
2945                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2946                         nesqp = *((struct nes_qp **)&context);
2947                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2948                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2949                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2950                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2951                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2952                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2953                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2954                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2955                                                 async_event_id, nesqp->last_aeq, tcp_state);
2956                         }
2957                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2958                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2959                                 /* FIN Received but tcp state or IB state moved on,
2960                                                 should expect a close complete */
2961                                 return;
2962                         }
2963                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2964                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2965                 case NES_AEQE_AEID_TERMINATE_SENT:
2966                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2967                 case NES_AEQE_AEID_RESET_SENT:
2968                         nesqp = *((struct nes_qp **)&context);
2969                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2970                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2971                         }
2972                         spin_lock_irqsave(&nesqp->lock, flags);
2973                         nesqp->hw_iwarp_state = iwarp_state;
2974                         nesqp->hw_tcp_state = tcp_state;
2975                         nesqp->last_aeq = async_event_id;
2976
2977                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2978                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2979                                 nesqp->hte_added = 0;
2980                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2981                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2982                                                 nesqp->hwqp.qp_id);
2983                                 nes_hw_modify_qp(nesdev, nesqp,
2984                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2985                                 spin_lock_irqsave(&nesqp->lock, flags);
2986                         }
2987
2988                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2989                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2990                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2991                                 switch (nesqp->hw_iwarp_state) {
2992                                         case NES_AEQE_IWARP_STATE_RTS:
2993                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2994                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2995                                                 break;
2996                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2997                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2998                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2999                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
3000                                                         next_iwarp_state |= 0x02000000;
3001                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3002                                                 }
3003                                                 break;
3004                                         default:
3005                                                 next_iwarp_state = 0;
3006                                 }
3007                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3008                                 if (next_iwarp_state) {
3009                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3010                                                         " also added another reference\n",
3011                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3012                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3013                                 }
3014                                 nes_cm_disconn(nesqp);
3015                         } else {
3016                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
3017                                         /* FIN Received but ib state not RTS,
3018                                                         close complete will be on its way */
3019                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3020                                         return;
3021                                 }
3022                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3023                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
3024                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
3025                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3026                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3027                                                         " also added another reference\n",
3028                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3029                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3030                                 }
3031                                 nes_cm_disconn(nesqp);
3032                         }
3033                         break;
3034                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3035                         nesqp = *((struct nes_qp **)&context);
3036                         spin_lock_irqsave(&nesqp->lock, flags);
3037                         nesqp->hw_iwarp_state = iwarp_state;
3038                         nesqp->hw_tcp_state = tcp_state;
3039                         nesqp->last_aeq = async_event_id;
3040                         spin_unlock_irqrestore(&nesqp->lock, flags);
3041                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
3042                                         " event on QP%u \n  Q2 Data:\n",
3043                                         nesqp->hwqp.qp_id);
3044                         if (nesqp->ibqp.event_handler) {
3045                                 ibevent.device = nesqp->ibqp.device;
3046                                 ibevent.element.qp = &nesqp->ibqp;
3047                                 ibevent.event = IB_EVENT_QP_FATAL;
3048                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3049                         }
3050                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3051                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
3052                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
3053                                 nes_cm_disconn(nesqp);
3054                         } else {
3055                                 nesqp->in_disconnect = 0;
3056                                 wake_up(&nesqp->kick_waitq);
3057                         }
3058                         break;
3059                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3060                         nesqp = *((struct nes_qp **)&context);
3061                         spin_lock_irqsave(&nesqp->lock, flags);
3062                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
3063                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3064                         nesqp->last_aeq = async_event_id;
3065                         if (nesqp->cm_id) {
3066                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3067                                                 " event on QP%u, remote IP = 0x%08X \n",
3068                                                 nesqp->hwqp.qp_id,
3069                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
3070                         } else {
3071                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3072                                                 " event on QP%u \n",
3073                                                 nesqp->hwqp.qp_id);
3074                         }
3075                         spin_unlock_irqrestore(&nesqp->lock, flags);
3076                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
3077                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3078                         if (nesqp->ibqp.event_handler) {
3079                                 ibevent.device = nesqp->ibqp.device;
3080                                 ibevent.element.qp = &nesqp->ibqp;
3081                                 ibevent.event = IB_EVENT_QP_FATAL;
3082                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3083                         }
3084                         break;
3085                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3086                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
3087                                 nesqp = nesadapter->qp_table[le32_to_cpu(
3088                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3089                         } else {
3090                                 /* TODO: get the actual WQE and mask off wqe index */
3091                                 context &= ~((u64)511);
3092                                 nesqp = *((struct nes_qp **)&context);
3093                         }
3094                         spin_lock_irqsave(&nesqp->lock, flags);
3095                         nesqp->hw_iwarp_state = iwarp_state;
3096                         nesqp->hw_tcp_state = tcp_state;
3097                         nesqp->last_aeq = async_event_id;
3098                         spin_unlock_irqrestore(&nesqp->lock, flags);
3099                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
3100                                         nesqp->hwqp.qp_id);
3101                         if (nesqp->ibqp.event_handler) {
3102                                 ibevent.device = nesqp->ibqp.device;
3103                                 ibevent.element.qp = &nesqp->ibqp;
3104                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3105                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3106                         }
3107                         break;
3108                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3109                         nesqp = *((struct nes_qp **)&context);
3110                         spin_lock_irqsave(&nesqp->lock, flags);
3111                         nesqp->hw_iwarp_state = iwarp_state;
3112                         nesqp->hw_tcp_state = tcp_state;
3113                         nesqp->last_aeq = async_event_id;
3114                         spin_unlock_irqrestore(&nesqp->lock, flags);
3115                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
3116                                         nesqp->hwqp.qp_id);
3117                         if (nesqp->ibqp.event_handler) {
3118                                 ibevent.device = nesqp->ibqp.device;
3119                                 ibevent.element.qp = &nesqp->ibqp;
3120                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3121                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3122                         }
3123                         break;
3124                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3125                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
3126                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3127                         spin_lock_irqsave(&nesqp->lock, flags);
3128                         nesqp->hw_iwarp_state = iwarp_state;
3129                         nesqp->hw_tcp_state = tcp_state;
3130                         nesqp->last_aeq = async_event_id;
3131                         spin_unlock_irqrestore(&nesqp->lock, flags);
3132                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
3133                                         " nesqp = %p, AE reported %p\n",
3134                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
3135                         if (nesqp->ibqp.event_handler) {
3136                                 ibevent.device = nesqp->ibqp.device;
3137                                 ibevent.element.qp = &nesqp->ibqp;
3138                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3139                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3140                         }
3141                         break;
3142                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3143                         context <<= 1;
3144                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3145                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3146                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3147                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3148                         if (resource_allocated) {
3149                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3150                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3151                         }
3152                         break;
3153                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3154                         nesqp = nesadapter->qp_table[le32_to_cpu(
3155                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3156                         spin_lock_irqsave(&nesqp->lock, flags);
3157                         nesqp->hw_iwarp_state = iwarp_state;
3158                         nesqp->hw_tcp_state = tcp_state;
3159                         nesqp->last_aeq = async_event_id;
3160                         spin_unlock_irqrestore(&nesqp->lock, flags);
3161                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
3162                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
3163                                         nesqp->hwqp.qp_id);
3164                         if (nesqp->ibqp.event_handler) {
3165                                 ibevent.device = nesqp->ibqp.device;
3166                                 ibevent.element.qp = &nesqp->ibqp;
3167                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3168                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3169                         }
3170                         /* tell cm to disconnect, cm will queue work to thread */
3171                         nes_cm_disconn(nesqp);
3172                         break;
3173                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3174                         nesqp = *((struct nes_qp **)&context);
3175                         spin_lock_irqsave(&nesqp->lock, flags);
3176                         nesqp->hw_iwarp_state = iwarp_state;
3177                         nesqp->hw_tcp_state = tcp_state;
3178                         nesqp->last_aeq = async_event_id;
3179                         spin_unlock_irqrestore(&nesqp->lock, flags);
3180                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
3181                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
3182                                         nesqp->hwqp.qp_id);
3183                         if (nesqp->ibqp.event_handler) {
3184                                 ibevent.device = nesqp->ibqp.device;
3185                                 ibevent.element.qp = &nesqp->ibqp;
3186                                 ibevent.event = IB_EVENT_QP_FATAL;
3187                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3188                         }
3189                         /* tell cm to disconnect, cm will queue work to thread */
3190                         nes_cm_disconn(nesqp);
3191                         break;
3192                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3193                         nesqp = *((struct nes_qp **)&context);
3194                         spin_lock_irqsave(&nesqp->lock, flags);
3195                         nesqp->hw_iwarp_state = iwarp_state;
3196                         nesqp->hw_tcp_state = tcp_state;
3197                         nesqp->last_aeq = async_event_id;
3198                         spin_unlock_irqrestore(&nesqp->lock, flags);
3199                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
3200                                         " event on QP%u \n  Q2 Data:\n",
3201                                         nesqp->hwqp.qp_id);
3202                         if (nesqp->ibqp.event_handler) {
3203                                 ibevent.device = nesqp->ibqp.device;
3204                                 ibevent.element.qp = &nesqp->ibqp;
3205                                 ibevent.event = IB_EVENT_QP_FATAL;
3206                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3207                         }
3208                         /* tell cm to disconnect, cm will queue work to thread */
3209                         nes_cm_disconn(nesqp);
3210                         break;
3211                         /* TODO: additional AEs need to be here */
3212                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3213                         nesqp = *((struct nes_qp **)&context);
3214                         spin_lock_irqsave(&nesqp->lock, flags);
3215                         nesqp->hw_iwarp_state = iwarp_state;
3216                         nesqp->hw_tcp_state = tcp_state;
3217                         nesqp->last_aeq = async_event_id;
3218                         spin_unlock_irqrestore(&nesqp->lock, flags);
3219                         if (nesqp->ibqp.event_handler) {
3220                                 ibevent.device = nesqp->ibqp.device;
3221                                 ibevent.element.qp = &nesqp->ibqp;
3222                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3223                                 nesqp->ibqp.event_handler(&ibevent,
3224                                                 nesqp->ibqp.qp_context);
3225                         }
3226                         nes_cm_disconn(nesqp);
3227                         break;
3228                 default:
3229                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3230                                         async_event_id);
3231                         break;
3232         }
3233
3234 }
3235
3236
3237 /**
3238  * nes_iwarp_ce_handler
3239  */
3240 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3241 {
3242         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3243
3244         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3245                         nescq->hw_cq.cq_number); */
3246         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3247
3248         if (nescq->ibcq.comp_handler)
3249                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3250
3251         return;
3252 }
3253
3254
3255 /**
3256  * nes_manage_apbvt()
3257  */
3258 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3259                 u32 nic_index, u32 add_port)
3260 {
3261         struct nes_device *nesdev = nesvnic->nesdev;
3262         struct nes_hw_cqp_wqe *cqp_wqe;
3263         struct nes_cqp_request *cqp_request;
3264         int ret = 0;
3265         u16 major_code;
3266
3267         /* Send manage APBVT request to CQP */
3268         cqp_request = nes_get_cqp_request(nesdev);
3269         if (cqp_request == NULL) {
3270                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3271                 return -ENOMEM;
3272         }
3273         cqp_request->waiting = 1;
3274         cqp_wqe = &cqp_request->cqp_wqe;
3275
3276         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3277                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3278                         accel_local_port, accel_local_port, nic_index);
3279
3280         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3281         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3282                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3283         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3284                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3285
3286         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3287
3288         atomic_set(&cqp_request->refcount, 2);
3289         nes_post_cqp_request(nesdev, cqp_request);
3290
3291         if (add_port == NES_MANAGE_APBVT_ADD)
3292                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3293                                 NES_EVENT_TIMEOUT);
3294         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3295                         ret, cqp_request->major_code, cqp_request->minor_code);
3296         major_code = cqp_request->major_code;
3297
3298         nes_put_cqp_request(nesdev, cqp_request);
3299
3300         if (!ret)
3301                 return -ETIME;
3302         else if (major_code)
3303                 return -EIO;
3304         else
3305                 return 0;
3306 }
3307
3308
3309 /**
3310  * nes_manage_arp_cache
3311  */
3312 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3313                 u32 ip_addr, u32 action)
3314 {
3315         struct nes_hw_cqp_wqe *cqp_wqe;
3316         struct nes_vnic *nesvnic = netdev_priv(netdev);
3317         struct nes_device *nesdev;
3318         struct nes_cqp_request *cqp_request;
3319         int arp_index;
3320
3321         nesdev = nesvnic->nesdev;
3322         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3323         if (arp_index == -1) {
3324                 return;
3325         }
3326
3327         /* update the ARP entry */
3328         cqp_request = nes_get_cqp_request(nesdev);
3329         if (cqp_request == NULL) {
3330                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3331                 return;
3332         }
3333         cqp_request->waiting = 0;
3334         cqp_wqe = &cqp_request->cqp_wqe;
3335         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3336
3337         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3338                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3339         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3340                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3341         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3342
3343         if (action == NES_ARP_ADD) {
3344                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3345                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3346                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3347                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3348                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3349                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3350         } else {
3351                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3352                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3353         }
3354
3355         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3356                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3357
3358         atomic_set(&cqp_request->refcount, 1);
3359         nes_post_cqp_request(nesdev, cqp_request);
3360 }
3361
3362
3363 /**
3364  * flush_wqes
3365  */
3366 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3367                 u32 which_wq, u32 wait_completion)
3368 {
3369         struct nes_cqp_request *cqp_request;
3370         struct nes_hw_cqp_wqe *cqp_wqe;
3371         int ret;
3372
3373         cqp_request = nes_get_cqp_request(nesdev);
3374         if (cqp_request == NULL) {
3375                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3376                 return;
3377         }
3378         if (wait_completion) {
3379                 cqp_request->waiting = 1;
3380                 atomic_set(&cqp_request->refcount, 2);
3381         } else {
3382                 cqp_request->waiting = 0;
3383         }
3384         cqp_wqe = &cqp_request->cqp_wqe;
3385         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3386
3387         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3388                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3389         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3390
3391         nes_post_cqp_request(nesdev, cqp_request);
3392
3393         if (wait_completion) {
3394                 /* Wait for CQP */
3395                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3396                                 NES_EVENT_TIMEOUT);
3397                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3398                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3399                                 ret, cqp_request->major_code, cqp_request->minor_code);
3400                 nes_put_cqp_request(nesdev, cqp_request);
3401         }
3402 }