drivers/staging/benet/be_int.c

   1 /*
   2  * Copyright (C) 2005 - 2008 ServerEngines
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@serverengines.com
  12  *
  13  * ServerEngines
  14  * 209 N. Fair Oaks Ave
  15  * Sunnyvale, CA 94085
  16  */
  17 #include <linux/if_vlan.h>
  18 #include <linux/inet_lro.h>
  19
  20 #include "benet.h"
  21
  22 /* number of bytes of RX frame that are copied to skb->data */
  23 #define BE_HDR_LEN 64
  24
  25 #define NETIF_RX(skb) netif_receive_skb(skb)
  26 #define VLAN_ACCEL_RX(skb, pnob, vt) \
  27                 vlan_hwaccel_rx(skb, pnob->vlan_grp, vt)
  28
  29 /*
  30     This function notifies BladeEngine of the number of completion
  31     entries processed from the specified completion queue by writing
  32     the number of popped entries to the door bell.
  33
  34     pnob        - Pointer to the NetObject structure
  35     n           - Number of completion entries processed
  36     cq_id       - Queue ID of the completion queue for which notification
  37                         is being done.
  38     re_arm      - 1  - rearm the completion ring to generate an event.
  39                 - 0  - dont rearm the completion ring to generate an event
  40 */
  41 void be_notify_cmpl(struct be_net_object *pnob, int n, int cq_id, int re_arm)
  42 {
  43         struct CQ_DB_AMAP cqdb;
  44
  45         cqdb.dw[0] = 0;
  46         AMAP_SET_BITS_PTR(CQ_DB, qid, &cqdb, cq_id);
  47         AMAP_SET_BITS_PTR(CQ_DB, rearm, &cqdb, re_arm);
  48         AMAP_SET_BITS_PTR(CQ_DB, num_popped, &cqdb, n);
  49         PD_WRITE(&pnob->fn_obj, cq_db, cqdb.dw[0]);
  50 }
  51
  52 /*
  53  * adds additional receive frags indicated by BE starting from given
  54  * frag index (fi) to specified skb's frag list
  55  */
  56 static void
  57 add_skb_frags(struct be_net_object *pnob, struct sk_buff *skb,
  58               u32 nresid, u32 fi)
  59 {
  60         struct be_adapter *adapter = pnob->adapter;
  61         u32 sk_frag_idx, n;
  62         struct be_rx_page_info *rx_page_info;
  63         u32 frag_sz = pnob->rx_buf_size;
  64
  65         sk_frag_idx = skb_shinfo(skb)->nr_frags;
  66         while (nresid) {
  67                 index_inc(&fi, pnob->rx_q_len);
  68
  69                 rx_page_info = (struct be_rx_page_info *)pnob->rx_ctxt[fi];
  70                 pnob->rx_ctxt[fi] = NULL;
  71                 if ((rx_page_info->page_offset) ||
  72                     (pnob->rx_pg_shared == false)) {
  73                         pci_unmap_page(adapter->pdev,
  74                                        pci_unmap_addr(rx_page_info, bus),
  75                                        frag_sz, PCI_DMA_FROMDEVICE);
  76                 }
  77
  78                 n = min(nresid, frag_sz);
  79                 skb_shinfo(skb)->frags[sk_frag_idx].page = rx_page_info->page;
  80                 skb_shinfo(skb)->frags[sk_frag_idx].page_offset
  81                     = rx_page_info->page_offset;
  82                 skb_shinfo(skb)->frags[sk_frag_idx].size = n;
  83
  84                 sk_frag_idx++;
  85                 skb->len += n;
  86                 skb->data_len += n;
  87                 skb_shinfo(skb)->nr_frags++;
  88                 nresid -= n;
  89
  90                 memset(rx_page_info, 0, sizeof(struct be_rx_page_info));
  91                 atomic_dec(&pnob->rx_q_posted);
  92         }
  93 }
  94
  95 /*
  96  * This function processes incoming nic packets over various Rx queues.
  97  * This function takes the adapter, the current Rx status descriptor
  98  * entry and the Rx completion queue ID as argument.
  99  */
 100 static inline int process_nic_rx_completion(struct be_net_object *pnob,
 101                                             struct ETH_RX_COMPL_AMAP *rxcp)
 102 {
 103         struct be_adapter *adapter = pnob->adapter;
 104         struct sk_buff *skb;
 105         int udpcksm, tcpcksm;
 106         int n;
 107         u32 nresid, fi;
 108         u32 frag_sz = pnob->rx_buf_size;
 109         u8 *va;
 110         struct be_rx_page_info *rx_page_info;
 111         u32 numfrags, vtp, vtm, vlan_tag, pktsize;
 112
 113         fi = AMAP_GET_BITS_PTR(ETH_RX_COMPL, fragndx, rxcp);
 114         BUG_ON(fi >= (int)pnob->rx_q_len);
 115         BUG_ON(fi < 0);
 116
 117         rx_page_info = (struct be_rx_page_info *)pnob->rx_ctxt[fi];
 118         BUG_ON(!rx_page_info->page);
 119         pnob->rx_ctxt[fi] = NULL;
 120
 121         /*
 122          * If one page is used per fragment or if this is the second half of
 123          *  of the page, unmap the page here
 124          */
 125         if ((rx_page_info->page_offset) || (pnob->rx_pg_shared == false)) {
 126                 pci_unmap_page(adapter->pdev,
 127                                pci_unmap_addr(rx_page_info, bus), frag_sz,
 128                                PCI_DMA_FROMDEVICE);
 129         }
 130
 131         atomic_dec(&pnob->rx_q_posted);
 132         udpcksm = AMAP_GET_BITS_PTR(ETH_RX_COMPL, udpcksm, rxcp);
 133         tcpcksm = AMAP_GET_BITS_PTR(ETH_RX_COMPL, tcpcksm, rxcp);
 134         pktsize = AMAP_GET_BITS_PTR(ETH_RX_COMPL, pktsize, rxcp);
 135         /*
 136          * get rid of RX flush completions first.
 137          */
 138         if ((tcpcksm) && (udpcksm) && (pktsize == 32)) {
 139                 put_page(rx_page_info->page);
 140                 memset(rx_page_info, 0, sizeof(struct be_rx_page_info));
 141                 return 0;
 142         }
 143         skb = netdev_alloc_skb(pnob->netdev, BE_HDR_LEN + NET_IP_ALIGN);
 144         if (skb == NULL) {
 145                 dev_info(&pnob->netdev->dev, "alloc_skb() failed\n");
 146                 put_page(rx_page_info->page);
 147                 memset(rx_page_info, 0, sizeof(struct be_rx_page_info));
 148                 goto free_frags;
 149         }
 150         skb_reserve(skb, NET_IP_ALIGN);
 151
 152         skb->dev = pnob->netdev;
 153
 154         n = min(pktsize, frag_sz);
 155
 156         va = page_address(rx_page_info->page) + rx_page_info->page_offset;
 157         prefetch(va);
 158
 159         skb->len = skb->data_len = n;
 160         if (n <= BE_HDR_LEN) {
 161                 memcpy(skb->data, va, n);
 162                 put_page(rx_page_info->page);
 163                 skb->data_len -= n;
 164                 skb->tail += n;
 165         } else {
 166
 167                 /* Setup the SKB with page buffer information */
 168                 skb_shinfo(skb)->frags[0].page = rx_page_info->page;
 169                 skb_shinfo(skb)->nr_frags++;
 170
 171                 /* Copy the header into the skb_data */
 172                 memcpy(skb->data, va, BE_HDR_LEN);
 173                 skb_shinfo(skb)->frags[0].page_offset =
 174                     rx_page_info->page_offset + BE_HDR_LEN;
 175                 skb_shinfo(skb)->frags[0].size = n - BE_HDR_LEN;
 176                 skb->data_len -= BE_HDR_LEN;
 177                 skb->tail += BE_HDR_LEN;
 178         }
 179         memset(rx_page_info, 0, sizeof(struct be_rx_page_info));
 180         nresid = pktsize - n;
 181
 182         skb->protocol = eth_type_trans(skb, pnob->netdev);
 183
 184         if ((tcpcksm || udpcksm) && adapter->rx_csum)
 185                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 186         else
 187                 skb->ip_summed = CHECKSUM_NONE;
 188         /*
 189          * if we have more bytes left, the frame has been
 190          * given to us in multiple fragments.  This happens
 191          * with Jumbo frames. Add the remaining fragments to
 192          * skb->frags[] array.
 193          */
 194         if (nresid)
 195                 add_skb_frags(pnob, skb, nresid, fi);
 196
 197         /* update the the true size of the skb. */
 198         skb->truesize = skb->len + sizeof(struct sk_buff);
 199
 200         /*
 201          * If a 802.3 frame or 802.2 LLC frame
 202          * (i.e) contains length field in MAC Hdr
 203          * and frame len is greater than 64 bytes
 204          */
 205         if (((skb->protocol == ntohs(ETH_P_802_2)) ||
 206              (skb->protocol == ntohs(ETH_P_802_3)))
 207             && (pktsize > BE_HDR_LEN)) {
 208                 /*
 209                  * If the length given in Mac Hdr is less than frame size
 210                  * Erraneous frame, Drop it
 211                  */
 212                 if ((ntohs(*(u16 *) (va + 12)) + ETH_HLEN) < pktsize) {
 213                         /* Increment Non Ether type II frames dropped */
 214                         adapter->be_stat.bes_802_3_dropped_frames++;
 215
 216                         kfree_skb(skb);
 217                         return 0;
 218                 }
 219                 /*
 220                  * else if the length given in Mac Hdr is greater than
 221                  * frame size, should not be seeing this sort of frames
 222                  * dump the pkt and pass to stack
 223                  */
 224                 else if ((ntohs(*(u16 *) (va + 12)) + ETH_HLEN) > pktsize) {
 225                         /* Increment Non Ether type II frames malformed */
 226                         adapter->be_stat.bes_802_3_malformed_frames++;
 227                 }
 228         }
 229
 230         vtp = AMAP_GET_BITS_PTR(ETH_RX_COMPL, vtp, rxcp);
 231         vtm = AMAP_GET_BITS_PTR(ETH_RX_COMPL, vtm, rxcp);
 232         if (vtp && vtm) {
 233                 /* Vlan tag present in pkt and BE found
 234                  * that the tag matched an entry in VLAN table
 235                  */
 236                 if (!pnob->vlan_grp || pnob->num_vlans == 0) {
 237                         /* But we have no VLANs configured.
 238                          * This should never happen.  Drop the packet.
 239                          */
 240                         dev_info(&pnob->netdev->dev,
 241                                "BladeEngine: Unexpected vlan tagged packet\n");
 242                         kfree_skb(skb);
 243                         return 0;
 244                 }
 245                 /* pass the VLAN packet to stack */
 246                 vlan_tag = AMAP_GET_BITS_PTR(ETH_RX_COMPL, vlan_tag, rxcp);
 247                 VLAN_ACCEL_RX(skb, pnob, be16_to_cpu(vlan_tag));
 248
 249         } else {
 250                 NETIF_RX(skb);
 251         }
 252         return 0;
 253
 254 free_frags:
 255         /* free all frags associated with the current rxcp */
 256         numfrags = AMAP_GET_BITS_PTR(ETH_RX_COMPL, numfrags, rxcp);
 257         while (numfrags-- > 1) {
 258                 index_inc(&fi, pnob->rx_q_len);
 259
 260                 rx_page_info = (struct be_rx_page_info *)
 261                     pnob->rx_ctxt[fi];
 262                 pnob->rx_ctxt[fi] = (void *)NULL;
 263                 if (rx_page_info->page_offset || !pnob->rx_pg_shared) {
 264                         pci_unmap_page(adapter->pdev,
 265                                        pci_unmap_addr(rx_page_info, bus),
 266                                        frag_sz, PCI_DMA_FROMDEVICE);
 267                 }
 268
 269                 put_page(rx_page_info->page);
 270                 memset(rx_page_info, 0, sizeof(struct be_rx_page_info));
 271                 atomic_dec(&pnob->rx_q_posted);
 272         }
 273         return -ENOMEM;
 274 }
 275
 276 static void process_nic_rx_completion_lro(struct be_net_object *pnob,
 277                                           struct ETH_RX_COMPL_AMAP *rxcp)
 278 {
 279         struct be_adapter *adapter = pnob->adapter;
 280         struct skb_frag_struct rx_frags[BE_MAX_FRAGS_PER_FRAME];
 281         unsigned int udpcksm, tcpcksm;
 282         u32 numfrags, vlanf, vtm, vlan_tag, nresid;
 283         u16 vlant;
 284         unsigned int fi, idx, n;
 285         struct be_rx_page_info *rx_page_info;
 286         u32 frag_sz = pnob->rx_buf_size, pktsize;
 287         bool rx_coal = (adapter->max_rx_coal <= 1) ? 0 : 1;
 288         u8 err, *va;
 289         __wsum csum = 0;
 290
 291         if (AMAP_GET_BITS_PTR(ETH_RX_COMPL, ipsec, rxcp)) {
 292                 /*  Drop the pkt and move to the next completion.  */
 293                 adapter->be_stat.bes_rx_misc_pkts++;
 294                 return;
 295         }
 296         err = AMAP_GET_BITS_PTR(ETH_RX_COMPL, err, rxcp);
 297         if (err || !rx_coal) {
 298                 /* We won't coalesce Rx pkts if the err bit set.
 299                  * take the path of normal completion processing */
 300                 process_nic_rx_completion(pnob, rxcp);
 301                 return;
 302         }
 303
 304         fi = AMAP_GET_BITS_PTR(ETH_RX_COMPL, fragndx, rxcp);
 305         BUG_ON(fi >= (int)pnob->rx_q_len);
 306         BUG_ON(fi < 0);
 307         rx_page_info = (struct be_rx_page_info *)pnob->rx_ctxt[fi];
 308         BUG_ON(!rx_page_info->page);
 309         pnob->rx_ctxt[fi] = (void *)NULL;
 310         /*  If one page is used per fragment or if this is the
 311          * second half of the page, unmap the page here
 312          */
 313         if (rx_page_info->page_offset || !pnob->rx_pg_shared) {
 314                 pci_unmap_page(adapter->pdev,
 315                                pci_unmap_addr(rx_page_info, bus),
 316                                frag_sz, PCI_DMA_FROMDEVICE);
 317         }
 318
 319         numfrags = AMAP_GET_BITS_PTR(ETH_RX_COMPL, numfrags, rxcp);
 320         udpcksm = AMAP_GET_BITS_PTR(ETH_RX_COMPL, udpcksm, rxcp);
 321         tcpcksm = AMAP_GET_BITS_PTR(ETH_RX_COMPL, tcpcksm, rxcp);
 322         vlan_tag = AMAP_GET_BITS_PTR(ETH_RX_COMPL, vlan_tag, rxcp);
 323         vlant = be16_to_cpu(vlan_tag);
 324         vlanf = AMAP_GET_BITS_PTR(ETH_RX_COMPL, vtp, rxcp);
 325         vtm = AMAP_GET_BITS_PTR(ETH_RX_COMPL, vtm, rxcp);
 326         pktsize = AMAP_GET_BITS_PTR(ETH_RX_COMPL, pktsize, rxcp);
 327
 328         atomic_dec(&pnob->rx_q_posted);
 329
 330         if (tcpcksm && udpcksm && pktsize == 32) {
 331                 /* flush completion entries */
 332                 put_page(rx_page_info->page);
 333                 memset(rx_page_info, 0, sizeof(struct be_rx_page_info));
 334                 return;
 335         }
 336         /* Only one of udpcksum and tcpcksum can be set */
 337         BUG_ON(udpcksm && tcpcksm);
 338
 339         /* jumbo frames could come in multiple fragments */
 340         BUG_ON(numfrags != ((pktsize + (frag_sz - 1)) / frag_sz));
 341         n = min(pktsize, frag_sz);
 342         nresid = pktsize - n;   /* will be useful for jumbo pkts */
 343         idx = 0;
 344
 345         va = page_address(rx_page_info->page) + rx_page_info->page_offset;
 346         prefetch(va);
 347         rx_frags[idx].page = rx_page_info->page;
 348         rx_frags[idx].page_offset = (rx_page_info->page_offset);
 349         rx_frags[idx].size = n;
 350         memset(rx_page_info, 0, sizeof(struct be_rx_page_info));
 351
 352         /* If we got multiple fragments, we have more data. */
 353         while (nresid) {
 354                 idx++;
 355                 index_inc(&fi, pnob->rx_q_len);
 356
 357                 rx_page_info = (struct be_rx_page_info *)pnob->rx_ctxt[fi];
 358                 pnob->rx_ctxt[fi] = (void *)NULL;
 359                 if (rx_page_info->page_offset || !pnob->rx_pg_shared) {
 360                         pci_unmap_page(adapter->pdev,
 361                                        pci_unmap_addr(rx_page_info, bus),
 362                                        frag_sz, PCI_DMA_FROMDEVICE);
 363                 }
 364
 365                 n = min(nresid, frag_sz);
 366                 rx_frags[idx].page = rx_page_info->page;
 367                 rx_frags[idx].page_offset = (rx_page_info->page_offset);
 368                 rx_frags[idx].size = n;
 369
 370                 nresid -= n;
 371                 memset(rx_page_info, 0, sizeof(struct be_rx_page_info));
 372                 atomic_dec(&pnob->rx_q_posted);
 373         }
 374
 375         if (likely(!(vlanf && vtm))) {
 376                 lro_receive_frags(&pnob->lro_mgr, rx_frags,
 377                                   pktsize, pktsize,
 378                                   (void *)(unsigned long)csum, csum);
 379         } else {
 380                 /* Vlan tag present in pkt and BE found
 381                  * that the tag matched an entry in VLAN table
 382                  */
 383                 if (unlikely(!pnob->vlan_grp || pnob->num_vlans == 0)) {
 384                         /* But we have no VLANs configured.
 385                          * This should never happen.  Drop the packet.
 386                          */
 387                         dev_info(&pnob->netdev->dev,
 388                                "BladeEngine: Unexpected vlan tagged packet\n");
 389                         return;
 390                 }
 391                 /* pass the VLAN packet to stack */
 392                 lro_vlan_hwaccel_receive_frags(&pnob->lro_mgr,
 393                                                rx_frags, pktsize, pktsize,
 394                                                pnob->vlan_grp, vlant,
 395                                                (void *)(unsigned long)csum,
 396                                                csum);
 397         }
 398
 399         adapter->be_stat.bes_rx_coal++;
 400 }
 401
 402 struct ETH_RX_COMPL_AMAP *be_get_rx_cmpl(struct be_net_object *pnob)
 403 {
 404         struct ETH_RX_COMPL_AMAP *rxcp = &pnob->rx_cq[pnob->rx_cq_tl];
 405         u32 valid, ct;
 406
 407         valid = AMAP_GET_BITS_PTR(ETH_RX_COMPL, valid, rxcp);
 408         if (valid == 0)
 409                 return NULL;
 410
 411         ct = AMAP_GET_BITS_PTR(ETH_RX_COMPL, ct, rxcp);
 412         if (ct != 0) {
 413                 /* Invalid chute #. treat as error */
 414                 AMAP_SET_BITS_PTR(ETH_RX_COMPL, err, rxcp, 1);
 415         }
 416
 417         be_adv_rxcq_tl(pnob);
 418         AMAP_SET_BITS_PTR(ETH_RX_COMPL, valid, rxcp, 0);
 419         return rxcp;
 420 }
 421
 422 static void update_rx_rate(struct be_adapter *adapter)
 423 {
 424         /* update the rate once in two seconds */
 425         if ((jiffies - adapter->eth_rx_jiffies) > 2 * (HZ)) {
 426                 u32 r;
 427                 r = adapter->eth_rx_bytes /
 428                     ((jiffies - adapter->eth_rx_jiffies) / (HZ));
 429                 r = (r / 1000000);      /* MB/Sec */
 430
 431                 /* Mega Bits/Sec */
 432                 adapter->be_stat.bes_eth_rx_rate = (r * 8);
 433                 adapter->eth_rx_jiffies = jiffies;
 434                 adapter->eth_rx_bytes = 0;
 435         }
 436 }
 437
 438 static int process_rx_completions(struct be_net_object *pnob, int max_work)
 439 {
 440         struct be_adapter *adapter = pnob->adapter;
 441         struct ETH_RX_COMPL_AMAP *rxcp;
 442         u32 nc = 0;
 443         unsigned int pktsize;
 444
 445         while (max_work && (rxcp = be_get_rx_cmpl(pnob))) {
 446                 prefetch(rxcp);
 447                 pktsize = AMAP_GET_BITS_PTR(ETH_RX_COMPL, pktsize, rxcp);
 448                 process_nic_rx_completion_lro(pnob, rxcp);
 449                 adapter->eth_rx_bytes += pktsize;
 450                 update_rx_rate(adapter);
 451                 nc++;
 452                 max_work--;
 453                 adapter->be_stat.bes_rx_compl++;
 454         }
 455         if (likely(adapter->max_rx_coal > 1)) {
 456                 adapter->be_stat.bes_rx_flush++;
 457                 lro_flush_all(&pnob->lro_mgr);
 458         }
 459
 460         /* Refill the queue */
 461         if (atomic_read(&pnob->rx_q_posted) < 900)
 462                 be_post_eth_rx_buffs(pnob);
 463
 464         return nc;
 465 }
 466
 467 static struct ETH_TX_COMPL_AMAP *be_get_tx_cmpl(struct be_net_object *pnob)
 468 {
 469         struct ETH_TX_COMPL_AMAP *txcp = &pnob->tx_cq[pnob->tx_cq_tl];
 470         u32 valid;
 471
 472         valid = AMAP_GET_BITS_PTR(ETH_TX_COMPL, valid, txcp);
 473         if (valid == 0)
 474                 return NULL;
 475
 476         AMAP_SET_BITS_PTR(ETH_TX_COMPL, valid, txcp, 0);
 477         be_adv_txcq_tl(pnob);
 478         return txcp;
 479
 480 }
 481
 482 void process_one_tx_compl(struct be_net_object *pnob, u32 end_idx)
 483 {
 484         struct be_adapter *adapter = pnob->adapter;
 485         int cur_index, tx_wrbs_completed = 0;
 486         struct sk_buff *skb;
 487         u64 busaddr, pa, pa_lo, pa_hi;
 488         struct ETH_WRB_AMAP *wrb;
 489         u32 frag_len, last_index, j;
 490
 491         last_index = tx_compl_lastwrb_idx_get(pnob);
 492         BUG_ON(last_index != end_idx);
 493         pnob->tx_ctxt[pnob->tx_q_tl] = NULL;
 494         do {
 495                 cur_index = pnob->tx_q_tl;
 496                 wrb = &pnob->tx_q[cur_index];
 497                 pa_hi = AMAP_GET_BITS_PTR(ETH_WRB, frag_pa_hi, wrb);
 498                 pa_lo = AMAP_GET_BITS_PTR(ETH_WRB, frag_pa_lo, wrb);
 499                 frag_len = AMAP_GET_BITS_PTR(ETH_WRB, frag_len, wrb);
 500                 busaddr = (pa_hi << 32) | pa_lo;
 501                 if (busaddr != 0) {
 502                         pa = le64_to_cpu(busaddr);
 503                         pci_unmap_single(adapter->pdev, pa,
 504                                          frag_len, PCI_DMA_TODEVICE);
 505                 }
 506                 if (cur_index == last_index) {
 507                         skb = (struct sk_buff *)pnob->tx_ctxt[cur_index];
 508                         BUG_ON(!skb);
 509                         for (j = 0; j < skb_shinfo(skb)->nr_frags; j++) {
 510                                 struct skb_frag_struct *frag;
 511                                 frag = &skb_shinfo(skb)->frags[j];
 512                                 pci_unmap_page(adapter->pdev,
 513                                                (ulong) frag->page, frag->size,
 514                                                PCI_DMA_TODEVICE);
 515                         }
 516                         kfree_skb(skb);
 517                         pnob->tx_ctxt[cur_index] = NULL;
 518                 } else {
 519                         BUG_ON(pnob->tx_ctxt[cur_index]);
 520                 }
 521                 tx_wrbs_completed++;
 522                 be_adv_txq_tl(pnob);
 523         } while (cur_index != last_index);
 524         atomic_sub(tx_wrbs_completed, &pnob->tx_q_used);
 525 }
 526
 527 /* there is no need to take an SMP lock here since currently
 528  * we have only one instance of the tasklet that does completion
 529  * processing.
 530  */
 531 static void process_nic_tx_completions(struct be_net_object *pnob)
 532 {
 533         struct be_adapter *adapter = pnob->adapter;
 534         struct ETH_TX_COMPL_AMAP *txcp;
 535         struct net_device *netdev = pnob->netdev;
 536         u32 end_idx, num_processed = 0;
 537
 538         adapter->be_stat.bes_tx_events++;
 539
 540         while ((txcp = be_get_tx_cmpl(pnob))) {
 541                 end_idx = AMAP_GET_BITS_PTR(ETH_TX_COMPL, wrb_index, txcp);
 542                 process_one_tx_compl(pnob, end_idx);
 543                 num_processed++;
 544                 adapter->be_stat.bes_tx_compl++;
 545         }
 546         be_notify_cmpl(pnob, num_processed, pnob->tx_cq_id, 1);
 547         /*
 548          * We got Tx completions and have usable WRBs.
 549          * If the netdev's queue has been stopped
 550          * because we had run out of WRBs, wake it now.
 551          */
 552         spin_lock(&adapter->txq_lock);
 553         if (netif_queue_stopped(netdev)
 554             && atomic_read(&pnob->tx_q_used) < pnob->tx_q_len / 2) {
 555                 netif_wake_queue(netdev);
 556         }
 557         spin_unlock(&adapter->txq_lock);
 558 }
 559
 560 static u32 post_rx_buffs(struct be_net_object *pnob, struct list_head *rxbl)
 561 {
 562         u32 nposted = 0;
 563         struct ETH_RX_D_AMAP *rxd = NULL;
 564         struct be_recv_buffer *rxbp;
 565         void **rx_ctxp;
 566         struct RQ_DB_AMAP rqdb;
 567
 568         rx_ctxp = pnob->rx_ctxt;
 569
 570         while (!list_empty(rxbl) &&
 571                (rx_ctxp[pnob->rx_q_hd] == NULL) && nposted < 255) {
 572
 573                 rxbp = list_first_entry(rxbl, struct be_recv_buffer, rxb_list);
 574                 list_del(&rxbp->rxb_list);
 575                 rxd = pnob->rx_q + pnob->rx_q_hd;
 576                 AMAP_SET_BITS_PTR(ETH_RX_D, fragpa_lo, rxd, rxbp->rxb_pa_lo);
 577                 AMAP_SET_BITS_PTR(ETH_RX_D, fragpa_hi, rxd, rxbp->rxb_pa_hi);
 578
 579                 rx_ctxp[pnob->rx_q_hd] = rxbp->rxb_ctxt;
 580                 be_adv_rxq_hd(pnob);
 581                 nposted++;
 582         }
 583
 584         if (nposted) {
 585                 /* Now press the door bell to notify BladeEngine. */
 586                 rqdb.dw[0] = 0;
 587                 AMAP_SET_BITS_PTR(RQ_DB, numPosted, &rqdb, nposted);
 588                 AMAP_SET_BITS_PTR(RQ_DB, rq, &rqdb, pnob->rx_q_id);
 589                 PD_WRITE(&pnob->fn_obj, erx_rq_db, rqdb.dw[0]);
 590         }
 591         atomic_add(nposted, &pnob->rx_q_posted);
 592         return nposted;
 593 }
 594
 595 void be_post_eth_rx_buffs(struct be_net_object *pnob)
 596 {
 597         struct be_adapter *adapter = pnob->adapter;
 598         u32 num_bufs, r;
 599         u64 busaddr = 0, tmp_pa;
 600         u32 max_bufs, pg_hd;
 601         u32 frag_size;
 602         struct be_recv_buffer *rxbp;
 603         struct list_head rxbl;
 604         struct be_rx_page_info *rx_page_info;
 605         struct page *page = NULL;
 606         u32 page_order = 0;
 607         gfp_t alloc_flags = GFP_ATOMIC;
 608
 609         BUG_ON(!adapter);
 610
 611         max_bufs = 64;          /* should be even # <= 255. */
 612
 613         frag_size = pnob->rx_buf_size;
 614         page_order = get_order(frag_size);
 615
 616         if (frag_size == 8192)
 617                 alloc_flags |= (gfp_t) __GFP_COMP;
 618         /*
 619          * Form a linked list of RECV_BUFFFER structure to be be posted.
 620          * We will post even number of buffer so that pages can be
 621          * shared.
 622          */
 623         INIT_LIST_HEAD(&rxbl);
 624
 625         for (num_bufs = 0; num_bufs < max_bufs; ++num_bufs) {
 626
 627                 rxbp = &pnob->eth_rx_bufs[num_bufs];
 628                 pg_hd = pnob->rx_pg_info_hd;
 629                 rx_page_info = &pnob->rx_page_info[pg_hd];
 630
 631                 if (!page) {
 632                         /*
 633                          * before we allocate a page make sure that we
 634                          * have space in the RX queue to post the buffer.
 635                          * We check for two vacant slots since with
 636                          * 2K frags, we will need two slots.
 637                          */
 638                         if ((pnob->rx_ctxt[(pnob->rx_q_hd + num_bufs) &
 639                                            (pnob->rx_q_len - 1)] != NULL)
 640                             || (pnob->rx_ctxt[(pnob->rx_q_hd + num_bufs + 1) %
 641                                               pnob->rx_q_len] != NULL)) {
 642                                 break;
 643                         }
 644                         page = alloc_pages(alloc_flags, page_order);
 645                         if (unlikely(page == NULL)) {
 646                                 adapter->be_stat.bes_ethrx_post_fail++;
 647                                 pnob->rxbuf_post_fail++;
 648                                 break;
 649                         }
 650                         pnob->rxbuf_post_fail = 0;
 651                         busaddr = pci_map_page(adapter->pdev, page, 0,
 652                                                frag_size, PCI_DMA_FROMDEVICE);
 653                         rx_page_info->page_offset = 0;
 654                         rx_page_info->page = page;
 655                         /*
 656                          * If we are sharing a page among two skbs,
 657                          * alloc a new one on the next iteration
 658                          */
 659                         if (pnob->rx_pg_shared == false)
 660                                 page = NULL;
 661                 } else {
 662                         get_page(page);
 663                         rx_page_info->page_offset += frag_size;
 664                         rx_page_info->page = page;
 665                         /*
 666                          * We are finished with the alloced page,
 667                          * Alloc a new one on the next iteration
 668                          */
 669                         page = NULL;
 670                 }
 671                 rxbp->rxb_ctxt = (void *)rx_page_info;
 672                 index_inc(&pnob->rx_pg_info_hd, pnob->rx_q_len);
 673
 674                 pci_unmap_addr_set(rx_page_info, bus, busaddr);
 675                 tmp_pa = busaddr + rx_page_info->page_offset;
 676                 rxbp->rxb_pa_lo = (tmp_pa & 0xFFFFFFFF);
 677                 rxbp->rxb_pa_hi = (tmp_pa >> 32);
 678                 rxbp->rxb_len = frag_size;
 679                 list_add_tail(&rxbp->rxb_list, &rxbl);
 680         }                       /* End of for */
 681
 682         r = post_rx_buffs(pnob, &rxbl);
 683         BUG_ON(r != num_bufs);
 684         return;
 685 }
 686
 687 /*
 688  * Interrupt service for network function.  We just schedule the
 689  * tasklet which does all completion processing.
 690  */
 691 irqreturn_t be_int(int irq, void *dev)
 692 {
 693         struct net_device *netdev = dev;
 694         struct be_net_object *pnob = netdev_priv(netdev);
 695         struct be_adapter *adapter = pnob->adapter;
 696         u32 isr;
 697
 698         isr = CSR_READ(&pnob->fn_obj, cev.isr1);
 699         if (unlikely(!isr))
 700                 return IRQ_NONE;
 701
 702         spin_lock(&adapter->int_lock);
 703         adapter->isr |= isr;
 704         spin_unlock(&adapter->int_lock);
 705
 706         adapter->be_stat.bes_ints++;
 707
 708         tasklet_schedule(&adapter->sts_handler);
 709         return IRQ_HANDLED;
 710 }
 711
 712 /*
 713  * Poll function called by NAPI with a work budget.
 714  * We process as many UC. BC and MC receive completions
 715  * as the budget allows and return the actual number of
 716  * RX ststutses processed.
 717  */
 718 int be_poll(struct napi_struct *napi, int budget)
 719 {
 720         struct net_device *netdev = napi->dev;
 721         struct be_net_object *pnob = netdev_priv(netdev);
 722         struct be_adapter *adapter = pnob->adapter;
 723         u32 work_done;
 724
 725         adapter->be_stat.bes_polls++;
 726         work_done = process_rx_completions(pnob, budget);
 727         BUG_ON(work_done > budget);
 728
 729         /* All consumed */
 730         if (work_done < budget) {
 731                 netif_rx_complete(napi);
 732                 /* enable intr */
 733                 be_notify_cmpl(pnob, work_done, pnob->rx_cq_id, 1);
 734         } else {
 735                 /* More to be consumed; continue with interrupts disabled */
 736                 be_notify_cmpl(pnob, work_done, pnob->rx_cq_id, 0);
 737         }
 738         return work_done;
 739 }
 740
 741 static struct EQ_ENTRY_AMAP *get_event(struct be_net_object *pnob)
 742 {
 743         struct EQ_ENTRY_AMAP *eqp = &(pnob->event_q[pnob->event_q_tl]);
 744         if (!AMAP_GET_BITS_PTR(EQ_ENTRY, Valid, eqp))
 745                 return NULL;
 746         be_adv_eq_tl(pnob);
 747         return eqp;
 748 }
 749
 750 /*
 751  * Processes all valid events in the event ring associated with given
 752  * NetObject.  Also, notifies BE the number of events processed.
 753  */
 754 static inline u32 process_events(struct be_net_object *pnob)
 755 {
 756         struct be_adapter *adapter = pnob->adapter;
 757         struct EQ_ENTRY_AMAP *eqp;
 758         u32 rid, num_events = 0;
 759         struct net_device *netdev = pnob->netdev;
 760
 761         while ((eqp = get_event(pnob)) != NULL) {
 762                 adapter->be_stat.bes_events++;
 763                 rid = AMAP_GET_BITS_PTR(EQ_ENTRY, ResourceID, eqp);
 764                 if (rid == pnob->rx_cq_id) {
 765                         adapter->be_stat.bes_rx_events++;
 766                         netif_rx_schedule(&pnob->napi);
 767                 } else if (rid == pnob->tx_cq_id) {
 768                         process_nic_tx_completions(pnob);
 769                 } else if (rid == pnob->mcc_cq_id) {
 770                         be_mcc_process_cq(&pnob->mcc_q_obj, 1);
 771                 } else {
 772                         dev_info(&netdev->dev,
 773                                         "Invalid EQ ResourceID %d\n", rid);
 774                 }
 775                 AMAP_SET_BITS_PTR(EQ_ENTRY, Valid, eqp, 0);
 776                 AMAP_SET_BITS_PTR(EQ_ENTRY, ResourceID, eqp, 0);
 777                 num_events++;
 778         }
 779         return num_events;
 780 }
 781
 782 static void update_eqd(struct be_adapter *adapter, struct be_net_object *pnob)
 783 {
 784         int status;
 785         struct be_eq_object *eq_objectp;
 786
 787         /* update once a second */
 788         if ((jiffies - adapter->ips_jiffies) > 1 * (HZ)) {
 789                 /* One second elapsed since last update  */
 790                 u32 r, new_eqd = -1;
 791                 r = adapter->be_stat.bes_ints - adapter->be_stat.bes_prev_ints;
 792                 r = r / ((jiffies - adapter->ips_jiffies) / (HZ));
 793                 adapter->be_stat.bes_ips = r;
 794                 adapter->ips_jiffies = jiffies;
 795                 adapter->be_stat.bes_prev_ints = adapter->be_stat.bes_ints;
 796                 if (r > IPS_HI_WM && adapter->cur_eqd < adapter->max_eqd)
 797                         new_eqd = (adapter->cur_eqd + 8);
 798                 if (r < IPS_LO_WM && adapter->cur_eqd > adapter->min_eqd)
 799                         new_eqd = (adapter->cur_eqd - 8);
 800                 if (adapter->enable_aic && new_eqd != -1) {
 801                         eq_objectp = &pnob->event_q_obj;
 802                         status = be_eq_modify_delay(&pnob->fn_obj, 1,
 803                                                     &eq_objectp, &new_eqd, NULL,
 804                                                     NULL, NULL);
 805                         if (status == BE_SUCCESS)
 806                                 adapter->cur_eqd = new_eqd;
 807                 }
 808         }
 809 }
 810
 811 /*
 812     This function notifies BladeEngine of how many events were processed
 813     from the event queue by ringing the corresponding door bell and
 814     optionally re-arms the event queue.
 815     n           - number of events processed
 816     re_arm      - 1 - re-arm the EQ, 0 - do not re-arm the EQ
 817
 818 */
 819 static void be_notify_event(struct be_net_object *pnob, int n, int re_arm)
 820 {
 821         struct CQ_DB_AMAP eqdb;
 822         eqdb.dw[0] = 0;
 823
 824         AMAP_SET_BITS_PTR(CQ_DB, qid, &eqdb, pnob->event_q_id);
 825         AMAP_SET_BITS_PTR(CQ_DB, rearm, &eqdb, re_arm);
 826         AMAP_SET_BITS_PTR(CQ_DB, event, &eqdb, 1);
 827         AMAP_SET_BITS_PTR(CQ_DB, num_popped, &eqdb, n);
 828         /*
 829          * Under some situations we see an interrupt and no valid
 830          * EQ entry.  To keep going, we need to ring the DB even if
 831          * numPOsted is 0.
 832          */
 833         PD_WRITE(&pnob->fn_obj, cq_db, eqdb.dw[0]);
 834         return;
 835 }
 836
 837 /*
 838  * Called from the tasklet scheduled by ISR.  All real interrupt processing
 839  * is done here.
 840  */
 841 void be_process_intr(unsigned long context)
 842 {
 843         struct be_adapter *adapter = (struct be_adapter *)context;
 844         struct be_net_object *pnob = adapter->net_obj;
 845         u32 isr, n;
 846         ulong flags = 0;
 847
 848         isr = adapter->isr;
 849
 850         /*
 851          * we create only one NIC event queue in Linux. Event is
 852          * expected only in the first event queue
 853          */
 854         BUG_ON(isr & 0xfffffffe);
 855         if ((isr & 1) == 0)
 856                 return;         /* not our interrupt */
 857         n = process_events(pnob);
 858         /*
 859          * Clear the event bit. adapter->isr is  set by
 860          * hard interrupt.  Prevent race with lock.
 861          */
 862         spin_lock_irqsave(&adapter->int_lock, flags);
 863         adapter->isr &= ~1;
 864         spin_unlock_irqrestore(&adapter->int_lock, flags);
 865         be_notify_event(pnob, n, 1);
 866         /*
 867          * If previous allocation attempts had failed and
 868          * BE has used up all posted buffers, post RX buffers here
 869          */
 870         if (pnob->rxbuf_post_fail && atomic_read(&pnob->rx_q_posted) == 0)
 871                 be_post_eth_rx_buffs(pnob);
 872         update_eqd(adapter, pnob);
 873         return;
 874 }