drivers/infiniband/hw/ehca/ehca_reqs.c

   1 /*
   2  *  IBM eServer eHCA Infiniband device driver for Linux on POWER
   3  *
   4  *  post_send/recv, poll_cq, req_notify
   5  *
   6  *  Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
   7  *           Waleri Fomin <fomin@de.ibm.com>
   8  *           Joachim Fenkes <fenkes@de.ibm.com>
   9  *           Reinhard Ernst <rernst@de.ibm.com>
  10  *
  11  *  Copyright (c) 2005 IBM Corporation
  12  *
  13  *  All rights reserved.
  14  *
  15  *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
  16  *  BSD.
  17  *
  18  * OpenIB BSD License
  19  *
  20  * Redistribution and use in source and binary forms, with or without
  21  * modification, are permitted provided that the following conditions are met:
  22  *
  23  * Redistributions of source code must retain the above copyright notice, this
  24  * list of conditions and the following disclaimer.
  25  *
  26  * Redistributions in binary form must reproduce the above copyright notice,
  27  * this list of conditions and the following disclaimer in the documentation
  28  * and/or other materials
  29  * provided with the distribution.
  30  *
  31  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  32  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  33  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  34  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  35  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  36  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  37  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  38  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  39  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  40  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  41  * POSSIBILITY OF SUCH DAMAGE.
  42  */
  43
  44
  45 #include <asm-powerpc/system.h>
  46 #include "ehca_classes.h"
  47 #include "ehca_tools.h"
  48 #include "ehca_qes.h"
  49 #include "ehca_iverbs.h"
  50 #include "hcp_if.h"
  51 #include "hipz_fns.h"
  52
  53 static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
  54                                   struct ehca_wqe *wqe_p,
  55                                   struct ib_recv_wr *recv_wr)
  56 {
  57         u8 cnt_ds;
  58         if (unlikely((recv_wr->num_sge < 0) ||
  59                      (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
  60                 ehca_gen_err("Invalid number of WQE SGE. "
  61                          "num_sqe=%x max_nr_of_sg=%x",
  62                          recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
  63                 return -EINVAL; /* invalid SG list length */
  64         }
  65
  66         /* clear wqe header until sglist */
  67         memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
  68
  69         wqe_p->work_request_id = recv_wr->wr_id;
  70         wqe_p->nr_of_data_seg = recv_wr->num_sge;
  71
  72         for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
  73                 wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
  74                         recv_wr->sg_list[cnt_ds].addr;
  75                 wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
  76                         recv_wr->sg_list[cnt_ds].lkey;
  77                 wqe_p->u.all_rcv.sg_list[cnt_ds].length =
  78                         recv_wr->sg_list[cnt_ds].length;
  79         }
  80
  81         if (ehca_debug_level) {
  82                 ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p", ipz_rqueue);
  83                 ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
  84         }
  85
  86         return 0;
  87 }
  88
  89 #if defined(DEBUG_GSI_SEND_WR)
  90
  91 /* need ib_mad struct */
  92 #include <rdma/ib_mad.h>
  93
  94 static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
  95 {
  96         int idx;
  97         int j;
  98         while (send_wr) {
  99                 struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
 100                 struct ib_sge *sge = send_wr->sg_list;
 101                 ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
 102                              "send_flags=%x opcode=%x",idx, send_wr->wr_id,
 103                              send_wr->num_sge, send_wr->send_flags,
 104                              send_wr->opcode);
 105                 if (mad_hdr) {
 106                         ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
 107                                      "mgmt_class=%x class_version=%x method=%x "
 108                                      "status=%x class_specific=%x tid=%lx "
 109                                      "attr_id=%x resv=%x attr_mod=%x",
 110                                      idx, mad_hdr->base_version,
 111                                      mad_hdr->mgmt_class,
 112                                      mad_hdr->class_version, mad_hdr->method,
 113                                      mad_hdr->status, mad_hdr->class_specific,
 114                                      mad_hdr->tid, mad_hdr->attr_id,
 115                                      mad_hdr->resv,
 116                                      mad_hdr->attr_mod);
 117                 }
 118                 for (j = 0; j < send_wr->num_sge; j++) {
 119                         u8 *data = (u8 *) abs_to_virt(sge->addr);
 120                         ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
 121                                      "lkey=%x",
 122                                      idx, j, data, sge->length, sge->lkey);
 123                         /* assume length is n*16 */
 124                         ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
 125                                  idx, j);
 126                         sge++;
 127                 } /* eof for j */
 128                 idx++;
 129                 send_wr = send_wr->next;
 130         } /* eof while send_wr */
 131 }
 132
 133 #endif /* DEBUG_GSI_SEND_WR */
 134
 135 static inline int ehca_write_swqe(struct ehca_qp *qp,
 136                                   struct ehca_wqe *wqe_p,
 137                                   const struct ib_send_wr *send_wr)
 138 {
 139         u32 idx;
 140         u64 dma_length;
 141         struct ehca_av *my_av;
 142         u32 remote_qkey = send_wr->wr.ud.remote_qkey;
 143
 144         if (unlikely((send_wr->num_sge < 0) ||
 145                      (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
 146                 ehca_gen_err("Invalid number of WQE SGE. "
 147                          "num_sqe=%x max_nr_of_sg=%x",
 148                          send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
 149                 return -EINVAL; /* invalid SG list length */
 150         }
 151
 152         /* clear wqe header until sglist */
 153         memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
 154
 155         wqe_p->work_request_id = send_wr->wr_id;
 156
 157         switch (send_wr->opcode) {
 158         case IB_WR_SEND:
 159         case IB_WR_SEND_WITH_IMM:
 160                 wqe_p->optype = WQE_OPTYPE_SEND;
 161                 break;
 162         case IB_WR_RDMA_WRITE:
 163         case IB_WR_RDMA_WRITE_WITH_IMM:
 164                 wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
 165                 break;
 166         case IB_WR_RDMA_READ:
 167                 wqe_p->optype = WQE_OPTYPE_RDMAREAD;
 168                 break;
 169         default:
 170                 ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
 171                 return -EINVAL; /* invalid opcode */
 172         }
 173
 174         wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
 175
 176         wqe_p->wr_flag = 0;
 177
 178         if (send_wr->send_flags & IB_SEND_SIGNALED)
 179                 wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
 180
 181         if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
 182             send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
 183                 /* this might not work as long as HW does not support it */
 184                 wqe_p->immediate_data = be32_to_cpu(send_wr->imm_data);
 185                 wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
 186         }
 187
 188         wqe_p->nr_of_data_seg = send_wr->num_sge;
 189
 190         switch (qp->qp_type) {
 191         case IB_QPT_SMI:
 192         case IB_QPT_GSI:
 193                 /* no break is intential here */
 194         case IB_QPT_UD:
 195                 /* IB 1.2 spec C10-15 compliance */
 196                 if (send_wr->wr.ud.remote_qkey & 0x80000000)
 197                         remote_qkey = qp->qkey;
 198
 199                 wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
 200                 wqe_p->local_ee_context_qkey = remote_qkey;
 201                 if (!send_wr->wr.ud.ah) {
 202                         ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
 203                         return -EINVAL;
 204                 }
 205                 my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
 206                 wqe_p->u.ud_av.ud_av = my_av->av;
 207
 208                 /*
 209                  * omitted check of IB_SEND_INLINE
 210                  * since HW does not support it
 211                  */
 212                 for (idx = 0; idx < send_wr->num_sge; idx++) {
 213                         wqe_p->u.ud_av.sg_list[idx].vaddr =
 214                                 send_wr->sg_list[idx].addr;
 215                         wqe_p->u.ud_av.sg_list[idx].lkey =
 216                                 send_wr->sg_list[idx].lkey;
 217                         wqe_p->u.ud_av.sg_list[idx].length =
 218                                 send_wr->sg_list[idx].length;
 219                 } /* eof for idx */
 220                 if (qp->qp_type == IB_QPT_SMI ||
 221                     qp->qp_type == IB_QPT_GSI)
 222                         wqe_p->u.ud_av.ud_av.pmtu = 1;
 223                 if (qp->qp_type == IB_QPT_GSI) {
 224                         wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
 225 #ifdef DEBUG_GSI_SEND_WR
 226                         trace_send_wr_ud(send_wr);
 227 #endif /* DEBUG_GSI_SEND_WR */
 228                 }
 229                 break;
 230
 231         case IB_QPT_UC:
 232                 if (send_wr->send_flags & IB_SEND_FENCE)
 233                         wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
 234                 /* no break is intentional here */
 235         case IB_QPT_RC:
 236                 /* TODO: atomic not implemented */
 237                 wqe_p->u.nud.remote_virtual_adress =
 238                         send_wr->wr.rdma.remote_addr;
 239                 wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
 240
 241                 /*
 242                  * omitted checking of IB_SEND_INLINE
 243                  * since HW does not support it
 244                  */
 245                 dma_length = 0;
 246                 for (idx = 0; idx < send_wr->num_sge; idx++) {
 247                         wqe_p->u.nud.sg_list[idx].vaddr =
 248                                 send_wr->sg_list[idx].addr;
 249                         wqe_p->u.nud.sg_list[idx].lkey =
 250                                 send_wr->sg_list[idx].lkey;
 251                         wqe_p->u.nud.sg_list[idx].length =
 252                                 send_wr->sg_list[idx].length;
 253                         dma_length += send_wr->sg_list[idx].length;
 254                 } /* eof idx */
 255                 wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
 256
 257                 break;
 258
 259         default:
 260                 ehca_gen_err("Invalid qptype=%x", qp->qp_type);
 261                 return -EINVAL;
 262         }
 263
 264         if (ehca_debug_level) {
 265                 ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
 266                 ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
 267         }
 268         return 0;
 269 }
 270
 271 /* map_ib_wc_status converts raw cqe_status to ib_wc_status */
 272 static inline void map_ib_wc_status(u32 cqe_status,
 273                                     enum ib_wc_status *wc_status)
 274 {
 275         if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) {
 276                 switch (cqe_status & 0x3F) {
 277                 case 0x01:
 278                 case 0x21:
 279                         *wc_status = IB_WC_LOC_LEN_ERR;
 280                         break;
 281                 case 0x02:
 282                 case 0x22:
 283                         *wc_status = IB_WC_LOC_QP_OP_ERR;
 284                         break;
 285                 case 0x03:
 286                 case 0x23:
 287                         *wc_status = IB_WC_LOC_EEC_OP_ERR;
 288                         break;
 289                 case 0x04:
 290                 case 0x24:
 291                         *wc_status = IB_WC_LOC_PROT_ERR;
 292                         break;
 293                 case 0x05:
 294                 case 0x25:
 295                         *wc_status = IB_WC_WR_FLUSH_ERR;
 296                         break;
 297                 case 0x06:
 298                         *wc_status = IB_WC_MW_BIND_ERR;
 299                         break;
 300                 case 0x07: /* remote error - look into bits 20:24 */
 301                         switch ((cqe_status
 302                                  & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) {
 303                         case 0x0:
 304                                 /*
 305                                  * PSN Sequence Error!
 306                                  * couldn't find a matching status!
 307                                  */
 308                                 *wc_status = IB_WC_GENERAL_ERR;
 309                                 break;
 310                         case 0x1:
 311                                 *wc_status = IB_WC_REM_INV_REQ_ERR;
 312                                 break;
 313                         case 0x2:
 314                                 *wc_status = IB_WC_REM_ACCESS_ERR;
 315                                 break;
 316                         case 0x3:
 317                                 *wc_status = IB_WC_REM_OP_ERR;
 318                                 break;
 319                         case 0x4:
 320                                 *wc_status = IB_WC_REM_INV_RD_REQ_ERR;
 321                                 break;
 322                         }
 323                         break;
 324                 case 0x08:
 325                         *wc_status = IB_WC_RETRY_EXC_ERR;
 326                         break;
 327                 case 0x09:
 328                         *wc_status = IB_WC_RNR_RETRY_EXC_ERR;
 329                         break;
 330                 case 0x0A:
 331                 case 0x2D:
 332                         *wc_status = IB_WC_REM_ABORT_ERR;
 333                         break;
 334                 case 0x0B:
 335                 case 0x2E:
 336                         *wc_status = IB_WC_INV_EECN_ERR;
 337                         break;
 338                 case 0x0C:
 339                 case 0x2F:
 340                         *wc_status = IB_WC_INV_EEC_STATE_ERR;
 341                         break;
 342                 case 0x0D:
 343                         *wc_status = IB_WC_BAD_RESP_ERR;
 344                         break;
 345                 case 0x10:
 346                         /* WQE purged */
 347                         *wc_status = IB_WC_WR_FLUSH_ERR;
 348                         break;
 349                 default:
 350                         *wc_status = IB_WC_FATAL_ERR;
 351
 352                 }
 353         } else
 354                 *wc_status = IB_WC_SUCCESS;
 355 }
 356
 357 int ehca_post_send(struct ib_qp *qp,
 358                    struct ib_send_wr *send_wr,
 359                    struct ib_send_wr **bad_send_wr)
 360 {
 361         struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
 362         struct ib_send_wr *cur_send_wr;
 363         struct ehca_wqe *wqe_p;
 364         int wqe_cnt = 0;
 365         int ret = 0;
 366         unsigned long flags;
 367
 368         /* LOCK the QUEUE */
 369         spin_lock_irqsave(&my_qp->spinlock_s, flags);
 370
 371         /* loop processes list of send reqs */
 372         for (cur_send_wr = send_wr; cur_send_wr != NULL;
 373              cur_send_wr = cur_send_wr->next) {
 374                 u64 start_offset = my_qp->ipz_squeue.current_q_offset;
 375                 /* get pointer next to free WQE */
 376                 wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
 377                 if (unlikely(!wqe_p)) {
 378                         /* too many posted work requests: queue overflow */
 379                         if (bad_send_wr)
 380                                 *bad_send_wr = cur_send_wr;
 381                         if (wqe_cnt == 0) {
 382                                 ret = -ENOMEM;
 383                                 ehca_err(qp->device, "Too many posted WQEs "
 384                                          "qp_num=%x", qp->qp_num);
 385                         }
 386                         goto post_send_exit0;
 387                 }
 388                 /* write a SEND WQE into the QUEUE */
 389                 ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr);
 390                 /*
 391                  * if something failed,
 392                  * reset the free entry pointer to the start value
 393                  */
 394                 if (unlikely(ret)) {
 395                         my_qp->ipz_squeue.current_q_offset = start_offset;
 396                         *bad_send_wr = cur_send_wr;
 397                         if (wqe_cnt == 0) {
 398                                 ret = -EINVAL;
 399                                 ehca_err(qp->device, "Could not write WQE "
 400                                          "qp_num=%x", qp->qp_num);
 401                         }
 402                         goto post_send_exit0;
 403                 }
 404                 wqe_cnt++;
 405                 ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d",
 406                          my_qp, qp->qp_num, wqe_cnt);
 407         } /* eof for cur_send_wr */
 408
 409 post_send_exit0:
 410         /* UNLOCK the QUEUE */
 411         spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
 412         iosync(); /* serialize GAL register access */
 413         hipz_update_sqa(my_qp, wqe_cnt);
 414         return ret;
 415 }
 416
 417 static int internal_post_recv(struct ehca_qp *my_qp,
 418                               struct ib_device *dev,
 419                               struct ib_recv_wr *recv_wr,
 420                               struct ib_recv_wr **bad_recv_wr)
 421 {
 422         struct ib_recv_wr *cur_recv_wr;
 423         struct ehca_wqe *wqe_p;
 424         int wqe_cnt = 0;
 425         int ret = 0;
 426         unsigned long flags;
 427
 428         if (unlikely(!HAS_RQ(my_qp))) {
 429                 ehca_err(dev, "QP has no RQ  ehca_qp=%p qp_num=%x ext_type=%d",
 430                          my_qp, my_qp->real_qp_num, my_qp->ext_type);
 431                 return -ENODEV;
 432         }
 433
 434         /* LOCK the QUEUE */
 435         spin_lock_irqsave(&my_qp->spinlock_r, flags);
 436
 437         /* loop processes list of send reqs */
 438         for (cur_recv_wr = recv_wr; cur_recv_wr != NULL;
 439              cur_recv_wr = cur_recv_wr->next) {
 440                 u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
 441                 /* get pointer next to free WQE */
 442                 wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
 443                 if (unlikely(!wqe_p)) {
 444                         /* too many posted work requests: queue overflow */
 445                         if (bad_recv_wr)
 446                                 *bad_recv_wr = cur_recv_wr;
 447                         if (wqe_cnt == 0) {
 448                                 ret = -ENOMEM;
 449                                 ehca_err(dev, "Too many posted WQEs "
 450                                          "qp_num=%x", my_qp->real_qp_num);
 451                         }
 452                         goto post_recv_exit0;
 453                 }
 454                 /* write a RECV WQE into the QUEUE */
 455                 ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, cur_recv_wr);
 456                 /*
 457                  * if something failed,
 458                  * reset the free entry pointer to the start value
 459                  */
 460                 if (unlikely(ret)) {
 461                         my_qp->ipz_rqueue.current_q_offset = start_offset;
 462                         *bad_recv_wr = cur_recv_wr;
 463                         if (wqe_cnt == 0) {
 464                                 ret = -EINVAL;
 465                                 ehca_err(dev, "Could not write WQE "
 466                                          "qp_num=%x", my_qp->real_qp_num);
 467                         }
 468                         goto post_recv_exit0;
 469                 }
 470                 wqe_cnt++;
 471                 ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d",
 472                          my_qp, my_qp->real_qp_num, wqe_cnt);
 473         } /* eof for cur_recv_wr */
 474
 475 post_recv_exit0:
 476         spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
 477         iosync(); /* serialize GAL register access */
 478         hipz_update_rqa(my_qp, wqe_cnt);
 479         return ret;
 480 }
 481
 482 int ehca_post_recv(struct ib_qp *qp,
 483                    struct ib_recv_wr *recv_wr,
 484                    struct ib_recv_wr **bad_recv_wr)
 485 {
 486         return internal_post_recv(container_of(qp, struct ehca_qp, ib_qp),
 487                                   qp->device, recv_wr, bad_recv_wr);
 488 }
 489
 490 int ehca_post_srq_recv(struct ib_srq *srq,
 491                        struct ib_recv_wr *recv_wr,
 492                        struct ib_recv_wr **bad_recv_wr)
 493 {
 494         return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq),
 495                                   srq->device, recv_wr, bad_recv_wr);
 496 }
 497
 498 /*
 499  * ib_wc_opcode table converts ehca wc opcode to ib
 500  * Since we use zero to indicate invalid opcode, the actual ib opcode must
 501  * be decremented!!!
 502  */
 503 static const u8 ib_wc_opcode[255] = {
 504         [0x01] = IB_WC_RECV+1,
 505         [0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
 506         [0x04] = IB_WC_BIND_MW+1,
 507         [0x08] = IB_WC_FETCH_ADD+1,
 508         [0x10] = IB_WC_COMP_SWAP+1,
 509         [0x20] = IB_WC_RDMA_WRITE+1,
 510         [0x40] = IB_WC_RDMA_READ+1,
 511         [0x80] = IB_WC_SEND+1
 512 };
 513
 514 /* internal function to poll one entry of cq */
 515 static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
 516 {
 517         int ret = 0;
 518         struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
 519         struct ehca_cqe *cqe;
 520         int cqe_count = 0;
 521
 522 poll_cq_one_read_cqe:
 523         cqe = (struct ehca_cqe *)
 524                 ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
 525         if (!cqe) {
 526                 ret = -EAGAIN;
 527                 ehca_dbg(cq->device, "Completion queue is empty ehca_cq=%p "
 528                          "cq_num=%x ret=%x", my_cq, my_cq->cq_number, ret);
 529                 goto  poll_cq_one_exit0;
 530         }
 531
 532         /* prevents loads being reordered across this point */
 533         rmb();
 534
 535         cqe_count++;
 536         if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
 537                 struct ehca_qp *qp=ehca_cq_get_qp(my_cq, cqe->local_qp_number);
 538                 int purgeflag;
 539                 unsigned long flags;
 540                 if (!qp) {
 541                         ehca_err(cq->device, "cq_num=%x qp_num=%x "
 542                                  "could not find qp -> ignore cqe",
 543                                  my_cq->cq_number, cqe->local_qp_number);
 544                         ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
 545                                  my_cq->cq_number, cqe->local_qp_number);
 546                         /* ignore this purged cqe */
 547                         goto poll_cq_one_read_cqe;
 548                 }
 549                 spin_lock_irqsave(&qp->spinlock_s, flags);
 550                 purgeflag = qp->sqerr_purgeflag;
 551                 spin_unlock_irqrestore(&qp->spinlock_s, flags);
 552
 553                 if (purgeflag) {
 554                         ehca_dbg(cq->device, "Got CQE with purged bit qp_num=%x "
 555                                  "src_qp=%x",
 556                                  cqe->local_qp_number, cqe->remote_qp_number);
 557                         if (ehca_debug_level)
 558                                 ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
 559                                          cqe->local_qp_number,
 560                                          cqe->remote_qp_number);
 561                         /*
 562                          * ignore this to avoid double cqes of bad wqe
 563                          * that caused sqe and turn off purge flag
 564                          */
 565                         qp->sqerr_purgeflag = 0;
 566                         goto poll_cq_one_read_cqe;
 567                 }
 568         }
 569
 570         /* tracing cqe */
 571         if (ehca_debug_level) {
 572                 ehca_dbg(cq->device,
 573                          "Received COMPLETION ehca_cq=%p cq_num=%x -----",
 574                          my_cq, my_cq->cq_number);
 575                 ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
 576                          my_cq, my_cq->cq_number);
 577                 ehca_dbg(cq->device,
 578                          "ehca_cq=%p cq_num=%x -------------------------",
 579                          my_cq, my_cq->cq_number);
 580         }
 581
 582         /* we got a completion! */
 583         wc->wr_id = cqe->work_request_id;
 584
 585         /* eval ib_wc_opcode */
 586         wc->opcode = ib_wc_opcode[cqe->optype]-1;
 587         if (unlikely(wc->opcode == -1)) {
 588                 ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
 589                          "ehca_cq=%p cq_num=%x",
 590                          cqe->optype, cqe->status, my_cq, my_cq->cq_number);
 591                 /* dump cqe for other infos */
 592                 ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
 593                          my_cq, my_cq->cq_number);
 594                 /* update also queue adder to throw away this entry!!! */
 595                 goto poll_cq_one_exit0;
 596         }
 597         /* eval ib_wc_status */
 598         if (unlikely(cqe->status & WC_STATUS_ERROR_BIT)) {
 599                 /* complete with errors */
 600                 map_ib_wc_status(cqe->status, &wc->status);
 601                 wc->vendor_err = wc->status;
 602         } else
 603                 wc->status = IB_WC_SUCCESS;
 604
 605         wc->qp = NULL;
 606         wc->byte_len = cqe->nr_bytes_transferred;
 607         wc->pkey_index = cqe->pkey_index;
 608         wc->slid = cqe->rlid;
 609         wc->dlid_path_bits = cqe->dlid;
 610         wc->src_qp = cqe->remote_qp_number;
 611         wc->wc_flags = cqe->w_completion_flags;
 612         wc->imm_data = cpu_to_be32(cqe->immediate_data);
 613         wc->sl = cqe->service_level;
 614
 615         if (wc->status != IB_WC_SUCCESS)
 616                 ehca_dbg(cq->device,
 617                          "ehca_cq=%p cq_num=%x WARNING unsuccessful cqe "
 618                          "OPType=%x status=%x qp_num=%x src_qp=%x wr_id=%lx "
 619                          "cqe=%p", my_cq, my_cq->cq_number, cqe->optype,
 620                          cqe->status, cqe->local_qp_number,
 621                          cqe->remote_qp_number, cqe->work_request_id, cqe);
 622
 623 poll_cq_one_exit0:
 624         if (cqe_count > 0)
 625                 hipz_update_feca(my_cq, cqe_count);
 626
 627         return ret;
 628 }
 629
 630 int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
 631 {
 632         struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
 633         int nr;
 634         struct ib_wc *current_wc = wc;
 635         int ret = 0;
 636         unsigned long flags;
 637
 638         if (num_entries < 1) {
 639                 ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
 640                          "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
 641                 ret = -EINVAL;
 642                 goto poll_cq_exit0;
 643         }
 644
 645         spin_lock_irqsave(&my_cq->spinlock, flags);
 646         for (nr = 0; nr < num_entries; nr++) {
 647                 ret = ehca_poll_cq_one(cq, current_wc);
 648                 if (ret)
 649                         break;
 650                 current_wc++;
 651         } /* eof for nr */
 652         spin_unlock_irqrestore(&my_cq->spinlock, flags);
 653         if (ret == -EAGAIN  || !ret)
 654                 ret = nr;
 655
 656 poll_cq_exit0:
 657         return ret;
 658 }
 659
 660 int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags)
 661 {
 662         struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
 663         int ret = 0;
 664
 665         switch (notify_flags & IB_CQ_SOLICITED_MASK) {
 666         case IB_CQ_SOLICITED:
 667                 hipz_set_cqx_n0(my_cq, 1);
 668                 break;
 669         case IB_CQ_NEXT_COMP:
 670                 hipz_set_cqx_n1(my_cq, 1);
 671                 break;
 672         default:
 673                 return -EINVAL;
 674         }
 675
 676         if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
 677                 unsigned long spl_flags;
 678                 spin_lock_irqsave(&my_cq->spinlock, spl_flags);
 679                 ret = ipz_qeit_is_valid(&my_cq->ipz_queue);
 680                 spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
 681         }
 682
 683         return ret;
 684 }