2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * $Id: mthca_srq.c 3047 2005-08-10 03:59:35Z roland $
35 #include <linux/slab.h>
36 #include <linux/string.h>
38 #include "mthca_dev.h"
39 #include "mthca_cmd.h"
40 #include "mthca_memfree.h"
41 #include "mthca_wqe.h"
44 MTHCA_MAX_DIRECT_SRQ_SIZE
= 4 * PAGE_SIZE
47 struct mthca_tavor_srq_context
{
48 __be64 wqe_base_ds
; /* low 6 bits is descriptor size */
52 __be16 limit_watermark
;
57 struct mthca_arbel_srq_context
{
58 __be32 state_logsize_srqn
;
61 __be32 logstride_usrpage
;
64 __be16 limit_watermark
;
71 static void *get_wqe(struct mthca_srq
*srq
, int n
)
74 return srq
->queue
.direct
.buf
+ (n
<< srq
->wqe_shift
);
76 return srq
->queue
.page_list
[(n
<< srq
->wqe_shift
) >> PAGE_SHIFT
].buf
+
77 ((n
<< srq
->wqe_shift
) & (PAGE_SIZE
- 1));
81 * Return a pointer to the location within a WQE that we're using as a
82 * link when the WQE is in the free list. We use the imm field
83 * because in the Tavor case, posting a WQE may overwrite the next
84 * segment of the previous WQE, but a receive WQE will never touch the
85 * imm field. This avoids corrupting our free list if the previous
86 * WQE has already completed and been put on the free list when we
89 static inline int *wqe_to_link(void *wqe
)
91 return (int *) (wqe
+ offsetof(struct mthca_next_seg
, imm
));
94 static void mthca_tavor_init_srq_context(struct mthca_dev
*dev
,
96 struct mthca_srq
*srq
,
97 struct mthca_tavor_srq_context
*context
)
99 memset(context
, 0, sizeof *context
);
101 context
->wqe_base_ds
= cpu_to_be64(1 << (srq
->wqe_shift
- 4));
102 context
->state_pd
= cpu_to_be32(pd
->pd_num
);
103 context
->lkey
= cpu_to_be32(srq
->mr
.ibmr
.lkey
);
105 if (pd
->ibpd
.uobject
)
107 cpu_to_be32(to_mucontext(pd
->ibpd
.uobject
->context
)->uar
.index
);
109 context
->uar
= cpu_to_be32(dev
->driver_uar
.index
);
112 static void mthca_arbel_init_srq_context(struct mthca_dev
*dev
,
114 struct mthca_srq
*srq
,
115 struct mthca_arbel_srq_context
*context
)
119 memset(context
, 0, sizeof *context
);
121 logsize
= long_log2(srq
->max
) + srq
->wqe_shift
;
122 context
->state_logsize_srqn
= cpu_to_be32(logsize
<< 24 | srq
->srqn
);
123 context
->lkey
= cpu_to_be32(srq
->mr
.ibmr
.lkey
);
124 context
->db_index
= cpu_to_be32(srq
->db_index
);
125 context
->logstride_usrpage
= cpu_to_be32((srq
->wqe_shift
- 4) << 29);
126 if (pd
->ibpd
.uobject
)
127 context
->logstride_usrpage
|=
128 cpu_to_be32(to_mucontext(pd
->ibpd
.uobject
->context
)->uar
.index
);
130 context
->logstride_usrpage
|= cpu_to_be32(dev
->driver_uar
.index
);
131 context
->eq_pd
= cpu_to_be32(MTHCA_EQ_ASYNC
<< 24 | pd
->pd_num
);
134 static void mthca_free_srq_buf(struct mthca_dev
*dev
, struct mthca_srq
*srq
)
136 mthca_buf_free(dev
, srq
->max
<< srq
->wqe_shift
, &srq
->queue
,
137 srq
->is_direct
, &srq
->mr
);
141 static int mthca_alloc_srq_buf(struct mthca_dev
*dev
, struct mthca_pd
*pd
,
142 struct mthca_srq
*srq
)
144 struct mthca_data_seg
*scatter
;
149 if (pd
->ibpd
.uobject
)
152 srq
->wrid
= kmalloc(srq
->max
* sizeof (u64
), GFP_KERNEL
);
156 err
= mthca_buf_alloc(dev
, srq
->max
<< srq
->wqe_shift
,
157 MTHCA_MAX_DIRECT_SRQ_SIZE
,
158 &srq
->queue
, &srq
->is_direct
, pd
, 1, &srq
->mr
);
165 * Now initialize the SRQ buffer so that all of the WQEs are
166 * linked into the list of free WQEs. In addition, set the
167 * scatter list L_Keys to the sentry value of 0x100.
169 for (i
= 0; i
< srq
->max
; ++i
) {
170 wqe
= get_wqe(srq
, i
);
172 *wqe_to_link(wqe
) = i
< srq
->max
- 1 ? i
+ 1 : -1;
174 for (scatter
= wqe
+ sizeof (struct mthca_next_seg
);
175 (void *) scatter
< wqe
+ (1 << srq
->wqe_shift
);
177 scatter
->lkey
= cpu_to_be32(MTHCA_INVAL_LKEY
);
180 srq
->last
= get_wqe(srq
, srq
->max
- 1);
185 int mthca_alloc_srq(struct mthca_dev
*dev
, struct mthca_pd
*pd
,
186 struct ib_srq_attr
*attr
, struct mthca_srq
*srq
)
188 struct mthca_mailbox
*mailbox
;
193 /* Sanity check SRQ size before proceeding */
194 if (attr
->max_wr
> dev
->limits
.max_srq_wqes
||
195 attr
->max_sge
> dev
->limits
.max_srq_sge
)
198 srq
->max
= attr
->max_wr
;
199 srq
->max_gs
= attr
->max_sge
;
202 if (mthca_is_memfree(dev
))
203 srq
->max
= roundup_pow_of_two(srq
->max
+ 1);
205 srq
->max
= srq
->max
+ 1;
208 roundup_pow_of_two(sizeof (struct mthca_next_seg
) +
209 srq
->max_gs
* sizeof (struct mthca_data_seg
)));
211 if (!mthca_is_memfree(dev
) && (ds
> dev
->limits
.max_desc_sz
))
214 srq
->wqe_shift
= long_log2(ds
);
216 srq
->srqn
= mthca_alloc(&dev
->srq_table
.alloc
);
220 if (mthca_is_memfree(dev
)) {
221 err
= mthca_table_get(dev
, dev
->srq_table
.table
, srq
->srqn
);
225 if (!pd
->ibpd
.uobject
) {
226 srq
->db_index
= mthca_alloc_db(dev
, MTHCA_DB_TYPE_SRQ
,
227 srq
->srqn
, &srq
->db
);
228 if (srq
->db_index
< 0) {
235 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
236 if (IS_ERR(mailbox
)) {
237 err
= PTR_ERR(mailbox
);
241 err
= mthca_alloc_srq_buf(dev
, pd
, srq
);
243 goto err_out_mailbox
;
245 spin_lock_init(&srq
->lock
);
247 init_waitqueue_head(&srq
->wait
);
248 mutex_init(&srq
->mutex
);
250 if (mthca_is_memfree(dev
))
251 mthca_arbel_init_srq_context(dev
, pd
, srq
, mailbox
->buf
);
253 mthca_tavor_init_srq_context(dev
, pd
, srq
, mailbox
->buf
);
255 err
= mthca_SW2HW_SRQ(dev
, mailbox
, srq
->srqn
, &status
);
258 mthca_warn(dev
, "SW2HW_SRQ failed (%d)\n", err
);
259 goto err_out_free_buf
;
262 mthca_warn(dev
, "SW2HW_SRQ returned status 0x%02x\n",
265 goto err_out_free_buf
;
268 spin_lock_irq(&dev
->srq_table
.lock
);
269 if (mthca_array_set(&dev
->srq_table
.srq
,
270 srq
->srqn
& (dev
->limits
.num_srqs
- 1),
272 spin_unlock_irq(&dev
->srq_table
.lock
);
273 goto err_out_free_srq
;
275 spin_unlock_irq(&dev
->srq_table
.lock
);
277 mthca_free_mailbox(dev
, mailbox
);
280 srq
->last_free
= srq
->max
- 1;
282 attr
->max_wr
= srq
->max
- 1;
283 attr
->max_sge
= srq
->max_gs
;
288 err
= mthca_HW2SW_SRQ(dev
, mailbox
, srq
->srqn
, &status
);
290 mthca_warn(dev
, "HW2SW_SRQ failed (%d)\n", err
);
292 mthca_warn(dev
, "HW2SW_SRQ returned status 0x%02x\n", status
);
295 if (!pd
->ibpd
.uobject
)
296 mthca_free_srq_buf(dev
, srq
);
299 mthca_free_mailbox(dev
, mailbox
);
302 if (!pd
->ibpd
.uobject
&& mthca_is_memfree(dev
))
303 mthca_free_db(dev
, MTHCA_DB_TYPE_SRQ
, srq
->db_index
);
306 mthca_table_put(dev
, dev
->srq_table
.table
, srq
->srqn
);
309 mthca_free(&dev
->srq_table
.alloc
, srq
->srqn
);
314 static inline int get_srq_refcount(struct mthca_dev
*dev
, struct mthca_srq
*srq
)
318 spin_lock_irq(&dev
->srq_table
.lock
);
320 spin_unlock_irq(&dev
->srq_table
.lock
);
325 void mthca_free_srq(struct mthca_dev
*dev
, struct mthca_srq
*srq
)
327 struct mthca_mailbox
*mailbox
;
331 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
332 if (IS_ERR(mailbox
)) {
333 mthca_warn(dev
, "No memory for mailbox to free SRQ.\n");
337 err
= mthca_HW2SW_SRQ(dev
, mailbox
, srq
->srqn
, &status
);
339 mthca_warn(dev
, "HW2SW_SRQ failed (%d)\n", err
);
341 mthca_warn(dev
, "HW2SW_SRQ returned status 0x%02x\n", status
);
343 spin_lock_irq(&dev
->srq_table
.lock
);
344 mthca_array_clear(&dev
->srq_table
.srq
,
345 srq
->srqn
& (dev
->limits
.num_srqs
- 1));
347 spin_unlock_irq(&dev
->srq_table
.lock
);
349 wait_event(srq
->wait
, !get_srq_refcount(dev
, srq
));
351 if (!srq
->ibsrq
.uobject
) {
352 mthca_free_srq_buf(dev
, srq
);
353 if (mthca_is_memfree(dev
))
354 mthca_free_db(dev
, MTHCA_DB_TYPE_SRQ
, srq
->db_index
);
357 mthca_table_put(dev
, dev
->srq_table
.table
, srq
->srqn
);
358 mthca_free(&dev
->srq_table
.alloc
, srq
->srqn
);
359 mthca_free_mailbox(dev
, mailbox
);
362 int mthca_modify_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*attr
,
363 enum ib_srq_attr_mask attr_mask
, struct ib_udata
*udata
)
365 struct mthca_dev
*dev
= to_mdev(ibsrq
->device
);
366 struct mthca_srq
*srq
= to_msrq(ibsrq
);
370 /* We don't support resizing SRQs (yet?) */
371 if (attr_mask
& IB_SRQ_MAX_WR
)
374 if (attr_mask
& IB_SRQ_LIMIT
) {
375 u32 max_wr
= mthca_is_memfree(dev
) ? srq
->max
- 1 : srq
->max
;
376 if (attr
->srq_limit
> max_wr
)
379 mutex_lock(&srq
->mutex
);
380 ret
= mthca_ARM_SRQ(dev
, srq
->srqn
, attr
->srq_limit
, &status
);
381 mutex_unlock(&srq
->mutex
);
392 int mthca_query_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*srq_attr
)
394 struct mthca_dev
*dev
= to_mdev(ibsrq
->device
);
395 struct mthca_srq
*srq
= to_msrq(ibsrq
);
396 struct mthca_mailbox
*mailbox
;
397 struct mthca_arbel_srq_context
*arbel_ctx
;
398 struct mthca_tavor_srq_context
*tavor_ctx
;
402 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
404 return PTR_ERR(mailbox
);
406 err
= mthca_QUERY_SRQ(dev
, srq
->srqn
, mailbox
, &status
);
410 if (mthca_is_memfree(dev
)) {
411 arbel_ctx
= mailbox
->buf
;
412 srq_attr
->srq_limit
= be16_to_cpu(arbel_ctx
->limit_watermark
);
414 tavor_ctx
= mailbox
->buf
;
415 srq_attr
->srq_limit
= be16_to_cpu(tavor_ctx
->limit_watermark
);
418 srq_attr
->max_wr
= srq
->max
- 1;
419 srq_attr
->max_sge
= srq
->max_gs
;
422 mthca_free_mailbox(dev
, mailbox
);
427 void mthca_srq_event(struct mthca_dev
*dev
, u32 srqn
,
428 enum ib_event_type event_type
)
430 struct mthca_srq
*srq
;
431 struct ib_event event
;
433 spin_lock(&dev
->srq_table
.lock
);
434 srq
= mthca_array_get(&dev
->srq_table
.srq
, srqn
& (dev
->limits
.num_srqs
- 1));
437 spin_unlock(&dev
->srq_table
.lock
);
440 mthca_warn(dev
, "Async event for bogus SRQ %08x\n", srqn
);
444 if (!srq
->ibsrq
.event_handler
)
447 event
.device
= &dev
->ib_dev
;
448 event
.event
= event_type
;
449 event
.element
.srq
= &srq
->ibsrq
;
450 srq
->ibsrq
.event_handler(&event
, srq
->ibsrq
.srq_context
);
453 spin_lock(&dev
->srq_table
.lock
);
454 if (!--srq
->refcount
)
456 spin_unlock(&dev
->srq_table
.lock
);
460 * This function must be called with IRQs disabled.
462 void mthca_free_srq_wqe(struct mthca_srq
*srq
, u32 wqe_addr
)
466 ind
= wqe_addr
>> srq
->wqe_shift
;
468 spin_lock(&srq
->lock
);
470 if (likely(srq
->first_free
>= 0))
471 *wqe_to_link(get_wqe(srq
, srq
->last_free
)) = ind
;
473 srq
->first_free
= ind
;
475 *wqe_to_link(get_wqe(srq
, ind
)) = -1;
476 srq
->last_free
= ind
;
478 spin_unlock(&srq
->lock
);
481 int mthca_tavor_post_srq_recv(struct ib_srq
*ibsrq
, struct ib_recv_wr
*wr
,
482 struct ib_recv_wr
**bad_wr
)
484 struct mthca_dev
*dev
= to_mdev(ibsrq
->device
);
485 struct mthca_srq
*srq
= to_msrq(ibsrq
);
497 spin_lock_irqsave(&srq
->lock
, flags
);
499 first_ind
= srq
->first_free
;
501 for (nreq
= 0; wr
; wr
= wr
->next
) {
502 ind
= srq
->first_free
;
505 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
511 wqe
= get_wqe(srq
, ind
);
512 next_ind
= *wqe_to_link(wqe
);
515 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
521 prev_wqe
= srq
->last
;
524 ((struct mthca_next_seg
*) wqe
)->nda_op
= 0;
525 ((struct mthca_next_seg
*) wqe
)->ee_nds
= 0;
526 /* flags field will always remain 0 */
528 wqe
+= sizeof (struct mthca_next_seg
);
530 if (unlikely(wr
->num_sge
> srq
->max_gs
)) {
533 srq
->last
= prev_wqe
;
537 for (i
= 0; i
< wr
->num_sge
; ++i
) {
538 ((struct mthca_data_seg
*) wqe
)->byte_count
=
539 cpu_to_be32(wr
->sg_list
[i
].length
);
540 ((struct mthca_data_seg
*) wqe
)->lkey
=
541 cpu_to_be32(wr
->sg_list
[i
].lkey
);
542 ((struct mthca_data_seg
*) wqe
)->addr
=
543 cpu_to_be64(wr
->sg_list
[i
].addr
);
544 wqe
+= sizeof (struct mthca_data_seg
);
547 if (i
< srq
->max_gs
) {
548 ((struct mthca_data_seg
*) wqe
)->byte_count
= 0;
549 ((struct mthca_data_seg
*) wqe
)->lkey
= cpu_to_be32(MTHCA_INVAL_LKEY
);
550 ((struct mthca_data_seg
*) wqe
)->addr
= 0;
553 ((struct mthca_next_seg
*) prev_wqe
)->nda_op
=
554 cpu_to_be32((ind
<< srq
->wqe_shift
) | 1);
556 ((struct mthca_next_seg
*) prev_wqe
)->ee_nds
=
557 cpu_to_be32(MTHCA_NEXT_DBD
);
559 srq
->wrid
[ind
] = wr
->wr_id
;
560 srq
->first_free
= next_ind
;
563 if (unlikely(nreq
== MTHCA_TAVOR_MAX_WQES_PER_RECV_DB
)) {
566 doorbell
[0] = cpu_to_be32(first_ind
<< srq
->wqe_shift
);
567 doorbell
[1] = cpu_to_be32(srq
->srqn
<< 8);
570 * Make sure that descriptors are written
571 * before doorbell is rung.
575 mthca_write64(doorbell
,
576 dev
->kar
+ MTHCA_RECEIVE_DOORBELL
,
577 MTHCA_GET_DOORBELL_LOCK(&dev
->doorbell_lock
));
579 first_ind
= srq
->first_free
;
584 doorbell
[0] = cpu_to_be32(first_ind
<< srq
->wqe_shift
);
585 doorbell
[1] = cpu_to_be32((srq
->srqn
<< 8) | nreq
);
588 * Make sure that descriptors are written before
593 mthca_write64(doorbell
,
594 dev
->kar
+ MTHCA_RECEIVE_DOORBELL
,
595 MTHCA_GET_DOORBELL_LOCK(&dev
->doorbell_lock
));
598 spin_unlock_irqrestore(&srq
->lock
, flags
);
602 int mthca_arbel_post_srq_recv(struct ib_srq
*ibsrq
, struct ib_recv_wr
*wr
,
603 struct ib_recv_wr
**bad_wr
)
605 struct mthca_dev
*dev
= to_mdev(ibsrq
->device
);
606 struct mthca_srq
*srq
= to_msrq(ibsrq
);
615 spin_lock_irqsave(&srq
->lock
, flags
);
617 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
618 ind
= srq
->first_free
;
621 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
627 wqe
= get_wqe(srq
, ind
);
628 next_ind
= *wqe_to_link(wqe
);
631 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
637 ((struct mthca_next_seg
*) wqe
)->nda_op
=
638 cpu_to_be32((next_ind
<< srq
->wqe_shift
) | 1);
639 ((struct mthca_next_seg
*) wqe
)->ee_nds
= 0;
640 /* flags field will always remain 0 */
642 wqe
+= sizeof (struct mthca_next_seg
);
644 if (unlikely(wr
->num_sge
> srq
->max_gs
)) {
650 for (i
= 0; i
< wr
->num_sge
; ++i
) {
651 ((struct mthca_data_seg
*) wqe
)->byte_count
=
652 cpu_to_be32(wr
->sg_list
[i
].length
);
653 ((struct mthca_data_seg
*) wqe
)->lkey
=
654 cpu_to_be32(wr
->sg_list
[i
].lkey
);
655 ((struct mthca_data_seg
*) wqe
)->addr
=
656 cpu_to_be64(wr
->sg_list
[i
].addr
);
657 wqe
+= sizeof (struct mthca_data_seg
);
660 if (i
< srq
->max_gs
) {
661 ((struct mthca_data_seg
*) wqe
)->byte_count
= 0;
662 ((struct mthca_data_seg
*) wqe
)->lkey
= cpu_to_be32(MTHCA_INVAL_LKEY
);
663 ((struct mthca_data_seg
*) wqe
)->addr
= 0;
666 srq
->wrid
[ind
] = wr
->wr_id
;
667 srq
->first_free
= next_ind
;
671 srq
->counter
+= nreq
;
674 * Make sure that descriptors are written before
675 * we write doorbell record.
678 *srq
->db
= cpu_to_be32(srq
->counter
);
681 spin_unlock_irqrestore(&srq
->lock
, flags
);
685 int mthca_max_srq_sge(struct mthca_dev
*dev
)
687 if (mthca_is_memfree(dev
))
688 return dev
->limits
.max_sg
;
691 * SRQ allocations are based on powers of 2 for Tavor,
692 * (although they only need to be multiples of 16 bytes).
694 * Therefore, we need to base the max number of sg entries on
695 * the largest power of 2 descriptor size that is <= to the
696 * actual max WQE descriptor size, rather than return the
697 * max_sg value given by the firmware (which is based on WQE
698 * sizes as multiples of 16, not powers of 2).
700 * If SRQ implementation is changed for Tavor to be based on
701 * multiples of 16, the calculation below can be deleted and
702 * the FW max_sg value returned.
704 return min_t(int, dev
->limits
.max_sg
,
705 ((1 << (fls(dev
->limits
.max_desc_sz
) - 1)) -
706 sizeof (struct mthca_next_seg
)) /
707 sizeof (struct mthca_data_seg
));
710 int __devinit
mthca_init_srq_table(struct mthca_dev
*dev
)
714 if (!(dev
->mthca_flags
& MTHCA_FLAG_SRQ
))
717 spin_lock_init(&dev
->srq_table
.lock
);
719 err
= mthca_alloc_init(&dev
->srq_table
.alloc
,
720 dev
->limits
.num_srqs
,
721 dev
->limits
.num_srqs
- 1,
722 dev
->limits
.reserved_srqs
);
726 err
= mthca_array_init(&dev
->srq_table
.srq
,
727 dev
->limits
.num_srqs
);
729 mthca_alloc_cleanup(&dev
->srq_table
.alloc
);
734 void mthca_cleanup_srq_table(struct mthca_dev
*dev
)
736 if (!(dev
->mthca_flags
& MTHCA_FLAG_SRQ
))
739 mthca_array_cleanup(&dev
->srq_table
.srq
, dev
->limits
.num_srqs
);
740 mthca_alloc_cleanup(&dev
->srq_table
.alloc
);