2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/file.h>
38 #include <linux/slab.h>
40 #include <asm/uaccess.h>
44 static struct lock_class_key pd_lock_key
;
45 static struct lock_class_key mr_lock_key
;
46 static struct lock_class_key cq_lock_key
;
47 static struct lock_class_key qp_lock_key
;
48 static struct lock_class_key ah_lock_key
;
49 static struct lock_class_key srq_lock_key
;
50 static struct lock_class_key xrcd_lock_key
;
52 #define INIT_UDATA(udata, ibuf, obuf, ilen, olen) \
54 (udata)->inbuf = (void __user *) (ibuf); \
55 (udata)->outbuf = (void __user *) (obuf); \
56 (udata)->inlen = (ilen); \
57 (udata)->outlen = (olen); \
61 * The ib_uobject locking scheme is as follows:
63 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it
64 * needs to be held during all idr operations. When an object is
65 * looked up, a reference must be taken on the object's kref before
68 * - Each object also has an rwsem. This rwsem must be held for
69 * reading while an operation that uses the object is performed.
70 * For example, while registering an MR, the associated PD's
71 * uobject.mutex must be held for reading. The rwsem must be held
72 * for writing while initializing or destroying an object.
74 * - In addition, each object has a "live" flag. If this flag is not
75 * set, then lookups of the object will fail even if it is found in
76 * the idr. This handles a reader that blocks and does not acquire
77 * the rwsem until after the object is destroyed. The destroy
78 * operation will set the live flag to 0 and then drop the rwsem;
79 * this will allow the reader to acquire the rwsem, see that the
80 * live flag is 0, and then drop the rwsem and its reference to
81 * object. The underlying storage will not be freed until the last
82 * reference to the object is dropped.
85 static void init_uobj(struct ib_uobject
*uobj
, u64 user_handle
,
86 struct ib_ucontext
*context
, struct lock_class_key
*key
)
88 uobj
->user_handle
= user_handle
;
89 uobj
->context
= context
;
90 kref_init(&uobj
->ref
);
91 init_rwsem(&uobj
->mutex
);
92 lockdep_set_class(&uobj
->mutex
, key
);
96 static void release_uobj(struct kref
*kref
)
98 kfree(container_of(kref
, struct ib_uobject
, ref
));
101 static void put_uobj(struct ib_uobject
*uobj
)
103 kref_put(&uobj
->ref
, release_uobj
);
106 static void put_uobj_read(struct ib_uobject
*uobj
)
108 up_read(&uobj
->mutex
);
112 static void put_uobj_write(struct ib_uobject
*uobj
)
114 up_write(&uobj
->mutex
);
118 static int idr_add_uobj(struct idr
*idr
, struct ib_uobject
*uobj
)
123 if (!idr_pre_get(idr
, GFP_KERNEL
))
126 spin_lock(&ib_uverbs_idr_lock
);
127 ret
= idr_get_new(idr
, uobj
, &uobj
->id
);
128 spin_unlock(&ib_uverbs_idr_lock
);
136 void idr_remove_uobj(struct idr
*idr
, struct ib_uobject
*uobj
)
138 spin_lock(&ib_uverbs_idr_lock
);
139 idr_remove(idr
, uobj
->id
);
140 spin_unlock(&ib_uverbs_idr_lock
);
143 static struct ib_uobject
*__idr_get_uobj(struct idr
*idr
, int id
,
144 struct ib_ucontext
*context
)
146 struct ib_uobject
*uobj
;
148 spin_lock(&ib_uverbs_idr_lock
);
149 uobj
= idr_find(idr
, id
);
151 if (uobj
->context
== context
)
152 kref_get(&uobj
->ref
);
156 spin_unlock(&ib_uverbs_idr_lock
);
161 static struct ib_uobject
*idr_read_uobj(struct idr
*idr
, int id
,
162 struct ib_ucontext
*context
, int nested
)
164 struct ib_uobject
*uobj
;
166 uobj
= __idr_get_uobj(idr
, id
, context
);
171 down_read_nested(&uobj
->mutex
, SINGLE_DEPTH_NESTING
);
173 down_read(&uobj
->mutex
);
182 static struct ib_uobject
*idr_write_uobj(struct idr
*idr
, int id
,
183 struct ib_ucontext
*context
)
185 struct ib_uobject
*uobj
;
187 uobj
= __idr_get_uobj(idr
, id
, context
);
191 down_write(&uobj
->mutex
);
193 put_uobj_write(uobj
);
200 static void *idr_read_obj(struct idr
*idr
, int id
, struct ib_ucontext
*context
,
203 struct ib_uobject
*uobj
;
205 uobj
= idr_read_uobj(idr
, id
, context
, nested
);
206 return uobj
? uobj
->object
: NULL
;
209 static struct ib_pd
*idr_read_pd(int pd_handle
, struct ib_ucontext
*context
)
211 return idr_read_obj(&ib_uverbs_pd_idr
, pd_handle
, context
, 0);
214 static void put_pd_read(struct ib_pd
*pd
)
216 put_uobj_read(pd
->uobject
);
219 static struct ib_cq
*idr_read_cq(int cq_handle
, struct ib_ucontext
*context
, int nested
)
221 return idr_read_obj(&ib_uverbs_cq_idr
, cq_handle
, context
, nested
);
224 static void put_cq_read(struct ib_cq
*cq
)
226 put_uobj_read(cq
->uobject
);
229 static struct ib_ah
*idr_read_ah(int ah_handle
, struct ib_ucontext
*context
)
231 return idr_read_obj(&ib_uverbs_ah_idr
, ah_handle
, context
, 0);
234 static void put_ah_read(struct ib_ah
*ah
)
236 put_uobj_read(ah
->uobject
);
239 static struct ib_qp
*idr_read_qp(int qp_handle
, struct ib_ucontext
*context
)
241 return idr_read_obj(&ib_uverbs_qp_idr
, qp_handle
, context
, 0);
244 static void put_qp_read(struct ib_qp
*qp
)
246 put_uobj_read(qp
->uobject
);
249 static struct ib_srq
*idr_read_srq(int srq_handle
, struct ib_ucontext
*context
)
251 return idr_read_obj(&ib_uverbs_srq_idr
, srq_handle
, context
, 0);
254 static void put_srq_read(struct ib_srq
*srq
)
256 put_uobj_read(srq
->uobject
);
259 static struct ib_xrcd
*idr_read_xrcd(int xrcd_handle
, struct ib_ucontext
*context
,
260 struct ib_uobject
**uobj
)
262 *uobj
= idr_read_uobj(&ib_uverbs_xrcd_idr
, xrcd_handle
, context
, 0);
263 return *uobj
? (*uobj
)->object
: NULL
;
266 static void put_xrcd_read(struct ib_uobject
*uobj
)
271 ssize_t
ib_uverbs_get_context(struct ib_uverbs_file
*file
,
272 const char __user
*buf
,
273 int in_len
, int out_len
)
275 struct ib_uverbs_get_context cmd
;
276 struct ib_uverbs_get_context_resp resp
;
277 struct ib_udata udata
;
278 struct ib_device
*ibdev
= file
->device
->ib_dev
;
279 struct ib_ucontext
*ucontext
;
283 if (out_len
< sizeof resp
)
286 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
289 mutex_lock(&file
->mutex
);
291 if (file
->ucontext
) {
296 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
297 (unsigned long) cmd
.response
+ sizeof resp
,
298 in_len
- sizeof cmd
, out_len
- sizeof resp
);
300 ucontext
= ibdev
->alloc_ucontext(ibdev
, &udata
);
301 if (IS_ERR(ucontext
)) {
302 ret
= PTR_ERR(ucontext
);
306 ucontext
->device
= ibdev
;
307 INIT_LIST_HEAD(&ucontext
->pd_list
);
308 INIT_LIST_HEAD(&ucontext
->mr_list
);
309 INIT_LIST_HEAD(&ucontext
->mw_list
);
310 INIT_LIST_HEAD(&ucontext
->cq_list
);
311 INIT_LIST_HEAD(&ucontext
->qp_list
);
312 INIT_LIST_HEAD(&ucontext
->srq_list
);
313 INIT_LIST_HEAD(&ucontext
->ah_list
);
314 INIT_LIST_HEAD(&ucontext
->xrcd_list
);
315 ucontext
->closing
= 0;
317 resp
.num_comp_vectors
= file
->device
->num_comp_vectors
;
319 ret
= get_unused_fd();
324 filp
= ib_uverbs_alloc_event_file(file
, 1);
330 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
331 &resp
, sizeof resp
)) {
336 file
->async_file
= filp
->private_data
;
338 INIT_IB_EVENT_HANDLER(&file
->event_handler
, file
->device
->ib_dev
,
339 ib_uverbs_event_handler
);
340 ret
= ib_register_event_handler(&file
->event_handler
);
344 kref_get(&file
->async_file
->ref
);
345 kref_get(&file
->ref
);
346 file
->ucontext
= ucontext
;
348 fd_install(resp
.async_fd
, filp
);
350 mutex_unlock(&file
->mutex
);
358 put_unused_fd(resp
.async_fd
);
361 ibdev
->dealloc_ucontext(ucontext
);
364 mutex_unlock(&file
->mutex
);
368 ssize_t
ib_uverbs_query_device(struct ib_uverbs_file
*file
,
369 const char __user
*buf
,
370 int in_len
, int out_len
)
372 struct ib_uverbs_query_device cmd
;
373 struct ib_uverbs_query_device_resp resp
;
374 struct ib_device_attr attr
;
377 if (out_len
< sizeof resp
)
380 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
383 ret
= ib_query_device(file
->device
->ib_dev
, &attr
);
387 memset(&resp
, 0, sizeof resp
);
389 resp
.fw_ver
= attr
.fw_ver
;
390 resp
.node_guid
= file
->device
->ib_dev
->node_guid
;
391 resp
.sys_image_guid
= attr
.sys_image_guid
;
392 resp
.max_mr_size
= attr
.max_mr_size
;
393 resp
.page_size_cap
= attr
.page_size_cap
;
394 resp
.vendor_id
= attr
.vendor_id
;
395 resp
.vendor_part_id
= attr
.vendor_part_id
;
396 resp
.hw_ver
= attr
.hw_ver
;
397 resp
.max_qp
= attr
.max_qp
;
398 resp
.max_qp_wr
= attr
.max_qp_wr
;
399 resp
.device_cap_flags
= attr
.device_cap_flags
;
400 resp
.max_sge
= attr
.max_sge
;
401 resp
.max_sge_rd
= attr
.max_sge_rd
;
402 resp
.max_cq
= attr
.max_cq
;
403 resp
.max_cqe
= attr
.max_cqe
;
404 resp
.max_mr
= attr
.max_mr
;
405 resp
.max_pd
= attr
.max_pd
;
406 resp
.max_qp_rd_atom
= attr
.max_qp_rd_atom
;
407 resp
.max_ee_rd_atom
= attr
.max_ee_rd_atom
;
408 resp
.max_res_rd_atom
= attr
.max_res_rd_atom
;
409 resp
.max_qp_init_rd_atom
= attr
.max_qp_init_rd_atom
;
410 resp
.max_ee_init_rd_atom
= attr
.max_ee_init_rd_atom
;
411 resp
.atomic_cap
= attr
.atomic_cap
;
412 resp
.max_ee
= attr
.max_ee
;
413 resp
.max_rdd
= attr
.max_rdd
;
414 resp
.max_mw
= attr
.max_mw
;
415 resp
.max_raw_ipv6_qp
= attr
.max_raw_ipv6_qp
;
416 resp
.max_raw_ethy_qp
= attr
.max_raw_ethy_qp
;
417 resp
.max_mcast_grp
= attr
.max_mcast_grp
;
418 resp
.max_mcast_qp_attach
= attr
.max_mcast_qp_attach
;
419 resp
.max_total_mcast_qp_attach
= attr
.max_total_mcast_qp_attach
;
420 resp
.max_ah
= attr
.max_ah
;
421 resp
.max_fmr
= attr
.max_fmr
;
422 resp
.max_map_per_fmr
= attr
.max_map_per_fmr
;
423 resp
.max_srq
= attr
.max_srq
;
424 resp
.max_srq_wr
= attr
.max_srq_wr
;
425 resp
.max_srq_sge
= attr
.max_srq_sge
;
426 resp
.max_pkeys
= attr
.max_pkeys
;
427 resp
.local_ca_ack_delay
= attr
.local_ca_ack_delay
;
428 resp
.phys_port_cnt
= file
->device
->ib_dev
->phys_port_cnt
;
430 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
437 ssize_t
ib_uverbs_query_port(struct ib_uverbs_file
*file
,
438 const char __user
*buf
,
439 int in_len
, int out_len
)
441 struct ib_uverbs_query_port cmd
;
442 struct ib_uverbs_query_port_resp resp
;
443 struct ib_port_attr attr
;
446 if (out_len
< sizeof resp
)
449 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
452 ret
= ib_query_port(file
->device
->ib_dev
, cmd
.port_num
, &attr
);
456 memset(&resp
, 0, sizeof resp
);
458 resp
.state
= attr
.state
;
459 resp
.max_mtu
= attr
.max_mtu
;
460 resp
.active_mtu
= attr
.active_mtu
;
461 resp
.gid_tbl_len
= attr
.gid_tbl_len
;
462 resp
.port_cap_flags
= attr
.port_cap_flags
;
463 resp
.max_msg_sz
= attr
.max_msg_sz
;
464 resp
.bad_pkey_cntr
= attr
.bad_pkey_cntr
;
465 resp
.qkey_viol_cntr
= attr
.qkey_viol_cntr
;
466 resp
.pkey_tbl_len
= attr
.pkey_tbl_len
;
468 resp
.sm_lid
= attr
.sm_lid
;
470 resp
.max_vl_num
= attr
.max_vl_num
;
471 resp
.sm_sl
= attr
.sm_sl
;
472 resp
.subnet_timeout
= attr
.subnet_timeout
;
473 resp
.init_type_reply
= attr
.init_type_reply
;
474 resp
.active_width
= attr
.active_width
;
475 resp
.active_speed
= attr
.active_speed
;
476 resp
.phys_state
= attr
.phys_state
;
477 resp
.link_layer
= rdma_port_get_link_layer(file
->device
->ib_dev
,
480 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
487 ssize_t
ib_uverbs_alloc_pd(struct ib_uverbs_file
*file
,
488 const char __user
*buf
,
489 int in_len
, int out_len
)
491 struct ib_uverbs_alloc_pd cmd
;
492 struct ib_uverbs_alloc_pd_resp resp
;
493 struct ib_udata udata
;
494 struct ib_uobject
*uobj
;
498 if (out_len
< sizeof resp
)
501 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
504 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
505 (unsigned long) cmd
.response
+ sizeof resp
,
506 in_len
- sizeof cmd
, out_len
- sizeof resp
);
508 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
512 init_uobj(uobj
, 0, file
->ucontext
, &pd_lock_key
);
513 down_write(&uobj
->mutex
);
515 pd
= file
->device
->ib_dev
->alloc_pd(file
->device
->ib_dev
,
516 file
->ucontext
, &udata
);
522 pd
->device
= file
->device
->ib_dev
;
524 atomic_set(&pd
->usecnt
, 0);
527 ret
= idr_add_uobj(&ib_uverbs_pd_idr
, uobj
);
531 memset(&resp
, 0, sizeof resp
);
532 resp
.pd_handle
= uobj
->id
;
534 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
535 &resp
, sizeof resp
)) {
540 mutex_lock(&file
->mutex
);
541 list_add_tail(&uobj
->list
, &file
->ucontext
->pd_list
);
542 mutex_unlock(&file
->mutex
);
546 up_write(&uobj
->mutex
);
551 idr_remove_uobj(&ib_uverbs_pd_idr
, uobj
);
557 put_uobj_write(uobj
);
561 ssize_t
ib_uverbs_dealloc_pd(struct ib_uverbs_file
*file
,
562 const char __user
*buf
,
563 int in_len
, int out_len
)
565 struct ib_uverbs_dealloc_pd cmd
;
566 struct ib_uobject
*uobj
;
569 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
572 uobj
= idr_write_uobj(&ib_uverbs_pd_idr
, cmd
.pd_handle
, file
->ucontext
);
576 ret
= ib_dealloc_pd(uobj
->object
);
580 put_uobj_write(uobj
);
585 idr_remove_uobj(&ib_uverbs_pd_idr
, uobj
);
587 mutex_lock(&file
->mutex
);
588 list_del(&uobj
->list
);
589 mutex_unlock(&file
->mutex
);
596 struct xrcd_table_entry
{
598 struct ib_xrcd
*xrcd
;
602 static int xrcd_table_insert(struct ib_uverbs_device
*dev
,
604 struct ib_xrcd
*xrcd
)
606 struct xrcd_table_entry
*entry
, *scan
;
607 struct rb_node
**p
= &dev
->xrcd_tree
.rb_node
;
608 struct rb_node
*parent
= NULL
;
610 entry
= kmalloc(sizeof *entry
, GFP_KERNEL
);
615 entry
->inode
= inode
;
619 scan
= rb_entry(parent
, struct xrcd_table_entry
, node
);
621 if (inode
< scan
->inode
) {
623 } else if (inode
> scan
->inode
) {
631 rb_link_node(&entry
->node
, parent
, p
);
632 rb_insert_color(&entry
->node
, &dev
->xrcd_tree
);
637 static struct xrcd_table_entry
*xrcd_table_search(struct ib_uverbs_device
*dev
,
640 struct xrcd_table_entry
*entry
;
641 struct rb_node
*p
= dev
->xrcd_tree
.rb_node
;
644 entry
= rb_entry(p
, struct xrcd_table_entry
, node
);
646 if (inode
< entry
->inode
)
648 else if (inode
> entry
->inode
)
657 static struct ib_xrcd
*find_xrcd(struct ib_uverbs_device
*dev
, struct inode
*inode
)
659 struct xrcd_table_entry
*entry
;
661 entry
= xrcd_table_search(dev
, inode
);
668 static void xrcd_table_delete(struct ib_uverbs_device
*dev
,
671 struct xrcd_table_entry
*entry
;
673 entry
= xrcd_table_search(dev
, inode
);
676 rb_erase(&entry
->node
, &dev
->xrcd_tree
);
681 ssize_t
ib_uverbs_open_xrcd(struct ib_uverbs_file
*file
,
682 const char __user
*buf
, int in_len
,
685 struct ib_uverbs_open_xrcd cmd
;
686 struct ib_uverbs_open_xrcd_resp resp
;
687 struct ib_udata udata
;
688 struct ib_uxrcd_object
*obj
;
689 struct ib_xrcd
*xrcd
= NULL
;
690 struct file
*f
= NULL
;
691 struct inode
*inode
= NULL
;
695 if (out_len
< sizeof resp
)
698 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
701 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
702 (unsigned long) cmd
.response
+ sizeof resp
,
703 in_len
- sizeof cmd
, out_len
- sizeof resp
);
705 mutex_lock(&file
->device
->xrcd_tree_mutex
);
708 /* search for file descriptor */
712 goto err_tree_mutex_unlock
;
715 inode
= f
->f_dentry
->d_inode
;
718 goto err_tree_mutex_unlock
;
721 xrcd
= find_xrcd(file
->device
, inode
);
722 if (!xrcd
&& !(cmd
.oflags
& O_CREAT
)) {
723 /* no file descriptor. Need CREATE flag */
725 goto err_tree_mutex_unlock
;
728 if (xrcd
&& cmd
.oflags
& O_EXCL
) {
730 goto err_tree_mutex_unlock
;
734 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
737 goto err_tree_mutex_unlock
;
740 init_uobj(&obj
->uobject
, 0, file
->ucontext
, &xrcd_lock_key
);
742 down_write(&obj
->uobject
.mutex
);
745 xrcd
= file
->device
->ib_dev
->alloc_xrcd(file
->device
->ib_dev
,
746 file
->ucontext
, &udata
);
753 xrcd
->device
= file
->device
->ib_dev
;
754 atomic_set(&xrcd
->usecnt
, 0);
755 mutex_init(&xrcd
->tgt_qp_mutex
);
756 INIT_LIST_HEAD(&xrcd
->tgt_qp_list
);
760 atomic_set(&obj
->refcnt
, 0);
761 obj
->uobject
.object
= xrcd
;
762 ret
= idr_add_uobj(&ib_uverbs_xrcd_idr
, &obj
->uobject
);
766 memset(&resp
, 0, sizeof resp
);
767 resp
.xrcd_handle
= obj
->uobject
.id
;
771 /* create new inode/xrcd table entry */
772 ret
= xrcd_table_insert(file
->device
, inode
, xrcd
);
774 goto err_insert_xrcd
;
776 atomic_inc(&xrcd
->usecnt
);
779 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
780 &resp
, sizeof resp
)) {
788 mutex_lock(&file
->mutex
);
789 list_add_tail(&obj
->uobject
.list
, &file
->ucontext
->xrcd_list
);
790 mutex_unlock(&file
->mutex
);
792 obj
->uobject
.live
= 1;
793 up_write(&obj
->uobject
.mutex
);
795 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
801 xrcd_table_delete(file
->device
, inode
);
802 atomic_dec(&xrcd
->usecnt
);
806 idr_remove_uobj(&ib_uverbs_xrcd_idr
, &obj
->uobject
);
809 ib_dealloc_xrcd(xrcd
);
812 put_uobj_write(&obj
->uobject
);
814 err_tree_mutex_unlock
:
818 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
823 ssize_t
ib_uverbs_close_xrcd(struct ib_uverbs_file
*file
,
824 const char __user
*buf
, int in_len
,
827 struct ib_uverbs_close_xrcd cmd
;
828 struct ib_uobject
*uobj
;
829 struct ib_xrcd
*xrcd
= NULL
;
830 struct inode
*inode
= NULL
;
831 struct ib_uxrcd_object
*obj
;
835 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
838 mutex_lock(&file
->device
->xrcd_tree_mutex
);
839 uobj
= idr_write_uobj(&ib_uverbs_xrcd_idr
, cmd
.xrcd_handle
, file
->ucontext
);
847 obj
= container_of(uobj
, struct ib_uxrcd_object
, uobject
);
848 if (atomic_read(&obj
->refcnt
)) {
849 put_uobj_write(uobj
);
854 if (!inode
|| atomic_dec_and_test(&xrcd
->usecnt
)) {
855 ret
= ib_dealloc_xrcd(uobj
->object
);
862 atomic_inc(&xrcd
->usecnt
);
864 put_uobj_write(uobj
);
870 xrcd_table_delete(file
->device
, inode
);
872 idr_remove_uobj(&ib_uverbs_xrcd_idr
, uobj
);
873 mutex_lock(&file
->mutex
);
874 list_del(&uobj
->list
);
875 mutex_unlock(&file
->mutex
);
881 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
885 void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device
*dev
,
886 struct ib_xrcd
*xrcd
)
891 if (inode
&& !atomic_dec_and_test(&xrcd
->usecnt
))
894 ib_dealloc_xrcd(xrcd
);
897 xrcd_table_delete(dev
, inode
);
900 ssize_t
ib_uverbs_reg_mr(struct ib_uverbs_file
*file
,
901 const char __user
*buf
, int in_len
,
904 struct ib_uverbs_reg_mr cmd
;
905 struct ib_uverbs_reg_mr_resp resp
;
906 struct ib_udata udata
;
907 struct ib_uobject
*uobj
;
912 if (out_len
< sizeof resp
)
915 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
918 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
919 (unsigned long) cmd
.response
+ sizeof resp
,
920 in_len
- sizeof cmd
, out_len
- sizeof resp
);
922 if ((cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
))
926 * Local write permission is required if remote write or
927 * remote atomic permission is also requested.
929 if (cmd
.access_flags
& (IB_ACCESS_REMOTE_ATOMIC
| IB_ACCESS_REMOTE_WRITE
) &&
930 !(cmd
.access_flags
& IB_ACCESS_LOCAL_WRITE
))
933 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
937 init_uobj(uobj
, 0, file
->ucontext
, &mr_lock_key
);
938 down_write(&uobj
->mutex
);
940 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
946 mr
= pd
->device
->reg_user_mr(pd
, cmd
.start
, cmd
.length
, cmd
.hca_va
,
947 cmd
.access_flags
, &udata
);
953 mr
->device
= pd
->device
;
956 atomic_inc(&pd
->usecnt
);
957 atomic_set(&mr
->usecnt
, 0);
960 ret
= idr_add_uobj(&ib_uverbs_mr_idr
, uobj
);
964 memset(&resp
, 0, sizeof resp
);
965 resp
.lkey
= mr
->lkey
;
966 resp
.rkey
= mr
->rkey
;
967 resp
.mr_handle
= uobj
->id
;
969 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
970 &resp
, sizeof resp
)) {
977 mutex_lock(&file
->mutex
);
978 list_add_tail(&uobj
->list
, &file
->ucontext
->mr_list
);
979 mutex_unlock(&file
->mutex
);
983 up_write(&uobj
->mutex
);
988 idr_remove_uobj(&ib_uverbs_mr_idr
, uobj
);
997 put_uobj_write(uobj
);
1001 ssize_t
ib_uverbs_dereg_mr(struct ib_uverbs_file
*file
,
1002 const char __user
*buf
, int in_len
,
1005 struct ib_uverbs_dereg_mr cmd
;
1007 struct ib_uobject
*uobj
;
1010 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1013 uobj
= idr_write_uobj(&ib_uverbs_mr_idr
, cmd
.mr_handle
, file
->ucontext
);
1019 ret
= ib_dereg_mr(mr
);
1023 put_uobj_write(uobj
);
1028 idr_remove_uobj(&ib_uverbs_mr_idr
, uobj
);
1030 mutex_lock(&file
->mutex
);
1031 list_del(&uobj
->list
);
1032 mutex_unlock(&file
->mutex
);
1039 ssize_t
ib_uverbs_create_comp_channel(struct ib_uverbs_file
*file
,
1040 const char __user
*buf
, int in_len
,
1043 struct ib_uverbs_create_comp_channel cmd
;
1044 struct ib_uverbs_create_comp_channel_resp resp
;
1048 if (out_len
< sizeof resp
)
1051 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1054 ret
= get_unused_fd();
1059 filp
= ib_uverbs_alloc_event_file(file
, 0);
1061 put_unused_fd(resp
.fd
);
1062 return PTR_ERR(filp
);
1065 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1066 &resp
, sizeof resp
)) {
1067 put_unused_fd(resp
.fd
);
1072 fd_install(resp
.fd
, filp
);
1076 ssize_t
ib_uverbs_create_cq(struct ib_uverbs_file
*file
,
1077 const char __user
*buf
, int in_len
,
1080 struct ib_uverbs_create_cq cmd
;
1081 struct ib_uverbs_create_cq_resp resp
;
1082 struct ib_udata udata
;
1083 struct ib_ucq_object
*obj
;
1084 struct ib_uverbs_event_file
*ev_file
= NULL
;
1088 if (out_len
< sizeof resp
)
1091 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1094 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1095 (unsigned long) cmd
.response
+ sizeof resp
,
1096 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1098 if (cmd
.comp_vector
>= file
->device
->num_comp_vectors
)
1101 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
1105 init_uobj(&obj
->uobject
, cmd
.user_handle
, file
->ucontext
, &cq_lock_key
);
1106 down_write(&obj
->uobject
.mutex
);
1108 if (cmd
.comp_channel
>= 0) {
1109 ev_file
= ib_uverbs_lookup_comp_file(cmd
.comp_channel
);
1116 obj
->uverbs_file
= file
;
1117 obj
->comp_events_reported
= 0;
1118 obj
->async_events_reported
= 0;
1119 INIT_LIST_HEAD(&obj
->comp_list
);
1120 INIT_LIST_HEAD(&obj
->async_list
);
1122 cq
= file
->device
->ib_dev
->create_cq(file
->device
->ib_dev
, cmd
.cqe
,
1124 file
->ucontext
, &udata
);
1130 cq
->device
= file
->device
->ib_dev
;
1131 cq
->uobject
= &obj
->uobject
;
1132 cq
->comp_handler
= ib_uverbs_comp_handler
;
1133 cq
->event_handler
= ib_uverbs_cq_event_handler
;
1134 cq
->cq_context
= ev_file
;
1135 atomic_set(&cq
->usecnt
, 0);
1137 obj
->uobject
.object
= cq
;
1138 ret
= idr_add_uobj(&ib_uverbs_cq_idr
, &obj
->uobject
);
1142 memset(&resp
, 0, sizeof resp
);
1143 resp
.cq_handle
= obj
->uobject
.id
;
1146 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1147 &resp
, sizeof resp
)) {
1152 mutex_lock(&file
->mutex
);
1153 list_add_tail(&obj
->uobject
.list
, &file
->ucontext
->cq_list
);
1154 mutex_unlock(&file
->mutex
);
1156 obj
->uobject
.live
= 1;
1158 up_write(&obj
->uobject
.mutex
);
1163 idr_remove_uobj(&ib_uverbs_cq_idr
, &obj
->uobject
);
1170 ib_uverbs_release_ucq(file
, ev_file
, obj
);
1173 put_uobj_write(&obj
->uobject
);
1177 ssize_t
ib_uverbs_resize_cq(struct ib_uverbs_file
*file
,
1178 const char __user
*buf
, int in_len
,
1181 struct ib_uverbs_resize_cq cmd
;
1182 struct ib_uverbs_resize_cq_resp resp
;
1183 struct ib_udata udata
;
1187 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1190 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1191 (unsigned long) cmd
.response
+ sizeof resp
,
1192 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1194 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
1198 ret
= cq
->device
->resize_cq(cq
, cmd
.cqe
, &udata
);
1204 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1205 &resp
, sizeof resp
.cqe
))
1211 return ret
? ret
: in_len
;
1214 static int copy_wc_to_user(void __user
*dest
, struct ib_wc
*wc
)
1216 struct ib_uverbs_wc tmp
;
1218 tmp
.wr_id
= wc
->wr_id
;
1219 tmp
.status
= wc
->status
;
1220 tmp
.opcode
= wc
->opcode
;
1221 tmp
.vendor_err
= wc
->vendor_err
;
1222 tmp
.byte_len
= wc
->byte_len
;
1223 tmp
.ex
.imm_data
= (__u32 __force
) wc
->ex
.imm_data
;
1224 tmp
.qp_num
= wc
->qp
->qp_num
;
1225 tmp
.src_qp
= wc
->src_qp
;
1226 tmp
.wc_flags
= wc
->wc_flags
;
1227 tmp
.pkey_index
= wc
->pkey_index
;
1228 tmp
.slid
= wc
->slid
;
1230 tmp
.dlid_path_bits
= wc
->dlid_path_bits
;
1231 tmp
.port_num
= wc
->port_num
;
1234 if (copy_to_user(dest
, &tmp
, sizeof tmp
))
1240 ssize_t
ib_uverbs_poll_cq(struct ib_uverbs_file
*file
,
1241 const char __user
*buf
, int in_len
,
1244 struct ib_uverbs_poll_cq cmd
;
1245 struct ib_uverbs_poll_cq_resp resp
;
1246 u8 __user
*header_ptr
;
1247 u8 __user
*data_ptr
;
1252 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1255 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
1259 /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1260 header_ptr
= (void __user
*)(unsigned long) cmd
.response
;
1261 data_ptr
= header_ptr
+ sizeof resp
;
1263 memset(&resp
, 0, sizeof resp
);
1264 while (resp
.count
< cmd
.ne
) {
1265 ret
= ib_poll_cq(cq
, 1, &wc
);
1271 ret
= copy_wc_to_user(data_ptr
, &wc
);
1275 data_ptr
+= sizeof(struct ib_uverbs_wc
);
1279 if (copy_to_user(header_ptr
, &resp
, sizeof resp
)) {
1291 ssize_t
ib_uverbs_req_notify_cq(struct ib_uverbs_file
*file
,
1292 const char __user
*buf
, int in_len
,
1295 struct ib_uverbs_req_notify_cq cmd
;
1298 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1301 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
1305 ib_req_notify_cq(cq
, cmd
.solicited_only
?
1306 IB_CQ_SOLICITED
: IB_CQ_NEXT_COMP
);
1313 ssize_t
ib_uverbs_destroy_cq(struct ib_uverbs_file
*file
,
1314 const char __user
*buf
, int in_len
,
1317 struct ib_uverbs_destroy_cq cmd
;
1318 struct ib_uverbs_destroy_cq_resp resp
;
1319 struct ib_uobject
*uobj
;
1321 struct ib_ucq_object
*obj
;
1322 struct ib_uverbs_event_file
*ev_file
;
1325 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1328 uobj
= idr_write_uobj(&ib_uverbs_cq_idr
, cmd
.cq_handle
, file
->ucontext
);
1332 ev_file
= cq
->cq_context
;
1333 obj
= container_of(cq
->uobject
, struct ib_ucq_object
, uobject
);
1335 ret
= ib_destroy_cq(cq
);
1339 put_uobj_write(uobj
);
1344 idr_remove_uobj(&ib_uverbs_cq_idr
, uobj
);
1346 mutex_lock(&file
->mutex
);
1347 list_del(&uobj
->list
);
1348 mutex_unlock(&file
->mutex
);
1350 ib_uverbs_release_ucq(file
, ev_file
, obj
);
1352 memset(&resp
, 0, sizeof resp
);
1353 resp
.comp_events_reported
= obj
->comp_events_reported
;
1354 resp
.async_events_reported
= obj
->async_events_reported
;
1358 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1359 &resp
, sizeof resp
))
1365 ssize_t
ib_uverbs_create_qp(struct ib_uverbs_file
*file
,
1366 const char __user
*buf
, int in_len
,
1369 struct ib_uverbs_create_qp cmd
;
1370 struct ib_uverbs_create_qp_resp resp
;
1371 struct ib_udata udata
;
1372 struct ib_uqp_object
*obj
;
1373 struct ib_device
*device
;
1374 struct ib_pd
*pd
= NULL
;
1375 struct ib_xrcd
*xrcd
= NULL
;
1376 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
1377 struct ib_cq
*scq
= NULL
, *rcq
= NULL
;
1378 struct ib_srq
*srq
= NULL
;
1380 struct ib_qp_init_attr attr
;
1383 if (out_len
< sizeof resp
)
1386 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1389 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1390 (unsigned long) cmd
.response
+ sizeof resp
,
1391 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1393 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
1397 init_uobj(&obj
->uevent
.uobject
, cmd
.user_handle
, file
->ucontext
, &qp_lock_key
);
1398 down_write(&obj
->uevent
.uobject
.mutex
);
1400 if (cmd
.qp_type
== IB_QPT_XRC_TGT
) {
1401 xrcd
= idr_read_xrcd(cmd
.pd_handle
, file
->ucontext
, &xrcd_uobj
);
1406 device
= xrcd
->device
;
1408 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
1409 scq
= idr_read_cq(cmd
.send_cq_handle
, file
->ucontext
, 0);
1415 if (cmd
.qp_type
== IB_QPT_XRC_INI
) {
1416 cmd
.max_recv_wr
= cmd
.max_recv_sge
= 0;
1419 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
1420 if (!srq
|| srq
->srq_type
!= IB_SRQT_BASIC
) {
1425 rcq
= (cmd
.recv_cq_handle
== cmd
.send_cq_handle
) ?
1426 scq
: idr_read_cq(cmd
.recv_cq_handle
, file
->ucontext
, 1);
1432 device
= pd
->device
;
1435 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1436 attr
.qp_context
= file
;
1441 attr
.sq_sig_type
= cmd
.sq_sig_all
? IB_SIGNAL_ALL_WR
: IB_SIGNAL_REQ_WR
;
1442 attr
.qp_type
= cmd
.qp_type
;
1443 attr
.create_flags
= 0;
1445 attr
.cap
.max_send_wr
= cmd
.max_send_wr
;
1446 attr
.cap
.max_recv_wr
= cmd
.max_recv_wr
;
1447 attr
.cap
.max_send_sge
= cmd
.max_send_sge
;
1448 attr
.cap
.max_recv_sge
= cmd
.max_recv_sge
;
1449 attr
.cap
.max_inline_data
= cmd
.max_inline_data
;
1451 obj
->uevent
.events_reported
= 0;
1452 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1453 INIT_LIST_HEAD(&obj
->mcast_list
);
1455 if (cmd
.qp_type
== IB_QPT_XRC_TGT
)
1456 qp
= ib_create_qp(pd
, &attr
);
1458 qp
= device
->create_qp(pd
, &attr
, &udata
);
1465 if (cmd
.qp_type
!= IB_QPT_XRC_TGT
) {
1467 qp
->device
= device
;
1469 qp
->send_cq
= attr
.send_cq
;
1470 qp
->recv_cq
= attr
.recv_cq
;
1472 qp
->event_handler
= attr
.event_handler
;
1473 qp
->qp_context
= attr
.qp_context
;
1474 qp
->qp_type
= attr
.qp_type
;
1475 atomic_inc(&pd
->usecnt
);
1476 atomic_inc(&attr
.send_cq
->usecnt
);
1478 atomic_inc(&attr
.recv_cq
->usecnt
);
1480 atomic_inc(&attr
.srq
->usecnt
);
1482 qp
->uobject
= &obj
->uevent
.uobject
;
1484 obj
->uevent
.uobject
.object
= qp
;
1485 ret
= idr_add_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
1489 memset(&resp
, 0, sizeof resp
);
1490 resp
.qpn
= qp
->qp_num
;
1491 resp
.qp_handle
= obj
->uevent
.uobject
.id
;
1492 resp
.max_recv_sge
= attr
.cap
.max_recv_sge
;
1493 resp
.max_send_sge
= attr
.cap
.max_send_sge
;
1494 resp
.max_recv_wr
= attr
.cap
.max_recv_wr
;
1495 resp
.max_send_wr
= attr
.cap
.max_send_wr
;
1496 resp
.max_inline_data
= attr
.cap
.max_inline_data
;
1498 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1499 &resp
, sizeof resp
)) {
1505 put_xrcd_read(xrcd_uobj
);
1510 if (rcq
&& rcq
!= scq
)
1515 mutex_lock(&file
->mutex
);
1516 list_add_tail(&obj
->uevent
.uobject
.list
, &file
->ucontext
->qp_list
);
1517 mutex_unlock(&file
->mutex
);
1519 obj
->uevent
.uobject
.live
= 1;
1521 up_write(&obj
->uevent
.uobject
.mutex
);
1526 idr_remove_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
1533 put_xrcd_read(xrcd_uobj
);
1538 if (rcq
&& rcq
!= scq
)
1543 put_uobj_write(&obj
->uevent
.uobject
);
1547 ssize_t
ib_uverbs_open_qp(struct ib_uverbs_file
*file
,
1548 const char __user
*buf
, int in_len
, int out_len
)
1550 struct ib_uverbs_open_qp cmd
;
1551 struct ib_uverbs_create_qp_resp resp
;
1552 struct ib_udata udata
;
1553 struct ib_uqp_object
*obj
;
1554 struct ib_xrcd
*xrcd
;
1555 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
1557 struct ib_qp_open_attr attr
;
1560 if (out_len
< sizeof resp
)
1563 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1566 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1567 (unsigned long) cmd
.response
+ sizeof resp
,
1568 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1570 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
1574 init_uobj(&obj
->uevent
.uobject
, cmd
.user_handle
, file
->ucontext
, &qp_lock_key
);
1575 down_write(&obj
->uevent
.uobject
.mutex
);
1577 xrcd
= idr_read_xrcd(cmd
.pd_handle
, file
->ucontext
, &xrcd_uobj
);
1583 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1584 attr
.qp_context
= file
;
1585 attr
.qp_num
= cmd
.qpn
;
1586 attr
.qp_type
= cmd
.qp_type
;
1588 obj
->uevent
.events_reported
= 0;
1589 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1590 INIT_LIST_HEAD(&obj
->mcast_list
);
1592 qp
= ib_open_qp(xrcd
, &attr
);
1598 qp
->uobject
= &obj
->uevent
.uobject
;
1600 obj
->uevent
.uobject
.object
= qp
;
1601 ret
= idr_add_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
1605 memset(&resp
, 0, sizeof resp
);
1606 resp
.qpn
= qp
->qp_num
;
1607 resp
.qp_handle
= obj
->uevent
.uobject
.id
;
1609 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1610 &resp
, sizeof resp
)) {
1615 put_xrcd_read(xrcd_uobj
);
1617 mutex_lock(&file
->mutex
);
1618 list_add_tail(&obj
->uevent
.uobject
.list
, &file
->ucontext
->qp_list
);
1619 mutex_unlock(&file
->mutex
);
1621 obj
->uevent
.uobject
.live
= 1;
1623 up_write(&obj
->uevent
.uobject
.mutex
);
1628 idr_remove_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
1634 put_xrcd_read(xrcd_uobj
);
1635 put_uobj_write(&obj
->uevent
.uobject
);
1639 ssize_t
ib_uverbs_query_qp(struct ib_uverbs_file
*file
,
1640 const char __user
*buf
, int in_len
,
1643 struct ib_uverbs_query_qp cmd
;
1644 struct ib_uverbs_query_qp_resp resp
;
1646 struct ib_qp_attr
*attr
;
1647 struct ib_qp_init_attr
*init_attr
;
1650 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1653 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
1654 init_attr
= kmalloc(sizeof *init_attr
, GFP_KERNEL
);
1655 if (!attr
|| !init_attr
) {
1660 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
1666 ret
= ib_query_qp(qp
, attr
, cmd
.attr_mask
, init_attr
);
1673 memset(&resp
, 0, sizeof resp
);
1675 resp
.qp_state
= attr
->qp_state
;
1676 resp
.cur_qp_state
= attr
->cur_qp_state
;
1677 resp
.path_mtu
= attr
->path_mtu
;
1678 resp
.path_mig_state
= attr
->path_mig_state
;
1679 resp
.qkey
= attr
->qkey
;
1680 resp
.rq_psn
= attr
->rq_psn
;
1681 resp
.sq_psn
= attr
->sq_psn
;
1682 resp
.dest_qp_num
= attr
->dest_qp_num
;
1683 resp
.qp_access_flags
= attr
->qp_access_flags
;
1684 resp
.pkey_index
= attr
->pkey_index
;
1685 resp
.alt_pkey_index
= attr
->alt_pkey_index
;
1686 resp
.sq_draining
= attr
->sq_draining
;
1687 resp
.max_rd_atomic
= attr
->max_rd_atomic
;
1688 resp
.max_dest_rd_atomic
= attr
->max_dest_rd_atomic
;
1689 resp
.min_rnr_timer
= attr
->min_rnr_timer
;
1690 resp
.port_num
= attr
->port_num
;
1691 resp
.timeout
= attr
->timeout
;
1692 resp
.retry_cnt
= attr
->retry_cnt
;
1693 resp
.rnr_retry
= attr
->rnr_retry
;
1694 resp
.alt_port_num
= attr
->alt_port_num
;
1695 resp
.alt_timeout
= attr
->alt_timeout
;
1697 memcpy(resp
.dest
.dgid
, attr
->ah_attr
.grh
.dgid
.raw
, 16);
1698 resp
.dest
.flow_label
= attr
->ah_attr
.grh
.flow_label
;
1699 resp
.dest
.sgid_index
= attr
->ah_attr
.grh
.sgid_index
;
1700 resp
.dest
.hop_limit
= attr
->ah_attr
.grh
.hop_limit
;
1701 resp
.dest
.traffic_class
= attr
->ah_attr
.grh
.traffic_class
;
1702 resp
.dest
.dlid
= attr
->ah_attr
.dlid
;
1703 resp
.dest
.sl
= attr
->ah_attr
.sl
;
1704 resp
.dest
.src_path_bits
= attr
->ah_attr
.src_path_bits
;
1705 resp
.dest
.static_rate
= attr
->ah_attr
.static_rate
;
1706 resp
.dest
.is_global
= !!(attr
->ah_attr
.ah_flags
& IB_AH_GRH
);
1707 resp
.dest
.port_num
= attr
->ah_attr
.port_num
;
1709 memcpy(resp
.alt_dest
.dgid
, attr
->alt_ah_attr
.grh
.dgid
.raw
, 16);
1710 resp
.alt_dest
.flow_label
= attr
->alt_ah_attr
.grh
.flow_label
;
1711 resp
.alt_dest
.sgid_index
= attr
->alt_ah_attr
.grh
.sgid_index
;
1712 resp
.alt_dest
.hop_limit
= attr
->alt_ah_attr
.grh
.hop_limit
;
1713 resp
.alt_dest
.traffic_class
= attr
->alt_ah_attr
.grh
.traffic_class
;
1714 resp
.alt_dest
.dlid
= attr
->alt_ah_attr
.dlid
;
1715 resp
.alt_dest
.sl
= attr
->alt_ah_attr
.sl
;
1716 resp
.alt_dest
.src_path_bits
= attr
->alt_ah_attr
.src_path_bits
;
1717 resp
.alt_dest
.static_rate
= attr
->alt_ah_attr
.static_rate
;
1718 resp
.alt_dest
.is_global
= !!(attr
->alt_ah_attr
.ah_flags
& IB_AH_GRH
);
1719 resp
.alt_dest
.port_num
= attr
->alt_ah_attr
.port_num
;
1721 resp
.max_send_wr
= init_attr
->cap
.max_send_wr
;
1722 resp
.max_recv_wr
= init_attr
->cap
.max_recv_wr
;
1723 resp
.max_send_sge
= init_attr
->cap
.max_send_sge
;
1724 resp
.max_recv_sge
= init_attr
->cap
.max_recv_sge
;
1725 resp
.max_inline_data
= init_attr
->cap
.max_inline_data
;
1726 resp
.sq_sig_all
= init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
;
1728 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1729 &resp
, sizeof resp
))
1736 return ret
? ret
: in_len
;
1739 /* Remove ignored fields set in the attribute mask */
1740 static int modify_qp_mask(enum ib_qp_type qp_type
, int mask
)
1743 case IB_QPT_XRC_INI
:
1744 return mask
& ~(IB_QP_MAX_DEST_RD_ATOMIC
| IB_QP_MIN_RNR_TIMER
);
1745 case IB_QPT_XRC_TGT
:
1746 return mask
& ~(IB_QP_MAX_QP_RD_ATOMIC
| IB_QP_RETRY_CNT
|
1753 ssize_t
ib_uverbs_modify_qp(struct ib_uverbs_file
*file
,
1754 const char __user
*buf
, int in_len
,
1757 struct ib_uverbs_modify_qp cmd
;
1758 struct ib_udata udata
;
1760 struct ib_qp_attr
*attr
;
1763 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1766 INIT_UDATA(&udata
, buf
+ sizeof cmd
, NULL
, in_len
- sizeof cmd
,
1769 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
1773 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
1779 attr
->qp_state
= cmd
.qp_state
;
1780 attr
->cur_qp_state
= cmd
.cur_qp_state
;
1781 attr
->path_mtu
= cmd
.path_mtu
;
1782 attr
->path_mig_state
= cmd
.path_mig_state
;
1783 attr
->qkey
= cmd
.qkey
;
1784 attr
->rq_psn
= cmd
.rq_psn
;
1785 attr
->sq_psn
= cmd
.sq_psn
;
1786 attr
->dest_qp_num
= cmd
.dest_qp_num
;
1787 attr
->qp_access_flags
= cmd
.qp_access_flags
;
1788 attr
->pkey_index
= cmd
.pkey_index
;
1789 attr
->alt_pkey_index
= cmd
.alt_pkey_index
;
1790 attr
->en_sqd_async_notify
= cmd
.en_sqd_async_notify
;
1791 attr
->max_rd_atomic
= cmd
.max_rd_atomic
;
1792 attr
->max_dest_rd_atomic
= cmd
.max_dest_rd_atomic
;
1793 attr
->min_rnr_timer
= cmd
.min_rnr_timer
;
1794 attr
->port_num
= cmd
.port_num
;
1795 attr
->timeout
= cmd
.timeout
;
1796 attr
->retry_cnt
= cmd
.retry_cnt
;
1797 attr
->rnr_retry
= cmd
.rnr_retry
;
1798 attr
->alt_port_num
= cmd
.alt_port_num
;
1799 attr
->alt_timeout
= cmd
.alt_timeout
;
1801 memcpy(attr
->ah_attr
.grh
.dgid
.raw
, cmd
.dest
.dgid
, 16);
1802 attr
->ah_attr
.grh
.flow_label
= cmd
.dest
.flow_label
;
1803 attr
->ah_attr
.grh
.sgid_index
= cmd
.dest
.sgid_index
;
1804 attr
->ah_attr
.grh
.hop_limit
= cmd
.dest
.hop_limit
;
1805 attr
->ah_attr
.grh
.traffic_class
= cmd
.dest
.traffic_class
;
1806 attr
->ah_attr
.dlid
= cmd
.dest
.dlid
;
1807 attr
->ah_attr
.sl
= cmd
.dest
.sl
;
1808 attr
->ah_attr
.src_path_bits
= cmd
.dest
.src_path_bits
;
1809 attr
->ah_attr
.static_rate
= cmd
.dest
.static_rate
;
1810 attr
->ah_attr
.ah_flags
= cmd
.dest
.is_global
? IB_AH_GRH
: 0;
1811 attr
->ah_attr
.port_num
= cmd
.dest
.port_num
;
1813 memcpy(attr
->alt_ah_attr
.grh
.dgid
.raw
, cmd
.alt_dest
.dgid
, 16);
1814 attr
->alt_ah_attr
.grh
.flow_label
= cmd
.alt_dest
.flow_label
;
1815 attr
->alt_ah_attr
.grh
.sgid_index
= cmd
.alt_dest
.sgid_index
;
1816 attr
->alt_ah_attr
.grh
.hop_limit
= cmd
.alt_dest
.hop_limit
;
1817 attr
->alt_ah_attr
.grh
.traffic_class
= cmd
.alt_dest
.traffic_class
;
1818 attr
->alt_ah_attr
.dlid
= cmd
.alt_dest
.dlid
;
1819 attr
->alt_ah_attr
.sl
= cmd
.alt_dest
.sl
;
1820 attr
->alt_ah_attr
.src_path_bits
= cmd
.alt_dest
.src_path_bits
;
1821 attr
->alt_ah_attr
.static_rate
= cmd
.alt_dest
.static_rate
;
1822 attr
->alt_ah_attr
.ah_flags
= cmd
.alt_dest
.is_global
? IB_AH_GRH
: 0;
1823 attr
->alt_ah_attr
.port_num
= cmd
.alt_dest
.port_num
;
1825 if (qp
->real_qp
== qp
) {
1826 ret
= qp
->device
->modify_qp(qp
, attr
,
1827 modify_qp_mask(qp
->qp_type
, cmd
.attr_mask
), &udata
);
1829 ret
= ib_modify_qp(qp
, attr
, modify_qp_mask(qp
->qp_type
, cmd
.attr_mask
));
1845 ssize_t
ib_uverbs_destroy_qp(struct ib_uverbs_file
*file
,
1846 const char __user
*buf
, int in_len
,
1849 struct ib_uverbs_destroy_qp cmd
;
1850 struct ib_uverbs_destroy_qp_resp resp
;
1851 struct ib_uobject
*uobj
;
1853 struct ib_uqp_object
*obj
;
1856 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1859 memset(&resp
, 0, sizeof resp
);
1861 uobj
= idr_write_uobj(&ib_uverbs_qp_idr
, cmd
.qp_handle
, file
->ucontext
);
1865 obj
= container_of(uobj
, struct ib_uqp_object
, uevent
.uobject
);
1867 if (!list_empty(&obj
->mcast_list
)) {
1868 put_uobj_write(uobj
);
1872 ret
= ib_destroy_qp(qp
);
1876 put_uobj_write(uobj
);
1881 idr_remove_uobj(&ib_uverbs_qp_idr
, uobj
);
1883 mutex_lock(&file
->mutex
);
1884 list_del(&uobj
->list
);
1885 mutex_unlock(&file
->mutex
);
1887 ib_uverbs_release_uevent(file
, &obj
->uevent
);
1889 resp
.events_reported
= obj
->uevent
.events_reported
;
1893 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1894 &resp
, sizeof resp
))
1900 ssize_t
ib_uverbs_post_send(struct ib_uverbs_file
*file
,
1901 const char __user
*buf
, int in_len
,
1904 struct ib_uverbs_post_send cmd
;
1905 struct ib_uverbs_post_send_resp resp
;
1906 struct ib_uverbs_send_wr
*user_wr
;
1907 struct ib_send_wr
*wr
= NULL
, *last
, *next
, *bad_wr
;
1911 ssize_t ret
= -EINVAL
;
1913 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1916 if (in_len
< sizeof cmd
+ cmd
.wqe_size
* cmd
.wr_count
+
1917 cmd
.sge_count
* sizeof (struct ib_uverbs_sge
))
1920 if (cmd
.wqe_size
< sizeof (struct ib_uverbs_send_wr
))
1923 user_wr
= kmalloc(cmd
.wqe_size
, GFP_KERNEL
);
1927 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
1931 is_ud
= qp
->qp_type
== IB_QPT_UD
;
1934 for (i
= 0; i
< cmd
.wr_count
; ++i
) {
1935 if (copy_from_user(user_wr
,
1936 buf
+ sizeof cmd
+ i
* cmd
.wqe_size
,
1942 if (user_wr
->num_sge
+ sg_ind
> cmd
.sge_count
) {
1947 next
= kmalloc(ALIGN(sizeof *next
, sizeof (struct ib_sge
)) +
1948 user_wr
->num_sge
* sizeof (struct ib_sge
),
1962 next
->wr_id
= user_wr
->wr_id
;
1963 next
->num_sge
= user_wr
->num_sge
;
1964 next
->opcode
= user_wr
->opcode
;
1965 next
->send_flags
= user_wr
->send_flags
;
1968 next
->wr
.ud
.ah
= idr_read_ah(user_wr
->wr
.ud
.ah
,
1970 if (!next
->wr
.ud
.ah
) {
1974 next
->wr
.ud
.remote_qpn
= user_wr
->wr
.ud
.remote_qpn
;
1975 next
->wr
.ud
.remote_qkey
= user_wr
->wr
.ud
.remote_qkey
;
1977 switch (next
->opcode
) {
1978 case IB_WR_RDMA_WRITE_WITH_IMM
:
1980 (__be32 __force
) user_wr
->ex
.imm_data
;
1981 case IB_WR_RDMA_WRITE
:
1982 case IB_WR_RDMA_READ
:
1983 next
->wr
.rdma
.remote_addr
=
1984 user_wr
->wr
.rdma
.remote_addr
;
1985 next
->wr
.rdma
.rkey
=
1986 user_wr
->wr
.rdma
.rkey
;
1988 case IB_WR_SEND_WITH_IMM
:
1990 (__be32 __force
) user_wr
->ex
.imm_data
;
1992 case IB_WR_SEND_WITH_INV
:
1993 next
->ex
.invalidate_rkey
=
1994 user_wr
->ex
.invalidate_rkey
;
1996 case IB_WR_ATOMIC_CMP_AND_SWP
:
1997 case IB_WR_ATOMIC_FETCH_AND_ADD
:
1998 next
->wr
.atomic
.remote_addr
=
1999 user_wr
->wr
.atomic
.remote_addr
;
2000 next
->wr
.atomic
.compare_add
=
2001 user_wr
->wr
.atomic
.compare_add
;
2002 next
->wr
.atomic
.swap
= user_wr
->wr
.atomic
.swap
;
2003 next
->wr
.atomic
.rkey
= user_wr
->wr
.atomic
.rkey
;
2010 if (next
->num_sge
) {
2011 next
->sg_list
= (void *) next
+
2012 ALIGN(sizeof *next
, sizeof (struct ib_sge
));
2013 if (copy_from_user(next
->sg_list
,
2015 cmd
.wr_count
* cmd
.wqe_size
+
2016 sg_ind
* sizeof (struct ib_sge
),
2017 next
->num_sge
* sizeof (struct ib_sge
))) {
2021 sg_ind
+= next
->num_sge
;
2023 next
->sg_list
= NULL
;
2027 ret
= qp
->device
->post_send(qp
->real_qp
, wr
, &bad_wr
);
2029 for (next
= wr
; next
; next
= next
->next
) {
2035 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2036 &resp
, sizeof resp
))
2043 if (is_ud
&& wr
->wr
.ud
.ah
)
2044 put_ah_read(wr
->wr
.ud
.ah
);
2053 return ret
? ret
: in_len
;
2056 static struct ib_recv_wr
*ib_uverbs_unmarshall_recv(const char __user
*buf
,
2062 struct ib_uverbs_recv_wr
*user_wr
;
2063 struct ib_recv_wr
*wr
= NULL
, *last
, *next
;
2068 if (in_len
< wqe_size
* wr_count
+
2069 sge_count
* sizeof (struct ib_uverbs_sge
))
2070 return ERR_PTR(-EINVAL
);
2072 if (wqe_size
< sizeof (struct ib_uverbs_recv_wr
))
2073 return ERR_PTR(-EINVAL
);
2075 user_wr
= kmalloc(wqe_size
, GFP_KERNEL
);
2077 return ERR_PTR(-ENOMEM
);
2081 for (i
= 0; i
< wr_count
; ++i
) {
2082 if (copy_from_user(user_wr
, buf
+ i
* wqe_size
,
2088 if (user_wr
->num_sge
+ sg_ind
> sge_count
) {
2093 next
= kmalloc(ALIGN(sizeof *next
, sizeof (struct ib_sge
)) +
2094 user_wr
->num_sge
* sizeof (struct ib_sge
),
2108 next
->wr_id
= user_wr
->wr_id
;
2109 next
->num_sge
= user_wr
->num_sge
;
2111 if (next
->num_sge
) {
2112 next
->sg_list
= (void *) next
+
2113 ALIGN(sizeof *next
, sizeof (struct ib_sge
));
2114 if (copy_from_user(next
->sg_list
,
2115 buf
+ wr_count
* wqe_size
+
2116 sg_ind
* sizeof (struct ib_sge
),
2117 next
->num_sge
* sizeof (struct ib_sge
))) {
2121 sg_ind
+= next
->num_sge
;
2123 next
->sg_list
= NULL
;
2138 return ERR_PTR(ret
);
2141 ssize_t
ib_uverbs_post_recv(struct ib_uverbs_file
*file
,
2142 const char __user
*buf
, int in_len
,
2145 struct ib_uverbs_post_recv cmd
;
2146 struct ib_uverbs_post_recv_resp resp
;
2147 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
2149 ssize_t ret
= -EINVAL
;
2151 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2154 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
2155 in_len
- sizeof cmd
, cmd
.wr_count
,
2156 cmd
.sge_count
, cmd
.wqe_size
);
2160 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2165 ret
= qp
->device
->post_recv(qp
->real_qp
, wr
, &bad_wr
);
2170 for (next
= wr
; next
; next
= next
->next
) {
2176 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2177 &resp
, sizeof resp
))
2187 return ret
? ret
: in_len
;
2190 ssize_t
ib_uverbs_post_srq_recv(struct ib_uverbs_file
*file
,
2191 const char __user
*buf
, int in_len
,
2194 struct ib_uverbs_post_srq_recv cmd
;
2195 struct ib_uverbs_post_srq_recv_resp resp
;
2196 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
2198 ssize_t ret
= -EINVAL
;
2200 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2203 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
2204 in_len
- sizeof cmd
, cmd
.wr_count
,
2205 cmd
.sge_count
, cmd
.wqe_size
);
2209 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
2214 ret
= srq
->device
->post_srq_recv(srq
, wr
, &bad_wr
);
2219 for (next
= wr
; next
; next
= next
->next
) {
2225 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2226 &resp
, sizeof resp
))
2236 return ret
? ret
: in_len
;
2239 ssize_t
ib_uverbs_create_ah(struct ib_uverbs_file
*file
,
2240 const char __user
*buf
, int in_len
,
2243 struct ib_uverbs_create_ah cmd
;
2244 struct ib_uverbs_create_ah_resp resp
;
2245 struct ib_uobject
*uobj
;
2248 struct ib_ah_attr attr
;
2251 if (out_len
< sizeof resp
)
2254 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2257 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
2261 init_uobj(uobj
, cmd
.user_handle
, file
->ucontext
, &ah_lock_key
);
2262 down_write(&uobj
->mutex
);
2264 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
2270 attr
.dlid
= cmd
.attr
.dlid
;
2271 attr
.sl
= cmd
.attr
.sl
;
2272 attr
.src_path_bits
= cmd
.attr
.src_path_bits
;
2273 attr
.static_rate
= cmd
.attr
.static_rate
;
2274 attr
.ah_flags
= cmd
.attr
.is_global
? IB_AH_GRH
: 0;
2275 attr
.port_num
= cmd
.attr
.port_num
;
2276 attr
.grh
.flow_label
= cmd
.attr
.grh
.flow_label
;
2277 attr
.grh
.sgid_index
= cmd
.attr
.grh
.sgid_index
;
2278 attr
.grh
.hop_limit
= cmd
.attr
.grh
.hop_limit
;
2279 attr
.grh
.traffic_class
= cmd
.attr
.grh
.traffic_class
;
2280 memcpy(attr
.grh
.dgid
.raw
, cmd
.attr
.grh
.dgid
, 16);
2282 ah
= ib_create_ah(pd
, &attr
);
2291 ret
= idr_add_uobj(&ib_uverbs_ah_idr
, uobj
);
2295 resp
.ah_handle
= uobj
->id
;
2297 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2298 &resp
, sizeof resp
)) {
2305 mutex_lock(&file
->mutex
);
2306 list_add_tail(&uobj
->list
, &file
->ucontext
->ah_list
);
2307 mutex_unlock(&file
->mutex
);
2311 up_write(&uobj
->mutex
);
2316 idr_remove_uobj(&ib_uverbs_ah_idr
, uobj
);
2325 put_uobj_write(uobj
);
2329 ssize_t
ib_uverbs_destroy_ah(struct ib_uverbs_file
*file
,
2330 const char __user
*buf
, int in_len
, int out_len
)
2332 struct ib_uverbs_destroy_ah cmd
;
2334 struct ib_uobject
*uobj
;
2337 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2340 uobj
= idr_write_uobj(&ib_uverbs_ah_idr
, cmd
.ah_handle
, file
->ucontext
);
2345 ret
= ib_destroy_ah(ah
);
2349 put_uobj_write(uobj
);
2354 idr_remove_uobj(&ib_uverbs_ah_idr
, uobj
);
2356 mutex_lock(&file
->mutex
);
2357 list_del(&uobj
->list
);
2358 mutex_unlock(&file
->mutex
);
2365 ssize_t
ib_uverbs_attach_mcast(struct ib_uverbs_file
*file
,
2366 const char __user
*buf
, int in_len
,
2369 struct ib_uverbs_attach_mcast cmd
;
2371 struct ib_uqp_object
*obj
;
2372 struct ib_uverbs_mcast_entry
*mcast
;
2375 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2378 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2382 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2384 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2385 if (cmd
.mlid
== mcast
->lid
&&
2386 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2391 mcast
= kmalloc(sizeof *mcast
, GFP_KERNEL
);
2397 mcast
->lid
= cmd
.mlid
;
2398 memcpy(mcast
->gid
.raw
, cmd
.gid
, sizeof mcast
->gid
.raw
);
2400 ret
= ib_attach_mcast(qp
, &mcast
->gid
, cmd
.mlid
);
2402 list_add_tail(&mcast
->list
, &obj
->mcast_list
);
2409 return ret
? ret
: in_len
;
2412 ssize_t
ib_uverbs_detach_mcast(struct ib_uverbs_file
*file
,
2413 const char __user
*buf
, int in_len
,
2416 struct ib_uverbs_detach_mcast cmd
;
2417 struct ib_uqp_object
*obj
;
2419 struct ib_uverbs_mcast_entry
*mcast
;
2422 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2425 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2429 ret
= ib_detach_mcast(qp
, (union ib_gid
*) cmd
.gid
, cmd
.mlid
);
2433 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2435 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2436 if (cmd
.mlid
== mcast
->lid
&&
2437 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2438 list_del(&mcast
->list
);
2446 return ret
? ret
: in_len
;
2449 int __uverbs_create_xsrq(struct ib_uverbs_file
*file
,
2450 struct ib_uverbs_create_xsrq
*cmd
,
2451 struct ib_udata
*udata
)
2453 struct ib_uverbs_create_srq_resp resp
;
2454 struct ib_usrq_object
*obj
;
2457 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
2458 struct ib_srq_init_attr attr
;
2461 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
2465 init_uobj(&obj
->uevent
.uobject
, cmd
->user_handle
, file
->ucontext
, &srq_lock_key
);
2466 down_write(&obj
->uevent
.uobject
.mutex
);
2468 pd
= idr_read_pd(cmd
->pd_handle
, file
->ucontext
);
2474 if (cmd
->srq_type
== IB_SRQT_XRC
) {
2475 attr
.ext
.xrc
.cq
= idr_read_cq(cmd
->cq_handle
, file
->ucontext
, 0);
2476 if (!attr
.ext
.xrc
.cq
) {
2481 attr
.ext
.xrc
.xrcd
= idr_read_xrcd(cmd
->xrcd_handle
, file
->ucontext
, &xrcd_uobj
);
2482 if (!attr
.ext
.xrc
.xrcd
) {
2487 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
, uobject
);
2488 atomic_inc(&obj
->uxrcd
->refcnt
);
2491 attr
.event_handler
= ib_uverbs_srq_event_handler
;
2492 attr
.srq_context
= file
;
2493 attr
.srq_type
= cmd
->srq_type
;
2494 attr
.attr
.max_wr
= cmd
->max_wr
;
2495 attr
.attr
.max_sge
= cmd
->max_sge
;
2496 attr
.attr
.srq_limit
= cmd
->srq_limit
;
2498 obj
->uevent
.events_reported
= 0;
2499 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
2501 srq
= pd
->device
->create_srq(pd
, &attr
, udata
);
2507 srq
->device
= pd
->device
;
2509 srq
->srq_type
= cmd
->srq_type
;
2510 srq
->uobject
= &obj
->uevent
.uobject
;
2511 srq
->event_handler
= attr
.event_handler
;
2512 srq
->srq_context
= attr
.srq_context
;
2514 if (cmd
->srq_type
== IB_SRQT_XRC
) {
2515 srq
->ext
.xrc
.cq
= attr
.ext
.xrc
.cq
;
2516 srq
->ext
.xrc
.xrcd
= attr
.ext
.xrc
.xrcd
;
2517 atomic_inc(&attr
.ext
.xrc
.cq
->usecnt
);
2518 atomic_inc(&attr
.ext
.xrc
.xrcd
->usecnt
);
2521 atomic_inc(&pd
->usecnt
);
2522 atomic_set(&srq
->usecnt
, 0);
2524 obj
->uevent
.uobject
.object
= srq
;
2525 ret
= idr_add_uobj(&ib_uverbs_srq_idr
, &obj
->uevent
.uobject
);
2529 memset(&resp
, 0, sizeof resp
);
2530 resp
.srq_handle
= obj
->uevent
.uobject
.id
;
2531 resp
.max_wr
= attr
.attr
.max_wr
;
2532 resp
.max_sge
= attr
.attr
.max_sge
;
2533 if (cmd
->srq_type
== IB_SRQT_XRC
)
2534 resp
.srqn
= srq
->ext
.xrc
.srq_num
;
2536 if (copy_to_user((void __user
*) (unsigned long) cmd
->response
,
2537 &resp
, sizeof resp
)) {
2542 if (cmd
->srq_type
== IB_SRQT_XRC
) {
2543 put_uobj_read(xrcd_uobj
);
2544 put_cq_read(attr
.ext
.xrc
.cq
);
2548 mutex_lock(&file
->mutex
);
2549 list_add_tail(&obj
->uevent
.uobject
.list
, &file
->ucontext
->srq_list
);
2550 mutex_unlock(&file
->mutex
);
2552 obj
->uevent
.uobject
.live
= 1;
2554 up_write(&obj
->uevent
.uobject
.mutex
);
2559 idr_remove_uobj(&ib_uverbs_srq_idr
, &obj
->uevent
.uobject
);
2562 ib_destroy_srq(srq
);
2565 if (cmd
->srq_type
== IB_SRQT_XRC
) {
2566 atomic_dec(&obj
->uxrcd
->refcnt
);
2567 put_uobj_read(xrcd_uobj
);
2571 if (cmd
->srq_type
== IB_SRQT_XRC
)
2572 put_cq_read(attr
.ext
.xrc
.cq
);
2578 put_uobj_write(&obj
->uevent
.uobject
);
2582 ssize_t
ib_uverbs_create_srq(struct ib_uverbs_file
*file
,
2583 const char __user
*buf
, int in_len
,
2586 struct ib_uverbs_create_srq cmd
;
2587 struct ib_uverbs_create_xsrq xcmd
;
2588 struct ib_uverbs_create_srq_resp resp
;
2589 struct ib_udata udata
;
2592 if (out_len
< sizeof resp
)
2595 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2598 xcmd
.response
= cmd
.response
;
2599 xcmd
.user_handle
= cmd
.user_handle
;
2600 xcmd
.srq_type
= IB_SRQT_BASIC
;
2601 xcmd
.pd_handle
= cmd
.pd_handle
;
2602 xcmd
.max_wr
= cmd
.max_wr
;
2603 xcmd
.max_sge
= cmd
.max_sge
;
2604 xcmd
.srq_limit
= cmd
.srq_limit
;
2606 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
2607 (unsigned long) cmd
.response
+ sizeof resp
,
2608 in_len
- sizeof cmd
, out_len
- sizeof resp
);
2610 ret
= __uverbs_create_xsrq(file
, &xcmd
, &udata
);
2617 ssize_t
ib_uverbs_create_xsrq(struct ib_uverbs_file
*file
,
2618 const char __user
*buf
, int in_len
, int out_len
)
2620 struct ib_uverbs_create_xsrq cmd
;
2621 struct ib_uverbs_create_srq_resp resp
;
2622 struct ib_udata udata
;
2625 if (out_len
< sizeof resp
)
2628 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2631 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
2632 (unsigned long) cmd
.response
+ sizeof resp
,
2633 in_len
- sizeof cmd
, out_len
- sizeof resp
);
2635 ret
= __uverbs_create_xsrq(file
, &cmd
, &udata
);
2642 ssize_t
ib_uverbs_modify_srq(struct ib_uverbs_file
*file
,
2643 const char __user
*buf
, int in_len
,
2646 struct ib_uverbs_modify_srq cmd
;
2647 struct ib_udata udata
;
2649 struct ib_srq_attr attr
;
2652 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2655 INIT_UDATA(&udata
, buf
+ sizeof cmd
, NULL
, in_len
- sizeof cmd
,
2658 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
2662 attr
.max_wr
= cmd
.max_wr
;
2663 attr
.srq_limit
= cmd
.srq_limit
;
2665 ret
= srq
->device
->modify_srq(srq
, &attr
, cmd
.attr_mask
, &udata
);
2669 return ret
? ret
: in_len
;
2672 ssize_t
ib_uverbs_query_srq(struct ib_uverbs_file
*file
,
2673 const char __user
*buf
,
2674 int in_len
, int out_len
)
2676 struct ib_uverbs_query_srq cmd
;
2677 struct ib_uverbs_query_srq_resp resp
;
2678 struct ib_srq_attr attr
;
2682 if (out_len
< sizeof resp
)
2685 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2688 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
2692 ret
= ib_query_srq(srq
, &attr
);
2699 memset(&resp
, 0, sizeof resp
);
2701 resp
.max_wr
= attr
.max_wr
;
2702 resp
.max_sge
= attr
.max_sge
;
2703 resp
.srq_limit
= attr
.srq_limit
;
2705 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2706 &resp
, sizeof resp
))
2712 ssize_t
ib_uverbs_destroy_srq(struct ib_uverbs_file
*file
,
2713 const char __user
*buf
, int in_len
,
2716 struct ib_uverbs_destroy_srq cmd
;
2717 struct ib_uverbs_destroy_srq_resp resp
;
2718 struct ib_uobject
*uobj
;
2720 struct ib_uevent_object
*obj
;
2723 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2726 uobj
= idr_write_uobj(&ib_uverbs_srq_idr
, cmd
.srq_handle
, file
->ucontext
);
2730 obj
= container_of(uobj
, struct ib_uevent_object
, uobject
);
2732 ret
= ib_destroy_srq(srq
);
2736 put_uobj_write(uobj
);
2741 idr_remove_uobj(&ib_uverbs_srq_idr
, uobj
);
2743 mutex_lock(&file
->mutex
);
2744 list_del(&uobj
->list
);
2745 mutex_unlock(&file
->mutex
);
2747 ib_uverbs_release_uevent(file
, obj
);
2749 memset(&resp
, 0, sizeof resp
);
2750 resp
.events_reported
= obj
->events_reported
;
2754 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2755 &resp
, sizeof resp
))
2758 return ret
? ret
: in_len
;