2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/file.h>
38 #include <linux/slab.h>
39 #include <linux/sched.h>
41 #include <asm/uaccess.h>
44 #include "core_priv.h"
46 struct uverbs_lock_class
{
47 struct lock_class_key key
;
51 static struct uverbs_lock_class pd_lock_class
= { .name
= "PD-uobj" };
52 static struct uverbs_lock_class mr_lock_class
= { .name
= "MR-uobj" };
53 static struct uverbs_lock_class mw_lock_class
= { .name
= "MW-uobj" };
54 static struct uverbs_lock_class cq_lock_class
= { .name
= "CQ-uobj" };
55 static struct uverbs_lock_class qp_lock_class
= { .name
= "QP-uobj" };
56 static struct uverbs_lock_class ah_lock_class
= { .name
= "AH-uobj" };
57 static struct uverbs_lock_class srq_lock_class
= { .name
= "SRQ-uobj" };
58 static struct uverbs_lock_class xrcd_lock_class
= { .name
= "XRCD-uobj" };
59 static struct uverbs_lock_class rule_lock_class
= { .name
= "RULE-uobj" };
62 * The ib_uobject locking scheme is as follows:
64 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it
65 * needs to be held during all idr operations. When an object is
66 * looked up, a reference must be taken on the object's kref before
69 * - Each object also has an rwsem. This rwsem must be held for
70 * reading while an operation that uses the object is performed.
71 * For example, while registering an MR, the associated PD's
72 * uobject.mutex must be held for reading. The rwsem must be held
73 * for writing while initializing or destroying an object.
75 * - In addition, each object has a "live" flag. If this flag is not
76 * set, then lookups of the object will fail even if it is found in
77 * the idr. This handles a reader that blocks and does not acquire
78 * the rwsem until after the object is destroyed. The destroy
79 * operation will set the live flag to 0 and then drop the rwsem;
80 * this will allow the reader to acquire the rwsem, see that the
81 * live flag is 0, and then drop the rwsem and its reference to
82 * object. The underlying storage will not be freed until the last
83 * reference to the object is dropped.
86 static void init_uobj(struct ib_uobject
*uobj
, u64 user_handle
,
87 struct ib_ucontext
*context
, struct uverbs_lock_class
*c
)
89 uobj
->user_handle
= user_handle
;
90 uobj
->context
= context
;
91 kref_init(&uobj
->ref
);
92 init_rwsem(&uobj
->mutex
);
93 lockdep_set_class_and_name(&uobj
->mutex
, &c
->key
, c
->name
);
97 static void release_uobj(struct kref
*kref
)
99 kfree(container_of(kref
, struct ib_uobject
, ref
));
102 static void put_uobj(struct ib_uobject
*uobj
)
104 kref_put(&uobj
->ref
, release_uobj
);
107 static void put_uobj_read(struct ib_uobject
*uobj
)
109 up_read(&uobj
->mutex
);
113 static void put_uobj_write(struct ib_uobject
*uobj
)
115 up_write(&uobj
->mutex
);
119 static int idr_add_uobj(struct idr
*idr
, struct ib_uobject
*uobj
)
123 idr_preload(GFP_KERNEL
);
124 spin_lock(&ib_uverbs_idr_lock
);
126 ret
= idr_alloc(idr
, uobj
, 0, 0, GFP_NOWAIT
);
130 spin_unlock(&ib_uverbs_idr_lock
);
133 return ret
< 0 ? ret
: 0;
136 void idr_remove_uobj(struct idr
*idr
, struct ib_uobject
*uobj
)
138 spin_lock(&ib_uverbs_idr_lock
);
139 idr_remove(idr
, uobj
->id
);
140 spin_unlock(&ib_uverbs_idr_lock
);
143 static struct ib_uobject
*__idr_get_uobj(struct idr
*idr
, int id
,
144 struct ib_ucontext
*context
)
146 struct ib_uobject
*uobj
;
148 spin_lock(&ib_uverbs_idr_lock
);
149 uobj
= idr_find(idr
, id
);
151 if (uobj
->context
== context
)
152 kref_get(&uobj
->ref
);
156 spin_unlock(&ib_uverbs_idr_lock
);
161 static struct ib_uobject
*idr_read_uobj(struct idr
*idr
, int id
,
162 struct ib_ucontext
*context
, int nested
)
164 struct ib_uobject
*uobj
;
166 uobj
= __idr_get_uobj(idr
, id
, context
);
171 down_read_nested(&uobj
->mutex
, SINGLE_DEPTH_NESTING
);
173 down_read(&uobj
->mutex
);
182 static struct ib_uobject
*idr_write_uobj(struct idr
*idr
, int id
,
183 struct ib_ucontext
*context
)
185 struct ib_uobject
*uobj
;
187 uobj
= __idr_get_uobj(idr
, id
, context
);
191 down_write(&uobj
->mutex
);
193 put_uobj_write(uobj
);
200 static void *idr_read_obj(struct idr
*idr
, int id
, struct ib_ucontext
*context
,
203 struct ib_uobject
*uobj
;
205 uobj
= idr_read_uobj(idr
, id
, context
, nested
);
206 return uobj
? uobj
->object
: NULL
;
209 static struct ib_pd
*idr_read_pd(int pd_handle
, struct ib_ucontext
*context
)
211 return idr_read_obj(&ib_uverbs_pd_idr
, pd_handle
, context
, 0);
214 static void put_pd_read(struct ib_pd
*pd
)
216 put_uobj_read(pd
->uobject
);
219 static struct ib_cq
*idr_read_cq(int cq_handle
, struct ib_ucontext
*context
, int nested
)
221 return idr_read_obj(&ib_uverbs_cq_idr
, cq_handle
, context
, nested
);
224 static void put_cq_read(struct ib_cq
*cq
)
226 put_uobj_read(cq
->uobject
);
229 static struct ib_ah
*idr_read_ah(int ah_handle
, struct ib_ucontext
*context
)
231 return idr_read_obj(&ib_uverbs_ah_idr
, ah_handle
, context
, 0);
234 static void put_ah_read(struct ib_ah
*ah
)
236 put_uobj_read(ah
->uobject
);
239 static struct ib_qp
*idr_read_qp(int qp_handle
, struct ib_ucontext
*context
)
241 return idr_read_obj(&ib_uverbs_qp_idr
, qp_handle
, context
, 0);
244 static struct ib_qp
*idr_write_qp(int qp_handle
, struct ib_ucontext
*context
)
246 struct ib_uobject
*uobj
;
248 uobj
= idr_write_uobj(&ib_uverbs_qp_idr
, qp_handle
, context
);
249 return uobj
? uobj
->object
: NULL
;
252 static void put_qp_read(struct ib_qp
*qp
)
254 put_uobj_read(qp
->uobject
);
257 static void put_qp_write(struct ib_qp
*qp
)
259 put_uobj_write(qp
->uobject
);
262 static struct ib_srq
*idr_read_srq(int srq_handle
, struct ib_ucontext
*context
)
264 return idr_read_obj(&ib_uverbs_srq_idr
, srq_handle
, context
, 0);
267 static void put_srq_read(struct ib_srq
*srq
)
269 put_uobj_read(srq
->uobject
);
272 static struct ib_xrcd
*idr_read_xrcd(int xrcd_handle
, struct ib_ucontext
*context
,
273 struct ib_uobject
**uobj
)
275 *uobj
= idr_read_uobj(&ib_uverbs_xrcd_idr
, xrcd_handle
, context
, 0);
276 return *uobj
? (*uobj
)->object
: NULL
;
279 static void put_xrcd_read(struct ib_uobject
*uobj
)
284 ssize_t
ib_uverbs_get_context(struct ib_uverbs_file
*file
,
285 const char __user
*buf
,
286 int in_len
, int out_len
)
288 struct ib_uverbs_get_context cmd
;
289 struct ib_uverbs_get_context_resp resp
;
290 struct ib_udata udata
;
291 struct ib_device
*ibdev
= file
->device
->ib_dev
;
292 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
293 struct ib_device_attr dev_attr
;
295 struct ib_ucontext
*ucontext
;
299 if (out_len
< sizeof resp
)
302 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
305 mutex_lock(&file
->mutex
);
307 if (file
->ucontext
) {
312 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
313 (unsigned long) cmd
.response
+ sizeof resp
,
314 in_len
- sizeof cmd
, out_len
- sizeof resp
);
316 ucontext
= ibdev
->alloc_ucontext(ibdev
, &udata
);
317 if (IS_ERR(ucontext
)) {
318 ret
= PTR_ERR(ucontext
);
322 ucontext
->device
= ibdev
;
323 INIT_LIST_HEAD(&ucontext
->pd_list
);
324 INIT_LIST_HEAD(&ucontext
->mr_list
);
325 INIT_LIST_HEAD(&ucontext
->mw_list
);
326 INIT_LIST_HEAD(&ucontext
->cq_list
);
327 INIT_LIST_HEAD(&ucontext
->qp_list
);
328 INIT_LIST_HEAD(&ucontext
->srq_list
);
329 INIT_LIST_HEAD(&ucontext
->ah_list
);
330 INIT_LIST_HEAD(&ucontext
->xrcd_list
);
331 INIT_LIST_HEAD(&ucontext
->rule_list
);
333 ucontext
->tgid
= get_task_pid(current
->group_leader
, PIDTYPE_PID
);
335 ucontext
->closing
= 0;
337 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
338 ucontext
->umem_tree
= RB_ROOT
;
339 init_rwsem(&ucontext
->umem_rwsem
);
340 ucontext
->odp_mrs_count
= 0;
341 INIT_LIST_HEAD(&ucontext
->no_private_counters
);
343 ret
= ib_query_device(ibdev
, &dev_attr
);
346 if (!(dev_attr
.device_cap_flags
& IB_DEVICE_ON_DEMAND_PAGING
))
347 ucontext
->invalidate_range
= NULL
;
351 resp
.num_comp_vectors
= file
->device
->num_comp_vectors
;
353 ret
= get_unused_fd_flags(O_CLOEXEC
);
358 filp
= ib_uverbs_alloc_event_file(file
, 1);
364 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
365 &resp
, sizeof resp
)) {
370 file
->async_file
= filp
->private_data
;
372 INIT_IB_EVENT_HANDLER(&file
->event_handler
, file
->device
->ib_dev
,
373 ib_uverbs_event_handler
);
374 ret
= ib_register_event_handler(&file
->event_handler
);
378 kref_get(&file
->async_file
->ref
);
379 kref_get(&file
->ref
);
380 file
->ucontext
= ucontext
;
382 fd_install(resp
.async_fd
, filp
);
384 mutex_unlock(&file
->mutex
);
392 put_unused_fd(resp
.async_fd
);
395 put_pid(ucontext
->tgid
);
396 ibdev
->dealloc_ucontext(ucontext
);
399 mutex_unlock(&file
->mutex
);
403 static void copy_query_dev_fields(struct ib_uverbs_file
*file
,
404 struct ib_uverbs_query_device_resp
*resp
,
405 struct ib_device_attr
*attr
)
407 resp
->fw_ver
= attr
->fw_ver
;
408 resp
->node_guid
= file
->device
->ib_dev
->node_guid
;
409 resp
->sys_image_guid
= attr
->sys_image_guid
;
410 resp
->max_mr_size
= attr
->max_mr_size
;
411 resp
->page_size_cap
= attr
->page_size_cap
;
412 resp
->vendor_id
= attr
->vendor_id
;
413 resp
->vendor_part_id
= attr
->vendor_part_id
;
414 resp
->hw_ver
= attr
->hw_ver
;
415 resp
->max_qp
= attr
->max_qp
;
416 resp
->max_qp_wr
= attr
->max_qp_wr
;
417 resp
->device_cap_flags
= attr
->device_cap_flags
;
418 resp
->max_sge
= attr
->max_sge
;
419 resp
->max_sge_rd
= attr
->max_sge_rd
;
420 resp
->max_cq
= attr
->max_cq
;
421 resp
->max_cqe
= attr
->max_cqe
;
422 resp
->max_mr
= attr
->max_mr
;
423 resp
->max_pd
= attr
->max_pd
;
424 resp
->max_qp_rd_atom
= attr
->max_qp_rd_atom
;
425 resp
->max_ee_rd_atom
= attr
->max_ee_rd_atom
;
426 resp
->max_res_rd_atom
= attr
->max_res_rd_atom
;
427 resp
->max_qp_init_rd_atom
= attr
->max_qp_init_rd_atom
;
428 resp
->max_ee_init_rd_atom
= attr
->max_ee_init_rd_atom
;
429 resp
->atomic_cap
= attr
->atomic_cap
;
430 resp
->max_ee
= attr
->max_ee
;
431 resp
->max_rdd
= attr
->max_rdd
;
432 resp
->max_mw
= attr
->max_mw
;
433 resp
->max_raw_ipv6_qp
= attr
->max_raw_ipv6_qp
;
434 resp
->max_raw_ethy_qp
= attr
->max_raw_ethy_qp
;
435 resp
->max_mcast_grp
= attr
->max_mcast_grp
;
436 resp
->max_mcast_qp_attach
= attr
->max_mcast_qp_attach
;
437 resp
->max_total_mcast_qp_attach
= attr
->max_total_mcast_qp_attach
;
438 resp
->max_ah
= attr
->max_ah
;
439 resp
->max_fmr
= attr
->max_fmr
;
440 resp
->max_map_per_fmr
= attr
->max_map_per_fmr
;
441 resp
->max_srq
= attr
->max_srq
;
442 resp
->max_srq_wr
= attr
->max_srq_wr
;
443 resp
->max_srq_sge
= attr
->max_srq_sge
;
444 resp
->max_pkeys
= attr
->max_pkeys
;
445 resp
->local_ca_ack_delay
= attr
->local_ca_ack_delay
;
446 resp
->phys_port_cnt
= file
->device
->ib_dev
->phys_port_cnt
;
449 ssize_t
ib_uverbs_query_device(struct ib_uverbs_file
*file
,
450 const char __user
*buf
,
451 int in_len
, int out_len
)
453 struct ib_uverbs_query_device cmd
;
454 struct ib_uverbs_query_device_resp resp
;
455 struct ib_device_attr attr
;
458 if (out_len
< sizeof resp
)
461 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
464 ret
= ib_query_device(file
->device
->ib_dev
, &attr
);
468 memset(&resp
, 0, sizeof resp
);
469 copy_query_dev_fields(file
, &resp
, &attr
);
471 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
478 ssize_t
ib_uverbs_query_port(struct ib_uverbs_file
*file
,
479 const char __user
*buf
,
480 int in_len
, int out_len
)
482 struct ib_uverbs_query_port cmd
;
483 struct ib_uverbs_query_port_resp resp
;
484 struct ib_port_attr attr
;
487 if (out_len
< sizeof resp
)
490 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
493 ret
= ib_query_port(file
->device
->ib_dev
, cmd
.port_num
, &attr
);
497 memset(&resp
, 0, sizeof resp
);
499 resp
.state
= attr
.state
;
500 resp
.max_mtu
= attr
.max_mtu
;
501 resp
.active_mtu
= attr
.active_mtu
;
502 resp
.gid_tbl_len
= attr
.gid_tbl_len
;
503 resp
.port_cap_flags
= attr
.port_cap_flags
;
504 resp
.max_msg_sz
= attr
.max_msg_sz
;
505 resp
.bad_pkey_cntr
= attr
.bad_pkey_cntr
;
506 resp
.qkey_viol_cntr
= attr
.qkey_viol_cntr
;
507 resp
.pkey_tbl_len
= attr
.pkey_tbl_len
;
509 resp
.sm_lid
= attr
.sm_lid
;
511 resp
.max_vl_num
= attr
.max_vl_num
;
512 resp
.sm_sl
= attr
.sm_sl
;
513 resp
.subnet_timeout
= attr
.subnet_timeout
;
514 resp
.init_type_reply
= attr
.init_type_reply
;
515 resp
.active_width
= attr
.active_width
;
516 resp
.active_speed
= attr
.active_speed
;
517 resp
.phys_state
= attr
.phys_state
;
518 resp
.link_layer
= rdma_port_get_link_layer(file
->device
->ib_dev
,
521 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
528 ssize_t
ib_uverbs_alloc_pd(struct ib_uverbs_file
*file
,
529 const char __user
*buf
,
530 int in_len
, int out_len
)
532 struct ib_uverbs_alloc_pd cmd
;
533 struct ib_uverbs_alloc_pd_resp resp
;
534 struct ib_udata udata
;
535 struct ib_uobject
*uobj
;
539 if (out_len
< sizeof resp
)
542 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
545 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
546 (unsigned long) cmd
.response
+ sizeof resp
,
547 in_len
- sizeof cmd
, out_len
- sizeof resp
);
549 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
553 init_uobj(uobj
, 0, file
->ucontext
, &pd_lock_class
);
554 down_write(&uobj
->mutex
);
556 pd
= file
->device
->ib_dev
->alloc_pd(file
->device
->ib_dev
,
557 file
->ucontext
, &udata
);
563 pd
->device
= file
->device
->ib_dev
;
565 atomic_set(&pd
->usecnt
, 0);
568 ret
= idr_add_uobj(&ib_uverbs_pd_idr
, uobj
);
572 memset(&resp
, 0, sizeof resp
);
573 resp
.pd_handle
= uobj
->id
;
575 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
576 &resp
, sizeof resp
)) {
581 mutex_lock(&file
->mutex
);
582 list_add_tail(&uobj
->list
, &file
->ucontext
->pd_list
);
583 mutex_unlock(&file
->mutex
);
587 up_write(&uobj
->mutex
);
592 idr_remove_uobj(&ib_uverbs_pd_idr
, uobj
);
598 put_uobj_write(uobj
);
602 ssize_t
ib_uverbs_dealloc_pd(struct ib_uverbs_file
*file
,
603 const char __user
*buf
,
604 int in_len
, int out_len
)
606 struct ib_uverbs_dealloc_pd cmd
;
607 struct ib_uobject
*uobj
;
610 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
613 uobj
= idr_write_uobj(&ib_uverbs_pd_idr
, cmd
.pd_handle
, file
->ucontext
);
617 ret
= ib_dealloc_pd(uobj
->object
);
621 put_uobj_write(uobj
);
626 idr_remove_uobj(&ib_uverbs_pd_idr
, uobj
);
628 mutex_lock(&file
->mutex
);
629 list_del(&uobj
->list
);
630 mutex_unlock(&file
->mutex
);
637 struct xrcd_table_entry
{
639 struct ib_xrcd
*xrcd
;
643 static int xrcd_table_insert(struct ib_uverbs_device
*dev
,
645 struct ib_xrcd
*xrcd
)
647 struct xrcd_table_entry
*entry
, *scan
;
648 struct rb_node
**p
= &dev
->xrcd_tree
.rb_node
;
649 struct rb_node
*parent
= NULL
;
651 entry
= kmalloc(sizeof *entry
, GFP_KERNEL
);
656 entry
->inode
= inode
;
660 scan
= rb_entry(parent
, struct xrcd_table_entry
, node
);
662 if (inode
< scan
->inode
) {
664 } else if (inode
> scan
->inode
) {
672 rb_link_node(&entry
->node
, parent
, p
);
673 rb_insert_color(&entry
->node
, &dev
->xrcd_tree
);
678 static struct xrcd_table_entry
*xrcd_table_search(struct ib_uverbs_device
*dev
,
681 struct xrcd_table_entry
*entry
;
682 struct rb_node
*p
= dev
->xrcd_tree
.rb_node
;
685 entry
= rb_entry(p
, struct xrcd_table_entry
, node
);
687 if (inode
< entry
->inode
)
689 else if (inode
> entry
->inode
)
698 static struct ib_xrcd
*find_xrcd(struct ib_uverbs_device
*dev
, struct inode
*inode
)
700 struct xrcd_table_entry
*entry
;
702 entry
= xrcd_table_search(dev
, inode
);
709 static void xrcd_table_delete(struct ib_uverbs_device
*dev
,
712 struct xrcd_table_entry
*entry
;
714 entry
= xrcd_table_search(dev
, inode
);
717 rb_erase(&entry
->node
, &dev
->xrcd_tree
);
722 ssize_t
ib_uverbs_open_xrcd(struct ib_uverbs_file
*file
,
723 const char __user
*buf
, int in_len
,
726 struct ib_uverbs_open_xrcd cmd
;
727 struct ib_uverbs_open_xrcd_resp resp
;
728 struct ib_udata udata
;
729 struct ib_uxrcd_object
*obj
;
730 struct ib_xrcd
*xrcd
= NULL
;
731 struct fd f
= {NULL
, 0};
732 struct inode
*inode
= NULL
;
736 if (out_len
< sizeof resp
)
739 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
742 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
743 (unsigned long) cmd
.response
+ sizeof resp
,
744 in_len
- sizeof cmd
, out_len
- sizeof resp
);
746 mutex_lock(&file
->device
->xrcd_tree_mutex
);
749 /* search for file descriptor */
753 goto err_tree_mutex_unlock
;
756 inode
= file_inode(f
.file
);
757 xrcd
= find_xrcd(file
->device
, inode
);
758 if (!xrcd
&& !(cmd
.oflags
& O_CREAT
)) {
759 /* no file descriptor. Need CREATE flag */
761 goto err_tree_mutex_unlock
;
764 if (xrcd
&& cmd
.oflags
& O_EXCL
) {
766 goto err_tree_mutex_unlock
;
770 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
773 goto err_tree_mutex_unlock
;
776 init_uobj(&obj
->uobject
, 0, file
->ucontext
, &xrcd_lock_class
);
778 down_write(&obj
->uobject
.mutex
);
781 xrcd
= file
->device
->ib_dev
->alloc_xrcd(file
->device
->ib_dev
,
782 file
->ucontext
, &udata
);
789 xrcd
->device
= file
->device
->ib_dev
;
790 atomic_set(&xrcd
->usecnt
, 0);
791 mutex_init(&xrcd
->tgt_qp_mutex
);
792 INIT_LIST_HEAD(&xrcd
->tgt_qp_list
);
796 atomic_set(&obj
->refcnt
, 0);
797 obj
->uobject
.object
= xrcd
;
798 ret
= idr_add_uobj(&ib_uverbs_xrcd_idr
, &obj
->uobject
);
802 memset(&resp
, 0, sizeof resp
);
803 resp
.xrcd_handle
= obj
->uobject
.id
;
807 /* create new inode/xrcd table entry */
808 ret
= xrcd_table_insert(file
->device
, inode
, xrcd
);
810 goto err_insert_xrcd
;
812 atomic_inc(&xrcd
->usecnt
);
815 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
816 &resp
, sizeof resp
)) {
824 mutex_lock(&file
->mutex
);
825 list_add_tail(&obj
->uobject
.list
, &file
->ucontext
->xrcd_list
);
826 mutex_unlock(&file
->mutex
);
828 obj
->uobject
.live
= 1;
829 up_write(&obj
->uobject
.mutex
);
831 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
837 xrcd_table_delete(file
->device
, inode
);
838 atomic_dec(&xrcd
->usecnt
);
842 idr_remove_uobj(&ib_uverbs_xrcd_idr
, &obj
->uobject
);
845 ib_dealloc_xrcd(xrcd
);
848 put_uobj_write(&obj
->uobject
);
850 err_tree_mutex_unlock
:
854 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
859 ssize_t
ib_uverbs_close_xrcd(struct ib_uverbs_file
*file
,
860 const char __user
*buf
, int in_len
,
863 struct ib_uverbs_close_xrcd cmd
;
864 struct ib_uobject
*uobj
;
865 struct ib_xrcd
*xrcd
= NULL
;
866 struct inode
*inode
= NULL
;
867 struct ib_uxrcd_object
*obj
;
871 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
874 mutex_lock(&file
->device
->xrcd_tree_mutex
);
875 uobj
= idr_write_uobj(&ib_uverbs_xrcd_idr
, cmd
.xrcd_handle
, file
->ucontext
);
883 obj
= container_of(uobj
, struct ib_uxrcd_object
, uobject
);
884 if (atomic_read(&obj
->refcnt
)) {
885 put_uobj_write(uobj
);
890 if (!inode
|| atomic_dec_and_test(&xrcd
->usecnt
)) {
891 ret
= ib_dealloc_xrcd(uobj
->object
);
898 atomic_inc(&xrcd
->usecnt
);
900 put_uobj_write(uobj
);
906 xrcd_table_delete(file
->device
, inode
);
908 idr_remove_uobj(&ib_uverbs_xrcd_idr
, uobj
);
909 mutex_lock(&file
->mutex
);
910 list_del(&uobj
->list
);
911 mutex_unlock(&file
->mutex
);
917 mutex_unlock(&file
->device
->xrcd_tree_mutex
);
921 void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device
*dev
,
922 struct ib_xrcd
*xrcd
)
927 if (inode
&& !atomic_dec_and_test(&xrcd
->usecnt
))
930 ib_dealloc_xrcd(xrcd
);
933 xrcd_table_delete(dev
, inode
);
936 ssize_t
ib_uverbs_reg_mr(struct ib_uverbs_file
*file
,
937 const char __user
*buf
, int in_len
,
940 struct ib_uverbs_reg_mr cmd
;
941 struct ib_uverbs_reg_mr_resp resp
;
942 struct ib_udata udata
;
943 struct ib_uobject
*uobj
;
948 if (out_len
< sizeof resp
)
951 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
954 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
955 (unsigned long) cmd
.response
+ sizeof resp
,
956 in_len
- sizeof cmd
, out_len
- sizeof resp
);
958 if ((cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
))
961 ret
= ib_check_mr_access(cmd
.access_flags
);
965 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
969 init_uobj(uobj
, 0, file
->ucontext
, &mr_lock_class
);
970 down_write(&uobj
->mutex
);
972 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
978 if (cmd
.access_flags
& IB_ACCESS_ON_DEMAND
) {
979 struct ib_device_attr attr
;
981 ret
= ib_query_device(pd
->device
, &attr
);
982 if (ret
|| !(attr
.device_cap_flags
&
983 IB_DEVICE_ON_DEMAND_PAGING
)) {
984 pr_debug("ODP support not available\n");
990 mr
= pd
->device
->reg_user_mr(pd
, cmd
.start
, cmd
.length
, cmd
.hca_va
,
991 cmd
.access_flags
, &udata
);
997 mr
->device
= pd
->device
;
1000 atomic_inc(&pd
->usecnt
);
1001 atomic_set(&mr
->usecnt
, 0);
1004 ret
= idr_add_uobj(&ib_uverbs_mr_idr
, uobj
);
1008 memset(&resp
, 0, sizeof resp
);
1009 resp
.lkey
= mr
->lkey
;
1010 resp
.rkey
= mr
->rkey
;
1011 resp
.mr_handle
= uobj
->id
;
1013 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1014 &resp
, sizeof resp
)) {
1021 mutex_lock(&file
->mutex
);
1022 list_add_tail(&uobj
->list
, &file
->ucontext
->mr_list
);
1023 mutex_unlock(&file
->mutex
);
1027 up_write(&uobj
->mutex
);
1032 idr_remove_uobj(&ib_uverbs_mr_idr
, uobj
);
1041 put_uobj_write(uobj
);
1045 ssize_t
ib_uverbs_rereg_mr(struct ib_uverbs_file
*file
,
1046 const char __user
*buf
, int in_len
,
1049 struct ib_uverbs_rereg_mr cmd
;
1050 struct ib_uverbs_rereg_mr_resp resp
;
1051 struct ib_udata udata
;
1052 struct ib_pd
*pd
= NULL
;
1054 struct ib_pd
*old_pd
;
1056 struct ib_uobject
*uobj
;
1058 if (out_len
< sizeof(resp
))
1061 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1064 INIT_UDATA(&udata
, buf
+ sizeof(cmd
),
1065 (unsigned long) cmd
.response
+ sizeof(resp
),
1066 in_len
- sizeof(cmd
), out_len
- sizeof(resp
));
1068 if (cmd
.flags
& ~IB_MR_REREG_SUPPORTED
|| !cmd
.flags
)
1071 if ((cmd
.flags
& IB_MR_REREG_TRANS
) &&
1072 (!cmd
.start
|| !cmd
.hca_va
|| 0 >= cmd
.length
||
1073 (cmd
.start
& ~PAGE_MASK
) != (cmd
.hca_va
& ~PAGE_MASK
)))
1076 uobj
= idr_write_uobj(&ib_uverbs_mr_idr
, cmd
.mr_handle
,
1084 if (cmd
.flags
& IB_MR_REREG_ACCESS
) {
1085 ret
= ib_check_mr_access(cmd
.access_flags
);
1090 if (cmd
.flags
& IB_MR_REREG_PD
) {
1091 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
1098 if (atomic_read(&mr
->usecnt
)) {
1104 ret
= mr
->device
->rereg_user_mr(mr
, cmd
.flags
, cmd
.start
,
1105 cmd
.length
, cmd
.hca_va
,
1106 cmd
.access_flags
, pd
, &udata
);
1108 if (cmd
.flags
& IB_MR_REREG_PD
) {
1109 atomic_inc(&pd
->usecnt
);
1111 atomic_dec(&old_pd
->usecnt
);
1117 memset(&resp
, 0, sizeof(resp
));
1118 resp
.lkey
= mr
->lkey
;
1119 resp
.rkey
= mr
->rkey
;
1121 if (copy_to_user((void __user
*)(unsigned long)cmd
.response
,
1122 &resp
, sizeof(resp
)))
1128 if (cmd
.flags
& IB_MR_REREG_PD
)
1133 put_uobj_write(mr
->uobject
);
1138 ssize_t
ib_uverbs_dereg_mr(struct ib_uverbs_file
*file
,
1139 const char __user
*buf
, int in_len
,
1142 struct ib_uverbs_dereg_mr cmd
;
1144 struct ib_uobject
*uobj
;
1147 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1150 uobj
= idr_write_uobj(&ib_uverbs_mr_idr
, cmd
.mr_handle
, file
->ucontext
);
1156 ret
= ib_dereg_mr(mr
);
1160 put_uobj_write(uobj
);
1165 idr_remove_uobj(&ib_uverbs_mr_idr
, uobj
);
1167 mutex_lock(&file
->mutex
);
1168 list_del(&uobj
->list
);
1169 mutex_unlock(&file
->mutex
);
1176 ssize_t
ib_uverbs_alloc_mw(struct ib_uverbs_file
*file
,
1177 const char __user
*buf
, int in_len
,
1180 struct ib_uverbs_alloc_mw cmd
;
1181 struct ib_uverbs_alloc_mw_resp resp
;
1182 struct ib_uobject
*uobj
;
1187 if (out_len
< sizeof(resp
))
1190 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1193 uobj
= kmalloc(sizeof(*uobj
), GFP_KERNEL
);
1197 init_uobj(uobj
, 0, file
->ucontext
, &mw_lock_class
);
1198 down_write(&uobj
->mutex
);
1200 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
1206 mw
= pd
->device
->alloc_mw(pd
, cmd
.mw_type
);
1212 mw
->device
= pd
->device
;
1215 atomic_inc(&pd
->usecnt
);
1218 ret
= idr_add_uobj(&ib_uverbs_mw_idr
, uobj
);
1222 memset(&resp
, 0, sizeof(resp
));
1223 resp
.rkey
= mw
->rkey
;
1224 resp
.mw_handle
= uobj
->id
;
1226 if (copy_to_user((void __user
*)(unsigned long)cmd
.response
,
1227 &resp
, sizeof(resp
))) {
1234 mutex_lock(&file
->mutex
);
1235 list_add_tail(&uobj
->list
, &file
->ucontext
->mw_list
);
1236 mutex_unlock(&file
->mutex
);
1240 up_write(&uobj
->mutex
);
1245 idr_remove_uobj(&ib_uverbs_mw_idr
, uobj
);
1254 put_uobj_write(uobj
);
1258 ssize_t
ib_uverbs_dealloc_mw(struct ib_uverbs_file
*file
,
1259 const char __user
*buf
, int in_len
,
1262 struct ib_uverbs_dealloc_mw cmd
;
1264 struct ib_uobject
*uobj
;
1267 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1270 uobj
= idr_write_uobj(&ib_uverbs_mw_idr
, cmd
.mw_handle
, file
->ucontext
);
1276 ret
= ib_dealloc_mw(mw
);
1280 put_uobj_write(uobj
);
1285 idr_remove_uobj(&ib_uverbs_mw_idr
, uobj
);
1287 mutex_lock(&file
->mutex
);
1288 list_del(&uobj
->list
);
1289 mutex_unlock(&file
->mutex
);
1296 ssize_t
ib_uverbs_create_comp_channel(struct ib_uverbs_file
*file
,
1297 const char __user
*buf
, int in_len
,
1300 struct ib_uverbs_create_comp_channel cmd
;
1301 struct ib_uverbs_create_comp_channel_resp resp
;
1305 if (out_len
< sizeof resp
)
1308 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1311 ret
= get_unused_fd_flags(O_CLOEXEC
);
1316 filp
= ib_uverbs_alloc_event_file(file
, 0);
1318 put_unused_fd(resp
.fd
);
1319 return PTR_ERR(filp
);
1322 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1323 &resp
, sizeof resp
)) {
1324 put_unused_fd(resp
.fd
);
1329 fd_install(resp
.fd
, filp
);
1333 static struct ib_ucq_object
*create_cq(struct ib_uverbs_file
*file
,
1334 struct ib_udata
*ucore
,
1335 struct ib_udata
*uhw
,
1336 struct ib_uverbs_ex_create_cq
*cmd
,
1338 int (*cb
)(struct ib_uverbs_file
*file
,
1339 struct ib_ucq_object
*obj
,
1340 struct ib_uverbs_ex_create_cq_resp
*resp
,
1341 struct ib_udata
*udata
,
1345 struct ib_ucq_object
*obj
;
1346 struct ib_uverbs_event_file
*ev_file
= NULL
;
1349 struct ib_uverbs_ex_create_cq_resp resp
;
1350 struct ib_cq_init_attr attr
= {};
1352 if (cmd
->comp_vector
>= file
->device
->num_comp_vectors
)
1353 return ERR_PTR(-EINVAL
);
1355 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
1357 return ERR_PTR(-ENOMEM
);
1359 init_uobj(&obj
->uobject
, cmd
->user_handle
, file
->ucontext
, &cq_lock_class
);
1360 down_write(&obj
->uobject
.mutex
);
1362 if (cmd
->comp_channel
>= 0) {
1363 ev_file
= ib_uverbs_lookup_comp_file(cmd
->comp_channel
);
1370 obj
->uverbs_file
= file
;
1371 obj
->comp_events_reported
= 0;
1372 obj
->async_events_reported
= 0;
1373 INIT_LIST_HEAD(&obj
->comp_list
);
1374 INIT_LIST_HEAD(&obj
->async_list
);
1376 attr
.cqe
= cmd
->cqe
;
1377 attr
.comp_vector
= cmd
->comp_vector
;
1379 if (cmd_sz
> offsetof(typeof(*cmd
), flags
) + sizeof(cmd
->flags
))
1380 attr
.flags
= cmd
->flags
;
1382 cq
= file
->device
->ib_dev
->create_cq(file
->device
->ib_dev
, &attr
,
1383 file
->ucontext
, uhw
);
1389 cq
->device
= file
->device
->ib_dev
;
1390 cq
->uobject
= &obj
->uobject
;
1391 cq
->comp_handler
= ib_uverbs_comp_handler
;
1392 cq
->event_handler
= ib_uverbs_cq_event_handler
;
1393 cq
->cq_context
= ev_file
;
1394 atomic_set(&cq
->usecnt
, 0);
1396 obj
->uobject
.object
= cq
;
1397 ret
= idr_add_uobj(&ib_uverbs_cq_idr
, &obj
->uobject
);
1401 memset(&resp
, 0, sizeof resp
);
1402 resp
.base
.cq_handle
= obj
->uobject
.id
;
1403 resp
.base
.cqe
= cq
->cqe
;
1405 resp
.response_length
= offsetof(typeof(resp
), response_length
) +
1406 sizeof(resp
.response_length
);
1408 ret
= cb(file
, obj
, &resp
, ucore
, context
);
1412 mutex_lock(&file
->mutex
);
1413 list_add_tail(&obj
->uobject
.list
, &file
->ucontext
->cq_list
);
1414 mutex_unlock(&file
->mutex
);
1416 obj
->uobject
.live
= 1;
1418 up_write(&obj
->uobject
.mutex
);
1423 idr_remove_uobj(&ib_uverbs_cq_idr
, &obj
->uobject
);
1430 ib_uverbs_release_ucq(file
, ev_file
, obj
);
1433 put_uobj_write(&obj
->uobject
);
1435 return ERR_PTR(ret
);
1438 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file
*file
,
1439 struct ib_ucq_object
*obj
,
1440 struct ib_uverbs_ex_create_cq_resp
*resp
,
1441 struct ib_udata
*ucore
, void *context
)
1443 if (ib_copy_to_udata(ucore
, &resp
->base
, sizeof(resp
->base
)))
1449 ssize_t
ib_uverbs_create_cq(struct ib_uverbs_file
*file
,
1450 const char __user
*buf
, int in_len
,
1453 struct ib_uverbs_create_cq cmd
;
1454 struct ib_uverbs_ex_create_cq cmd_ex
;
1455 struct ib_uverbs_create_cq_resp resp
;
1456 struct ib_udata ucore
;
1457 struct ib_udata uhw
;
1458 struct ib_ucq_object
*obj
;
1460 if (out_len
< sizeof(resp
))
1463 if (copy_from_user(&cmd
, buf
, sizeof(cmd
)))
1466 INIT_UDATA(&ucore
, buf
, cmd
.response
, sizeof(cmd
), sizeof(resp
));
1468 INIT_UDATA(&uhw
, buf
+ sizeof(cmd
),
1469 (unsigned long)cmd
.response
+ sizeof(resp
),
1470 in_len
- sizeof(cmd
), out_len
- sizeof(resp
));
1472 memset(&cmd_ex
, 0, sizeof(cmd_ex
));
1473 cmd_ex
.user_handle
= cmd
.user_handle
;
1474 cmd_ex
.cqe
= cmd
.cqe
;
1475 cmd_ex
.comp_vector
= cmd
.comp_vector
;
1476 cmd_ex
.comp_channel
= cmd
.comp_channel
;
1478 obj
= create_cq(file
, &ucore
, &uhw
, &cmd_ex
,
1479 offsetof(typeof(cmd_ex
), comp_channel
) +
1480 sizeof(cmd
.comp_channel
), ib_uverbs_create_cq_cb
,
1484 return PTR_ERR(obj
);
1489 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file
*file
,
1490 struct ib_ucq_object
*obj
,
1491 struct ib_uverbs_ex_create_cq_resp
*resp
,
1492 struct ib_udata
*ucore
, void *context
)
1494 if (ib_copy_to_udata(ucore
, resp
, resp
->response_length
))
1500 int ib_uverbs_ex_create_cq(struct ib_uverbs_file
*file
,
1501 struct ib_udata
*ucore
,
1502 struct ib_udata
*uhw
)
1504 struct ib_uverbs_ex_create_cq_resp resp
;
1505 struct ib_uverbs_ex_create_cq cmd
;
1506 struct ib_ucq_object
*obj
;
1509 if (ucore
->inlen
< sizeof(cmd
))
1512 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
1522 if (ucore
->outlen
< (offsetof(typeof(resp
), response_length
) +
1523 sizeof(resp
.response_length
)))
1526 obj
= create_cq(file
, ucore
, uhw
, &cmd
,
1527 min(ucore
->inlen
, sizeof(cmd
)),
1528 ib_uverbs_ex_create_cq_cb
, NULL
);
1531 return PTR_ERR(obj
);
1536 ssize_t
ib_uverbs_resize_cq(struct ib_uverbs_file
*file
,
1537 const char __user
*buf
, int in_len
,
1540 struct ib_uverbs_resize_cq cmd
;
1541 struct ib_uverbs_resize_cq_resp resp
;
1542 struct ib_udata udata
;
1546 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1549 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1550 (unsigned long) cmd
.response
+ sizeof resp
,
1551 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1553 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
1557 ret
= cq
->device
->resize_cq(cq
, cmd
.cqe
, &udata
);
1563 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1564 &resp
, sizeof resp
.cqe
))
1570 return ret
? ret
: in_len
;
1573 static int copy_wc_to_user(void __user
*dest
, struct ib_wc
*wc
)
1575 struct ib_uverbs_wc tmp
;
1577 tmp
.wr_id
= wc
->wr_id
;
1578 tmp
.status
= wc
->status
;
1579 tmp
.opcode
= wc
->opcode
;
1580 tmp
.vendor_err
= wc
->vendor_err
;
1581 tmp
.byte_len
= wc
->byte_len
;
1582 tmp
.ex
.imm_data
= (__u32 __force
) wc
->ex
.imm_data
;
1583 tmp
.qp_num
= wc
->qp
->qp_num
;
1584 tmp
.src_qp
= wc
->src_qp
;
1585 tmp
.wc_flags
= wc
->wc_flags
;
1586 tmp
.pkey_index
= wc
->pkey_index
;
1587 tmp
.slid
= wc
->slid
;
1589 tmp
.dlid_path_bits
= wc
->dlid_path_bits
;
1590 tmp
.port_num
= wc
->port_num
;
1593 if (copy_to_user(dest
, &tmp
, sizeof tmp
))
1599 ssize_t
ib_uverbs_poll_cq(struct ib_uverbs_file
*file
,
1600 const char __user
*buf
, int in_len
,
1603 struct ib_uverbs_poll_cq cmd
;
1604 struct ib_uverbs_poll_cq_resp resp
;
1605 u8 __user
*header_ptr
;
1606 u8 __user
*data_ptr
;
1611 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1614 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
1618 /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1619 header_ptr
= (void __user
*)(unsigned long) cmd
.response
;
1620 data_ptr
= header_ptr
+ sizeof resp
;
1622 memset(&resp
, 0, sizeof resp
);
1623 while (resp
.count
< cmd
.ne
) {
1624 ret
= ib_poll_cq(cq
, 1, &wc
);
1630 ret
= copy_wc_to_user(data_ptr
, &wc
);
1634 data_ptr
+= sizeof(struct ib_uverbs_wc
);
1638 if (copy_to_user(header_ptr
, &resp
, sizeof resp
)) {
1650 ssize_t
ib_uverbs_req_notify_cq(struct ib_uverbs_file
*file
,
1651 const char __user
*buf
, int in_len
,
1654 struct ib_uverbs_req_notify_cq cmd
;
1657 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1660 cq
= idr_read_cq(cmd
.cq_handle
, file
->ucontext
, 0);
1664 ib_req_notify_cq(cq
, cmd
.solicited_only
?
1665 IB_CQ_SOLICITED
: IB_CQ_NEXT_COMP
);
1672 ssize_t
ib_uverbs_destroy_cq(struct ib_uverbs_file
*file
,
1673 const char __user
*buf
, int in_len
,
1676 struct ib_uverbs_destroy_cq cmd
;
1677 struct ib_uverbs_destroy_cq_resp resp
;
1678 struct ib_uobject
*uobj
;
1680 struct ib_ucq_object
*obj
;
1681 struct ib_uverbs_event_file
*ev_file
;
1684 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1687 uobj
= idr_write_uobj(&ib_uverbs_cq_idr
, cmd
.cq_handle
, file
->ucontext
);
1691 ev_file
= cq
->cq_context
;
1692 obj
= container_of(cq
->uobject
, struct ib_ucq_object
, uobject
);
1694 ret
= ib_destroy_cq(cq
);
1698 put_uobj_write(uobj
);
1703 idr_remove_uobj(&ib_uverbs_cq_idr
, uobj
);
1705 mutex_lock(&file
->mutex
);
1706 list_del(&uobj
->list
);
1707 mutex_unlock(&file
->mutex
);
1709 ib_uverbs_release_ucq(file
, ev_file
, obj
);
1711 memset(&resp
, 0, sizeof resp
);
1712 resp
.comp_events_reported
= obj
->comp_events_reported
;
1713 resp
.async_events_reported
= obj
->async_events_reported
;
1717 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1718 &resp
, sizeof resp
))
1724 ssize_t
ib_uverbs_create_qp(struct ib_uverbs_file
*file
,
1725 const char __user
*buf
, int in_len
,
1728 struct ib_uverbs_create_qp cmd
;
1729 struct ib_uverbs_create_qp_resp resp
;
1730 struct ib_udata udata
;
1731 struct ib_uqp_object
*obj
;
1732 struct ib_device
*device
;
1733 struct ib_pd
*pd
= NULL
;
1734 struct ib_xrcd
*xrcd
= NULL
;
1735 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
1736 struct ib_cq
*scq
= NULL
, *rcq
= NULL
;
1737 struct ib_srq
*srq
= NULL
;
1739 struct ib_qp_init_attr attr
;
1742 if (out_len
< sizeof resp
)
1745 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1748 if (cmd
.qp_type
== IB_QPT_RAW_PACKET
&& !capable(CAP_NET_RAW
))
1751 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1752 (unsigned long) cmd
.response
+ sizeof resp
,
1753 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1755 obj
= kzalloc(sizeof *obj
, GFP_KERNEL
);
1759 init_uobj(&obj
->uevent
.uobject
, cmd
.user_handle
, file
->ucontext
, &qp_lock_class
);
1760 down_write(&obj
->uevent
.uobject
.mutex
);
1762 if (cmd
.qp_type
== IB_QPT_XRC_TGT
) {
1763 xrcd
= idr_read_xrcd(cmd
.pd_handle
, file
->ucontext
, &xrcd_uobj
);
1768 device
= xrcd
->device
;
1770 if (cmd
.qp_type
== IB_QPT_XRC_INI
) {
1771 cmd
.max_recv_wr
= cmd
.max_recv_sge
= 0;
1774 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
1775 if (!srq
|| srq
->srq_type
!= IB_SRQT_BASIC
) {
1781 if (cmd
.recv_cq_handle
!= cmd
.send_cq_handle
) {
1782 rcq
= idr_read_cq(cmd
.recv_cq_handle
, file
->ucontext
, 0);
1790 scq
= idr_read_cq(cmd
.send_cq_handle
, file
->ucontext
, !!rcq
);
1792 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
1798 device
= pd
->device
;
1801 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1802 attr
.qp_context
= file
;
1807 attr
.sq_sig_type
= cmd
.sq_sig_all
? IB_SIGNAL_ALL_WR
: IB_SIGNAL_REQ_WR
;
1808 attr
.qp_type
= cmd
.qp_type
;
1809 attr
.create_flags
= 0;
1811 attr
.cap
.max_send_wr
= cmd
.max_send_wr
;
1812 attr
.cap
.max_recv_wr
= cmd
.max_recv_wr
;
1813 attr
.cap
.max_send_sge
= cmd
.max_send_sge
;
1814 attr
.cap
.max_recv_sge
= cmd
.max_recv_sge
;
1815 attr
.cap
.max_inline_data
= cmd
.max_inline_data
;
1817 obj
->uevent
.events_reported
= 0;
1818 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1819 INIT_LIST_HEAD(&obj
->mcast_list
);
1821 if (cmd
.qp_type
== IB_QPT_XRC_TGT
)
1822 qp
= ib_create_qp(pd
, &attr
);
1824 qp
= device
->create_qp(pd
, &attr
, &udata
);
1831 if (cmd
.qp_type
!= IB_QPT_XRC_TGT
) {
1833 qp
->device
= device
;
1835 qp
->send_cq
= attr
.send_cq
;
1836 qp
->recv_cq
= attr
.recv_cq
;
1838 qp
->event_handler
= attr
.event_handler
;
1839 qp
->qp_context
= attr
.qp_context
;
1840 qp
->qp_type
= attr
.qp_type
;
1841 atomic_set(&qp
->usecnt
, 0);
1842 atomic_inc(&pd
->usecnt
);
1843 atomic_inc(&attr
.send_cq
->usecnt
);
1845 atomic_inc(&attr
.recv_cq
->usecnt
);
1847 atomic_inc(&attr
.srq
->usecnt
);
1849 qp
->uobject
= &obj
->uevent
.uobject
;
1851 obj
->uevent
.uobject
.object
= qp
;
1852 ret
= idr_add_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
1856 memset(&resp
, 0, sizeof resp
);
1857 resp
.qpn
= qp
->qp_num
;
1858 resp
.qp_handle
= obj
->uevent
.uobject
.id
;
1859 resp
.max_recv_sge
= attr
.cap
.max_recv_sge
;
1860 resp
.max_send_sge
= attr
.cap
.max_send_sge
;
1861 resp
.max_recv_wr
= attr
.cap
.max_recv_wr
;
1862 resp
.max_send_wr
= attr
.cap
.max_send_wr
;
1863 resp
.max_inline_data
= attr
.cap
.max_inline_data
;
1865 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1866 &resp
, sizeof resp
)) {
1872 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
,
1874 atomic_inc(&obj
->uxrcd
->refcnt
);
1875 put_xrcd_read(xrcd_uobj
);
1882 if (rcq
&& rcq
!= scq
)
1887 mutex_lock(&file
->mutex
);
1888 list_add_tail(&obj
->uevent
.uobject
.list
, &file
->ucontext
->qp_list
);
1889 mutex_unlock(&file
->mutex
);
1891 obj
->uevent
.uobject
.live
= 1;
1893 up_write(&obj
->uevent
.uobject
.mutex
);
1898 idr_remove_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
1905 put_xrcd_read(xrcd_uobj
);
1910 if (rcq
&& rcq
!= scq
)
1915 put_uobj_write(&obj
->uevent
.uobject
);
1919 ssize_t
ib_uverbs_open_qp(struct ib_uverbs_file
*file
,
1920 const char __user
*buf
, int in_len
, int out_len
)
1922 struct ib_uverbs_open_qp cmd
;
1923 struct ib_uverbs_create_qp_resp resp
;
1924 struct ib_udata udata
;
1925 struct ib_uqp_object
*obj
;
1926 struct ib_xrcd
*xrcd
;
1927 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
1929 struct ib_qp_open_attr attr
;
1932 if (out_len
< sizeof resp
)
1935 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
1938 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
1939 (unsigned long) cmd
.response
+ sizeof resp
,
1940 in_len
- sizeof cmd
, out_len
- sizeof resp
);
1942 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
1946 init_uobj(&obj
->uevent
.uobject
, cmd
.user_handle
, file
->ucontext
, &qp_lock_class
);
1947 down_write(&obj
->uevent
.uobject
.mutex
);
1949 xrcd
= idr_read_xrcd(cmd
.pd_handle
, file
->ucontext
, &xrcd_uobj
);
1955 attr
.event_handler
= ib_uverbs_qp_event_handler
;
1956 attr
.qp_context
= file
;
1957 attr
.qp_num
= cmd
.qpn
;
1958 attr
.qp_type
= cmd
.qp_type
;
1960 obj
->uevent
.events_reported
= 0;
1961 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
1962 INIT_LIST_HEAD(&obj
->mcast_list
);
1964 qp
= ib_open_qp(xrcd
, &attr
);
1970 qp
->uobject
= &obj
->uevent
.uobject
;
1972 obj
->uevent
.uobject
.object
= qp
;
1973 ret
= idr_add_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
1977 memset(&resp
, 0, sizeof resp
);
1978 resp
.qpn
= qp
->qp_num
;
1979 resp
.qp_handle
= obj
->uevent
.uobject
.id
;
1981 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
1982 &resp
, sizeof resp
)) {
1987 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
, uobject
);
1988 atomic_inc(&obj
->uxrcd
->refcnt
);
1989 put_xrcd_read(xrcd_uobj
);
1991 mutex_lock(&file
->mutex
);
1992 list_add_tail(&obj
->uevent
.uobject
.list
, &file
->ucontext
->qp_list
);
1993 mutex_unlock(&file
->mutex
);
1995 obj
->uevent
.uobject
.live
= 1;
1997 up_write(&obj
->uevent
.uobject
.mutex
);
2002 idr_remove_uobj(&ib_uverbs_qp_idr
, &obj
->uevent
.uobject
);
2008 put_xrcd_read(xrcd_uobj
);
2009 put_uobj_write(&obj
->uevent
.uobject
);
2013 ssize_t
ib_uverbs_query_qp(struct ib_uverbs_file
*file
,
2014 const char __user
*buf
, int in_len
,
2017 struct ib_uverbs_query_qp cmd
;
2018 struct ib_uverbs_query_qp_resp resp
;
2020 struct ib_qp_attr
*attr
;
2021 struct ib_qp_init_attr
*init_attr
;
2024 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2027 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
2028 init_attr
= kmalloc(sizeof *init_attr
, GFP_KERNEL
);
2029 if (!attr
|| !init_attr
) {
2034 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2040 ret
= ib_query_qp(qp
, attr
, cmd
.attr_mask
, init_attr
);
2047 memset(&resp
, 0, sizeof resp
);
2049 resp
.qp_state
= attr
->qp_state
;
2050 resp
.cur_qp_state
= attr
->cur_qp_state
;
2051 resp
.path_mtu
= attr
->path_mtu
;
2052 resp
.path_mig_state
= attr
->path_mig_state
;
2053 resp
.qkey
= attr
->qkey
;
2054 resp
.rq_psn
= attr
->rq_psn
;
2055 resp
.sq_psn
= attr
->sq_psn
;
2056 resp
.dest_qp_num
= attr
->dest_qp_num
;
2057 resp
.qp_access_flags
= attr
->qp_access_flags
;
2058 resp
.pkey_index
= attr
->pkey_index
;
2059 resp
.alt_pkey_index
= attr
->alt_pkey_index
;
2060 resp
.sq_draining
= attr
->sq_draining
;
2061 resp
.max_rd_atomic
= attr
->max_rd_atomic
;
2062 resp
.max_dest_rd_atomic
= attr
->max_dest_rd_atomic
;
2063 resp
.min_rnr_timer
= attr
->min_rnr_timer
;
2064 resp
.port_num
= attr
->port_num
;
2065 resp
.timeout
= attr
->timeout
;
2066 resp
.retry_cnt
= attr
->retry_cnt
;
2067 resp
.rnr_retry
= attr
->rnr_retry
;
2068 resp
.alt_port_num
= attr
->alt_port_num
;
2069 resp
.alt_timeout
= attr
->alt_timeout
;
2071 memcpy(resp
.dest
.dgid
, attr
->ah_attr
.grh
.dgid
.raw
, 16);
2072 resp
.dest
.flow_label
= attr
->ah_attr
.grh
.flow_label
;
2073 resp
.dest
.sgid_index
= attr
->ah_attr
.grh
.sgid_index
;
2074 resp
.dest
.hop_limit
= attr
->ah_attr
.grh
.hop_limit
;
2075 resp
.dest
.traffic_class
= attr
->ah_attr
.grh
.traffic_class
;
2076 resp
.dest
.dlid
= attr
->ah_attr
.dlid
;
2077 resp
.dest
.sl
= attr
->ah_attr
.sl
;
2078 resp
.dest
.src_path_bits
= attr
->ah_attr
.src_path_bits
;
2079 resp
.dest
.static_rate
= attr
->ah_attr
.static_rate
;
2080 resp
.dest
.is_global
= !!(attr
->ah_attr
.ah_flags
& IB_AH_GRH
);
2081 resp
.dest
.port_num
= attr
->ah_attr
.port_num
;
2083 memcpy(resp
.alt_dest
.dgid
, attr
->alt_ah_attr
.grh
.dgid
.raw
, 16);
2084 resp
.alt_dest
.flow_label
= attr
->alt_ah_attr
.grh
.flow_label
;
2085 resp
.alt_dest
.sgid_index
= attr
->alt_ah_attr
.grh
.sgid_index
;
2086 resp
.alt_dest
.hop_limit
= attr
->alt_ah_attr
.grh
.hop_limit
;
2087 resp
.alt_dest
.traffic_class
= attr
->alt_ah_attr
.grh
.traffic_class
;
2088 resp
.alt_dest
.dlid
= attr
->alt_ah_attr
.dlid
;
2089 resp
.alt_dest
.sl
= attr
->alt_ah_attr
.sl
;
2090 resp
.alt_dest
.src_path_bits
= attr
->alt_ah_attr
.src_path_bits
;
2091 resp
.alt_dest
.static_rate
= attr
->alt_ah_attr
.static_rate
;
2092 resp
.alt_dest
.is_global
= !!(attr
->alt_ah_attr
.ah_flags
& IB_AH_GRH
);
2093 resp
.alt_dest
.port_num
= attr
->alt_ah_attr
.port_num
;
2095 resp
.max_send_wr
= init_attr
->cap
.max_send_wr
;
2096 resp
.max_recv_wr
= init_attr
->cap
.max_recv_wr
;
2097 resp
.max_send_sge
= init_attr
->cap
.max_send_sge
;
2098 resp
.max_recv_sge
= init_attr
->cap
.max_recv_sge
;
2099 resp
.max_inline_data
= init_attr
->cap
.max_inline_data
;
2100 resp
.sq_sig_all
= init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
;
2102 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2103 &resp
, sizeof resp
))
2110 return ret
? ret
: in_len
;
2113 /* Remove ignored fields set in the attribute mask */
2114 static int modify_qp_mask(enum ib_qp_type qp_type
, int mask
)
2117 case IB_QPT_XRC_INI
:
2118 return mask
& ~(IB_QP_MAX_DEST_RD_ATOMIC
| IB_QP_MIN_RNR_TIMER
);
2119 case IB_QPT_XRC_TGT
:
2120 return mask
& ~(IB_QP_MAX_QP_RD_ATOMIC
| IB_QP_RETRY_CNT
|
2127 ssize_t
ib_uverbs_modify_qp(struct ib_uverbs_file
*file
,
2128 const char __user
*buf
, int in_len
,
2131 struct ib_uverbs_modify_qp cmd
;
2132 struct ib_udata udata
;
2134 struct ib_qp_attr
*attr
;
2137 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2140 INIT_UDATA(&udata
, buf
+ sizeof cmd
, NULL
, in_len
- sizeof cmd
,
2143 attr
= kmalloc(sizeof *attr
, GFP_KERNEL
);
2147 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2153 attr
->qp_state
= cmd
.qp_state
;
2154 attr
->cur_qp_state
= cmd
.cur_qp_state
;
2155 attr
->path_mtu
= cmd
.path_mtu
;
2156 attr
->path_mig_state
= cmd
.path_mig_state
;
2157 attr
->qkey
= cmd
.qkey
;
2158 attr
->rq_psn
= cmd
.rq_psn
;
2159 attr
->sq_psn
= cmd
.sq_psn
;
2160 attr
->dest_qp_num
= cmd
.dest_qp_num
;
2161 attr
->qp_access_flags
= cmd
.qp_access_flags
;
2162 attr
->pkey_index
= cmd
.pkey_index
;
2163 attr
->alt_pkey_index
= cmd
.alt_pkey_index
;
2164 attr
->en_sqd_async_notify
= cmd
.en_sqd_async_notify
;
2165 attr
->max_rd_atomic
= cmd
.max_rd_atomic
;
2166 attr
->max_dest_rd_atomic
= cmd
.max_dest_rd_atomic
;
2167 attr
->min_rnr_timer
= cmd
.min_rnr_timer
;
2168 attr
->port_num
= cmd
.port_num
;
2169 attr
->timeout
= cmd
.timeout
;
2170 attr
->retry_cnt
= cmd
.retry_cnt
;
2171 attr
->rnr_retry
= cmd
.rnr_retry
;
2172 attr
->alt_port_num
= cmd
.alt_port_num
;
2173 attr
->alt_timeout
= cmd
.alt_timeout
;
2175 memcpy(attr
->ah_attr
.grh
.dgid
.raw
, cmd
.dest
.dgid
, 16);
2176 attr
->ah_attr
.grh
.flow_label
= cmd
.dest
.flow_label
;
2177 attr
->ah_attr
.grh
.sgid_index
= cmd
.dest
.sgid_index
;
2178 attr
->ah_attr
.grh
.hop_limit
= cmd
.dest
.hop_limit
;
2179 attr
->ah_attr
.grh
.traffic_class
= cmd
.dest
.traffic_class
;
2180 attr
->ah_attr
.dlid
= cmd
.dest
.dlid
;
2181 attr
->ah_attr
.sl
= cmd
.dest
.sl
;
2182 attr
->ah_attr
.src_path_bits
= cmd
.dest
.src_path_bits
;
2183 attr
->ah_attr
.static_rate
= cmd
.dest
.static_rate
;
2184 attr
->ah_attr
.ah_flags
= cmd
.dest
.is_global
? IB_AH_GRH
: 0;
2185 attr
->ah_attr
.port_num
= cmd
.dest
.port_num
;
2187 memcpy(attr
->alt_ah_attr
.grh
.dgid
.raw
, cmd
.alt_dest
.dgid
, 16);
2188 attr
->alt_ah_attr
.grh
.flow_label
= cmd
.alt_dest
.flow_label
;
2189 attr
->alt_ah_attr
.grh
.sgid_index
= cmd
.alt_dest
.sgid_index
;
2190 attr
->alt_ah_attr
.grh
.hop_limit
= cmd
.alt_dest
.hop_limit
;
2191 attr
->alt_ah_attr
.grh
.traffic_class
= cmd
.alt_dest
.traffic_class
;
2192 attr
->alt_ah_attr
.dlid
= cmd
.alt_dest
.dlid
;
2193 attr
->alt_ah_attr
.sl
= cmd
.alt_dest
.sl
;
2194 attr
->alt_ah_attr
.src_path_bits
= cmd
.alt_dest
.src_path_bits
;
2195 attr
->alt_ah_attr
.static_rate
= cmd
.alt_dest
.static_rate
;
2196 attr
->alt_ah_attr
.ah_flags
= cmd
.alt_dest
.is_global
? IB_AH_GRH
: 0;
2197 attr
->alt_ah_attr
.port_num
= cmd
.alt_dest
.port_num
;
2199 if (qp
->real_qp
== qp
) {
2200 ret
= ib_resolve_eth_l2_attrs(qp
, attr
, &cmd
.attr_mask
);
2203 ret
= qp
->device
->modify_qp(qp
, attr
,
2204 modify_qp_mask(qp
->qp_type
, cmd
.attr_mask
), &udata
);
2206 ret
= ib_modify_qp(qp
, attr
, modify_qp_mask(qp
->qp_type
, cmd
.attr_mask
));
2223 ssize_t
ib_uverbs_destroy_qp(struct ib_uverbs_file
*file
,
2224 const char __user
*buf
, int in_len
,
2227 struct ib_uverbs_destroy_qp cmd
;
2228 struct ib_uverbs_destroy_qp_resp resp
;
2229 struct ib_uobject
*uobj
;
2231 struct ib_uqp_object
*obj
;
2234 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2237 memset(&resp
, 0, sizeof resp
);
2239 uobj
= idr_write_uobj(&ib_uverbs_qp_idr
, cmd
.qp_handle
, file
->ucontext
);
2243 obj
= container_of(uobj
, struct ib_uqp_object
, uevent
.uobject
);
2245 if (!list_empty(&obj
->mcast_list
)) {
2246 put_uobj_write(uobj
);
2250 ret
= ib_destroy_qp(qp
);
2254 put_uobj_write(uobj
);
2260 atomic_dec(&obj
->uxrcd
->refcnt
);
2262 idr_remove_uobj(&ib_uverbs_qp_idr
, uobj
);
2264 mutex_lock(&file
->mutex
);
2265 list_del(&uobj
->list
);
2266 mutex_unlock(&file
->mutex
);
2268 ib_uverbs_release_uevent(file
, &obj
->uevent
);
2270 resp
.events_reported
= obj
->uevent
.events_reported
;
2274 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2275 &resp
, sizeof resp
))
2281 ssize_t
ib_uverbs_post_send(struct ib_uverbs_file
*file
,
2282 const char __user
*buf
, int in_len
,
2285 struct ib_uverbs_post_send cmd
;
2286 struct ib_uverbs_post_send_resp resp
;
2287 struct ib_uverbs_send_wr
*user_wr
;
2288 struct ib_send_wr
*wr
= NULL
, *last
, *next
, *bad_wr
;
2292 ssize_t ret
= -EINVAL
;
2294 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2297 if (in_len
< sizeof cmd
+ cmd
.wqe_size
* cmd
.wr_count
+
2298 cmd
.sge_count
* sizeof (struct ib_uverbs_sge
))
2301 if (cmd
.wqe_size
< sizeof (struct ib_uverbs_send_wr
))
2304 user_wr
= kmalloc(cmd
.wqe_size
, GFP_KERNEL
);
2308 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2312 is_ud
= qp
->qp_type
== IB_QPT_UD
;
2315 for (i
= 0; i
< cmd
.wr_count
; ++i
) {
2316 if (copy_from_user(user_wr
,
2317 buf
+ sizeof cmd
+ i
* cmd
.wqe_size
,
2323 if (user_wr
->num_sge
+ sg_ind
> cmd
.sge_count
) {
2328 next
= kmalloc(ALIGN(sizeof *next
, sizeof (struct ib_sge
)) +
2329 user_wr
->num_sge
* sizeof (struct ib_sge
),
2343 next
->wr_id
= user_wr
->wr_id
;
2344 next
->num_sge
= user_wr
->num_sge
;
2345 next
->opcode
= user_wr
->opcode
;
2346 next
->send_flags
= user_wr
->send_flags
;
2349 next
->wr
.ud
.ah
= idr_read_ah(user_wr
->wr
.ud
.ah
,
2351 if (!next
->wr
.ud
.ah
) {
2355 next
->wr
.ud
.remote_qpn
= user_wr
->wr
.ud
.remote_qpn
;
2356 next
->wr
.ud
.remote_qkey
= user_wr
->wr
.ud
.remote_qkey
;
2357 if (next
->opcode
== IB_WR_SEND_WITH_IMM
)
2359 (__be32 __force
) user_wr
->ex
.imm_data
;
2361 switch (next
->opcode
) {
2362 case IB_WR_RDMA_WRITE_WITH_IMM
:
2364 (__be32 __force
) user_wr
->ex
.imm_data
;
2365 case IB_WR_RDMA_WRITE
:
2366 case IB_WR_RDMA_READ
:
2367 next
->wr
.rdma
.remote_addr
=
2368 user_wr
->wr
.rdma
.remote_addr
;
2369 next
->wr
.rdma
.rkey
=
2370 user_wr
->wr
.rdma
.rkey
;
2372 case IB_WR_SEND_WITH_IMM
:
2374 (__be32 __force
) user_wr
->ex
.imm_data
;
2376 case IB_WR_SEND_WITH_INV
:
2377 next
->ex
.invalidate_rkey
=
2378 user_wr
->ex
.invalidate_rkey
;
2380 case IB_WR_ATOMIC_CMP_AND_SWP
:
2381 case IB_WR_ATOMIC_FETCH_AND_ADD
:
2382 next
->wr
.atomic
.remote_addr
=
2383 user_wr
->wr
.atomic
.remote_addr
;
2384 next
->wr
.atomic
.compare_add
=
2385 user_wr
->wr
.atomic
.compare_add
;
2386 next
->wr
.atomic
.swap
= user_wr
->wr
.atomic
.swap
;
2387 next
->wr
.atomic
.rkey
= user_wr
->wr
.atomic
.rkey
;
2394 if (next
->num_sge
) {
2395 next
->sg_list
= (void *) next
+
2396 ALIGN(sizeof *next
, sizeof (struct ib_sge
));
2397 if (copy_from_user(next
->sg_list
,
2399 cmd
.wr_count
* cmd
.wqe_size
+
2400 sg_ind
* sizeof (struct ib_sge
),
2401 next
->num_sge
* sizeof (struct ib_sge
))) {
2405 sg_ind
+= next
->num_sge
;
2407 next
->sg_list
= NULL
;
2411 ret
= qp
->device
->post_send(qp
->real_qp
, wr
, &bad_wr
);
2413 for (next
= wr
; next
; next
= next
->next
) {
2419 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2420 &resp
, sizeof resp
))
2427 if (is_ud
&& wr
->wr
.ud
.ah
)
2428 put_ah_read(wr
->wr
.ud
.ah
);
2437 return ret
? ret
: in_len
;
2440 static struct ib_recv_wr
*ib_uverbs_unmarshall_recv(const char __user
*buf
,
2446 struct ib_uverbs_recv_wr
*user_wr
;
2447 struct ib_recv_wr
*wr
= NULL
, *last
, *next
;
2452 if (in_len
< wqe_size
* wr_count
+
2453 sge_count
* sizeof (struct ib_uverbs_sge
))
2454 return ERR_PTR(-EINVAL
);
2456 if (wqe_size
< sizeof (struct ib_uverbs_recv_wr
))
2457 return ERR_PTR(-EINVAL
);
2459 user_wr
= kmalloc(wqe_size
, GFP_KERNEL
);
2461 return ERR_PTR(-ENOMEM
);
2465 for (i
= 0; i
< wr_count
; ++i
) {
2466 if (copy_from_user(user_wr
, buf
+ i
* wqe_size
,
2472 if (user_wr
->num_sge
+ sg_ind
> sge_count
) {
2477 next
= kmalloc(ALIGN(sizeof *next
, sizeof (struct ib_sge
)) +
2478 user_wr
->num_sge
* sizeof (struct ib_sge
),
2492 next
->wr_id
= user_wr
->wr_id
;
2493 next
->num_sge
= user_wr
->num_sge
;
2495 if (next
->num_sge
) {
2496 next
->sg_list
= (void *) next
+
2497 ALIGN(sizeof *next
, sizeof (struct ib_sge
));
2498 if (copy_from_user(next
->sg_list
,
2499 buf
+ wr_count
* wqe_size
+
2500 sg_ind
* sizeof (struct ib_sge
),
2501 next
->num_sge
* sizeof (struct ib_sge
))) {
2505 sg_ind
+= next
->num_sge
;
2507 next
->sg_list
= NULL
;
2522 return ERR_PTR(ret
);
2525 ssize_t
ib_uverbs_post_recv(struct ib_uverbs_file
*file
,
2526 const char __user
*buf
, int in_len
,
2529 struct ib_uverbs_post_recv cmd
;
2530 struct ib_uverbs_post_recv_resp resp
;
2531 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
2533 ssize_t ret
= -EINVAL
;
2535 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2538 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
2539 in_len
- sizeof cmd
, cmd
.wr_count
,
2540 cmd
.sge_count
, cmd
.wqe_size
);
2544 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2549 ret
= qp
->device
->post_recv(qp
->real_qp
, wr
, &bad_wr
);
2554 for (next
= wr
; next
; next
= next
->next
) {
2560 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2561 &resp
, sizeof resp
))
2571 return ret
? ret
: in_len
;
2574 ssize_t
ib_uverbs_post_srq_recv(struct ib_uverbs_file
*file
,
2575 const char __user
*buf
, int in_len
,
2578 struct ib_uverbs_post_srq_recv cmd
;
2579 struct ib_uverbs_post_srq_recv_resp resp
;
2580 struct ib_recv_wr
*wr
, *next
, *bad_wr
;
2582 ssize_t ret
= -EINVAL
;
2584 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2587 wr
= ib_uverbs_unmarshall_recv(buf
+ sizeof cmd
,
2588 in_len
- sizeof cmd
, cmd
.wr_count
,
2589 cmd
.sge_count
, cmd
.wqe_size
);
2593 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
2598 ret
= srq
->device
->post_srq_recv(srq
, wr
, &bad_wr
);
2603 for (next
= wr
; next
; next
= next
->next
) {
2609 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2610 &resp
, sizeof resp
))
2620 return ret
? ret
: in_len
;
2623 ssize_t
ib_uverbs_create_ah(struct ib_uverbs_file
*file
,
2624 const char __user
*buf
, int in_len
,
2627 struct ib_uverbs_create_ah cmd
;
2628 struct ib_uverbs_create_ah_resp resp
;
2629 struct ib_uobject
*uobj
;
2632 struct ib_ah_attr attr
;
2635 if (out_len
< sizeof resp
)
2638 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2641 uobj
= kmalloc(sizeof *uobj
, GFP_KERNEL
);
2645 init_uobj(uobj
, cmd
.user_handle
, file
->ucontext
, &ah_lock_class
);
2646 down_write(&uobj
->mutex
);
2648 pd
= idr_read_pd(cmd
.pd_handle
, file
->ucontext
);
2654 attr
.dlid
= cmd
.attr
.dlid
;
2655 attr
.sl
= cmd
.attr
.sl
;
2656 attr
.src_path_bits
= cmd
.attr
.src_path_bits
;
2657 attr
.static_rate
= cmd
.attr
.static_rate
;
2658 attr
.ah_flags
= cmd
.attr
.is_global
? IB_AH_GRH
: 0;
2659 attr
.port_num
= cmd
.attr
.port_num
;
2660 attr
.grh
.flow_label
= cmd
.attr
.grh
.flow_label
;
2661 attr
.grh
.sgid_index
= cmd
.attr
.grh
.sgid_index
;
2662 attr
.grh
.hop_limit
= cmd
.attr
.grh
.hop_limit
;
2663 attr
.grh
.traffic_class
= cmd
.attr
.grh
.traffic_class
;
2665 memset(&attr
.dmac
, 0, sizeof(attr
.dmac
));
2666 memcpy(attr
.grh
.dgid
.raw
, cmd
.attr
.grh
.dgid
, 16);
2668 ah
= ib_create_ah(pd
, &attr
);
2677 ret
= idr_add_uobj(&ib_uverbs_ah_idr
, uobj
);
2681 resp
.ah_handle
= uobj
->id
;
2683 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
2684 &resp
, sizeof resp
)) {
2691 mutex_lock(&file
->mutex
);
2692 list_add_tail(&uobj
->list
, &file
->ucontext
->ah_list
);
2693 mutex_unlock(&file
->mutex
);
2697 up_write(&uobj
->mutex
);
2702 idr_remove_uobj(&ib_uverbs_ah_idr
, uobj
);
2711 put_uobj_write(uobj
);
2715 ssize_t
ib_uverbs_destroy_ah(struct ib_uverbs_file
*file
,
2716 const char __user
*buf
, int in_len
, int out_len
)
2718 struct ib_uverbs_destroy_ah cmd
;
2720 struct ib_uobject
*uobj
;
2723 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2726 uobj
= idr_write_uobj(&ib_uverbs_ah_idr
, cmd
.ah_handle
, file
->ucontext
);
2731 ret
= ib_destroy_ah(ah
);
2735 put_uobj_write(uobj
);
2740 idr_remove_uobj(&ib_uverbs_ah_idr
, uobj
);
2742 mutex_lock(&file
->mutex
);
2743 list_del(&uobj
->list
);
2744 mutex_unlock(&file
->mutex
);
2751 ssize_t
ib_uverbs_attach_mcast(struct ib_uverbs_file
*file
,
2752 const char __user
*buf
, int in_len
,
2755 struct ib_uverbs_attach_mcast cmd
;
2757 struct ib_uqp_object
*obj
;
2758 struct ib_uverbs_mcast_entry
*mcast
;
2761 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2764 qp
= idr_write_qp(cmd
.qp_handle
, file
->ucontext
);
2768 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2770 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2771 if (cmd
.mlid
== mcast
->lid
&&
2772 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2777 mcast
= kmalloc(sizeof *mcast
, GFP_KERNEL
);
2783 mcast
->lid
= cmd
.mlid
;
2784 memcpy(mcast
->gid
.raw
, cmd
.gid
, sizeof mcast
->gid
.raw
);
2786 ret
= ib_attach_mcast(qp
, &mcast
->gid
, cmd
.mlid
);
2788 list_add_tail(&mcast
->list
, &obj
->mcast_list
);
2795 return ret
? ret
: in_len
;
2798 ssize_t
ib_uverbs_detach_mcast(struct ib_uverbs_file
*file
,
2799 const char __user
*buf
, int in_len
,
2802 struct ib_uverbs_detach_mcast cmd
;
2803 struct ib_uqp_object
*obj
;
2805 struct ib_uverbs_mcast_entry
*mcast
;
2808 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
2811 qp
= idr_write_qp(cmd
.qp_handle
, file
->ucontext
);
2815 ret
= ib_detach_mcast(qp
, (union ib_gid
*) cmd
.gid
, cmd
.mlid
);
2819 obj
= container_of(qp
->uobject
, struct ib_uqp_object
, uevent
.uobject
);
2821 list_for_each_entry(mcast
, &obj
->mcast_list
, list
)
2822 if (cmd
.mlid
== mcast
->lid
&&
2823 !memcmp(cmd
.gid
, mcast
->gid
.raw
, sizeof mcast
->gid
.raw
)) {
2824 list_del(&mcast
->list
);
2832 return ret
? ret
: in_len
;
2835 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec
*kern_spec
,
2836 union ib_flow_spec
*ib_spec
)
2838 if (kern_spec
->reserved
)
2841 ib_spec
->type
= kern_spec
->type
;
2843 switch (ib_spec
->type
) {
2844 case IB_FLOW_SPEC_ETH
:
2845 ib_spec
->eth
.size
= sizeof(struct ib_flow_spec_eth
);
2846 if (ib_spec
->eth
.size
!= kern_spec
->eth
.size
)
2848 memcpy(&ib_spec
->eth
.val
, &kern_spec
->eth
.val
,
2849 sizeof(struct ib_flow_eth_filter
));
2850 memcpy(&ib_spec
->eth
.mask
, &kern_spec
->eth
.mask
,
2851 sizeof(struct ib_flow_eth_filter
));
2853 case IB_FLOW_SPEC_IPV4
:
2854 ib_spec
->ipv4
.size
= sizeof(struct ib_flow_spec_ipv4
);
2855 if (ib_spec
->ipv4
.size
!= kern_spec
->ipv4
.size
)
2857 memcpy(&ib_spec
->ipv4
.val
, &kern_spec
->ipv4
.val
,
2858 sizeof(struct ib_flow_ipv4_filter
));
2859 memcpy(&ib_spec
->ipv4
.mask
, &kern_spec
->ipv4
.mask
,
2860 sizeof(struct ib_flow_ipv4_filter
));
2862 case IB_FLOW_SPEC_TCP
:
2863 case IB_FLOW_SPEC_UDP
:
2864 ib_spec
->tcp_udp
.size
= sizeof(struct ib_flow_spec_tcp_udp
);
2865 if (ib_spec
->tcp_udp
.size
!= kern_spec
->tcp_udp
.size
)
2867 memcpy(&ib_spec
->tcp_udp
.val
, &kern_spec
->tcp_udp
.val
,
2868 sizeof(struct ib_flow_tcp_udp_filter
));
2869 memcpy(&ib_spec
->tcp_udp
.mask
, &kern_spec
->tcp_udp
.mask
,
2870 sizeof(struct ib_flow_tcp_udp_filter
));
2878 int ib_uverbs_ex_create_flow(struct ib_uverbs_file
*file
,
2879 struct ib_udata
*ucore
,
2880 struct ib_udata
*uhw
)
2882 struct ib_uverbs_create_flow cmd
;
2883 struct ib_uverbs_create_flow_resp resp
;
2884 struct ib_uobject
*uobj
;
2885 struct ib_flow
*flow_id
;
2886 struct ib_uverbs_flow_attr
*kern_flow_attr
;
2887 struct ib_flow_attr
*flow_attr
;
2894 if (ucore
->inlen
< sizeof(cmd
))
2897 if (ucore
->outlen
< sizeof(resp
))
2900 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
2904 ucore
->inbuf
+= sizeof(cmd
);
2905 ucore
->inlen
-= sizeof(cmd
);
2910 if ((cmd
.flow_attr
.type
== IB_FLOW_ATTR_SNIFFER
&&
2911 !capable(CAP_NET_ADMIN
)) || !capable(CAP_NET_RAW
))
2914 if (cmd
.flow_attr
.num_of_specs
> IB_FLOW_SPEC_SUPPORT_LAYERS
)
2917 if (cmd
.flow_attr
.size
> ucore
->inlen
||
2918 cmd
.flow_attr
.size
>
2919 (cmd
.flow_attr
.num_of_specs
* sizeof(struct ib_uverbs_flow_spec
)))
2922 if (cmd
.flow_attr
.reserved
[0] ||
2923 cmd
.flow_attr
.reserved
[1])
2926 if (cmd
.flow_attr
.num_of_specs
) {
2927 kern_flow_attr
= kmalloc(sizeof(*kern_flow_attr
) + cmd
.flow_attr
.size
,
2929 if (!kern_flow_attr
)
2932 memcpy(kern_flow_attr
, &cmd
.flow_attr
, sizeof(*kern_flow_attr
));
2933 err
= ib_copy_from_udata(kern_flow_attr
+ 1, ucore
,
2934 cmd
.flow_attr
.size
);
2938 kern_flow_attr
= &cmd
.flow_attr
;
2941 uobj
= kmalloc(sizeof(*uobj
), GFP_KERNEL
);
2946 init_uobj(uobj
, 0, file
->ucontext
, &rule_lock_class
);
2947 down_write(&uobj
->mutex
);
2949 qp
= idr_read_qp(cmd
.qp_handle
, file
->ucontext
);
2955 flow_attr
= kmalloc(sizeof(*flow_attr
) + cmd
.flow_attr
.size
, GFP_KERNEL
);
2961 flow_attr
->type
= kern_flow_attr
->type
;
2962 flow_attr
->priority
= kern_flow_attr
->priority
;
2963 flow_attr
->num_of_specs
= kern_flow_attr
->num_of_specs
;
2964 flow_attr
->port
= kern_flow_attr
->port
;
2965 flow_attr
->flags
= kern_flow_attr
->flags
;
2966 flow_attr
->size
= sizeof(*flow_attr
);
2968 kern_spec
= kern_flow_attr
+ 1;
2969 ib_spec
= flow_attr
+ 1;
2970 for (i
= 0; i
< flow_attr
->num_of_specs
&&
2971 cmd
.flow_attr
.size
> offsetof(struct ib_uverbs_flow_spec
, reserved
) &&
2972 cmd
.flow_attr
.size
>=
2973 ((struct ib_uverbs_flow_spec
*)kern_spec
)->size
; i
++) {
2974 err
= kern_spec_to_ib_spec(kern_spec
, ib_spec
);
2978 ((union ib_flow_spec
*) ib_spec
)->size
;
2979 cmd
.flow_attr
.size
-= ((struct ib_uverbs_flow_spec
*)kern_spec
)->size
;
2980 kern_spec
+= ((struct ib_uverbs_flow_spec
*) kern_spec
)->size
;
2981 ib_spec
+= ((union ib_flow_spec
*) ib_spec
)->size
;
2983 if (cmd
.flow_attr
.size
|| (i
!= flow_attr
->num_of_specs
)) {
2984 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
2985 i
, cmd
.flow_attr
.size
);
2989 flow_id
= ib_create_flow(qp
, flow_attr
, IB_FLOW_DOMAIN_USER
);
2990 if (IS_ERR(flow_id
)) {
2991 err
= PTR_ERR(flow_id
);
2995 flow_id
->uobject
= uobj
;
2996 uobj
->object
= flow_id
;
2998 err
= idr_add_uobj(&ib_uverbs_rule_idr
, uobj
);
3002 memset(&resp
, 0, sizeof(resp
));
3003 resp
.flow_handle
= uobj
->id
;
3005 err
= ib_copy_to_udata(ucore
,
3006 &resp
, sizeof(resp
));
3011 mutex_lock(&file
->mutex
);
3012 list_add_tail(&uobj
->list
, &file
->ucontext
->rule_list
);
3013 mutex_unlock(&file
->mutex
);
3017 up_write(&uobj
->mutex
);
3019 if (cmd
.flow_attr
.num_of_specs
)
3020 kfree(kern_flow_attr
);
3023 idr_remove_uobj(&ib_uverbs_rule_idr
, uobj
);
3025 ib_destroy_flow(flow_id
);
3031 put_uobj_write(uobj
);
3033 if (cmd
.flow_attr
.num_of_specs
)
3034 kfree(kern_flow_attr
);
3038 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file
*file
,
3039 struct ib_udata
*ucore
,
3040 struct ib_udata
*uhw
)
3042 struct ib_uverbs_destroy_flow cmd
;
3043 struct ib_flow
*flow_id
;
3044 struct ib_uobject
*uobj
;
3047 if (ucore
->inlen
< sizeof(cmd
))
3050 ret
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
3057 uobj
= idr_write_uobj(&ib_uverbs_rule_idr
, cmd
.flow_handle
,
3061 flow_id
= uobj
->object
;
3063 ret
= ib_destroy_flow(flow_id
);
3067 put_uobj_write(uobj
);
3069 idr_remove_uobj(&ib_uverbs_rule_idr
, uobj
);
3071 mutex_lock(&file
->mutex
);
3072 list_del(&uobj
->list
);
3073 mutex_unlock(&file
->mutex
);
3080 static int __uverbs_create_xsrq(struct ib_uverbs_file
*file
,
3081 struct ib_uverbs_create_xsrq
*cmd
,
3082 struct ib_udata
*udata
)
3084 struct ib_uverbs_create_srq_resp resp
;
3085 struct ib_usrq_object
*obj
;
3088 struct ib_uobject
*uninitialized_var(xrcd_uobj
);
3089 struct ib_srq_init_attr attr
;
3092 obj
= kmalloc(sizeof *obj
, GFP_KERNEL
);
3096 init_uobj(&obj
->uevent
.uobject
, cmd
->user_handle
, file
->ucontext
, &srq_lock_class
);
3097 down_write(&obj
->uevent
.uobject
.mutex
);
3099 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3100 attr
.ext
.xrc
.xrcd
= idr_read_xrcd(cmd
->xrcd_handle
, file
->ucontext
, &xrcd_uobj
);
3101 if (!attr
.ext
.xrc
.xrcd
) {
3106 obj
->uxrcd
= container_of(xrcd_uobj
, struct ib_uxrcd_object
, uobject
);
3107 atomic_inc(&obj
->uxrcd
->refcnt
);
3109 attr
.ext
.xrc
.cq
= idr_read_cq(cmd
->cq_handle
, file
->ucontext
, 0);
3110 if (!attr
.ext
.xrc
.cq
) {
3116 pd
= idr_read_pd(cmd
->pd_handle
, file
->ucontext
);
3122 attr
.event_handler
= ib_uverbs_srq_event_handler
;
3123 attr
.srq_context
= file
;
3124 attr
.srq_type
= cmd
->srq_type
;
3125 attr
.attr
.max_wr
= cmd
->max_wr
;
3126 attr
.attr
.max_sge
= cmd
->max_sge
;
3127 attr
.attr
.srq_limit
= cmd
->srq_limit
;
3129 obj
->uevent
.events_reported
= 0;
3130 INIT_LIST_HEAD(&obj
->uevent
.event_list
);
3132 srq
= pd
->device
->create_srq(pd
, &attr
, udata
);
3138 srq
->device
= pd
->device
;
3140 srq
->srq_type
= cmd
->srq_type
;
3141 srq
->uobject
= &obj
->uevent
.uobject
;
3142 srq
->event_handler
= attr
.event_handler
;
3143 srq
->srq_context
= attr
.srq_context
;
3145 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3146 srq
->ext
.xrc
.cq
= attr
.ext
.xrc
.cq
;
3147 srq
->ext
.xrc
.xrcd
= attr
.ext
.xrc
.xrcd
;
3148 atomic_inc(&attr
.ext
.xrc
.cq
->usecnt
);
3149 atomic_inc(&attr
.ext
.xrc
.xrcd
->usecnt
);
3152 atomic_inc(&pd
->usecnt
);
3153 atomic_set(&srq
->usecnt
, 0);
3155 obj
->uevent
.uobject
.object
= srq
;
3156 ret
= idr_add_uobj(&ib_uverbs_srq_idr
, &obj
->uevent
.uobject
);
3160 memset(&resp
, 0, sizeof resp
);
3161 resp
.srq_handle
= obj
->uevent
.uobject
.id
;
3162 resp
.max_wr
= attr
.attr
.max_wr
;
3163 resp
.max_sge
= attr
.attr
.max_sge
;
3164 if (cmd
->srq_type
== IB_SRQT_XRC
)
3165 resp
.srqn
= srq
->ext
.xrc
.srq_num
;
3167 if (copy_to_user((void __user
*) (unsigned long) cmd
->response
,
3168 &resp
, sizeof resp
)) {
3173 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3174 put_uobj_read(xrcd_uobj
);
3175 put_cq_read(attr
.ext
.xrc
.cq
);
3179 mutex_lock(&file
->mutex
);
3180 list_add_tail(&obj
->uevent
.uobject
.list
, &file
->ucontext
->srq_list
);
3181 mutex_unlock(&file
->mutex
);
3183 obj
->uevent
.uobject
.live
= 1;
3185 up_write(&obj
->uevent
.uobject
.mutex
);
3190 idr_remove_uobj(&ib_uverbs_srq_idr
, &obj
->uevent
.uobject
);
3193 ib_destroy_srq(srq
);
3199 if (cmd
->srq_type
== IB_SRQT_XRC
)
3200 put_cq_read(attr
.ext
.xrc
.cq
);
3203 if (cmd
->srq_type
== IB_SRQT_XRC
) {
3204 atomic_dec(&obj
->uxrcd
->refcnt
);
3205 put_uobj_read(xrcd_uobj
);
3209 put_uobj_write(&obj
->uevent
.uobject
);
3213 ssize_t
ib_uverbs_create_srq(struct ib_uverbs_file
*file
,
3214 const char __user
*buf
, int in_len
,
3217 struct ib_uverbs_create_srq cmd
;
3218 struct ib_uverbs_create_xsrq xcmd
;
3219 struct ib_uverbs_create_srq_resp resp
;
3220 struct ib_udata udata
;
3223 if (out_len
< sizeof resp
)
3226 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3229 xcmd
.response
= cmd
.response
;
3230 xcmd
.user_handle
= cmd
.user_handle
;
3231 xcmd
.srq_type
= IB_SRQT_BASIC
;
3232 xcmd
.pd_handle
= cmd
.pd_handle
;
3233 xcmd
.max_wr
= cmd
.max_wr
;
3234 xcmd
.max_sge
= cmd
.max_sge
;
3235 xcmd
.srq_limit
= cmd
.srq_limit
;
3237 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
3238 (unsigned long) cmd
.response
+ sizeof resp
,
3239 in_len
- sizeof cmd
, out_len
- sizeof resp
);
3241 ret
= __uverbs_create_xsrq(file
, &xcmd
, &udata
);
3248 ssize_t
ib_uverbs_create_xsrq(struct ib_uverbs_file
*file
,
3249 const char __user
*buf
, int in_len
, int out_len
)
3251 struct ib_uverbs_create_xsrq cmd
;
3252 struct ib_uverbs_create_srq_resp resp
;
3253 struct ib_udata udata
;
3256 if (out_len
< sizeof resp
)
3259 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3262 INIT_UDATA(&udata
, buf
+ sizeof cmd
,
3263 (unsigned long) cmd
.response
+ sizeof resp
,
3264 in_len
- sizeof cmd
, out_len
- sizeof resp
);
3266 ret
= __uverbs_create_xsrq(file
, &cmd
, &udata
);
3273 ssize_t
ib_uverbs_modify_srq(struct ib_uverbs_file
*file
,
3274 const char __user
*buf
, int in_len
,
3277 struct ib_uverbs_modify_srq cmd
;
3278 struct ib_udata udata
;
3280 struct ib_srq_attr attr
;
3283 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3286 INIT_UDATA(&udata
, buf
+ sizeof cmd
, NULL
, in_len
- sizeof cmd
,
3289 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
3293 attr
.max_wr
= cmd
.max_wr
;
3294 attr
.srq_limit
= cmd
.srq_limit
;
3296 ret
= srq
->device
->modify_srq(srq
, &attr
, cmd
.attr_mask
, &udata
);
3300 return ret
? ret
: in_len
;
3303 ssize_t
ib_uverbs_query_srq(struct ib_uverbs_file
*file
,
3304 const char __user
*buf
,
3305 int in_len
, int out_len
)
3307 struct ib_uverbs_query_srq cmd
;
3308 struct ib_uverbs_query_srq_resp resp
;
3309 struct ib_srq_attr attr
;
3313 if (out_len
< sizeof resp
)
3316 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3319 srq
= idr_read_srq(cmd
.srq_handle
, file
->ucontext
);
3323 ret
= ib_query_srq(srq
, &attr
);
3330 memset(&resp
, 0, sizeof resp
);
3332 resp
.max_wr
= attr
.max_wr
;
3333 resp
.max_sge
= attr
.max_sge
;
3334 resp
.srq_limit
= attr
.srq_limit
;
3336 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
3337 &resp
, sizeof resp
))
3343 ssize_t
ib_uverbs_destroy_srq(struct ib_uverbs_file
*file
,
3344 const char __user
*buf
, int in_len
,
3347 struct ib_uverbs_destroy_srq cmd
;
3348 struct ib_uverbs_destroy_srq_resp resp
;
3349 struct ib_uobject
*uobj
;
3351 struct ib_uevent_object
*obj
;
3353 struct ib_usrq_object
*us
;
3354 enum ib_srq_type srq_type
;
3356 if (copy_from_user(&cmd
, buf
, sizeof cmd
))
3359 uobj
= idr_write_uobj(&ib_uverbs_srq_idr
, cmd
.srq_handle
, file
->ucontext
);
3363 obj
= container_of(uobj
, struct ib_uevent_object
, uobject
);
3364 srq_type
= srq
->srq_type
;
3366 ret
= ib_destroy_srq(srq
);
3370 put_uobj_write(uobj
);
3375 if (srq_type
== IB_SRQT_XRC
) {
3376 us
= container_of(obj
, struct ib_usrq_object
, uevent
);
3377 atomic_dec(&us
->uxrcd
->refcnt
);
3380 idr_remove_uobj(&ib_uverbs_srq_idr
, uobj
);
3382 mutex_lock(&file
->mutex
);
3383 list_del(&uobj
->list
);
3384 mutex_unlock(&file
->mutex
);
3386 ib_uverbs_release_uevent(file
, obj
);
3388 memset(&resp
, 0, sizeof resp
);
3389 resp
.events_reported
= obj
->events_reported
;
3393 if (copy_to_user((void __user
*) (unsigned long) cmd
.response
,
3394 &resp
, sizeof resp
))
3397 return ret
? ret
: in_len
;
3400 int ib_uverbs_ex_query_device(struct ib_uverbs_file
*file
,
3401 struct ib_udata
*ucore
,
3402 struct ib_udata
*uhw
)
3404 struct ib_uverbs_ex_query_device_resp resp
;
3405 struct ib_uverbs_ex_query_device cmd
;
3406 struct ib_device_attr attr
;
3407 struct ib_device
*device
;
3410 device
= file
->device
->ib_dev
;
3411 if (ucore
->inlen
< sizeof(cmd
))
3414 err
= ib_copy_from_udata(&cmd
, ucore
, sizeof(cmd
));
3424 resp
.response_length
= offsetof(typeof(resp
), odp_caps
);
3426 if (ucore
->outlen
< resp
.response_length
)
3429 memset(&attr
, 0, sizeof(attr
));
3431 err
= device
->query_device(device
, &attr
, uhw
);
3435 copy_query_dev_fields(file
, &resp
.base
, &attr
);
3438 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.odp_caps
))
3441 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
3442 resp
.odp_caps
.general_caps
= attr
.odp_caps
.general_caps
;
3443 resp
.odp_caps
.per_transport_caps
.rc_odp_caps
=
3444 attr
.odp_caps
.per_transport_caps
.rc_odp_caps
;
3445 resp
.odp_caps
.per_transport_caps
.uc_odp_caps
=
3446 attr
.odp_caps
.per_transport_caps
.uc_odp_caps
;
3447 resp
.odp_caps
.per_transport_caps
.ud_odp_caps
=
3448 attr
.odp_caps
.per_transport_caps
.ud_odp_caps
;
3449 resp
.odp_caps
.reserved
= 0;
3451 memset(&resp
.odp_caps
, 0, sizeof(resp
.odp_caps
));
3453 resp
.response_length
+= sizeof(resp
.odp_caps
);
3455 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.timestamp_mask
))
3458 resp
.timestamp_mask
= attr
.timestamp_mask
;
3459 resp
.response_length
+= sizeof(resp
.timestamp_mask
);
3461 if (ucore
->outlen
< resp
.response_length
+ sizeof(resp
.hca_core_clock
))
3464 resp
.hca_core_clock
= attr
.hca_core_clock
;
3465 resp
.response_length
+= sizeof(resp
.hca_core_clock
);
3468 err
= ib_copy_to_udata(ucore
, &resp
, resp
.response_length
);