projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
ALSA: oxfw: add stream format quirk for SCS.1 models
[deliverable/linux.git] / drivers / infiniband / core / uverbs_cmd.c
1 /*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 */
35
36 #include <linux/file.h>
37 #include <linux/fs.h>
38 #include <linux/slab.h>
39 #include <linux/sched.h>
40
41 #include <asm/uaccess.h>
42
43 #include "uverbs.h"
44 #include "core_priv.h"
45
46 struct uverbs_lock_class {
47 struct lock_class_key key;
48 char name[16];
49 };
50
51 static struct uverbs_lock_class pd_lock_class = { .name = "PD-uobj" };
52 static struct uverbs_lock_class mr_lock_class = { .name = "MR-uobj" };
53 static struct uverbs_lock_class mw_lock_class = { .name = "MW-uobj" };
54 static struct uverbs_lock_class cq_lock_class = { .name = "CQ-uobj" };
55 static struct uverbs_lock_class qp_lock_class = { .name = "QP-uobj" };
56 static struct uverbs_lock_class ah_lock_class = { .name = "AH-uobj" };
57 static struct uverbs_lock_class srq_lock_class = { .name = "SRQ-uobj" };
58 static struct uverbs_lock_class xrcd_lock_class = { .name = "XRCD-uobj" };
59 static struct uverbs_lock_class rule_lock_class = { .name = "RULE-uobj" };
60
61 /*
62 * The ib_uobject locking scheme is as follows:
63 *
64 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it
65 * needs to be held during all idr operations. When an object is
66 * looked up, a reference must be taken on the object's kref before
67 * dropping this lock.
68 *
69 * - Each object also has an rwsem. This rwsem must be held for
70 * reading while an operation that uses the object is performed.
71 * For example, while registering an MR, the associated PD's
72 * uobject.mutex must be held for reading. The rwsem must be held
73 * for writing while initializing or destroying an object.
74 *
75 * - In addition, each object has a "live" flag. If this flag is not
76 * set, then lookups of the object will fail even if it is found in
77 * the idr. This handles a reader that blocks and does not acquire
78 * the rwsem until after the object is destroyed. The destroy
79 * operation will set the live flag to 0 and then drop the rwsem;
80 * this will allow the reader to acquire the rwsem, see that the
81 * live flag is 0, and then drop the rwsem and its reference to
82 * object. The underlying storage will not be freed until the last
83 * reference to the object is dropped.
84 */
85
86 static void init_uobj(struct ib_uobject *uobj, u64 user_handle,
87 struct ib_ucontext *context, struct uverbs_lock_class *c)
88 {
89 uobj->user_handle = user_handle;
90 uobj->context = context;
91 kref_init(&uobj->ref);
92 init_rwsem(&uobj->mutex);
93 lockdep_set_class_and_name(&uobj->mutex, &c->key, c->name);
94 uobj->live = 0;
95 }
96
97 static void release_uobj(struct kref *kref)
98 {
99 kfree(container_of(kref, struct ib_uobject, ref));
100 }
101
102 static void put_uobj(struct ib_uobject *uobj)
103 {
104 kref_put(&uobj->ref, release_uobj);
105 }
106
107 static void put_uobj_read(struct ib_uobject *uobj)
108 {
109 up_read(&uobj->mutex);
110 put_uobj(uobj);
111 }
112
113 static void put_uobj_write(struct ib_uobject *uobj)
114 {
115 up_write(&uobj->mutex);
116 put_uobj(uobj);
117 }
118
119 static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj)
120 {
121 int ret;
122
123 idr_preload(GFP_KERNEL);
124 spin_lock(&ib_uverbs_idr_lock);
125
126 ret = idr_alloc(idr, uobj, 0, 0, GFP_NOWAIT);
127 if (ret >= 0)
128 uobj->id = ret;
129
130 spin_unlock(&ib_uverbs_idr_lock);
131 idr_preload_end();
132
133 return ret < 0 ? ret : 0;
134 }
135
136 void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj)
137 {
138 spin_lock(&ib_uverbs_idr_lock);
139 idr_remove(idr, uobj->id);
140 spin_unlock(&ib_uverbs_idr_lock);
141 }
142
143 static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id,
144 struct ib_ucontext *context)
145 {
146 struct ib_uobject *uobj;
147
148 spin_lock(&ib_uverbs_idr_lock);
149 uobj = idr_find(idr, id);
150 if (uobj) {
151 if (uobj->context == context)
152 kref_get(&uobj->ref);
153 else
154 uobj = NULL;
155 }
156 spin_unlock(&ib_uverbs_idr_lock);
157
158 return uobj;
159 }
160
161 static struct ib_uobject *idr_read_uobj(struct idr *idr, int id,
162 struct ib_ucontext *context, int nested)
163 {
164 struct ib_uobject *uobj;
165
166 uobj = __idr_get_uobj(idr, id, context);
167 if (!uobj)
168 return NULL;
169
170 if (nested)
171 down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING);
172 else
173 down_read(&uobj->mutex);
174 if (!uobj->live) {
175 put_uobj_read(uobj);
176 return NULL;
177 }
178
179 return uobj;
180 }
181
182 static struct ib_uobject *idr_write_uobj(struct idr *idr, int id,
183 struct ib_ucontext *context)
184 {
185 struct ib_uobject *uobj;
186
187 uobj = __idr_get_uobj(idr, id, context);
188 if (!uobj)
189 return NULL;
190
191 down_write(&uobj->mutex);
192 if (!uobj->live) {
193 put_uobj_write(uobj);
194 return NULL;
195 }
196
197 return uobj;
198 }
199
200 static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context,
201 int nested)
202 {
203 struct ib_uobject *uobj;
204
205 uobj = idr_read_uobj(idr, id, context, nested);
206 return uobj ? uobj->object : NULL;
207 }
208
209 static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context)
210 {
211 return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0);
212 }
213
214 static void put_pd_read(struct ib_pd *pd)
215 {
216 put_uobj_read(pd->uobject);
217 }
218
219 static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested)
220 {
221 return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested);
222 }
223
224 static void put_cq_read(struct ib_cq *cq)
225 {
226 put_uobj_read(cq->uobject);
227 }
228
229 static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context)
230 {
231 return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0);
232 }
233
234 static void put_ah_read(struct ib_ah *ah)
235 {
236 put_uobj_read(ah->uobject);
237 }
238
239 static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context)
240 {
241 return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0);
242 }
243
244 static struct ib_qp *idr_write_qp(int qp_handle, struct ib_ucontext *context)
245 {
246 struct ib_uobject *uobj;
247
248 uobj = idr_write_uobj(&ib_uverbs_qp_idr, qp_handle, context);
249 return uobj ? uobj->object : NULL;
250 }
251
252 static void put_qp_read(struct ib_qp *qp)
253 {
254 put_uobj_read(qp->uobject);
255 }
256
257 static void put_qp_write(struct ib_qp *qp)
258 {
259 put_uobj_write(qp->uobject);
260 }
261
262 static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context)
263 {
264 return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0);
265 }
266
267 static void put_srq_read(struct ib_srq *srq)
268 {
269 put_uobj_read(srq->uobject);
270 }
271
272 static struct ib_xrcd *idr_read_xrcd(int xrcd_handle, struct ib_ucontext *context,
273 struct ib_uobject **uobj)
274 {
275 *uobj = idr_read_uobj(&ib_uverbs_xrcd_idr, xrcd_handle, context, 0);
276 return *uobj ? (*uobj)->object : NULL;
277 }
278
279 static void put_xrcd_read(struct ib_uobject *uobj)
280 {
281 put_uobj_read(uobj);
282 }
283
284 ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
285 struct ib_device *ib_dev,
286 const char __user *buf,
287 int in_len, int out_len)
288 {
289 struct ib_uverbs_get_context cmd;
290 struct ib_uverbs_get_context_resp resp;
291 struct ib_udata udata;
292 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
293 struct ib_device_attr dev_attr;
294 #endif
295 struct ib_ucontext *ucontext;
296 struct file *filp;
297 int ret;
298
299 if (out_len < sizeof resp)
300 return -ENOSPC;
301
302 if (copy_from_user(&cmd, buf, sizeof cmd))
303 return -EFAULT;
304
305 mutex_lock(&file->mutex);
306
307 if (file->ucontext) {
308 ret = -EINVAL;
309 goto err;
310 }
311
312 INIT_UDATA(&udata, buf + sizeof cmd,
313 (unsigned long) cmd.response + sizeof resp,
314 in_len - sizeof cmd, out_len - sizeof resp);
315
316 ucontext = ib_dev->alloc_ucontext(ib_dev, &udata);
317 if (IS_ERR(ucontext)) {
318 ret = PTR_ERR(ucontext);
319 goto err;
320 }
321
322 ucontext->device = ib_dev;
323 INIT_LIST_HEAD(&ucontext->pd_list);
324 INIT_LIST_HEAD(&ucontext->mr_list);
325 INIT_LIST_HEAD(&ucontext->mw_list);
326 INIT_LIST_HEAD(&ucontext->cq_list);
327 INIT_LIST_HEAD(&ucontext->qp_list);
328 INIT_LIST_HEAD(&ucontext->srq_list);
329 INIT_LIST_HEAD(&ucontext->ah_list);
330 INIT_LIST_HEAD(&ucontext->xrcd_list);
331 INIT_LIST_HEAD(&ucontext->rule_list);
332 rcu_read_lock();
333 ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
334 rcu_read_unlock();
335 ucontext->closing = 0;
336
337 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
338 ucontext->umem_tree = RB_ROOT;
339 init_rwsem(&ucontext->umem_rwsem);
340 ucontext->odp_mrs_count = 0;
341 INIT_LIST_HEAD(&ucontext->no_private_counters);
342
343 ret = ib_query_device(ib_dev, &dev_attr);
344 if (ret)
345 goto err_free;
346 if (!(dev_attr.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
347 ucontext->invalidate_range = NULL;
348
349 #endif
350
351 resp.num_comp_vectors = file->device->num_comp_vectors;
352
353 ret = get_unused_fd_flags(O_CLOEXEC);
354 if (ret < 0)
355 goto err_free;
356 resp.async_fd = ret;
357
358 filp = ib_uverbs_alloc_event_file(file, ib_dev, 1);
359 if (IS_ERR(filp)) {
360 ret = PTR_ERR(filp);
361 goto err_fd;
362 }
363
364 if (copy_to_user((void __user *) (unsigned long) cmd.response,
365 &resp, sizeof resp)) {
366 ret = -EFAULT;
367 goto err_file;
368 }
369
370 file->ucontext = ucontext;
371
372 fd_install(resp.async_fd, filp);
373
374 mutex_unlock(&file->mutex);
375
376 return in_len;
377
378 err_file:
379 ib_uverbs_free_async_event_file(file);
380 fput(filp);
381
382 err_fd:
383 put_unused_fd(resp.async_fd);
384
385 err_free:
386 put_pid(ucontext->tgid);
387 ib_dev->dealloc_ucontext(ucontext);
388
389 err:
390 mutex_unlock(&file->mutex);
391 return ret;
392 }
393
394 static void copy_query_dev_fields(struct ib_uverbs_file *file,
395 struct ib_device *ib_dev,
396 struct ib_uverbs_query_device_resp *resp,
397 struct ib_device_attr *attr)
398 {
399 resp->fw_ver = attr->fw_ver;
400 resp->node_guid = ib_dev->node_guid;
401 resp->sys_image_guid = attr->sys_image_guid;
402 resp->max_mr_size = attr->max_mr_size;
403 resp->page_size_cap = attr->page_size_cap;
404 resp->vendor_id = attr->vendor_id;
405 resp->vendor_part_id = attr->vendor_part_id;
406 resp->hw_ver = attr->hw_ver;
407 resp->max_qp = attr->max_qp;
408 resp->max_qp_wr = attr->max_qp_wr;
409 resp->device_cap_flags = attr->device_cap_flags;
410 resp->max_sge = attr->max_sge;
411 resp->max_sge_rd = attr->max_sge_rd;
412 resp->max_cq = attr->max_cq;
413 resp->max_cqe = attr->max_cqe;
414 resp->max_mr = attr->max_mr;
415 resp->max_pd = attr->max_pd;
416 resp->max_qp_rd_atom = attr->max_qp_rd_atom;
417 resp->max_ee_rd_atom = attr->max_ee_rd_atom;
418 resp->max_res_rd_atom = attr->max_res_rd_atom;
419 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom;
420 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom;
421 resp->atomic_cap = attr->atomic_cap;
422 resp->max_ee = attr->max_ee;
423 resp->max_rdd = attr->max_rdd;
424 resp->max_mw = attr->max_mw;
425 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp;
426 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp;
427 resp->max_mcast_grp = attr->max_mcast_grp;
428 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach;
429 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach;
430 resp->max_ah = attr->max_ah;
431 resp->max_fmr = attr->max_fmr;
432 resp->max_map_per_fmr = attr->max_map_per_fmr;
433 resp->max_srq = attr->max_srq;
434 resp->max_srq_wr = attr->max_srq_wr;
435 resp->max_srq_sge = attr->max_srq_sge;
436 resp->max_pkeys = attr->max_pkeys;
437 resp->local_ca_ack_delay = attr->local_ca_ack_delay;
438 resp->phys_port_cnt = ib_dev->phys_port_cnt;
439 }
440
441 ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file,
442 struct ib_device *ib_dev,
443 const char __user *buf,
444 int in_len, int out_len)
445 {
446 struct ib_uverbs_query_device cmd;
447 struct ib_uverbs_query_device_resp resp;
448 struct ib_device_attr attr;
449 int ret;
450
451 if (out_len < sizeof resp)
452 return -ENOSPC;
453
454 if (copy_from_user(&cmd, buf, sizeof cmd))
455 return -EFAULT;
456
457 ret = ib_query_device(ib_dev, &attr);
458 if (ret)
459 return ret;
460
461 memset(&resp, 0, sizeof resp);
462 copy_query_dev_fields(file, ib_dev, &resp, &attr);
463
464 if (copy_to_user((void __user *) (unsigned long) cmd.response,
465 &resp, sizeof resp))
466 return -EFAULT;
467
468 return in_len;
469 }
470
471 ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file,
472 struct ib_device *ib_dev,
473 const char __user *buf,
474 int in_len, int out_len)
475 {
476 struct ib_uverbs_query_port cmd;
477 struct ib_uverbs_query_port_resp resp;
478 struct ib_port_attr attr;
479 int ret;
480
481 if (out_len < sizeof resp)
482 return -ENOSPC;
483
484 if (copy_from_user(&cmd, buf, sizeof cmd))
485 return -EFAULT;
486
487 ret = ib_query_port(ib_dev, cmd.port_num, &attr);
488 if (ret)
489 return ret;
490
491 memset(&resp, 0, sizeof resp);
492
493 resp.state = attr.state;
494 resp.max_mtu = attr.max_mtu;
495 resp.active_mtu = attr.active_mtu;
496 resp.gid_tbl_len = attr.gid_tbl_len;
497 resp.port_cap_flags = attr.port_cap_flags;
498 resp.max_msg_sz = attr.max_msg_sz;
499 resp.bad_pkey_cntr = attr.bad_pkey_cntr;
500 resp.qkey_viol_cntr = attr.qkey_viol_cntr;
501 resp.pkey_tbl_len = attr.pkey_tbl_len;
502 resp.lid = attr.lid;
503 resp.sm_lid = attr.sm_lid;
504 resp.lmc = attr.lmc;
505 resp.max_vl_num = attr.max_vl_num;
506 resp.sm_sl = attr.sm_sl;
507 resp.subnet_timeout = attr.subnet_timeout;
508 resp.init_type_reply = attr.init_type_reply;
509 resp.active_width = attr.active_width;
510 resp.active_speed = attr.active_speed;
511 resp.phys_state = attr.phys_state;
512 resp.link_layer = rdma_port_get_link_layer(ib_dev,
513 cmd.port_num);
514
515 if (copy_to_user((void __user *) (unsigned long) cmd.response,
516 &resp, sizeof resp))
517 return -EFAULT;
518
519 return in_len;
520 }
521
522 ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file,
523 struct ib_device *ib_dev,
524 const char __user *buf,
525 int in_len, int out_len)
526 {
527 struct ib_uverbs_alloc_pd cmd;
528 struct ib_uverbs_alloc_pd_resp resp;
529 struct ib_udata udata;
530 struct ib_uobject *uobj;
531 struct ib_pd *pd;
532 int ret;
533
534 if (out_len < sizeof resp)
535 return -ENOSPC;
536
537 if (copy_from_user(&cmd, buf, sizeof cmd))
538 return -EFAULT;
539
540 INIT_UDATA(&udata, buf + sizeof cmd,
541 (unsigned long) cmd.response + sizeof resp,
542 in_len - sizeof cmd, out_len - sizeof resp);
543
544 uobj = kmalloc(sizeof *uobj, GFP_KERNEL);
545 if (!uobj)
546 return -ENOMEM;
547
548 init_uobj(uobj, 0, file->ucontext, &pd_lock_class);
549 down_write(&uobj->mutex);
550
551 pd = ib_dev->alloc_pd(ib_dev, file->ucontext, &udata);
552 if (IS_ERR(pd)) {
553 ret = PTR_ERR(pd);
554 goto err;
555 }
556
557 pd->device = ib_dev;
558 pd->uobject = uobj;
559 pd->local_mr = NULL;
560 atomic_set(&pd->usecnt, 0);
561
562 uobj->object = pd;
563 ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj);
564 if (ret)
565 goto err_idr;
566
567 memset(&resp, 0, sizeof resp);
568 resp.pd_handle = uobj->id;
569
570 if (copy_to_user((void __user *) (unsigned long) cmd.response,
571 &resp, sizeof resp)) {
572 ret = -EFAULT;
573 goto err_copy;
574 }
575
576 mutex_lock(&file->mutex);
577 list_add_tail(&uobj->list, &file->ucontext->pd_list);
578 mutex_unlock(&file->mutex);
579
580 uobj->live = 1;
581
582 up_write(&uobj->mutex);
583
584 return in_len;
585
586 err_copy:
587 idr_remove_uobj(&ib_uverbs_pd_idr, uobj);
588
589 err_idr:
590 ib_dealloc_pd(pd);
591
592 err:
593 put_uobj_write(uobj);
594 return ret;
595 }
596
597 ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file,
598 struct ib_device *ib_dev,
599 const char __user *buf,
600 int in_len, int out_len)
601 {
602 struct ib_uverbs_dealloc_pd cmd;
603 struct ib_uobject *uobj;
604 struct ib_pd *pd;
605 int ret;
606
607 if (copy_from_user(&cmd, buf, sizeof cmd))
608 return -EFAULT;
609
610 uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext);
611 if (!uobj)
612 return -EINVAL;
613 pd = uobj->object;
614
615 if (atomic_read(&pd->usecnt)) {
616 ret = -EBUSY;
617 goto err_put;
618 }
619
620 ret = pd->device->dealloc_pd(uobj->object);
621 WARN_ONCE(ret, "Infiniband HW driver failed dealloc_pd");
622 if (ret)
623 goto err_put;
624
625 uobj->live = 0;
626 put_uobj_write(uobj);
627
628 idr_remove_uobj(&ib_uverbs_pd_idr, uobj);
629
630 mutex_lock(&file->mutex);
631 list_del(&uobj->list);
632 mutex_unlock(&file->mutex);
633
634 put_uobj(uobj);
635
636 return in_len;
637
638 err_put:
639 put_uobj_write(uobj);
640 return ret;
641 }
642
643 struct xrcd_table_entry {
644 struct rb_node node;
645 struct ib_xrcd *xrcd;
646 struct inode *inode;
647 };
648
649 static int xrcd_table_insert(struct ib_uverbs_device *dev,
650 struct inode *inode,
651 struct ib_xrcd *xrcd)
652 {
653 struct xrcd_table_entry *entry, *scan;
654 struct rb_node **p = &dev->xrcd_tree.rb_node;
655 struct rb_node *parent = NULL;
656
657 entry = kmalloc(sizeof *entry, GFP_KERNEL);
658 if (!entry)
659 return -ENOMEM;
660
661 entry->xrcd = xrcd;
662 entry->inode = inode;
663
664 while (*p) {
665 parent = *p;
666 scan = rb_entry(parent, struct xrcd_table_entry, node);
667
668 if (inode < scan->inode) {
669 p = &(*p)->rb_left;
670 } else if (inode > scan->inode) {
671 p = &(*p)->rb_right;
672 } else {
673 kfree(entry);
674 return -EEXIST;
675 }
676 }
677
678 rb_link_node(&entry->node, parent, p);
679 rb_insert_color(&entry->node, &dev->xrcd_tree);
680 igrab(inode);
681 return 0;
682 }
683
684 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev,
685 struct inode *inode)
686 {
687 struct xrcd_table_entry *entry;
688 struct rb_node *p = dev->xrcd_tree.rb_node;
689
690 while (p) {
691 entry = rb_entry(p, struct xrcd_table_entry, node);
692
693 if (inode < entry->inode)
694 p = p->rb_left;
695 else if (inode > entry->inode)
696 p = p->rb_right;
697 else
698 return entry;
699 }
700
701 return NULL;
702 }
703
704 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode)
705 {
706 struct xrcd_table_entry *entry;
707
708 entry = xrcd_table_search(dev, inode);
709 if (!entry)
710 return NULL;
711
712 return entry->xrcd;
713 }
714
715 static void xrcd_table_delete(struct ib_uverbs_device *dev,
716 struct inode *inode)
717 {
718 struct xrcd_table_entry *entry;
719
720 entry = xrcd_table_search(dev, inode);
721 if (entry) {
722 iput(inode);
723 rb_erase(&entry->node, &dev->xrcd_tree);
724 kfree(entry);
725 }
726 }
727
728 ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
729 struct ib_device *ib_dev,
730 const char __user *buf, int in_len,
731 int out_len)
732 {
733 struct ib_uverbs_open_xrcd cmd;
734 struct ib_uverbs_open_xrcd_resp resp;
735 struct ib_udata udata;
736 struct ib_uxrcd_object *obj;
737 struct ib_xrcd *xrcd = NULL;
738 struct fd f = {NULL, 0};
739 struct inode *inode = NULL;
740 int ret = 0;
741 int new_xrcd = 0;
742
743 if (out_len < sizeof resp)
744 return -ENOSPC;
745
746 if (copy_from_user(&cmd, buf, sizeof cmd))
747 return -EFAULT;
748
749 INIT_UDATA(&udata, buf + sizeof cmd,
750 (unsigned long) cmd.response + sizeof resp,
751 in_len - sizeof cmd, out_len - sizeof resp);
752
753 mutex_lock(&file->device->xrcd_tree_mutex);
754
755 if (cmd.fd != -1) {
756 /* search for file descriptor */
757 f = fdget(cmd.fd);
758 if (!f.file) {
759 ret = -EBADF;
760 goto err_tree_mutex_unlock;
761 }
762
763 inode = file_inode(f.file);
764 xrcd = find_xrcd(file->device, inode);
765 if (!xrcd && !(cmd.oflags & O_CREAT)) {
766 /* no file descriptor. Need CREATE flag */
767 ret = -EAGAIN;
768 goto err_tree_mutex_unlock;
769 }
770
771 if (xrcd && cmd.oflags & O_EXCL) {
772 ret = -EINVAL;
773 goto err_tree_mutex_unlock;
774 }
775 }
776
777 obj = kmalloc(sizeof *obj, GFP_KERNEL);
778 if (!obj) {
779 ret = -ENOMEM;
780 goto err_tree_mutex_unlock;
781 }
782
783 init_uobj(&obj->uobject, 0, file->ucontext, &xrcd_lock_class);
784
785 down_write(&obj->uobject.mutex);
786
787 if (!xrcd) {
788 xrcd = ib_dev->alloc_xrcd(ib_dev, file->ucontext, &udata);
789 if (IS_ERR(xrcd)) {
790 ret = PTR_ERR(xrcd);
791 goto err;
792 }
793
794 xrcd->inode = inode;
795 xrcd->device = ib_dev;
796 atomic_set(&xrcd->usecnt, 0);
797 mutex_init(&xrcd->tgt_qp_mutex);
798 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
799 new_xrcd = 1;
800 }
801
802 atomic_set(&obj->refcnt, 0);
803 obj->uobject.object = xrcd;
804 ret = idr_add_uobj(&ib_uverbs_xrcd_idr, &obj->uobject);
805 if (ret)
806 goto err_idr;
807
808 memset(&resp, 0, sizeof resp);
809 resp.xrcd_handle = obj->uobject.id;
810
811 if (inode) {
812 if (new_xrcd) {
813 /* create new inode/xrcd table entry */
814 ret = xrcd_table_insert(file->device, inode, xrcd);
815 if (ret)
816 goto err_insert_xrcd;
817 }
818 atomic_inc(&xrcd->usecnt);
819 }
820
821 if (copy_to_user((void __user *) (unsigned long) cmd.response,
822 &resp, sizeof resp)) {
823 ret = -EFAULT;
824 goto err_copy;
825 }
826
827 if (f.file)
828 fdput(f);
829
830 mutex_lock(&file->mutex);
831 list_add_tail(&obj->uobject.list, &file->ucontext->xrcd_list);
832 mutex_unlock(&file->mutex);
833
834 obj->uobject.live = 1;
835 up_write(&obj->uobject.mutex);
836
837 mutex_unlock(&file->device->xrcd_tree_mutex);
838 return in_len;
839
840 err_copy:
841 if (inode) {
842 if (new_xrcd)
843 xrcd_table_delete(file->device, inode);
844 atomic_dec(&xrcd->usecnt);
845 }
846
847 err_insert_xrcd:
848 idr_remove_uobj(&ib_uverbs_xrcd_idr, &obj->uobject);
849
850 err_idr:
851 ib_dealloc_xrcd(xrcd);
852
853 err:
854 put_uobj_write(&obj->uobject);
855
856 err_tree_mutex_unlock:
857 if (f.file)
858 fdput(f);
859
860 mutex_unlock(&file->device->xrcd_tree_mutex);
861
862 return ret;
863 }
864
865 ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file,
866 struct ib_device *ib_dev,
867 const char __user *buf, int in_len,
868 int out_len)
869 {
870 struct ib_uverbs_close_xrcd cmd;
871 struct ib_uobject *uobj;
872 struct ib_xrcd *xrcd = NULL;
873 struct inode *inode = NULL;
874 struct ib_uxrcd_object *obj;
875 int live;
876 int ret = 0;
877
878 if (copy_from_user(&cmd, buf, sizeof cmd))
879 return -EFAULT;
880
881 mutex_lock(&file->device->xrcd_tree_mutex);
882 uobj = idr_write_uobj(&ib_uverbs_xrcd_idr, cmd.xrcd_handle, file->ucontext);
883 if (!uobj) {
884 ret = -EINVAL;
885 goto out;
886 }
887
888 xrcd = uobj->object;
889 inode = xrcd->inode;
890 obj = container_of(uobj, struct ib_uxrcd_object, uobject);
891 if (atomic_read(&obj->refcnt)) {
892 put_uobj_write(uobj);
893 ret = -EBUSY;
894 goto out;
895 }
896
897 if (!inode || atomic_dec_and_test(&xrcd->usecnt)) {
898 ret = ib_dealloc_xrcd(uobj->object);
899 if (!ret)
900 uobj->live = 0;
901 }
902
903 live = uobj->live;
904 if (inode && ret)
905 atomic_inc(&xrcd->usecnt);
906
907 put_uobj_write(uobj);
908
909 if (ret)
910 goto out;
911
912 if (inode && !live)
913 xrcd_table_delete(file->device, inode);
914
915 idr_remove_uobj(&ib_uverbs_xrcd_idr, uobj);
916 mutex_lock(&file->mutex);
917 list_del(&uobj->list);
918 mutex_unlock(&file->mutex);
919
920 put_uobj(uobj);
921 ret = in_len;
922
923 out:
924 mutex_unlock(&file->device->xrcd_tree_mutex);
925 return ret;
926 }
927
928 void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev,
929 struct ib_xrcd *xrcd)
930 {
931 struct inode *inode;
932
933 inode = xrcd->inode;
934 if (inode && !atomic_dec_and_test(&xrcd->usecnt))
935 return;
936
937 ib_dealloc_xrcd(xrcd);
938
939 if (inode)
940 xrcd_table_delete(dev, inode);
941 }
942
943 ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file,
944 struct ib_device *ib_dev,
945 const char __user *buf, int in_len,
946 int out_len)
947 {
948 struct ib_uverbs_reg_mr cmd;
949 struct ib_uverbs_reg_mr_resp resp;
950 struct ib_udata udata;
951 struct ib_uobject *uobj;
952 struct ib_pd *pd;
953 struct ib_mr *mr;
954 int ret;
955
956 if (out_len < sizeof resp)
957 return -ENOSPC;
958
959 if (copy_from_user(&cmd, buf, sizeof cmd))
960 return -EFAULT;
961
962 INIT_UDATA(&udata, buf + sizeof cmd,
963 (unsigned long) cmd.response + sizeof resp,
964 in_len - sizeof cmd, out_len - sizeof resp);
965
966 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
967 return -EINVAL;
968
969 ret = ib_check_mr_access(cmd.access_flags);
970 if (ret)
971 return ret;
972
973 uobj = kmalloc(sizeof *uobj, GFP_KERNEL);
974 if (!uobj)
975 return -ENOMEM;
976
977 init_uobj(uobj, 0, file->ucontext, &mr_lock_class);
978 down_write(&uobj->mutex);
979
980 pd = idr_read_pd(cmd.pd_handle, file->ucontext);
981 if (!pd) {
982 ret = -EINVAL;
983 goto err_free;
984 }
985
986 if (cmd.access_flags & IB_ACCESS_ON_DEMAND) {
987 struct ib_device_attr attr;
988
989 ret = ib_query_device(pd->device, &attr);
990 if (ret || !(attr.device_cap_flags &
991 IB_DEVICE_ON_DEMAND_PAGING)) {
992 pr_debug("ODP support not available\n");
993 ret = -EINVAL;
994 goto err_put;
995 }
996 }
997
998 mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
999 cmd.access_flags, &udata);
1000 if (IS_ERR(mr)) {
1001 ret = PTR_ERR(mr);
1002 goto err_put;
1003 }
1004
1005 mr->device = pd->device;
1006 mr->pd = pd;
1007 mr->uobject = uobj;
1008 atomic_inc(&pd->usecnt);
1009 atomic_set(&mr->usecnt, 0);
1010
1011 uobj->object = mr;
1012 ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj);
1013 if (ret)
1014 goto err_unreg;
1015
1016 memset(&resp, 0, sizeof resp);
1017 resp.lkey = mr->lkey;
1018 resp.rkey = mr->rkey;
1019 resp.mr_handle = uobj->id;
1020
1021 if (copy_to_user((void __user *) (unsigned long) cmd.response,
1022 &resp, sizeof resp)) {
1023 ret = -EFAULT;
1024 goto err_copy;
1025 }
1026
1027 put_pd_read(pd);
1028
1029 mutex_lock(&file->mutex);
1030 list_add_tail(&uobj->list, &file->ucontext->mr_list);
1031 mutex_unlock(&file->mutex);
1032
1033 uobj->live = 1;
1034
1035 up_write(&uobj->mutex);
1036
1037 return in_len;
1038
1039 err_copy:
1040 idr_remove_uobj(&ib_uverbs_mr_idr, uobj);
1041
1042 err_unreg:
1043 ib_dereg_mr(mr);
1044
1045 err_put:
1046 put_pd_read(pd);
1047
1048 err_free:
1049 put_uobj_write(uobj);
1050 return ret;
1051 }
1052
1053 ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file,
1054 struct ib_device *ib_dev,
1055 const char __user *buf, int in_len,
1056 int out_len)
1057 {
1058 struct ib_uverbs_rereg_mr cmd;
1059 struct ib_uverbs_rereg_mr_resp resp;
1060 struct ib_udata udata;
1061 struct ib_pd *pd = NULL;
1062 struct ib_mr *mr;
1063 struct ib_pd *old_pd;
1064 int ret;
1065 struct ib_uobject *uobj;
1066
1067 if (out_len < sizeof(resp))
1068 return -ENOSPC;
1069
1070 if (copy_from_user(&cmd, buf, sizeof(cmd)))
1071 return -EFAULT;
1072
1073 INIT_UDATA(&udata, buf + sizeof(cmd),
1074 (unsigned long) cmd.response + sizeof(resp),
1075 in_len - sizeof(cmd), out_len - sizeof(resp));
1076
1077 if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags)
1078 return -EINVAL;
1079
1080 if ((cmd.flags & IB_MR_REREG_TRANS) &&
1081 (!cmd.start || !cmd.hca_va || 0 >= cmd.length ||
1082 (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)))
1083 return -EINVAL;
1084
1085 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle,
1086 file->ucontext);
1087
1088 if (!uobj)
1089 return -EINVAL;
1090
1091 mr = uobj->object;
1092
1093 if (cmd.flags & IB_MR_REREG_ACCESS) {
1094 ret = ib_check_mr_access(cmd.access_flags);
1095 if (ret)
1096 goto put_uobjs;
1097 }
1098
1099 if (cmd.flags & IB_MR_REREG_PD) {
1100 pd = idr_read_pd(cmd.pd_handle, file->ucontext);
1101 if (!pd) {
1102 ret = -EINVAL;
1103 goto put_uobjs;
1104 }
1105 }
1106
1107 if (atomic_read(&mr->usecnt)) {
1108 ret = -EBUSY;
1109 goto put_uobj_pd;
1110 }
1111
1112 old_pd = mr->pd;
1113 ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start,
1114 cmd.length, cmd.hca_va,
1115 cmd.access_flags, pd, &udata);
1116 if (!ret) {
1117 if (cmd.flags & IB_MR_REREG_PD) {
1118 atomic_inc(&pd->usecnt);
1119 mr->pd = pd;
1120 atomic_dec(&old_pd->usecnt);
1121 }
1122 } else {
1123 goto put_uobj_pd;
1124 }
1125
1126 memset(&resp, 0, sizeof(resp));
1127 resp.lkey = mr->lkey;
1128 resp.rkey = mr->rkey;
1129
1130 if (copy_to_user((void __user *)(unsigned long)cmd.response,
1131 &resp, sizeof(resp)))
1132 ret = -EFAULT;
1133 else
1134 ret = in_len;
1135
1136 put_uobj_pd:
1137 if (cmd.flags & IB_MR_REREG_PD)
1138 put_pd_read(pd);
1139
1140 put_uobjs:
1141
1142 put_uobj_write(mr->uobject);
1143
1144 return ret;
1145 }
1146
1147 ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file,
1148 struct ib_device *ib_dev,
1149 const char __user *buf, int in_len,
1150 int out_len)
1151 {
1152 struct ib_uverbs_dereg_mr cmd;
1153 struct ib_mr *mr;
1154 struct ib_uobject *uobj;
1155 int ret = -EINVAL;
1156
1157 if (copy_from_user(&cmd, buf, sizeof cmd))
1158 return -EFAULT;
1159
1160 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext);
1161 if (!uobj)
1162 return -EINVAL;
1163
1164 mr = uobj->object;
1165
1166 ret = ib_dereg_mr(mr);
1167 if (!ret)
1168 uobj->live = 0;
1169
1170 put_uobj_write(uobj);
1171
1172 if (ret)
1173 return ret;
1174
1175 idr_remove_uobj(&ib_uverbs_mr_idr, uobj);
1176
1177 mutex_lock(&file->mutex);
1178 list_del(&uobj->list);
1179 mutex_unlock(&file->mutex);
1180
1181 put_uobj(uobj);
1182
1183 return in_len;
1184 }
1185
1186 ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file,
1187 struct ib_device *ib_dev,
1188 const char __user *buf, int in_len,
1189 int out_len)
1190 {
1191 struct ib_uverbs_alloc_mw cmd;
1192 struct ib_uverbs_alloc_mw_resp resp;
1193 struct ib_uobject *uobj;
1194 struct ib_pd *pd;
1195 struct ib_mw *mw;
1196 int ret;
1197
1198 if (out_len < sizeof(resp))
1199 return -ENOSPC;
1200
1201 if (copy_from_user(&cmd, buf, sizeof(cmd)))
1202 return -EFAULT;
1203
1204 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL);
1205 if (!uobj)
1206 return -ENOMEM;
1207
1208 init_uobj(uobj, 0, file->ucontext, &mw_lock_class);
1209 down_write(&uobj->mutex);
1210
1211 pd = idr_read_pd(cmd.pd_handle, file->ucontext);
1212 if (!pd) {
1213 ret = -EINVAL;
1214 goto err_free;
1215 }
1216
1217 mw = pd->device->alloc_mw(pd, cmd.mw_type);
1218 if (IS_ERR(mw)) {
1219 ret = PTR_ERR(mw);
1220 goto err_put;
1221 }
1222
1223 mw->device = pd->device;
1224 mw->pd = pd;
1225 mw->uobject = uobj;
1226 atomic_inc(&pd->usecnt);
1227
1228 uobj->object = mw;
1229 ret = idr_add_uobj(&ib_uverbs_mw_idr, uobj);
1230 if (ret)
1231 goto err_unalloc;
1232
1233 memset(&resp, 0, sizeof(resp));
1234 resp.rkey = mw->rkey;
1235 resp.mw_handle = uobj->id;
1236
1237 if (copy_to_user((void __user *)(unsigned long)cmd.response,
1238 &resp, sizeof(resp))) {
1239 ret = -EFAULT;
1240 goto err_copy;
1241 }
1242
1243 put_pd_read(pd);
1244
1245 mutex_lock(&file->mutex);
1246 list_add_tail(&uobj->list, &file->ucontext->mw_list);
1247 mutex_unlock(&file->mutex);
1248
1249 uobj->live = 1;
1250
1251 up_write(&uobj->mutex);
1252
1253 return in_len;
1254
1255 err_copy:
1256 idr_remove_uobj(&ib_uverbs_mw_idr, uobj);
1257
1258 err_unalloc:
1259 ib_dealloc_mw(mw);
1260
1261 err_put:
1262 put_pd_read(pd);
1263
1264 err_free:
1265 put_uobj_write(uobj);
1266 return ret;
1267 }
1268
1269 ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file,
1270 struct ib_device *ib_dev,
1271 const char __user *buf, int in_len,
1272 int out_len)
1273 {
1274 struct ib_uverbs_dealloc_mw cmd;
1275 struct ib_mw *mw;
1276 struct ib_uobject *uobj;
1277 int ret = -EINVAL;
1278
1279 if (copy_from_user(&cmd, buf, sizeof(cmd)))
1280 return -EFAULT;
1281
1282 uobj = idr_write_uobj(&ib_uverbs_mw_idr, cmd.mw_handle, file->ucontext);
1283 if (!uobj)
1284 return -EINVAL;
1285
1286 mw = uobj->object;
1287
1288 ret = ib_dealloc_mw(mw);
1289 if (!ret)
1290 uobj->live = 0;
1291
1292 put_uobj_write(uobj);
1293
1294 if (ret)
1295 return ret;
1296
1297 idr_remove_uobj(&ib_uverbs_mw_idr, uobj);
1298
1299 mutex_lock(&file->mutex);
1300 list_del(&uobj->list);
1301 mutex_unlock(&file->mutex);
1302
1303 put_uobj(uobj);
1304
1305 return in_len;
1306 }
1307
1308 ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file,
1309 struct ib_device *ib_dev,
1310 const char __user *buf, int in_len,
1311 int out_len)
1312 {
1313 struct ib_uverbs_create_comp_channel cmd;
1314 struct ib_uverbs_create_comp_channel_resp resp;
1315 struct file *filp;
1316 int ret;
1317
1318 if (out_len < sizeof resp)
1319 return -ENOSPC;
1320
1321 if (copy_from_user(&cmd, buf, sizeof cmd))
1322 return -EFAULT;
1323
1324 ret = get_unused_fd_flags(O_CLOEXEC);
1325 if (ret < 0)
1326 return ret;
1327 resp.fd = ret;
1328
1329 filp = ib_uverbs_alloc_event_file(file, ib_dev, 0);
1330 if (IS_ERR(filp)) {
1331 put_unused_fd(resp.fd);
1332 return PTR_ERR(filp);
1333 }
1334
1335 if (copy_to_user((void __user *) (unsigned long) cmd.response,
1336 &resp, sizeof resp)) {
1337 put_unused_fd(resp.fd);
1338 fput(filp);
1339 return -EFAULT;
1340 }
1341
1342 fd_install(resp.fd, filp);
1343 return in_len;
1344 }
1345
1346 static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file,
1347 struct ib_device *ib_dev,
1348 struct ib_udata *ucore,
1349 struct ib_udata *uhw,
1350 struct ib_uverbs_ex_create_cq *cmd,
1351 size_t cmd_sz,
1352 int (*cb)(struct ib_uverbs_file *file,
1353 struct ib_ucq_object *obj,
1354 struct ib_uverbs_ex_create_cq_resp *resp,
1355 struct ib_udata *udata,
1356 void *context),
1357 void *context)
1358 {
1359 struct ib_ucq_object *obj;
1360 struct ib_uverbs_event_file *ev_file = NULL;
1361 struct ib_cq *cq;
1362 int ret;
1363 struct ib_uverbs_ex_create_cq_resp resp;
1364 struct ib_cq_init_attr attr = {};
1365
1366 if (cmd->comp_vector >= file->device->num_comp_vectors)
1367 return ERR_PTR(-EINVAL);
1368
1369 obj = kmalloc(sizeof *obj, GFP_KERNEL);
1370 if (!obj)
1371 return ERR_PTR(-ENOMEM);
1372
1373 init_uobj(&obj->uobject, cmd->user_handle, file->ucontext, &cq_lock_class);
1374 down_write(&obj->uobject.mutex);
1375
1376 if (cmd->comp_channel >= 0) {
1377 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel);
1378 if (!ev_file) {
1379 ret = -EINVAL;
1380 goto err;
1381 }
1382 }
1383
1384 obj->uverbs_file = file;
1385 obj->comp_events_reported = 0;
1386 obj->async_events_reported = 0;
1387 INIT_LIST_HEAD(&obj->comp_list);
1388 INIT_LIST_HEAD(&obj->async_list);
1389
1390 attr.cqe = cmd->cqe;
1391 attr.comp_vector = cmd->comp_vector;
1392
1393 if (cmd_sz > offsetof(typeof(*cmd), flags) + sizeof(cmd->flags))
1394 attr.flags = cmd->flags;
1395
1396 cq = ib_dev->create_cq(ib_dev, &attr,
1397 file->ucontext, uhw);
1398 if (IS_ERR(cq)) {
1399 ret = PTR_ERR(cq);
1400 goto err_file;
1401 }
1402
1403 cq->device = ib_dev;
1404 cq->uobject = &obj->uobject;
1405 cq->comp_handler = ib_uverbs_comp_handler;
1406 cq->event_handler = ib_uverbs_cq_event_handler;
1407 cq->cq_context = ev_file;
1408 atomic_set(&cq->usecnt, 0);
1409
1410 obj->uobject.object = cq;
1411 ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject);
1412 if (ret)
1413 goto err_free;
1414
1415 memset(&resp, 0, sizeof resp);
1416 resp.base.cq_handle = obj->uobject.id;
1417 resp.base.cqe = cq->cqe;
1418
1419 resp.response_length = offsetof(typeof(resp), response_length) +
1420 sizeof(resp.response_length);
1421
1422 ret = cb(file, obj, &resp, ucore, context);
1423 if (ret)
1424 goto err_cb;
1425
1426 mutex_lock(&file->mutex);
1427 list_add_tail(&obj->uobject.list, &file->ucontext->cq_list);
1428 mutex_unlock(&file->mutex);
1429
1430 obj->uobject.live = 1;
1431
1432 up_write(&obj->uobject.mutex);
1433
1434 return obj;
1435
1436 err_cb:
1437 idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject);
1438
1439 err_free:
1440 ib_destroy_cq(cq);
1441
1442 err_file:
1443 if (ev_file)
1444 ib_uverbs_release_ucq(file, ev_file, obj);
1445
1446 err:
1447 put_uobj_write(&obj->uobject);
1448
1449 return ERR_PTR(ret);
1450 }
1451
1452 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file *file,
1453 struct ib_ucq_object *obj,
1454 struct ib_uverbs_ex_create_cq_resp *resp,
1455 struct ib_udata *ucore, void *context)
1456 {
1457 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base)))
1458 return -EFAULT;
1459
1460 return 0;
1461 }
1462
1463 ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file,
1464 struct ib_device *ib_dev,
1465 const char __user *buf, int in_len,
1466 int out_len)
1467 {
1468 struct ib_uverbs_create_cq cmd;
1469 struct ib_uverbs_ex_create_cq cmd_ex;
1470 struct ib_uverbs_create_cq_resp resp;
1471 struct ib_udata ucore;
1472 struct ib_udata uhw;
1473 struct ib_ucq_object *obj;
1474
1475 if (out_len < sizeof(resp))
1476 return -ENOSPC;
1477
1478 if (copy_from_user(&cmd, buf, sizeof(cmd)))
1479 return -EFAULT;
1480
1481 INIT_UDATA(&ucore, buf, (unsigned long)cmd.response, sizeof(cmd), sizeof(resp));
1482
1483 INIT_UDATA(&uhw, buf + sizeof(cmd),
1484 (unsigned long)cmd.response + sizeof(resp),
1485 in_len - sizeof(cmd), out_len - sizeof(resp));
1486
1487 memset(&cmd_ex, 0, sizeof(cmd_ex));
1488 cmd_ex.user_handle = cmd.user_handle;
1489 cmd_ex.cqe = cmd.cqe;
1490 cmd_ex.comp_vector = cmd.comp_vector;
1491 cmd_ex.comp_channel = cmd.comp_channel;
1492
1493 obj = create_cq(file, ib_dev, &ucore, &uhw, &cmd_ex,
1494 offsetof(typeof(cmd_ex), comp_channel) +
1495 sizeof(cmd.comp_channel), ib_uverbs_create_cq_cb,
1496 NULL);
1497
1498 if (IS_ERR(obj))
1499 return PTR_ERR(obj);
1500
1501 return in_len;
1502 }
1503
1504 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file *file,
1505 struct ib_ucq_object *obj,
1506 struct ib_uverbs_ex_create_cq_resp *resp,
1507 struct ib_udata *ucore, void *context)
1508 {
1509 if (ib_copy_to_udata(ucore, resp, resp->response_length))
1510 return -EFAULT;
1511
1512 return 0;
1513 }
1514
1515 int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file,
1516 struct ib_device *ib_dev,
1517 struct ib_udata *ucore,
1518 struct ib_udata *uhw)
1519 {
1520 struct ib_uverbs_ex_create_cq_resp resp;
1521 struct ib_uverbs_ex_create_cq cmd;
1522 struct ib_ucq_object *obj;
1523 int err;
1524
1525 if (ucore->inlen < sizeof(cmd))
1526 return -EINVAL;
1527
1528 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
1529 if (err)
1530 return err;
1531
1532 if (cmd.comp_mask)
1533 return -EINVAL;
1534
1535 if (cmd.reserved)
1536 return -EINVAL;
1537
1538 if (ucore->outlen < (offsetof(typeof(resp), response_length) +
1539 sizeof(resp.response_length)))
1540 return -ENOSPC;
1541
1542 obj = create_cq(file, ib_dev, ucore, uhw, &cmd,
1543 min(ucore->inlen, sizeof(cmd)),
1544 ib_uverbs_ex_create_cq_cb, NULL);
1545
1546 if (IS_ERR(obj))
1547 return PTR_ERR(obj);
1548
1549 return 0;
1550 }
1551
1552 ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file,
1553 struct ib_device *ib_dev,
1554 const char __user *buf, int in_len,
1555 int out_len)
1556 {
1557 struct ib_uverbs_resize_cq cmd;
1558 struct ib_uverbs_resize_cq_resp resp;
1559 struct ib_udata udata;
1560 struct ib_cq *cq;
1561 int ret = -EINVAL;
1562
1563 if (copy_from_user(&cmd, buf, sizeof cmd))
1564 return -EFAULT;
1565
1566 INIT_UDATA(&udata, buf + sizeof cmd,
1567 (unsigned long) cmd.response + sizeof resp,
1568 in_len - sizeof cmd, out_len - sizeof resp);
1569
1570 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
1571 if (!cq)
1572 return -EINVAL;
1573
1574 ret = cq->device->resize_cq(cq, cmd.cqe, &udata);
1575 if (ret)
1576 goto out;
1577
1578 resp.cqe = cq->cqe;
1579
1580 if (copy_to_user((void __user *) (unsigned long) cmd.response,
1581 &resp, sizeof resp.cqe))
1582 ret = -EFAULT;
1583
1584 out:
1585 put_cq_read(cq);
1586
1587 return ret ? ret : in_len;
1588 }
1589
1590 static int copy_wc_to_user(void __user *dest, struct ib_wc *wc)
1591 {
1592 struct ib_uverbs_wc tmp;
1593
1594 tmp.wr_id = wc->wr_id;
1595 tmp.status = wc->status;
1596 tmp.opcode = wc->opcode;
1597 tmp.vendor_err = wc->vendor_err;
1598 tmp.byte_len = wc->byte_len;
1599 tmp.ex.imm_data = (__u32 __force) wc->ex.imm_data;
1600 tmp.qp_num = wc->qp->qp_num;
1601 tmp.src_qp = wc->src_qp;
1602 tmp.wc_flags = wc->wc_flags;
1603 tmp.pkey_index = wc->pkey_index;
1604 tmp.slid = wc->slid;
1605 tmp.sl = wc->sl;
1606 tmp.dlid_path_bits = wc->dlid_path_bits;
1607 tmp.port_num = wc->port_num;
1608 tmp.reserved = 0;
1609
1610 if (copy_to_user(dest, &tmp, sizeof tmp))
1611 return -EFAULT;
1612
1613 return 0;
1614 }
1615
1616 ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file,
1617 struct ib_device *ib_dev,
1618 const char __user *buf, int in_len,
1619 int out_len)
1620 {
1621 struct ib_uverbs_poll_cq cmd;
1622 struct ib_uverbs_poll_cq_resp resp;
1623 u8 __user *header_ptr;
1624 u8 __user *data_ptr;
1625 struct ib_cq *cq;
1626 struct ib_wc wc;
1627 int ret;
1628
1629 if (copy_from_user(&cmd, buf, sizeof cmd))
1630 return -EFAULT;
1631
1632 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
1633 if (!cq)
1634 return -EINVAL;
1635
1636 /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1637 header_ptr = (void __user *)(unsigned long) cmd.response;
1638 data_ptr = header_ptr + sizeof resp;
1639
1640 memset(&resp, 0, sizeof resp);
1641 while (resp.count < cmd.ne) {
1642 ret = ib_poll_cq(cq, 1, &wc);
1643 if (ret < 0)
1644 goto out_put;
1645 if (!ret)
1646 break;
1647
1648 ret = copy_wc_to_user(data_ptr, &wc);
1649 if (ret)
1650 goto out_put;
1651
1652 data_ptr += sizeof(struct ib_uverbs_wc);
1653 ++resp.count;
1654 }
1655
1656 if (copy_to_user(header_ptr, &resp, sizeof resp)) {
1657 ret = -EFAULT;
1658 goto out_put;
1659 }
1660
1661 ret = in_len;
1662
1663 out_put:
1664 put_cq_read(cq);
1665 return ret;
1666 }
1667
1668 ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file,
1669 struct ib_device *ib_dev,
1670 const char __user *buf, int in_len,
1671 int out_len)
1672 {
1673 struct ib_uverbs_req_notify_cq cmd;
1674 struct ib_cq *cq;
1675
1676 if (copy_from_user(&cmd, buf, sizeof cmd))
1677 return -EFAULT;
1678
1679 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
1680 if (!cq)
1681 return -EINVAL;
1682
1683 ib_req_notify_cq(cq, cmd.solicited_only ?
1684 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1685
1686 put_cq_read(cq);
1687
1688 return in_len;
1689 }
1690
1691 ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file,
1692 struct ib_device *ib_dev,
1693 const char __user *buf, int in_len,
1694 int out_len)
1695 {
1696 struct ib_uverbs_destroy_cq cmd;
1697 struct ib_uverbs_destroy_cq_resp resp;
1698 struct ib_uobject *uobj;
1699 struct ib_cq *cq;
1700 struct ib_ucq_object *obj;
1701 struct ib_uverbs_event_file *ev_file;
1702 int ret = -EINVAL;
1703
1704 if (copy_from_user(&cmd, buf, sizeof cmd))
1705 return -EFAULT;
1706
1707 uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext);
1708 if (!uobj)
1709 return -EINVAL;
1710 cq = uobj->object;
1711 ev_file = cq->cq_context;
1712 obj = container_of(cq->uobject, struct ib_ucq_object, uobject);
1713
1714 ret = ib_destroy_cq(cq);
1715 if (!ret)
1716 uobj->live = 0;
1717
1718 put_uobj_write(uobj);
1719
1720 if (ret)
1721 return ret;
1722
1723 idr_remove_uobj(&ib_uverbs_cq_idr, uobj);
1724
1725 mutex_lock(&file->mutex);
1726 list_del(&uobj->list);
1727 mutex_unlock(&file->mutex);
1728
1729 ib_uverbs_release_ucq(file, ev_file, obj);
1730
1731 memset(&resp, 0, sizeof resp);
1732 resp.comp_events_reported = obj->comp_events_reported;
1733 resp.async_events_reported = obj->async_events_reported;
1734
1735 put_uobj(uobj);
1736
1737 if (copy_to_user((void __user *) (unsigned long) cmd.response,
1738 &resp, sizeof resp))
1739 return -EFAULT;
1740
1741 return in_len;
1742 }
1743
1744 static int create_qp(struct ib_uverbs_file *file,
1745 struct ib_udata *ucore,
1746 struct ib_udata *uhw,
1747 struct ib_uverbs_ex_create_qp *cmd,
1748 size_t cmd_sz,
1749 int (*cb)(struct ib_uverbs_file *file,
1750 struct ib_uverbs_ex_create_qp_resp *resp,
1751 struct ib_udata *udata),
1752 void *context)
1753 {
1754 struct ib_uqp_object *obj;
1755 struct ib_device *device;
1756 struct ib_pd *pd = NULL;
1757 struct ib_xrcd *xrcd = NULL;
1758 struct ib_uobject *uninitialized_var(xrcd_uobj);
1759 struct ib_cq *scq = NULL, *rcq = NULL;
1760 struct ib_srq *srq = NULL;
1761 struct ib_qp *qp;
1762 char *buf;
1763 struct ib_qp_init_attr attr;
1764 struct ib_uverbs_ex_create_qp_resp resp;
1765 int ret;
1766
1767 if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
1768 return -EPERM;
1769
1770 obj = kzalloc(sizeof *obj, GFP_KERNEL);
1771 if (!obj)
1772 return -ENOMEM;
1773
1774 init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext,
1775 &qp_lock_class);
1776 down_write(&obj->uevent.uobject.mutex);
1777
1778 if (cmd->qp_type == IB_QPT_XRC_TGT) {
1779 xrcd = idr_read_xrcd(cmd->pd_handle, file->ucontext,
1780 &xrcd_uobj);
1781 if (!xrcd) {
1782 ret = -EINVAL;
1783 goto err_put;
1784 }
1785 device = xrcd->device;
1786 } else {
1787 if (cmd->qp_type == IB_QPT_XRC_INI) {
1788 cmd->max_recv_wr = 0;
1789 cmd->max_recv_sge = 0;
1790 } else {
1791 if (cmd->is_srq) {
1792 srq = idr_read_srq(cmd->srq_handle,
1793 file->ucontext);
1794 if (!srq || srq->srq_type != IB_SRQT_BASIC) {
1795 ret = -EINVAL;
1796 goto err_put;
1797 }
1798 }
1799
1800 if (cmd->recv_cq_handle != cmd->send_cq_handle) {
1801 rcq = idr_read_cq(cmd->recv_cq_handle,
1802 file->ucontext, 0);
1803 if (!rcq) {
1804 ret = -EINVAL;
1805 goto err_put;
1806 }
1807 }
1808 }
1809
1810 scq = idr_read_cq(cmd->send_cq_handle, file->ucontext, !!rcq);
1811 rcq = rcq ?: scq;
1812 pd = idr_read_pd(cmd->pd_handle, file->ucontext);
1813 if (!pd || !scq) {
1814 ret = -EINVAL;
1815 goto err_put;
1816 }
1817
1818 device = pd->device;
1819 }
1820
1821 attr.event_handler = ib_uverbs_qp_event_handler;
1822 attr.qp_context = file;
1823 attr.send_cq = scq;
1824 attr.recv_cq = rcq;
1825 attr.srq = srq;
1826 attr.xrcd = xrcd;
1827 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :
1828 IB_SIGNAL_REQ_WR;
1829 attr.qp_type = cmd->qp_type;
1830 attr.create_flags = 0;
1831
1832 attr.cap.max_send_wr = cmd->max_send_wr;
1833 attr.cap.max_recv_wr = cmd->max_recv_wr;
1834 attr.cap.max_send_sge = cmd->max_send_sge;
1835 attr.cap.max_recv_sge = cmd->max_recv_sge;
1836 attr.cap.max_inline_data = cmd->max_inline_data;
1837
1838 obj->uevent.events_reported = 0;
1839 INIT_LIST_HEAD(&obj->uevent.event_list);
1840 INIT_LIST_HEAD(&obj->mcast_list);
1841
1842 if (cmd_sz >= offsetof(typeof(*cmd), create_flags) +
1843 sizeof(cmd->create_flags))
1844 attr.create_flags = cmd->create_flags;
1845
1846 if (attr.create_flags & ~IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
1847 ret = -EINVAL;
1848 goto err_put;
1849 }
1850
1851 buf = (void *)cmd + sizeof(*cmd);
1852 if (cmd_sz > sizeof(*cmd))
1853 if (!(buf[0] == 0 && !memcmp(buf, buf + 1,
1854 cmd_sz - sizeof(*cmd) - 1))) {
1855 ret = -EINVAL;
1856 goto err_put;
1857 }
1858
1859 if (cmd->qp_type == IB_QPT_XRC_TGT)
1860 qp = ib_create_qp(pd, &attr);
1861 else
1862 qp = device->create_qp(pd, &attr, uhw);
1863
1864 if (IS_ERR(qp)) {
1865 ret = PTR_ERR(qp);
1866 goto err_put;
1867 }
1868
1869 if (cmd->qp_type != IB_QPT_XRC_TGT) {
1870 qp->real_qp = qp;
1871 qp->device = device;
1872 qp->pd = pd;
1873 qp->send_cq = attr.send_cq;
1874 qp->recv_cq = attr.recv_cq;
1875 qp->srq = attr.srq;
1876 qp->event_handler = attr.event_handler;
1877 qp->qp_context = attr.qp_context;
1878 qp->qp_type = attr.qp_type;
1879 atomic_set(&qp->usecnt, 0);
1880 atomic_inc(&pd->usecnt);
1881 atomic_inc(&attr.send_cq->usecnt);
1882 if (attr.recv_cq)
1883 atomic_inc(&attr.recv_cq->usecnt);
1884 if (attr.srq)
1885 atomic_inc(&attr.srq->usecnt);
1886 }
1887 qp->uobject = &obj->uevent.uobject;
1888
1889 obj->uevent.uobject.object = qp;
1890 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
1891 if (ret)
1892 goto err_destroy;
1893
1894 memset(&resp, 0, sizeof resp);
1895 resp.base.qpn = qp->qp_num;
1896 resp.base.qp_handle = obj->uevent.uobject.id;
1897 resp.base.max_recv_sge = attr.cap.max_recv_sge;
1898 resp.base.max_send_sge = attr.cap.max_send_sge;
1899 resp.base.max_recv_wr = attr.cap.max_recv_wr;
1900 resp.base.max_send_wr = attr.cap.max_send_wr;
1901 resp.base.max_inline_data = attr.cap.max_inline_data;
1902
1903 resp.response_length = offsetof(typeof(resp), response_length) +
1904 sizeof(resp.response_length);
1905
1906 ret = cb(file, &resp, ucore);
1907 if (ret)
1908 goto err_cb;
1909
1910 if (xrcd) {
1911 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
1912 uobject);
1913 atomic_inc(&obj->uxrcd->refcnt);
1914 put_xrcd_read(xrcd_uobj);
1915 }
1916
1917 if (pd)
1918 put_pd_read(pd);
1919 if (scq)
1920 put_cq_read(scq);
1921 if (rcq && rcq != scq)
1922 put_cq_read(rcq);
1923 if (srq)
1924 put_srq_read(srq);
1925
1926 mutex_lock(&file->mutex);
1927 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list);
1928 mutex_unlock(&file->mutex);
1929
1930 obj->uevent.uobject.live = 1;
1931
1932 up_write(&obj->uevent.uobject.mutex);
1933
1934 return 0;
1935 err_cb:
1936 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
1937
1938 err_destroy:
1939 ib_destroy_qp(qp);
1940
1941 err_put:
1942 if (xrcd)
1943 put_xrcd_read(xrcd_uobj);
1944 if (pd)
1945 put_pd_read(pd);
1946 if (scq)
1947 put_cq_read(scq);
1948 if (rcq && rcq != scq)
1949 put_cq_read(rcq);
1950 if (srq)
1951 put_srq_read(srq);
1952
1953 put_uobj_write(&obj->uevent.uobject);
1954 return ret;
1955 }
1956
1957 static int ib_uverbs_create_qp_cb(struct ib_uverbs_file *file,
1958 struct ib_uverbs_ex_create_qp_resp *resp,
1959 struct ib_udata *ucore)
1960 {
1961 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base)))
1962 return -EFAULT;
1963
1964 return 0;
1965 }
1966
1967 ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
1968 struct ib_device *ib_dev,
1969 const char __user *buf, int in_len,
1970 int out_len)
1971 {
1972 struct ib_uverbs_create_qp cmd;
1973 struct ib_uverbs_ex_create_qp cmd_ex;
1974 struct ib_udata ucore;
1975 struct ib_udata uhw;
1976 ssize_t resp_size = sizeof(struct ib_uverbs_create_qp_resp);
1977 int err;
1978
1979 if (out_len < resp_size)
1980 return -ENOSPC;
1981
1982 if (copy_from_user(&cmd, buf, sizeof(cmd)))
1983 return -EFAULT;
1984
1985 INIT_UDATA(&ucore, buf, (unsigned long)cmd.response, sizeof(cmd),
1986 resp_size);
1987 INIT_UDATA(&uhw, buf + sizeof(cmd),
1988 (unsigned long)cmd.response + resp_size,
1989 in_len - sizeof(cmd), out_len - resp_size);
1990
1991 memset(&cmd_ex, 0, sizeof(cmd_ex));
1992 cmd_ex.user_handle = cmd.user_handle;
1993 cmd_ex.pd_handle = cmd.pd_handle;
1994 cmd_ex.send_cq_handle = cmd.send_cq_handle;
1995 cmd_ex.recv_cq_handle = cmd.recv_cq_handle;
1996 cmd_ex.srq_handle = cmd.srq_handle;
1997 cmd_ex.max_send_wr = cmd.max_send_wr;
1998 cmd_ex.max_recv_wr = cmd.max_recv_wr;
1999 cmd_ex.max_send_sge = cmd.max_send_sge;
2000 cmd_ex.max_recv_sge = cmd.max_recv_sge;
2001 cmd_ex.max_inline_data = cmd.max_inline_data;
2002 cmd_ex.sq_sig_all = cmd.sq_sig_all;
2003 cmd_ex.qp_type = cmd.qp_type;
2004 cmd_ex.is_srq = cmd.is_srq;
2005
2006 err = create_qp(file, &ucore, &uhw, &cmd_ex,
2007 offsetof(typeof(cmd_ex), is_srq) +
2008 sizeof(cmd.is_srq), ib_uverbs_create_qp_cb,
2009 NULL);
2010
2011 if (err)
2012 return err;
2013
2014 return in_len;
2015 }
2016
2017 static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file *file,
2018 struct ib_uverbs_ex_create_qp_resp *resp,
2019 struct ib_udata *ucore)
2020 {
2021 if (ib_copy_to_udata(ucore, resp, resp->response_length))
2022 return -EFAULT;
2023
2024 return 0;
2025 }
2026
2027 int ib_uverbs_ex_create_qp(struct ib_uverbs_file *file,
2028 struct ib_device *ib_dev,
2029 struct ib_udata *ucore,
2030 struct ib_udata *uhw)
2031 {
2032 struct ib_uverbs_ex_create_qp_resp resp;
2033 struct ib_uverbs_ex_create_qp cmd = {0};
2034 int err;
2035
2036 if (ucore->inlen < (offsetof(typeof(cmd), comp_mask) +
2037 sizeof(cmd.comp_mask)))
2038 return -EINVAL;
2039
2040 err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
2041 if (err)
2042 return err;
2043
2044 if (cmd.comp_mask)
2045 return -EINVAL;
2046
2047 if (cmd.reserved)
2048 return -EINVAL;
2049
2050 if (ucore->outlen < (offsetof(typeof(resp), response_length) +
2051 sizeof(resp.response_length)))
2052 return -ENOSPC;
2053
2054 err = create_qp(file, ucore, uhw, &cmd,
2055 min(ucore->inlen, sizeof(cmd)),
2056 ib_uverbs_ex_create_qp_cb, NULL);
2057
2058 if (err)
2059 return err;
2060
2061 return 0;
2062 }
2063
2064 ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file,
2065 struct ib_device *ib_dev,
2066 const char __user *buf, int in_len, int out_len)
2067 {
2068 struct ib_uverbs_open_qp cmd;
2069 struct ib_uverbs_create_qp_resp resp;
2070 struct ib_udata udata;
2071 struct ib_uqp_object *obj;
2072 struct ib_xrcd *xrcd;
2073 struct ib_uobject *uninitialized_var(xrcd_uobj);
2074 struct ib_qp *qp;
2075 struct ib_qp_open_attr attr;
2076 int ret;
2077
2078 if (out_len < sizeof resp)
2079 return -ENOSPC;
2080
2081 if (copy_from_user(&cmd, buf, sizeof cmd))
2082 return -EFAULT;
2083
2084 INIT_UDATA(&udata, buf + sizeof cmd,
2085 (unsigned long) cmd.response + sizeof resp,
2086 in_len - sizeof cmd, out_len - sizeof resp);
2087
2088 obj = kmalloc(sizeof *obj, GFP_KERNEL);
2089 if (!obj)
2090 return -ENOMEM;
2091
2092 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_class);
2093 down_write(&obj->uevent.uobject.mutex);
2094
2095 xrcd = idr_read_xrcd(cmd.pd_handle, file->ucontext, &xrcd_uobj);
2096 if (!xrcd) {
2097 ret = -EINVAL;
2098 goto err_put;
2099 }
2100
2101 attr.event_handler = ib_uverbs_qp_event_handler;
2102 attr.qp_context = file;
2103 attr.qp_num = cmd.qpn;
2104 attr.qp_type = cmd.qp_type;
2105
2106 obj->uevent.events_reported = 0;
2107 INIT_LIST_HEAD(&obj->uevent.event_list);
2108 INIT_LIST_HEAD(&obj->mcast_list);
2109
2110 qp = ib_open_qp(xrcd, &attr);
2111 if (IS_ERR(qp)) {
2112 ret = PTR_ERR(qp);
2113 goto err_put;
2114 }
2115
2116 qp->uobject = &obj->uevent.uobject;
2117
2118 obj->uevent.uobject.object = qp;
2119 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
2120 if (ret)
2121 goto err_destroy;
2122
2123 memset(&resp, 0, sizeof resp);
2124 resp.qpn = qp->qp_num;
2125 resp.qp_handle = obj->uevent.uobject.id;
2126
2127 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2128 &resp, sizeof resp)) {
2129 ret = -EFAULT;
2130 goto err_remove;
2131 }
2132
2133 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
2134 atomic_inc(&obj->uxrcd->refcnt);
2135 put_xrcd_read(xrcd_uobj);
2136
2137 mutex_lock(&file->mutex);
2138 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list);
2139 mutex_unlock(&file->mutex);
2140
2141 obj->uevent.uobject.live = 1;
2142
2143 up_write(&obj->uevent.uobject.mutex);
2144
2145 return in_len;
2146
2147 err_remove:
2148 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
2149
2150 err_destroy:
2151 ib_destroy_qp(qp);
2152
2153 err_put:
2154 put_xrcd_read(xrcd_uobj);
2155 put_uobj_write(&obj->uevent.uobject);
2156 return ret;
2157 }
2158
2159 ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file,
2160 struct ib_device *ib_dev,
2161 const char __user *buf, int in_len,
2162 int out_len)
2163 {
2164 struct ib_uverbs_query_qp cmd;
2165 struct ib_uverbs_query_qp_resp resp;
2166 struct ib_qp *qp;
2167 struct ib_qp_attr *attr;
2168 struct ib_qp_init_attr *init_attr;
2169 int ret;
2170
2171 if (copy_from_user(&cmd, buf, sizeof cmd))
2172 return -EFAULT;
2173
2174 attr = kmalloc(sizeof *attr, GFP_KERNEL);
2175 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
2176 if (!attr || !init_attr) {
2177 ret = -ENOMEM;
2178 goto out;
2179 }
2180
2181 qp = idr_read_qp(cmd.qp_handle, file->ucontext);
2182 if (!qp) {
2183 ret = -EINVAL;
2184 goto out;
2185 }
2186
2187 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);
2188
2189 put_qp_read(qp);
2190
2191 if (ret)
2192 goto out;
2193
2194 memset(&resp, 0, sizeof resp);
2195
2196 resp.qp_state = attr->qp_state;
2197 resp.cur_qp_state = attr->cur_qp_state;
2198 resp.path_mtu = attr->path_mtu;
2199 resp.path_mig_state = attr->path_mig_state;
2200 resp.qkey = attr->qkey;
2201 resp.rq_psn = attr->rq_psn;
2202 resp.sq_psn = attr->sq_psn;
2203 resp.dest_qp_num = attr->dest_qp_num;
2204 resp.qp_access_flags = attr->qp_access_flags;
2205 resp.pkey_index = attr->pkey_index;
2206 resp.alt_pkey_index = attr->alt_pkey_index;
2207 resp.sq_draining = attr->sq_draining;
2208 resp.max_rd_atomic = attr->max_rd_atomic;
2209 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic;
2210 resp.min_rnr_timer = attr->min_rnr_timer;
2211 resp.port_num = attr->port_num;
2212 resp.timeout = attr->timeout;
2213 resp.retry_cnt = attr->retry_cnt;
2214 resp.rnr_retry = attr->rnr_retry;
2215 resp.alt_port_num = attr->alt_port_num;
2216 resp.alt_timeout = attr->alt_timeout;
2217
2218 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16);
2219 resp.dest.flow_label = attr->ah_attr.grh.flow_label;
2220 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index;
2221 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit;
2222 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class;
2223 resp.dest.dlid = attr->ah_attr.dlid;
2224 resp.dest.sl = attr->ah_attr.sl;
2225 resp.dest.src_path_bits = attr->ah_attr.src_path_bits;
2226 resp.dest.static_rate = attr->ah_attr.static_rate;
2227 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH);
2228 resp.dest.port_num = attr->ah_attr.port_num;
2229
2230 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16);
2231 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label;
2232 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index;
2233 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit;
2234 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class;
2235 resp.alt_dest.dlid = attr->alt_ah_attr.dlid;
2236 resp.alt_dest.sl = attr->alt_ah_attr.sl;
2237 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits;
2238 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate;
2239 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH);
2240 resp.alt_dest.port_num = attr->alt_ah_attr.port_num;
2241
2242 resp.max_send_wr = init_attr->cap.max_send_wr;
2243 resp.max_recv_wr = init_attr->cap.max_recv_wr;
2244 resp.max_send_sge = init_attr->cap.max_send_sge;
2245 resp.max_recv_sge = init_attr->cap.max_recv_sge;
2246 resp.max_inline_data = init_attr->cap.max_inline_data;
2247 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
2248
2249 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2250 &resp, sizeof resp))
2251 ret = -EFAULT;
2252
2253 out:
2254 kfree(attr);
2255 kfree(init_attr);
2256
2257 return ret ? ret : in_len;
2258 }
2259
2260 /* Remove ignored fields set in the attribute mask */
2261 static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
2262 {
2263 switch (qp_type) {
2264 case IB_QPT_XRC_INI:
2265 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);
2266 case IB_QPT_XRC_TGT:
2267 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |
2268 IB_QP_RNR_RETRY);
2269 default:
2270 return mask;
2271 }
2272 }
2273
2274 ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
2275 struct ib_device *ib_dev,
2276 const char __user *buf, int in_len,
2277 int out_len)
2278 {
2279 struct ib_uverbs_modify_qp cmd;
2280 struct ib_udata udata;
2281 struct ib_qp *qp;
2282 struct ib_qp_attr *attr;
2283 int ret;
2284
2285 if (copy_from_user(&cmd, buf, sizeof cmd))
2286 return -EFAULT;
2287
2288 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd,
2289 out_len);
2290
2291 attr = kmalloc(sizeof *attr, GFP_KERNEL);
2292 if (!attr)
2293 return -ENOMEM;
2294
2295 qp = idr_read_qp(cmd.qp_handle, file->ucontext);
2296 if (!qp) {
2297 ret = -EINVAL;
2298 goto out;
2299 }
2300
2301 attr->qp_state = cmd.qp_state;
2302 attr->cur_qp_state = cmd.cur_qp_state;
2303 attr->path_mtu = cmd.path_mtu;
2304 attr->path_mig_state = cmd.path_mig_state;
2305 attr->qkey = cmd.qkey;
2306 attr->rq_psn = cmd.rq_psn;
2307 attr->sq_psn = cmd.sq_psn;
2308 attr->dest_qp_num = cmd.dest_qp_num;
2309 attr->qp_access_flags = cmd.qp_access_flags;
2310 attr->pkey_index = cmd.pkey_index;
2311 attr->alt_pkey_index = cmd.alt_pkey_index;
2312 attr->en_sqd_async_notify = cmd.en_sqd_async_notify;
2313 attr->max_rd_atomic = cmd.max_rd_atomic;
2314 attr->max_dest_rd_atomic = cmd.max_dest_rd_atomic;
2315 attr->min_rnr_timer = cmd.min_rnr_timer;
2316 attr->port_num = cmd.port_num;
2317 attr->timeout = cmd.timeout;
2318 attr->retry_cnt = cmd.retry_cnt;
2319 attr->rnr_retry = cmd.rnr_retry;
2320 attr->alt_port_num = cmd.alt_port_num;
2321 attr->alt_timeout = cmd.alt_timeout;
2322
2323 memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16);
2324 attr->ah_attr.grh.flow_label = cmd.dest.flow_label;
2325 attr->ah_attr.grh.sgid_index = cmd.dest.sgid_index;
2326 attr->ah_attr.grh.hop_limit = cmd.dest.hop_limit;
2327 attr->ah_attr.grh.traffic_class = cmd.dest.traffic_class;
2328 attr->ah_attr.dlid = cmd.dest.dlid;
2329 attr->ah_attr.sl = cmd.dest.sl;
2330 attr->ah_attr.src_path_bits = cmd.dest.src_path_bits;
2331 attr->ah_attr.static_rate = cmd.dest.static_rate;
2332 attr->ah_attr.ah_flags = cmd.dest.is_global ? IB_AH_GRH : 0;
2333 attr->ah_attr.port_num = cmd.dest.port_num;
2334
2335 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16);
2336 attr->alt_ah_attr.grh.flow_label = cmd.alt_dest.flow_label;
2337 attr->alt_ah_attr.grh.sgid_index = cmd.alt_dest.sgid_index;
2338 attr->alt_ah_attr.grh.hop_limit = cmd.alt_dest.hop_limit;
2339 attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class;
2340 attr->alt_ah_attr.dlid = cmd.alt_dest.dlid;
2341 attr->alt_ah_attr.sl = cmd.alt_dest.sl;
2342 attr->alt_ah_attr.src_path_bits = cmd.alt_dest.src_path_bits;
2343 attr->alt_ah_attr.static_rate = cmd.alt_dest.static_rate;
2344 attr->alt_ah_attr.ah_flags = cmd.alt_dest.is_global ? IB_AH_GRH : 0;
2345 attr->alt_ah_attr.port_num = cmd.alt_dest.port_num;
2346
2347 if (qp->real_qp == qp) {
2348 ret = ib_resolve_eth_dmac(qp, attr, &cmd.attr_mask);
2349 if (ret)
2350 goto release_qp;
2351 ret = qp->device->modify_qp(qp, attr,
2352 modify_qp_mask(qp->qp_type, cmd.attr_mask), &udata);
2353 } else {
2354 ret = ib_modify_qp(qp, attr, modify_qp_mask(qp->qp_type, cmd.attr_mask));
2355 }
2356
2357 if (ret)
2358 goto release_qp;
2359
2360 ret = in_len;
2361
2362 release_qp:
2363 put_qp_read(qp);
2364
2365 out:
2366 kfree(attr);
2367
2368 return ret;
2369 }
2370
2371 ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file,
2372 struct ib_device *ib_dev,
2373 const char __user *buf, int in_len,
2374 int out_len)
2375 {
2376 struct ib_uverbs_destroy_qp cmd;
2377 struct ib_uverbs_destroy_qp_resp resp;
2378 struct ib_uobject *uobj;
2379 struct ib_qp *qp;
2380 struct ib_uqp_object *obj;
2381 int ret = -EINVAL;
2382
2383 if (copy_from_user(&cmd, buf, sizeof cmd))
2384 return -EFAULT;
2385
2386 memset(&resp, 0, sizeof resp);
2387
2388 uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext);
2389 if (!uobj)
2390 return -EINVAL;
2391 qp = uobj->object;
2392 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject);
2393
2394 if (!list_empty(&obj->mcast_list)) {
2395 put_uobj_write(uobj);
2396 return -EBUSY;
2397 }
2398
2399 ret = ib_destroy_qp(qp);
2400 if (!ret)
2401 uobj->live = 0;
2402
2403 put_uobj_write(uobj);
2404
2405 if (ret)
2406 return ret;
2407
2408 if (obj->uxrcd)
2409 atomic_dec(&obj->uxrcd->refcnt);
2410
2411 idr_remove_uobj(&ib_uverbs_qp_idr, uobj);
2412
2413 mutex_lock(&file->mutex);
2414 list_del(&uobj->list);
2415 mutex_unlock(&file->mutex);
2416
2417 ib_uverbs_release_uevent(file, &obj->uevent);
2418
2419 resp.events_reported = obj->uevent.events_reported;
2420
2421 put_uobj(uobj);
2422
2423 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2424 &resp, sizeof resp))
2425 return -EFAULT;
2426
2427 return in_len;
2428 }
2429
2430 static void *alloc_wr(size_t wr_size, __u32 num_sge)
2431 {
2432 return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) +
2433 num_sge * sizeof (struct ib_sge), GFP_KERNEL);
2434 };
2435
2436 ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file,
2437 struct ib_device *ib_dev,
2438 const char __user *buf, int in_len,
2439 int out_len)
2440 {
2441 struct ib_uverbs_post_send cmd;
2442 struct ib_uverbs_post_send_resp resp;
2443 struct ib_uverbs_send_wr *user_wr;
2444 struct ib_send_wr *wr = NULL, *last, *next, *bad_wr;
2445 struct ib_qp *qp;
2446 int i, sg_ind;
2447 int is_ud;
2448 ssize_t ret = -EINVAL;
2449
2450 if (copy_from_user(&cmd, buf, sizeof cmd))
2451 return -EFAULT;
2452
2453 if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count +
2454 cmd.sge_count * sizeof (struct ib_uverbs_sge))
2455 return -EINVAL;
2456
2457 if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr))
2458 return -EINVAL;
2459
2460 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
2461 if (!user_wr)
2462 return -ENOMEM;
2463
2464 qp = idr_read_qp(cmd.qp_handle, file->ucontext);
2465 if (!qp)
2466 goto out;
2467
2468 is_ud = qp->qp_type == IB_QPT_UD;
2469 sg_ind = 0;
2470 last = NULL;
2471 for (i = 0; i < cmd.wr_count; ++i) {
2472 if (copy_from_user(user_wr,
2473 buf + sizeof cmd + i * cmd.wqe_size,
2474 cmd.wqe_size)) {
2475 ret = -EFAULT;
2476 goto out_put;
2477 }
2478
2479 if (user_wr->num_sge + sg_ind > cmd.sge_count) {
2480 ret = -EINVAL;
2481 goto out_put;
2482 }
2483
2484 if (is_ud) {
2485 struct ib_ud_wr *ud;
2486
2487 if (user_wr->opcode != IB_WR_SEND &&
2488 user_wr->opcode != IB_WR_SEND_WITH_IMM) {
2489 ret = -EINVAL;
2490 goto out_put;
2491 }
2492
2493 ud = alloc_wr(sizeof(*ud), user_wr->num_sge);
2494 if (!ud) {
2495 ret = -ENOMEM;
2496 goto out_put;
2497 }
2498
2499 ud->ah = idr_read_ah(user_wr->wr.ud.ah, file->ucontext);
2500 if (!ud->ah) {
2501 kfree(ud);
2502 ret = -EINVAL;
2503 goto out_put;
2504 }
2505 ud->remote_qpn = user_wr->wr.ud.remote_qpn;
2506 ud->remote_qkey = user_wr->wr.ud.remote_qkey;
2507
2508 next = &ud->wr;
2509 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2510 user_wr->opcode == IB_WR_RDMA_WRITE ||
2511 user_wr->opcode == IB_WR_RDMA_READ) {
2512 struct ib_rdma_wr *rdma;
2513
2514 rdma = alloc_wr(sizeof(*rdma), user_wr->num_sge);
2515 if (!rdma) {
2516 ret = -ENOMEM;
2517 goto out_put;
2518 }
2519
2520 rdma->remote_addr = user_wr->wr.rdma.remote_addr;
2521 rdma->rkey = user_wr->wr.rdma.rkey;
2522
2523 next = &rdma->wr;
2524 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2525 user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2526 struct ib_atomic_wr *atomic;
2527
2528 atomic = alloc_wr(sizeof(*atomic), user_wr->num_sge);
2529 if (!atomic) {
2530 ret = -ENOMEM;
2531 goto out_put;
2532 }
2533
2534 atomic->remote_addr = user_wr->wr.atomic.remote_addr;
2535 atomic->compare_add = user_wr->wr.atomic.compare_add;
2536 atomic->swap = user_wr->wr.atomic.swap;
2537 atomic->rkey = user_wr->wr.atomic.rkey;
2538
2539 next = &atomic->wr;
2540 } else if (user_wr->opcode == IB_WR_SEND ||
2541 user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2542 user_wr->opcode == IB_WR_SEND_WITH_INV) {
2543 next = alloc_wr(sizeof(*next), user_wr->num_sge);
2544 if (!next) {
2545 ret = -ENOMEM;
2546 goto out_put;
2547 }
2548 } else {
2549 ret = -EINVAL;
2550 goto out_put;
2551 }
2552
2553 if (user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2554 user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
2555 next->ex.imm_data =
2556 (__be32 __force) user_wr->ex.imm_data;
2557 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) {
2558 next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey;
2559 }
2560
2561 if (!last)
2562 wr = next;
2563 else
2564 last->next = next;
2565 last = next;
2566
2567 next->next = NULL;
2568 next->wr_id = user_wr->wr_id;
2569 next->num_sge = user_wr->num_sge;
2570 next->opcode = user_wr->opcode;
2571 next->send_flags = user_wr->send_flags;
2572
2573 if (next->num_sge) {
2574 next->sg_list = (void *) next +
2575 ALIGN(sizeof *next, sizeof (struct ib_sge));
2576 if (copy_from_user(next->sg_list,
2577 buf + sizeof cmd +
2578 cmd.wr_count * cmd.wqe_size +
2579 sg_ind * sizeof (struct ib_sge),
2580 next->num_sge * sizeof (struct ib_sge))) {
2581 ret = -EFAULT;
2582 goto out_put;
2583 }
2584 sg_ind += next->num_sge;
2585 } else
2586 next->sg_list = NULL;
2587 }
2588
2589 resp.bad_wr = 0;
2590 ret = qp->device->post_send(qp->real_qp, wr, &bad_wr);
2591 if (ret)
2592 for (next = wr; next; next = next->next) {
2593 ++resp.bad_wr;
2594 if (next == bad_wr)
2595 break;
2596 }
2597
2598 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2599 &resp, sizeof resp))
2600 ret = -EFAULT;
2601
2602 out_put:
2603 put_qp_read(qp);
2604
2605 while (wr) {
2606 if (is_ud && ud_wr(wr)->ah)
2607 put_ah_read(ud_wr(wr)->ah);
2608 next = wr->next;
2609 kfree(wr);
2610 wr = next;
2611 }
2612
2613 out:
2614 kfree(user_wr);
2615
2616 return ret ? ret : in_len;
2617 }
2618
2619 static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf,
2620 int in_len,
2621 u32 wr_count,
2622 u32 sge_count,
2623 u32 wqe_size)
2624 {
2625 struct ib_uverbs_recv_wr *user_wr;
2626 struct ib_recv_wr *wr = NULL, *last, *next;
2627 int sg_ind;
2628 int i;
2629 int ret;
2630
2631 if (in_len < wqe_size * wr_count +
2632 sge_count * sizeof (struct ib_uverbs_sge))
2633 return ERR_PTR(-EINVAL);
2634
2635 if (wqe_size < sizeof (struct ib_uverbs_recv_wr))
2636 return ERR_PTR(-EINVAL);
2637
2638 user_wr = kmalloc(wqe_size, GFP_KERNEL);
2639 if (!user_wr)
2640 return ERR_PTR(-ENOMEM);
2641
2642 sg_ind = 0;
2643 last = NULL;
2644 for (i = 0; i < wr_count; ++i) {
2645 if (copy_from_user(user_wr, buf + i * wqe_size,
2646 wqe_size)) {
2647 ret = -EFAULT;
2648 goto err;
2649 }
2650
2651 if (user_wr->num_sge + sg_ind > sge_count) {
2652 ret = -EINVAL;
2653 goto err;
2654 }
2655
2656 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) +
2657 user_wr->num_sge * sizeof (struct ib_sge),
2658 GFP_KERNEL);
2659 if (!next) {
2660 ret = -ENOMEM;
2661 goto err;
2662 }
2663
2664 if (!last)
2665 wr = next;
2666 else
2667 last->next = next;
2668 last = next;
2669
2670 next->next = NULL;
2671 next->wr_id = user_wr->wr_id;
2672 next->num_sge = user_wr->num_sge;
2673
2674 if (next->num_sge) {
2675 next->sg_list = (void *) next +
2676 ALIGN(sizeof *next, sizeof (struct ib_sge));
2677 if (copy_from_user(next->sg_list,
2678 buf + wr_count * wqe_size +
2679 sg_ind * sizeof (struct ib_sge),
2680 next->num_sge * sizeof (struct ib_sge))) {
2681 ret = -EFAULT;
2682 goto err;
2683 }
2684 sg_ind += next->num_sge;
2685 } else
2686 next->sg_list = NULL;
2687 }
2688
2689 kfree(user_wr);
2690 return wr;
2691
2692 err:
2693 kfree(user_wr);
2694
2695 while (wr) {
2696 next = wr->next;
2697 kfree(wr);
2698 wr = next;
2699 }
2700
2701 return ERR_PTR(ret);
2702 }
2703
2704 ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file,
2705 struct ib_device *ib_dev,
2706 const char __user *buf, int in_len,
2707 int out_len)
2708 {
2709 struct ib_uverbs_post_recv cmd;
2710 struct ib_uverbs_post_recv_resp resp;
2711 struct ib_recv_wr *wr, *next, *bad_wr;
2712 struct ib_qp *qp;
2713 ssize_t ret = -EINVAL;
2714
2715 if (copy_from_user(&cmd, buf, sizeof cmd))
2716 return -EFAULT;
2717
2718 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2719 in_len - sizeof cmd, cmd.wr_count,
2720 cmd.sge_count, cmd.wqe_size);
2721 if (IS_ERR(wr))
2722 return PTR_ERR(wr);
2723
2724 qp = idr_read_qp(cmd.qp_handle, file->ucontext);
2725 if (!qp)
2726 goto out;
2727
2728 resp.bad_wr = 0;
2729 ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr);
2730
2731 put_qp_read(qp);
2732
2733 if (ret)
2734 for (next = wr; next; next = next->next) {
2735 ++resp.bad_wr;
2736 if (next == bad_wr)
2737 break;
2738 }
2739
2740 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2741 &resp, sizeof resp))
2742 ret = -EFAULT;
2743
2744 out:
2745 while (wr) {
2746 next = wr->next;
2747 kfree(wr);
2748 wr = next;
2749 }
2750
2751 return ret ? ret : in_len;
2752 }
2753
2754 ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file,
2755 struct ib_device *ib_dev,
2756 const char __user *buf, int in_len,
2757 int out_len)
2758 {
2759 struct ib_uverbs_post_srq_recv cmd;
2760 struct ib_uverbs_post_srq_recv_resp resp;
2761 struct ib_recv_wr *wr, *next, *bad_wr;
2762 struct ib_srq *srq;
2763 ssize_t ret = -EINVAL;
2764
2765 if (copy_from_user(&cmd, buf, sizeof cmd))
2766 return -EFAULT;
2767
2768 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2769 in_len - sizeof cmd, cmd.wr_count,
2770 cmd.sge_count, cmd.wqe_size);
2771 if (IS_ERR(wr))
2772 return PTR_ERR(wr);
2773
2774 srq = idr_read_srq(cmd.srq_handle, file->ucontext);
2775 if (!srq)
2776 goto out;
2777
2778 resp.bad_wr = 0;
2779 ret = srq->device->post_srq_recv(srq, wr, &bad_wr);
2780
2781 put_srq_read(srq);
2782
2783 if (ret)
2784 for (next = wr; next; next = next->next) {
2785 ++resp.bad_wr;
2786 if (next == bad_wr)
2787 break;
2788 }
2789
2790 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2791 &resp, sizeof resp))
2792 ret = -EFAULT;
2793
2794 out:
2795 while (wr) {
2796 next = wr->next;
2797 kfree(wr);
2798 wr = next;
2799 }
2800
2801 return ret ? ret : in_len;
2802 }
2803
2804 ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
2805 struct ib_device *ib_dev,
2806 const char __user *buf, int in_len,
2807 int out_len)
2808 {
2809 struct ib_uverbs_create_ah cmd;
2810 struct ib_uverbs_create_ah_resp resp;
2811 struct ib_uobject *uobj;
2812 struct ib_pd *pd;
2813 struct ib_ah *ah;
2814 struct ib_ah_attr attr;
2815 int ret;
2816
2817 if (out_len < sizeof resp)
2818 return -ENOSPC;
2819
2820 if (copy_from_user(&cmd, buf, sizeof cmd))
2821 return -EFAULT;
2822
2823 uobj = kmalloc(sizeof *uobj, GFP_KERNEL);
2824 if (!uobj)
2825 return -ENOMEM;
2826
2827 init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_class);
2828 down_write(&uobj->mutex);
2829
2830 pd = idr_read_pd(cmd.pd_handle, file->ucontext);
2831 if (!pd) {
2832 ret = -EINVAL;
2833 goto err;
2834 }
2835
2836 attr.dlid = cmd.attr.dlid;
2837 attr.sl = cmd.attr.sl;
2838 attr.src_path_bits = cmd.attr.src_path_bits;
2839 attr.static_rate = cmd.attr.static_rate;
2840 attr.ah_flags = cmd.attr.is_global ? IB_AH_GRH : 0;
2841 attr.port_num = cmd.attr.port_num;
2842 attr.grh.flow_label = cmd.attr.grh.flow_label;
2843 attr.grh.sgid_index = cmd.attr.grh.sgid_index;
2844 attr.grh.hop_limit = cmd.attr.grh.hop_limit;
2845 attr.grh.traffic_class = cmd.attr.grh.traffic_class;
2846 memset(&attr.dmac, 0, sizeof(attr.dmac));
2847 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16);
2848
2849 ah = ib_create_ah(pd, &attr);
2850 if (IS_ERR(ah)) {
2851 ret = PTR_ERR(ah);
2852 goto err_put;
2853 }
2854
2855 ah->uobject = uobj;
2856 uobj->object = ah;
2857
2858 ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj);
2859 if (ret)
2860 goto err_destroy;
2861
2862 resp.ah_handle = uobj->id;
2863
2864 if (copy_to_user((void __user *) (unsigned long) cmd.response,
2865 &resp, sizeof resp)) {
2866 ret = -EFAULT;
2867 goto err_copy;
2868 }
2869
2870 put_pd_read(pd);
2871
2872 mutex_lock(&file->mutex);
2873 list_add_tail(&uobj->list, &file->ucontext->ah_list);
2874 mutex_unlock(&file->mutex);
2875
2876 uobj->live = 1;
2877
2878 up_write(&uobj->mutex);
2879
2880 return in_len;
2881
2882 err_copy:
2883 idr_remove_uobj(&ib_uverbs_ah_idr, uobj);
2884
2885 err_destroy:
2886 ib_destroy_ah(ah);
2887
2888 err_put:
2889 put_pd_read(pd);
2890
2891 err:
2892 put_uobj_write(uobj);
2893 return ret;
2894 }
2895
2896 ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file,
2897 struct ib_device *ib_dev,
2898 const char __user *buf, int in_len, int out_len)
2899 {
2900 struct ib_uverbs_destroy_ah cmd;
2901 struct ib_ah *ah;
2902 struct ib_uobject *uobj;
2903 int ret;
2904
2905 if (copy_from_user(&cmd, buf, sizeof cmd))
2906 return -EFAULT;
2907
2908 uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext);
2909 if (!uobj)
2910 return -EINVAL;
2911 ah = uobj->object;
2912
2913 ret = ib_destroy_ah(ah);
2914 if (!ret)
2915 uobj->live = 0;
2916
2917 put_uobj_write(uobj);
2918
2919 if (ret)
2920 return ret;
2921
2922 idr_remove_uobj(&ib_uverbs_ah_idr, uobj);
2923
2924 mutex_lock(&file->mutex);
2925 list_del(&uobj->list);
2926 mutex_unlock(&file->mutex);
2927
2928 put_uobj(uobj);
2929
2930 return in_len;
2931 }
2932
2933 ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file,
2934 struct ib_device *ib_dev,
2935 const char __user *buf, int in_len,
2936 int out_len)
2937 {
2938 struct ib_uverbs_attach_mcast cmd;
2939 struct ib_qp *qp;
2940 struct ib_uqp_object *obj;
2941 struct ib_uverbs_mcast_entry *mcast;
2942 int ret;
2943
2944 if (copy_from_user(&cmd, buf, sizeof cmd))
2945 return -EFAULT;
2946
2947 qp = idr_write_qp(cmd.qp_handle, file->ucontext);
2948 if (!qp)
2949 return -EINVAL;
2950
2951 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2952
2953 list_for_each_entry(mcast, &obj->mcast_list, list)
2954 if (cmd.mlid == mcast->lid &&
2955 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2956 ret = 0;
2957 goto out_put;
2958 }
2959
2960 mcast = kmalloc(sizeof *mcast, GFP_KERNEL);
2961 if (!mcast) {
2962 ret = -ENOMEM;
2963 goto out_put;
2964 }
2965
2966 mcast->lid = cmd.mlid;
2967 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw);
2968
2969 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid);
2970 if (!ret)
2971 list_add_tail(&mcast->list, &obj->mcast_list);
2972 else
2973 kfree(mcast);
2974
2975 out_put:
2976 put_qp_write(qp);
2977
2978 return ret ? ret : in_len;
2979 }
2980
2981 ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file,
2982 struct ib_device *ib_dev,
2983 const char __user *buf, int in_len,
2984 int out_len)
2985 {
2986 struct ib_uverbs_detach_mcast cmd;
2987 struct ib_uqp_object *obj;
2988 struct ib_qp *qp;
2989 struct ib_uverbs_mcast_entry *mcast;
2990 int ret = -EINVAL;
2991
2992 if (copy_from_user(&cmd, buf, sizeof cmd))
2993 return -EFAULT;
2994
2995 qp = idr_write_qp(cmd.qp_handle, file->ucontext);
2996 if (!qp)
2997 return -EINVAL;
2998
2999 ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid);
3000 if (ret)
3001 goto out_put;
3002
3003 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
3004
3005 list_for_each_entry(mcast, &obj->mcast_list, list)
3006 if (cmd.mlid == mcast->lid &&
3007 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
3008 list_del(&mcast->list);
3009 kfree(mcast);
3010 break;
3011 }
3012
3013 out_put:
3014 put_qp_write(qp);
3015
3016 return ret ? ret : in_len;
3017 }
3018
3019 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec,
3020 union ib_flow_spec *ib_spec)
3021 {
3022 if (kern_spec->reserved)
3023 return -EINVAL;
3024
3025 ib_spec->type = kern_spec->type;
3026
3027 switch (ib_spec->type) {
3028 case IB_FLOW_SPEC_ETH:
3029 ib_spec->eth.size = sizeof(struct ib_flow_spec_eth);
3030 if (ib_spec->eth.size != kern_spec->eth.size)
3031 return -EINVAL;
3032 memcpy(&ib_spec->eth.val, &kern_spec->eth.val,
3033 sizeof(struct ib_flow_eth_filter));
3034 memcpy(&ib_spec->eth.mask, &kern_spec->eth.mask,
3035 sizeof(struct ib_flow_eth_filter));
3036 break;
3037 case IB_FLOW_SPEC_IPV4:
3038 ib_spec->ipv4.size = sizeof(struct ib_flow_spec_ipv4);
3039 if (ib_spec->ipv4.size != kern_spec->ipv4.size)
3040 return -EINVAL;
3041 memcpy(&ib_spec->ipv4.val, &kern_spec->ipv4.val,
3042 sizeof(struct ib_flow_ipv4_filter));
3043 memcpy(&ib_spec->ipv4.mask, &kern_spec->ipv4.mask,
3044 sizeof(struct ib_flow_ipv4_filter));
3045 break;
3046 case IB_FLOW_SPEC_TCP:
3047 case IB_FLOW_SPEC_UDP:
3048 ib_spec->tcp_udp.size = sizeof(struct ib_flow_spec_tcp_udp);
3049 if (ib_spec->tcp_udp.size != kern_spec->tcp_udp.size)
3050 return -EINVAL;
3051 memcpy(&ib_spec->tcp_udp.val, &kern_spec->tcp_udp.val,
3052 sizeof(struct ib_flow_tcp_udp_filter));
3053 memcpy(&ib_spec->tcp_udp.mask, &kern_spec->tcp_udp.mask,
3054 sizeof(struct ib_flow_tcp_udp_filter));
3055 break;
3056 default:
3057 return -EINVAL;
3058 }
3059 return 0;
3060 }
3061
3062 int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file,
3063 struct ib_device *ib_dev,
3064 struct ib_udata *ucore,
3065 struct ib_udata *uhw)
3066 {
3067 struct ib_uverbs_create_flow cmd;
3068 struct ib_uverbs_create_flow_resp resp;
3069 struct ib_uobject *uobj;
3070 struct ib_flow *flow_id;
3071 struct ib_uverbs_flow_attr *kern_flow_attr;
3072 struct ib_flow_attr *flow_attr;
3073 struct ib_qp *qp;
3074 int err = 0;
3075 void *kern_spec;
3076 void *ib_spec;
3077 int i;
3078
3079 if (ucore->inlen < sizeof(cmd))
3080 return -EINVAL;
3081
3082 if (ucore->outlen < sizeof(resp))
3083 return -ENOSPC;
3084
3085 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
3086 if (err)
3087 return err;
3088
3089 ucore->inbuf += sizeof(cmd);
3090 ucore->inlen -= sizeof(cmd);
3091
3092 if (cmd.comp_mask)
3093 return -EINVAL;
3094
3095 if ((cmd.flow_attr.type == IB_FLOW_ATTR_SNIFFER &&
3096 !capable(CAP_NET_ADMIN)) || !capable(CAP_NET_RAW))
3097 return -EPERM;
3098
3099 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
3100 return -EINVAL;
3101
3102 if (cmd.flow_attr.size > ucore->inlen ||
3103 cmd.flow_attr.size >
3104 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
3105 return -EINVAL;
3106
3107 if (cmd.flow_attr.reserved[0] ||
3108 cmd.flow_attr.reserved[1])
3109 return -EINVAL;
3110
3111 if (cmd.flow_attr.num_of_specs) {
3112 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size,
3113 GFP_KERNEL);
3114 if (!kern_flow_attr)
3115 return -ENOMEM;
3116
3117 memcpy(kern_flow_attr, &cmd.flow_attr, sizeof(*kern_flow_attr));
3118 err = ib_copy_from_udata(kern_flow_attr + 1, ucore,
3119 cmd.flow_attr.size);
3120 if (err)
3121 goto err_free_attr;
3122 } else {
3123 kern_flow_attr = &cmd.flow_attr;
3124 }
3125
3126 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL);
3127 if (!uobj) {
3128 err = -ENOMEM;
3129 goto err_free_attr;
3130 }
3131 init_uobj(uobj, 0, file->ucontext, &rule_lock_class);
3132 down_write(&uobj->mutex);
3133
3134 qp = idr_read_qp(cmd.qp_handle, file->ucontext);
3135 if (!qp) {
3136 err = -EINVAL;
3137 goto err_uobj;
3138 }
3139
3140 flow_attr = kmalloc(sizeof(*flow_attr) + cmd.flow_attr.size, GFP_KERNEL);
3141 if (!flow_attr) {
3142 err = -ENOMEM;
3143 goto err_put;
3144 }
3145
3146 flow_attr->type = kern_flow_attr->type;
3147 flow_attr->priority = kern_flow_attr->priority;
3148 flow_attr->num_of_specs = kern_flow_attr->num_of_specs;
3149 flow_attr->port = kern_flow_attr->port;
3150 flow_attr->flags = kern_flow_attr->flags;
3151 flow_attr->size = sizeof(*flow_attr);
3152
3153 kern_spec = kern_flow_attr + 1;
3154 ib_spec = flow_attr + 1;
3155 for (i = 0; i < flow_attr->num_of_specs &&
3156 cmd.flow_attr.size > offsetof(struct ib_uverbs_flow_spec, reserved) &&
3157 cmd.flow_attr.size >=
3158 ((struct ib_uverbs_flow_spec *)kern_spec)->size; i++) {
3159 err = kern_spec_to_ib_spec(kern_spec, ib_spec);
3160 if (err)
3161 goto err_free;
3162 flow_attr->size +=
3163 ((union ib_flow_spec *) ib_spec)->size;
3164 cmd.flow_attr.size -= ((struct ib_uverbs_flow_spec *)kern_spec)->size;
3165 kern_spec += ((struct ib_uverbs_flow_spec *) kern_spec)->size;
3166 ib_spec += ((union ib_flow_spec *) ib_spec)->size;
3167 }
3168 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) {
3169 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3170 i, cmd.flow_attr.size);
3171 err = -EINVAL;
3172 goto err_free;
3173 }
3174 flow_id = ib_create_flow(qp, flow_attr, IB_FLOW_DOMAIN_USER);
3175 if (IS_ERR(flow_id)) {
3176 err = PTR_ERR(flow_id);
3177 goto err_free;
3178 }
3179 flow_id->qp = qp;
3180 flow_id->uobject = uobj;
3181 uobj->object = flow_id;
3182
3183 err = idr_add_uobj(&ib_uverbs_rule_idr, uobj);
3184 if (err)
3185 goto destroy_flow;
3186
3187 memset(&resp, 0, sizeof(resp));
3188 resp.flow_handle = uobj->id;
3189
3190 err = ib_copy_to_udata(ucore,
3191 &resp, sizeof(resp));
3192 if (err)
3193 goto err_copy;
3194
3195 put_qp_read(qp);
3196 mutex_lock(&file->mutex);
3197 list_add_tail(&uobj->list, &file->ucontext->rule_list);
3198 mutex_unlock(&file->mutex);
3199
3200 uobj->live = 1;
3201
3202 up_write(&uobj->mutex);
3203 kfree(flow_attr);
3204 if (cmd.flow_attr.num_of_specs)
3205 kfree(kern_flow_attr);
3206 return 0;
3207 err_copy:
3208 idr_remove_uobj(&ib_uverbs_rule_idr, uobj);
3209 destroy_flow:
3210 ib_destroy_flow(flow_id);
3211 err_free:
3212 kfree(flow_attr);
3213 err_put:
3214 put_qp_read(qp);
3215 err_uobj:
3216 put_uobj_write(uobj);
3217 err_free_attr:
3218 if (cmd.flow_attr.num_of_specs)
3219 kfree(kern_flow_attr);
3220 return err;
3221 }
3222
3223 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file,
3224 struct ib_device *ib_dev,
3225 struct ib_udata *ucore,
3226 struct ib_udata *uhw)
3227 {
3228 struct ib_uverbs_destroy_flow cmd;
3229 struct ib_flow *flow_id;
3230 struct ib_uobject *uobj;
3231 int ret;
3232
3233 if (ucore->inlen < sizeof(cmd))
3234 return -EINVAL;
3235
3236 ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
3237 if (ret)
3238 return ret;
3239
3240 if (cmd.comp_mask)
3241 return -EINVAL;
3242
3243 uobj = idr_write_uobj(&ib_uverbs_rule_idr, cmd.flow_handle,
3244 file->ucontext);
3245 if (!uobj)
3246 return -EINVAL;
3247 flow_id = uobj->object;
3248
3249 ret = ib_destroy_flow(flow_id);
3250 if (!ret)
3251 uobj->live = 0;
3252
3253 put_uobj_write(uobj);
3254
3255 idr_remove_uobj(&ib_uverbs_rule_idr, uobj);
3256
3257 mutex_lock(&file->mutex);
3258 list_del(&uobj->list);
3259 mutex_unlock(&file->mutex);
3260
3261 put_uobj(uobj);
3262
3263 return ret;
3264 }
3265
3266 static int __uverbs_create_xsrq(struct ib_uverbs_file *file,
3267 struct ib_device *ib_dev,
3268 struct ib_uverbs_create_xsrq *cmd,
3269 struct ib_udata *udata)
3270 {
3271 struct ib_uverbs_create_srq_resp resp;
3272 struct ib_usrq_object *obj;
3273 struct ib_pd *pd;
3274 struct ib_srq *srq;
3275 struct ib_uobject *uninitialized_var(xrcd_uobj);
3276 struct ib_srq_init_attr attr;
3277 int ret;
3278
3279 obj = kmalloc(sizeof *obj, GFP_KERNEL);
3280 if (!obj)
3281 return -ENOMEM;
3282
3283 init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, &srq_lock_class);
3284 down_write(&obj->uevent.uobject.mutex);
3285
3286 if (cmd->srq_type == IB_SRQT_XRC) {
3287 attr.ext.xrc.xrcd = idr_read_xrcd(cmd->xrcd_handle, file->ucontext, &xrcd_uobj);
3288 if (!attr.ext.xrc.xrcd) {
3289 ret = -EINVAL;
3290 goto err;
3291 }
3292
3293 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
3294 atomic_inc(&obj->uxrcd->refcnt);
3295
3296 attr.ext.xrc.cq = idr_read_cq(cmd->cq_handle, file->ucontext, 0);
3297 if (!attr.ext.xrc.cq) {
3298 ret = -EINVAL;
3299 goto err_put_xrcd;
3300 }
3301 }
3302
3303 pd = idr_read_pd(cmd->pd_handle, file->ucontext);
3304 if (!pd) {
3305 ret = -EINVAL;
3306 goto err_put_cq;
3307 }
3308
3309 attr.event_handler = ib_uverbs_srq_event_handler;
3310 attr.srq_context = file;
3311 attr.srq_type = cmd->srq_type;
3312 attr.attr.max_wr = cmd->max_wr;
3313 attr.attr.max_sge = cmd->max_sge;
3314 attr.attr.srq_limit = cmd->srq_limit;
3315
3316 obj->uevent.events_reported = 0;
3317 INIT_LIST_HEAD(&obj->uevent.event_list);
3318
3319 srq = pd->device->create_srq(pd, &attr, udata);
3320 if (IS_ERR(srq)) {
3321 ret = PTR_ERR(srq);
3322 goto err_put;
3323 }
3324
3325 srq->device = pd->device;
3326 srq->pd = pd;
3327 srq->srq_type = cmd->srq_type;
3328 srq->uobject = &obj->uevent.uobject;
3329 srq->event_handler = attr.event_handler;
3330 srq->srq_context = attr.srq_context;
3331
3332 if (cmd->srq_type == IB_SRQT_XRC) {
3333 srq->ext.xrc.cq = attr.ext.xrc.cq;
3334 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
3335 atomic_inc(&attr.ext.xrc.cq->usecnt);
3336 atomic_inc(&attr.ext.xrc.xrcd->usecnt);
3337 }
3338
3339 atomic_inc(&pd->usecnt);
3340 atomic_set(&srq->usecnt, 0);
3341
3342 obj->uevent.uobject.object = srq;
3343 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject);
3344 if (ret)
3345 goto err_destroy;
3346
3347 memset(&resp, 0, sizeof resp);
3348 resp.srq_handle = obj->uevent.uobject.id;
3349 resp.max_wr = attr.attr.max_wr;
3350 resp.max_sge = attr.attr.max_sge;
3351 if (cmd->srq_type == IB_SRQT_XRC)
3352 resp.srqn = srq->ext.xrc.srq_num;
3353
3354 if (copy_to_user((void __user *) (unsigned long) cmd->response,
3355 &resp, sizeof resp)) {
3356 ret = -EFAULT;
3357 goto err_copy;
3358 }
3359
3360 if (cmd->srq_type == IB_SRQT_XRC) {
3361 put_uobj_read(xrcd_uobj);
3362 put_cq_read(attr.ext.xrc.cq);
3363 }
3364 put_pd_read(pd);
3365
3366 mutex_lock(&file->mutex);
3367 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->srq_list);
3368 mutex_unlock(&file->mutex);
3369
3370 obj->uevent.uobject.live = 1;
3371
3372 up_write(&obj->uevent.uobject.mutex);
3373
3374 return 0;
3375
3376 err_copy:
3377 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject);
3378
3379 err_destroy:
3380 ib_destroy_srq(srq);
3381
3382 err_put:
3383 put_pd_read(pd);
3384
3385 err_put_cq:
3386 if (cmd->srq_type == IB_SRQT_XRC)
3387 put_cq_read(attr.ext.xrc.cq);
3388
3389 err_put_xrcd:
3390 if (cmd->srq_type == IB_SRQT_XRC) {
3391 atomic_dec(&obj->uxrcd->refcnt);
3392 put_uobj_read(xrcd_uobj);
3393 }
3394
3395 err:
3396 put_uobj_write(&obj->uevent.uobject);
3397 return ret;
3398 }
3399
3400 ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file,
3401 struct ib_device *ib_dev,
3402 const char __user *buf, int in_len,
3403 int out_len)
3404 {
3405 struct ib_uverbs_create_srq cmd;
3406 struct ib_uverbs_create_xsrq xcmd;
3407 struct ib_uverbs_create_srq_resp resp;
3408 struct ib_udata udata;
3409 int ret;
3410
3411 if (out_len < sizeof resp)
3412 return -ENOSPC;
3413
3414 if (copy_from_user(&cmd, buf, sizeof cmd))
3415 return -EFAULT;
3416
3417 xcmd.response = cmd.response;
3418 xcmd.user_handle = cmd.user_handle;
3419 xcmd.srq_type = IB_SRQT_BASIC;
3420 xcmd.pd_handle = cmd.pd_handle;
3421 xcmd.max_wr = cmd.max_wr;
3422 xcmd.max_sge = cmd.max_sge;
3423 xcmd.srq_limit = cmd.srq_limit;
3424
3425 INIT_UDATA(&udata, buf + sizeof cmd,
3426 (unsigned long) cmd.response + sizeof resp,
3427 in_len - sizeof cmd, out_len - sizeof resp);
3428
3429 ret = __uverbs_create_xsrq(file, ib_dev, &xcmd, &udata);
3430 if (ret)
3431 return ret;
3432
3433 return in_len;
3434 }
3435
3436 ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file,
3437 struct ib_device *ib_dev,
3438 const char __user *buf, int in_len, int out_len)
3439 {
3440 struct ib_uverbs_create_xsrq cmd;
3441 struct ib_uverbs_create_srq_resp resp;
3442 struct ib_udata udata;
3443 int ret;
3444
3445 if (out_len < sizeof resp)
3446 return -ENOSPC;
3447
3448 if (copy_from_user(&cmd, buf, sizeof cmd))
3449 return -EFAULT;
3450
3451 INIT_UDATA(&udata, buf + sizeof cmd,
3452 (unsigned long) cmd.response + sizeof resp,
3453 in_len - sizeof cmd, out_len - sizeof resp);
3454
3455 ret = __uverbs_create_xsrq(file, ib_dev, &cmd, &udata);
3456 if (ret)
3457 return ret;
3458
3459 return in_len;
3460 }
3461
3462 ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file,
3463 struct ib_device *ib_dev,
3464 const char __user *buf, int in_len,
3465 int out_len)
3466 {
3467 struct ib_uverbs_modify_srq cmd;
3468 struct ib_udata udata;
3469 struct ib_srq *srq;
3470 struct ib_srq_attr attr;
3471 int ret;
3472
3473 if (copy_from_user(&cmd, buf, sizeof cmd))
3474 return -EFAULT;
3475
3476 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd,
3477 out_len);
3478
3479 srq = idr_read_srq(cmd.srq_handle, file->ucontext);
3480 if (!srq)
3481 return -EINVAL;
3482
3483 attr.max_wr = cmd.max_wr;
3484 attr.srq_limit = cmd.srq_limit;
3485
3486 ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata);
3487
3488 put_srq_read(srq);
3489
3490 return ret ? ret : in_len;
3491 }
3492
3493 ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file,
3494 struct ib_device *ib_dev,
3495 const char __user *buf,
3496 int in_len, int out_len)
3497 {
3498 struct ib_uverbs_query_srq cmd;
3499 struct ib_uverbs_query_srq_resp resp;
3500 struct ib_srq_attr attr;
3501 struct ib_srq *srq;
3502 int ret;
3503
3504 if (out_len < sizeof resp)
3505 return -ENOSPC;
3506
3507 if (copy_from_user(&cmd, buf, sizeof cmd))
3508 return -EFAULT;
3509
3510 srq = idr_read_srq(cmd.srq_handle, file->ucontext);
3511 if (!srq)
3512 return -EINVAL;
3513
3514 ret = ib_query_srq(srq, &attr);
3515
3516 put_srq_read(srq);
3517
3518 if (ret)
3519 return ret;
3520
3521 memset(&resp, 0, sizeof resp);
3522
3523 resp.max_wr = attr.max_wr;
3524 resp.max_sge = attr.max_sge;
3525 resp.srq_limit = attr.srq_limit;
3526
3527 if (copy_to_user((void __user *) (unsigned long) cmd.response,
3528 &resp, sizeof resp))
3529 return -EFAULT;
3530
3531 return in_len;
3532 }
3533
3534 ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file,
3535 struct ib_device *ib_dev,
3536 const char __user *buf, int in_len,
3537 int out_len)
3538 {
3539 struct ib_uverbs_destroy_srq cmd;
3540 struct ib_uverbs_destroy_srq_resp resp;
3541 struct ib_uobject *uobj;
3542 struct ib_srq *srq;
3543 struct ib_uevent_object *obj;
3544 int ret = -EINVAL;
3545 struct ib_usrq_object *us;
3546 enum ib_srq_type srq_type;
3547
3548 if (copy_from_user(&cmd, buf, sizeof cmd))
3549 return -EFAULT;
3550
3551 uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext);
3552 if (!uobj)
3553 return -EINVAL;
3554 srq = uobj->object;
3555 obj = container_of(uobj, struct ib_uevent_object, uobject);
3556 srq_type = srq->srq_type;
3557
3558 ret = ib_destroy_srq(srq);
3559 if (!ret)
3560 uobj->live = 0;
3561
3562 put_uobj_write(uobj);
3563
3564 if (ret)
3565 return ret;
3566
3567 if (srq_type == IB_SRQT_XRC) {
3568 us = container_of(obj, struct ib_usrq_object, uevent);
3569 atomic_dec(&us->uxrcd->refcnt);
3570 }
3571
3572 idr_remove_uobj(&ib_uverbs_srq_idr, uobj);
3573
3574 mutex_lock(&file->mutex);
3575 list_del(&uobj->list);
3576 mutex_unlock(&file->mutex);
3577
3578 ib_uverbs_release_uevent(file, obj);
3579
3580 memset(&resp, 0, sizeof resp);
3581 resp.events_reported = obj->events_reported;
3582
3583 put_uobj(uobj);
3584
3585 if (copy_to_user((void __user *) (unsigned long) cmd.response,
3586 &resp, sizeof resp))
3587 ret = -EFAULT;
3588
3589 return ret ? ret : in_len;
3590 }
3591
3592 int ib_uverbs_ex_query_device(struct ib_uverbs_file *file,
3593 struct ib_device *ib_dev,
3594 struct ib_udata *ucore,
3595 struct ib_udata *uhw)
3596 {
3597 struct ib_uverbs_ex_query_device_resp resp;
3598 struct ib_uverbs_ex_query_device cmd;
3599 struct ib_device_attr attr;
3600 int err;
3601
3602 if (ucore->inlen < sizeof(cmd))
3603 return -EINVAL;
3604
3605 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
3606 if (err)
3607 return err;
3608
3609 if (cmd.comp_mask)
3610 return -EINVAL;
3611
3612 if (cmd.reserved)
3613 return -EINVAL;
3614
3615 resp.response_length = offsetof(typeof(resp), odp_caps);
3616
3617 if (ucore->outlen < resp.response_length)
3618 return -ENOSPC;
3619
3620 memset(&attr, 0, sizeof(attr));
3621
3622 err = ib_dev->query_device(ib_dev, &attr, uhw);
3623 if (err)
3624 return err;
3625
3626 copy_query_dev_fields(file, ib_dev, &resp.base, &attr);
3627 resp.comp_mask = 0;
3628
3629 if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps))
3630 goto end;
3631
3632 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
3633 resp.odp_caps.general_caps = attr.odp_caps.general_caps;
3634 resp.odp_caps.per_transport_caps.rc_odp_caps =
3635 attr.odp_caps.per_transport_caps.rc_odp_caps;
3636 resp.odp_caps.per_transport_caps.uc_odp_caps =
3637 attr.odp_caps.per_transport_caps.uc_odp_caps;
3638 resp.odp_caps.per_transport_caps.ud_odp_caps =
3639 attr.odp_caps.per_transport_caps.ud_odp_caps;
3640 resp.odp_caps.reserved = 0;
3641 #else
3642 memset(&resp.odp_caps, 0, sizeof(resp.odp_caps));
3643 #endif
3644 resp.response_length += sizeof(resp.odp_caps);
3645
3646 if (ucore->outlen < resp.response_length + sizeof(resp.timestamp_mask))
3647 goto end;
3648
3649 resp.timestamp_mask = attr.timestamp_mask;
3650 resp.response_length += sizeof(resp.timestamp_mask);
3651
3652 if (ucore->outlen < resp.response_length + sizeof(resp.hca_core_clock))
3653 goto end;
3654
3655 resp.hca_core_clock = attr.hca_core_clock;
3656 resp.response_length += sizeof(resp.hca_core_clock);
3657
3658 end:
3659 err = ib_copy_to_udata(ucore, &resp, resp.response_length);
3660 if (err)
3661 return err;
3662
3663 return 0;
3664 }
Linux kernel - Deliverables
This page took 0.143025 seconds and 5 git commands to generate.