projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/mlx5: Fix eq names to display nicely in /proc/interrupts
[deliverable/linux.git] / drivers / infiniband / hw / mlx5 / mr.c
CommitLineData
e126ba97
EC		 1/*
2 * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33
34#include <linux/kref.h>
35#include <linux/random.h>
36#include <linux/debugfs.h>
37#include <linux/export.h>
38#include <rdma/ib_umem.h>
39#include "mlx5_ib.h"
40
41enum {
42 DEF_CACHE_SIZE = 10,
43};
44
45static __be64 *mr_align(__be64 *ptr, int align)
46{
47 unsigned long mask = align - 1;
48
49 return (__be64 *)(((unsigned long)ptr + mask) & ~mask);
50}
51
52static int order2idx(struct mlx5_ib_dev *dev, int order)
53{
54 struct mlx5_mr_cache *cache = &dev->cache;
55
56 if (order < cache->ent[0].order)
57 return 0;
58 else
59 return order - cache->ent[0].order;
60}
61
62static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
63{
e126ba97
EC		 64 struct mlx5_mr_cache *cache = &dev->cache;
65 struct mlx5_cache_ent *ent = &cache->ent[c];
66 struct mlx5_create_mkey_mbox_in *in;
67 struct mlx5_ib_mr *mr;
68 int npages = 1 << ent->order;
e126ba97
EC		 69 int err = 0;
70 int i;
71
72 in = kzalloc(sizeof(*in), GFP_KERNEL);
73 if (!in)
74 return -ENOMEM;
75
76 for (i = 0; i < num; i++) {
77 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
78 if (!mr) {
79 err = -ENOMEM;
80 goto out;
81 }
82 mr->order = ent->order;
83 mr->umred = 1;
e126ba97
EC		 84 in->seg.status = 1 << 6;
85 in->seg.xlt_oct_size = cpu_to_be32((npages + 1) / 2);
86 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
87 in->seg.flags = MLX5_ACCESS_MODE_MTT | MLX5_PERM_UMR_EN;
88 in->seg.log2_page_size = 12;
89
90 err = mlx5_core_create_mkey(&dev->mdev, &mr->mmr, in,
91 sizeof(*in));
92 if (err) {
93 mlx5_ib_warn(dev, "create mkey failed %d\n", err);
e126ba97
EC		 94 kfree(mr);
95 goto out;
96 }
97 cache->last_add = jiffies;
98
99 spin_lock(&ent->lock);
100 list_add_tail(&mr->list, &ent->head);
101 ent->cur++;
102 ent->size++;
103 spin_unlock(&ent->lock);
104 }
105
106out:
107 kfree(in);
108 return err;
109}
110
111static void remove_keys(struct mlx5_ib_dev *dev, int c, int num)
112{
e126ba97
EC		 113 struct mlx5_mr_cache *cache = &dev->cache;
114 struct mlx5_cache_ent *ent = &cache->ent[c];
115 struct mlx5_ib_mr *mr;
e126ba97
EC		 116 int err;
117 int i;
118
119 for (i = 0; i < num; i++) {
120 spin_lock(&ent->lock);
121 if (list_empty(&ent->head)) {
122 spin_unlock(&ent->lock);
123 return;
124 }
125 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
126 list_del(&mr->list);
127 ent->cur--;
128 ent->size--;
129 spin_unlock(&ent->lock);
130 err = mlx5_core_destroy_mkey(&dev->mdev, &mr->mmr);
203099fd 131 if (err)
e126ba97 132 mlx5_ib_warn(dev, "failed destroy mkey\n");
203099fd 133 else
e126ba97 134 kfree(mr);
e126ba97
EC		 135 }
136}
137
138static ssize_t size_write(struct file *filp, const char __user *buf,
139 size_t count, loff_t *pos)
140{
141 struct mlx5_cache_ent *ent = filp->private_data;
142 struct mlx5_ib_dev *dev = ent->dev;
143 char lbuf[20];
144 u32 var;
145 int err;
146 int c;
147
148 if (copy_from_user(lbuf, buf, sizeof(lbuf)))
5e631a03 149 return -EFAULT;
e126ba97
EC		 150
151 c = order2idx(dev, ent->order);
152 lbuf[sizeof(lbuf) - 1] = 0;
153
154 if (sscanf(lbuf, "%u", &var) != 1)
155 return -EINVAL;
156
157 if (var < ent->limit)
158 return -EINVAL;
159
160 if (var > ent->size) {
161 err = add_keys(dev, c, var - ent->size);
162 if (err)
163 return err;
164 } else if (var < ent->size) {
165 remove_keys(dev, c, ent->size - var);
166 }
167
168 return count;
169}
170
171static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
172 loff_t *pos)
173{
174 struct mlx5_cache_ent *ent = filp->private_data;
175 char lbuf[20];
176 int err;
177
178 if (*pos)
179 return 0;
180
181 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->size);
182 if (err < 0)
183 return err;
184
185 if (copy_to_user(buf, lbuf, err))
5e631a03 186 return -EFAULT;
e126ba97
EC		 187
188 *pos += err;
189
190 return err;
191}
192
193static const struct file_operations size_fops = {
194 .owner = THIS_MODULE,
195 .open = simple_open,
196 .write = size_write,
197 .read = size_read,
198};
199
200static ssize_t limit_write(struct file *filp, const char __user *buf,
201 size_t count, loff_t *pos)
202{
203 struct mlx5_cache_ent *ent = filp->private_data;
204 struct mlx5_ib_dev *dev = ent->dev;
205 char lbuf[20];
206 u32 var;
207 int err;
208 int c;
209
210 if (copy_from_user(lbuf, buf, sizeof(lbuf)))
5e631a03 211 return -EFAULT;
e126ba97
EC		 212
213 c = order2idx(dev, ent->order);
214 lbuf[sizeof(lbuf) - 1] = 0;
215
216 if (sscanf(lbuf, "%u", &var) != 1)
217 return -EINVAL;
218
219 if (var > ent->size)
220 return -EINVAL;
221
222 ent->limit = var;
223
224 if (ent->cur < ent->limit) {
225 err = add_keys(dev, c, 2 * ent->limit - ent->cur);
226 if (err)
227 return err;
228 }
229
230 return count;
231}
232
233static ssize_t limit_read(struct file *filp, char __user *buf, size_t count,
234 loff_t *pos)
235{
236 struct mlx5_cache_ent *ent = filp->private_data;
237 char lbuf[20];
238 int err;
239
240 if (*pos)
241 return 0;
242
243 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit);
244 if (err < 0)
245 return err;
246
247 if (copy_to_user(buf, lbuf, err))
5e631a03 248 return -EFAULT;
e126ba97
EC		 249
250 *pos += err;
251
252 return err;
253}
254
255static const struct file_operations limit_fops = {
256 .owner = THIS_MODULE,
257 .open = simple_open,
258 .write = limit_write,
259 .read = limit_read,
260};
261
262static int someone_adding(struct mlx5_mr_cache *cache)
263{
264 int i;
265
266 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
267 if (cache->ent[i].cur < cache->ent[i].limit)
268 return 1;
269 }
270
271 return 0;
272}
273
274static void __cache_work_func(struct mlx5_cache_ent *ent)
275{
276 struct mlx5_ib_dev *dev = ent->dev;
277 struct mlx5_mr_cache *cache = &dev->cache;
278 int i = order2idx(dev, ent->order);
279
280 if (cache->stopped)
281 return;
282
283 ent = &dev->cache.ent[i];
284 if (ent->cur < 2 * ent->limit) {
285 add_keys(dev, i, 1);
286 if (ent->cur < 2 * ent->limit)
287 queue_work(cache->wq, &ent->work);
288 } else if (ent->cur > 2 * ent->limit) {
289 if (!someone_adding(cache) &&
290 time_after(jiffies, cache->last_add + 60 * HZ)) {
291 remove_keys(dev, i, 1);
292 if (ent->cur > ent->limit)
293 queue_work(cache->wq, &ent->work);
294 } else {
295 queue_delayed_work(cache->wq, &ent->dwork, 60 * HZ);
296 }
297 }
298}
299
300static void delayed_cache_work_func(struct work_struct *work)
301{
302 struct mlx5_cache_ent *ent;
303
304 ent = container_of(work, struct mlx5_cache_ent, dwork.work);
305 __cache_work_func(ent);
306}
307
308static void cache_work_func(struct work_struct *work)
309{
310 struct mlx5_cache_ent *ent;
311
312 ent = container_of(work, struct mlx5_cache_ent, work);
313 __cache_work_func(ent);
314}
315
316static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
317{
318 struct mlx5_mr_cache *cache = &dev->cache;
319 struct mlx5_ib_mr *mr = NULL;
320 struct mlx5_cache_ent *ent;
321 int c;
322 int i;
323
324 c = order2idx(dev, order);
325 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) {
326 mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c);
327 return NULL;
328 }
329
330 for (i = c; i < MAX_MR_CACHE_ENTRIES; i++) {
331 ent = &cache->ent[i];
332
333 mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i);
334
335 spin_lock(&ent->lock);
336 if (!list_empty(&ent->head)) {
337 mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
338 list);
339 list_del(&mr->list);
340 ent->cur--;
341 spin_unlock(&ent->lock);
342 if (ent->cur < ent->limit)
343 queue_work(cache->wq, &ent->work);
344 break;
345 }
346 spin_unlock(&ent->lock);
347
348 queue_work(cache->wq, &ent->work);
349
350 if (mr)
351 break;
352 }
353
354 if (!mr)
355 cache->ent[c].miss++;
356
357 return mr;
358}
359
360static void free_cached_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
361{
362 struct mlx5_mr_cache *cache = &dev->cache;
363 struct mlx5_cache_ent *ent;
364 int shrink = 0;
365 int c;
366
367 c = order2idx(dev, mr->order);
368 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) {
369 mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c);
370 return;
371 }
372 ent = &cache->ent[c];
373 spin_lock(&ent->lock);
374 list_add_tail(&mr->list, &ent->head);
375 ent->cur++;
376 if (ent->cur > 2 * ent->limit)
377 shrink = 1;
378 spin_unlock(&ent->lock);
379
380 if (shrink)
381 queue_work(cache->wq, &ent->work);
382}
383
384static void clean_keys(struct mlx5_ib_dev *dev, int c)
385{
e126ba97
EC		 386 struct mlx5_mr_cache *cache = &dev->cache;
387 struct mlx5_cache_ent *ent = &cache->ent[c];
388 struct mlx5_ib_mr *mr;
e126ba97
EC		 389 int err;
390
3c461911 391 cancel_delayed_work(&ent->dwork);
e126ba97
EC		 392 while (1) {
393 spin_lock(&ent->lock);
394 if (list_empty(&ent->head)) {
395 spin_unlock(&ent->lock);
396 return;
397 }
398 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
399 list_del(&mr->list);
400 ent->cur--;
401 ent->size--;
402 spin_unlock(&ent->lock);
403 err = mlx5_core_destroy_mkey(&dev->mdev, &mr->mmr);
203099fd 404 if (err)
e126ba97 405 mlx5_ib_warn(dev, "failed destroy mkey\n");
203099fd 406 else
e126ba97 407 kfree(mr);
e126ba97
EC		 408 }
409}
410
411static int mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
412{
413 struct mlx5_mr_cache *cache = &dev->cache;
414 struct mlx5_cache_ent *ent;
415 int i;
416
417 if (!mlx5_debugfs_root)
418 return 0;
419
420 cache->root = debugfs_create_dir("mr_cache", dev->mdev.priv.dbg_root);
421 if (!cache->root)
422 return -ENOMEM;
423
424 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
425 ent = &cache->ent[i];
426 sprintf(ent->name, "%d", ent->order);
427 ent->dir = debugfs_create_dir(ent->name, cache->root);
428 if (!ent->dir)
429 return -ENOMEM;
430
431 ent->fsize = debugfs_create_file("size", 0600, ent->dir, ent,
432 &size_fops);
433 if (!ent->fsize)
434 return -ENOMEM;
435
436 ent->flimit = debugfs_create_file("limit", 0600, ent->dir, ent,
437 &limit_fops);
438 if (!ent->flimit)
439 return -ENOMEM;
440
441 ent->fcur = debugfs_create_u32("cur", 0400, ent->dir,
442 &ent->cur);
443 if (!ent->fcur)
444 return -ENOMEM;
445
446 ent->fmiss = debugfs_create_u32("miss", 0600, ent->dir,
447 &ent->miss);
448 if (!ent->fmiss)
449 return -ENOMEM;
450 }
451
452 return 0;
453}
454
455static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
456{
457 if (!mlx5_debugfs_root)
458 return;
459
460 debugfs_remove_recursive(dev->cache.root);
461}
462
463int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
464{
465 struct mlx5_mr_cache *cache = &dev->cache;
466 struct mlx5_cache_ent *ent;
467 int limit;
468 int size;
469 int err;
470 int i;
471
472 cache->wq = create_singlethread_workqueue("mkey_cache");
473 if (!cache->wq) {
474 mlx5_ib_warn(dev, "failed to create work queue\n");
475 return -ENOMEM;
476 }
477
478 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
479 INIT_LIST_HEAD(&cache->ent[i].head);
480 spin_lock_init(&cache->ent[i].lock);
481
482 ent = &cache->ent[i];
483 INIT_LIST_HEAD(&ent->head);
484 spin_lock_init(&ent->lock);
485 ent->order = i + 2;
486 ent->dev = dev;
487
488 if (dev->mdev.profile->mask & MLX5_PROF_MASK_MR_CACHE) {
489 size = dev->mdev.profile->mr_cache[i].size;
490 limit = dev->mdev.profile->mr_cache[i].limit;
491 } else {
492 size = DEF_CACHE_SIZE;
493 limit = 0;
494 }
495 INIT_WORK(&ent->work, cache_work_func);
496 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
497 ent->limit = limit;
498 queue_work(cache->wq, &ent->work);
499 }
500
501 err = mlx5_mr_cache_debugfs_init(dev);
502 if (err)
503 mlx5_ib_warn(dev, "cache debugfs failure\n");
504
505 return 0;
506}
507
508int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
509{
510 int i;
511
512 dev->cache.stopped = 1;
3c461911 513 flush_workqueue(dev->cache.wq);
e126ba97
EC		 514
515 mlx5_mr_cache_debugfs_cleanup(dev);
516
517 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++)
518 clean_keys(dev, i);
519
3c461911
ML		 520 destroy_workqueue(dev->cache.wq);
521
e126ba97
EC		 522 return 0;
523}
524
525struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
526{
527 struct mlx5_ib_dev *dev = to_mdev(pd->device);
528 struct mlx5_core_dev *mdev = &dev->mdev;
529 struct mlx5_create_mkey_mbox_in *in;
530 struct mlx5_mkey_seg *seg;
531 struct mlx5_ib_mr *mr;
532 int err;
533
534 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
535 if (!mr)
536 return ERR_PTR(-ENOMEM);
537
538 in = kzalloc(sizeof(*in), GFP_KERNEL);
539 if (!in) {
540 err = -ENOMEM;
541 goto err_free;
542 }
543
544 seg = &in->seg;
545 seg->flags = convert_access(acc) | MLX5_ACCESS_MODE_PA;
546 seg->flags_pd = cpu_to_be32(to_mpd(pd)->pdn | MLX5_MKEY_LEN64);
547 seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
548 seg->start_addr = 0;
549
550 err = mlx5_core_create_mkey(mdev, &mr->mmr, in, sizeof(*in));
551 if (err)
552 goto err_in;
553
554 kfree(in);
555 mr->ibmr.lkey = mr->mmr.key;
556 mr->ibmr.rkey = mr->mmr.key;
557 mr->umem = NULL;
558
559 return &mr->ibmr;
560
561err_in:
562 kfree(in);
563
564err_free:
565 kfree(mr);
566
567 return ERR_PTR(err);
568}
569
570static int get_octo_len(u64 addr, u64 len, int page_size)
571{
572 u64 offset;
573 int npages;
574
575 offset = addr & (page_size - 1);
576 npages = ALIGN(len + offset, page_size) >> ilog2(page_size);
577 return (npages + 1) / 2;
578}
579
580static int use_umr(int order)
581{
582 return order <= 17;
583}
584
585static void prep_umr_reg_wqe(struct ib_pd *pd, struct ib_send_wr *wr,
586 struct ib_sge *sg, u64 dma, int n, u32 key,
587 int page_shift, u64 virt_addr, u64 len,
588 int access_flags)
589{
590 struct mlx5_ib_dev *dev = to_mdev(pd->device);
591 struct ib_mr *mr = dev->umrc.mr;
592
593 sg->addr = dma;
594 sg->length = ALIGN(sizeof(u64) * n, 64);
595 sg->lkey = mr->lkey;
596
597 wr->next = NULL;
598 wr->send_flags = 0;
599 wr->sg_list = sg;
600 if (n)
601 wr->num_sge = 1;
602 else
603 wr->num_sge = 0;
604
605 wr->opcode = MLX5_IB_WR_UMR;
606 wr->wr.fast_reg.page_list_len = n;
607 wr->wr.fast_reg.page_shift = page_shift;
608 wr->wr.fast_reg.rkey = key;
609 wr->wr.fast_reg.iova_start = virt_addr;
610 wr->wr.fast_reg.length = len;
611 wr->wr.fast_reg.access_flags = access_flags;
612 wr->wr.fast_reg.page_list = (struct ib_fast_reg_page_list *)pd;
613}
614
615static void prep_umr_unreg_wqe(struct mlx5_ib_dev *dev,
616 struct ib_send_wr *wr, u32 key)
617{
618 wr->send_flags = MLX5_IB_SEND_UMR_UNREG;
619 wr->opcode = MLX5_IB_WR_UMR;
620 wr->wr.fast_reg.rkey = key;
621}
622
623void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context)
624{
625 struct mlx5_ib_mr *mr;
626 struct ib_wc wc;
627 int err;
628
629 while (1) {
630 err = ib_poll_cq(cq, 1, &wc);
631 if (err < 0) {
632 pr_warn("poll cq error %d\n", err);
633 return;
634 }
635 if (err == 0)
636 break;
637
638 mr = (struct mlx5_ib_mr *)(unsigned long)wc.wr_id;
639 mr->status = wc.status;
640 complete(&mr->done);
641 }
642 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
643}
644
645static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
646 u64 virt_addr, u64 len, int npages,
647 int page_shift, int order, int access_flags)
648{
649 struct mlx5_ib_dev *dev = to_mdev(pd->device);
203099fd 650 struct device *ddev = dev->ib_dev.dma_device;
e126ba97
EC		 651 struct umr_common *umrc = &dev->umrc;
652 struct ib_send_wr wr, *bad;
653 struct mlx5_ib_mr *mr;
654 struct ib_sge sg;
203099fd 655 int size = sizeof(u64) * npages;
e126ba97
EC		 656 int err;
657 int i;
658
659 for (i = 0; i < 10; i++) {
660 mr = alloc_cached_mr(dev, order);
661 if (mr)
662 break;
663
664 err = add_keys(dev, order2idx(dev, order), 1);
665 if (err) {
666 mlx5_ib_warn(dev, "add_keys failed\n");
667 break;
668 }
669 }
670
671 if (!mr)
672 return ERR_PTR(-EAGAIN);
673
203099fd
EC		 674 mr->pas = kmalloc(size + 0x3f, GFP_KERNEL);
675 if (!mr->pas) {
676 err = -ENOMEM;
677 goto error;
678 }
679 mr->dma = dma_map_single(ddev, mr_align(mr->pas, 0x40), size,
680 DMA_TO_DEVICE);
681 if (dma_mapping_error(ddev, mr->dma)) {
682 kfree(mr->pas);
683 err = -ENOMEM;
684 goto error;
685 }
686
e126ba97
EC		 687 mlx5_ib_populate_pas(dev, umem, page_shift, mr_align(mr->pas, 0x40), 1);
688
689 memset(&wr, 0, sizeof(wr));
690 wr.wr_id = (u64)(unsigned long)mr;
691 prep_umr_reg_wqe(pd, &wr, &sg, mr->dma, npages, mr->mmr.key, page_shift, virt_addr, len, access_flags);
692
693 /* We serialize polls so one process does not kidnap another's
694 * completion. This is not a problem since wr is completed in
695 * around 1 usec
696 */
697 down(&umrc->sem);
698 init_completion(&mr->done);
699 err = ib_post_send(umrc->qp, &wr, &bad);
700 if (err) {
701 mlx5_ib_warn(dev, "post send failed, err %d\n", err);
702 up(&umrc->sem);
703 goto error;
704 }
705 wait_for_completion(&mr->done);
706 up(&umrc->sem);
707
203099fd
EC		 708 dma_unmap_single(ddev, mr->dma, size, DMA_TO_DEVICE);
709 kfree(mr->pas);
710
e126ba97
EC		 711 if (mr->status != IB_WC_SUCCESS) {
712 mlx5_ib_warn(dev, "reg umr failed\n");
713 err = -EFAULT;
714 goto error;
715 }
716
717 return mr;
718
719error:
720 free_cached_mr(dev, mr);
721 return ERR_PTR(err);
722}
723
724static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, u64 virt_addr,
725 u64 length, struct ib_umem *umem,
726 int npages, int page_shift,
727 int access_flags)
728{
729 struct mlx5_ib_dev *dev = to_mdev(pd->device);
730 struct mlx5_create_mkey_mbox_in *in;
731 struct mlx5_ib_mr *mr;
732 int inlen;
733 int err;
734
735 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
736 if (!mr)
737 return ERR_PTR(-ENOMEM);
738
739 inlen = sizeof(*in) + sizeof(*in->pas) * ((npages + 1) / 2) * 2;
740 in = mlx5_vzalloc(inlen);
741 if (!in) {
742 err = -ENOMEM;
743 goto err_1;
744 }
745 mlx5_ib_populate_pas(dev, umem, page_shift, in->pas, 0);
746
747 in->seg.flags = convert_access(access_flags) |
748 MLX5_ACCESS_MODE_MTT;
749 in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
750 in->seg.start_addr = cpu_to_be64(virt_addr);
751 in->seg.len = cpu_to_be64(length);
752 in->seg.bsfs_octo_size = 0;
753 in->seg.xlt_oct_size = cpu_to_be32(get_octo_len(virt_addr, length, 1 << page_shift));
754 in->seg.log2_page_size = page_shift;
755 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
756 in->xlat_oct_act_size = cpu_to_be32(get_octo_len(virt_addr, length, 1 << page_shift));
757 err = mlx5_core_create_mkey(&dev->mdev, &mr->mmr, in, inlen);
758 if (err) {
759 mlx5_ib_warn(dev, "create mkey failed\n");
760 goto err_2;
761 }
762 mr->umem = umem;
763 mlx5_vfree(in);
764
765 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmr.key);
766
767 return mr;
768
769err_2:
770 mlx5_vfree(in);
771
772err_1:
773 kfree(mr);
774
775 return ERR_PTR(err);
776}
777
778struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
779 u64 virt_addr, int access_flags,
780 struct ib_udata *udata)
781{
782 struct mlx5_ib_dev *dev = to_mdev(pd->device);
783 struct mlx5_ib_mr *mr = NULL;
784 struct ib_umem *umem;
785 int page_shift;
786 int npages;
787 int ncont;
788 int order;
789 int err;
790
791 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx\n",
792 start, virt_addr, length);
793 umem = ib_umem_get(pd->uobject->context, start, length, access_flags,
794 0);
795 if (IS_ERR(umem)) {
796 mlx5_ib_dbg(dev, "umem get failed\n");
797 return (void *)umem;
798 }
799
800 mlx5_ib_cont_pages(umem, start, &npages, &page_shift, &ncont, &order);
801 if (!npages) {
802 mlx5_ib_warn(dev, "avoid zero region\n");
803 err = -EINVAL;
804 goto error;
805 }
806
807 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
808 npages, ncont, order, page_shift);
809
810 if (use_umr(order)) {
811 mr = reg_umr(pd, umem, virt_addr, length, ncont, page_shift,
812 order, access_flags);
813 if (PTR_ERR(mr) == -EAGAIN) {
814 mlx5_ib_dbg(dev, "cache empty for order %d", order);
815 mr = NULL;
816 }
817 }
818
819 if (!mr)
820 mr = reg_create(pd, virt_addr, length, umem, ncont, page_shift,
821 access_flags);
822
823 if (IS_ERR(mr)) {
824 err = PTR_ERR(mr);
825 goto error;
826 }
827
828 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmr.key);
829
830 mr->umem = umem;
831 mr->npages = npages;
832 spin_lock(&dev->mr_lock);
833 dev->mdev.priv.reg_pages += npages;
834 spin_unlock(&dev->mr_lock);
835 mr->ibmr.lkey = mr->mmr.key;
836 mr->ibmr.rkey = mr->mmr.key;
837
838 return &mr->ibmr;
839
840error:
841 ib_umem_release(umem);
842 return ERR_PTR(err);
843}
844
845static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
846{
847 struct umr_common *umrc = &dev->umrc;
848 struct ib_send_wr wr, *bad;
849 int err;
850
851 memset(&wr, 0, sizeof(wr));
852 wr.wr_id = (u64)(unsigned long)mr;
853 prep_umr_unreg_wqe(dev, &wr, mr->mmr.key);
854
855 down(&umrc->sem);
856 init_completion(&mr->done);
857 err = ib_post_send(umrc->qp, &wr, &bad);
858 if (err) {
859 up(&umrc->sem);
860 mlx5_ib_dbg(dev, "err %d\n", err);
861 goto error;
862 }
863 wait_for_completion(&mr->done);
864 up(&umrc->sem);
865 if (mr->status != IB_WC_SUCCESS) {
866 mlx5_ib_warn(dev, "unreg umr failed\n");
867 err = -EFAULT;
868 goto error;
869 }
870 return 0;
871
872error:
873 return err;
874}
875
876int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
877{
878 struct mlx5_ib_dev *dev = to_mdev(ibmr->device);
879 struct mlx5_ib_mr *mr = to_mmr(ibmr);
880 struct ib_umem *umem = mr->umem;
881 int npages = mr->npages;
882 int umred = mr->umred;
883 int err;
884
885 if (!umred) {
886 err = mlx5_core_destroy_mkey(&dev->mdev, &mr->mmr);
887 if (err) {
888 mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n",
889 mr->mmr.key, err);
890 return err;
891 }
892 } else {
893 err = unreg_umr(dev, mr);
894 if (err) {
895 mlx5_ib_warn(dev, "failed unregister\n");
896 return err;
897 }
898 free_cached_mr(dev, mr);
899 }
900
901 if (umem) {
902 ib_umem_release(umem);
903 spin_lock(&dev->mr_lock);
904 dev->mdev.priv.reg_pages -= npages;
905 spin_unlock(&dev->mr_lock);
906 }
907
908 if (!umred)
909 kfree(mr);
910
911 return 0;
912}
913
914struct ib_mr *mlx5_ib_alloc_fast_reg_mr(struct ib_pd *pd,
915 int max_page_list_len)
916{
917 struct mlx5_ib_dev *dev = to_mdev(pd->device);
918 struct mlx5_create_mkey_mbox_in *in;
919 struct mlx5_ib_mr *mr;
920 int err;
921
922 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
923 if (!mr)
924 return ERR_PTR(-ENOMEM);
925
926 in = kzalloc(sizeof(*in), GFP_KERNEL);
927 if (!in) {
928 err = -ENOMEM;
929 goto err_free;
930 }
931
932 in->seg.status = 1 << 6; /* free */
933 in->seg.xlt_oct_size = cpu_to_be32((max_page_list_len + 1) / 2);
934 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
935 in->seg.flags = MLX5_PERM_UMR_EN | MLX5_ACCESS_MODE_MTT;
936 in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
937 /*
938 * TBD not needed - issue 197292 */
939 in->seg.log2_page_size = PAGE_SHIFT;
940
941 err = mlx5_core_create_mkey(&dev->mdev, &mr->mmr, in, sizeof(*in));
942 kfree(in);
943 if (err)
944 goto err_free;
945
946 mr->ibmr.lkey = mr->mmr.key;
947 mr->ibmr.rkey = mr->mmr.key;
948 mr->umem = NULL;
949
950 return &mr->ibmr;
951
952err_free:
953 kfree(mr);
954 return ERR_PTR(err);
955}
956
957struct ib_fast_reg_page_list *mlx5_ib_alloc_fast_reg_page_list(struct ib_device *ibdev,
958 int page_list_len)
959{
960 struct mlx5_ib_fast_reg_page_list *mfrpl;
961 int size = page_list_len * sizeof(u64);
962
963 mfrpl = kmalloc(sizeof(*mfrpl), GFP_KERNEL);
964 if (!mfrpl)
965 return ERR_PTR(-ENOMEM);
966
967 mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL);
968 if (!mfrpl->ibfrpl.page_list)
969 goto err_free;
970
971 mfrpl->mapped_page_list = dma_alloc_coherent(ibdev->dma_device,
972 size, &mfrpl->map,
973 GFP_KERNEL);
974 if (!mfrpl->mapped_page_list)
975 goto err_free;
976
977 WARN_ON(mfrpl->map & 0x3f);
978
979 return &mfrpl->ibfrpl;
980
981err_free:
982 kfree(mfrpl->ibfrpl.page_list);
983 kfree(mfrpl);
984 return ERR_PTR(-ENOMEM);
985}
986
987void mlx5_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
988{
989 struct mlx5_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list);
990 struct mlx5_ib_dev *dev = to_mdev(page_list->device);
991 int size = page_list->max_page_list_len * sizeof(u64);
992
993 dma_free_coherent(&dev->mdev.pdev->dev, size, mfrpl->mapped_page_list,
994 mfrpl->map);
995 kfree(mfrpl->ibfrpl.page_list);
996 kfree(mfrpl);
997}
Linux kernel - Deliverables
This page took 0.07667 seconds and 5 git commands to generate.