projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
IB/srp: Introduce target->mr_pool_size
[deliverable/linux.git] / drivers / infiniband / ulp / srp / ib_srp.c
1 /*
2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34
35 #include <linux/module.h>
36 #include <linux/init.h>
37 #include <linux/slab.h>
38 #include <linux/err.h>
39 #include <linux/string.h>
40 #include <linux/parser.h>
41 #include <linux/random.h>
42 #include <linux/jiffies.h>
43 #include <rdma/ib_cache.h>
44
45 #include <linux/atomic.h>
46
47 #include <scsi/scsi.h>
48 #include <scsi/scsi_device.h>
49 #include <scsi/scsi_dbg.h>
50 #include <scsi/scsi_tcq.h>
51 #include <scsi/srp.h>
52 #include <scsi/scsi_transport_srp.h>
53
54 #include "ib_srp.h"
55
56 #define DRV_NAME "ib_srp"
57 #define PFX DRV_NAME ": "
58 #define DRV_VERSION "2.0"
59 #define DRV_RELDATE "July 26, 2015"
60
61 MODULE_AUTHOR("Roland Dreier");
62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
63 MODULE_LICENSE("Dual BSD/GPL");
64 MODULE_VERSION(DRV_VERSION);
65 MODULE_INFO(release_date, DRV_RELDATE);
66
67 static unsigned int srp_sg_tablesize;
68 static unsigned int cmd_sg_entries;
69 static unsigned int indirect_sg_entries;
70 static bool allow_ext_sg;
71 static bool prefer_fr = true;
72 static bool register_always = true;
73 static int topspin_workarounds = 1;
74
75 module_param(srp_sg_tablesize, uint, 0444);
76 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
77
78 module_param(cmd_sg_entries, uint, 0444);
79 MODULE_PARM_DESC(cmd_sg_entries,
80 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
81
82 module_param(indirect_sg_entries, uint, 0444);
83 MODULE_PARM_DESC(indirect_sg_entries,
84 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");
85
86 module_param(allow_ext_sg, bool, 0444);
87 MODULE_PARM_DESC(allow_ext_sg,
88 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
89
90 module_param(topspin_workarounds, int, 0444);
91 MODULE_PARM_DESC(topspin_workarounds,
92 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
93
94 module_param(prefer_fr, bool, 0444);
95 MODULE_PARM_DESC(prefer_fr,
96 "Whether to use fast registration if both FMR and fast registration are supported");
97
98 module_param(register_always, bool, 0444);
99 MODULE_PARM_DESC(register_always,
100 "Use memory registration even for contiguous memory regions");
101
102 static const struct kernel_param_ops srp_tmo_ops;
103
104 static int srp_reconnect_delay = 10;
105 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
106 S_IRUGO | S_IWUSR);
107 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
108
109 static int srp_fast_io_fail_tmo = 15;
110 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
111 S_IRUGO | S_IWUSR);
112 MODULE_PARM_DESC(fast_io_fail_tmo,
113 "Number of seconds between the observation of a transport"
114 " layer error and failing all I/O. \"off\" means that this"
115 " functionality is disabled.");
116
117 static int srp_dev_loss_tmo = 600;
118 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
119 S_IRUGO | S_IWUSR);
120 MODULE_PARM_DESC(dev_loss_tmo,
121 "Maximum number of seconds that the SRP transport should"
122 " insulate transport layer errors. After this time has been"
123 " exceeded the SCSI host is removed. Should be"
124 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
125 " if fast_io_fail_tmo has not been set. \"off\" means that"
126 " this functionality is disabled.");
127
128 static unsigned ch_count;
129 module_param(ch_count, uint, 0444);
130 MODULE_PARM_DESC(ch_count,
131 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
132
133 static void srp_add_one(struct ib_device *device);
134 static void srp_remove_one(struct ib_device *device, void *client_data);
135 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
136 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
137 const char *opname);
138 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
139
140 static struct scsi_transport_template *ib_srp_transport_template;
141 static struct workqueue_struct *srp_remove_wq;
142
143 static struct ib_client srp_client = {
144 .name = "srp",
145 .add = srp_add_one,
146 .remove = srp_remove_one
147 };
148
149 static struct ib_sa_client srp_sa_client;
150
151 static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
152 {
153 int tmo = *(int *)kp->arg;
154
155 if (tmo >= 0)
156 return sprintf(buffer, "%d", tmo);
157 else
158 return sprintf(buffer, "off");
159 }
160
161 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
162 {
163 int tmo, res;
164
165 res = srp_parse_tmo(&tmo, val);
166 if (res)
167 goto out;
168
169 if (kp->arg == &srp_reconnect_delay)
170 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
171 srp_dev_loss_tmo);
172 else if (kp->arg == &srp_fast_io_fail_tmo)
173 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
174 else
175 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
176 tmo);
177 if (res)
178 goto out;
179 *(int *)kp->arg = tmo;
180
181 out:
182 return res;
183 }
184
185 static const struct kernel_param_ops srp_tmo_ops = {
186 .get = srp_tmo_get,
187 .set = srp_tmo_set,
188 };
189
190 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
191 {
192 return (struct srp_target_port *) host->hostdata;
193 }
194
195 static const char *srp_target_info(struct Scsi_Host *host)
196 {
197 return host_to_target(host)->target_name;
198 }
199
200 static int srp_target_is_topspin(struct srp_target_port *target)
201 {
202 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
203 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d };
204
205 return topspin_workarounds &&
206 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
207 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
208 }
209
210 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
211 gfp_t gfp_mask,
212 enum dma_data_direction direction)
213 {
214 struct srp_iu *iu;
215
216 iu = kmalloc(sizeof *iu, gfp_mask);
217 if (!iu)
218 goto out;
219
220 iu->buf = kzalloc(size, gfp_mask);
221 if (!iu->buf)
222 goto out_free_iu;
223
224 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
225 direction);
226 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
227 goto out_free_buf;
228
229 iu->size = size;
230 iu->direction = direction;
231
232 return iu;
233
234 out_free_buf:
235 kfree(iu->buf);
236 out_free_iu:
237 kfree(iu);
238 out:
239 return NULL;
240 }
241
242 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
243 {
244 if (!iu)
245 return;
246
247 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
248 iu->direction);
249 kfree(iu->buf);
250 kfree(iu);
251 }
252
253 static void srp_qp_event(struct ib_event *event, void *context)
254 {
255 pr_debug("QP event %s (%d)\n",
256 ib_event_msg(event->event), event->event);
257 }
258
259 static int srp_init_qp(struct srp_target_port *target,
260 struct ib_qp *qp)
261 {
262 struct ib_qp_attr *attr;
263 int ret;
264
265 attr = kmalloc(sizeof *attr, GFP_KERNEL);
266 if (!attr)
267 return -ENOMEM;
268
269 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
270 target->srp_host->port,
271 be16_to_cpu(target->pkey),
272 &attr->pkey_index);
273 if (ret)
274 goto out;
275
276 attr->qp_state = IB_QPS_INIT;
277 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
278 IB_ACCESS_REMOTE_WRITE);
279 attr->port_num = target->srp_host->port;
280
281 ret = ib_modify_qp(qp, attr,
282 IB_QP_STATE |
283 IB_QP_PKEY_INDEX |
284 IB_QP_ACCESS_FLAGS |
285 IB_QP_PORT);
286
287 out:
288 kfree(attr);
289 return ret;
290 }
291
292 static int srp_new_cm_id(struct srp_rdma_ch *ch)
293 {
294 struct srp_target_port *target = ch->target;
295 struct ib_cm_id *new_cm_id;
296
297 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
298 srp_cm_handler, ch);
299 if (IS_ERR(new_cm_id))
300 return PTR_ERR(new_cm_id);
301
302 if (ch->cm_id)
303 ib_destroy_cm_id(ch->cm_id);
304 ch->cm_id = new_cm_id;
305 ch->path.sgid = target->sgid;
306 ch->path.dgid = target->orig_dgid;
307 ch->path.pkey = target->pkey;
308 ch->path.service_id = target->service_id;
309
310 return 0;
311 }
312
313 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
314 {
315 struct srp_device *dev = target->srp_host->srp_dev;
316 struct ib_fmr_pool_param fmr_param;
317
318 memset(&fmr_param, 0, sizeof(fmr_param));
319 fmr_param.pool_size = target->mr_pool_size;
320 fmr_param.dirty_watermark = fmr_param.pool_size / 4;
321 fmr_param.cache = 1;
322 fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
323 fmr_param.page_shift = ilog2(dev->mr_page_size);
324 fmr_param.access = (IB_ACCESS_LOCAL_WRITE |
325 IB_ACCESS_REMOTE_WRITE |
326 IB_ACCESS_REMOTE_READ);
327
328 return ib_create_fmr_pool(dev->pd, &fmr_param);
329 }
330
331 /**
332 * srp_destroy_fr_pool() - free the resources owned by a pool
333 * @pool: Fast registration pool to be destroyed.
334 */
335 static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
336 {
337 int i;
338 struct srp_fr_desc *d;
339
340 if (!pool)
341 return;
342
343 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
344 if (d->mr)
345 ib_dereg_mr(d->mr);
346 }
347 kfree(pool);
348 }
349
350 /**
351 * srp_create_fr_pool() - allocate and initialize a pool for fast registration
352 * @device: IB device to allocate fast registration descriptors for.
353 * @pd: Protection domain associated with the FR descriptors.
354 * @pool_size: Number of descriptors to allocate.
355 * @max_page_list_len: Maximum fast registration work request page list length.
356 */
357 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
358 struct ib_pd *pd, int pool_size,
359 int max_page_list_len)
360 {
361 struct srp_fr_pool *pool;
362 struct srp_fr_desc *d;
363 struct ib_mr *mr;
364 int i, ret = -EINVAL;
365
366 if (pool_size <= 0)
367 goto err;
368 ret = -ENOMEM;
369 pool = kzalloc(sizeof(struct srp_fr_pool) +
370 pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
371 if (!pool)
372 goto err;
373 pool->size = pool_size;
374 pool->max_page_list_len = max_page_list_len;
375 spin_lock_init(&pool->lock);
376 INIT_LIST_HEAD(&pool->free_list);
377
378 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
379 mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
380 max_page_list_len);
381 if (IS_ERR(mr)) {
382 ret = PTR_ERR(mr);
383 goto destroy_pool;
384 }
385 d->mr = mr;
386 list_add_tail(&d->entry, &pool->free_list);
387 }
388
389 out:
390 return pool;
391
392 destroy_pool:
393 srp_destroy_fr_pool(pool);
394
395 err:
396 pool = ERR_PTR(ret);
397 goto out;
398 }
399
400 /**
401 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
402 * @pool: Pool to obtain descriptor from.
403 */
404 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
405 {
406 struct srp_fr_desc *d = NULL;
407 unsigned long flags;
408
409 spin_lock_irqsave(&pool->lock, flags);
410 if (!list_empty(&pool->free_list)) {
411 d = list_first_entry(&pool->free_list, typeof(*d), entry);
412 list_del(&d->entry);
413 }
414 spin_unlock_irqrestore(&pool->lock, flags);
415
416 return d;
417 }
418
419 /**
420 * srp_fr_pool_put() - put an FR descriptor back in the free list
421 * @pool: Pool the descriptor was allocated from.
422 * @desc: Pointer to an array of fast registration descriptor pointers.
423 * @n: Number of descriptors to put back.
424 *
425 * Note: The caller must already have queued an invalidation request for
426 * desc->mr->rkey before calling this function.
427 */
428 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
429 int n)
430 {
431 unsigned long flags;
432 int i;
433
434 spin_lock_irqsave(&pool->lock, flags);
435 for (i = 0; i < n; i++)
436 list_add(&desc[i]->entry, &pool->free_list);
437 spin_unlock_irqrestore(&pool->lock, flags);
438 }
439
440 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
441 {
442 struct srp_device *dev = target->srp_host->srp_dev;
443
444 return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size,
445 dev->max_pages_per_mr);
446 }
447
448 /**
449 * srp_destroy_qp() - destroy an RDMA queue pair
450 * @ch: SRP RDMA channel.
451 *
452 * Drain the qp before destroying it. This avoids that the receive
453 * completion handler can access the queue pair while it is
454 * being destroyed.
455 */
456 static void srp_destroy_qp(struct srp_rdma_ch *ch)
457 {
458 ib_drain_rq(ch->qp);
459 ib_destroy_qp(ch->qp);
460 }
461
462 static int srp_create_ch_ib(struct srp_rdma_ch *ch)
463 {
464 struct srp_target_port *target = ch->target;
465 struct srp_device *dev = target->srp_host->srp_dev;
466 struct ib_qp_init_attr *init_attr;
467 struct ib_cq *recv_cq, *send_cq;
468 struct ib_qp *qp;
469 struct ib_fmr_pool *fmr_pool = NULL;
470 struct srp_fr_pool *fr_pool = NULL;
471 const int m = dev->use_fast_reg ? 3 : 1;
472 int ret;
473
474 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
475 if (!init_attr)
476 return -ENOMEM;
477
478 /* queue_size + 1 for ib_drain_rq() */
479 recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1,
480 ch->comp_vector, IB_POLL_SOFTIRQ);
481 if (IS_ERR(recv_cq)) {
482 ret = PTR_ERR(recv_cq);
483 goto err;
484 }
485
486 send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size,
487 ch->comp_vector, IB_POLL_DIRECT);
488 if (IS_ERR(send_cq)) {
489 ret = PTR_ERR(send_cq);
490 goto err_recv_cq;
491 }
492
493 init_attr->event_handler = srp_qp_event;
494 init_attr->cap.max_send_wr = m * target->queue_size;
495 init_attr->cap.max_recv_wr = target->queue_size + 1;
496 init_attr->cap.max_recv_sge = 1;
497 init_attr->cap.max_send_sge = 1;
498 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
499 init_attr->qp_type = IB_QPT_RC;
500 init_attr->send_cq = send_cq;
501 init_attr->recv_cq = recv_cq;
502
503 qp = ib_create_qp(dev->pd, init_attr);
504 if (IS_ERR(qp)) {
505 ret = PTR_ERR(qp);
506 goto err_send_cq;
507 }
508
509 ret = srp_init_qp(target, qp);
510 if (ret)
511 goto err_qp;
512
513 if (dev->use_fast_reg) {
514 fr_pool = srp_alloc_fr_pool(target);
515 if (IS_ERR(fr_pool)) {
516 ret = PTR_ERR(fr_pool);
517 shost_printk(KERN_WARNING, target->scsi_host, PFX
518 "FR pool allocation failed (%d)\n", ret);
519 goto err_qp;
520 }
521 } else if (dev->use_fmr) {
522 fmr_pool = srp_alloc_fmr_pool(target);
523 if (IS_ERR(fmr_pool)) {
524 ret = PTR_ERR(fmr_pool);
525 shost_printk(KERN_WARNING, target->scsi_host, PFX
526 "FMR pool allocation failed (%d)\n", ret);
527 goto err_qp;
528 }
529 }
530
531 if (ch->qp)
532 srp_destroy_qp(ch);
533 if (ch->recv_cq)
534 ib_free_cq(ch->recv_cq);
535 if (ch->send_cq)
536 ib_free_cq(ch->send_cq);
537
538 ch->qp = qp;
539 ch->recv_cq = recv_cq;
540 ch->send_cq = send_cq;
541
542 if (dev->use_fast_reg) {
543 if (ch->fr_pool)
544 srp_destroy_fr_pool(ch->fr_pool);
545 ch->fr_pool = fr_pool;
546 } else if (dev->use_fmr) {
547 if (ch->fmr_pool)
548 ib_destroy_fmr_pool(ch->fmr_pool);
549 ch->fmr_pool = fmr_pool;
550 }
551
552 kfree(init_attr);
553 return 0;
554
555 err_qp:
556 srp_destroy_qp(ch);
557
558 err_send_cq:
559 ib_free_cq(send_cq);
560
561 err_recv_cq:
562 ib_free_cq(recv_cq);
563
564 err:
565 kfree(init_attr);
566 return ret;
567 }
568
569 /*
570 * Note: this function may be called without srp_alloc_iu_bufs() having been
571 * invoked. Hence the ch->[rt]x_ring checks.
572 */
573 static void srp_free_ch_ib(struct srp_target_port *target,
574 struct srp_rdma_ch *ch)
575 {
576 struct srp_device *dev = target->srp_host->srp_dev;
577 int i;
578
579 if (!ch->target)
580 return;
581
582 if (ch->cm_id) {
583 ib_destroy_cm_id(ch->cm_id);
584 ch->cm_id = NULL;
585 }
586
587 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
588 if (!ch->qp)
589 return;
590
591 if (dev->use_fast_reg) {
592 if (ch->fr_pool)
593 srp_destroy_fr_pool(ch->fr_pool);
594 } else if (dev->use_fmr) {
595 if (ch->fmr_pool)
596 ib_destroy_fmr_pool(ch->fmr_pool);
597 }
598
599 srp_destroy_qp(ch);
600 ib_free_cq(ch->send_cq);
601 ib_free_cq(ch->recv_cq);
602
603 /*
604 * Avoid that the SCSI error handler tries to use this channel after
605 * it has been freed. The SCSI error handler can namely continue
606 * trying to perform recovery actions after scsi_remove_host()
607 * returned.
608 */
609 ch->target = NULL;
610
611 ch->qp = NULL;
612 ch->send_cq = ch->recv_cq = NULL;
613
614 if (ch->rx_ring) {
615 for (i = 0; i < target->queue_size; ++i)
616 srp_free_iu(target->srp_host, ch->rx_ring[i]);
617 kfree(ch->rx_ring);
618 ch->rx_ring = NULL;
619 }
620 if (ch->tx_ring) {
621 for (i = 0; i < target->queue_size; ++i)
622 srp_free_iu(target->srp_host, ch->tx_ring[i]);
623 kfree(ch->tx_ring);
624 ch->tx_ring = NULL;
625 }
626 }
627
628 static void srp_path_rec_completion(int status,
629 struct ib_sa_path_rec *pathrec,
630 void *ch_ptr)
631 {
632 struct srp_rdma_ch *ch = ch_ptr;
633 struct srp_target_port *target = ch->target;
634
635 ch->status = status;
636 if (status)
637 shost_printk(KERN_ERR, target->scsi_host,
638 PFX "Got failed path rec status %d\n", status);
639 else
640 ch->path = *pathrec;
641 complete(&ch->done);
642 }
643
644 static int srp_lookup_path(struct srp_rdma_ch *ch)
645 {
646 struct srp_target_port *target = ch->target;
647 int ret;
648
649 ch->path.numb_path = 1;
650
651 init_completion(&ch->done);
652
653 ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
654 target->srp_host->srp_dev->dev,
655 target->srp_host->port,
656 &ch->path,
657 IB_SA_PATH_REC_SERVICE_ID |
658 IB_SA_PATH_REC_DGID |
659 IB_SA_PATH_REC_SGID |
660 IB_SA_PATH_REC_NUMB_PATH |
661 IB_SA_PATH_REC_PKEY,
662 SRP_PATH_REC_TIMEOUT_MS,
663 GFP_KERNEL,
664 srp_path_rec_completion,
665 ch, &ch->path_query);
666 if (ch->path_query_id < 0)
667 return ch->path_query_id;
668
669 ret = wait_for_completion_interruptible(&ch->done);
670 if (ret < 0)
671 return ret;
672
673 if (ch->status < 0)
674 shost_printk(KERN_WARNING, target->scsi_host,
675 PFX "Path record query failed\n");
676
677 return ch->status;
678 }
679
680 static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
681 {
682 struct srp_target_port *target = ch->target;
683 struct {
684 struct ib_cm_req_param param;
685 struct srp_login_req priv;
686 } *req = NULL;
687 int status;
688
689 req = kzalloc(sizeof *req, GFP_KERNEL);
690 if (!req)
691 return -ENOMEM;
692
693 req->param.primary_path = &ch->path;
694 req->param.alternate_path = NULL;
695 req->param.service_id = target->service_id;
696 req->param.qp_num = ch->qp->qp_num;
697 req->param.qp_type = ch->qp->qp_type;
698 req->param.private_data = &req->priv;
699 req->param.private_data_len = sizeof req->priv;
700 req->param.flow_control = 1;
701
702 get_random_bytes(&req->param.starting_psn, 4);
703 req->param.starting_psn &= 0xffffff;
704
705 /*
706 * Pick some arbitrary defaults here; we could make these
707 * module parameters if anyone cared about setting them.
708 */
709 req->param.responder_resources = 4;
710 req->param.remote_cm_response_timeout = 20;
711 req->param.local_cm_response_timeout = 20;
712 req->param.retry_count = target->tl_retry_count;
713 req->param.rnr_retry_count = 7;
714 req->param.max_cm_retries = 15;
715
716 req->priv.opcode = SRP_LOGIN_REQ;
717 req->priv.tag = 0;
718 req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
719 req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
720 SRP_BUF_FORMAT_INDIRECT);
721 req->priv.req_flags = (multich ? SRP_MULTICHAN_MULTI :
722 SRP_MULTICHAN_SINGLE);
723 /*
724 * In the published SRP specification (draft rev. 16a), the
725 * port identifier format is 8 bytes of ID extension followed
726 * by 8 bytes of GUID. Older drafts put the two halves in the
727 * opposite order, so that the GUID comes first.
728 *
729 * Targets conforming to these obsolete drafts can be
730 * recognized by the I/O Class they report.
731 */
732 if (target->io_class == SRP_REV10_IB_IO_CLASS) {
733 memcpy(req->priv.initiator_port_id,
734 &target->sgid.global.interface_id, 8);
735 memcpy(req->priv.initiator_port_id + 8,
736 &target->initiator_ext, 8);
737 memcpy(req->priv.target_port_id, &target->ioc_guid, 8);
738 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
739 } else {
740 memcpy(req->priv.initiator_port_id,
741 &target->initiator_ext, 8);
742 memcpy(req->priv.initiator_port_id + 8,
743 &target->sgid.global.interface_id, 8);
744 memcpy(req->priv.target_port_id, &target->id_ext, 8);
745 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
746 }
747
748 /*
749 * Topspin/Cisco SRP targets will reject our login unless we
750 * zero out the first 8 bytes of our initiator port ID and set
751 * the second 8 bytes to the local node GUID.
752 */
753 if (srp_target_is_topspin(target)) {
754 shost_printk(KERN_DEBUG, target->scsi_host,
755 PFX "Topspin/Cisco initiator port ID workaround "
756 "activated for target GUID %016llx\n",
757 be64_to_cpu(target->ioc_guid));
758 memset(req->priv.initiator_port_id, 0, 8);
759 memcpy(req->priv.initiator_port_id + 8,
760 &target->srp_host->srp_dev->dev->node_guid, 8);
761 }
762
763 status = ib_send_cm_req(ch->cm_id, &req->param);
764
765 kfree(req);
766
767 return status;
768 }
769
770 static bool srp_queue_remove_work(struct srp_target_port *target)
771 {
772 bool changed = false;
773
774 spin_lock_irq(&target->lock);
775 if (target->state != SRP_TARGET_REMOVED) {
776 target->state = SRP_TARGET_REMOVED;
777 changed = true;
778 }
779 spin_unlock_irq(&target->lock);
780
781 if (changed)
782 queue_work(srp_remove_wq, &target->remove_work);
783
784 return changed;
785 }
786
787 static void srp_disconnect_target(struct srp_target_port *target)
788 {
789 struct srp_rdma_ch *ch;
790 int i;
791
792 /* XXX should send SRP_I_LOGOUT request */
793
794 for (i = 0; i < target->ch_count; i++) {
795 ch = &target->ch[i];
796 ch->connected = false;
797 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
798 shost_printk(KERN_DEBUG, target->scsi_host,
799 PFX "Sending CM DREQ failed\n");
800 }
801 }
802 }
803
804 static void srp_free_req_data(struct srp_target_port *target,
805 struct srp_rdma_ch *ch)
806 {
807 struct srp_device *dev = target->srp_host->srp_dev;
808 struct ib_device *ibdev = dev->dev;
809 struct srp_request *req;
810 int i;
811
812 if (!ch->req_ring)
813 return;
814
815 for (i = 0; i < target->req_ring_size; ++i) {
816 req = &ch->req_ring[i];
817 if (dev->use_fast_reg) {
818 kfree(req->fr_list);
819 } else {
820 kfree(req->fmr_list);
821 kfree(req->map_page);
822 }
823 if (req->indirect_dma_addr) {
824 ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
825 target->indirect_size,
826 DMA_TO_DEVICE);
827 }
828 kfree(req->indirect_desc);
829 }
830
831 kfree(ch->req_ring);
832 ch->req_ring = NULL;
833 }
834
835 static int srp_alloc_req_data(struct srp_rdma_ch *ch)
836 {
837 struct srp_target_port *target = ch->target;
838 struct srp_device *srp_dev = target->srp_host->srp_dev;
839 struct ib_device *ibdev = srp_dev->dev;
840 struct srp_request *req;
841 void *mr_list;
842 dma_addr_t dma_addr;
843 int i, ret = -ENOMEM;
844
845 ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
846 GFP_KERNEL);
847 if (!ch->req_ring)
848 goto out;
849
850 for (i = 0; i < target->req_ring_size; ++i) {
851 req = &ch->req_ring[i];
852 mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
853 GFP_KERNEL);
854 if (!mr_list)
855 goto out;
856 if (srp_dev->use_fast_reg) {
857 req->fr_list = mr_list;
858 } else {
859 req->fmr_list = mr_list;
860 req->map_page = kmalloc(srp_dev->max_pages_per_mr *
861 sizeof(void *), GFP_KERNEL);
862 if (!req->map_page)
863 goto out;
864 }
865 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
866 if (!req->indirect_desc)
867 goto out;
868
869 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
870 target->indirect_size,
871 DMA_TO_DEVICE);
872 if (ib_dma_mapping_error(ibdev, dma_addr))
873 goto out;
874
875 req->indirect_dma_addr = dma_addr;
876 }
877 ret = 0;
878
879 out:
880 return ret;
881 }
882
883 /**
884 * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
885 * @shost: SCSI host whose attributes to remove from sysfs.
886 *
887 * Note: Any attributes defined in the host template and that did not exist
888 * before invocation of this function will be ignored.
889 */
890 static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
891 {
892 struct device_attribute **attr;
893
894 for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
895 device_remove_file(&shost->shost_dev, *attr);
896 }
897
898 static void srp_remove_target(struct srp_target_port *target)
899 {
900 struct srp_rdma_ch *ch;
901 int i;
902
903 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
904
905 srp_del_scsi_host_attr(target->scsi_host);
906 srp_rport_get(target->rport);
907 srp_remove_host(target->scsi_host);
908 scsi_remove_host(target->scsi_host);
909 srp_stop_rport_timers(target->rport);
910 srp_disconnect_target(target);
911 for (i = 0; i < target->ch_count; i++) {
912 ch = &target->ch[i];
913 srp_free_ch_ib(target, ch);
914 }
915 cancel_work_sync(&target->tl_err_work);
916 srp_rport_put(target->rport);
917 for (i = 0; i < target->ch_count; i++) {
918 ch = &target->ch[i];
919 srp_free_req_data(target, ch);
920 }
921 kfree(target->ch);
922 target->ch = NULL;
923
924 spin_lock(&target->srp_host->target_lock);
925 list_del(&target->list);
926 spin_unlock(&target->srp_host->target_lock);
927
928 scsi_host_put(target->scsi_host);
929 }
930
931 static void srp_remove_work(struct work_struct *work)
932 {
933 struct srp_target_port *target =
934 container_of(work, struct srp_target_port, remove_work);
935
936 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
937
938 srp_remove_target(target);
939 }
940
941 static void srp_rport_delete(struct srp_rport *rport)
942 {
943 struct srp_target_port *target = rport->lld_data;
944
945 srp_queue_remove_work(target);
946 }
947
948 /**
949 * srp_connected_ch() - number of connected channels
950 * @target: SRP target port.
951 */
952 static int srp_connected_ch(struct srp_target_port *target)
953 {
954 int i, c = 0;
955
956 for (i = 0; i < target->ch_count; i++)
957 c += target->ch[i].connected;
958
959 return c;
960 }
961
962 static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
963 {
964 struct srp_target_port *target = ch->target;
965 int ret;
966
967 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
968
969 ret = srp_lookup_path(ch);
970 if (ret)
971 goto out;
972
973 while (1) {
974 init_completion(&ch->done);
975 ret = srp_send_req(ch, multich);
976 if (ret)
977 goto out;
978 ret = wait_for_completion_interruptible(&ch->done);
979 if (ret < 0)
980 goto out;
981
982 /*
983 * The CM event handling code will set status to
984 * SRP_PORT_REDIRECT if we get a port redirect REJ
985 * back, or SRP_DLID_REDIRECT if we get a lid/qp
986 * redirect REJ back.
987 */
988 ret = ch->status;
989 switch (ret) {
990 case 0:
991 ch->connected = true;
992 goto out;
993
994 case SRP_PORT_REDIRECT:
995 ret = srp_lookup_path(ch);
996 if (ret)
997 goto out;
998 break;
999
1000 case SRP_DLID_REDIRECT:
1001 break;
1002
1003 case SRP_STALE_CONN:
1004 shost_printk(KERN_ERR, target->scsi_host, PFX
1005 "giving up on stale connection\n");
1006 ret = -ECONNRESET;
1007 goto out;
1008
1009 default:
1010 goto out;
1011 }
1012 }
1013
1014 out:
1015 return ret <= 0 ? ret : -ENODEV;
1016 }
1017
1018 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc)
1019 {
1020 srp_handle_qp_err(cq, wc, "INV RKEY");
1021 }
1022
1023 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch,
1024 u32 rkey)
1025 {
1026 struct ib_send_wr *bad_wr;
1027 struct ib_send_wr wr = {
1028 .opcode = IB_WR_LOCAL_INV,
1029 .next = NULL,
1030 .num_sge = 0,
1031 .send_flags = 0,
1032 .ex.invalidate_rkey = rkey,
1033 };
1034
1035 wr.wr_cqe = &req->reg_cqe;
1036 req->reg_cqe.done = srp_inv_rkey_err_done;
1037 return ib_post_send(ch->qp, &wr, &bad_wr);
1038 }
1039
1040 static void srp_unmap_data(struct scsi_cmnd *scmnd,
1041 struct srp_rdma_ch *ch,
1042 struct srp_request *req)
1043 {
1044 struct srp_target_port *target = ch->target;
1045 struct srp_device *dev = target->srp_host->srp_dev;
1046 struct ib_device *ibdev = dev->dev;
1047 int i, res;
1048
1049 if (!scsi_sglist(scmnd) ||
1050 (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1051 scmnd->sc_data_direction != DMA_FROM_DEVICE))
1052 return;
1053
1054 if (dev->use_fast_reg) {
1055 struct srp_fr_desc **pfr;
1056
1057 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
1058 res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey);
1059 if (res < 0) {
1060 shost_printk(KERN_ERR, target->scsi_host, PFX
1061 "Queueing INV WR for rkey %#x failed (%d)\n",
1062 (*pfr)->mr->rkey, res);
1063 queue_work(system_long_wq,
1064 &target->tl_err_work);
1065 }
1066 }
1067 if (req->nmdesc)
1068 srp_fr_pool_put(ch->fr_pool, req->fr_list,
1069 req->nmdesc);
1070 } else if (dev->use_fmr) {
1071 struct ib_pool_fmr **pfmr;
1072
1073 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1074 ib_fmr_pool_unmap(*pfmr);
1075 }
1076
1077 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1078 scmnd->sc_data_direction);
1079 }
1080
1081 /**
1082 * srp_claim_req - Take ownership of the scmnd associated with a request.
1083 * @ch: SRP RDMA channel.
1084 * @req: SRP request.
1085 * @sdev: If not NULL, only take ownership for this SCSI device.
1086 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1087 * ownership of @req->scmnd if it equals @scmnd.
1088 *
1089 * Return value:
1090 * Either NULL or a pointer to the SCSI command the caller became owner of.
1091 */
1092 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
1093 struct srp_request *req,
1094 struct scsi_device *sdev,
1095 struct scsi_cmnd *scmnd)
1096 {
1097 unsigned long flags;
1098
1099 spin_lock_irqsave(&ch->lock, flags);
1100 if (req->scmnd &&
1101 (!sdev || req->scmnd->device == sdev) &&
1102 (!scmnd || req->scmnd == scmnd)) {
1103 scmnd = req->scmnd;
1104 req->scmnd = NULL;
1105 } else {
1106 scmnd = NULL;
1107 }
1108 spin_unlock_irqrestore(&ch->lock, flags);
1109
1110 return scmnd;
1111 }
1112
1113 /**
1114 * srp_free_req() - Unmap data and adjust ch->req_lim.
1115 * @ch: SRP RDMA channel.
1116 * @req: Request to be freed.
1117 * @scmnd: SCSI command associated with @req.
1118 * @req_lim_delta: Amount to be added to @target->req_lim.
1119 */
1120 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1121 struct scsi_cmnd *scmnd, s32 req_lim_delta)
1122 {
1123 unsigned long flags;
1124
1125 srp_unmap_data(scmnd, ch, req);
1126
1127 spin_lock_irqsave(&ch->lock, flags);
1128 ch->req_lim += req_lim_delta;
1129 spin_unlock_irqrestore(&ch->lock, flags);
1130 }
1131
1132 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1133 struct scsi_device *sdev, int result)
1134 {
1135 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
1136
1137 if (scmnd) {
1138 srp_free_req(ch, req, scmnd, 0);
1139 scmnd->result = result;
1140 scmnd->scsi_done(scmnd);
1141 }
1142 }
1143
1144 static void srp_terminate_io(struct srp_rport *rport)
1145 {
1146 struct srp_target_port *target = rport->lld_data;
1147 struct srp_rdma_ch *ch;
1148 struct Scsi_Host *shost = target->scsi_host;
1149 struct scsi_device *sdev;
1150 int i, j;
1151
1152 /*
1153 * Invoking srp_terminate_io() while srp_queuecommand() is running
1154 * is not safe. Hence the warning statement below.
1155 */
1156 shost_for_each_device(sdev, shost)
1157 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1158
1159 for (i = 0; i < target->ch_count; i++) {
1160 ch = &target->ch[i];
1161
1162 for (j = 0; j < target->req_ring_size; ++j) {
1163 struct srp_request *req = &ch->req_ring[j];
1164
1165 srp_finish_req(ch, req, NULL,
1166 DID_TRANSPORT_FAILFAST << 16);
1167 }
1168 }
1169 }
1170
1171 /*
1172 * It is up to the caller to ensure that srp_rport_reconnect() calls are
1173 * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1174 * srp_reset_device() or srp_reset_host() calls will occur while this function
1175 * is in progress. One way to realize that is not to call this function
1176 * directly but to call srp_reconnect_rport() instead since that last function
1177 * serializes calls of this function via rport->mutex and also blocks
1178 * srp_queuecommand() calls before invoking this function.
1179 */
1180 static int srp_rport_reconnect(struct srp_rport *rport)
1181 {
1182 struct srp_target_port *target = rport->lld_data;
1183 struct srp_rdma_ch *ch;
1184 int i, j, ret = 0;
1185 bool multich = false;
1186
1187 srp_disconnect_target(target);
1188
1189 if (target->state == SRP_TARGET_SCANNING)
1190 return -ENODEV;
1191
1192 /*
1193 * Now get a new local CM ID so that we avoid confusing the target in
1194 * case things are really fouled up. Doing so also ensures that all CM
1195 * callbacks will have finished before a new QP is allocated.
1196 */
1197 for (i = 0; i < target->ch_count; i++) {
1198 ch = &target->ch[i];
1199 ret += srp_new_cm_id(ch);
1200 }
1201 for (i = 0; i < target->ch_count; i++) {
1202 ch = &target->ch[i];
1203 for (j = 0; j < target->req_ring_size; ++j) {
1204 struct srp_request *req = &ch->req_ring[j];
1205
1206 srp_finish_req(ch, req, NULL, DID_RESET << 16);
1207 }
1208 }
1209 for (i = 0; i < target->ch_count; i++) {
1210 ch = &target->ch[i];
1211 /*
1212 * Whether or not creating a new CM ID succeeded, create a new
1213 * QP. This guarantees that all completion callback function
1214 * invocations have finished before request resetting starts.
1215 */
1216 ret += srp_create_ch_ib(ch);
1217
1218 INIT_LIST_HEAD(&ch->free_tx);
1219 for (j = 0; j < target->queue_size; ++j)
1220 list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1221 }
1222
1223 target->qp_in_error = false;
1224
1225 for (i = 0; i < target->ch_count; i++) {
1226 ch = &target->ch[i];
1227 if (ret)
1228 break;
1229 ret = srp_connect_ch(ch, multich);
1230 multich = true;
1231 }
1232
1233 if (ret == 0)
1234 shost_printk(KERN_INFO, target->scsi_host,
1235 PFX "reconnect succeeded\n");
1236
1237 return ret;
1238 }
1239
1240 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1241 unsigned int dma_len, u32 rkey)
1242 {
1243 struct srp_direct_buf *desc = state->desc;
1244
1245 WARN_ON_ONCE(!dma_len);
1246
1247 desc->va = cpu_to_be64(dma_addr);
1248 desc->key = cpu_to_be32(rkey);
1249 desc->len = cpu_to_be32(dma_len);
1250
1251 state->total_len += dma_len;
1252 state->desc++;
1253 state->ndesc++;
1254 }
1255
1256 static int srp_map_finish_fmr(struct srp_map_state *state,
1257 struct srp_rdma_ch *ch)
1258 {
1259 struct srp_target_port *target = ch->target;
1260 struct srp_device *dev = target->srp_host->srp_dev;
1261 struct ib_pool_fmr *fmr;
1262 u64 io_addr = 0;
1263
1264 if (state->fmr.next >= state->fmr.end)
1265 return -ENOMEM;
1266
1267 WARN_ON_ONCE(!dev->use_fmr);
1268
1269 if (state->npages == 0)
1270 return 0;
1271
1272 if (state->npages == 1 && target->global_mr) {
1273 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1274 target->global_mr->rkey);
1275 goto reset_state;
1276 }
1277
1278 fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
1279 state->npages, io_addr);
1280 if (IS_ERR(fmr))
1281 return PTR_ERR(fmr);
1282
1283 *state->fmr.next++ = fmr;
1284 state->nmdesc++;
1285
1286 srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
1287 state->dma_len, fmr->fmr->rkey);
1288
1289 reset_state:
1290 state->npages = 0;
1291 state->dma_len = 0;
1292
1293 return 0;
1294 }
1295
1296 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
1297 {
1298 srp_handle_qp_err(cq, wc, "FAST REG");
1299 }
1300
1301 static int srp_map_finish_fr(struct srp_map_state *state,
1302 struct srp_request *req,
1303 struct srp_rdma_ch *ch, int sg_nents)
1304 {
1305 struct srp_target_port *target = ch->target;
1306 struct srp_device *dev = target->srp_host->srp_dev;
1307 struct ib_send_wr *bad_wr;
1308 struct ib_reg_wr wr;
1309 struct srp_fr_desc *desc;
1310 u32 rkey;
1311 int n, err;
1312
1313 if (state->fr.next >= state->fr.end)
1314 return -ENOMEM;
1315
1316 WARN_ON_ONCE(!dev->use_fast_reg);
1317
1318 if (sg_nents == 0)
1319 return 0;
1320
1321 if (sg_nents == 1 && target->global_mr) {
1322 srp_map_desc(state, sg_dma_address(state->sg),
1323 sg_dma_len(state->sg),
1324 target->global_mr->rkey);
1325 return 1;
1326 }
1327
1328 desc = srp_fr_pool_get(ch->fr_pool);
1329 if (!desc)
1330 return -ENOMEM;
1331
1332 rkey = ib_inc_rkey(desc->mr->rkey);
1333 ib_update_fast_reg_key(desc->mr, rkey);
1334
1335 n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, dev->mr_page_size);
1336 if (unlikely(n < 0))
1337 return n;
1338
1339 req->reg_cqe.done = srp_reg_mr_err_done;
1340
1341 wr.wr.next = NULL;
1342 wr.wr.opcode = IB_WR_REG_MR;
1343 wr.wr.wr_cqe = &req->reg_cqe;
1344 wr.wr.num_sge = 0;
1345 wr.wr.send_flags = 0;
1346 wr.mr = desc->mr;
1347 wr.key = desc->mr->rkey;
1348 wr.access = (IB_ACCESS_LOCAL_WRITE |
1349 IB_ACCESS_REMOTE_READ |
1350 IB_ACCESS_REMOTE_WRITE);
1351
1352 *state->fr.next++ = desc;
1353 state->nmdesc++;
1354
1355 srp_map_desc(state, desc->mr->iova,
1356 desc->mr->length, desc->mr->rkey);
1357
1358 err = ib_post_send(ch->qp, &wr.wr, &bad_wr);
1359 if (unlikely(err))
1360 return err;
1361
1362 return n;
1363 }
1364
1365 static int srp_map_sg_entry(struct srp_map_state *state,
1366 struct srp_rdma_ch *ch,
1367 struct scatterlist *sg, int sg_index)
1368 {
1369 struct srp_target_port *target = ch->target;
1370 struct srp_device *dev = target->srp_host->srp_dev;
1371 struct ib_device *ibdev = dev->dev;
1372 dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1373 unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1374 unsigned int len = 0;
1375 int ret;
1376
1377 WARN_ON_ONCE(!dma_len);
1378
1379 while (dma_len) {
1380 unsigned offset = dma_addr & ~dev->mr_page_mask;
1381 if (state->npages == dev->max_pages_per_mr || offset != 0) {
1382 ret = srp_map_finish_fmr(state, ch);
1383 if (ret)
1384 return ret;
1385 }
1386
1387 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1388
1389 if (!state->npages)
1390 state->base_dma_addr = dma_addr;
1391 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1392 state->dma_len += len;
1393 dma_addr += len;
1394 dma_len -= len;
1395 }
1396
1397 /*
1398 * If the last entry of the MR wasn't a full page, then we need to
1399 * close it out and start a new one -- we can only merge at page
1400 * boundaries.
1401 */
1402 ret = 0;
1403 if (len != dev->mr_page_size)
1404 ret = srp_map_finish_fmr(state, ch);
1405 return ret;
1406 }
1407
1408 static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1409 struct srp_request *req, struct scatterlist *scat,
1410 int count)
1411 {
1412 struct scatterlist *sg;
1413 int i, ret;
1414
1415 state->desc = req->indirect_desc;
1416 state->pages = req->map_page;
1417 state->fmr.next = req->fmr_list;
1418 state->fmr.end = req->fmr_list + ch->target->cmd_sg_cnt;
1419
1420 for_each_sg(scat, sg, count, i) {
1421 ret = srp_map_sg_entry(state, ch, sg, i);
1422 if (ret)
1423 return ret;
1424 }
1425
1426 ret = srp_map_finish_fmr(state, ch);
1427 if (ret)
1428 return ret;
1429
1430 return 0;
1431 }
1432
1433 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1434 struct srp_request *req, struct scatterlist *scat,
1435 int count)
1436 {
1437 state->desc = req->indirect_desc;
1438 state->fr.next = req->fr_list;
1439 state->fr.end = req->fr_list + ch->target->cmd_sg_cnt;
1440 state->sg = scat;
1441
1442 while (count) {
1443 int i, n;
1444
1445 n = srp_map_finish_fr(state, req, ch, count);
1446 if (unlikely(n < 0))
1447 return n;
1448
1449 count -= n;
1450 for (i = 0; i < n; i++)
1451 state->sg = sg_next(state->sg);
1452 }
1453
1454 return 0;
1455 }
1456
1457 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
1458 struct srp_request *req, struct scatterlist *scat,
1459 int count)
1460 {
1461 struct srp_target_port *target = ch->target;
1462 struct srp_device *dev = target->srp_host->srp_dev;
1463 struct scatterlist *sg;
1464 int i;
1465
1466 state->desc = req->indirect_desc;
1467 for_each_sg(scat, sg, count, i) {
1468 srp_map_desc(state, ib_sg_dma_address(dev->dev, sg),
1469 ib_sg_dma_len(dev->dev, sg),
1470 target->global_mr->rkey);
1471 }
1472
1473 return 0;
1474 }
1475
1476 /*
1477 * Register the indirect data buffer descriptor with the HCA.
1478 *
1479 * Note: since the indirect data buffer descriptor has been allocated with
1480 * kmalloc() it is guaranteed that this buffer is a physically contiguous
1481 * memory buffer.
1482 */
1483 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
1484 void **next_mr, void **end_mr, u32 idb_len,
1485 __be32 *idb_rkey)
1486 {
1487 struct srp_target_port *target = ch->target;
1488 struct srp_device *dev = target->srp_host->srp_dev;
1489 struct srp_map_state state;
1490 struct srp_direct_buf idb_desc;
1491 u64 idb_pages[1];
1492 struct scatterlist idb_sg[1];
1493 int ret;
1494
1495 memset(&state, 0, sizeof(state));
1496 memset(&idb_desc, 0, sizeof(idb_desc));
1497 state.gen.next = next_mr;
1498 state.gen.end = end_mr;
1499 state.desc = &idb_desc;
1500 state.base_dma_addr = req->indirect_dma_addr;
1501 state.dma_len = idb_len;
1502
1503 if (dev->use_fast_reg) {
1504 state.sg = idb_sg;
1505 sg_set_buf(idb_sg, req->indirect_desc, idb_len);
1506 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */
1507 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1508 idb_sg->dma_length = idb_sg->length; /* hack^2 */
1509 #endif
1510 ret = srp_map_finish_fr(&state, req, ch, 1);
1511 if (ret < 0)
1512 return ret;
1513 } else if (dev->use_fmr) {
1514 state.pages = idb_pages;
1515 state.pages[0] = (req->indirect_dma_addr &
1516 dev->mr_page_mask);
1517 state.npages = 1;
1518 ret = srp_map_finish_fmr(&state, ch);
1519 if (ret < 0)
1520 return ret;
1521 } else {
1522 return -EINVAL;
1523 }
1524
1525 *idb_rkey = idb_desc.key;
1526
1527 return 0;
1528 }
1529
1530 /**
1531 * srp_map_data() - map SCSI data buffer onto an SRP request
1532 * @scmnd: SCSI command to map
1533 * @ch: SRP RDMA channel
1534 * @req: SRP request
1535 *
1536 * Returns the length in bytes of the SRP_CMD IU or a negative value if
1537 * mapping failed.
1538 */
1539 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
1540 struct srp_request *req)
1541 {
1542 struct srp_target_port *target = ch->target;
1543 struct scatterlist *scat;
1544 struct srp_cmd *cmd = req->cmd->buf;
1545 int len, nents, count, ret;
1546 struct srp_device *dev;
1547 struct ib_device *ibdev;
1548 struct srp_map_state state;
1549 struct srp_indirect_buf *indirect_hdr;
1550 u32 idb_len, table_len;
1551 __be32 idb_rkey;
1552 u8 fmt;
1553
1554 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1555 return sizeof (struct srp_cmd);
1556
1557 if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1558 scmnd->sc_data_direction != DMA_TO_DEVICE) {
1559 shost_printk(KERN_WARNING, target->scsi_host,
1560 PFX "Unhandled data direction %d\n",
1561 scmnd->sc_data_direction);
1562 return -EINVAL;
1563 }
1564
1565 nents = scsi_sg_count(scmnd);
1566 scat = scsi_sglist(scmnd);
1567
1568 dev = target->srp_host->srp_dev;
1569 ibdev = dev->dev;
1570
1571 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1572 if (unlikely(count == 0))
1573 return -EIO;
1574
1575 fmt = SRP_DATA_DESC_DIRECT;
1576 len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
1577
1578 if (count == 1 && target->global_mr) {
1579 /*
1580 * The midlayer only generated a single gather/scatter
1581 * entry, or DMA mapping coalesced everything to a
1582 * single entry. So a direct descriptor along with
1583 * the DMA MR suffices.
1584 */
1585 struct srp_direct_buf *buf = (void *) cmd->add_data;
1586
1587 buf->va = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
1588 buf->key = cpu_to_be32(target->global_mr->rkey);
1589 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1590
1591 req->nmdesc = 0;
1592 goto map_complete;
1593 }
1594
1595 /*
1596 * We have more than one scatter/gather entry, so build our indirect
1597 * descriptor table, trying to merge as many entries as we can.
1598 */
1599 indirect_hdr = (void *) cmd->add_data;
1600
1601 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1602 target->indirect_size, DMA_TO_DEVICE);
1603
1604 memset(&state, 0, sizeof(state));
1605 if (dev->use_fast_reg)
1606 ret = srp_map_sg_fr(&state, ch, req, scat, count);
1607 else if (dev->use_fmr)
1608 ret = srp_map_sg_fmr(&state, ch, req, scat, count);
1609 else
1610 ret = srp_map_sg_dma(&state, ch, req, scat, count);
1611 req->nmdesc = state.nmdesc;
1612 if (ret < 0)
1613 goto unmap;
1614
1615 /* We've mapped the request, now pull as much of the indirect
1616 * descriptor table as we can into the command buffer. If this
1617 * target is not using an external indirect table, we are
1618 * guaranteed to fit into the command, as the SCSI layer won't
1619 * give us more S/G entries than we allow.
1620 */
1621 if (state.ndesc == 1) {
1622 /*
1623 * Memory registration collapsed the sg-list into one entry,
1624 * so use a direct descriptor.
1625 */
1626 struct srp_direct_buf *buf = (void *) cmd->add_data;
1627
1628 *buf = req->indirect_desc[0];
1629 goto map_complete;
1630 }
1631
1632 if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1633 !target->allow_ext_sg)) {
1634 shost_printk(KERN_ERR, target->scsi_host,
1635 "Could not fit S/G list into SRP_CMD\n");
1636 ret = -EIO;
1637 goto unmap;
1638 }
1639
1640 count = min(state.ndesc, target->cmd_sg_cnt);
1641 table_len = state.ndesc * sizeof (struct srp_direct_buf);
1642 idb_len = sizeof(struct srp_indirect_buf) + table_len;
1643
1644 fmt = SRP_DATA_DESC_INDIRECT;
1645 len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1646 len += count * sizeof (struct srp_direct_buf);
1647
1648 memcpy(indirect_hdr->desc_list, req->indirect_desc,
1649 count * sizeof (struct srp_direct_buf));
1650
1651 if (!target->global_mr) {
1652 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
1653 idb_len, &idb_rkey);
1654 if (ret < 0)
1655 goto unmap;
1656 req->nmdesc++;
1657 } else {
1658 idb_rkey = cpu_to_be32(target->global_mr->rkey);
1659 }
1660
1661 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1662 indirect_hdr->table_desc.key = idb_rkey;
1663 indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1664 indirect_hdr->len = cpu_to_be32(state.total_len);
1665
1666 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1667 cmd->data_out_desc_cnt = count;
1668 else
1669 cmd->data_in_desc_cnt = count;
1670
1671 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1672 DMA_TO_DEVICE);
1673
1674 map_complete:
1675 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1676 cmd->buf_fmt = fmt << 4;
1677 else
1678 cmd->buf_fmt = fmt;
1679
1680 return len;
1681
1682 unmap:
1683 srp_unmap_data(scmnd, ch, req);
1684 return ret;
1685 }
1686
1687 /*
1688 * Return an IU and possible credit to the free pool
1689 */
1690 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
1691 enum srp_iu_type iu_type)
1692 {
1693 unsigned long flags;
1694
1695 spin_lock_irqsave(&ch->lock, flags);
1696 list_add(&iu->list, &ch->free_tx);
1697 if (iu_type != SRP_IU_RSP)
1698 ++ch->req_lim;
1699 spin_unlock_irqrestore(&ch->lock, flags);
1700 }
1701
1702 /*
1703 * Must be called with ch->lock held to protect req_lim and free_tx.
1704 * If IU is not sent, it must be returned using srp_put_tx_iu().
1705 *
1706 * Note:
1707 * An upper limit for the number of allocated information units for each
1708 * request type is:
1709 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1710 * more than Scsi_Host.can_queue requests.
1711 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1712 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1713 * one unanswered SRP request to an initiator.
1714 */
1715 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
1716 enum srp_iu_type iu_type)
1717 {
1718 struct srp_target_port *target = ch->target;
1719 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1720 struct srp_iu *iu;
1721
1722 ib_process_cq_direct(ch->send_cq, -1);
1723
1724 if (list_empty(&ch->free_tx))
1725 return NULL;
1726
1727 /* Initiator responses to target requests do not consume credits */
1728 if (iu_type != SRP_IU_RSP) {
1729 if (ch->req_lim <= rsv) {
1730 ++target->zero_req_lim;
1731 return NULL;
1732 }
1733
1734 --ch->req_lim;
1735 }
1736
1737 iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
1738 list_del(&iu->list);
1739 return iu;
1740 }
1741
1742 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc)
1743 {
1744 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1745 struct srp_rdma_ch *ch = cq->cq_context;
1746
1747 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1748 srp_handle_qp_err(cq, wc, "SEND");
1749 return;
1750 }
1751
1752 list_add(&iu->list, &ch->free_tx);
1753 }
1754
1755 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
1756 {
1757 struct srp_target_port *target = ch->target;
1758 struct ib_sge list;
1759 struct ib_send_wr wr, *bad_wr;
1760
1761 list.addr = iu->dma;
1762 list.length = len;
1763 list.lkey = target->lkey;
1764
1765 iu->cqe.done = srp_send_done;
1766
1767 wr.next = NULL;
1768 wr.wr_cqe = &iu->cqe;
1769 wr.sg_list = &list;
1770 wr.num_sge = 1;
1771 wr.opcode = IB_WR_SEND;
1772 wr.send_flags = IB_SEND_SIGNALED;
1773
1774 return ib_post_send(ch->qp, &wr, &bad_wr);
1775 }
1776
1777 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
1778 {
1779 struct srp_target_port *target = ch->target;
1780 struct ib_recv_wr wr, *bad_wr;
1781 struct ib_sge list;
1782
1783 list.addr = iu->dma;
1784 list.length = iu->size;
1785 list.lkey = target->lkey;
1786
1787 iu->cqe.done = srp_recv_done;
1788
1789 wr.next = NULL;
1790 wr.wr_cqe = &iu->cqe;
1791 wr.sg_list = &list;
1792 wr.num_sge = 1;
1793
1794 return ib_post_recv(ch->qp, &wr, &bad_wr);
1795 }
1796
1797 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
1798 {
1799 struct srp_target_port *target = ch->target;
1800 struct srp_request *req;
1801 struct scsi_cmnd *scmnd;
1802 unsigned long flags;
1803
1804 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1805 spin_lock_irqsave(&ch->lock, flags);
1806 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1807 spin_unlock_irqrestore(&ch->lock, flags);
1808
1809 ch->tsk_mgmt_status = -1;
1810 if (be32_to_cpu(rsp->resp_data_len) >= 4)
1811 ch->tsk_mgmt_status = rsp->data[3];
1812 complete(&ch->tsk_mgmt_done);
1813 } else {
1814 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1815 if (scmnd) {
1816 req = (void *)scmnd->host_scribble;
1817 scmnd = srp_claim_req(ch, req, NULL, scmnd);
1818 }
1819 if (!scmnd) {
1820 shost_printk(KERN_ERR, target->scsi_host,
1821 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1822 rsp->tag, ch - target->ch, ch->qp->qp_num);
1823
1824 spin_lock_irqsave(&ch->lock, flags);
1825 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1826 spin_unlock_irqrestore(&ch->lock, flags);
1827
1828 return;
1829 }
1830 scmnd->result = rsp->status;
1831
1832 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1833 memcpy(scmnd->sense_buffer, rsp->data +
1834 be32_to_cpu(rsp->resp_data_len),
1835 min_t(int, be32_to_cpu(rsp->sense_data_len),
1836 SCSI_SENSE_BUFFERSIZE));
1837 }
1838
1839 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
1840 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
1841 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1842 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1843 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1844 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1845 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1846 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
1847
1848 srp_free_req(ch, req, scmnd,
1849 be32_to_cpu(rsp->req_lim_delta));
1850
1851 scmnd->host_scribble = NULL;
1852 scmnd->scsi_done(scmnd);
1853 }
1854 }
1855
1856 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
1857 void *rsp, int len)
1858 {
1859 struct srp_target_port *target = ch->target;
1860 struct ib_device *dev = target->srp_host->srp_dev->dev;
1861 unsigned long flags;
1862 struct srp_iu *iu;
1863 int err;
1864
1865 spin_lock_irqsave(&ch->lock, flags);
1866 ch->req_lim += req_delta;
1867 iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1868 spin_unlock_irqrestore(&ch->lock, flags);
1869
1870 if (!iu) {
1871 shost_printk(KERN_ERR, target->scsi_host, PFX
1872 "no IU available to send response\n");
1873 return 1;
1874 }
1875
1876 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1877 memcpy(iu->buf, rsp, len);
1878 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1879
1880 err = srp_post_send(ch, iu, len);
1881 if (err) {
1882 shost_printk(KERN_ERR, target->scsi_host, PFX
1883 "unable to post response: %d\n", err);
1884 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
1885 }
1886
1887 return err;
1888 }
1889
1890 static void srp_process_cred_req(struct srp_rdma_ch *ch,
1891 struct srp_cred_req *req)
1892 {
1893 struct srp_cred_rsp rsp = {
1894 .opcode = SRP_CRED_RSP,
1895 .tag = req->tag,
1896 };
1897 s32 delta = be32_to_cpu(req->req_lim_delta);
1898
1899 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1900 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
1901 "problems processing SRP_CRED_REQ\n");
1902 }
1903
1904 static void srp_process_aer_req(struct srp_rdma_ch *ch,
1905 struct srp_aer_req *req)
1906 {
1907 struct srp_target_port *target = ch->target;
1908 struct srp_aer_rsp rsp = {
1909 .opcode = SRP_AER_RSP,
1910 .tag = req->tag,
1911 };
1912 s32 delta = be32_to_cpu(req->req_lim_delta);
1913
1914 shost_printk(KERN_ERR, target->scsi_host, PFX
1915 "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
1916
1917 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1918 shost_printk(KERN_ERR, target->scsi_host, PFX
1919 "problems processing SRP_AER_REQ\n");
1920 }
1921
1922 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1923 {
1924 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1925 struct srp_rdma_ch *ch = cq->cq_context;
1926 struct srp_target_port *target = ch->target;
1927 struct ib_device *dev = target->srp_host->srp_dev->dev;
1928 int res;
1929 u8 opcode;
1930
1931 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1932 srp_handle_qp_err(cq, wc, "RECV");
1933 return;
1934 }
1935
1936 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
1937 DMA_FROM_DEVICE);
1938
1939 opcode = *(u8 *) iu->buf;
1940
1941 if (0) {
1942 shost_printk(KERN_ERR, target->scsi_host,
1943 PFX "recv completion, opcode 0x%02x\n", opcode);
1944 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
1945 iu->buf, wc->byte_len, true);
1946 }
1947
1948 switch (opcode) {
1949 case SRP_RSP:
1950 srp_process_rsp(ch, iu->buf);
1951 break;
1952
1953 case SRP_CRED_REQ:
1954 srp_process_cred_req(ch, iu->buf);
1955 break;
1956
1957 case SRP_AER_REQ:
1958 srp_process_aer_req(ch, iu->buf);
1959 break;
1960
1961 case SRP_T_LOGOUT:
1962 /* XXX Handle target logout */
1963 shost_printk(KERN_WARNING, target->scsi_host,
1964 PFX "Got target logout request\n");
1965 break;
1966
1967 default:
1968 shost_printk(KERN_WARNING, target->scsi_host,
1969 PFX "Unhandled SRP opcode 0x%02x\n", opcode);
1970 break;
1971 }
1972
1973 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
1974 DMA_FROM_DEVICE);
1975
1976 res = srp_post_recv(ch, iu);
1977 if (res != 0)
1978 shost_printk(KERN_ERR, target->scsi_host,
1979 PFX "Recv failed with error code %d\n", res);
1980 }
1981
1982 /**
1983 * srp_tl_err_work() - handle a transport layer error
1984 * @work: Work structure embedded in an SRP target port.
1985 *
1986 * Note: This function may get invoked before the rport has been created,
1987 * hence the target->rport test.
1988 */
1989 static void srp_tl_err_work(struct work_struct *work)
1990 {
1991 struct srp_target_port *target;
1992
1993 target = container_of(work, struct srp_target_port, tl_err_work);
1994 if (target->rport)
1995 srp_start_tl_fail_timers(target->rport);
1996 }
1997
1998 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
1999 const char *opname)
2000 {
2001 struct srp_rdma_ch *ch = cq->cq_context;
2002 struct srp_target_port *target = ch->target;
2003
2004 if (ch->connected && !target->qp_in_error) {
2005 shost_printk(KERN_ERR, target->scsi_host,
2006 PFX "failed %s status %s (%d) for CQE %p\n",
2007 opname, ib_wc_status_msg(wc->status), wc->status,
2008 wc->wr_cqe);
2009 queue_work(system_long_wq, &target->tl_err_work);
2010 }
2011 target->qp_in_error = true;
2012 }
2013
2014 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
2015 {
2016 struct srp_target_port *target = host_to_target(shost);
2017 struct srp_rport *rport = target->rport;
2018 struct srp_rdma_ch *ch;
2019 struct srp_request *req;
2020 struct srp_iu *iu;
2021 struct srp_cmd *cmd;
2022 struct ib_device *dev;
2023 unsigned long flags;
2024 u32 tag;
2025 u16 idx;
2026 int len, ret;
2027 const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
2028
2029 /*
2030 * The SCSI EH thread is the only context from which srp_queuecommand()
2031 * can get invoked for blocked devices (SDEV_BLOCK /
2032 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
2033 * locking the rport mutex if invoked from inside the SCSI EH.
2034 */
2035 if (in_scsi_eh)
2036 mutex_lock(&rport->mutex);
2037
2038 scmnd->result = srp_chkready(target->rport);
2039 if (unlikely(scmnd->result))
2040 goto err;
2041
2042 WARN_ON_ONCE(scmnd->request->tag < 0);
2043 tag = blk_mq_unique_tag(scmnd->request);
2044 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
2045 idx = blk_mq_unique_tag_to_tag(tag);
2046 WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2047 dev_name(&shost->shost_gendev), tag, idx,
2048 target->req_ring_size);
2049
2050 spin_lock_irqsave(&ch->lock, flags);
2051 iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
2052 spin_unlock_irqrestore(&ch->lock, flags);
2053
2054 if (!iu)
2055 goto err;
2056
2057 req = &ch->req_ring[idx];
2058 dev = target->srp_host->srp_dev->dev;
2059 ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
2060 DMA_TO_DEVICE);
2061
2062 scmnd->host_scribble = (void *) req;
2063
2064 cmd = iu->buf;
2065 memset(cmd, 0, sizeof *cmd);
2066
2067 cmd->opcode = SRP_CMD;
2068 int_to_scsilun(scmnd->device->lun, &cmd->lun);
2069 cmd->tag = tag;
2070 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2071
2072 req->scmnd = scmnd;
2073 req->cmd = iu;
2074
2075 len = srp_map_data(scmnd, ch, req);
2076 if (len < 0) {
2077 shost_printk(KERN_ERR, target->scsi_host,
2078 PFX "Failed to map data (%d)\n", len);
2079 /*
2080 * If we ran out of memory descriptors (-ENOMEM) because an
2081 * application is queuing many requests with more than
2082 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
2083 * to reduce queue depth temporarily.
2084 */
2085 scmnd->result = len == -ENOMEM ?
2086 DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
2087 goto err_iu;
2088 }
2089
2090 ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
2091 DMA_TO_DEVICE);
2092
2093 if (srp_post_send(ch, iu, len)) {
2094 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
2095 goto err_unmap;
2096 }
2097
2098 ret = 0;
2099
2100 unlock_rport:
2101 if (in_scsi_eh)
2102 mutex_unlock(&rport->mutex);
2103
2104 return ret;
2105
2106 err_unmap:
2107 srp_unmap_data(scmnd, ch, req);
2108
2109 err_iu:
2110 srp_put_tx_iu(ch, iu, SRP_IU_CMD);
2111
2112 /*
2113 * Avoid that the loops that iterate over the request ring can
2114 * encounter a dangling SCSI command pointer.
2115 */
2116 req->scmnd = NULL;
2117
2118 err:
2119 if (scmnd->result) {
2120 scmnd->scsi_done(scmnd);
2121 ret = 0;
2122 } else {
2123 ret = SCSI_MLQUEUE_HOST_BUSY;
2124 }
2125
2126 goto unlock_rport;
2127 }
2128
2129 /*
2130 * Note: the resources allocated in this function are freed in
2131 * srp_free_ch_ib().
2132 */
2133 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
2134 {
2135 struct srp_target_port *target = ch->target;
2136 int i;
2137
2138 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2139 GFP_KERNEL);
2140 if (!ch->rx_ring)
2141 goto err_no_ring;
2142 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2143 GFP_KERNEL);
2144 if (!ch->tx_ring)
2145 goto err_no_ring;
2146
2147 for (i = 0; i < target->queue_size; ++i) {
2148 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2149 ch->max_ti_iu_len,
2150 GFP_KERNEL, DMA_FROM_DEVICE);
2151 if (!ch->rx_ring[i])
2152 goto err;
2153 }
2154
2155 for (i = 0; i < target->queue_size; ++i) {
2156 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2157 target->max_iu_len,
2158 GFP_KERNEL, DMA_TO_DEVICE);
2159 if (!ch->tx_ring[i])
2160 goto err;
2161
2162 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
2163 }
2164
2165 return 0;
2166
2167 err:
2168 for (i = 0; i < target->queue_size; ++i) {
2169 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2170 srp_free_iu(target->srp_host, ch->tx_ring[i]);
2171 }
2172
2173
2174 err_no_ring:
2175 kfree(ch->tx_ring);
2176 ch->tx_ring = NULL;
2177 kfree(ch->rx_ring);
2178 ch->rx_ring = NULL;
2179
2180 return -ENOMEM;
2181 }
2182
2183 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2184 {
2185 uint64_t T_tr_ns, max_compl_time_ms;
2186 uint32_t rq_tmo_jiffies;
2187
2188 /*
2189 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2190 * table 91), both the QP timeout and the retry count have to be set
2191 * for RC QP's during the RTR to RTS transition.
2192 */
2193 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2194 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2195
2196 /*
2197 * Set target->rq_tmo_jiffies to one second more than the largest time
2198 * it can take before an error completion is generated. See also
2199 * C9-140..142 in the IBTA spec for more information about how to
2200 * convert the QP Local ACK Timeout value to nanoseconds.
2201 */
2202 T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2203 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2204 do_div(max_compl_time_ms, NSEC_PER_MSEC);
2205 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2206
2207 return rq_tmo_jiffies;
2208 }
2209
2210 static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2211 const struct srp_login_rsp *lrsp,
2212 struct srp_rdma_ch *ch)
2213 {
2214 struct srp_target_port *target = ch->target;
2215 struct ib_qp_attr *qp_attr = NULL;
2216 int attr_mask = 0;
2217 int ret;
2218 int i;
2219
2220 if (lrsp->opcode == SRP_LOGIN_RSP) {
2221 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2222 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta);
2223
2224 /*
2225 * Reserve credits for task management so we don't
2226 * bounce requests back to the SCSI mid-layer.
2227 */
2228 target->scsi_host->can_queue
2229 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2230 target->scsi_host->can_queue);
2231 target->scsi_host->cmd_per_lun
2232 = min_t(int, target->scsi_host->can_queue,
2233 target->scsi_host->cmd_per_lun);
2234 } else {
2235 shost_printk(KERN_WARNING, target->scsi_host,
2236 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2237 ret = -ECONNRESET;
2238 goto error;
2239 }
2240
2241 if (!ch->rx_ring) {
2242 ret = srp_alloc_iu_bufs(ch);
2243 if (ret)
2244 goto error;
2245 }
2246
2247 ret = -ENOMEM;
2248 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2249 if (!qp_attr)
2250 goto error;
2251
2252 qp_attr->qp_state = IB_QPS_RTR;
2253 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2254 if (ret)
2255 goto error_free;
2256
2257 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2258 if (ret)
2259 goto error_free;
2260
2261 for (i = 0; i < target->queue_size; i++) {
2262 struct srp_iu *iu = ch->rx_ring[i];
2263
2264 ret = srp_post_recv(ch, iu);
2265 if (ret)
2266 goto error_free;
2267 }
2268
2269 qp_attr->qp_state = IB_QPS_RTS;
2270 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2271 if (ret)
2272 goto error_free;
2273
2274 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2275
2276 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2277 if (ret)
2278 goto error_free;
2279
2280 ret = ib_send_cm_rtu(cm_id, NULL, 0);
2281
2282 error_free:
2283 kfree(qp_attr);
2284
2285 error:
2286 ch->status = ret;
2287 }
2288
2289 static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2290 struct ib_cm_event *event,
2291 struct srp_rdma_ch *ch)
2292 {
2293 struct srp_target_port *target = ch->target;
2294 struct Scsi_Host *shost = target->scsi_host;
2295 struct ib_class_port_info *cpi;
2296 int opcode;
2297
2298 switch (event->param.rej_rcvd.reason) {
2299 case IB_CM_REJ_PORT_CM_REDIRECT:
2300 cpi = event->param.rej_rcvd.ari;
2301 ch->path.dlid = cpi->redirect_lid;
2302 ch->path.pkey = cpi->redirect_pkey;
2303 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2304 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
2305
2306 ch->status = ch->path.dlid ?
2307 SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2308 break;
2309
2310 case IB_CM_REJ_PORT_REDIRECT:
2311 if (srp_target_is_topspin(target)) {
2312 /*
2313 * Topspin/Cisco SRP gateways incorrectly send
2314 * reject reason code 25 when they mean 24
2315 * (port redirect).
2316 */
2317 memcpy(ch->path.dgid.raw,
2318 event->param.rej_rcvd.ari, 16);
2319
2320 shost_printk(KERN_DEBUG, shost,
2321 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2322 be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2323 be64_to_cpu(ch->path.dgid.global.interface_id));
2324
2325 ch->status = SRP_PORT_REDIRECT;
2326 } else {
2327 shost_printk(KERN_WARNING, shost,
2328 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2329 ch->status = -ECONNRESET;
2330 }
2331 break;
2332
2333 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2334 shost_printk(KERN_WARNING, shost,
2335 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2336 ch->status = -ECONNRESET;
2337 break;
2338
2339 case IB_CM_REJ_CONSUMER_DEFINED:
2340 opcode = *(u8 *) event->private_data;
2341 if (opcode == SRP_LOGIN_REJ) {
2342 struct srp_login_rej *rej = event->private_data;
2343 u32 reason = be32_to_cpu(rej->reason);
2344
2345 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2346 shost_printk(KERN_WARNING, shost,
2347 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2348 else
2349 shost_printk(KERN_WARNING, shost, PFX
2350 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2351 target->sgid.raw,
2352 target->orig_dgid.raw, reason);
2353 } else
2354 shost_printk(KERN_WARNING, shost,
2355 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2356 " opcode 0x%02x\n", opcode);
2357 ch->status = -ECONNRESET;
2358 break;
2359
2360 case IB_CM_REJ_STALE_CONN:
2361 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n");
2362 ch->status = SRP_STALE_CONN;
2363 break;
2364
2365 default:
2366 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n",
2367 event->param.rej_rcvd.reason);
2368 ch->status = -ECONNRESET;
2369 }
2370 }
2371
2372 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2373 {
2374 struct srp_rdma_ch *ch = cm_id->context;
2375 struct srp_target_port *target = ch->target;
2376 int comp = 0;
2377
2378 switch (event->event) {
2379 case IB_CM_REQ_ERROR:
2380 shost_printk(KERN_DEBUG, target->scsi_host,
2381 PFX "Sending CM REQ failed\n");
2382 comp = 1;
2383 ch->status = -ECONNRESET;
2384 break;
2385
2386 case IB_CM_REP_RECEIVED:
2387 comp = 1;
2388 srp_cm_rep_handler(cm_id, event->private_data, ch);
2389 break;
2390
2391 case IB_CM_REJ_RECEIVED:
2392 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2393 comp = 1;
2394
2395 srp_cm_rej_handler(cm_id, event, ch);
2396 break;
2397
2398 case IB_CM_DREQ_RECEIVED:
2399 shost_printk(KERN_WARNING, target->scsi_host,
2400 PFX "DREQ received - connection closed\n");
2401 ch->connected = false;
2402 if (ib_send_cm_drep(cm_id, NULL, 0))
2403 shost_printk(KERN_ERR, target->scsi_host,
2404 PFX "Sending CM DREP failed\n");
2405 queue_work(system_long_wq, &target->tl_err_work);
2406 break;
2407
2408 case IB_CM_TIMEWAIT_EXIT:
2409 shost_printk(KERN_ERR, target->scsi_host,
2410 PFX "connection closed\n");
2411 comp = 1;
2412
2413 ch->status = 0;
2414 break;
2415
2416 case IB_CM_MRA_RECEIVED:
2417 case IB_CM_DREQ_ERROR:
2418 case IB_CM_DREP_RECEIVED:
2419 break;
2420
2421 default:
2422 shost_printk(KERN_WARNING, target->scsi_host,
2423 PFX "Unhandled CM event %d\n", event->event);
2424 break;
2425 }
2426
2427 if (comp)
2428 complete(&ch->done);
2429
2430 return 0;
2431 }
2432
2433 /**
2434 * srp_change_queue_depth - setting device queue depth
2435 * @sdev: scsi device struct
2436 * @qdepth: requested queue depth
2437 *
2438 * Returns queue depth.
2439 */
2440 static int
2441 srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
2442 {
2443 if (!sdev->tagged_supported)
2444 qdepth = 1;
2445 return scsi_change_queue_depth(sdev, qdepth);
2446 }
2447
2448 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2449 u8 func)
2450 {
2451 struct srp_target_port *target = ch->target;
2452 struct srp_rport *rport = target->rport;
2453 struct ib_device *dev = target->srp_host->srp_dev->dev;
2454 struct srp_iu *iu;
2455 struct srp_tsk_mgmt *tsk_mgmt;
2456
2457 if (!ch->connected || target->qp_in_error)
2458 return -1;
2459
2460 init_completion(&ch->tsk_mgmt_done);
2461
2462 /*
2463 * Lock the rport mutex to avoid that srp_create_ch_ib() is
2464 * invoked while a task management function is being sent.
2465 */
2466 mutex_lock(&rport->mutex);
2467 spin_lock_irq(&ch->lock);
2468 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2469 spin_unlock_irq(&ch->lock);
2470
2471 if (!iu) {
2472 mutex_unlock(&rport->mutex);
2473
2474 return -1;
2475 }
2476
2477 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2478 DMA_TO_DEVICE);
2479 tsk_mgmt = iu->buf;
2480 memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2481
2482 tsk_mgmt->opcode = SRP_TSK_MGMT;
2483 int_to_scsilun(lun, &tsk_mgmt->lun);
2484 tsk_mgmt->tag = req_tag | SRP_TAG_TSK_MGMT;
2485 tsk_mgmt->tsk_mgmt_func = func;
2486 tsk_mgmt->task_tag = req_tag;
2487
2488 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2489 DMA_TO_DEVICE);
2490 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2491 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
2492 mutex_unlock(&rport->mutex);
2493
2494 return -1;
2495 }
2496 mutex_unlock(&rport->mutex);
2497
2498 if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
2499 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
2500 return -1;
2501
2502 return 0;
2503 }
2504
2505 static int srp_abort(struct scsi_cmnd *scmnd)
2506 {
2507 struct srp_target_port *target = host_to_target(scmnd->device->host);
2508 struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
2509 u32 tag;
2510 u16 ch_idx;
2511 struct srp_rdma_ch *ch;
2512 int ret;
2513
2514 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2515
2516 if (!req)
2517 return SUCCESS;
2518 tag = blk_mq_unique_tag(scmnd->request);
2519 ch_idx = blk_mq_unique_tag_to_hwq(tag);
2520 if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2521 return SUCCESS;
2522 ch = &target->ch[ch_idx];
2523 if (!srp_claim_req(ch, req, NULL, scmnd))
2524 return SUCCESS;
2525 shost_printk(KERN_ERR, target->scsi_host,
2526 "Sending SRP abort for tag %#x\n", tag);
2527 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
2528 SRP_TSK_ABORT_TASK) == 0)
2529 ret = SUCCESS;
2530 else if (target->rport->state == SRP_RPORT_LOST)
2531 ret = FAST_IO_FAIL;
2532 else
2533 ret = FAILED;
2534 srp_free_req(ch, req, scmnd, 0);
2535 scmnd->result = DID_ABORT << 16;
2536 scmnd->scsi_done(scmnd);
2537
2538 return ret;
2539 }
2540
2541 static int srp_reset_device(struct scsi_cmnd *scmnd)
2542 {
2543 struct srp_target_port *target = host_to_target(scmnd->device->host);
2544 struct srp_rdma_ch *ch;
2545 int i;
2546
2547 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2548
2549 ch = &target->ch[0];
2550 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
2551 SRP_TSK_LUN_RESET))
2552 return FAILED;
2553 if (ch->tsk_mgmt_status)
2554 return FAILED;
2555
2556 for (i = 0; i < target->ch_count; i++) {
2557 ch = &target->ch[i];
2558 for (i = 0; i < target->req_ring_size; ++i) {
2559 struct srp_request *req = &ch->req_ring[i];
2560
2561 srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2562 }
2563 }
2564
2565 return SUCCESS;
2566 }
2567
2568 static int srp_reset_host(struct scsi_cmnd *scmnd)
2569 {
2570 struct srp_target_port *target = host_to_target(scmnd->device->host);
2571
2572 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2573
2574 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2575 }
2576
2577 static int srp_slave_configure(struct scsi_device *sdev)
2578 {
2579 struct Scsi_Host *shost = sdev->host;
2580 struct srp_target_port *target = host_to_target(shost);
2581 struct request_queue *q = sdev->request_queue;
2582 unsigned long timeout;
2583
2584 if (sdev->type == TYPE_DISK) {
2585 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2586 blk_queue_rq_timeout(q, timeout);
2587 }
2588
2589 return 0;
2590 }
2591
2592 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2593 char *buf)
2594 {
2595 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2596
2597 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
2598 }
2599
2600 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2601 char *buf)
2602 {
2603 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2604
2605 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
2606 }
2607
2608 static ssize_t show_service_id(struct device *dev,
2609 struct device_attribute *attr, char *buf)
2610 {
2611 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2612
2613 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
2614 }
2615
2616 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2617 char *buf)
2618 {
2619 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2620
2621 return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
2622 }
2623
2624 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2625 char *buf)
2626 {
2627 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2628
2629 return sprintf(buf, "%pI6\n", target->sgid.raw);
2630 }
2631
2632 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2633 char *buf)
2634 {
2635 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2636 struct srp_rdma_ch *ch = &target->ch[0];
2637
2638 return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
2639 }
2640
2641 static ssize_t show_orig_dgid(struct device *dev,
2642 struct device_attribute *attr, char *buf)
2643 {
2644 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2645
2646 return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
2647 }
2648
2649 static ssize_t show_req_lim(struct device *dev,
2650 struct device_attribute *attr, char *buf)
2651 {
2652 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2653 struct srp_rdma_ch *ch;
2654 int i, req_lim = INT_MAX;
2655
2656 for (i = 0; i < target->ch_count; i++) {
2657 ch = &target->ch[i];
2658 req_lim = min(req_lim, ch->req_lim);
2659 }
2660 return sprintf(buf, "%d\n", req_lim);
2661 }
2662
2663 static ssize_t show_zero_req_lim(struct device *dev,
2664 struct device_attribute *attr, char *buf)
2665 {
2666 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2667
2668 return sprintf(buf, "%d\n", target->zero_req_lim);
2669 }
2670
2671 static ssize_t show_local_ib_port(struct device *dev,
2672 struct device_attribute *attr, char *buf)
2673 {
2674 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2675
2676 return sprintf(buf, "%d\n", target->srp_host->port);
2677 }
2678
2679 static ssize_t show_local_ib_device(struct device *dev,
2680 struct device_attribute *attr, char *buf)
2681 {
2682 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2683
2684 return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
2685 }
2686
2687 static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2688 char *buf)
2689 {
2690 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2691
2692 return sprintf(buf, "%d\n", target->ch_count);
2693 }
2694
2695 static ssize_t show_comp_vector(struct device *dev,
2696 struct device_attribute *attr, char *buf)
2697 {
2698 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2699
2700 return sprintf(buf, "%d\n", target->comp_vector);
2701 }
2702
2703 static ssize_t show_tl_retry_count(struct device *dev,
2704 struct device_attribute *attr, char *buf)
2705 {
2706 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2707
2708 return sprintf(buf, "%d\n", target->tl_retry_count);
2709 }
2710
2711 static ssize_t show_cmd_sg_entries(struct device *dev,
2712 struct device_attribute *attr, char *buf)
2713 {
2714 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2715
2716 return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2717 }
2718
2719 static ssize_t show_allow_ext_sg(struct device *dev,
2720 struct device_attribute *attr, char *buf)
2721 {
2722 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2723
2724 return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2725 }
2726
2727 static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL);
2728 static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL);
2729 static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL);
2730 static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL);
2731 static DEVICE_ATTR(sgid, S_IRUGO, show_sgid, NULL);
2732 static DEVICE_ATTR(dgid, S_IRUGO, show_dgid, NULL);
2733 static DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL);
2734 static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL);
2735 static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
2736 static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
2737 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
2738 static DEVICE_ATTR(ch_count, S_IRUGO, show_ch_count, NULL);
2739 static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL);
2740 static DEVICE_ATTR(tl_retry_count, S_IRUGO, show_tl_retry_count, NULL);
2741 static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL);
2742 static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL);
2743
2744 static struct device_attribute *srp_host_attrs[] = {
2745 &dev_attr_id_ext,
2746 &dev_attr_ioc_guid,
2747 &dev_attr_service_id,
2748 &dev_attr_pkey,
2749 &dev_attr_sgid,
2750 &dev_attr_dgid,
2751 &dev_attr_orig_dgid,
2752 &dev_attr_req_lim,
2753 &dev_attr_zero_req_lim,
2754 &dev_attr_local_ib_port,
2755 &dev_attr_local_ib_device,
2756 &dev_attr_ch_count,
2757 &dev_attr_comp_vector,
2758 &dev_attr_tl_retry_count,
2759 &dev_attr_cmd_sg_entries,
2760 &dev_attr_allow_ext_sg,
2761 NULL
2762 };
2763
2764 static struct scsi_host_template srp_template = {
2765 .module = THIS_MODULE,
2766 .name = "InfiniBand SRP initiator",
2767 .proc_name = DRV_NAME,
2768 .slave_configure = srp_slave_configure,
2769 .info = srp_target_info,
2770 .queuecommand = srp_queuecommand,
2771 .change_queue_depth = srp_change_queue_depth,
2772 .eh_abort_handler = srp_abort,
2773 .eh_device_reset_handler = srp_reset_device,
2774 .eh_host_reset_handler = srp_reset_host,
2775 .skip_settle_delay = true,
2776 .sg_tablesize = SRP_DEF_SG_TABLESIZE,
2777 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE,
2778 .this_id = -1,
2779 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE,
2780 .use_clustering = ENABLE_CLUSTERING,
2781 .shost_attrs = srp_host_attrs,
2782 .track_queue_depth = 1,
2783 };
2784
2785 static int srp_sdev_count(struct Scsi_Host *host)
2786 {
2787 struct scsi_device *sdev;
2788 int c = 0;
2789
2790 shost_for_each_device(sdev, host)
2791 c++;
2792
2793 return c;
2794 }
2795
2796 /*
2797 * Return values:
2798 * < 0 upon failure. Caller is responsible for SRP target port cleanup.
2799 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
2800 * removal has been scheduled.
2801 * 0 and target->state != SRP_TARGET_REMOVED upon success.
2802 */
2803 static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2804 {
2805 struct srp_rport_identifiers ids;
2806 struct srp_rport *rport;
2807
2808 target->state = SRP_TARGET_SCANNING;
2809 sprintf(target->target_name, "SRP.T10:%016llX",
2810 be64_to_cpu(target->id_ext));
2811
2812 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
2813 return -ENODEV;
2814
2815 memcpy(ids.port_id, &target->id_ext, 8);
2816 memcpy(ids.port_id + 8, &target->ioc_guid, 8);
2817 ids.roles = SRP_RPORT_ROLE_TARGET;
2818 rport = srp_rport_add(target->scsi_host, &ids);
2819 if (IS_ERR(rport)) {
2820 scsi_remove_host(target->scsi_host);
2821 return PTR_ERR(rport);
2822 }
2823
2824 rport->lld_data = target;
2825 target->rport = rport;
2826
2827 spin_lock(&host->target_lock);
2828 list_add_tail(&target->list, &host->target_list);
2829 spin_unlock(&host->target_lock);
2830
2831 scsi_scan_target(&target->scsi_host->shost_gendev,
2832 0, target->scsi_id, SCAN_WILD_CARD, 0);
2833
2834 if (srp_connected_ch(target) < target->ch_count ||
2835 target->qp_in_error) {
2836 shost_printk(KERN_INFO, target->scsi_host,
2837 PFX "SCSI scan failed - removing SCSI host\n");
2838 srp_queue_remove_work(target);
2839 goto out;
2840 }
2841
2842 pr_debug(PFX "%s: SCSI scan succeeded - detected %d LUNs\n",
2843 dev_name(&target->scsi_host->shost_gendev),
2844 srp_sdev_count(target->scsi_host));
2845
2846 spin_lock_irq(&target->lock);
2847 if (target->state == SRP_TARGET_SCANNING)
2848 target->state = SRP_TARGET_LIVE;
2849 spin_unlock_irq(&target->lock);
2850
2851 out:
2852 return 0;
2853 }
2854
2855 static void srp_release_dev(struct device *dev)
2856 {
2857 struct srp_host *host =
2858 container_of(dev, struct srp_host, dev);
2859
2860 complete(&host->released);
2861 }
2862
2863 static struct class srp_class = {
2864 .name = "infiniband_srp",
2865 .dev_release = srp_release_dev
2866 };
2867
2868 /**
2869 * srp_conn_unique() - check whether the connection to a target is unique
2870 * @host: SRP host.
2871 * @target: SRP target port.
2872 */
2873 static bool srp_conn_unique(struct srp_host *host,
2874 struct srp_target_port *target)
2875 {
2876 struct srp_target_port *t;
2877 bool ret = false;
2878
2879 if (target->state == SRP_TARGET_REMOVED)
2880 goto out;
2881
2882 ret = true;
2883
2884 spin_lock(&host->target_lock);
2885 list_for_each_entry(t, &host->target_list, list) {
2886 if (t != target &&
2887 target->id_ext == t->id_ext &&
2888 target->ioc_guid == t->ioc_guid &&
2889 target->initiator_ext == t->initiator_ext) {
2890 ret = false;
2891 break;
2892 }
2893 }
2894 spin_unlock(&host->target_lock);
2895
2896 out:
2897 return ret;
2898 }
2899
2900 /*
2901 * Target ports are added by writing
2902 *
2903 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2904 * pkey=<P_Key>,service_id=<service ID>
2905 *
2906 * to the add_target sysfs attribute.
2907 */
2908 enum {
2909 SRP_OPT_ERR = 0,
2910 SRP_OPT_ID_EXT = 1 << 0,
2911 SRP_OPT_IOC_GUID = 1 << 1,
2912 SRP_OPT_DGID = 1 << 2,
2913 SRP_OPT_PKEY = 1 << 3,
2914 SRP_OPT_SERVICE_ID = 1 << 4,
2915 SRP_OPT_MAX_SECT = 1 << 5,
2916 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
2917 SRP_OPT_IO_CLASS = 1 << 7,
2918 SRP_OPT_INITIATOR_EXT = 1 << 8,
2919 SRP_OPT_CMD_SG_ENTRIES = 1 << 9,
2920 SRP_OPT_ALLOW_EXT_SG = 1 << 10,
2921 SRP_OPT_SG_TABLESIZE = 1 << 11,
2922 SRP_OPT_COMP_VECTOR = 1 << 12,
2923 SRP_OPT_TL_RETRY_COUNT = 1 << 13,
2924 SRP_OPT_QUEUE_SIZE = 1 << 14,
2925 SRP_OPT_ALL = (SRP_OPT_ID_EXT |
2926 SRP_OPT_IOC_GUID |
2927 SRP_OPT_DGID |
2928 SRP_OPT_PKEY |
2929 SRP_OPT_SERVICE_ID),
2930 };
2931
2932 static const match_table_t srp_opt_tokens = {
2933 { SRP_OPT_ID_EXT, "id_ext=%s" },
2934 { SRP_OPT_IOC_GUID, "ioc_guid=%s" },
2935 { SRP_OPT_DGID, "dgid=%s" },
2936 { SRP_OPT_PKEY, "pkey=%x" },
2937 { SRP_OPT_SERVICE_ID, "service_id=%s" },
2938 { SRP_OPT_MAX_SECT, "max_sect=%d" },
2939 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" },
2940 { SRP_OPT_IO_CLASS, "io_class=%x" },
2941 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" },
2942 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" },
2943 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" },
2944 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" },
2945 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" },
2946 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" },
2947 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" },
2948 { SRP_OPT_ERR, NULL }
2949 };
2950
2951 static int srp_parse_options(const char *buf, struct srp_target_port *target)
2952 {
2953 char *options, *sep_opt;
2954 char *p;
2955 char dgid[3];
2956 substring_t args[MAX_OPT_ARGS];
2957 int opt_mask = 0;
2958 int token;
2959 int ret = -EINVAL;
2960 int i;
2961
2962 options = kstrdup(buf, GFP_KERNEL);
2963 if (!options)
2964 return -ENOMEM;
2965
2966 sep_opt = options;
2967 while ((p = strsep(&sep_opt, ",\n")) != NULL) {
2968 if (!*p)
2969 continue;
2970
2971 token = match_token(p, srp_opt_tokens, args);
2972 opt_mask |= token;
2973
2974 switch (token) {
2975 case SRP_OPT_ID_EXT:
2976 p = match_strdup(args);
2977 if (!p) {
2978 ret = -ENOMEM;
2979 goto out;
2980 }
2981 target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2982 kfree(p);
2983 break;
2984
2985 case SRP_OPT_IOC_GUID:
2986 p = match_strdup(args);
2987 if (!p) {
2988 ret = -ENOMEM;
2989 goto out;
2990 }
2991 target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
2992 kfree(p);
2993 break;
2994
2995 case SRP_OPT_DGID:
2996 p = match_strdup(args);
2997 if (!p) {
2998 ret = -ENOMEM;
2999 goto out;
3000 }
3001 if (strlen(p) != 32) {
3002 pr_warn("bad dest GID parameter '%s'\n", p);
3003 kfree(p);
3004 goto out;
3005 }
3006
3007 for (i = 0; i < 16; ++i) {
3008 strlcpy(dgid, p + i * 2, sizeof(dgid));
3009 if (sscanf(dgid, "%hhx",
3010 &target->orig_dgid.raw[i]) < 1) {
3011 ret = -EINVAL;
3012 kfree(p);
3013 goto out;
3014 }
3015 }
3016 kfree(p);
3017 break;
3018
3019 case SRP_OPT_PKEY:
3020 if (match_hex(args, &token)) {
3021 pr_warn("bad P_Key parameter '%s'\n", p);
3022 goto out;
3023 }
3024 target->pkey = cpu_to_be16(token);
3025 break;
3026
3027 case SRP_OPT_SERVICE_ID:
3028 p = match_strdup(args);
3029 if (!p) {
3030 ret = -ENOMEM;
3031 goto out;
3032 }
3033 target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
3034 kfree(p);
3035 break;
3036
3037 case SRP_OPT_MAX_SECT:
3038 if (match_int(args, &token)) {
3039 pr_warn("bad max sect parameter '%s'\n", p);
3040 goto out;
3041 }
3042 target->scsi_host->max_sectors = token;
3043 break;
3044
3045 case SRP_OPT_QUEUE_SIZE:
3046 if (match_int(args, &token) || token < 1) {
3047 pr_warn("bad queue_size parameter '%s'\n", p);
3048 goto out;
3049 }
3050 target->scsi_host->can_queue = token;
3051 target->queue_size = token + SRP_RSP_SQ_SIZE +
3052 SRP_TSK_MGMT_SQ_SIZE;
3053 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3054 target->scsi_host->cmd_per_lun = token;
3055 break;
3056
3057 case SRP_OPT_MAX_CMD_PER_LUN:
3058 if (match_int(args, &token) || token < 1) {
3059 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3060 p);
3061 goto out;
3062 }
3063 target->scsi_host->cmd_per_lun = token;
3064 break;
3065
3066 case SRP_OPT_IO_CLASS:
3067 if (match_hex(args, &token)) {
3068 pr_warn("bad IO class parameter '%s'\n", p);
3069 goto out;
3070 }
3071 if (token != SRP_REV10_IB_IO_CLASS &&
3072 token != SRP_REV16A_IB_IO_CLASS) {
3073 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3074 token, SRP_REV10_IB_IO_CLASS,
3075 SRP_REV16A_IB_IO_CLASS);
3076 goto out;
3077 }
3078 target->io_class = token;
3079 break;
3080
3081 case SRP_OPT_INITIATOR_EXT:
3082 p = match_strdup(args);
3083 if (!p) {
3084 ret = -ENOMEM;
3085 goto out;
3086 }
3087 target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3088 kfree(p);
3089 break;
3090
3091 case SRP_OPT_CMD_SG_ENTRIES:
3092 if (match_int(args, &token) || token < 1 || token > 255) {
3093 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3094 p);
3095 goto out;
3096 }
3097 target->cmd_sg_cnt = token;
3098 break;
3099
3100 case SRP_OPT_ALLOW_EXT_SG:
3101 if (match_int(args, &token)) {
3102 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3103 goto out;
3104 }
3105 target->allow_ext_sg = !!token;
3106 break;
3107
3108 case SRP_OPT_SG_TABLESIZE:
3109 if (match_int(args, &token) || token < 1 ||
3110 token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
3111 pr_warn("bad max sg_tablesize parameter '%s'\n",
3112 p);
3113 goto out;
3114 }
3115 target->sg_tablesize = token;
3116 break;
3117
3118 case SRP_OPT_COMP_VECTOR:
3119 if (match_int(args, &token) || token < 0) {
3120 pr_warn("bad comp_vector parameter '%s'\n", p);
3121 goto out;
3122 }
3123 target->comp_vector = token;
3124 break;
3125
3126 case SRP_OPT_TL_RETRY_COUNT:
3127 if (match_int(args, &token) || token < 2 || token > 7) {
3128 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3129 p);
3130 goto out;
3131 }
3132 target->tl_retry_count = token;
3133 break;
3134
3135 default:
3136 pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3137 p);
3138 goto out;
3139 }
3140 }
3141
3142 if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3143 ret = 0;
3144 else
3145 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3146 if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3147 !(srp_opt_tokens[i].token & opt_mask))
3148 pr_warn("target creation request is missing parameter '%s'\n",
3149 srp_opt_tokens[i].pattern);
3150
3151 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3152 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3153 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3154 target->scsi_host->cmd_per_lun,
3155 target->scsi_host->can_queue);
3156
3157 out:
3158 kfree(options);
3159 return ret;
3160 }
3161
3162 static ssize_t srp_create_target(struct device *dev,
3163 struct device_attribute *attr,
3164 const char *buf, size_t count)
3165 {
3166 struct srp_host *host =
3167 container_of(dev, struct srp_host, dev);
3168 struct Scsi_Host *target_host;
3169 struct srp_target_port *target;
3170 struct srp_rdma_ch *ch;
3171 struct srp_device *srp_dev = host->srp_dev;
3172 struct ib_device *ibdev = srp_dev->dev;
3173 int ret, node_idx, node, cpu, i;
3174 bool multich = false;
3175
3176 target_host = scsi_host_alloc(&srp_template,
3177 sizeof (struct srp_target_port));
3178 if (!target_host)
3179 return -ENOMEM;
3180
3181 target_host->transportt = ib_srp_transport_template;
3182 target_host->max_channel = 0;
3183 target_host->max_id = 1;
3184 target_host->max_lun = -1LL;
3185 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
3186
3187 target = host_to_target(target_host);
3188
3189 target->io_class = SRP_REV16A_IB_IO_CLASS;
3190 target->scsi_host = target_host;
3191 target->srp_host = host;
3192 target->lkey = host->srp_dev->pd->local_dma_lkey;
3193 target->global_mr = host->srp_dev->global_mr;
3194 target->cmd_sg_cnt = cmd_sg_entries;
3195 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries;
3196 target->allow_ext_sg = allow_ext_sg;
3197 target->tl_retry_count = 7;
3198 target->queue_size = SRP_DEFAULT_QUEUE_SIZE;
3199
3200 /*
3201 * Avoid that the SCSI host can be removed by srp_remove_target()
3202 * before this function returns.
3203 */
3204 scsi_host_get(target->scsi_host);
3205
3206 mutex_lock(&host->add_target_mutex);
3207
3208 ret = srp_parse_options(buf, target);
3209 if (ret)
3210 goto out;
3211
3212 target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3213
3214 if (!srp_conn_unique(target->srp_host, target)) {
3215 shost_printk(KERN_INFO, target->scsi_host,
3216 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3217 be64_to_cpu(target->id_ext),
3218 be64_to_cpu(target->ioc_guid),
3219 be64_to_cpu(target->initiator_ext));
3220 ret = -EEXIST;
3221 goto out;
3222 }
3223
3224 if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3225 target->cmd_sg_cnt < target->sg_tablesize) {
3226 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3227 target->sg_tablesize = target->cmd_sg_cnt;
3228 }
3229
3230 target_host->sg_tablesize = target->sg_tablesize;
3231 target->mr_pool_size = target->scsi_host->can_queue;
3232 target->indirect_size = target->sg_tablesize *
3233 sizeof (struct srp_direct_buf);
3234 target->max_iu_len = sizeof (struct srp_cmd) +
3235 sizeof (struct srp_indirect_buf) +
3236 target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3237
3238 INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3239 INIT_WORK(&target->remove_work, srp_remove_work);
3240 spin_lock_init(&target->lock);
3241 ret = ib_query_gid(ibdev, host->port, 0, &target->sgid, NULL);
3242 if (ret)
3243 goto out;
3244
3245 ret = -ENOMEM;
3246 target->ch_count = max_t(unsigned, num_online_nodes(),
3247 min(ch_count ? :
3248 min(4 * num_online_nodes(),
3249 ibdev->num_comp_vectors),
3250 num_online_cpus()));
3251 target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3252 GFP_KERNEL);
3253 if (!target->ch)
3254 goto out;
3255
3256 node_idx = 0;
3257 for_each_online_node(node) {
3258 const int ch_start = (node_idx * target->ch_count /
3259 num_online_nodes());
3260 const int ch_end = ((node_idx + 1) * target->ch_count /
3261 num_online_nodes());
3262 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3263 num_online_nodes() + target->comp_vector)
3264 % ibdev->num_comp_vectors;
3265 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3266 num_online_nodes() + target->comp_vector)
3267 % ibdev->num_comp_vectors;
3268 int cpu_idx = 0;
3269
3270 for_each_online_cpu(cpu) {
3271 if (cpu_to_node(cpu) != node)
3272 continue;
3273 if (ch_start + cpu_idx >= ch_end)
3274 continue;
3275 ch = &target->ch[ch_start + cpu_idx];
3276 ch->target = target;
3277 ch->comp_vector = cv_start == cv_end ? cv_start :
3278 cv_start + cpu_idx % (cv_end - cv_start);
3279 spin_lock_init(&ch->lock);
3280 INIT_LIST_HEAD(&ch->free_tx);
3281 ret = srp_new_cm_id(ch);
3282 if (ret)
3283 goto err_disconnect;
3284
3285 ret = srp_create_ch_ib(ch);
3286 if (ret)
3287 goto err_disconnect;
3288
3289 ret = srp_alloc_req_data(ch);
3290 if (ret)
3291 goto err_disconnect;
3292
3293 ret = srp_connect_ch(ch, multich);
3294 if (ret) {
3295 shost_printk(KERN_ERR, target->scsi_host,
3296 PFX "Connection %d/%d failed\n",
3297 ch_start + cpu_idx,
3298 target->ch_count);
3299 if (node_idx == 0 && cpu_idx == 0) {
3300 goto err_disconnect;
3301 } else {
3302 srp_free_ch_ib(target, ch);
3303 srp_free_req_data(target, ch);
3304 target->ch_count = ch - target->ch;
3305 goto connected;
3306 }
3307 }
3308
3309 multich = true;
3310 cpu_idx++;
3311 }
3312 node_idx++;
3313 }
3314
3315 connected:
3316 target->scsi_host->nr_hw_queues = target->ch_count;
3317
3318 ret = srp_add_target(host, target);
3319 if (ret)
3320 goto err_disconnect;
3321
3322 if (target->state != SRP_TARGET_REMOVED) {
3323 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3324 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3325 be64_to_cpu(target->id_ext),
3326 be64_to_cpu(target->ioc_guid),
3327 be16_to_cpu(target->pkey),
3328 be64_to_cpu(target->service_id),
3329 target->sgid.raw, target->orig_dgid.raw);
3330 }
3331
3332 ret = count;
3333
3334 out:
3335 mutex_unlock(&host->add_target_mutex);
3336
3337 scsi_host_put(target->scsi_host);
3338 if (ret < 0)
3339 scsi_host_put(target->scsi_host);
3340
3341 return ret;
3342
3343 err_disconnect:
3344 srp_disconnect_target(target);
3345
3346 for (i = 0; i < target->ch_count; i++) {
3347 ch = &target->ch[i];
3348 srp_free_ch_ib(target, ch);
3349 srp_free_req_data(target, ch);
3350 }
3351
3352 kfree(target->ch);
3353 goto out;
3354 }
3355
3356 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
3357
3358 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3359 char *buf)
3360 {
3361 struct srp_host *host = container_of(dev, struct srp_host, dev);
3362
3363 return sprintf(buf, "%s\n", host->srp_dev->dev->name);
3364 }
3365
3366 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
3367
3368 static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3369 char *buf)
3370 {
3371 struct srp_host *host = container_of(dev, struct srp_host, dev);
3372
3373 return sprintf(buf, "%d\n", host->port);
3374 }
3375
3376 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
3377
3378 static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
3379 {
3380 struct srp_host *host;
3381
3382 host = kzalloc(sizeof *host, GFP_KERNEL);
3383 if (!host)
3384 return NULL;
3385
3386 INIT_LIST_HEAD(&host->target_list);
3387 spin_lock_init(&host->target_lock);
3388 init_completion(&host->released);
3389 mutex_init(&host->add_target_mutex);
3390 host->srp_dev = device;
3391 host->port = port;
3392
3393 host->dev.class = &srp_class;
3394 host->dev.parent = device->dev->dma_device;
3395 dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
3396
3397 if (device_register(&host->dev))
3398 goto free_host;
3399 if (device_create_file(&host->dev, &dev_attr_add_target))
3400 goto err_class;
3401 if (device_create_file(&host->dev, &dev_attr_ibdev))
3402 goto err_class;
3403 if (device_create_file(&host->dev, &dev_attr_port))
3404 goto err_class;
3405
3406 return host;
3407
3408 err_class:
3409 device_unregister(&host->dev);
3410
3411 free_host:
3412 kfree(host);
3413
3414 return NULL;
3415 }
3416
3417 static void srp_add_one(struct ib_device *device)
3418 {
3419 struct srp_device *srp_dev;
3420 struct srp_host *host;
3421 int mr_page_shift, p;
3422 u64 max_pages_per_mr;
3423
3424 srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
3425 if (!srp_dev)
3426 return;
3427
3428 srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3429 device->map_phys_fmr && device->unmap_fmr);
3430 srp_dev->has_fr = (device->attrs.device_cap_flags &
3431 IB_DEVICE_MEM_MGT_EXTENSIONS);
3432 if (!srp_dev->has_fmr && !srp_dev->has_fr)
3433 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3434
3435 srp_dev->use_fast_reg = (srp_dev->has_fr &&
3436 (!srp_dev->has_fmr || prefer_fr));
3437 srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
3438
3439 /*
3440 * Use the smallest page size supported by the HCA, down to a
3441 * minimum of 4096 bytes. We're unlikely to build large sglists
3442 * out of smaller entries.
3443 */
3444 mr_page_shift = max(12, ffs(device->attrs.page_size_cap) - 1);
3445 srp_dev->mr_page_size = 1 << mr_page_shift;
3446 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1);
3447 max_pages_per_mr = device->attrs.max_mr_size;
3448 do_div(max_pages_per_mr, srp_dev->mr_page_size);
3449 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3450 max_pages_per_mr);
3451 if (srp_dev->use_fast_reg) {
3452 srp_dev->max_pages_per_mr =
3453 min_t(u32, srp_dev->max_pages_per_mr,
3454 device->attrs.max_fast_reg_page_list_len);
3455 }
3456 srp_dev->mr_max_size = srp_dev->mr_page_size *
3457 srp_dev->max_pages_per_mr;
3458 pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
3459 device->name, mr_page_shift, device->attrs.max_mr_size,
3460 device->attrs.max_fast_reg_page_list_len,
3461 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
3462
3463 INIT_LIST_HEAD(&srp_dev->dev_list);
3464
3465 srp_dev->dev = device;
3466 srp_dev->pd = ib_alloc_pd(device);
3467 if (IS_ERR(srp_dev->pd))
3468 goto free_dev;
3469
3470 if (!register_always || (!srp_dev->has_fmr && !srp_dev->has_fr)) {
3471 srp_dev->global_mr = ib_get_dma_mr(srp_dev->pd,
3472 IB_ACCESS_LOCAL_WRITE |
3473 IB_ACCESS_REMOTE_READ |
3474 IB_ACCESS_REMOTE_WRITE);
3475 if (IS_ERR(srp_dev->global_mr))
3476 goto err_pd;
3477 } else {
3478 srp_dev->global_mr = NULL;
3479 }
3480
3481 for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
3482 host = srp_add_port(srp_dev, p);
3483 if (host)
3484 list_add_tail(&host->list, &srp_dev->dev_list);
3485 }
3486
3487 ib_set_client_data(device, &srp_client, srp_dev);
3488 return;
3489
3490 err_pd:
3491 ib_dealloc_pd(srp_dev->pd);
3492
3493 free_dev:
3494 kfree(srp_dev);
3495 }
3496
3497 static void srp_remove_one(struct ib_device *device, void *client_data)
3498 {
3499 struct srp_device *srp_dev;
3500 struct srp_host *host, *tmp_host;
3501 struct srp_target_port *target;
3502
3503 srp_dev = client_data;
3504 if (!srp_dev)
3505 return;
3506
3507 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
3508 device_unregister(&host->dev);
3509 /*
3510 * Wait for the sysfs entry to go away, so that no new
3511 * target ports can be created.
3512 */
3513 wait_for_completion(&host->released);
3514
3515 /*
3516 * Remove all target ports.
3517 */
3518 spin_lock(&host->target_lock);
3519 list_for_each_entry(target, &host->target_list, list)
3520 srp_queue_remove_work(target);
3521 spin_unlock(&host->target_lock);
3522
3523 /*
3524 * Wait for tl_err and target port removal tasks.
3525 */
3526 flush_workqueue(system_long_wq);
3527 flush_workqueue(srp_remove_wq);
3528
3529 kfree(host);
3530 }
3531
3532 if (srp_dev->global_mr)
3533 ib_dereg_mr(srp_dev->global_mr);
3534 ib_dealloc_pd(srp_dev->pd);
3535
3536 kfree(srp_dev);
3537 }
3538
3539 static struct srp_function_template ib_srp_transport_functions = {
3540 .has_rport_state = true,
3541 .reset_timer_if_blocked = true,
3542 .reconnect_delay = &srp_reconnect_delay,
3543 .fast_io_fail_tmo = &srp_fast_io_fail_tmo,
3544 .dev_loss_tmo = &srp_dev_loss_tmo,
3545 .reconnect = srp_rport_reconnect,
3546 .rport_delete = srp_rport_delete,
3547 .terminate_rport_io = srp_terminate_io,
3548 };
3549
3550 static int __init srp_init_module(void)
3551 {
3552 int ret;
3553
3554 if (srp_sg_tablesize) {
3555 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
3556 if (!cmd_sg_entries)
3557 cmd_sg_entries = srp_sg_tablesize;
3558 }
3559
3560 if (!cmd_sg_entries)
3561 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3562
3563 if (cmd_sg_entries > 255) {
3564 pr_warn("Clamping cmd_sg_entries to 255\n");
3565 cmd_sg_entries = 255;
3566 }
3567
3568 if (!indirect_sg_entries)
3569 indirect_sg_entries = cmd_sg_entries;
3570 else if (indirect_sg_entries < cmd_sg_entries) {
3571 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3572 cmd_sg_entries);
3573 indirect_sg_entries = cmd_sg_entries;
3574 }
3575
3576 srp_remove_wq = create_workqueue("srp_remove");
3577 if (!srp_remove_wq) {
3578 ret = -ENOMEM;
3579 goto out;
3580 }
3581
3582 ret = -ENOMEM;
3583 ib_srp_transport_template =
3584 srp_attach_transport(&ib_srp_transport_functions);
3585 if (!ib_srp_transport_template)
3586 goto destroy_wq;
3587
3588 ret = class_register(&srp_class);
3589 if (ret) {
3590 pr_err("couldn't register class infiniband_srp\n");
3591 goto release_tr;
3592 }
3593
3594 ib_sa_register_client(&srp_sa_client);
3595
3596 ret = ib_register_client(&srp_client);
3597 if (ret) {
3598 pr_err("couldn't register IB client\n");
3599 goto unreg_sa;
3600 }
3601
3602 out:
3603 return ret;
3604
3605 unreg_sa:
3606 ib_sa_unregister_client(&srp_sa_client);
3607 class_unregister(&srp_class);
3608
3609 release_tr:
3610 srp_release_transport(ib_srp_transport_template);
3611
3612 destroy_wq:
3613 destroy_workqueue(srp_remove_wq);
3614 goto out;
3615 }
3616
3617 static void __exit srp_cleanup_module(void)
3618 {
3619 ib_unregister_client(&srp_client);
3620 ib_sa_unregister_client(&srp_sa_client);
3621 class_unregister(&srp_class);
3622 srp_release_transport(ib_srp_transport_template);
3623 destroy_workqueue(srp_remove_wq);
3624 }
3625
3626 module_init(srp_init_module);
3627 module_exit(srp_cleanup_module);
Linux kernel - Deliverables
This page took 0.137862 seconds and 6 git commands to generate.