projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/srp: Fix srp_map_data() error paths
[deliverable/linux.git] / drivers / infiniband / ulp / srp / ib_srp.c
CommitLineData
aef9ec39
RD		 1/*
2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
aef9ec39
RD		 31 */
32
d236cd0e 33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
e0bda7d8 34
aef9ec39
RD		 35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/err.h>
39#include <linux/string.h>
40#include <linux/parser.h>
41#include <linux/random.h>
de25968c 42#include <linux/jiffies.h>
56b5390c 43#include <rdma/ib_cache.h>
aef9ec39 44
60063497 45#include <linux/atomic.h>
aef9ec39
RD		 46
47#include <scsi/scsi.h>
48#include <scsi/scsi_device.h>
49#include <scsi/scsi_dbg.h>
71444b97 50#include <scsi/scsi_tcq.h>
aef9ec39 51#include <scsi/srp.h>
3236822b 52#include <scsi/scsi_transport_srp.h>
aef9ec39 53
aef9ec39
RD		 54#include "ib_srp.h"
55
56#define DRV_NAME "ib_srp"
57#define PFX DRV_NAME ": "
713ef24e
BVA		 58#define DRV_VERSION "2.0"
59#define DRV_RELDATE "July 26, 2015"
aef9ec39
RD		 60
61MODULE_AUTHOR("Roland Dreier");
33ab3e5b 62MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
aef9ec39 63MODULE_LICENSE("Dual BSD/GPL");
33ab3e5b
BVA		 64MODULE_VERSION(DRV_VERSION);
65MODULE_INFO(release_date, DRV_RELDATE);
aef9ec39 66
49248644
DD		 67static unsigned int srp_sg_tablesize;
68static unsigned int cmd_sg_entries;
c07d424d
DD		 69static unsigned int indirect_sg_entries;
70static bool allow_ext_sg;
03f6fb93
BVA		 71static bool prefer_fr = true;
72static bool register_always = true;
49248644 73static int topspin_workarounds = 1;
74b0a15b 74
49248644
DD		 75module_param(srp_sg_tablesize, uint, 0444);
76MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
74b0a15b 77
49248644
DD		 78module_param(cmd_sg_entries, uint, 0444);
79MODULE_PARM_DESC(cmd_sg_entries,
80 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
aef9ec39 81
c07d424d
DD		 82module_param(indirect_sg_entries, uint, 0444);
83MODULE_PARM_DESC(indirect_sg_entries,
84 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");
85
86module_param(allow_ext_sg, bool, 0444);
87MODULE_PARM_DESC(allow_ext_sg,
88 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
89
aef9ec39
RD		 90module_param(topspin_workarounds, int, 0444);
91MODULE_PARM_DESC(topspin_workarounds,
92 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
93
5cfb1782
BVA		 94module_param(prefer_fr, bool, 0444);
95MODULE_PARM_DESC(prefer_fr,
96"Whether to use fast registration if both FMR and fast registration are supported");
97
b1b8854d
BVA		 98module_param(register_always, bool, 0444);
99MODULE_PARM_DESC(register_always,
100 "Use memory registration even for contiguous memory regions");
101
9c27847d 102static const struct kernel_param_ops srp_tmo_ops;
ed9b2264 103
a95cadb9
BVA		 104static int srp_reconnect_delay = 10;
105module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
106 S_IRUGO | S_IWUSR);
107MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
108
ed9b2264
BVA		 109static int srp_fast_io_fail_tmo = 15;
110module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
111 S_IRUGO | S_IWUSR);
112MODULE_PARM_DESC(fast_io_fail_tmo,
113 "Number of seconds between the observation of a transport"
114 " layer error and failing all I/O. \"off\" means that this"
115 " functionality is disabled.");
116
a95cadb9 117static int srp_dev_loss_tmo = 600;
ed9b2264
BVA		 118module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
119 S_IRUGO | S_IWUSR);
120MODULE_PARM_DESC(dev_loss_tmo,
121 "Maximum number of seconds that the SRP transport should"
122 " insulate transport layer errors. After this time has been"
123 " exceeded the SCSI host is removed. Should be"
124 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
125 " if fast_io_fail_tmo has not been set. \"off\" means that"
126 " this functionality is disabled.");
127
d92c0da7
BVA		 128static unsigned ch_count;
129module_param(ch_count, uint, 0444);
130MODULE_PARM_DESC(ch_count,
131 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
132
aef9ec39 133static void srp_add_one(struct ib_device *device);
7c1eb45a 134static void srp_remove_one(struct ib_device *device, void *client_data);
1dc7b1f1
CH		 135static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
136static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
137 const char *opname);
aef9ec39
RD		 138static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
139
3236822b 140static struct scsi_transport_template *ib_srp_transport_template;
bcc05910 141static struct workqueue_struct *srp_remove_wq;
3236822b 142
aef9ec39
RD		 143static struct ib_client srp_client = {
144 .name = "srp",
145 .add = srp_add_one,
146 .remove = srp_remove_one
147};
148
c1a0b23b
MT		 149static struct ib_sa_client srp_sa_client;
150
ed9b2264
BVA		 151static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
152{
153 int tmo = *(int *)kp->arg;
154
155 if (tmo >= 0)
156 return sprintf(buffer, "%d", tmo);
157 else
158 return sprintf(buffer, "off");
159}
160
161static int srp_tmo_set(const char *val, const struct kernel_param *kp)
162{
163 int tmo, res;
164
3fdf70ac
SG		 165 res = srp_parse_tmo(&tmo, val);
166 if (res)
167 goto out;
168
a95cadb9
BVA		 169 if (kp->arg == &srp_reconnect_delay)
170 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
171 srp_dev_loss_tmo);
172 else if (kp->arg == &srp_fast_io_fail_tmo)
173 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
ed9b2264 174 else
a95cadb9
BVA		 175 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
176 tmo);
ed9b2264
BVA		 177 if (res)
178 goto out;
179 *(int *)kp->arg = tmo;
180
181out:
182 return res;
183}
184
9c27847d 185static const struct kernel_param_ops srp_tmo_ops = {
ed9b2264
BVA		 186 .get = srp_tmo_get,
187 .set = srp_tmo_set,
188};
189
aef9ec39
RD		 190static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
191{
192 return (struct srp_target_port *) host->hostdata;
193}
194
195static const char *srp_target_info(struct Scsi_Host *host)
196{
197 return host_to_target(host)->target_name;
198}
199
5d7cbfd6
RD		 200static int srp_target_is_topspin(struct srp_target_port *target)
201{
202 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
3d1ff48d 203 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d };
5d7cbfd6
RD		 204
205 return topspin_workarounds &&
3d1ff48d
RK		 206 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
207 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
5d7cbfd6
RD		 208}
209
aef9ec39
RD		 210static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
211 gfp_t gfp_mask,
212 enum dma_data_direction direction)
213{
214 struct srp_iu *iu;
215
216 iu = kmalloc(sizeof *iu, gfp_mask);
217 if (!iu)
218 goto out;
219
220 iu->buf = kzalloc(size, gfp_mask);
221 if (!iu->buf)
222 goto out_free_iu;
223
05321937
GKH		 224 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
225 direction);
226 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
aef9ec39
RD		 227 goto out_free_buf;
228
229 iu->size = size;
230 iu->direction = direction;
231
232 return iu;
233
234out_free_buf:
235 kfree(iu->buf);
236out_free_iu:
237 kfree(iu);
238out:
239 return NULL;
240}
241
242static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
243{
244 if (!iu)
245 return;
246
05321937
GKH		 247 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
248 iu->direction);
aef9ec39
RD		 249 kfree(iu->buf);
250 kfree(iu);
251}
252
253static void srp_qp_event(struct ib_event *event, void *context)
254{
57363d98
SG		 255 pr_debug("QP event %s (%d)\n",
256 ib_event_msg(event->event), event->event);
aef9ec39
RD		 257}
258
259static int srp_init_qp(struct srp_target_port *target,
260 struct ib_qp *qp)
261{
262 struct ib_qp_attr *attr;
263 int ret;
264
265 attr = kmalloc(sizeof *attr, GFP_KERNEL);
266 if (!attr)
267 return -ENOMEM;
268
56b5390c
BVA		 269 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
270 target->srp_host->port,
271 be16_to_cpu(target->pkey),
272 &attr->pkey_index);
aef9ec39
RD		 273 if (ret)
274 goto out;
275
276 attr->qp_state = IB_QPS_INIT;
277 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
278 IB_ACCESS_REMOTE_WRITE);
279 attr->port_num = target->srp_host->port;
280
281 ret = ib_modify_qp(qp, attr,
282 IB_QP_STATE |
283 IB_QP_PKEY_INDEX |
284 IB_QP_ACCESS_FLAGS |
285 IB_QP_PORT);
286
287out:
288 kfree(attr);
289 return ret;
290}
291
509c07bc 292static int srp_new_cm_id(struct srp_rdma_ch *ch)
9fe4bcf4 293{
509c07bc 294 struct srp_target_port *target = ch->target;
9fe4bcf4
DD		 295 struct ib_cm_id *new_cm_id;
296
05321937 297 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
509c07bc 298 srp_cm_handler, ch);
9fe4bcf4
DD		 299 if (IS_ERR(new_cm_id))
300 return PTR_ERR(new_cm_id);
301
509c07bc
BVA		 302 if (ch->cm_id)
303 ib_destroy_cm_id(ch->cm_id);
304 ch->cm_id = new_cm_id;
305 ch->path.sgid = target->sgid;
306 ch->path.dgid = target->orig_dgid;
307 ch->path.pkey = target->pkey;
308 ch->path.service_id = target->service_id;
9fe4bcf4
DD		 309
310 return 0;
311}
312
d1b4289e
BVA		 313static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
314{
315 struct srp_device *dev = target->srp_host->srp_dev;
316 struct ib_fmr_pool_param fmr_param;
317
318 memset(&fmr_param, 0, sizeof(fmr_param));
319 fmr_param.pool_size = target->scsi_host->can_queue;
320 fmr_param.dirty_watermark = fmr_param.pool_size / 4;
321 fmr_param.cache = 1;
52ede08f
BVA		 322 fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
323 fmr_param.page_shift = ilog2(dev->mr_page_size);
d1b4289e
BVA		 324 fmr_param.access = (IB_ACCESS_LOCAL_WRITE |
325 IB_ACCESS_REMOTE_WRITE |
326 IB_ACCESS_REMOTE_READ);
327
328 return ib_create_fmr_pool(dev->pd, &fmr_param);
329}
330
5cfb1782
BVA		 331/**
332 * srp_destroy_fr_pool() - free the resources owned by a pool
333 * @pool: Fast registration pool to be destroyed.
334 */
335static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
336{
337 int i;
338 struct srp_fr_desc *d;
339
340 if (!pool)
341 return;
342
343 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
5cfb1782
BVA		 344 if (d->mr)
345 ib_dereg_mr(d->mr);
346 }
347 kfree(pool);
348}
349
350/**
351 * srp_create_fr_pool() - allocate and initialize a pool for fast registration
352 * @device: IB device to allocate fast registration descriptors for.
353 * @pd: Protection domain associated with the FR descriptors.
354 * @pool_size: Number of descriptors to allocate.
355 * @max_page_list_len: Maximum fast registration work request page list length.
356 */
357static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
358 struct ib_pd *pd, int pool_size,
359 int max_page_list_len)
360{
361 struct srp_fr_pool *pool;
362 struct srp_fr_desc *d;
363 struct ib_mr *mr;
5cfb1782
BVA		 364 int i, ret = -EINVAL;
365
366 if (pool_size <= 0)
367 goto err;
368 ret = -ENOMEM;
369 pool = kzalloc(sizeof(struct srp_fr_pool) +
370 pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
371 if (!pool)
372 goto err;
373 pool->size = pool_size;
374 pool->max_page_list_len = max_page_list_len;
375 spin_lock_init(&pool->lock);
376 INIT_LIST_HEAD(&pool->free_list);
377
378 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
563b67c5
SG		 379 mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
380 max_page_list_len);
5cfb1782
BVA		 381 if (IS_ERR(mr)) {
382 ret = PTR_ERR(mr);
383 goto destroy_pool;
384 }
385 d->mr = mr;
5cfb1782
BVA		 386 list_add_tail(&d->entry, &pool->free_list);
387 }
388
389out:
390 return pool;
391
392destroy_pool:
393 srp_destroy_fr_pool(pool);
394
395err:
396 pool = ERR_PTR(ret);
397 goto out;
398}
399
400/**
401 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
402 * @pool: Pool to obtain descriptor from.
403 */
404static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
405{
406 struct srp_fr_desc *d = NULL;
407 unsigned long flags;
408
409 spin_lock_irqsave(&pool->lock, flags);
410 if (!list_empty(&pool->free_list)) {
411 d = list_first_entry(&pool->free_list, typeof(*d), entry);
412 list_del(&d->entry);
413 }
414 spin_unlock_irqrestore(&pool->lock, flags);
415
416 return d;
417}
418
419/**
420 * srp_fr_pool_put() - put an FR descriptor back in the free list
421 * @pool: Pool the descriptor was allocated from.
422 * @desc: Pointer to an array of fast registration descriptor pointers.
423 * @n: Number of descriptors to put back.
424 *
425 * Note: The caller must already have queued an invalidation request for
426 * desc->mr->rkey before calling this function.
427 */
428static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
429 int n)
430{
431 unsigned long flags;
432 int i;
433
434 spin_lock_irqsave(&pool->lock, flags);
435 for (i = 0; i < n; i++)
436 list_add(&desc[i]->entry, &pool->free_list);
437 spin_unlock_irqrestore(&pool->lock, flags);
438}
439
440static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
441{
442 struct srp_device *dev = target->srp_host->srp_dev;
443
444 return srp_create_fr_pool(dev->dev, dev->pd,
445 target->scsi_host->can_queue,
446 dev->max_pages_per_mr);
447}
448
7dad6b2e
BVA		 449/**
450 * srp_destroy_qp() - destroy an RDMA queue pair
451 * @ch: SRP RDMA channel.
452 *
561392d4
SW		 453 * Drain the qp before destroying it. This avoids that the receive
454 * completion handler can access the queue pair while it is
7dad6b2e
BVA		 455 * being destroyed.
456 */
457static void srp_destroy_qp(struct srp_rdma_ch *ch)
458{
561392d4 459 ib_drain_rq(ch->qp);
7dad6b2e
BVA		 460 ib_destroy_qp(ch->qp);
461}
462
509c07bc 463static int srp_create_ch_ib(struct srp_rdma_ch *ch)
aef9ec39 464{
509c07bc 465 struct srp_target_port *target = ch->target;
62154b2e 466 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39 467 struct ib_qp_init_attr *init_attr;
73aa89ed
IR		 468 struct ib_cq *recv_cq, *send_cq;
469 struct ib_qp *qp;
d1b4289e 470 struct ib_fmr_pool *fmr_pool = NULL;
5cfb1782 471 struct srp_fr_pool *fr_pool = NULL;
09c0c0be 472 const int m = dev->use_fast_reg ? 3 : 1;
aef9ec39
RD		 473 int ret;
474
475 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
476 if (!init_attr)
477 return -ENOMEM;
478
561392d4 479 /* queue_size + 1 for ib_drain_rq() */
1dc7b1f1
CH		 480 recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1,
481 ch->comp_vector, IB_POLL_SOFTIRQ);
73aa89ed
IR		 482 if (IS_ERR(recv_cq)) {
483 ret = PTR_ERR(recv_cq);
da9d2f07 484 goto err;
aef9ec39
RD		 485 }
486
1dc7b1f1
CH		 487 send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size,
488 ch->comp_vector, IB_POLL_DIRECT);
73aa89ed
IR		 489 if (IS_ERR(send_cq)) {
490 ret = PTR_ERR(send_cq);
da9d2f07 491 goto err_recv_cq;
9c03dc9f
BVA		 492 }
493
aef9ec39 494 init_attr->event_handler = srp_qp_event;
5cfb1782 495 init_attr->cap.max_send_wr = m * target->queue_size;
7dad6b2e 496 init_attr->cap.max_recv_wr = target->queue_size + 1;
aef9ec39
RD		 497 init_attr->cap.max_recv_sge = 1;
498 init_attr->cap.max_send_sge = 1;
5cfb1782 499 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
aef9ec39 500 init_attr->qp_type = IB_QPT_RC;
73aa89ed
IR		 501 init_attr->send_cq = send_cq;
502 init_attr->recv_cq = recv_cq;
aef9ec39 503
62154b2e 504 qp = ib_create_qp(dev->pd, init_attr);
73aa89ed
IR		 505 if (IS_ERR(qp)) {
506 ret = PTR_ERR(qp);
da9d2f07 507 goto err_send_cq;
aef9ec39
RD		 508 }
509
73aa89ed 510 ret = srp_init_qp(target, qp);
da9d2f07
RD		 511 if (ret)
512 goto err_qp;
aef9ec39 513
002f1567 514 if (dev->use_fast_reg) {
5cfb1782
BVA		 515 fr_pool = srp_alloc_fr_pool(target);
516 if (IS_ERR(fr_pool)) {
517 ret = PTR_ERR(fr_pool);
518 shost_printk(KERN_WARNING, target->scsi_host, PFX
519 "FR pool allocation failed (%d)\n", ret);
520 goto err_qp;
521 }
002f1567 522 } else if (dev->use_fmr) {
d1b4289e
BVA		 523 fmr_pool = srp_alloc_fmr_pool(target);
524 if (IS_ERR(fmr_pool)) {
525 ret = PTR_ERR(fmr_pool);
526 shost_printk(KERN_WARNING, target->scsi_host, PFX
527 "FMR pool allocation failed (%d)\n", ret);
528 goto err_qp;
529 }
d1b4289e
BVA		 530 }
531
509c07bc 532 if (ch->qp)
7dad6b2e 533 srp_destroy_qp(ch);
509c07bc 534 if (ch->recv_cq)
1dc7b1f1 535 ib_free_cq(ch->recv_cq);
509c07bc 536 if (ch->send_cq)
1dc7b1f1 537 ib_free_cq(ch->send_cq);
73aa89ed 538
509c07bc
BVA		 539 ch->qp = qp;
540 ch->recv_cq = recv_cq;
541 ch->send_cq = send_cq;
73aa89ed 542
7fbc67df
SG		 543 if (dev->use_fast_reg) {
544 if (ch->fr_pool)
545 srp_destroy_fr_pool(ch->fr_pool);
546 ch->fr_pool = fr_pool;
547 } else if (dev->use_fmr) {
548 if (ch->fmr_pool)
549 ib_destroy_fmr_pool(ch->fmr_pool);
550 ch->fmr_pool = fmr_pool;
551 }
552
da9d2f07
RD		 553 kfree(init_attr);
554 return 0;
555
556err_qp:
1dc7b1f1 557 srp_destroy_qp(ch);
da9d2f07
RD		 558
559err_send_cq:
1dc7b1f1 560 ib_free_cq(send_cq);
da9d2f07
RD		 561
562err_recv_cq:
1dc7b1f1 563 ib_free_cq(recv_cq);
da9d2f07
RD		 564
565err:
aef9ec39
RD		 566 kfree(init_attr);
567 return ret;
568}
569
4d73f95f
BVA		 570/*
571 * Note: this function may be called without srp_alloc_iu_bufs() having been
509c07bc 572 * invoked. Hence the ch->[rt]x_ring checks.
4d73f95f 573 */
509c07bc
BVA		 574static void srp_free_ch_ib(struct srp_target_port *target,
575 struct srp_rdma_ch *ch)
aef9ec39 576{
5cfb1782 577 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39
RD		 578 int i;
579
d92c0da7
BVA		 580 if (!ch->target)
581 return;
582
509c07bc
BVA		 583 if (ch->cm_id) {
584 ib_destroy_cm_id(ch->cm_id);
585 ch->cm_id = NULL;
394c595e
BVA		 586 }
587
d92c0da7
BVA		 588 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
589 if (!ch->qp)
590 return;
591
5cfb1782 592 if (dev->use_fast_reg) {
509c07bc
BVA		 593 if (ch->fr_pool)
594 srp_destroy_fr_pool(ch->fr_pool);
002f1567 595 } else if (dev->use_fmr) {
509c07bc
BVA		 596 if (ch->fmr_pool)
597 ib_destroy_fmr_pool(ch->fmr_pool);
5cfb1782 598 }
1dc7b1f1 599
7dad6b2e 600 srp_destroy_qp(ch);
1dc7b1f1
CH		 601 ib_free_cq(ch->send_cq);
602 ib_free_cq(ch->recv_cq);
aef9ec39 603
d92c0da7
BVA		 604 /*
605 * Avoid that the SCSI error handler tries to use this channel after
606 * it has been freed. The SCSI error handler can namely continue
607 * trying to perform recovery actions after scsi_remove_host()
608 * returned.
609 */
610 ch->target = NULL;
611
509c07bc
BVA		 612 ch->qp = NULL;
613 ch->send_cq = ch->recv_cq = NULL;
73aa89ed 614
509c07bc 615 if (ch->rx_ring) {
4d73f95f 616 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 617 srp_free_iu(target->srp_host, ch->rx_ring[i]);
618 kfree(ch->rx_ring);
619 ch->rx_ring = NULL;
4d73f95f 620 }
509c07bc 621 if (ch->tx_ring) {
4d73f95f 622 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 623 srp_free_iu(target->srp_host, ch->tx_ring[i]);
624 kfree(ch->tx_ring);
625 ch->tx_ring = NULL;
4d73f95f 626 }
aef9ec39
RD		 627}
628
629static void srp_path_rec_completion(int status,
630 struct ib_sa_path_rec *pathrec,
509c07bc 631 void *ch_ptr)
aef9ec39 632{
509c07bc
BVA		 633 struct srp_rdma_ch *ch = ch_ptr;
634 struct srp_target_port *target = ch->target;
aef9ec39 635
509c07bc 636 ch->status = status;
aef9ec39 637 if (status)
7aa54bd7
DD		 638 shost_printk(KERN_ERR, target->scsi_host,
639 PFX "Got failed path rec status %d\n", status);
aef9ec39 640 else
509c07bc
BVA		 641 ch->path = *pathrec;
642 complete(&ch->done);
aef9ec39
RD		 643}
644
509c07bc 645static int srp_lookup_path(struct srp_rdma_ch *ch)
aef9ec39 646{
509c07bc 647 struct srp_target_port *target = ch->target;
a702adce
BVA		 648 int ret;
649
509c07bc
BVA		 650 ch->path.numb_path = 1;
651
652 init_completion(&ch->done);
653
654 ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
655 target->srp_host->srp_dev->dev,
656 target->srp_host->port,
657 &ch->path,
658 IB_SA_PATH_REC_SERVICE_ID |
659 IB_SA_PATH_REC_DGID |
660 IB_SA_PATH_REC_SGID |
661 IB_SA_PATH_REC_NUMB_PATH |
662 IB_SA_PATH_REC_PKEY,
663 SRP_PATH_REC_TIMEOUT_MS,
664 GFP_KERNEL,
665 srp_path_rec_completion,
666 ch, &ch->path_query);
667 if (ch->path_query_id < 0)
668 return ch->path_query_id;
669
670 ret = wait_for_completion_interruptible(&ch->done);
a702adce
BVA		 671 if (ret < 0)
672 return ret;
aef9ec39 673
509c07bc 674 if (ch->status < 0)
7aa54bd7
DD		 675 shost_printk(KERN_WARNING, target->scsi_host,
676 PFX "Path record query failed\n");
aef9ec39 677
509c07bc 678 return ch->status;
aef9ec39
RD		 679}
680
d92c0da7 681static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
aef9ec39 682{
509c07bc 683 struct srp_target_port *target = ch->target;
aef9ec39
RD		 684 struct {
685 struct ib_cm_req_param param;
686 struct srp_login_req priv;
687 } *req = NULL;
688 int status;
689
690 req = kzalloc(sizeof *req, GFP_KERNEL);
691 if (!req)
692 return -ENOMEM;
693
509c07bc 694 req->param.primary_path = &ch->path;
aef9ec39
RD		 695 req->param.alternate_path = NULL;
696 req->param.service_id = target->service_id;
509c07bc
BVA		 697 req->param.qp_num = ch->qp->qp_num;
698 req->param.qp_type = ch->qp->qp_type;
aef9ec39
RD		 699 req->param.private_data = &req->priv;
700 req->param.private_data_len = sizeof req->priv;
701 req->param.flow_control = 1;
702
703 get_random_bytes(&req->param.starting_psn, 4);
704 req->param.starting_psn &= 0xffffff;
705
706 /*
707 * Pick some arbitrary defaults here; we could make these
708 * module parameters if anyone cared about setting them.
709 */
710 req->param.responder_resources = 4;
711 req->param.remote_cm_response_timeout = 20;
712 req->param.local_cm_response_timeout = 20;
7bb312e4 713 req->param.retry_count = target->tl_retry_count;
aef9ec39
RD		 714 req->param.rnr_retry_count = 7;
715 req->param.max_cm_retries = 15;
716
717 req->priv.opcode = SRP_LOGIN_REQ;
718 req->priv.tag = 0;
49248644 719 req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
aef9ec39
RD		 720 req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
721 SRP_BUF_FORMAT_INDIRECT);
d92c0da7
BVA		 722 req->priv.req_flags = (multich ? SRP_MULTICHAN_MULTI :
723 SRP_MULTICHAN_SINGLE);
0c0450db 724 /*
3cd96564 725 * In the published SRP specification (draft rev. 16a), the
0c0450db
R		 726 * port identifier format is 8 bytes of ID extension followed
727 * by 8 bytes of GUID. Older drafts put the two halves in the
728 * opposite order, so that the GUID comes first.
729 *
730 * Targets conforming to these obsolete drafts can be
731 * recognized by the I/O Class they report.
732 */
733 if (target->io_class == SRP_REV10_IB_IO_CLASS) {
734 memcpy(req->priv.initiator_port_id,
747fe000 735 &target->sgid.global.interface_id, 8);
0c0450db 736 memcpy(req->priv.initiator_port_id + 8,
01cb9bcb 737 &target->initiator_ext, 8);
0c0450db
R		 738 memcpy(req->priv.target_port_id, &target->ioc_guid, 8);
739 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
740 } else {
741 memcpy(req->priv.initiator_port_id,
01cb9bcb
IR		 742 &target->initiator_ext, 8);
743 memcpy(req->priv.initiator_port_id + 8,
747fe000 744 &target->sgid.global.interface_id, 8);
0c0450db
R		 745 memcpy(req->priv.target_port_id, &target->id_ext, 8);
746 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
747 }
748
aef9ec39
RD		 749 /*
750 * Topspin/Cisco SRP targets will reject our login unless we
01cb9bcb
IR		 751 * zero out the first 8 bytes of our initiator port ID and set
752 * the second 8 bytes to the local node GUID.
aef9ec39 753 */
5d7cbfd6 754 if (srp_target_is_topspin(target)) {
7aa54bd7
DD		 755 shost_printk(KERN_DEBUG, target->scsi_host,
756 PFX "Topspin/Cisco initiator port ID workaround "
757 "activated for target GUID %016llx\n",
45c37cad 758 be64_to_cpu(target->ioc_guid));
aef9ec39 759 memset(req->priv.initiator_port_id, 0, 8);
01cb9bcb 760 memcpy(req->priv.initiator_port_id + 8,
05321937 761 &target->srp_host->srp_dev->dev->node_guid, 8);
aef9ec39 762 }
aef9ec39 763
509c07bc 764 status = ib_send_cm_req(ch->cm_id, &req->param);
aef9ec39
RD		 765
766 kfree(req);
767
768 return status;
769}
770
ef6c49d8
BVA		 771static bool srp_queue_remove_work(struct srp_target_port *target)
772{
773 bool changed = false;
774
775 spin_lock_irq(&target->lock);
776 if (target->state != SRP_TARGET_REMOVED) {
777 target->state = SRP_TARGET_REMOVED;
778 changed = true;
779 }
780 spin_unlock_irq(&target->lock);
781
782 if (changed)
bcc05910 783 queue_work(srp_remove_wq, &target->remove_work);
ef6c49d8
BVA		 784
785 return changed;
786}
787
aef9ec39
RD		 788static void srp_disconnect_target(struct srp_target_port *target)
789{
d92c0da7
BVA		 790 struct srp_rdma_ch *ch;
791 int i;
509c07bc 792
c014c8cd 793 /* XXX should send SRP_I_LOGOUT request */
aef9ec39 794
c014c8cd
BVA		 795 for (i = 0; i < target->ch_count; i++) {
796 ch = &target->ch[i];
797 ch->connected = false;
798 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
799 shost_printk(KERN_DEBUG, target->scsi_host,
800 PFX "Sending CM DREQ failed\n");
294c875a 801 }
e6581056 802 }
aef9ec39
RD		 803}
804
509c07bc
BVA		 805static void srp_free_req_data(struct srp_target_port *target,
806 struct srp_rdma_ch *ch)
8f26c9ff 807{
5cfb1782
BVA		 808 struct srp_device *dev = target->srp_host->srp_dev;
809 struct ib_device *ibdev = dev->dev;
8f26c9ff
DD		 810 struct srp_request *req;
811 int i;
812
47513cf4 813 if (!ch->req_ring)
4d73f95f
BVA		 814 return;
815
816 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 817 req = &ch->req_ring[i];
9a21be53 818 if (dev->use_fast_reg) {
5cfb1782 819 kfree(req->fr_list);
9a21be53 820 } else {
5cfb1782 821 kfree(req->fmr_list);
9a21be53
SG		 822 kfree(req->map_page);
823 }
c07d424d
DD		 824 if (req->indirect_dma_addr) {
825 ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
826 target->indirect_size,
827 DMA_TO_DEVICE);
828 }
829 kfree(req->indirect_desc);
8f26c9ff 830 }
4d73f95f 831
509c07bc
BVA		 832 kfree(ch->req_ring);
833 ch->req_ring = NULL;
8f26c9ff
DD		 834}
835
509c07bc 836static int srp_alloc_req_data(struct srp_rdma_ch *ch)
b81d00bd 837{
509c07bc 838 struct srp_target_port *target = ch->target;
b81d00bd
BVA		 839 struct srp_device *srp_dev = target->srp_host->srp_dev;
840 struct ib_device *ibdev = srp_dev->dev;
841 struct srp_request *req;
5cfb1782 842 void *mr_list;
b81d00bd
BVA		 843 dma_addr_t dma_addr;
844 int i, ret = -ENOMEM;
845
509c07bc
BVA		 846 ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
847 GFP_KERNEL);
848 if (!ch->req_ring)
4d73f95f
BVA		 849 goto out;
850
851 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 852 req = &ch->req_ring[i];
5cfb1782
BVA		 853 mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
854 GFP_KERNEL);
855 if (!mr_list)
856 goto out;
9a21be53 857 if (srp_dev->use_fast_reg) {
5cfb1782 858 req->fr_list = mr_list;
9a21be53 859 } else {
5cfb1782 860 req->fmr_list = mr_list;
9a21be53
SG		 861 req->map_page = kmalloc(srp_dev->max_pages_per_mr *
862 sizeof(void *), GFP_KERNEL);
863 if (!req->map_page)
864 goto out;
865 }
b81d00bd 866 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
5cfb1782 867 if (!req->indirect_desc)
b81d00bd
BVA		 868 goto out;
869
870 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
871 target->indirect_size,
872 DMA_TO_DEVICE);
873 if (ib_dma_mapping_error(ibdev, dma_addr))
874 goto out;
875
876 req->indirect_dma_addr = dma_addr;
b81d00bd
BVA		 877 }
878 ret = 0;
879
880out:
881 return ret;
882}
883
683b159a
BVA		 884/**
885 * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
886 * @shost: SCSI host whose attributes to remove from sysfs.
887 *
888 * Note: Any attributes defined in the host template and that did not exist
889 * before invocation of this function will be ignored.
890 */
891static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
892{
893 struct device_attribute **attr;
894
895 for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
896 device_remove_file(&shost->shost_dev, *attr);
897}
898
ee12d6a8
BVA		 899static void srp_remove_target(struct srp_target_port *target)
900{
d92c0da7
BVA		 901 struct srp_rdma_ch *ch;
902 int i;
509c07bc 903
ef6c49d8
BVA		 904 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
905
ee12d6a8 906 srp_del_scsi_host_attr(target->scsi_host);
9dd69a60 907 srp_rport_get(target->rport);
ee12d6a8
BVA		 908 srp_remove_host(target->scsi_host);
909 scsi_remove_host(target->scsi_host);
93079162 910 srp_stop_rport_timers(target->rport);
ef6c49d8 911 srp_disconnect_target(target);
d92c0da7
BVA		 912 for (i = 0; i < target->ch_count; i++) {
913 ch = &target->ch[i];
914 srp_free_ch_ib(target, ch);
915 }
c1120f89 916 cancel_work_sync(&target->tl_err_work);
9dd69a60 917 srp_rport_put(target->rport);
d92c0da7
BVA		 918 for (i = 0; i < target->ch_count; i++) {
919 ch = &target->ch[i];
920 srp_free_req_data(target, ch);
921 }
922 kfree(target->ch);
923 target->ch = NULL;
65d7dd2f
VP		 924
925 spin_lock(&target->srp_host->target_lock);
926 list_del(&target->list);
927 spin_unlock(&target->srp_host->target_lock);
928
ee12d6a8
BVA		 929 scsi_host_put(target->scsi_host);
930}
931
c4028958 932static void srp_remove_work(struct work_struct *work)
aef9ec39 933{
c4028958 934 struct srp_target_port *target =
ef6c49d8 935 container_of(work, struct srp_target_port, remove_work);
aef9ec39 936
ef6c49d8 937 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
aef9ec39 938
96fc248a 939 srp_remove_target(target);
aef9ec39
RD		 940}
941
dc1bdbd9
BVA		 942static void srp_rport_delete(struct srp_rport *rport)
943{
944 struct srp_target_port *target = rport->lld_data;
945
946 srp_queue_remove_work(target);
947}
948
c014c8cd
BVA		 949/**
950 * srp_connected_ch() - number of connected channels
951 * @target: SRP target port.
952 */
953static int srp_connected_ch(struct srp_target_port *target)
954{
955 int i, c = 0;
956
957 for (i = 0; i < target->ch_count; i++)
958 c += target->ch[i].connected;
959
960 return c;
961}
962
d92c0da7 963static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
aef9ec39 964{
509c07bc 965 struct srp_target_port *target = ch->target;
aef9ec39
RD		 966 int ret;
967
c014c8cd 968 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
294c875a 969
509c07bc 970 ret = srp_lookup_path(ch);
aef9ec39 971 if (ret)
4d59ad29 972 goto out;
aef9ec39
RD		 973
974 while (1) {
509c07bc 975 init_completion(&ch->done);
d92c0da7 976 ret = srp_send_req(ch, multich);
aef9ec39 977 if (ret)
4d59ad29 978 goto out;
509c07bc 979 ret = wait_for_completion_interruptible(&ch->done);
a702adce 980 if (ret < 0)
4d59ad29 981 goto out;
aef9ec39
RD		 982
983 /*
984 * The CM event handling code will set status to
985 * SRP_PORT_REDIRECT if we get a port redirect REJ
986 * back, or SRP_DLID_REDIRECT if we get a lid/qp
987 * redirect REJ back.
988 */
4d59ad29
BVA		 989 ret = ch->status;
990 switch (ret) {
aef9ec39 991 case 0:
c014c8cd 992 ch->connected = true;
4d59ad29 993 goto out;
aef9ec39
RD		 994
995 case SRP_PORT_REDIRECT:
509c07bc 996 ret = srp_lookup_path(ch);
aef9ec39 997 if (ret)
4d59ad29 998 goto out;
aef9ec39
RD		 999 break;
1000
1001 case SRP_DLID_REDIRECT:
1002 break;
1003
9fe4bcf4 1004 case SRP_STALE_CONN:
9fe4bcf4 1005 shost_printk(KERN_ERR, target->scsi_host, PFX
205619f2 1006 "giving up on stale connection\n");
4d59ad29
BVA		 1007 ret = -ECONNRESET;
1008 goto out;
9fe4bcf4 1009
aef9ec39 1010 default:
4d59ad29 1011 goto out;
aef9ec39
RD		 1012 }
1013 }
4d59ad29
BVA		 1014
1015out:
1016 return ret <= 0 ? ret : -ENODEV;
aef9ec39
RD		 1017}
1018
1dc7b1f1
CH		 1019static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc)
1020{
1021 srp_handle_qp_err(cq, wc, "INV RKEY");
1022}
1023
1024static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch,
1025 u32 rkey)
5cfb1782
BVA		 1026{
1027 struct ib_send_wr *bad_wr;
1028 struct ib_send_wr wr = {
1029 .opcode = IB_WR_LOCAL_INV,
5cfb1782
BVA		 1030 .next = NULL,
1031 .num_sge = 0,
1032 .send_flags = 0,
1033 .ex.invalidate_rkey = rkey,
1034 };
1035
1dc7b1f1
CH		 1036 wr.wr_cqe = &req->reg_cqe;
1037 req->reg_cqe.done = srp_inv_rkey_err_done;
509c07bc 1038 return ib_post_send(ch->qp, &wr, &bad_wr);
5cfb1782
BVA		 1039}
1040
d945e1df 1041static void srp_unmap_data(struct scsi_cmnd *scmnd,
509c07bc 1042 struct srp_rdma_ch *ch,
d945e1df
RD		 1043 struct srp_request *req)
1044{
509c07bc 1045 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1046 struct srp_device *dev = target->srp_host->srp_dev;
1047 struct ib_device *ibdev = dev->dev;
1048 int i, res;
8f26c9ff 1049
bb350d1d 1050 if (!scsi_sglist(scmnd) ||
d945e1df
RD		 1051 (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1052 scmnd->sc_data_direction != DMA_FROM_DEVICE))
1053 return;
1054
5cfb1782
BVA		 1055 if (dev->use_fast_reg) {
1056 struct srp_fr_desc **pfr;
1057
1058 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
1dc7b1f1 1059 res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey);
5cfb1782
BVA		 1060 if (res < 0) {
1061 shost_printk(KERN_ERR, target->scsi_host, PFX
1062 "Queueing INV WR for rkey %#x failed (%d)\n",
1063 (*pfr)->mr->rkey, res);
1064 queue_work(system_long_wq,
1065 &target->tl_err_work);
1066 }
1067 }
1068 if (req->nmdesc)
509c07bc 1069 srp_fr_pool_put(ch->fr_pool, req->fr_list,
5cfb1782 1070 req->nmdesc);
002f1567 1071 } else if (dev->use_fmr) {
5cfb1782
BVA		 1072 struct ib_pool_fmr **pfmr;
1073
1074 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1075 ib_fmr_pool_unmap(*pfmr);
1076 }
f5358a17 1077
8f26c9ff
DD		 1078 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1079 scmnd->sc_data_direction);
d945e1df
RD		 1080}
1081
22032991
BVA		 1082/**
1083 * srp_claim_req - Take ownership of the scmnd associated with a request.
509c07bc 1084 * @ch: SRP RDMA channel.
22032991 1085 * @req: SRP request.
b3fe628d 1086 * @sdev: If not NULL, only take ownership for this SCSI device.
22032991
BVA		 1087 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1088 * ownership of @req->scmnd if it equals @scmnd.
1089 *
1090 * Return value:
1091 * Either NULL or a pointer to the SCSI command the caller became owner of.
1092 */
509c07bc 1093static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
22032991 1094 struct srp_request *req,
b3fe628d 1095 struct scsi_device *sdev,
22032991
BVA		 1096 struct scsi_cmnd *scmnd)
1097{
1098 unsigned long flags;
1099
509c07bc 1100 spin_lock_irqsave(&ch->lock, flags);
b3fe628d
BVA		 1101 if (req->scmnd &&
1102 (!sdev || req->scmnd->device == sdev) &&
1103 (!scmnd || req->scmnd == scmnd)) {
22032991
BVA		 1104 scmnd = req->scmnd;
1105 req->scmnd = NULL;
22032991
BVA		 1106 } else {
1107 scmnd = NULL;
1108 }
509c07bc 1109 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1110
1111 return scmnd;
1112}
1113
1114/**
6ec2ba02 1115 * srp_free_req() - Unmap data and adjust ch->req_lim.
509c07bc 1116 * @ch: SRP RDMA channel.
af24663b
BVA		 1117 * @req: Request to be freed.
1118 * @scmnd: SCSI command associated with @req.
1119 * @req_lim_delta: Amount to be added to @target->req_lim.
22032991 1120 */
509c07bc
BVA		 1121static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1122 struct scsi_cmnd *scmnd, s32 req_lim_delta)
526b4caa 1123{
94a9174c
BVA		 1124 unsigned long flags;
1125
509c07bc 1126 srp_unmap_data(scmnd, ch, req);
22032991 1127
509c07bc
BVA		 1128 spin_lock_irqsave(&ch->lock, flags);
1129 ch->req_lim += req_lim_delta;
509c07bc 1130 spin_unlock_irqrestore(&ch->lock, flags);
526b4caa
IR		 1131}
1132
509c07bc
BVA		 1133static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1134 struct scsi_device *sdev, int result)
526b4caa 1135{
509c07bc 1136 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
22032991
BVA		 1137
1138 if (scmnd) {
509c07bc 1139 srp_free_req(ch, req, scmnd, 0);
ed9b2264 1140 scmnd->result = result;
22032991 1141 scmnd->scsi_done(scmnd);
22032991 1142 }
526b4caa
IR		 1143}
1144
ed9b2264 1145static void srp_terminate_io(struct srp_rport *rport)
aef9ec39 1146{
ed9b2264 1147 struct srp_target_port *target = rport->lld_data;
d92c0da7 1148 struct srp_rdma_ch *ch;
b3fe628d
BVA		 1149 struct Scsi_Host *shost = target->scsi_host;
1150 struct scsi_device *sdev;
d92c0da7 1151 int i, j;
ed9b2264 1152
b3fe628d
BVA		 1153 /*
1154 * Invoking srp_terminate_io() while srp_queuecommand() is running
1155 * is not safe. Hence the warning statement below.
1156 */
1157 shost_for_each_device(sdev, shost)
1158 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1159
d92c0da7
BVA		 1160 for (i = 0; i < target->ch_count; i++) {
1161 ch = &target->ch[i];
509c07bc 1162
d92c0da7
BVA		 1163 for (j = 0; j < target->req_ring_size; ++j) {
1164 struct srp_request *req = &ch->req_ring[j];
1165
1166 srp_finish_req(ch, req, NULL,
1167 DID_TRANSPORT_FAILFAST << 16);
1168 }
ed9b2264
BVA		 1169 }
1170}
aef9ec39 1171
ed9b2264
BVA		 1172/*
1173 * It is up to the caller to ensure that srp_rport_reconnect() calls are
1174 * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1175 * srp_reset_device() or srp_reset_host() calls will occur while this function
1176 * is in progress. One way to realize that is not to call this function
1177 * directly but to call srp_reconnect_rport() instead since that last function
1178 * serializes calls of this function via rport->mutex and also blocks
1179 * srp_queuecommand() calls before invoking this function.
1180 */
1181static int srp_rport_reconnect(struct srp_rport *rport)
1182{
1183 struct srp_target_port *target = rport->lld_data;
d92c0da7
BVA		 1184 struct srp_rdma_ch *ch;
1185 int i, j, ret = 0;
1186 bool multich = false;
09be70a2 1187
aef9ec39 1188 srp_disconnect_target(target);
34aa654e
BVA		 1189
1190 if (target->state == SRP_TARGET_SCANNING)
1191 return -ENODEV;
1192
aef9ec39 1193 /*
c7c4e7ff
BVA		 1194 * Now get a new local CM ID so that we avoid confusing the target in
1195 * case things are really fouled up. Doing so also ensures that all CM
1196 * callbacks will have finished before a new QP is allocated.
aef9ec39 1197 */
d92c0da7
BVA		 1198 for (i = 0; i < target->ch_count; i++) {
1199 ch = &target->ch[i];
d92c0da7 1200 ret += srp_new_cm_id(ch);
536ae14e 1201 }
d92c0da7
BVA		 1202 for (i = 0; i < target->ch_count; i++) {
1203 ch = &target->ch[i];
d92c0da7
BVA		 1204 for (j = 0; j < target->req_ring_size; ++j) {
1205 struct srp_request *req = &ch->req_ring[j];
aef9ec39 1206
d92c0da7
BVA		 1207 srp_finish_req(ch, req, NULL, DID_RESET << 16);
1208 }
1209 }
1210 for (i = 0; i < target->ch_count; i++) {
1211 ch = &target->ch[i];
d92c0da7
BVA		 1212 /*
1213 * Whether or not creating a new CM ID succeeded, create a new
1214 * QP. This guarantees that all completion callback function
1215 * invocations have finished before request resetting starts.
1216 */
1217 ret += srp_create_ch_ib(ch);
aef9ec39 1218
d92c0da7
BVA		 1219 INIT_LIST_HEAD(&ch->free_tx);
1220 for (j = 0; j < target->queue_size; ++j)
1221 list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1222 }
8de9fe3a
BVA		 1223
1224 target->qp_in_error = false;
1225
d92c0da7
BVA		 1226 for (i = 0; i < target->ch_count; i++) {
1227 ch = &target->ch[i];
bbac5ccf 1228 if (ret)
d92c0da7 1229 break;
d92c0da7
BVA		 1230 ret = srp_connect_ch(ch, multich);
1231 multich = true;
1232 }
09be70a2 1233
ed9b2264
BVA		 1234 if (ret == 0)
1235 shost_printk(KERN_INFO, target->scsi_host,
1236 PFX "reconnect succeeded\n");
aef9ec39
RD		 1237
1238 return ret;
1239}
1240
8f26c9ff
DD		 1241static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1242 unsigned int dma_len, u32 rkey)
f5358a17 1243{
8f26c9ff 1244 struct srp_direct_buf *desc = state->desc;
f5358a17 1245
3ae95da8
BVA		 1246 WARN_ON_ONCE(!dma_len);
1247
8f26c9ff
DD		 1248 desc->va = cpu_to_be64(dma_addr);
1249 desc->key = cpu_to_be32(rkey);
1250 desc->len = cpu_to_be32(dma_len);
f5358a17 1251
8f26c9ff
DD		 1252 state->total_len += dma_len;
1253 state->desc++;
1254 state->ndesc++;
1255}
559ce8f1 1256
8f26c9ff 1257static int srp_map_finish_fmr(struct srp_map_state *state,
509c07bc 1258 struct srp_rdma_ch *ch)
8f26c9ff 1259{
186fbc66
BVA		 1260 struct srp_target_port *target = ch->target;
1261 struct srp_device *dev = target->srp_host->srp_dev;
8f26c9ff
DD		 1262 struct ib_pool_fmr *fmr;
1263 u64 io_addr = 0;
85507bcc 1264
f731ed62
BVA		 1265 if (state->fmr.next >= state->fmr.end)
1266 return -ENOMEM;
1267
26630e8a
SG		 1268 WARN_ON_ONCE(!dev->use_fmr);
1269
1270 if (state->npages == 0)
1271 return 0;
1272
1273 if (state->npages == 1 && target->global_mr) {
1274 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1275 target->global_mr->rkey);
1276 goto reset_state;
1277 }
1278
509c07bc 1279 fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
8f26c9ff
DD		 1280 state->npages, io_addr);
1281 if (IS_ERR(fmr))
1282 return PTR_ERR(fmr);
f5358a17 1283
f731ed62 1284 *state->fmr.next++ = fmr;
52ede08f 1285 state->nmdesc++;
f5358a17 1286
186fbc66
BVA		 1287 srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
1288 state->dma_len, fmr->fmr->rkey);
539dde6f 1289
26630e8a
SG		 1290reset_state:
1291 state->npages = 0;
1292 state->dma_len = 0;
1293
8f26c9ff
DD		 1294 return 0;
1295}
1296
1dc7b1f1
CH		 1297static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
1298{
1299 srp_handle_qp_err(cq, wc, "FAST REG");
1300}
1301
5cfb1782 1302static int srp_map_finish_fr(struct srp_map_state *state,
1dc7b1f1 1303 struct srp_request *req,
57b0be9c 1304 struct srp_rdma_ch *ch, int sg_nents)
5cfb1782 1305{
509c07bc 1306 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1307 struct srp_device *dev = target->srp_host->srp_dev;
1308 struct ib_send_wr *bad_wr;
f7f7aab1 1309 struct ib_reg_wr wr;
5cfb1782
BVA		 1310 struct srp_fr_desc *desc;
1311 u32 rkey;
f7f7aab1 1312 int n, err;
5cfb1782 1313
f731ed62
BVA		 1314 if (state->fr.next >= state->fr.end)
1315 return -ENOMEM;
1316
26630e8a
SG		 1317 WARN_ON_ONCE(!dev->use_fast_reg);
1318
57b0be9c 1319 if (sg_nents == 0)
26630e8a
SG		 1320 return 0;
1321
57b0be9c 1322 if (sg_nents == 1 && target->global_mr) {
f7f7aab1
SG		 1323 srp_map_desc(state, sg_dma_address(state->sg),
1324 sg_dma_len(state->sg),
26630e8a 1325 target->global_mr->rkey);
f7f7aab1 1326 return 1;
26630e8a
SG		 1327 }
1328
509c07bc 1329 desc = srp_fr_pool_get(ch->fr_pool);
5cfb1782
BVA		 1330 if (!desc)
1331 return -ENOMEM;
1332
1333 rkey = ib_inc_rkey(desc->mr->rkey);
1334 ib_update_fast_reg_key(desc->mr, rkey);
1335
57b0be9c 1336 n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, dev->mr_page_size);
f7f7aab1
SG		 1337 if (unlikely(n < 0))
1338 return n;
5cfb1782 1339
1dc7b1f1
CH		 1340 req->reg_cqe.done = srp_reg_mr_err_done;
1341
f7f7aab1
SG		 1342 wr.wr.next = NULL;
1343 wr.wr.opcode = IB_WR_REG_MR;
1dc7b1f1 1344 wr.wr.wr_cqe = &req->reg_cqe;
f7f7aab1
SG		 1345 wr.wr.num_sge = 0;
1346 wr.wr.send_flags = 0;
1347 wr.mr = desc->mr;
1348 wr.key = desc->mr->rkey;
1349 wr.access = (IB_ACCESS_LOCAL_WRITE |
1350 IB_ACCESS_REMOTE_READ |
1351 IB_ACCESS_REMOTE_WRITE);
5cfb1782 1352
f731ed62 1353 *state->fr.next++ = desc;
5cfb1782
BVA		 1354 state->nmdesc++;
1355
f7f7aab1
SG		 1356 srp_map_desc(state, desc->mr->iova,
1357 desc->mr->length, desc->mr->rkey);
5cfb1782 1358
26630e8a 1359 err = ib_post_send(ch->qp, &wr.wr, &bad_wr);
f7f7aab1 1360 if (unlikely(err))
26630e8a
SG		 1361 return err;
1362
f7f7aab1 1363 return n;
5cfb1782
BVA		 1364}
1365
8f26c9ff 1366static int srp_map_sg_entry(struct srp_map_state *state,
509c07bc 1367 struct srp_rdma_ch *ch,
3ae95da8 1368 struct scatterlist *sg, int sg_index)
8f26c9ff 1369{
509c07bc 1370 struct srp_target_port *target = ch->target;
8f26c9ff
DD		 1371 struct srp_device *dev = target->srp_host->srp_dev;
1372 struct ib_device *ibdev = dev->dev;
1373 dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1374 unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
3ae95da8 1375 unsigned int len = 0;
8f26c9ff
DD		 1376 int ret;
1377
3ae95da8 1378 WARN_ON_ONCE(!dma_len);
f5358a17 1379
8f26c9ff 1380 while (dma_len) {
5cfb1782
BVA		 1381 unsigned offset = dma_addr & ~dev->mr_page_mask;
1382 if (state->npages == dev->max_pages_per_mr || offset != 0) {
f7f7aab1 1383 ret = srp_map_finish_fmr(state, ch);
8f26c9ff
DD		 1384 if (ret)
1385 return ret;
8f26c9ff
DD		 1386 }
1387
5cfb1782 1388 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
f5358a17 1389
8f26c9ff
DD		 1390 if (!state->npages)
1391 state->base_dma_addr = dma_addr;
5cfb1782 1392 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
52ede08f 1393 state->dma_len += len;
8f26c9ff
DD		 1394 dma_addr += len;
1395 dma_len -= len;
1396 }
1397
5cfb1782
BVA		 1398 /*
1399 * If the last entry of the MR wasn't a full page, then we need to
8f26c9ff 1400 * close it out and start a new one -- we can only merge at page
1d3d98c4 1401 * boundaries.
8f26c9ff
DD		 1402 */
1403 ret = 0;
0e0d3a48 1404 if (len != dev->mr_page_size)
f7f7aab1 1405 ret = srp_map_finish_fmr(state, ch);
f5358a17
RD		 1406 return ret;
1407}
1408
26630e8a
SG		 1409static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1410 struct srp_request *req, struct scatterlist *scat,
1411 int count)
76bc1e1d 1412{
76bc1e1d 1413 struct scatterlist *sg;
0e0d3a48 1414 int i, ret;
76bc1e1d 1415
26630e8a
SG		 1416 state->desc = req->indirect_desc;
1417 state->pages = req->map_page;
1418 state->fmr.next = req->fmr_list;
1419 state->fmr.end = req->fmr_list + ch->target->cmd_sg_cnt;
1420
1421 for_each_sg(scat, sg, count, i) {
1422 ret = srp_map_sg_entry(state, ch, sg, i);
1423 if (ret)
1424 return ret;
5cfb1782 1425 }
76bc1e1d 1426
f7f7aab1 1427 ret = srp_map_finish_fmr(state, ch);
26630e8a
SG		 1428 if (ret)
1429 return ret;
1430
26630e8a
SG		 1431 return 0;
1432}
1433
1434static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1435 struct srp_request *req, struct scatterlist *scat,
1436 int count)
1437{
26630e8a 1438 state->desc = req->indirect_desc;
f7f7aab1
SG		 1439 state->fr.next = req->fr_list;
1440 state->fr.end = req->fr_list + ch->target->cmd_sg_cnt;
1441 state->sg = scat;
26630e8a 1442
57b0be9c 1443 while (count) {
f7f7aab1 1444 int i, n;
26630e8a 1445
c6333f9f 1446 n = srp_map_finish_fr(state, req, ch, count);
f7f7aab1
SG		 1447 if (unlikely(n < 0))
1448 return n;
1449
57b0be9c 1450 count -= n;
f7f7aab1
SG		 1451 for (i = 0; i < n; i++)
1452 state->sg = sg_next(state->sg);
1453 }
26630e8a 1454
26630e8a
SG		 1455 return 0;
1456}
1457
1458static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
1459 struct srp_request *req, struct scatterlist *scat,
1460 int count)
1461{
1462 struct srp_target_port *target = ch->target;
1463 struct srp_device *dev = target->srp_host->srp_dev;
1464 struct scatterlist *sg;
1465 int i;
1466
1467 state->desc = req->indirect_desc;
1468 for_each_sg(scat, sg, count, i) {
1469 srp_map_desc(state, ib_sg_dma_address(dev->dev, sg),
1470 ib_sg_dma_len(dev->dev, sg),
1471 target->global_mr->rkey);
0e0d3a48 1472 }
76bc1e1d 1473
26630e8a 1474 return 0;
76bc1e1d
BVA		 1475}
1476
330179f2
BVA		 1477/*
1478 * Register the indirect data buffer descriptor with the HCA.
1479 *
1480 * Note: since the indirect data buffer descriptor has been allocated with
1481 * kmalloc() it is guaranteed that this buffer is a physically contiguous
1482 * memory buffer.
1483 */
1484static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
1485 void **next_mr, void **end_mr, u32 idb_len,
1486 __be32 *idb_rkey)
1487{
1488 struct srp_target_port *target = ch->target;
1489 struct srp_device *dev = target->srp_host->srp_dev;
1490 struct srp_map_state state;
1491 struct srp_direct_buf idb_desc;
1492 u64 idb_pages[1];
f7f7aab1 1493 struct scatterlist idb_sg[1];
330179f2
BVA		 1494 int ret;
1495
1496 memset(&state, 0, sizeof(state));
1497 memset(&idb_desc, 0, sizeof(idb_desc));
1498 state.gen.next = next_mr;
1499 state.gen.end = end_mr;
1500 state.desc = &idb_desc;
330179f2
BVA		 1501 state.base_dma_addr = req->indirect_dma_addr;
1502 state.dma_len = idb_len;
f7f7aab1
SG		 1503
1504 if (dev->use_fast_reg) {
1505 state.sg = idb_sg;
f7f7aab1
SG		 1506 sg_set_buf(idb_sg, req->indirect_desc, idb_len);
1507 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */
fc925518
CH		 1508#ifdef CONFIG_NEED_SG_DMA_LENGTH
1509 idb_sg->dma_length = idb_sg->length; /* hack^2 */
1510#endif
c6333f9f 1511 ret = srp_map_finish_fr(&state, req, ch, 1);
f7f7aab1
SG		 1512 if (ret < 0)
1513 return ret;
1514 } else if (dev->use_fmr) {
1515 state.pages = idb_pages;
1516 state.pages[0] = (req->indirect_dma_addr &
1517 dev->mr_page_mask);
1518 state.npages = 1;
1519 ret = srp_map_finish_fmr(&state, ch);
1520 if (ret < 0)
1521 return ret;
1522 } else {
1523 return -EINVAL;
1524 }
330179f2
BVA		 1525
1526 *idb_rkey = idb_desc.key;
1527
f7f7aab1 1528 return 0;
330179f2
BVA		 1529}
1530
77269cdf
BVA		 1531/**
1532 * srp_map_data() - map SCSI data buffer onto an SRP request
1533 * @scmnd: SCSI command to map
1534 * @ch: SRP RDMA channel
1535 * @req: SRP request
1536 *
1537 * Returns the length in bytes of the SRP_CMD IU or a negative value if
1538 * mapping failed.
1539 */
509c07bc 1540static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
aef9ec39
RD		 1541 struct srp_request *req)
1542{
509c07bc 1543 struct srp_target_port *target = ch->target;
76bc1e1d 1544 struct scatterlist *scat;
aef9ec39 1545 struct srp_cmd *cmd = req->cmd->buf;
330179f2 1546 int len, nents, count, ret;
85507bcc
RC		 1547 struct srp_device *dev;
1548 struct ib_device *ibdev;
8f26c9ff
DD		 1549 struct srp_map_state state;
1550 struct srp_indirect_buf *indirect_hdr;
330179f2
BVA		 1551 u32 idb_len, table_len;
1552 __be32 idb_rkey;
8f26c9ff 1553 u8 fmt;
aef9ec39 1554
bb350d1d 1555 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
aef9ec39
RD		 1556 return sizeof (struct srp_cmd);
1557
1558 if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1559 scmnd->sc_data_direction != DMA_TO_DEVICE) {
7aa54bd7
DD		 1560 shost_printk(KERN_WARNING, target->scsi_host,
1561 PFX "Unhandled data direction %d\n",
1562 scmnd->sc_data_direction);
aef9ec39
RD		 1563 return -EINVAL;
1564 }
1565
bb350d1d
FT		 1566 nents = scsi_sg_count(scmnd);
1567 scat = scsi_sglist(scmnd);
aef9ec39 1568
05321937 1569 dev = target->srp_host->srp_dev;
85507bcc
RC		 1570 ibdev = dev->dev;
1571
1572 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
8f26c9ff
DD		 1573 if (unlikely(count == 0))
1574 return -EIO;
f5358a17
RD		 1575
1576 fmt = SRP_DATA_DESC_DIRECT;
1577 len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
aef9ec39 1578
03f6fb93 1579 if (count == 1 && target->global_mr) {
f5358a17
RD		 1580 /*
1581 * The midlayer only generated a single gather/scatter
1582 * entry, or DMA mapping coalesced everything to a
1583 * single entry. So a direct descriptor along with
1584 * the DMA MR suffices.
1585 */
cf368713 1586 struct srp_direct_buf *buf = (void *) cmd->add_data;
aef9ec39 1587
85507bcc 1588 buf->va = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
03f6fb93 1589 buf->key = cpu_to_be32(target->global_mr->rkey);
85507bcc 1590 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
8f26c9ff 1591
52ede08f 1592 req->nmdesc = 0;
8f26c9ff
DD		 1593 goto map_complete;
1594 }
1595
5cfb1782
BVA		 1596 /*
1597 * We have more than one scatter/gather entry, so build our indirect
1598 * descriptor table, trying to merge as many entries as we can.
8f26c9ff
DD		 1599 */
1600 indirect_hdr = (void *) cmd->add_data;
1601
c07d424d
DD		 1602 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1603 target->indirect_size, DMA_TO_DEVICE);
1604
8f26c9ff 1605 memset(&state, 0, sizeof(state));
26630e8a 1606 if (dev->use_fast_reg)
e012f363 1607 ret = srp_map_sg_fr(&state, ch, req, scat, count);
26630e8a 1608 else if (dev->use_fmr)
e012f363 1609 ret = srp_map_sg_fmr(&state, ch, req, scat, count);
26630e8a 1610 else
e012f363
BVA		 1611 ret = srp_map_sg_dma(&state, ch, req, scat, count);
1612 req->nmdesc = state.nmdesc;
1613 if (ret < 0)
1614 goto unmap;
cf368713 1615
c07d424d
DD		 1616 /* We've mapped the request, now pull as much of the indirect
1617 * descriptor table as we can into the command buffer. If this
1618 * target is not using an external indirect table, we are
1619 * guaranteed to fit into the command, as the SCSI layer won't
1620 * give us more S/G entries than we allow.
8f26c9ff 1621 */
8f26c9ff 1622 if (state.ndesc == 1) {
5cfb1782
BVA		 1623 /*
1624 * Memory registration collapsed the sg-list into one entry,
8f26c9ff
DD		 1625 * so use a direct descriptor.
1626 */
1627 struct srp_direct_buf *buf = (void *) cmd->add_data;
cf368713 1628
c07d424d 1629 *buf = req->indirect_desc[0];
8f26c9ff 1630 goto map_complete;
aef9ec39
RD		 1631 }
1632
c07d424d
DD		 1633 if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1634 !target->allow_ext_sg)) {
1635 shost_printk(KERN_ERR, target->scsi_host,
1636 "Could not fit S/G list into SRP_CMD\n");
e012f363
BVA		 1637 ret = -EIO;
1638 goto unmap;
c07d424d
DD		 1639 }
1640
1641 count = min(state.ndesc, target->cmd_sg_cnt);
8f26c9ff 1642 table_len = state.ndesc * sizeof (struct srp_direct_buf);
330179f2 1643 idb_len = sizeof(struct srp_indirect_buf) + table_len;
8f26c9ff
DD		 1644
1645 fmt = SRP_DATA_DESC_INDIRECT;
1646 len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
c07d424d 1647 len += count * sizeof (struct srp_direct_buf);
8f26c9ff 1648
c07d424d
DD		 1649 memcpy(indirect_hdr->desc_list, req->indirect_desc,
1650 count * sizeof (struct srp_direct_buf));
8f26c9ff 1651
03f6fb93 1652 if (!target->global_mr) {
330179f2
BVA		 1653 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
1654 idb_len, &idb_rkey);
1655 if (ret < 0)
e012f363 1656 goto unmap;
330179f2
BVA		 1657 req->nmdesc++;
1658 } else {
a745f4f4 1659 idb_rkey = cpu_to_be32(target->global_mr->rkey);
330179f2
BVA		 1660 }
1661
c07d424d 1662 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
330179f2 1663 indirect_hdr->table_desc.key = idb_rkey;
8f26c9ff
DD		 1664 indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1665 indirect_hdr->len = cpu_to_be32(state.total_len);
1666
1667 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
c07d424d 1668 cmd->data_out_desc_cnt = count;
8f26c9ff 1669 else
c07d424d
DD		 1670 cmd->data_in_desc_cnt = count;
1671
1672 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1673 DMA_TO_DEVICE);
8f26c9ff
DD		 1674
1675map_complete:
aef9ec39
RD		 1676 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1677 cmd->buf_fmt = fmt << 4;
1678 else
1679 cmd->buf_fmt = fmt;
1680
aef9ec39 1681 return len;
e012f363
BVA		 1682
1683unmap:
1684 srp_unmap_data(scmnd, ch, req);
1685 return ret;
aef9ec39
RD		 1686}
1687
76c75b25
BVA		 1688/*
1689 * Return an IU and possible credit to the free pool
1690 */
509c07bc 1691static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
76c75b25
BVA		 1692 enum srp_iu_type iu_type)
1693{
1694 unsigned long flags;
1695
509c07bc
BVA		 1696 spin_lock_irqsave(&ch->lock, flags);
1697 list_add(&iu->list, &ch->free_tx);
76c75b25 1698 if (iu_type != SRP_IU_RSP)
509c07bc
BVA		 1699 ++ch->req_lim;
1700 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25
BVA		 1701}
1702
05a1d750 1703/*
509c07bc 1704 * Must be called with ch->lock held to protect req_lim and free_tx.
e9684678 1705 * If IU is not sent, it must be returned using srp_put_tx_iu().
05a1d750
DD		 1706 *
1707 * Note:
1708 * An upper limit for the number of allocated information units for each
1709 * request type is:
1710 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1711 * more than Scsi_Host.can_queue requests.
1712 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1713 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1714 * one unanswered SRP request to an initiator.
1715 */
509c07bc 1716static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
05a1d750
DD		 1717 enum srp_iu_type iu_type)
1718{
509c07bc 1719 struct srp_target_port *target = ch->target;
05a1d750
DD		 1720 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1721 struct srp_iu *iu;
1722
1dc7b1f1 1723 ib_process_cq_direct(ch->send_cq, -1);
05a1d750 1724
509c07bc 1725 if (list_empty(&ch->free_tx))
05a1d750
DD		 1726 return NULL;
1727
1728 /* Initiator responses to target requests do not consume credits */
76c75b25 1729 if (iu_type != SRP_IU_RSP) {
509c07bc 1730 if (ch->req_lim <= rsv) {
76c75b25
BVA		 1731 ++target->zero_req_lim;
1732 return NULL;
1733 }
1734
509c07bc 1735 --ch->req_lim;
05a1d750
DD		 1736 }
1737
509c07bc 1738 iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
76c75b25 1739 list_del(&iu->list);
05a1d750
DD		 1740 return iu;
1741}
1742
1dc7b1f1
CH		 1743static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc)
1744{
1745 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1746 struct srp_rdma_ch *ch = cq->cq_context;
1747
1748 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1749 srp_handle_qp_err(cq, wc, "SEND");
1750 return;
1751 }
1752
1753 list_add(&iu->list, &ch->free_tx);
1754}
1755
509c07bc 1756static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
05a1d750 1757{
509c07bc 1758 struct srp_target_port *target = ch->target;
05a1d750
DD		 1759 struct ib_sge list;
1760 struct ib_send_wr wr, *bad_wr;
05a1d750
DD		 1761
1762 list.addr = iu->dma;
1763 list.length = len;
9af76271 1764 list.lkey = target->lkey;
05a1d750 1765
1dc7b1f1
CH		 1766 iu->cqe.done = srp_send_done;
1767
05a1d750 1768 wr.next = NULL;
1dc7b1f1 1769 wr.wr_cqe = &iu->cqe;
05a1d750
DD		 1770 wr.sg_list = &list;
1771 wr.num_sge = 1;
1772 wr.opcode = IB_WR_SEND;
1773 wr.send_flags = IB_SEND_SIGNALED;
1774
509c07bc 1775 return ib_post_send(ch->qp, &wr, &bad_wr);
05a1d750
DD		 1776}
1777
509c07bc 1778static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
c996bb47 1779{
509c07bc 1780 struct srp_target_port *target = ch->target;
c996bb47 1781 struct ib_recv_wr wr, *bad_wr;
dcb4cb85 1782 struct ib_sge list;
c996bb47
BVA		 1783
1784 list.addr = iu->dma;
1785 list.length = iu->size;
9af76271 1786 list.lkey = target->lkey;
c996bb47 1787
1dc7b1f1
CH		 1788 iu->cqe.done = srp_recv_done;
1789
c996bb47 1790 wr.next = NULL;
1dc7b1f1 1791 wr.wr_cqe = &iu->cqe;
c996bb47
BVA		 1792 wr.sg_list = &list;
1793 wr.num_sge = 1;
1794
509c07bc 1795 return ib_post_recv(ch->qp, &wr, &bad_wr);
c996bb47
BVA		 1796}
1797
509c07bc 1798static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
aef9ec39 1799{
509c07bc 1800 struct srp_target_port *target = ch->target;
aef9ec39
RD		 1801 struct srp_request *req;
1802 struct scsi_cmnd *scmnd;
1803 unsigned long flags;
aef9ec39 1804
aef9ec39 1805 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
509c07bc
BVA		 1806 spin_lock_irqsave(&ch->lock, flags);
1807 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1808 spin_unlock_irqrestore(&ch->lock, flags);
94a9174c 1809
509c07bc 1810 ch->tsk_mgmt_status = -1;
f8b6e31e 1811 if (be32_to_cpu(rsp->resp_data_len) >= 4)
509c07bc
BVA		 1812 ch->tsk_mgmt_status = rsp->data[3];
1813 complete(&ch->tsk_mgmt_done);
aef9ec39 1814 } else {
77f2c1a4
BVA		 1815 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1816 if (scmnd) {
1817 req = (void *)scmnd->host_scribble;
1818 scmnd = srp_claim_req(ch, req, NULL, scmnd);
1819 }
22032991 1820 if (!scmnd) {
7aa54bd7 1821 shost_printk(KERN_ERR, target->scsi_host,
d92c0da7
BVA		 1822 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1823 rsp->tag, ch - target->ch, ch->qp->qp_num);
22032991 1824
509c07bc
BVA		 1825 spin_lock_irqsave(&ch->lock, flags);
1826 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1827 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1828
1829 return;
1830 }
aef9ec39
RD		 1831 scmnd->result = rsp->status;
1832
1833 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1834 memcpy(scmnd->sense_buffer, rsp->data +
1835 be32_to_cpu(rsp->resp_data_len),
1836 min_t(int, be32_to_cpu(rsp->sense_data_len),
1837 SCSI_SENSE_BUFFERSIZE));
1838 }
1839
e714531a 1840 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
bb350d1d 1841 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
e714531a
BVA		 1842 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1843 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1844 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1845 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1846 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1847 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
aef9ec39 1848
509c07bc 1849 srp_free_req(ch, req, scmnd,
22032991
BVA		 1850 be32_to_cpu(rsp->req_lim_delta));
1851
f8b6e31e
DD		 1852 scmnd->host_scribble = NULL;
1853 scmnd->scsi_done(scmnd);
aef9ec39 1854 }
aef9ec39
RD		 1855}
1856
509c07bc 1857static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
bb12588a
DD		 1858 void *rsp, int len)
1859{
509c07bc 1860 struct srp_target_port *target = ch->target;
76c75b25 1861 struct ib_device *dev = target->srp_host->srp_dev->dev;
bb12588a
DD		 1862 unsigned long flags;
1863 struct srp_iu *iu;
76c75b25 1864 int err;
bb12588a 1865
509c07bc
BVA		 1866 spin_lock_irqsave(&ch->lock, flags);
1867 ch->req_lim += req_delta;
1868 iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1869 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25 1870
bb12588a
DD		 1871 if (!iu) {
1872 shost_printk(KERN_ERR, target->scsi_host, PFX
1873 "no IU available to send response\n");
76c75b25 1874 return 1;
bb12588a
DD		 1875 }
1876
1877 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1878 memcpy(iu->buf, rsp, len);
1879 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1880
509c07bc 1881 err = srp_post_send(ch, iu, len);
76c75b25 1882 if (err) {
bb12588a
DD		 1883 shost_printk(KERN_ERR, target->scsi_host, PFX
1884 "unable to post response: %d\n", err);
509c07bc 1885 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
76c75b25 1886 }
bb12588a 1887
bb12588a
DD		 1888 return err;
1889}
1890
509c07bc 1891static void srp_process_cred_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1892 struct srp_cred_req *req)
1893{
1894 struct srp_cred_rsp rsp = {
1895 .opcode = SRP_CRED_RSP,
1896 .tag = req->tag,
1897 };
1898 s32 delta = be32_to_cpu(req->req_lim_delta);
1899
509c07bc
BVA		 1900 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1901 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
bb12588a
DD		 1902 "problems processing SRP_CRED_REQ\n");
1903}
1904
509c07bc 1905static void srp_process_aer_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1906 struct srp_aer_req *req)
1907{
509c07bc 1908 struct srp_target_port *target = ch->target;
bb12588a
DD		 1909 struct srp_aer_rsp rsp = {
1910 .opcode = SRP_AER_RSP,
1911 .tag = req->tag,
1912 };
1913 s32 delta = be32_to_cpu(req->req_lim_delta);
1914
1915 shost_printk(KERN_ERR, target->scsi_host, PFX
985aa495 1916 "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
bb12588a 1917
509c07bc 1918 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
bb12588a
DD		 1919 shost_printk(KERN_ERR, target->scsi_host, PFX
1920 "problems processing SRP_AER_REQ\n");
1921}
1922
1dc7b1f1 1923static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc)
aef9ec39 1924{
1dc7b1f1
CH		 1925 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1926 struct srp_rdma_ch *ch = cq->cq_context;
509c07bc 1927 struct srp_target_port *target = ch->target;
dcb4cb85 1928 struct ib_device *dev = target->srp_host->srp_dev->dev;
c996bb47 1929 int res;
aef9ec39
RD		 1930 u8 opcode;
1931
1dc7b1f1
CH		 1932 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1933 srp_handle_qp_err(cq, wc, "RECV");
1934 return;
1935 }
1936
509c07bc 1937 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1938 DMA_FROM_DEVICE);
aef9ec39
RD		 1939
1940 opcode = *(u8 *) iu->buf;
1941
1942 if (0) {
7aa54bd7
DD		 1943 shost_printk(KERN_ERR, target->scsi_host,
1944 PFX "recv completion, opcode 0x%02x\n", opcode);
7a700811
BVA		 1945 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
1946 iu->buf, wc->byte_len, true);
aef9ec39
RD		 1947 }
1948
1949 switch (opcode) {
1950 case SRP_RSP:
509c07bc 1951 srp_process_rsp(ch, iu->buf);
aef9ec39
RD		 1952 break;
1953
bb12588a 1954 case SRP_CRED_REQ:
509c07bc 1955 srp_process_cred_req(ch, iu->buf);
bb12588a
DD		 1956 break;
1957
1958 case SRP_AER_REQ:
509c07bc 1959 srp_process_aer_req(ch, iu->buf);
bb12588a
DD		 1960 break;
1961
aef9ec39
RD		 1962 case SRP_T_LOGOUT:
1963 /* XXX Handle target logout */
7aa54bd7
DD		 1964 shost_printk(KERN_WARNING, target->scsi_host,
1965 PFX "Got target logout request\n");
aef9ec39
RD		 1966 break;
1967
1968 default:
7aa54bd7
DD		 1969 shost_printk(KERN_WARNING, target->scsi_host,
1970 PFX "Unhandled SRP opcode 0x%02x\n", opcode);
aef9ec39
RD		 1971 break;
1972 }
1973
509c07bc 1974 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1975 DMA_FROM_DEVICE);
c996bb47 1976
509c07bc 1977 res = srp_post_recv(ch, iu);
c996bb47
BVA		 1978 if (res != 0)
1979 shost_printk(KERN_ERR, target->scsi_host,
1980 PFX "Recv failed with error code %d\n", res);
aef9ec39
RD		 1981}
1982
c1120f89
BVA		 1983/**
1984 * srp_tl_err_work() - handle a transport layer error
af24663b 1985 * @work: Work structure embedded in an SRP target port.
c1120f89
BVA		 1986 *
1987 * Note: This function may get invoked before the rport has been created,
1988 * hence the target->rport test.
1989 */
1990static void srp_tl_err_work(struct work_struct *work)
1991{
1992 struct srp_target_port *target;
1993
1994 target = container_of(work, struct srp_target_port, tl_err_work);
1995 if (target->rport)
1996 srp_start_tl_fail_timers(target->rport);
1997}
1998
1dc7b1f1
CH		 1999static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
2000 const char *opname)
948d1e88 2001{
1dc7b1f1 2002 struct srp_rdma_ch *ch = cq->cq_context;
7dad6b2e
BVA		 2003 struct srp_target_port *target = ch->target;
2004
c014c8cd 2005 if (ch->connected && !target->qp_in_error) {
1dc7b1f1
CH		 2006 shost_printk(KERN_ERR, target->scsi_host,
2007 PFX "failed %s status %s (%d) for CQE %p\n",
2008 opname, ib_wc_status_msg(wc->status), wc->status,
2009 wc->wr_cqe);
c1120f89 2010 queue_work(system_long_wq, &target->tl_err_work);
4f0af697 2011 }
948d1e88
BVA		 2012 target->qp_in_error = true;
2013}
2014
76c75b25 2015static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
aef9ec39 2016{
76c75b25 2017 struct srp_target_port *target = host_to_target(shost);
a95cadb9 2018 struct srp_rport *rport = target->rport;
509c07bc 2019 struct srp_rdma_ch *ch;
aef9ec39
RD		 2020 struct srp_request *req;
2021 struct srp_iu *iu;
2022 struct srp_cmd *cmd;
85507bcc 2023 struct ib_device *dev;
76c75b25 2024 unsigned long flags;
77f2c1a4
BVA		 2025 u32 tag;
2026 u16 idx;
d1b4289e 2027 int len, ret;
a95cadb9
BVA		 2028 const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
2029
2030 /*
2031 * The SCSI EH thread is the only context from which srp_queuecommand()
2032 * can get invoked for blocked devices (SDEV_BLOCK /
2033 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
2034 * locking the rport mutex if invoked from inside the SCSI EH.
2035 */
2036 if (in_scsi_eh)
2037 mutex_lock(&rport->mutex);
aef9ec39 2038
d1b4289e
BVA		 2039 scmnd->result = srp_chkready(target->rport);
2040 if (unlikely(scmnd->result))
2041 goto err;
2ce19e72 2042
77f2c1a4
BVA		 2043 WARN_ON_ONCE(scmnd->request->tag < 0);
2044 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7 2045 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
77f2c1a4
BVA		 2046 idx = blk_mq_unique_tag_to_tag(tag);
2047 WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2048 dev_name(&shost->shost_gendev), tag, idx,
2049 target->req_ring_size);
509c07bc
BVA		 2050
2051 spin_lock_irqsave(&ch->lock, flags);
2052 iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
509c07bc 2053 spin_unlock_irqrestore(&ch->lock, flags);
aef9ec39 2054
77f2c1a4
BVA		 2055 if (!iu)
2056 goto err;
2057
2058 req = &ch->req_ring[idx];
05321937 2059 dev = target->srp_host->srp_dev->dev;
49248644 2060 ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
85507bcc 2061 DMA_TO_DEVICE);
aef9ec39 2062
f8b6e31e 2063 scmnd->host_scribble = (void *) req;
aef9ec39
RD		 2064
2065 cmd = iu->buf;
2066 memset(cmd, 0, sizeof *cmd);
2067
2068 cmd->opcode = SRP_CMD;
985aa495 2069 int_to_scsilun(scmnd->device->lun, &cmd->lun);
77f2c1a4 2070 cmd->tag = tag;
aef9ec39
RD		 2071 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2072
aef9ec39
RD		 2073 req->scmnd = scmnd;
2074 req->cmd = iu;
aef9ec39 2075
509c07bc 2076 len = srp_map_data(scmnd, ch, req);
aef9ec39 2077 if (len < 0) {
7aa54bd7 2078 shost_printk(KERN_ERR, target->scsi_host,
d1b4289e
BVA		 2079 PFX "Failed to map data (%d)\n", len);
2080 /*
2081 * If we ran out of memory descriptors (-ENOMEM) because an
2082 * application is queuing many requests with more than
52ede08f 2083 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
d1b4289e
BVA		 2084 * to reduce queue depth temporarily.
2085 */
2086 scmnd->result = len == -ENOMEM ?
2087 DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
76c75b25 2088 goto err_iu;
aef9ec39
RD		 2089 }
2090
49248644 2091 ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
85507bcc 2092 DMA_TO_DEVICE);
aef9ec39 2093
509c07bc 2094 if (srp_post_send(ch, iu, len)) {
7aa54bd7 2095 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
aef9ec39
RD		 2096 goto err_unmap;
2097 }
2098
d1b4289e
BVA		 2099 ret = 0;
2100
a95cadb9
BVA		 2101unlock_rport:
2102 if (in_scsi_eh)
2103 mutex_unlock(&rport->mutex);
2104
d1b4289e 2105 return ret;
aef9ec39
RD		 2106
2107err_unmap:
509c07bc 2108 srp_unmap_data(scmnd, ch, req);
aef9ec39 2109
76c75b25 2110err_iu:
509c07bc 2111 srp_put_tx_iu(ch, iu, SRP_IU_CMD);
76c75b25 2112
024ca901
BVA		 2113 /*
2114 * Avoid that the loops that iterate over the request ring can
2115 * encounter a dangling SCSI command pointer.
2116 */
2117 req->scmnd = NULL;
2118
d1b4289e
BVA		 2119err:
2120 if (scmnd->result) {
2121 scmnd->scsi_done(scmnd);
2122 ret = 0;
2123 } else {
2124 ret = SCSI_MLQUEUE_HOST_BUSY;
2125 }
a95cadb9 2126
d1b4289e 2127 goto unlock_rport;
aef9ec39
RD		 2128}
2129
4d73f95f
BVA		 2130/*
2131 * Note: the resources allocated in this function are freed in
509c07bc 2132 * srp_free_ch_ib().
4d73f95f 2133 */
509c07bc 2134static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
aef9ec39 2135{
509c07bc 2136 struct srp_target_port *target = ch->target;
aef9ec39
RD		 2137 int i;
2138
509c07bc
BVA		 2139 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2140 GFP_KERNEL);
2141 if (!ch->rx_ring)
4d73f95f 2142 goto err_no_ring;
509c07bc
BVA		 2143 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2144 GFP_KERNEL);
2145 if (!ch->tx_ring)
4d73f95f
BVA		 2146 goto err_no_ring;
2147
2148 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2149 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2150 ch->max_ti_iu_len,
2151 GFP_KERNEL, DMA_FROM_DEVICE);
2152 if (!ch->rx_ring[i])
aef9ec39
RD		 2153 goto err;
2154 }
2155
4d73f95f 2156 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2157 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2158 target->max_iu_len,
2159 GFP_KERNEL, DMA_TO_DEVICE);
2160 if (!ch->tx_ring[i])
aef9ec39 2161 goto err;
dcb4cb85 2162
509c07bc 2163 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
aef9ec39
RD		 2164 }
2165
2166 return 0;
2167
2168err:
4d73f95f 2169 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2170 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2171 srp_free_iu(target->srp_host, ch->tx_ring[i]);
aef9ec39
RD		 2172 }
2173
4d73f95f
BVA		 2174
2175err_no_ring:
509c07bc
BVA		 2176 kfree(ch->tx_ring);
2177 ch->tx_ring = NULL;
2178 kfree(ch->rx_ring);
2179 ch->rx_ring = NULL;
4d73f95f 2180
aef9ec39
RD		 2181 return -ENOMEM;
2182}
2183
c9b03c1a
BVA		 2184static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2185{
2186 uint64_t T_tr_ns, max_compl_time_ms;
2187 uint32_t rq_tmo_jiffies;
2188
2189 /*
2190 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2191 * table 91), both the QP timeout and the retry count have to be set
2192 * for RC QP's during the RTR to RTS transition.
2193 */
2194 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2195 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2196
2197 /*
2198 * Set target->rq_tmo_jiffies to one second more than the largest time
2199 * it can take before an error completion is generated. See also
2200 * C9-140..142 in the IBTA spec for more information about how to
2201 * convert the QP Local ACK Timeout value to nanoseconds.
2202 */
2203 T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2204 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2205 do_div(max_compl_time_ms, NSEC_PER_MSEC);
2206 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2207
2208 return rq_tmo_jiffies;
2209}
2210
961e0be8 2211static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
e6300cbd 2212 const struct srp_login_rsp *lrsp,
509c07bc 2213 struct srp_rdma_ch *ch)
961e0be8 2214{
509c07bc 2215 struct srp_target_port *target = ch->target;
961e0be8
DD		 2216 struct ib_qp_attr *qp_attr = NULL;
2217 int attr_mask = 0;
2218 int ret;
2219 int i;
2220
2221 if (lrsp->opcode == SRP_LOGIN_RSP) {
509c07bc
BVA		 2222 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2223 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta);
961e0be8
DD		 2224
2225 /*
2226 * Reserve credits for task management so we don't
2227 * bounce requests back to the SCSI mid-layer.
2228 */
2229 target->scsi_host->can_queue
509c07bc 2230 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
961e0be8 2231 target->scsi_host->can_queue);
4d73f95f
BVA		 2232 target->scsi_host->cmd_per_lun
2233 = min_t(int, target->scsi_host->can_queue,
2234 target->scsi_host->cmd_per_lun);
961e0be8
DD		 2235 } else {
2236 shost_printk(KERN_WARNING, target->scsi_host,
2237 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2238 ret = -ECONNRESET;
2239 goto error;
2240 }
2241
509c07bc
BVA		 2242 if (!ch->rx_ring) {
2243 ret = srp_alloc_iu_bufs(ch);
961e0be8
DD		 2244 if (ret)
2245 goto error;
2246 }
2247
2248 ret = -ENOMEM;
2249 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2250 if (!qp_attr)
2251 goto error;
2252
2253 qp_attr->qp_state = IB_QPS_RTR;
2254 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2255 if (ret)
2256 goto error_free;
2257
509c07bc 2258 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2259 if (ret)
2260 goto error_free;
2261
4d73f95f 2262 for (i = 0; i < target->queue_size; i++) {
509c07bc
BVA		 2263 struct srp_iu *iu = ch->rx_ring[i];
2264
2265 ret = srp_post_recv(ch, iu);
961e0be8
DD		 2266 if (ret)
2267 goto error_free;
2268 }
2269
2270 qp_attr->qp_state = IB_QPS_RTS;
2271 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2272 if (ret)
2273 goto error_free;
2274
c9b03c1a
BVA		 2275 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2276
509c07bc 2277 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2278 if (ret)
2279 goto error_free;
2280
2281 ret = ib_send_cm_rtu(cm_id, NULL, 0);
2282
2283error_free:
2284 kfree(qp_attr);
2285
2286error:
509c07bc 2287 ch->status = ret;
961e0be8
DD		 2288}
2289
aef9ec39
RD		 2290static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2291 struct ib_cm_event *event,
509c07bc 2292 struct srp_rdma_ch *ch)
aef9ec39 2293{
509c07bc 2294 struct srp_target_port *target = ch->target;
7aa54bd7 2295 struct Scsi_Host *shost = target->scsi_host;
aef9ec39
RD		 2296 struct ib_class_port_info *cpi;
2297 int opcode;
2298
2299 switch (event->param.rej_rcvd.reason) {
2300 case IB_CM_REJ_PORT_CM_REDIRECT:
2301 cpi = event->param.rej_rcvd.ari;
509c07bc
BVA		 2302 ch->path.dlid = cpi->redirect_lid;
2303 ch->path.pkey = cpi->redirect_pkey;
aef9ec39 2304 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
509c07bc 2305 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
aef9ec39 2306
509c07bc 2307 ch->status = ch->path.dlid ?
aef9ec39
RD		 2308 SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2309 break;
2310
2311 case IB_CM_REJ_PORT_REDIRECT:
5d7cbfd6 2312 if (srp_target_is_topspin(target)) {
aef9ec39
RD		 2313 /*
2314 * Topspin/Cisco SRP gateways incorrectly send
2315 * reject reason code 25 when they mean 24
2316 * (port redirect).
2317 */
509c07bc 2318 memcpy(ch->path.dgid.raw,
aef9ec39
RD		 2319 event->param.rej_rcvd.ari, 16);
2320
7aa54bd7
DD		 2321 shost_printk(KERN_DEBUG, shost,
2322 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
509c07bc
BVA		 2323 be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2324 be64_to_cpu(ch->path.dgid.global.interface_id));
aef9ec39 2325
509c07bc 2326 ch->status = SRP_PORT_REDIRECT;
aef9ec39 2327 } else {
7aa54bd7
DD		 2328 shost_printk(KERN_WARNING, shost,
2329 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
509c07bc 2330 ch->status = -ECONNRESET;
aef9ec39
RD		 2331 }
2332 break;
2333
2334 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
7aa54bd7
DD		 2335 shost_printk(KERN_WARNING, shost,
2336 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
509c07bc 2337 ch->status = -ECONNRESET;
aef9ec39
RD		 2338 break;
2339
2340 case IB_CM_REJ_CONSUMER_DEFINED:
2341 opcode = *(u8 *) event->private_data;
2342 if (opcode == SRP_LOGIN_REJ) {
2343 struct srp_login_rej *rej = event->private_data;
2344 u32 reason = be32_to_cpu(rej->reason);
2345
2346 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
7aa54bd7
DD		 2347 shost_printk(KERN_WARNING, shost,
2348 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
aef9ec39 2349 else
e7ffde01
BVA		 2350 shost_printk(KERN_WARNING, shost, PFX
2351 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
747fe000
BVA		 2352 target->sgid.raw,
2353 target->orig_dgid.raw, reason);
aef9ec39 2354 } else
7aa54bd7
DD		 2355 shost_printk(KERN_WARNING, shost,
2356 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2357 " opcode 0x%02x\n", opcode);
509c07bc 2358 ch->status = -ECONNRESET;
aef9ec39
RD		 2359 break;
2360
9fe4bcf4
DD		 2361 case IB_CM_REJ_STALE_CONN:
2362 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n");
509c07bc 2363 ch->status = SRP_STALE_CONN;
9fe4bcf4
DD		 2364 break;
2365
aef9ec39 2366 default:
7aa54bd7
DD		 2367 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n",
2368 event->param.rej_rcvd.reason);
509c07bc 2369 ch->status = -ECONNRESET;
aef9ec39
RD		 2370 }
2371}
2372
2373static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2374{
509c07bc
BVA		 2375 struct srp_rdma_ch *ch = cm_id->context;
2376 struct srp_target_port *target = ch->target;
aef9ec39 2377 int comp = 0;
aef9ec39
RD		 2378
2379 switch (event->event) {
2380 case IB_CM_REQ_ERROR:
7aa54bd7
DD		 2381 shost_printk(KERN_DEBUG, target->scsi_host,
2382 PFX "Sending CM REQ failed\n");
aef9ec39 2383 comp = 1;
509c07bc 2384 ch->status = -ECONNRESET;
aef9ec39
RD		 2385 break;
2386
2387 case IB_CM_REP_RECEIVED:
2388 comp = 1;
509c07bc 2389 srp_cm_rep_handler(cm_id, event->private_data, ch);
aef9ec39
RD		 2390 break;
2391
2392 case IB_CM_REJ_RECEIVED:
7aa54bd7 2393 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
aef9ec39
RD		 2394 comp = 1;
2395
509c07bc 2396 srp_cm_rej_handler(cm_id, event, ch);
aef9ec39
RD		 2397 break;
2398
b7ac4ab4 2399 case IB_CM_DREQ_RECEIVED:
7aa54bd7
DD		 2400 shost_printk(KERN_WARNING, target->scsi_host,
2401 PFX "DREQ received - connection closed\n");
c014c8cd 2402 ch->connected = false;
b7ac4ab4 2403 if (ib_send_cm_drep(cm_id, NULL, 0))
7aa54bd7
DD		 2404 shost_printk(KERN_ERR, target->scsi_host,
2405 PFX "Sending CM DREP failed\n");
c1120f89 2406 queue_work(system_long_wq, &target->tl_err_work);
aef9ec39
RD		 2407 break;
2408
2409 case IB_CM_TIMEWAIT_EXIT:
7aa54bd7
DD		 2410 shost_printk(KERN_ERR, target->scsi_host,
2411 PFX "connection closed\n");
ac72d766 2412 comp = 1;
aef9ec39 2413
509c07bc 2414 ch->status = 0;
aef9ec39
RD		 2415 break;
2416
b7ac4ab4
IR		 2417 case IB_CM_MRA_RECEIVED:
2418 case IB_CM_DREQ_ERROR:
2419 case IB_CM_DREP_RECEIVED:
2420 break;
2421
aef9ec39 2422 default:
7aa54bd7
DD		 2423 shost_printk(KERN_WARNING, target->scsi_host,
2424 PFX "Unhandled CM event %d\n", event->event);
aef9ec39
RD		 2425 break;
2426 }
2427
2428 if (comp)
509c07bc 2429 complete(&ch->done);
aef9ec39 2430
aef9ec39
RD		 2431 return 0;
2432}
2433
71444b97
JW		 2434/**
2435 * srp_change_queue_depth - setting device queue depth
2436 * @sdev: scsi device struct
2437 * @qdepth: requested queue depth
71444b97
JW		 2438 *
2439 * Returns queue depth.
2440 */
2441static int
db5ed4df 2442srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
71444b97 2443{
c40ecc12 2444 if (!sdev->tagged_supported)
1e6f2416 2445 qdepth = 1;
db5ed4df 2446 return scsi_change_queue_depth(sdev, qdepth);
71444b97
JW		 2447}
2448
985aa495
BVA		 2449static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2450 u8 func)
aef9ec39 2451{
509c07bc 2452 struct srp_target_port *target = ch->target;
a95cadb9 2453 struct srp_rport *rport = target->rport;
19081f31 2454 struct ib_device *dev = target->srp_host->srp_dev->dev;
aef9ec39
RD		 2455 struct srp_iu *iu;
2456 struct srp_tsk_mgmt *tsk_mgmt;
aef9ec39 2457
c014c8cd 2458 if (!ch->connected || target->qp_in_error)
3780d1f0
BVA		 2459 return -1;
2460
509c07bc 2461 init_completion(&ch->tsk_mgmt_done);
aef9ec39 2462
a95cadb9 2463 /*
509c07bc 2464 * Lock the rport mutex to avoid that srp_create_ch_ib() is
a95cadb9
BVA		 2465 * invoked while a task management function is being sent.
2466 */
2467 mutex_lock(&rport->mutex);
509c07bc
BVA		 2468 spin_lock_irq(&ch->lock);
2469 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2470 spin_unlock_irq(&ch->lock);
76c75b25 2471
a95cadb9
BVA		 2472 if (!iu) {
2473 mutex_unlock(&rport->mutex);
2474
76c75b25 2475 return -1;
a95cadb9 2476 }
aef9ec39 2477
19081f31
DD		 2478 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2479 DMA_TO_DEVICE);
aef9ec39
RD		 2480 tsk_mgmt = iu->buf;
2481 memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2482
2483 tsk_mgmt->opcode = SRP_TSK_MGMT;
985aa495 2484 int_to_scsilun(lun, &tsk_mgmt->lun);
f8b6e31e 2485 tsk_mgmt->tag = req_tag | SRP_TAG_TSK_MGMT;
aef9ec39 2486 tsk_mgmt->tsk_mgmt_func = func;
f8b6e31e 2487 tsk_mgmt->task_tag = req_tag;
aef9ec39 2488
19081f31
DD		 2489 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2490 DMA_TO_DEVICE);
509c07bc
BVA		 2491 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2492 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
a95cadb9
BVA		 2493 mutex_unlock(&rport->mutex);
2494
76c75b25
BVA		 2495 return -1;
2496 }
a95cadb9 2497 mutex_unlock(&rport->mutex);
d945e1df 2498
509c07bc 2499 if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
aef9ec39 2500 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
d945e1df 2501 return -1;
aef9ec39 2502
d945e1df 2503 return 0;
d945e1df
RD		 2504}
2505
aef9ec39
RD		 2506static int srp_abort(struct scsi_cmnd *scmnd)
2507{
d945e1df 2508 struct srp_target_port *target = host_to_target(scmnd->device->host);
f8b6e31e 2509 struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
77f2c1a4 2510 u32 tag;
d92c0da7 2511 u16 ch_idx;
509c07bc 2512 struct srp_rdma_ch *ch;
086f44f5 2513 int ret;
d945e1df 2514
7aa54bd7 2515 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
aef9ec39 2516
d92c0da7 2517 if (!req)
99b6697a 2518 return SUCCESS;
77f2c1a4 2519 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7
BVA		 2520 ch_idx = blk_mq_unique_tag_to_hwq(tag);
2521 if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2522 return SUCCESS;
2523 ch = &target->ch[ch_idx];
2524 if (!srp_claim_req(ch, req, NULL, scmnd))
2525 return SUCCESS;
2526 shost_printk(KERN_ERR, target->scsi_host,
2527 "Sending SRP abort for tag %#x\n", tag);
77f2c1a4 2528 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
80d5e8a2 2529 SRP_TSK_ABORT_TASK) == 0)
086f44f5 2530 ret = SUCCESS;
ed9b2264 2531 else if (target->rport->state == SRP_RPORT_LOST)
99e1c139 2532 ret = FAST_IO_FAIL;
086f44f5
BVA		 2533 else
2534 ret = FAILED;
509c07bc 2535 srp_free_req(ch, req, scmnd, 0);
22032991 2536 scmnd->result = DID_ABORT << 16;
d8536670 2537 scmnd->scsi_done(scmnd);
d945e1df 2538
086f44f5 2539 return ret;
aef9ec39
RD		 2540}
2541
2542static int srp_reset_device(struct scsi_cmnd *scmnd)
2543{
d945e1df 2544 struct srp_target_port *target = host_to_target(scmnd->device->host);
d92c0da7 2545 struct srp_rdma_ch *ch;
536ae14e 2546 int i;
d945e1df 2547
7aa54bd7 2548 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
aef9ec39 2549
d92c0da7 2550 ch = &target->ch[0];
509c07bc 2551 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
f8b6e31e 2552 SRP_TSK_LUN_RESET))
d945e1df 2553 return FAILED;
509c07bc 2554 if (ch->tsk_mgmt_status)
d945e1df
RD		 2555 return FAILED;
2556
d92c0da7
BVA		 2557 for (i = 0; i < target->ch_count; i++) {
2558 ch = &target->ch[i];
2559 for (i = 0; i < target->req_ring_size; ++i) {
2560 struct srp_request *req = &ch->req_ring[i];
509c07bc 2561
d92c0da7
BVA		 2562 srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2563 }
536ae14e 2564 }
d945e1df 2565
d945e1df 2566 return SUCCESS;
aef9ec39
RD		 2567}
2568
2569static int srp_reset_host(struct scsi_cmnd *scmnd)
2570{
2571 struct srp_target_port *target = host_to_target(scmnd->device->host);
aef9ec39 2572
7aa54bd7 2573 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
aef9ec39 2574
ed9b2264 2575 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
aef9ec39
RD		 2576}
2577
c9b03c1a
BVA		 2578static int srp_slave_configure(struct scsi_device *sdev)
2579{
2580 struct Scsi_Host *shost = sdev->host;
2581 struct srp_target_port *target = host_to_target(shost);
2582 struct request_queue *q = sdev->request_queue;
2583 unsigned long timeout;
2584
2585 if (sdev->type == TYPE_DISK) {
2586 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2587 blk_queue_rq_timeout(q, timeout);
2588 }
2589
2590 return 0;
2591}
2592
ee959b00
TJ		 2593static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2594 char *buf)
6ecb0c84 2595{
ee959b00 2596 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2597
45c37cad 2598 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
6ecb0c84
RD		 2599}
2600
ee959b00
TJ		 2601static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2602 char *buf)
6ecb0c84 2603{
ee959b00 2604 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2605
45c37cad 2606 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
6ecb0c84
RD		 2607}
2608
ee959b00
TJ		 2609static ssize_t show_service_id(struct device *dev,
2610 struct device_attribute *attr, char *buf)
6ecb0c84 2611{
ee959b00 2612 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2613
45c37cad 2614 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
6ecb0c84
RD		 2615}
2616
ee959b00
TJ		 2617static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2618 char *buf)
6ecb0c84 2619{
ee959b00 2620 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2621
747fe000 2622 return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
6ecb0c84
RD		 2623}
2624
848b3082
BVA		 2625static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2626 char *buf)
2627{
2628 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2629
747fe000 2630 return sprintf(buf, "%pI6\n", target->sgid.raw);
848b3082
BVA		 2631}
2632
ee959b00
TJ		 2633static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2634 char *buf)
6ecb0c84 2635{
ee959b00 2636 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7 2637 struct srp_rdma_ch *ch = &target->ch[0];
6ecb0c84 2638
509c07bc 2639 return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
6ecb0c84
RD		 2640}
2641
ee959b00
TJ		 2642static ssize_t show_orig_dgid(struct device *dev,
2643 struct device_attribute *attr, char *buf)
3633b3d0 2644{
ee959b00 2645 struct srp_target_port *target = host_to_target(class_to_shost(dev));
3633b3d0 2646
747fe000 2647 return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
3633b3d0
IR		 2648}
2649
89de7486
BVA		 2650static ssize_t show_req_lim(struct device *dev,
2651 struct device_attribute *attr, char *buf)
2652{
2653 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7
BVA		 2654 struct srp_rdma_ch *ch;
2655 int i, req_lim = INT_MAX;
89de7486 2656
d92c0da7
BVA		 2657 for (i = 0; i < target->ch_count; i++) {
2658 ch = &target->ch[i];
2659 req_lim = min(req_lim, ch->req_lim);
2660 }
2661 return sprintf(buf, "%d\n", req_lim);
89de7486
BVA		 2662}
2663
ee959b00
TJ		 2664static ssize_t show_zero_req_lim(struct device *dev,
2665 struct device_attribute *attr, char *buf)
6bfa24fa 2666{
ee959b00 2667 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6bfa24fa 2668
6bfa24fa
RD		 2669 return sprintf(buf, "%d\n", target->zero_req_lim);
2670}
2671
ee959b00
TJ		 2672static ssize_t show_local_ib_port(struct device *dev,
2673 struct device_attribute *attr, char *buf)
ded7f1a1 2674{
ee959b00 2675 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1
IR		 2676
2677 return sprintf(buf, "%d\n", target->srp_host->port);
2678}
2679
ee959b00
TJ		 2680static ssize_t show_local_ib_device(struct device *dev,
2681 struct device_attribute *attr, char *buf)
ded7f1a1 2682{
ee959b00 2683 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1 2684
05321937 2685 return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
ded7f1a1
IR		 2686}
2687
d92c0da7
BVA		 2688static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2689 char *buf)
2690{
2691 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2692
2693 return sprintf(buf, "%d\n", target->ch_count);
2694}
2695
4b5e5f41
BVA		 2696static ssize_t show_comp_vector(struct device *dev,
2697 struct device_attribute *attr, char *buf)
2698{
2699 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2700
2701 return sprintf(buf, "%d\n", target->comp_vector);
2702}
2703
7bb312e4
VP		 2704static ssize_t show_tl_retry_count(struct device *dev,
2705 struct device_attribute *attr, char *buf)
2706{
2707 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2708
2709 return sprintf(buf, "%d\n", target->tl_retry_count);
2710}
2711
49248644
DD		 2712static ssize_t show_cmd_sg_entries(struct device *dev,
2713 struct device_attribute *attr, char *buf)
2714{
2715 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2716
2717 return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2718}
2719
c07d424d
DD		 2720static ssize_t show_allow_ext_sg(struct device *dev,
2721 struct device_attribute *attr, char *buf)
2722{
2723 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2724
2725 return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2726}
2727
ee959b00
TJ		 2728static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL);
2729static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL);
2730static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL);
2731static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL);
848b3082 2732static DEVICE_ATTR(sgid, S_IRUGO, show_sgid, NULL);
ee959b00
TJ		 2733static DEVICE_ATTR(dgid, S_IRUGO, show_dgid, NULL);
2734static DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL);
89de7486 2735static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL);
ee959b00
TJ		 2736static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
2737static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
2738static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
d92c0da7 2739static DEVICE_ATTR(ch_count, S_IRUGO, show_ch_count, NULL);
4b5e5f41 2740static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL);
7bb312e4 2741static DEVICE_ATTR(tl_retry_count, S_IRUGO, show_tl_retry_count, NULL);
49248644 2742static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL);
c07d424d 2743static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL);
ee959b00
TJ		 2744
2745static struct device_attribute *srp_host_attrs[] = {
2746 &dev_attr_id_ext,
2747 &dev_attr_ioc_guid,
2748 &dev_attr_service_id,
2749 &dev_attr_pkey,
848b3082 2750 &dev_attr_sgid,
ee959b00
TJ		 2751 &dev_attr_dgid,
2752 &dev_attr_orig_dgid,
89de7486 2753 &dev_attr_req_lim,
ee959b00
TJ		 2754 &dev_attr_zero_req_lim,
2755 &dev_attr_local_ib_port,
2756 &dev_attr_local_ib_device,
d92c0da7 2757 &dev_attr_ch_count,
4b5e5f41 2758 &dev_attr_comp_vector,
7bb312e4 2759 &dev_attr_tl_retry_count,
49248644 2760 &dev_attr_cmd_sg_entries,
c07d424d 2761 &dev_attr_allow_ext_sg,
6ecb0c84
RD		 2762 NULL
2763};
2764
aef9ec39
RD		 2765static struct scsi_host_template srp_template = {
2766 .module = THIS_MODULE,
b7f008fd
RD		 2767 .name = "InfiniBand SRP initiator",
2768 .proc_name = DRV_NAME,
c9b03c1a 2769 .slave_configure = srp_slave_configure,
aef9ec39
RD		 2770 .info = srp_target_info,
2771 .queuecommand = srp_queuecommand,
71444b97 2772 .change_queue_depth = srp_change_queue_depth,
aef9ec39
RD		 2773 .eh_abort_handler = srp_abort,
2774 .eh_device_reset_handler = srp_reset_device,
2775 .eh_host_reset_handler = srp_reset_host,
2742c1da 2776 .skip_settle_delay = true,
49248644 2777 .sg_tablesize = SRP_DEF_SG_TABLESIZE,
4d73f95f 2778 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE,
aef9ec39 2779 .this_id = -1,
4d73f95f 2780 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE,
6ecb0c84 2781 .use_clustering = ENABLE_CLUSTERING,
77f2c1a4 2782 .shost_attrs = srp_host_attrs,
c40ecc12 2783 .track_queue_depth = 1,
aef9ec39
RD		 2784};
2785
34aa654e
BVA		 2786static int srp_sdev_count(struct Scsi_Host *host)
2787{
2788 struct scsi_device *sdev;
2789 int c = 0;
2790
2791 shost_for_each_device(sdev, host)
2792 c++;
2793
2794 return c;
2795}
2796
bc44bd1d
BVA		 2797/*
2798 * Return values:
2799 * < 0 upon failure. Caller is responsible for SRP target port cleanup.
2800 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
2801 * removal has been scheduled.
2802 * 0 and target->state != SRP_TARGET_REMOVED upon success.
2803 */
aef9ec39
RD		 2804static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2805{
3236822b
FT		 2806 struct srp_rport_identifiers ids;
2807 struct srp_rport *rport;
2808
34aa654e 2809 target->state = SRP_TARGET_SCANNING;
aef9ec39 2810 sprintf(target->target_name, "SRP.T10:%016llX",
45c37cad 2811 be64_to_cpu(target->id_ext));
aef9ec39 2812
05321937 2813 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
aef9ec39
RD		 2814 return -ENODEV;
2815
3236822b
FT		 2816 memcpy(ids.port_id, &target->id_ext, 8);
2817 memcpy(ids.port_id + 8, &target->ioc_guid, 8);
aebd5e47 2818 ids.roles = SRP_RPORT_ROLE_TARGET;
3236822b
FT		 2819 rport = srp_rport_add(target->scsi_host, &ids);
2820 if (IS_ERR(rport)) {
2821 scsi_remove_host(target->scsi_host);
2822 return PTR_ERR(rport);
2823 }
2824
dc1bdbd9 2825 rport->lld_data = target;
9dd69a60 2826 target->rport = rport;
dc1bdbd9 2827
b3589fd4 2828 spin_lock(&host->target_lock);
aef9ec39 2829 list_add_tail(&target->list, &host->target_list);
b3589fd4 2830 spin_unlock(&host->target_lock);
aef9ec39 2831
aef9ec39 2832 scsi_scan_target(&target->scsi_host->shost_gendev,
1962a4a1 2833 0, target->scsi_id, SCAN_WILD_CARD, 0);
aef9ec39 2834
c014c8cd
BVA		 2835 if (srp_connected_ch(target) < target->ch_count ||
2836 target->qp_in_error) {
34aa654e
BVA		 2837 shost_printk(KERN_INFO, target->scsi_host,
2838 PFX "SCSI scan failed - removing SCSI host\n");
2839 srp_queue_remove_work(target);
2840 goto out;
2841 }
2842
2843 pr_debug(PFX "%s: SCSI scan succeeded - detected %d LUNs\n",
2844 dev_name(&target->scsi_host->shost_gendev),
2845 srp_sdev_count(target->scsi_host));
2846
2847 spin_lock_irq(&target->lock);
2848 if (target->state == SRP_TARGET_SCANNING)
2849 target->state = SRP_TARGET_LIVE;
2850 spin_unlock_irq(&target->lock);
2851
2852out:
aef9ec39
RD		 2853 return 0;
2854}
2855
ee959b00 2856static void srp_release_dev(struct device *dev)
aef9ec39
RD		 2857{
2858 struct srp_host *host =
ee959b00 2859 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 2860
2861 complete(&host->released);
2862}
2863
2864static struct class srp_class = {
2865 .name = "infiniband_srp",
ee959b00 2866 .dev_release = srp_release_dev
aef9ec39
RD		 2867};
2868
96fc248a
BVA		 2869/**
2870 * srp_conn_unique() - check whether the connection to a target is unique
af24663b
BVA		 2871 * @host: SRP host.
2872 * @target: SRP target port.
96fc248a
BVA		 2873 */
2874static bool srp_conn_unique(struct srp_host *host,
2875 struct srp_target_port *target)
2876{
2877 struct srp_target_port *t;
2878 bool ret = false;
2879
2880 if (target->state == SRP_TARGET_REMOVED)
2881 goto out;
2882
2883 ret = true;
2884
2885 spin_lock(&host->target_lock);
2886 list_for_each_entry(t, &host->target_list, list) {
2887 if (t != target &&
2888 target->id_ext == t->id_ext &&
2889 target->ioc_guid == t->ioc_guid &&
2890 target->initiator_ext == t->initiator_ext) {
2891 ret = false;
2892 break;
2893 }
2894 }
2895 spin_unlock(&host->target_lock);
2896
2897out:
2898 return ret;
2899}
2900
aef9ec39
RD		 2901/*
2902 * Target ports are added by writing
2903 *
2904 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2905 * pkey=<P_Key>,service_id=<service ID>
2906 *
2907 * to the add_target sysfs attribute.
2908 */
2909enum {
2910 SRP_OPT_ERR = 0,
2911 SRP_OPT_ID_EXT = 1 << 0,
2912 SRP_OPT_IOC_GUID = 1 << 1,
2913 SRP_OPT_DGID = 1 << 2,
2914 SRP_OPT_PKEY = 1 << 3,
2915 SRP_OPT_SERVICE_ID = 1 << 4,
2916 SRP_OPT_MAX_SECT = 1 << 5,
52fb2b50 2917 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
0c0450db 2918 SRP_OPT_IO_CLASS = 1 << 7,
01cb9bcb 2919 SRP_OPT_INITIATOR_EXT = 1 << 8,
49248644 2920 SRP_OPT_CMD_SG_ENTRIES = 1 << 9,
c07d424d
DD		 2921 SRP_OPT_ALLOW_EXT_SG = 1 << 10,
2922 SRP_OPT_SG_TABLESIZE = 1 << 11,
4b5e5f41 2923 SRP_OPT_COMP_VECTOR = 1 << 12,
7bb312e4 2924 SRP_OPT_TL_RETRY_COUNT = 1 << 13,
4d73f95f 2925 SRP_OPT_QUEUE_SIZE = 1 << 14,
aef9ec39
RD		 2926 SRP_OPT_ALL = (SRP_OPT_ID_EXT |
2927 SRP_OPT_IOC_GUID |
2928 SRP_OPT_DGID |
2929 SRP_OPT_PKEY |
2930 SRP_OPT_SERVICE_ID),
2931};
2932
a447c093 2933static const match_table_t srp_opt_tokens = {
52fb2b50
VP		 2934 { SRP_OPT_ID_EXT, "id_ext=%s" },
2935 { SRP_OPT_IOC_GUID, "ioc_guid=%s" },
2936 { SRP_OPT_DGID, "dgid=%s" },
2937 { SRP_OPT_PKEY, "pkey=%x" },
2938 { SRP_OPT_SERVICE_ID, "service_id=%s" },
2939 { SRP_OPT_MAX_SECT, "max_sect=%d" },
2940 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" },
0c0450db 2941 { SRP_OPT_IO_CLASS, "io_class=%x" },
01cb9bcb 2942 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" },
49248644 2943 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" },
c07d424d
DD		 2944 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" },
2945 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" },
4b5e5f41 2946 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" },
7bb312e4 2947 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" },
4d73f95f 2948 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" },
52fb2b50 2949 { SRP_OPT_ERR, NULL }
aef9ec39
RD		 2950};
2951
2952static int srp_parse_options(const char *buf, struct srp_target_port *target)
2953{
2954 char *options, *sep_opt;
2955 char *p;
2956 char dgid[3];
2957 substring_t args[MAX_OPT_ARGS];
2958 int opt_mask = 0;
2959 int token;
2960 int ret = -EINVAL;
2961 int i;
2962
2963 options = kstrdup(buf, GFP_KERNEL);
2964 if (!options)
2965 return -ENOMEM;
2966
2967 sep_opt = options;
7dcf9c19 2968 while ((p = strsep(&sep_opt, ",\n")) != NULL) {
aef9ec39
RD		 2969 if (!*p)
2970 continue;
2971
2972 token = match_token(p, srp_opt_tokens, args);
2973 opt_mask |= token;
2974
2975 switch (token) {
2976 case SRP_OPT_ID_EXT:
2977 p = match_strdup(args);
a20f3a6d
IR		 2978 if (!p) {
2979 ret = -ENOMEM;
2980 goto out;
2981 }
aef9ec39
RD		 2982 target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2983 kfree(p);
2984 break;
2985
2986 case SRP_OPT_IOC_GUID:
2987 p = match_strdup(args);
a20f3a6d
IR		 2988 if (!p) {
2989 ret = -ENOMEM;
2990 goto out;
2991 }
aef9ec39
RD		 2992 target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
2993 kfree(p);
2994 break;
2995
2996 case SRP_OPT_DGID:
2997 p = match_strdup(args);
a20f3a6d
IR		 2998 if (!p) {
2999 ret = -ENOMEM;
3000 goto out;
3001 }
aef9ec39 3002 if (strlen(p) != 32) {
e0bda7d8 3003 pr_warn("bad dest GID parameter '%s'\n", p);
ce1823f0 3004 kfree(p);
aef9ec39
RD		 3005 goto out;
3006 }
3007
3008 for (i = 0; i < 16; ++i) {
747fe000
BVA		 3009 strlcpy(dgid, p + i * 2, sizeof(dgid));
3010 if (sscanf(dgid, "%hhx",
3011 &target->orig_dgid.raw[i]) < 1) {
3012 ret = -EINVAL;
3013 kfree(p);
3014 goto out;
3015 }
aef9ec39 3016 }
bf17c1c7 3017 kfree(p);
aef9ec39
RD		 3018 break;
3019
3020 case SRP_OPT_PKEY:
3021 if (match_hex(args, &token)) {
e0bda7d8 3022 pr_warn("bad P_Key parameter '%s'\n", p);
aef9ec39
RD		 3023 goto out;
3024 }
747fe000 3025 target->pkey = cpu_to_be16(token);
aef9ec39
RD		 3026 break;
3027
3028 case SRP_OPT_SERVICE_ID:
3029 p = match_strdup(args);
a20f3a6d
IR		 3030 if (!p) {
3031 ret = -ENOMEM;
3032 goto out;
3033 }
aef9ec39
RD		 3034 target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
3035 kfree(p);
3036 break;
3037
3038 case SRP_OPT_MAX_SECT:
3039 if (match_int(args, &token)) {
e0bda7d8 3040 pr_warn("bad max sect parameter '%s'\n", p);
aef9ec39
RD		 3041 goto out;
3042 }
3043 target->scsi_host->max_sectors = token;
3044 break;
3045
4d73f95f
BVA		 3046 case SRP_OPT_QUEUE_SIZE:
3047 if (match_int(args, &token) || token < 1) {
3048 pr_warn("bad queue_size parameter '%s'\n", p);
3049 goto out;
3050 }
3051 target->scsi_host->can_queue = token;
3052 target->queue_size = token + SRP_RSP_SQ_SIZE +
3053 SRP_TSK_MGMT_SQ_SIZE;
3054 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3055 target->scsi_host->cmd_per_lun = token;
3056 break;
3057
52fb2b50 3058 case SRP_OPT_MAX_CMD_PER_LUN:
4d73f95f 3059 if (match_int(args, &token) || token < 1) {
e0bda7d8
BVA		 3060 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3061 p);
52fb2b50
VP		 3062 goto out;
3063 }
4d73f95f 3064 target->scsi_host->cmd_per_lun = token;
52fb2b50
VP		 3065 break;
3066
0c0450db
R		 3067 case SRP_OPT_IO_CLASS:
3068 if (match_hex(args, &token)) {
e0bda7d8 3069 pr_warn("bad IO class parameter '%s'\n", p);
0c0450db
R		 3070 goto out;
3071 }
3072 if (token != SRP_REV10_IB_IO_CLASS &&
3073 token != SRP_REV16A_IB_IO_CLASS) {
e0bda7d8
BVA		 3074 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3075 token, SRP_REV10_IB_IO_CLASS,
3076 SRP_REV16A_IB_IO_CLASS);
0c0450db
R		 3077 goto out;
3078 }
3079 target->io_class = token;
3080 break;
3081
01cb9bcb
IR		 3082 case SRP_OPT_INITIATOR_EXT:
3083 p = match_strdup(args);
a20f3a6d
IR		 3084 if (!p) {
3085 ret = -ENOMEM;
3086 goto out;
3087 }
01cb9bcb
IR		 3088 target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3089 kfree(p);
3090 break;
3091
49248644
DD		 3092 case SRP_OPT_CMD_SG_ENTRIES:
3093 if (match_int(args, &token) || token < 1 || token > 255) {
e0bda7d8
BVA		 3094 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3095 p);
49248644
DD		 3096 goto out;
3097 }
3098 target->cmd_sg_cnt = token;
3099 break;
3100
c07d424d
DD		 3101 case SRP_OPT_ALLOW_EXT_SG:
3102 if (match_int(args, &token)) {
e0bda7d8 3103 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
c07d424d
DD		 3104 goto out;
3105 }
3106 target->allow_ext_sg = !!token;
3107 break;
3108
3109 case SRP_OPT_SG_TABLESIZE:
3110 if (match_int(args, &token) || token < 1 ||
3111 token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
e0bda7d8
BVA		 3112 pr_warn("bad max sg_tablesize parameter '%s'\n",
3113 p);
c07d424d
DD		 3114 goto out;
3115 }
3116 target->sg_tablesize = token;
3117 break;
3118
4b5e5f41
BVA		 3119 case SRP_OPT_COMP_VECTOR:
3120 if (match_int(args, &token) || token < 0) {
3121 pr_warn("bad comp_vector parameter '%s'\n", p);
3122 goto out;
3123 }
3124 target->comp_vector = token;
3125 break;
3126
7bb312e4
VP		 3127 case SRP_OPT_TL_RETRY_COUNT:
3128 if (match_int(args, &token) || token < 2 || token > 7) {
3129 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3130 p);
3131 goto out;
3132 }
3133 target->tl_retry_count = token;
3134 break;
3135
aef9ec39 3136 default:
e0bda7d8
BVA		 3137 pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3138 p);
aef9ec39
RD		 3139 goto out;
3140 }
3141 }
3142
3143 if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3144 ret = 0;
3145 else
3146 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3147 if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3148 !(srp_opt_tokens[i].token & opt_mask))
e0bda7d8
BVA		 3149 pr_warn("target creation request is missing parameter '%s'\n",
3150 srp_opt_tokens[i].pattern);
aef9ec39 3151
4d73f95f
BVA		 3152 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3153 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3154 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3155 target->scsi_host->cmd_per_lun,
3156 target->scsi_host->can_queue);
3157
aef9ec39
RD		 3158out:
3159 kfree(options);
3160 return ret;
3161}
3162
ee959b00
TJ		 3163static ssize_t srp_create_target(struct device *dev,
3164 struct device_attribute *attr,
aef9ec39
RD		 3165 const char *buf, size_t count)
3166{
3167 struct srp_host *host =
ee959b00 3168 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3169 struct Scsi_Host *target_host;
3170 struct srp_target_port *target;
509c07bc 3171 struct srp_rdma_ch *ch;
d1b4289e
BVA		 3172 struct srp_device *srp_dev = host->srp_dev;
3173 struct ib_device *ibdev = srp_dev->dev;
d92c0da7
BVA		 3174 int ret, node_idx, node, cpu, i;
3175 bool multich = false;
aef9ec39
RD		 3176
3177 target_host = scsi_host_alloc(&srp_template,
3178 sizeof (struct srp_target_port));
3179 if (!target_host)
3180 return -ENOMEM;
3181
49248644 3182 target_host->transportt = ib_srp_transport_template;
fd1b6c4a
BVA		 3183 target_host->max_channel = 0;
3184 target_host->max_id = 1;
985aa495 3185 target_host->max_lun = -1LL;
3c8edf0e 3186 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
5f068992 3187
aef9ec39 3188 target = host_to_target(target_host);
aef9ec39 3189
49248644
DD		 3190 target->io_class = SRP_REV16A_IB_IO_CLASS;
3191 target->scsi_host = target_host;
3192 target->srp_host = host;
e6bf5f48 3193 target->lkey = host->srp_dev->pd->local_dma_lkey;
03f6fb93 3194 target->global_mr = host->srp_dev->global_mr;
49248644 3195 target->cmd_sg_cnt = cmd_sg_entries;
c07d424d
DD		 3196 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries;
3197 target->allow_ext_sg = allow_ext_sg;
7bb312e4 3198 target->tl_retry_count = 7;
4d73f95f 3199 target->queue_size = SRP_DEFAULT_QUEUE_SIZE;
aef9ec39 3200
34aa654e
BVA		 3201 /*
3202 * Avoid that the SCSI host can be removed by srp_remove_target()
3203 * before this function returns.
3204 */
3205 scsi_host_get(target->scsi_host);
3206
2d7091bc
BVA		 3207 mutex_lock(&host->add_target_mutex);
3208
aef9ec39
RD		 3209 ret = srp_parse_options(buf, target);
3210 if (ret)
fb49c8bb 3211 goto out;
aef9ec39 3212
4d73f95f
BVA		 3213 target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3214
96fc248a
BVA		 3215 if (!srp_conn_unique(target->srp_host, target)) {
3216 shost_printk(KERN_INFO, target->scsi_host,
3217 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3218 be64_to_cpu(target->id_ext),
3219 be64_to_cpu(target->ioc_guid),
3220 be64_to_cpu(target->initiator_ext));
3221 ret = -EEXIST;
fb49c8bb 3222 goto out;
96fc248a
BVA		 3223 }
3224
5cfb1782 3225 if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
d1b4289e 3226 target->cmd_sg_cnt < target->sg_tablesize) {
5cfb1782 3227 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
c07d424d
DD		 3228 target->sg_tablesize = target->cmd_sg_cnt;
3229 }
3230
3231 target_host->sg_tablesize = target->sg_tablesize;
3232 target->indirect_size = target->sg_tablesize *
3233 sizeof (struct srp_direct_buf);
49248644
DD		 3234 target->max_iu_len = sizeof (struct srp_cmd) +
3235 sizeof (struct srp_indirect_buf) +
3236 target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3237
c1120f89 3238 INIT_WORK(&target->tl_err_work, srp_tl_err_work);
ef6c49d8 3239 INIT_WORK(&target->remove_work, srp_remove_work);
8f26c9ff 3240 spin_lock_init(&target->lock);
55ee3ab2 3241 ret = ib_query_gid(ibdev, host->port, 0, &target->sgid, NULL);
2088ca66 3242 if (ret)
fb49c8bb 3243 goto out;
aef9ec39 3244
d92c0da7
BVA		 3245 ret = -ENOMEM;
3246 target->ch_count = max_t(unsigned, num_online_nodes(),
3247 min(ch_count ? :
3248 min(4 * num_online_nodes(),
3249 ibdev->num_comp_vectors),
3250 num_online_cpus()));
3251 target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3252 GFP_KERNEL);
3253 if (!target->ch)
fb49c8bb 3254 goto out;
aef9ec39 3255
d92c0da7
BVA		 3256 node_idx = 0;
3257 for_each_online_node(node) {
3258 const int ch_start = (node_idx * target->ch_count /
3259 num_online_nodes());
3260 const int ch_end = ((node_idx + 1) * target->ch_count /
3261 num_online_nodes());
3262 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3263 num_online_nodes() + target->comp_vector)
3264 % ibdev->num_comp_vectors;
3265 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3266 num_online_nodes() + target->comp_vector)
3267 % ibdev->num_comp_vectors;
3268 int cpu_idx = 0;
3269
3270 for_each_online_cpu(cpu) {
3271 if (cpu_to_node(cpu) != node)
3272 continue;
3273 if (ch_start + cpu_idx >= ch_end)
3274 continue;
3275 ch = &target->ch[ch_start + cpu_idx];
3276 ch->target = target;
3277 ch->comp_vector = cv_start == cv_end ? cv_start :
3278 cv_start + cpu_idx % (cv_end - cv_start);
3279 spin_lock_init(&ch->lock);
3280 INIT_LIST_HEAD(&ch->free_tx);
3281 ret = srp_new_cm_id(ch);
3282 if (ret)
3283 goto err_disconnect;
aef9ec39 3284
d92c0da7
BVA		 3285 ret = srp_create_ch_ib(ch);
3286 if (ret)
3287 goto err_disconnect;
3288
3289 ret = srp_alloc_req_data(ch);
3290 if (ret)
3291 goto err_disconnect;
3292
3293 ret = srp_connect_ch(ch, multich);
3294 if (ret) {
3295 shost_printk(KERN_ERR, target->scsi_host,
3296 PFX "Connection %d/%d failed\n",
3297 ch_start + cpu_idx,
3298 target->ch_count);
3299 if (node_idx == 0 && cpu_idx == 0) {
3300 goto err_disconnect;
3301 } else {
3302 srp_free_ch_ib(target, ch);
3303 srp_free_req_data(target, ch);
3304 target->ch_count = ch - target->ch;
c257ea6f 3305 goto connected;
d92c0da7
BVA		 3306 }
3307 }
3308
3309 multich = true;
3310 cpu_idx++;
3311 }
3312 node_idx++;
aef9ec39
RD		 3313 }
3314
c257ea6f 3315connected:
d92c0da7
BVA		 3316 target->scsi_host->nr_hw_queues = target->ch_count;
3317
aef9ec39
RD		 3318 ret = srp_add_target(host, target);
3319 if (ret)
3320 goto err_disconnect;
3321
34aa654e
BVA		 3322 if (target->state != SRP_TARGET_REMOVED) {
3323 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3324 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3325 be64_to_cpu(target->id_ext),
3326 be64_to_cpu(target->ioc_guid),
747fe000 3327 be16_to_cpu(target->pkey),
34aa654e 3328 be64_to_cpu(target->service_id),
747fe000 3329 target->sgid.raw, target->orig_dgid.raw);
34aa654e 3330 }
e7ffde01 3331
2d7091bc
BVA		 3332 ret = count;
3333
3334out:
3335 mutex_unlock(&host->add_target_mutex);
34aa654e
BVA		 3336
3337 scsi_host_put(target->scsi_host);
bc44bd1d
BVA		 3338 if (ret < 0)
3339 scsi_host_put(target->scsi_host);
34aa654e 3340
2d7091bc 3341 return ret;
aef9ec39
RD		 3342
3343err_disconnect:
3344 srp_disconnect_target(target);
3345
d92c0da7
BVA		 3346 for (i = 0; i < target->ch_count; i++) {
3347 ch = &target->ch[i];
3348 srp_free_ch_ib(target, ch);
3349 srp_free_req_data(target, ch);
3350 }
aef9ec39 3351
d92c0da7 3352 kfree(target->ch);
2d7091bc 3353 goto out;
aef9ec39
RD		 3354}
3355
ee959b00 3356static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
aef9ec39 3357
ee959b00
TJ		 3358static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3359 char *buf)
aef9ec39 3360{
ee959b00 3361 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39 3362
05321937 3363 return sprintf(buf, "%s\n", host->srp_dev->dev->name);
aef9ec39
RD		 3364}
3365
ee959b00 3366static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
aef9ec39 3367
ee959b00
TJ		 3368static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3369 char *buf)
aef9ec39 3370{
ee959b00 3371 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3372
3373 return sprintf(buf, "%d\n", host->port);
3374}
3375
ee959b00 3376static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
aef9ec39 3377
f5358a17 3378static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
aef9ec39
RD		 3379{
3380 struct srp_host *host;
3381
3382 host = kzalloc(sizeof *host, GFP_KERNEL);
3383 if (!host)
3384 return NULL;
3385
3386 INIT_LIST_HEAD(&host->target_list);
b3589fd4 3387 spin_lock_init(&host->target_lock);
aef9ec39 3388 init_completion(&host->released);
2d7091bc 3389 mutex_init(&host->add_target_mutex);
05321937 3390 host->srp_dev = device;
aef9ec39
RD		 3391 host->port = port;
3392
ee959b00
TJ		 3393 host->dev.class = &srp_class;
3394 host->dev.parent = device->dev->dma_device;
d927e38c 3395 dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
aef9ec39 3396
ee959b00 3397 if (device_register(&host->dev))
f5358a17 3398 goto free_host;
ee959b00 3399 if (device_create_file(&host->dev, &dev_attr_add_target))
aef9ec39 3400 goto err_class;
ee959b00 3401 if (device_create_file(&host->dev, &dev_attr_ibdev))
aef9ec39 3402 goto err_class;
ee959b00 3403 if (device_create_file(&host->dev, &dev_attr_port))
aef9ec39
RD		 3404 goto err_class;
3405
3406 return host;
3407
3408err_class:
ee959b00 3409 device_unregister(&host->dev);
aef9ec39 3410
f5358a17 3411free_host:
aef9ec39
RD		 3412 kfree(host);
3413
3414 return NULL;
3415}
3416
3417static void srp_add_one(struct ib_device *device)
3418{
f5358a17 3419 struct srp_device *srp_dev;
aef9ec39 3420 struct srp_host *host;
4139032b 3421 int mr_page_shift, p;
52ede08f 3422 u64 max_pages_per_mr;
aef9ec39 3423
f5358a17
RD		 3424 srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
3425 if (!srp_dev)
4a061b28 3426 return;
f5358a17 3427
d1b4289e
BVA		 3428 srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3429 device->map_phys_fmr && device->unmap_fmr);
4a061b28 3430 srp_dev->has_fr = (device->attrs.device_cap_flags &
5cfb1782
BVA		 3431 IB_DEVICE_MEM_MGT_EXTENSIONS);
3432 if (!srp_dev->has_fmr && !srp_dev->has_fr)
3433 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3434
3435 srp_dev->use_fast_reg = (srp_dev->has_fr &&
3436 (!srp_dev->has_fmr || prefer_fr));
002f1567 3437 srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
d1b4289e 3438
f5358a17
RD		 3439 /*
3440 * Use the smallest page size supported by the HCA, down to a
8f26c9ff
DD		 3441 * minimum of 4096 bytes. We're unlikely to build large sglists
3442 * out of smaller entries.
f5358a17 3443 */
4a061b28 3444 mr_page_shift = max(12, ffs(device->attrs.page_size_cap) - 1);
52ede08f
BVA		 3445 srp_dev->mr_page_size = 1 << mr_page_shift;
3446 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1);
4a061b28 3447 max_pages_per_mr = device->attrs.max_mr_size;
52ede08f
BVA		 3448 do_div(max_pages_per_mr, srp_dev->mr_page_size);
3449 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3450 max_pages_per_mr);
5cfb1782
BVA		 3451 if (srp_dev->use_fast_reg) {
3452 srp_dev->max_pages_per_mr =
3453 min_t(u32, srp_dev->max_pages_per_mr,
4a061b28 3454 device->attrs.max_fast_reg_page_list_len);
5cfb1782 3455 }
52ede08f
BVA		 3456 srp_dev->mr_max_size = srp_dev->mr_page_size *
3457 srp_dev->max_pages_per_mr;
4a061b28
OG		 3458 pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
3459 device->name, mr_page_shift, device->attrs.max_mr_size,
3460 device->attrs.max_fast_reg_page_list_len,
52ede08f 3461 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
f5358a17
RD		 3462
3463 INIT_LIST_HEAD(&srp_dev->dev_list);
3464
3465 srp_dev->dev = device;
3466 srp_dev->pd = ib_alloc_pd(device);
3467 if (IS_ERR(srp_dev->pd))
3468 goto free_dev;
3469
03f6fb93
BVA		 3470 if (!register_always || (!srp_dev->has_fmr && !srp_dev->has_fr)) {
3471 srp_dev->global_mr = ib_get_dma_mr(srp_dev->pd,
3472 IB_ACCESS_LOCAL_WRITE |
3473 IB_ACCESS_REMOTE_READ |
3474 IB_ACCESS_REMOTE_WRITE);
3475 if (IS_ERR(srp_dev->global_mr))
3476 goto err_pd;
3477 } else {
3478 srp_dev->global_mr = NULL;
3479 }
f5358a17 3480
4139032b 3481 for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
f5358a17 3482 host = srp_add_port(srp_dev, p);
aef9ec39 3483 if (host)
f5358a17 3484 list_add_tail(&host->list, &srp_dev->dev_list);
aef9ec39
RD		 3485 }
3486
f5358a17 3487 ib_set_client_data(device, &srp_client, srp_dev);
4a061b28 3488 return;
f5358a17
RD		 3489
3490err_pd:
3491 ib_dealloc_pd(srp_dev->pd);
3492
3493free_dev:
3494 kfree(srp_dev);
aef9ec39
RD		 3495}
3496
7c1eb45a 3497static void srp_remove_one(struct ib_device *device, void *client_data)
aef9ec39 3498{
f5358a17 3499 struct srp_device *srp_dev;
aef9ec39 3500 struct srp_host *host, *tmp_host;
ef6c49d8 3501 struct srp_target_port *target;
aef9ec39 3502
7c1eb45a 3503 srp_dev = client_data;
1fe0cb84
DB		 3504 if (!srp_dev)
3505 return;
aef9ec39 3506
f5358a17 3507 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
ee959b00 3508 device_unregister(&host->dev);
aef9ec39
RD		 3509 /*
3510 * Wait for the sysfs entry to go away, so that no new
3511 * target ports can be created.
3512 */
3513 wait_for_completion(&host->released);
3514
3515 /*
ef6c49d8 3516 * Remove all target ports.
aef9ec39 3517 */
b3589fd4 3518 spin_lock(&host->target_lock);
ef6c49d8
BVA		 3519 list_for_each_entry(target, &host->target_list, list)
3520 srp_queue_remove_work(target);
b3589fd4 3521 spin_unlock(&host->target_lock);
aef9ec39
RD		 3522
3523 /*
bcc05910 3524 * Wait for tl_err and target port removal tasks.
aef9ec39 3525 */
ef6c49d8 3526 flush_workqueue(system_long_wq);
bcc05910 3527 flush_workqueue(srp_remove_wq);
aef9ec39 3528
aef9ec39
RD		 3529 kfree(host);
3530 }
3531
03f6fb93
BVA		 3532 if (srp_dev->global_mr)
3533 ib_dereg_mr(srp_dev->global_mr);
f5358a17
RD		 3534 ib_dealloc_pd(srp_dev->pd);
3535
3536 kfree(srp_dev);
aef9ec39
RD		 3537}
3538
3236822b 3539static struct srp_function_template ib_srp_transport_functions = {
ed9b2264
BVA		 3540 .has_rport_state = true,
3541 .reset_timer_if_blocked = true,
a95cadb9 3542 .reconnect_delay = &srp_reconnect_delay,
ed9b2264
BVA		 3543 .fast_io_fail_tmo = &srp_fast_io_fail_tmo,
3544 .dev_loss_tmo = &srp_dev_loss_tmo,
3545 .reconnect = srp_rport_reconnect,
dc1bdbd9 3546 .rport_delete = srp_rport_delete,
ed9b2264 3547 .terminate_rport_io = srp_terminate_io,
3236822b
FT		 3548};
3549
aef9ec39
RD		 3550static int __init srp_init_module(void)
3551{
3552 int ret;
3553
49248644 3554 if (srp_sg_tablesize) {
e0bda7d8 3555 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
49248644
DD		 3556 if (!cmd_sg_entries)
3557 cmd_sg_entries = srp_sg_tablesize;
3558 }
3559
3560 if (!cmd_sg_entries)
3561 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3562
3563 if (cmd_sg_entries > 255) {
e0bda7d8 3564 pr_warn("Clamping cmd_sg_entries to 255\n");
49248644 3565 cmd_sg_entries = 255;
1e89a194
DD		 3566 }
3567
c07d424d
DD		 3568 if (!indirect_sg_entries)
3569 indirect_sg_entries = cmd_sg_entries;
3570 else if (indirect_sg_entries < cmd_sg_entries) {
e0bda7d8
BVA		 3571 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3572 cmd_sg_entries);
c07d424d
DD		 3573 indirect_sg_entries = cmd_sg_entries;
3574 }
3575
bcc05910 3576 srp_remove_wq = create_workqueue("srp_remove");
da05be29
WY		 3577 if (!srp_remove_wq) {
3578 ret = -ENOMEM;
bcc05910
BVA		 3579 goto out;
3580 }
3581
3582 ret = -ENOMEM;
3236822b
FT		 3583 ib_srp_transport_template =
3584 srp_attach_transport(&ib_srp_transport_functions);
3585 if (!ib_srp_transport_template)
bcc05910 3586 goto destroy_wq;
3236822b 3587
aef9ec39
RD		 3588 ret = class_register(&srp_class);
3589 if (ret) {
e0bda7d8 3590 pr_err("couldn't register class infiniband_srp\n");
bcc05910 3591 goto release_tr;
aef9ec39
RD		 3592 }
3593
c1a0b23b
MT		 3594 ib_sa_register_client(&srp_sa_client);
3595
aef9ec39
RD		 3596 ret = ib_register_client(&srp_client);
3597 if (ret) {
e0bda7d8 3598 pr_err("couldn't register IB client\n");
bcc05910 3599 goto unreg_sa;
aef9ec39
RD		 3600 }
3601
bcc05910
BVA		 3602out:
3603 return ret;
3604
3605unreg_sa:
3606 ib_sa_unregister_client(&srp_sa_client);
3607 class_unregister(&srp_class);
3608
3609release_tr:
3610 srp_release_transport(ib_srp_transport_template);
3611
3612destroy_wq:
3613 destroy_workqueue(srp_remove_wq);
3614 goto out;
aef9ec39
RD		 3615}
3616
3617static void __exit srp_cleanup_module(void)
3618{
3619 ib_unregister_client(&srp_client);
c1a0b23b 3620 ib_sa_unregister_client(&srp_sa_client);
aef9ec39 3621 class_unregister(&srp_class);
3236822b 3622 srp_release_transport(ib_srp_transport_template);
bcc05910 3623 destroy_workqueue(srp_remove_wq);
aef9ec39
RD		 3624}
3625
3626module_init(srp_init_module);
3627module_exit(srp_cleanup_module);
Linux kernel - Deliverables
This page took 0.994755 seconds and 5 git commands to generate.