projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/srp: Rearrange module description
[deliverable/linux.git] / drivers / infiniband / ulp / srp / ib_srp.c
CommitLineData
aef9ec39
RD		 1/*
2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
aef9ec39
RD		 31 */
32
d236cd0e 33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
e0bda7d8 34
aef9ec39
RD		 35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/err.h>
39#include <linux/string.h>
40#include <linux/parser.h>
41#include <linux/random.h>
de25968c 42#include <linux/jiffies.h>
56b5390c 43#include <rdma/ib_cache.h>
aef9ec39 44
60063497 45#include <linux/atomic.h>
aef9ec39
RD		 46
47#include <scsi/scsi.h>
48#include <scsi/scsi_device.h>
49#include <scsi/scsi_dbg.h>
71444b97 50#include <scsi/scsi_tcq.h>
aef9ec39 51#include <scsi/srp.h>
3236822b 52#include <scsi/scsi_transport_srp.h>
aef9ec39 53
aef9ec39
RD		 54#include "ib_srp.h"
55
56#define DRV_NAME "ib_srp"
57#define PFX DRV_NAME ": "
e8ca4135
VP		 58#define DRV_VERSION "1.0"
59#define DRV_RELDATE "July 1, 2013"
aef9ec39
RD		 60
61MODULE_AUTHOR("Roland Dreier");
33ab3e5b 62MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
aef9ec39 63MODULE_LICENSE("Dual BSD/GPL");
33ab3e5b
BVA		 64MODULE_VERSION(DRV_VERSION);
65MODULE_INFO(release_date, DRV_RELDATE);
aef9ec39 66
49248644
DD		 67static unsigned int srp_sg_tablesize;
68static unsigned int cmd_sg_entries;
c07d424d
DD		 69static unsigned int indirect_sg_entries;
70static bool allow_ext_sg;
5cfb1782 71static bool prefer_fr;
b1b8854d 72static bool register_always;
49248644 73static int topspin_workarounds = 1;
74b0a15b 74
49248644
DD		 75module_param(srp_sg_tablesize, uint, 0444);
76MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
74b0a15b 77
49248644
DD		 78module_param(cmd_sg_entries, uint, 0444);
79MODULE_PARM_DESC(cmd_sg_entries,
80 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
aef9ec39 81
c07d424d
DD		 82module_param(indirect_sg_entries, uint, 0444);
83MODULE_PARM_DESC(indirect_sg_entries,
84 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");
85
86module_param(allow_ext_sg, bool, 0444);
87MODULE_PARM_DESC(allow_ext_sg,
88 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
89
aef9ec39
RD		 90module_param(topspin_workarounds, int, 0444);
91MODULE_PARM_DESC(topspin_workarounds,
92 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
93
5cfb1782
BVA		 94module_param(prefer_fr, bool, 0444);
95MODULE_PARM_DESC(prefer_fr,
96"Whether to use fast registration if both FMR and fast registration are supported");
97
b1b8854d
BVA		 98module_param(register_always, bool, 0444);
99MODULE_PARM_DESC(register_always,
100 "Use memory registration even for contiguous memory regions");
101
ed9b2264
BVA		 102static struct kernel_param_ops srp_tmo_ops;
103
a95cadb9
BVA		 104static int srp_reconnect_delay = 10;
105module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
106 S_IRUGO | S_IWUSR);
107MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
108
ed9b2264
BVA		 109static int srp_fast_io_fail_tmo = 15;
110module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
111 S_IRUGO | S_IWUSR);
112MODULE_PARM_DESC(fast_io_fail_tmo,
113 "Number of seconds between the observation of a transport"
114 " layer error and failing all I/O. \"off\" means that this"
115 " functionality is disabled.");
116
a95cadb9 117static int srp_dev_loss_tmo = 600;
ed9b2264
BVA		 118module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
119 S_IRUGO | S_IWUSR);
120MODULE_PARM_DESC(dev_loss_tmo,
121 "Maximum number of seconds that the SRP transport should"
122 " insulate transport layer errors. After this time has been"
123 " exceeded the SCSI host is removed. Should be"
124 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
125 " if fast_io_fail_tmo has not been set. \"off\" means that"
126 " this functionality is disabled.");
127
d92c0da7
BVA		 128static unsigned ch_count;
129module_param(ch_count, uint, 0444);
130MODULE_PARM_DESC(ch_count,
131 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
132
aef9ec39
RD		 133static void srp_add_one(struct ib_device *device);
134static void srp_remove_one(struct ib_device *device);
509c07bc
BVA		 135static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr);
136static void srp_send_completion(struct ib_cq *cq, void *ch_ptr);
aef9ec39
RD		 137static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
138
3236822b 139static struct scsi_transport_template *ib_srp_transport_template;
bcc05910 140static struct workqueue_struct *srp_remove_wq;
3236822b 141
aef9ec39
RD		 142static struct ib_client srp_client = {
143 .name = "srp",
144 .add = srp_add_one,
145 .remove = srp_remove_one
146};
147
c1a0b23b
MT		 148static struct ib_sa_client srp_sa_client;
149
ed9b2264
BVA		 150static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
151{
152 int tmo = *(int *)kp->arg;
153
154 if (tmo >= 0)
155 return sprintf(buffer, "%d", tmo);
156 else
157 return sprintf(buffer, "off");
158}
159
160static int srp_tmo_set(const char *val, const struct kernel_param *kp)
161{
162 int tmo, res;
163
164 if (strncmp(val, "off", 3) != 0) {
165 res = kstrtoint(val, 0, &tmo);
166 if (res)
167 goto out;
168 } else {
169 tmo = -1;
170 }
a95cadb9
BVA		 171 if (kp->arg == &srp_reconnect_delay)
172 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
173 srp_dev_loss_tmo);
174 else if (kp->arg == &srp_fast_io_fail_tmo)
175 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
ed9b2264 176 else
a95cadb9
BVA		 177 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
178 tmo);
ed9b2264
BVA		 179 if (res)
180 goto out;
181 *(int *)kp->arg = tmo;
182
183out:
184 return res;
185}
186
187static struct kernel_param_ops srp_tmo_ops = {
188 .get = srp_tmo_get,
189 .set = srp_tmo_set,
190};
191
aef9ec39
RD		 192static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
193{
194 return (struct srp_target_port *) host->hostdata;
195}
196
197static const char *srp_target_info(struct Scsi_Host *host)
198{
199 return host_to_target(host)->target_name;
200}
201
5d7cbfd6
RD		 202static int srp_target_is_topspin(struct srp_target_port *target)
203{
204 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
3d1ff48d 205 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d };
5d7cbfd6
RD		 206
207 return topspin_workarounds &&
3d1ff48d
RK		 208 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
209 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
5d7cbfd6
RD		 210}
211
aef9ec39
RD		 212static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
213 gfp_t gfp_mask,
214 enum dma_data_direction direction)
215{
216 struct srp_iu *iu;
217
218 iu = kmalloc(sizeof *iu, gfp_mask);
219 if (!iu)
220 goto out;
221
222 iu->buf = kzalloc(size, gfp_mask);
223 if (!iu->buf)
224 goto out_free_iu;
225
05321937
GKH		 226 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
227 direction);
228 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
aef9ec39
RD		 229 goto out_free_buf;
230
231 iu->size = size;
232 iu->direction = direction;
233
234 return iu;
235
236out_free_buf:
237 kfree(iu->buf);
238out_free_iu:
239 kfree(iu);
240out:
241 return NULL;
242}
243
244static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
245{
246 if (!iu)
247 return;
248
05321937
GKH		 249 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
250 iu->direction);
aef9ec39
RD		 251 kfree(iu->buf);
252 kfree(iu);
253}
254
255static void srp_qp_event(struct ib_event *event, void *context)
256{
e0bda7d8 257 pr_debug("QP event %d\n", event->event);
aef9ec39
RD		 258}
259
260static int srp_init_qp(struct srp_target_port *target,
261 struct ib_qp *qp)
262{
263 struct ib_qp_attr *attr;
264 int ret;
265
266 attr = kmalloc(sizeof *attr, GFP_KERNEL);
267 if (!attr)
268 return -ENOMEM;
269
56b5390c
BVA		 270 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
271 target->srp_host->port,
272 be16_to_cpu(target->pkey),
273 &attr->pkey_index);
aef9ec39
RD		 274 if (ret)
275 goto out;
276
277 attr->qp_state = IB_QPS_INIT;
278 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
279 IB_ACCESS_REMOTE_WRITE);
280 attr->port_num = target->srp_host->port;
281
282 ret = ib_modify_qp(qp, attr,
283 IB_QP_STATE |
284 IB_QP_PKEY_INDEX |
285 IB_QP_ACCESS_FLAGS |
286 IB_QP_PORT);
287
288out:
289 kfree(attr);
290 return ret;
291}
292
509c07bc 293static int srp_new_cm_id(struct srp_rdma_ch *ch)
9fe4bcf4 294{
509c07bc 295 struct srp_target_port *target = ch->target;
9fe4bcf4
DD		 296 struct ib_cm_id *new_cm_id;
297
05321937 298 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
509c07bc 299 srp_cm_handler, ch);
9fe4bcf4
DD		 300 if (IS_ERR(new_cm_id))
301 return PTR_ERR(new_cm_id);
302
509c07bc
BVA		 303 if (ch->cm_id)
304 ib_destroy_cm_id(ch->cm_id);
305 ch->cm_id = new_cm_id;
306 ch->path.sgid = target->sgid;
307 ch->path.dgid = target->orig_dgid;
308 ch->path.pkey = target->pkey;
309 ch->path.service_id = target->service_id;
9fe4bcf4
DD		 310
311 return 0;
312}
313
d1b4289e
BVA		 314static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
315{
316 struct srp_device *dev = target->srp_host->srp_dev;
317 struct ib_fmr_pool_param fmr_param;
318
319 memset(&fmr_param, 0, sizeof(fmr_param));
320 fmr_param.pool_size = target->scsi_host->can_queue;
321 fmr_param.dirty_watermark = fmr_param.pool_size / 4;
322 fmr_param.cache = 1;
52ede08f
BVA		 323 fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
324 fmr_param.page_shift = ilog2(dev->mr_page_size);
d1b4289e
BVA		 325 fmr_param.access = (IB_ACCESS_LOCAL_WRITE |
326 IB_ACCESS_REMOTE_WRITE |
327 IB_ACCESS_REMOTE_READ);
328
329 return ib_create_fmr_pool(dev->pd, &fmr_param);
330}
331
5cfb1782
BVA		 332/**
333 * srp_destroy_fr_pool() - free the resources owned by a pool
334 * @pool: Fast registration pool to be destroyed.
335 */
336static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
337{
338 int i;
339 struct srp_fr_desc *d;
340
341 if (!pool)
342 return;
343
344 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
345 if (d->frpl)
346 ib_free_fast_reg_page_list(d->frpl);
347 if (d->mr)
348 ib_dereg_mr(d->mr);
349 }
350 kfree(pool);
351}
352
353/**
354 * srp_create_fr_pool() - allocate and initialize a pool for fast registration
355 * @device: IB device to allocate fast registration descriptors for.
356 * @pd: Protection domain associated with the FR descriptors.
357 * @pool_size: Number of descriptors to allocate.
358 * @max_page_list_len: Maximum fast registration work request page list length.
359 */
360static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
361 struct ib_pd *pd, int pool_size,
362 int max_page_list_len)
363{
364 struct srp_fr_pool *pool;
365 struct srp_fr_desc *d;
366 struct ib_mr *mr;
367 struct ib_fast_reg_page_list *frpl;
368 int i, ret = -EINVAL;
369
370 if (pool_size <= 0)
371 goto err;
372 ret = -ENOMEM;
373 pool = kzalloc(sizeof(struct srp_fr_pool) +
374 pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
375 if (!pool)
376 goto err;
377 pool->size = pool_size;
378 pool->max_page_list_len = max_page_list_len;
379 spin_lock_init(&pool->lock);
380 INIT_LIST_HEAD(&pool->free_list);
381
382 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
383 mr = ib_alloc_fast_reg_mr(pd, max_page_list_len);
384 if (IS_ERR(mr)) {
385 ret = PTR_ERR(mr);
386 goto destroy_pool;
387 }
388 d->mr = mr;
389 frpl = ib_alloc_fast_reg_page_list(device, max_page_list_len);
390 if (IS_ERR(frpl)) {
391 ret = PTR_ERR(frpl);
392 goto destroy_pool;
393 }
394 d->frpl = frpl;
395 list_add_tail(&d->entry, &pool->free_list);
396 }
397
398out:
399 return pool;
400
401destroy_pool:
402 srp_destroy_fr_pool(pool);
403
404err:
405 pool = ERR_PTR(ret);
406 goto out;
407}
408
409/**
410 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
411 * @pool: Pool to obtain descriptor from.
412 */
413static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
414{
415 struct srp_fr_desc *d = NULL;
416 unsigned long flags;
417
418 spin_lock_irqsave(&pool->lock, flags);
419 if (!list_empty(&pool->free_list)) {
420 d = list_first_entry(&pool->free_list, typeof(*d), entry);
421 list_del(&d->entry);
422 }
423 spin_unlock_irqrestore(&pool->lock, flags);
424
425 return d;
426}
427
428/**
429 * srp_fr_pool_put() - put an FR descriptor back in the free list
430 * @pool: Pool the descriptor was allocated from.
431 * @desc: Pointer to an array of fast registration descriptor pointers.
432 * @n: Number of descriptors to put back.
433 *
434 * Note: The caller must already have queued an invalidation request for
435 * desc->mr->rkey before calling this function.
436 */
437static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
438 int n)
439{
440 unsigned long flags;
441 int i;
442
443 spin_lock_irqsave(&pool->lock, flags);
444 for (i = 0; i < n; i++)
445 list_add(&desc[i]->entry, &pool->free_list);
446 spin_unlock_irqrestore(&pool->lock, flags);
447}
448
449static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
450{
451 struct srp_device *dev = target->srp_host->srp_dev;
452
453 return srp_create_fr_pool(dev->dev, dev->pd,
454 target->scsi_host->can_queue,
455 dev->max_pages_per_mr);
456}
457
7dad6b2e
BVA		 458/**
459 * srp_destroy_qp() - destroy an RDMA queue pair
460 * @ch: SRP RDMA channel.
461 *
462 * Change a queue pair into the error state and wait until all receive
463 * completions have been processed before destroying it. This avoids that
464 * the receive completion handler can access the queue pair while it is
465 * being destroyed.
466 */
467static void srp_destroy_qp(struct srp_rdma_ch *ch)
468{
7dad6b2e
BVA		 469 static struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
470 static struct ib_recv_wr wr = { .wr_id = SRP_LAST_WR_ID };
471 struct ib_recv_wr *bad_wr;
472 int ret;
473
474 /* Destroying a QP and reusing ch->done is only safe if not connected */
c014c8cd 475 WARN_ON_ONCE(ch->connected);
7dad6b2e
BVA		 476
477 ret = ib_modify_qp(ch->qp, &attr, IB_QP_STATE);
478 WARN_ONCE(ret, "ib_cm_init_qp_attr() returned %d\n", ret);
479 if (ret)
480 goto out;
481
482 init_completion(&ch->done);
483 ret = ib_post_recv(ch->qp, &wr, &bad_wr);
484 WARN_ONCE(ret, "ib_post_recv() returned %d\n", ret);
485 if (ret == 0)
486 wait_for_completion(&ch->done);
487
488out:
489 ib_destroy_qp(ch->qp);
490}
491
509c07bc 492static int srp_create_ch_ib(struct srp_rdma_ch *ch)
aef9ec39 493{
509c07bc 494 struct srp_target_port *target = ch->target;
62154b2e 495 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39 496 struct ib_qp_init_attr *init_attr;
73aa89ed
IR		 497 struct ib_cq *recv_cq, *send_cq;
498 struct ib_qp *qp;
d1b4289e 499 struct ib_fmr_pool *fmr_pool = NULL;
5cfb1782
BVA		 500 struct srp_fr_pool *fr_pool = NULL;
501 const int m = 1 + dev->use_fast_reg;
aef9ec39
RD		 502 int ret;
503
504 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
505 if (!init_attr)
506 return -ENOMEM;
507
7dad6b2e 508 /* + 1 for SRP_LAST_WR_ID */
509c07bc 509 recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, ch,
7dad6b2e 510 target->queue_size + 1, ch->comp_vector);
73aa89ed
IR		 511 if (IS_ERR(recv_cq)) {
512 ret = PTR_ERR(recv_cq);
da9d2f07 513 goto err;
aef9ec39
RD		 514 }
515
509c07bc
BVA		 516 send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, ch,
517 m * target->queue_size, ch->comp_vector);
73aa89ed
IR		 518 if (IS_ERR(send_cq)) {
519 ret = PTR_ERR(send_cq);
da9d2f07 520 goto err_recv_cq;
9c03dc9f
BVA		 521 }
522
73aa89ed 523 ib_req_notify_cq(recv_cq, IB_CQ_NEXT_COMP);
aef9ec39
RD		 524
525 init_attr->event_handler = srp_qp_event;
5cfb1782 526 init_attr->cap.max_send_wr = m * target->queue_size;
7dad6b2e 527 init_attr->cap.max_recv_wr = target->queue_size + 1;
aef9ec39
RD		 528 init_attr->cap.max_recv_sge = 1;
529 init_attr->cap.max_send_sge = 1;
5cfb1782 530 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
aef9ec39 531 init_attr->qp_type = IB_QPT_RC;
73aa89ed
IR		 532 init_attr->send_cq = send_cq;
533 init_attr->recv_cq = recv_cq;
aef9ec39 534
62154b2e 535 qp = ib_create_qp(dev->pd, init_attr);
73aa89ed
IR		 536 if (IS_ERR(qp)) {
537 ret = PTR_ERR(qp);
da9d2f07 538 goto err_send_cq;
aef9ec39
RD		 539 }
540
73aa89ed 541 ret = srp_init_qp(target, qp);
da9d2f07
RD		 542 if (ret)
543 goto err_qp;
aef9ec39 544
5cfb1782
BVA		 545 if (dev->use_fast_reg && dev->has_fr) {
546 fr_pool = srp_alloc_fr_pool(target);
547 if (IS_ERR(fr_pool)) {
548 ret = PTR_ERR(fr_pool);
549 shost_printk(KERN_WARNING, target->scsi_host, PFX
550 "FR pool allocation failed (%d)\n", ret);
551 goto err_qp;
552 }
509c07bc
BVA		 553 if (ch->fr_pool)
554 srp_destroy_fr_pool(ch->fr_pool);
555 ch->fr_pool = fr_pool;
5cfb1782 556 } else if (!dev->use_fast_reg && dev->has_fmr) {
d1b4289e
BVA		 557 fmr_pool = srp_alloc_fmr_pool(target);
558 if (IS_ERR(fmr_pool)) {
559 ret = PTR_ERR(fmr_pool);
560 shost_printk(KERN_WARNING, target->scsi_host, PFX
561 "FMR pool allocation failed (%d)\n", ret);
562 goto err_qp;
563 }
509c07bc
BVA		 564 if (ch->fmr_pool)
565 ib_destroy_fmr_pool(ch->fmr_pool);
566 ch->fmr_pool = fmr_pool;
d1b4289e
BVA		 567 }
568
509c07bc 569 if (ch->qp)
7dad6b2e 570 srp_destroy_qp(ch);
509c07bc
BVA		 571 if (ch->recv_cq)
572 ib_destroy_cq(ch->recv_cq);
573 if (ch->send_cq)
574 ib_destroy_cq(ch->send_cq);
73aa89ed 575
509c07bc
BVA		 576 ch->qp = qp;
577 ch->recv_cq = recv_cq;
578 ch->send_cq = send_cq;
73aa89ed 579
da9d2f07
RD		 580 kfree(init_attr);
581 return 0;
582
583err_qp:
73aa89ed 584 ib_destroy_qp(qp);
da9d2f07
RD		 585
586err_send_cq:
73aa89ed 587 ib_destroy_cq(send_cq);
da9d2f07
RD		 588
589err_recv_cq:
73aa89ed 590 ib_destroy_cq(recv_cq);
da9d2f07
RD		 591
592err:
aef9ec39
RD		 593 kfree(init_attr);
594 return ret;
595}
596
4d73f95f
BVA		 597/*
598 * Note: this function may be called without srp_alloc_iu_bufs() having been
509c07bc 599 * invoked. Hence the ch->[rt]x_ring checks.
4d73f95f 600 */
509c07bc
BVA		 601static void srp_free_ch_ib(struct srp_target_port *target,
602 struct srp_rdma_ch *ch)
aef9ec39 603{
5cfb1782 604 struct srp_device *dev = target->srp_host->srp_dev;
aef9ec39
RD		 605 int i;
606
d92c0da7
BVA		 607 if (!ch->target)
608 return;
609
509c07bc
BVA		 610 if (ch->cm_id) {
611 ib_destroy_cm_id(ch->cm_id);
612 ch->cm_id = NULL;
394c595e
BVA		 613 }
614
d92c0da7
BVA		 615 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
616 if (!ch->qp)
617 return;
618
5cfb1782 619 if (dev->use_fast_reg) {
509c07bc
BVA		 620 if (ch->fr_pool)
621 srp_destroy_fr_pool(ch->fr_pool);
5cfb1782 622 } else {
509c07bc
BVA		 623 if (ch->fmr_pool)
624 ib_destroy_fmr_pool(ch->fmr_pool);
5cfb1782 625 }
7dad6b2e 626 srp_destroy_qp(ch);
509c07bc
BVA		 627 ib_destroy_cq(ch->send_cq);
628 ib_destroy_cq(ch->recv_cq);
aef9ec39 629
d92c0da7
BVA		 630 /*
631 * Avoid that the SCSI error handler tries to use this channel after
632 * it has been freed. The SCSI error handler can namely continue
633 * trying to perform recovery actions after scsi_remove_host()
634 * returned.
635 */
636 ch->target = NULL;
637
509c07bc
BVA		 638 ch->qp = NULL;
639 ch->send_cq = ch->recv_cq = NULL;
73aa89ed 640
509c07bc 641 if (ch->rx_ring) {
4d73f95f 642 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 643 srp_free_iu(target->srp_host, ch->rx_ring[i]);
644 kfree(ch->rx_ring);
645 ch->rx_ring = NULL;
4d73f95f 646 }
509c07bc 647 if (ch->tx_ring) {
4d73f95f 648 for (i = 0; i < target->queue_size; ++i)
509c07bc
BVA		 649 srp_free_iu(target->srp_host, ch->tx_ring[i]);
650 kfree(ch->tx_ring);
651 ch->tx_ring = NULL;
4d73f95f 652 }
aef9ec39
RD		 653}
654
655static void srp_path_rec_completion(int status,
656 struct ib_sa_path_rec *pathrec,
509c07bc 657 void *ch_ptr)
aef9ec39 658{
509c07bc
BVA		 659 struct srp_rdma_ch *ch = ch_ptr;
660 struct srp_target_port *target = ch->target;
aef9ec39 661
509c07bc 662 ch->status = status;
aef9ec39 663 if (status)
7aa54bd7
DD		 664 shost_printk(KERN_ERR, target->scsi_host,
665 PFX "Got failed path rec status %d\n", status);
aef9ec39 666 else
509c07bc
BVA		 667 ch->path = *pathrec;
668 complete(&ch->done);
aef9ec39
RD		 669}
670
509c07bc 671static int srp_lookup_path(struct srp_rdma_ch *ch)
aef9ec39 672{
509c07bc 673 struct srp_target_port *target = ch->target;
a702adce
BVA		 674 int ret;
675
509c07bc
BVA		 676 ch->path.numb_path = 1;
677
678 init_completion(&ch->done);
679
680 ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
681 target->srp_host->srp_dev->dev,
682 target->srp_host->port,
683 &ch->path,
684 IB_SA_PATH_REC_SERVICE_ID |
685 IB_SA_PATH_REC_DGID |
686 IB_SA_PATH_REC_SGID |
687 IB_SA_PATH_REC_NUMB_PATH |
688 IB_SA_PATH_REC_PKEY,
689 SRP_PATH_REC_TIMEOUT_MS,
690 GFP_KERNEL,
691 srp_path_rec_completion,
692 ch, &ch->path_query);
693 if (ch->path_query_id < 0)
694 return ch->path_query_id;
695
696 ret = wait_for_completion_interruptible(&ch->done);
a702adce
BVA		 697 if (ret < 0)
698 return ret;
aef9ec39 699
509c07bc 700 if (ch->status < 0)
7aa54bd7
DD		 701 shost_printk(KERN_WARNING, target->scsi_host,
702 PFX "Path record query failed\n");
aef9ec39 703
509c07bc 704 return ch->status;
aef9ec39
RD		 705}
706
d92c0da7 707static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
aef9ec39 708{
509c07bc 709 struct srp_target_port *target = ch->target;
aef9ec39
RD		 710 struct {
711 struct ib_cm_req_param param;
712 struct srp_login_req priv;
713 } *req = NULL;
714 int status;
715
716 req = kzalloc(sizeof *req, GFP_KERNEL);
717 if (!req)
718 return -ENOMEM;
719
509c07bc 720 req->param.primary_path = &ch->path;
aef9ec39
RD		 721 req->param.alternate_path = NULL;
722 req->param.service_id = target->service_id;
509c07bc
BVA		 723 req->param.qp_num = ch->qp->qp_num;
724 req->param.qp_type = ch->qp->qp_type;
aef9ec39
RD		 725 req->param.private_data = &req->priv;
726 req->param.private_data_len = sizeof req->priv;
727 req->param.flow_control = 1;
728
729 get_random_bytes(&req->param.starting_psn, 4);
730 req->param.starting_psn &= 0xffffff;
731
732 /*
733 * Pick some arbitrary defaults here; we could make these
734 * module parameters if anyone cared about setting them.
735 */
736 req->param.responder_resources = 4;
737 req->param.remote_cm_response_timeout = 20;
738 req->param.local_cm_response_timeout = 20;
7bb312e4 739 req->param.retry_count = target->tl_retry_count;
aef9ec39
RD		 740 req->param.rnr_retry_count = 7;
741 req->param.max_cm_retries = 15;
742
743 req->priv.opcode = SRP_LOGIN_REQ;
744 req->priv.tag = 0;
49248644 745 req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
aef9ec39
RD		 746 req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
747 SRP_BUF_FORMAT_INDIRECT);
d92c0da7
BVA		 748 req->priv.req_flags = (multich ? SRP_MULTICHAN_MULTI :
749 SRP_MULTICHAN_SINGLE);
0c0450db 750 /*
3cd96564 751 * In the published SRP specification (draft rev. 16a), the
0c0450db
R		 752 * port identifier format is 8 bytes of ID extension followed
753 * by 8 bytes of GUID. Older drafts put the two halves in the
754 * opposite order, so that the GUID comes first.
755 *
756 * Targets conforming to these obsolete drafts can be
757 * recognized by the I/O Class they report.
758 */
759 if (target->io_class == SRP_REV10_IB_IO_CLASS) {
760 memcpy(req->priv.initiator_port_id,
747fe000 761 &target->sgid.global.interface_id, 8);
0c0450db 762 memcpy(req->priv.initiator_port_id + 8,
01cb9bcb 763 &target->initiator_ext, 8);
0c0450db
R		 764 memcpy(req->priv.target_port_id, &target->ioc_guid, 8);
765 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
766 } else {
767 memcpy(req->priv.initiator_port_id,
01cb9bcb
IR		 768 &target->initiator_ext, 8);
769 memcpy(req->priv.initiator_port_id + 8,
747fe000 770 &target->sgid.global.interface_id, 8);
0c0450db
R		 771 memcpy(req->priv.target_port_id, &target->id_ext, 8);
772 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
773 }
774
aef9ec39
RD		 775 /*
776 * Topspin/Cisco SRP targets will reject our login unless we
01cb9bcb
IR		 777 * zero out the first 8 bytes of our initiator port ID and set
778 * the second 8 bytes to the local node GUID.
aef9ec39 779 */
5d7cbfd6 780 if (srp_target_is_topspin(target)) {
7aa54bd7
DD		 781 shost_printk(KERN_DEBUG, target->scsi_host,
782 PFX "Topspin/Cisco initiator port ID workaround "
783 "activated for target GUID %016llx\n",
45c37cad 784 be64_to_cpu(target->ioc_guid));
aef9ec39 785 memset(req->priv.initiator_port_id, 0, 8);
01cb9bcb 786 memcpy(req->priv.initiator_port_id + 8,
05321937 787 &target->srp_host->srp_dev->dev->node_guid, 8);
aef9ec39 788 }
aef9ec39 789
509c07bc 790 status = ib_send_cm_req(ch->cm_id, &req->param);
aef9ec39
RD		 791
792 kfree(req);
793
794 return status;
795}
796
ef6c49d8
BVA		 797static bool srp_queue_remove_work(struct srp_target_port *target)
798{
799 bool changed = false;
800
801 spin_lock_irq(&target->lock);
802 if (target->state != SRP_TARGET_REMOVED) {
803 target->state = SRP_TARGET_REMOVED;
804 changed = true;
805 }
806 spin_unlock_irq(&target->lock);
807
808 if (changed)
bcc05910 809 queue_work(srp_remove_wq, &target->remove_work);
ef6c49d8
BVA		 810
811 return changed;
812}
813
aef9ec39
RD		 814static void srp_disconnect_target(struct srp_target_port *target)
815{
d92c0da7
BVA		 816 struct srp_rdma_ch *ch;
817 int i;
509c07bc 818
c014c8cd 819 /* XXX should send SRP_I_LOGOUT request */
aef9ec39 820
c014c8cd
BVA		 821 for (i = 0; i < target->ch_count; i++) {
822 ch = &target->ch[i];
823 ch->connected = false;
824 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
825 shost_printk(KERN_DEBUG, target->scsi_host,
826 PFX "Sending CM DREQ failed\n");
294c875a 827 }
e6581056 828 }
aef9ec39
RD		 829}
830
509c07bc
BVA		 831static void srp_free_req_data(struct srp_target_port *target,
832 struct srp_rdma_ch *ch)
8f26c9ff 833{
5cfb1782
BVA		 834 struct srp_device *dev = target->srp_host->srp_dev;
835 struct ib_device *ibdev = dev->dev;
8f26c9ff
DD		 836 struct srp_request *req;
837 int i;
838
d92c0da7 839 if (!ch->target || !ch->req_ring)
4d73f95f
BVA		 840 return;
841
842 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 843 req = &ch->req_ring[i];
5cfb1782
BVA		 844 if (dev->use_fast_reg)
845 kfree(req->fr_list);
846 else
847 kfree(req->fmr_list);
8f26c9ff 848 kfree(req->map_page);
c07d424d
DD		 849 if (req->indirect_dma_addr) {
850 ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
851 target->indirect_size,
852 DMA_TO_DEVICE);
853 }
854 kfree(req->indirect_desc);
8f26c9ff 855 }
4d73f95f 856
509c07bc
BVA		 857 kfree(ch->req_ring);
858 ch->req_ring = NULL;
8f26c9ff
DD		 859}
860
509c07bc 861static int srp_alloc_req_data(struct srp_rdma_ch *ch)
b81d00bd 862{
509c07bc 863 struct srp_target_port *target = ch->target;
b81d00bd
BVA		 864 struct srp_device *srp_dev = target->srp_host->srp_dev;
865 struct ib_device *ibdev = srp_dev->dev;
866 struct srp_request *req;
5cfb1782 867 void *mr_list;
b81d00bd
BVA		 868 dma_addr_t dma_addr;
869 int i, ret = -ENOMEM;
870
509c07bc
BVA		 871 ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
872 GFP_KERNEL);
873 if (!ch->req_ring)
4d73f95f
BVA		 874 goto out;
875
876 for (i = 0; i < target->req_ring_size; ++i) {
509c07bc 877 req = &ch->req_ring[i];
5cfb1782
BVA		 878 mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
879 GFP_KERNEL);
880 if (!mr_list)
881 goto out;
882 if (srp_dev->use_fast_reg)
883 req->fr_list = mr_list;
884 else
885 req->fmr_list = mr_list;
52ede08f 886 req->map_page = kmalloc(srp_dev->max_pages_per_mr *
d1b4289e 887 sizeof(void *), GFP_KERNEL);
5cfb1782
BVA		 888 if (!req->map_page)
889 goto out;
b81d00bd 890 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
5cfb1782 891 if (!req->indirect_desc)
b81d00bd
BVA		 892 goto out;
893
894 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
895 target->indirect_size,
896 DMA_TO_DEVICE);
897 if (ib_dma_mapping_error(ibdev, dma_addr))
898 goto out;
899
900 req->indirect_dma_addr = dma_addr;
b81d00bd
BVA		 901 }
902 ret = 0;
903
904out:
905 return ret;
906}
907
683b159a
BVA		 908/**
909 * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
910 * @shost: SCSI host whose attributes to remove from sysfs.
911 *
912 * Note: Any attributes defined in the host template and that did not exist
913 * before invocation of this function will be ignored.
914 */
915static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
916{
917 struct device_attribute **attr;
918
919 for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
920 device_remove_file(&shost->shost_dev, *attr);
921}
922
ee12d6a8
BVA		 923static void srp_remove_target(struct srp_target_port *target)
924{
d92c0da7
BVA		 925 struct srp_rdma_ch *ch;
926 int i;
509c07bc 927
ef6c49d8
BVA		 928 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
929
ee12d6a8 930 srp_del_scsi_host_attr(target->scsi_host);
9dd69a60 931 srp_rport_get(target->rport);
ee12d6a8
BVA		 932 srp_remove_host(target->scsi_host);
933 scsi_remove_host(target->scsi_host);
93079162 934 srp_stop_rport_timers(target->rport);
ef6c49d8 935 srp_disconnect_target(target);
d92c0da7
BVA		 936 for (i = 0; i < target->ch_count; i++) {
937 ch = &target->ch[i];
938 srp_free_ch_ib(target, ch);
939 }
c1120f89 940 cancel_work_sync(&target->tl_err_work);
9dd69a60 941 srp_rport_put(target->rport);
d92c0da7
BVA		 942 for (i = 0; i < target->ch_count; i++) {
943 ch = &target->ch[i];
944 srp_free_req_data(target, ch);
945 }
946 kfree(target->ch);
947 target->ch = NULL;
65d7dd2f
VP		 948
949 spin_lock(&target->srp_host->target_lock);
950 list_del(&target->list);
951 spin_unlock(&target->srp_host->target_lock);
952
ee12d6a8
BVA		 953 scsi_host_put(target->scsi_host);
954}
955
c4028958 956static void srp_remove_work(struct work_struct *work)
aef9ec39 957{
c4028958 958 struct srp_target_port *target =
ef6c49d8 959 container_of(work, struct srp_target_port, remove_work);
aef9ec39 960
ef6c49d8 961 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
aef9ec39 962
96fc248a 963 srp_remove_target(target);
aef9ec39
RD		 964}
965
dc1bdbd9
BVA		 966static void srp_rport_delete(struct srp_rport *rport)
967{
968 struct srp_target_port *target = rport->lld_data;
969
970 srp_queue_remove_work(target);
971}
972
c014c8cd
BVA		 973/**
974 * srp_connected_ch() - number of connected channels
975 * @target: SRP target port.
976 */
977static int srp_connected_ch(struct srp_target_port *target)
978{
979 int i, c = 0;
980
981 for (i = 0; i < target->ch_count; i++)
982 c += target->ch[i].connected;
983
984 return c;
985}
986
d92c0da7 987static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
aef9ec39 988{
509c07bc 989 struct srp_target_port *target = ch->target;
aef9ec39
RD		 990 int ret;
991
c014c8cd 992 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
294c875a 993
509c07bc 994 ret = srp_lookup_path(ch);
aef9ec39
RD		 995 if (ret)
996 return ret;
997
998 while (1) {
509c07bc 999 init_completion(&ch->done);
d92c0da7 1000 ret = srp_send_req(ch, multich);
aef9ec39
RD		 1001 if (ret)
1002 return ret;
509c07bc 1003 ret = wait_for_completion_interruptible(&ch->done);
a702adce
BVA		 1004 if (ret < 0)
1005 return ret;
aef9ec39
RD		 1006
1007 /*
1008 * The CM event handling code will set status to
1009 * SRP_PORT_REDIRECT if we get a port redirect REJ
1010 * back, or SRP_DLID_REDIRECT if we get a lid/qp
1011 * redirect REJ back.
1012 */
509c07bc 1013 switch (ch->status) {
aef9ec39 1014 case 0:
c014c8cd 1015 ch->connected = true;
aef9ec39
RD		 1016 return 0;
1017
1018 case SRP_PORT_REDIRECT:
509c07bc 1019 ret = srp_lookup_path(ch);
aef9ec39
RD		 1020 if (ret)
1021 return ret;
1022 break;
1023
1024 case SRP_DLID_REDIRECT:
1025 break;
1026
9fe4bcf4 1027 case SRP_STALE_CONN:
9fe4bcf4 1028 shost_printk(KERN_ERR, target->scsi_host, PFX
205619f2 1029 "giving up on stale connection\n");
509c07bc
BVA		 1030 ch->status = -ECONNRESET;
1031 return ch->status;
9fe4bcf4 1032
aef9ec39 1033 default:
509c07bc 1034 return ch->status;
aef9ec39
RD		 1035 }
1036 }
1037}
1038
509c07bc 1039static int srp_inv_rkey(struct srp_rdma_ch *ch, u32 rkey)
5cfb1782
BVA		 1040{
1041 struct ib_send_wr *bad_wr;
1042 struct ib_send_wr wr = {
1043 .opcode = IB_WR_LOCAL_INV,
1044 .wr_id = LOCAL_INV_WR_ID_MASK,
1045 .next = NULL,
1046 .num_sge = 0,
1047 .send_flags = 0,
1048 .ex.invalidate_rkey = rkey,
1049 };
1050
509c07bc 1051 return ib_post_send(ch->qp, &wr, &bad_wr);
5cfb1782
BVA		 1052}
1053
d945e1df 1054static void srp_unmap_data(struct scsi_cmnd *scmnd,
509c07bc 1055 struct srp_rdma_ch *ch,
d945e1df
RD		 1056 struct srp_request *req)
1057{
509c07bc 1058 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1059 struct srp_device *dev = target->srp_host->srp_dev;
1060 struct ib_device *ibdev = dev->dev;
1061 int i, res;
8f26c9ff 1062
bb350d1d 1063 if (!scsi_sglist(scmnd) ||
d945e1df
RD		 1064 (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1065 scmnd->sc_data_direction != DMA_FROM_DEVICE))
1066 return;
1067
5cfb1782
BVA		 1068 if (dev->use_fast_reg) {
1069 struct srp_fr_desc **pfr;
1070
1071 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
509c07bc 1072 res = srp_inv_rkey(ch, (*pfr)->mr->rkey);
5cfb1782
BVA		 1073 if (res < 0) {
1074 shost_printk(KERN_ERR, target->scsi_host, PFX
1075 "Queueing INV WR for rkey %#x failed (%d)\n",
1076 (*pfr)->mr->rkey, res);
1077 queue_work(system_long_wq,
1078 &target->tl_err_work);
1079 }
1080 }
1081 if (req->nmdesc)
509c07bc 1082 srp_fr_pool_put(ch->fr_pool, req->fr_list,
5cfb1782
BVA		 1083 req->nmdesc);
1084 } else {
1085 struct ib_pool_fmr **pfmr;
1086
1087 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1088 ib_fmr_pool_unmap(*pfmr);
1089 }
f5358a17 1090
8f26c9ff
DD		 1091 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1092 scmnd->sc_data_direction);
d945e1df
RD		 1093}
1094
22032991
BVA		 1095/**
1096 * srp_claim_req - Take ownership of the scmnd associated with a request.
509c07bc 1097 * @ch: SRP RDMA channel.
22032991 1098 * @req: SRP request.
b3fe628d 1099 * @sdev: If not NULL, only take ownership for this SCSI device.
22032991
BVA		 1100 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1101 * ownership of @req->scmnd if it equals @scmnd.
1102 *
1103 * Return value:
1104 * Either NULL or a pointer to the SCSI command the caller became owner of.
1105 */
509c07bc 1106static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
22032991 1107 struct srp_request *req,
b3fe628d 1108 struct scsi_device *sdev,
22032991
BVA		 1109 struct scsi_cmnd *scmnd)
1110{
1111 unsigned long flags;
1112
509c07bc 1113 spin_lock_irqsave(&ch->lock, flags);
b3fe628d
BVA		 1114 if (req->scmnd &&
1115 (!sdev || req->scmnd->device == sdev) &&
1116 (!scmnd || req->scmnd == scmnd)) {
22032991
BVA		 1117 scmnd = req->scmnd;
1118 req->scmnd = NULL;
22032991
BVA		 1119 } else {
1120 scmnd = NULL;
1121 }
509c07bc 1122 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1123
1124 return scmnd;
1125}
1126
1127/**
1128 * srp_free_req() - Unmap data and add request to the free request list.
509c07bc 1129 * @ch: SRP RDMA channel.
af24663b
BVA		 1130 * @req: Request to be freed.
1131 * @scmnd: SCSI command associated with @req.
1132 * @req_lim_delta: Amount to be added to @target->req_lim.
22032991 1133 */
509c07bc
BVA		 1134static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1135 struct scsi_cmnd *scmnd, s32 req_lim_delta)
526b4caa 1136{
94a9174c
BVA		 1137 unsigned long flags;
1138
509c07bc 1139 srp_unmap_data(scmnd, ch, req);
22032991 1140
509c07bc
BVA		 1141 spin_lock_irqsave(&ch->lock, flags);
1142 ch->req_lim += req_lim_delta;
509c07bc 1143 spin_unlock_irqrestore(&ch->lock, flags);
526b4caa
IR		 1144}
1145
509c07bc
BVA		 1146static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1147 struct scsi_device *sdev, int result)
526b4caa 1148{
509c07bc 1149 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
22032991
BVA		 1150
1151 if (scmnd) {
509c07bc 1152 srp_free_req(ch, req, scmnd, 0);
ed9b2264 1153 scmnd->result = result;
22032991 1154 scmnd->scsi_done(scmnd);
22032991 1155 }
526b4caa
IR		 1156}
1157
ed9b2264 1158static void srp_terminate_io(struct srp_rport *rport)
aef9ec39 1159{
ed9b2264 1160 struct srp_target_port *target = rport->lld_data;
d92c0da7 1161 struct srp_rdma_ch *ch;
b3fe628d
BVA		 1162 struct Scsi_Host *shost = target->scsi_host;
1163 struct scsi_device *sdev;
d92c0da7 1164 int i, j;
ed9b2264 1165
b3fe628d
BVA		 1166 /*
1167 * Invoking srp_terminate_io() while srp_queuecommand() is running
1168 * is not safe. Hence the warning statement below.
1169 */
1170 shost_for_each_device(sdev, shost)
1171 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1172
d92c0da7
BVA		 1173 for (i = 0; i < target->ch_count; i++) {
1174 ch = &target->ch[i];
509c07bc 1175
d92c0da7
BVA		 1176 for (j = 0; j < target->req_ring_size; ++j) {
1177 struct srp_request *req = &ch->req_ring[j];
1178
1179 srp_finish_req(ch, req, NULL,
1180 DID_TRANSPORT_FAILFAST << 16);
1181 }
ed9b2264
BVA		 1182 }
1183}
aef9ec39 1184
ed9b2264
BVA		 1185/*
1186 * It is up to the caller to ensure that srp_rport_reconnect() calls are
1187 * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1188 * srp_reset_device() or srp_reset_host() calls will occur while this function
1189 * is in progress. One way to realize that is not to call this function
1190 * directly but to call srp_reconnect_rport() instead since that last function
1191 * serializes calls of this function via rport->mutex and also blocks
1192 * srp_queuecommand() calls before invoking this function.
1193 */
1194static int srp_rport_reconnect(struct srp_rport *rport)
1195{
1196 struct srp_target_port *target = rport->lld_data;
d92c0da7
BVA		 1197 struct srp_rdma_ch *ch;
1198 int i, j, ret = 0;
1199 bool multich = false;
09be70a2 1200
aef9ec39 1201 srp_disconnect_target(target);
34aa654e
BVA		 1202
1203 if (target->state == SRP_TARGET_SCANNING)
1204 return -ENODEV;
1205
aef9ec39 1206 /*
c7c4e7ff
BVA		 1207 * Now get a new local CM ID so that we avoid confusing the target in
1208 * case things are really fouled up. Doing so also ensures that all CM
1209 * callbacks will have finished before a new QP is allocated.
aef9ec39 1210 */
d92c0da7
BVA		 1211 for (i = 0; i < target->ch_count; i++) {
1212 ch = &target->ch[i];
1213 if (!ch->target)
1214 break;
1215 ret += srp_new_cm_id(ch);
536ae14e 1216 }
d92c0da7
BVA		 1217 for (i = 0; i < target->ch_count; i++) {
1218 ch = &target->ch[i];
1219 if (!ch->target)
1220 break;
1221 for (j = 0; j < target->req_ring_size; ++j) {
1222 struct srp_request *req = &ch->req_ring[j];
aef9ec39 1223
d92c0da7
BVA		 1224 srp_finish_req(ch, req, NULL, DID_RESET << 16);
1225 }
1226 }
1227 for (i = 0; i < target->ch_count; i++) {
1228 ch = &target->ch[i];
1229 if (!ch->target)
1230 break;
1231 /*
1232 * Whether or not creating a new CM ID succeeded, create a new
1233 * QP. This guarantees that all completion callback function
1234 * invocations have finished before request resetting starts.
1235 */
1236 ret += srp_create_ch_ib(ch);
aef9ec39 1237
d92c0da7
BVA		 1238 INIT_LIST_HEAD(&ch->free_tx);
1239 for (j = 0; j < target->queue_size; ++j)
1240 list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1241 }
8de9fe3a
BVA		 1242
1243 target->qp_in_error = false;
1244
d92c0da7
BVA		 1245 for (i = 0; i < target->ch_count; i++) {
1246 ch = &target->ch[i];
a44074f1 1247 if (ret || !ch->target)
d92c0da7 1248 break;
d92c0da7
BVA		 1249 ret = srp_connect_ch(ch, multich);
1250 multich = true;
1251 }
09be70a2 1252
ed9b2264
BVA		 1253 if (ret == 0)
1254 shost_printk(KERN_INFO, target->scsi_host,
1255 PFX "reconnect succeeded\n");
aef9ec39
RD		 1256
1257 return ret;
1258}
1259
8f26c9ff
DD		 1260static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1261 unsigned int dma_len, u32 rkey)
f5358a17 1262{
8f26c9ff 1263 struct srp_direct_buf *desc = state->desc;
f5358a17 1264
8f26c9ff
DD		 1265 desc->va = cpu_to_be64(dma_addr);
1266 desc->key = cpu_to_be32(rkey);
1267 desc->len = cpu_to_be32(dma_len);
f5358a17 1268
8f26c9ff
DD		 1269 state->total_len += dma_len;
1270 state->desc++;
1271 state->ndesc++;
1272}
559ce8f1 1273
8f26c9ff 1274static int srp_map_finish_fmr(struct srp_map_state *state,
509c07bc 1275 struct srp_rdma_ch *ch)
8f26c9ff 1276{
8f26c9ff
DD		 1277 struct ib_pool_fmr *fmr;
1278 u64 io_addr = 0;
85507bcc 1279
509c07bc 1280 fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
8f26c9ff
DD		 1281 state->npages, io_addr);
1282 if (IS_ERR(fmr))
1283 return PTR_ERR(fmr);
f5358a17 1284
8f26c9ff 1285 *state->next_fmr++ = fmr;
52ede08f 1286 state->nmdesc++;
f5358a17 1287
52ede08f 1288 srp_map_desc(state, 0, state->dma_len, fmr->fmr->rkey);
539dde6f 1289
8f26c9ff
DD		 1290 return 0;
1291}
1292
5cfb1782 1293static int srp_map_finish_fr(struct srp_map_state *state,
509c07bc 1294 struct srp_rdma_ch *ch)
5cfb1782 1295{
509c07bc 1296 struct srp_target_port *target = ch->target;
5cfb1782
BVA		 1297 struct srp_device *dev = target->srp_host->srp_dev;
1298 struct ib_send_wr *bad_wr;
1299 struct ib_send_wr wr;
1300 struct srp_fr_desc *desc;
1301 u32 rkey;
1302
509c07bc 1303 desc = srp_fr_pool_get(ch->fr_pool);
5cfb1782
BVA		 1304 if (!desc)
1305 return -ENOMEM;
1306
1307 rkey = ib_inc_rkey(desc->mr->rkey);
1308 ib_update_fast_reg_key(desc->mr, rkey);
1309
1310 memcpy(desc->frpl->page_list, state->pages,
1311 sizeof(state->pages[0]) * state->npages);
1312
1313 memset(&wr, 0, sizeof(wr));
1314 wr.opcode = IB_WR_FAST_REG_MR;
1315 wr.wr_id = FAST_REG_WR_ID_MASK;
1316 wr.wr.fast_reg.iova_start = state->base_dma_addr;
1317 wr.wr.fast_reg.page_list = desc->frpl;
1318 wr.wr.fast_reg.page_list_len = state->npages;
1319 wr.wr.fast_reg.page_shift = ilog2(dev->mr_page_size);
1320 wr.wr.fast_reg.length = state->dma_len;
1321 wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE |
1322 IB_ACCESS_REMOTE_READ |
1323 IB_ACCESS_REMOTE_WRITE);
1324 wr.wr.fast_reg.rkey = desc->mr->lkey;
1325
1326 *state->next_fr++ = desc;
1327 state->nmdesc++;
1328
1329 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1330 desc->mr->rkey);
1331
509c07bc 1332 return ib_post_send(ch->qp, &wr, &bad_wr);
5cfb1782
BVA		 1333}
1334
539dde6f 1335static int srp_finish_mapping(struct srp_map_state *state,
509c07bc 1336 struct srp_rdma_ch *ch)
539dde6f 1337{
509c07bc 1338 struct srp_target_port *target = ch->target;
539dde6f
BVA		 1339 int ret = 0;
1340
1341 if (state->npages == 0)
1342 return 0;
1343
b1b8854d 1344 if (state->npages == 1 && !register_always)
52ede08f 1345 srp_map_desc(state, state->base_dma_addr, state->dma_len,
539dde6f
BVA		 1346 target->rkey);
1347 else
5cfb1782 1348 ret = target->srp_host->srp_dev->use_fast_reg ?
509c07bc
BVA		 1349 srp_map_finish_fr(state, ch) :
1350 srp_map_finish_fmr(state, ch);
539dde6f
BVA		 1351
1352 if (ret == 0) {
1353 state->npages = 0;
52ede08f 1354 state->dma_len = 0;
539dde6f
BVA		 1355 }
1356
1357 return ret;
1358}
1359
8f26c9ff
DD		 1360static void srp_map_update_start(struct srp_map_state *state,
1361 struct scatterlist *sg, int sg_index,
1362 dma_addr_t dma_addr)
1363{
1364 state->unmapped_sg = sg;
1365 state->unmapped_index = sg_index;
1366 state->unmapped_addr = dma_addr;
1367}
85507bcc 1368
8f26c9ff 1369static int srp_map_sg_entry(struct srp_map_state *state,
509c07bc 1370 struct srp_rdma_ch *ch,
8f26c9ff 1371 struct scatterlist *sg, int sg_index,
5cfb1782 1372 bool use_mr)
8f26c9ff 1373{
509c07bc 1374 struct srp_target_port *target = ch->target;
8f26c9ff
DD		 1375 struct srp_device *dev = target->srp_host->srp_dev;
1376 struct ib_device *ibdev = dev->dev;
1377 dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1378 unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1379 unsigned int len;
1380 int ret;
1381
1382 if (!dma_len)
1383 return 0;
1384
5cfb1782
BVA		 1385 if (!use_mr) {
1386 /*
1387 * Once we're in direct map mode for a request, we don't
1388 * go back to FMR or FR mode, so no need to update anything
8f26c9ff
DD		 1389 * other than the descriptor.
1390 */
1391 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1392 return 0;
85507bcc 1393 }
f5358a17 1394
5cfb1782
BVA		 1395 /*
1396 * Since not all RDMA HW drivers support non-zero page offsets for
1397 * FMR, if we start at an offset into a page, don't merge into the
1398 * current FMR mapping. Finish it out, and use the kernel's MR for
1399 * this sg entry.
8f26c9ff 1400 */
5cfb1782
BVA		 1401 if ((!dev->use_fast_reg && dma_addr & ~dev->mr_page_mask) ||
1402 dma_len > dev->mr_max_size) {
509c07bc 1403 ret = srp_finish_mapping(state, ch);
8f26c9ff
DD		 1404 if (ret)
1405 return ret;
1406
1407 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1408 srp_map_update_start(state, NULL, 0, 0);
1409 return 0;
f5358a17
RD		 1410 }
1411
5cfb1782
BVA		 1412 /*
1413 * If this is the first sg that will be mapped via FMR or via FR, save
1414 * our position. We need to know the first unmapped entry, its index,
1415 * and the first unmapped address within that entry to be able to
1416 * restart mapping after an error.
8f26c9ff
DD		 1417 */
1418 if (!state->unmapped_sg)
1419 srp_map_update_start(state, sg, sg_index, dma_addr);
f5358a17 1420
8f26c9ff 1421 while (dma_len) {
5cfb1782
BVA		 1422 unsigned offset = dma_addr & ~dev->mr_page_mask;
1423 if (state->npages == dev->max_pages_per_mr || offset != 0) {
509c07bc 1424 ret = srp_finish_mapping(state, ch);
8f26c9ff
DD		 1425 if (ret)
1426 return ret;
f5358a17 1427
8f26c9ff
DD		 1428 srp_map_update_start(state, sg, sg_index, dma_addr);
1429 }
1430
5cfb1782 1431 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
f5358a17 1432
8f26c9ff
DD		 1433 if (!state->npages)
1434 state->base_dma_addr = dma_addr;
5cfb1782 1435 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
52ede08f 1436 state->dma_len += len;
8f26c9ff
DD		 1437 dma_addr += len;
1438 dma_len -= len;
1439 }
1440
5cfb1782
BVA		 1441 /*
1442 * If the last entry of the MR wasn't a full page, then we need to
8f26c9ff
DD		 1443 * close it out and start a new one -- we can only merge at page
1444 * boundries.
1445 */
1446 ret = 0;
52ede08f 1447 if (len != dev->mr_page_size) {
509c07bc 1448 ret = srp_finish_mapping(state, ch);
8f26c9ff
DD		 1449 if (!ret)
1450 srp_map_update_start(state, NULL, 0, 0);
1451 }
f5358a17
RD		 1452 return ret;
1453}
1454
509c07bc
BVA		 1455static int srp_map_sg(struct srp_map_state *state, struct srp_rdma_ch *ch,
1456 struct srp_request *req, struct scatterlist *scat,
1457 int count)
76bc1e1d 1458{
509c07bc 1459 struct srp_target_port *target = ch->target;
76bc1e1d
BVA		 1460 struct srp_device *dev = target->srp_host->srp_dev;
1461 struct ib_device *ibdev = dev->dev;
1462 struct scatterlist *sg;
5cfb1782
BVA		 1463 int i;
1464 bool use_mr;
76bc1e1d
BVA		 1465
1466 state->desc = req->indirect_desc;
1467 state->pages = req->map_page;
5cfb1782
BVA		 1468 if (dev->use_fast_reg) {
1469 state->next_fr = req->fr_list;
509c07bc 1470 use_mr = !!ch->fr_pool;
5cfb1782
BVA		 1471 } else {
1472 state->next_fmr = req->fmr_list;
509c07bc 1473 use_mr = !!ch->fmr_pool;
5cfb1782 1474 }
76bc1e1d
BVA		 1475
1476 for_each_sg(scat, sg, count, i) {
509c07bc 1477 if (srp_map_sg_entry(state, ch, sg, i, use_mr)) {
5cfb1782
BVA		 1478 /*
1479 * Memory registration failed, so backtrack to the
1480 * first unmapped entry and continue on without using
1481 * memory registration.
76bc1e1d
BVA		 1482 */
1483 dma_addr_t dma_addr;
1484 unsigned int dma_len;
1485
1486backtrack:
1487 sg = state->unmapped_sg;
1488 i = state->unmapped_index;
1489
1490 dma_addr = ib_sg_dma_address(ibdev, sg);
1491 dma_len = ib_sg_dma_len(ibdev, sg);
1492 dma_len -= (state->unmapped_addr - dma_addr);
1493 dma_addr = state->unmapped_addr;
5cfb1782 1494 use_mr = false;
76bc1e1d
BVA		 1495 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1496 }
1497 }
1498
509c07bc 1499 if (use_mr && srp_finish_mapping(state, ch))
76bc1e1d
BVA		 1500 goto backtrack;
1501
52ede08f 1502 req->nmdesc = state->nmdesc;
5cfb1782
BVA		 1503
1504 return 0;
76bc1e1d
BVA		 1505}
1506
509c07bc 1507static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
aef9ec39
RD		 1508 struct srp_request *req)
1509{
509c07bc 1510 struct srp_target_port *target = ch->target;
76bc1e1d 1511 struct scatterlist *scat;
aef9ec39 1512 struct srp_cmd *cmd = req->cmd->buf;
76bc1e1d 1513 int len, nents, count;
85507bcc
RC		 1514 struct srp_device *dev;
1515 struct ib_device *ibdev;
8f26c9ff
DD		 1516 struct srp_map_state state;
1517 struct srp_indirect_buf *indirect_hdr;
8f26c9ff
DD		 1518 u32 table_len;
1519 u8 fmt;
aef9ec39 1520
bb350d1d 1521 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
aef9ec39
RD		 1522 return sizeof (struct srp_cmd);
1523
1524 if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1525 scmnd->sc_data_direction != DMA_TO_DEVICE) {
7aa54bd7
DD		 1526 shost_printk(KERN_WARNING, target->scsi_host,
1527 PFX "Unhandled data direction %d\n",
1528 scmnd->sc_data_direction);
aef9ec39
RD		 1529 return -EINVAL;
1530 }
1531
bb350d1d
FT		 1532 nents = scsi_sg_count(scmnd);
1533 scat = scsi_sglist(scmnd);
aef9ec39 1534
05321937 1535 dev = target->srp_host->srp_dev;
85507bcc
RC		 1536 ibdev = dev->dev;
1537
1538 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
8f26c9ff
DD		 1539 if (unlikely(count == 0))
1540 return -EIO;
f5358a17
RD		 1541
1542 fmt = SRP_DATA_DESC_DIRECT;
1543 len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
aef9ec39 1544
b1b8854d 1545 if (count == 1 && !register_always) {
f5358a17
RD		 1546 /*
1547 * The midlayer only generated a single gather/scatter
1548 * entry, or DMA mapping coalesced everything to a
1549 * single entry. So a direct descriptor along with
1550 * the DMA MR suffices.
1551 */
cf368713 1552 struct srp_direct_buf *buf = (void *) cmd->add_data;
aef9ec39 1553
85507bcc 1554 buf->va = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
9af76271 1555 buf->key = cpu_to_be32(target->rkey);
85507bcc 1556 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
8f26c9ff 1557
52ede08f 1558 req->nmdesc = 0;
8f26c9ff
DD		 1559 goto map_complete;
1560 }
1561
5cfb1782
BVA		 1562 /*
1563 * We have more than one scatter/gather entry, so build our indirect
1564 * descriptor table, trying to merge as many entries as we can.
8f26c9ff
DD		 1565 */
1566 indirect_hdr = (void *) cmd->add_data;
1567
c07d424d
DD		 1568 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1569 target->indirect_size, DMA_TO_DEVICE);
1570
8f26c9ff 1571 memset(&state, 0, sizeof(state));
509c07bc 1572 srp_map_sg(&state, ch, req, scat, count);
cf368713 1573
c07d424d
DD		 1574 /* We've mapped the request, now pull as much of the indirect
1575 * descriptor table as we can into the command buffer. If this
1576 * target is not using an external indirect table, we are
1577 * guaranteed to fit into the command, as the SCSI layer won't
1578 * give us more S/G entries than we allow.
8f26c9ff 1579 */
8f26c9ff 1580 if (state.ndesc == 1) {
5cfb1782
BVA		 1581 /*
1582 * Memory registration collapsed the sg-list into one entry,
8f26c9ff
DD		 1583 * so use a direct descriptor.
1584 */
1585 struct srp_direct_buf *buf = (void *) cmd->add_data;
cf368713 1586
c07d424d 1587 *buf = req->indirect_desc[0];
8f26c9ff 1588 goto map_complete;
aef9ec39
RD		 1589 }
1590
c07d424d
DD		 1591 if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1592 !target->allow_ext_sg)) {
1593 shost_printk(KERN_ERR, target->scsi_host,
1594 "Could not fit S/G list into SRP_CMD\n");
1595 return -EIO;
1596 }
1597
1598 count = min(state.ndesc, target->cmd_sg_cnt);
8f26c9ff
DD		 1599 table_len = state.ndesc * sizeof (struct srp_direct_buf);
1600
1601 fmt = SRP_DATA_DESC_INDIRECT;
1602 len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
c07d424d 1603 len += count * sizeof (struct srp_direct_buf);
8f26c9ff 1604
c07d424d
DD		 1605 memcpy(indirect_hdr->desc_list, req->indirect_desc,
1606 count * sizeof (struct srp_direct_buf));
8f26c9ff 1607
c07d424d 1608 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
8f26c9ff
DD		 1609 indirect_hdr->table_desc.key = cpu_to_be32(target->rkey);
1610 indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1611 indirect_hdr->len = cpu_to_be32(state.total_len);
1612
1613 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
c07d424d 1614 cmd->data_out_desc_cnt = count;
8f26c9ff 1615 else
c07d424d
DD		 1616 cmd->data_in_desc_cnt = count;
1617
1618 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1619 DMA_TO_DEVICE);
8f26c9ff
DD		 1620
1621map_complete:
aef9ec39
RD		 1622 if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1623 cmd->buf_fmt = fmt << 4;
1624 else
1625 cmd->buf_fmt = fmt;
1626
aef9ec39
RD		 1627 return len;
1628}
1629
76c75b25
BVA		 1630/*
1631 * Return an IU and possible credit to the free pool
1632 */
509c07bc 1633static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
76c75b25
BVA		 1634 enum srp_iu_type iu_type)
1635{
1636 unsigned long flags;
1637
509c07bc
BVA		 1638 spin_lock_irqsave(&ch->lock, flags);
1639 list_add(&iu->list, &ch->free_tx);
76c75b25 1640 if (iu_type != SRP_IU_RSP)
509c07bc
BVA		 1641 ++ch->req_lim;
1642 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25
BVA		 1643}
1644
05a1d750 1645/*
509c07bc 1646 * Must be called with ch->lock held to protect req_lim and free_tx.
e9684678 1647 * If IU is not sent, it must be returned using srp_put_tx_iu().
05a1d750
DD		 1648 *
1649 * Note:
1650 * An upper limit for the number of allocated information units for each
1651 * request type is:
1652 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1653 * more than Scsi_Host.can_queue requests.
1654 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1655 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1656 * one unanswered SRP request to an initiator.
1657 */
509c07bc 1658static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
05a1d750
DD		 1659 enum srp_iu_type iu_type)
1660{
509c07bc 1661 struct srp_target_port *target = ch->target;
05a1d750
DD		 1662 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1663 struct srp_iu *iu;
1664
509c07bc 1665 srp_send_completion(ch->send_cq, ch);
05a1d750 1666
509c07bc 1667 if (list_empty(&ch->free_tx))
05a1d750
DD		 1668 return NULL;
1669
1670 /* Initiator responses to target requests do not consume credits */
76c75b25 1671 if (iu_type != SRP_IU_RSP) {
509c07bc 1672 if (ch->req_lim <= rsv) {
76c75b25
BVA		 1673 ++target->zero_req_lim;
1674 return NULL;
1675 }
1676
509c07bc 1677 --ch->req_lim;
05a1d750
DD		 1678 }
1679
509c07bc 1680 iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
76c75b25 1681 list_del(&iu->list);
05a1d750
DD		 1682 return iu;
1683}
1684
509c07bc 1685static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
05a1d750 1686{
509c07bc 1687 struct srp_target_port *target = ch->target;
05a1d750
DD		 1688 struct ib_sge list;
1689 struct ib_send_wr wr, *bad_wr;
05a1d750
DD		 1690
1691 list.addr = iu->dma;
1692 list.length = len;
9af76271 1693 list.lkey = target->lkey;
05a1d750
DD		 1694
1695 wr.next = NULL;
dcb4cb85 1696 wr.wr_id = (uintptr_t) iu;
05a1d750
DD		 1697 wr.sg_list = &list;
1698 wr.num_sge = 1;
1699 wr.opcode = IB_WR_SEND;
1700 wr.send_flags = IB_SEND_SIGNALED;
1701
509c07bc 1702 return ib_post_send(ch->qp, &wr, &bad_wr);
05a1d750
DD		 1703}
1704
509c07bc 1705static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
c996bb47 1706{
509c07bc 1707 struct srp_target_port *target = ch->target;
c996bb47 1708 struct ib_recv_wr wr, *bad_wr;
dcb4cb85 1709 struct ib_sge list;
c996bb47
BVA		 1710
1711 list.addr = iu->dma;
1712 list.length = iu->size;
9af76271 1713 list.lkey = target->lkey;
c996bb47
BVA		 1714
1715 wr.next = NULL;
dcb4cb85 1716 wr.wr_id = (uintptr_t) iu;
c996bb47
BVA		 1717 wr.sg_list = &list;
1718 wr.num_sge = 1;
1719
509c07bc 1720 return ib_post_recv(ch->qp, &wr, &bad_wr);
c996bb47
BVA		 1721}
1722
509c07bc 1723static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
aef9ec39 1724{
509c07bc 1725 struct srp_target_port *target = ch->target;
aef9ec39
RD		 1726 struct srp_request *req;
1727 struct scsi_cmnd *scmnd;
1728 unsigned long flags;
aef9ec39 1729
aef9ec39 1730 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
509c07bc
BVA		 1731 spin_lock_irqsave(&ch->lock, flags);
1732 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1733 spin_unlock_irqrestore(&ch->lock, flags);
94a9174c 1734
509c07bc 1735 ch->tsk_mgmt_status = -1;
f8b6e31e 1736 if (be32_to_cpu(rsp->resp_data_len) >= 4)
509c07bc
BVA		 1737 ch->tsk_mgmt_status = rsp->data[3];
1738 complete(&ch->tsk_mgmt_done);
aef9ec39 1739 } else {
77f2c1a4
BVA		 1740 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1741 if (scmnd) {
1742 req = (void *)scmnd->host_scribble;
1743 scmnd = srp_claim_req(ch, req, NULL, scmnd);
1744 }
22032991 1745 if (!scmnd) {
7aa54bd7 1746 shost_printk(KERN_ERR, target->scsi_host,
d92c0da7
BVA		 1747 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1748 rsp->tag, ch - target->ch, ch->qp->qp_num);
22032991 1749
509c07bc
BVA		 1750 spin_lock_irqsave(&ch->lock, flags);
1751 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1752 spin_unlock_irqrestore(&ch->lock, flags);
22032991
BVA		 1753
1754 return;
1755 }
aef9ec39
RD		 1756 scmnd->result = rsp->status;
1757
1758 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1759 memcpy(scmnd->sense_buffer, rsp->data +
1760 be32_to_cpu(rsp->resp_data_len),
1761 min_t(int, be32_to_cpu(rsp->sense_data_len),
1762 SCSI_SENSE_BUFFERSIZE));
1763 }
1764
e714531a 1765 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
bb350d1d 1766 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
e714531a
BVA		 1767 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1768 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1769 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1770 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1771 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1772 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
aef9ec39 1773
509c07bc 1774 srp_free_req(ch, req, scmnd,
22032991
BVA		 1775 be32_to_cpu(rsp->req_lim_delta));
1776
f8b6e31e
DD		 1777 scmnd->host_scribble = NULL;
1778 scmnd->scsi_done(scmnd);
aef9ec39 1779 }
aef9ec39
RD		 1780}
1781
509c07bc 1782static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
bb12588a
DD		 1783 void *rsp, int len)
1784{
509c07bc 1785 struct srp_target_port *target = ch->target;
76c75b25 1786 struct ib_device *dev = target->srp_host->srp_dev->dev;
bb12588a
DD		 1787 unsigned long flags;
1788 struct srp_iu *iu;
76c75b25 1789 int err;
bb12588a 1790
509c07bc
BVA		 1791 spin_lock_irqsave(&ch->lock, flags);
1792 ch->req_lim += req_delta;
1793 iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1794 spin_unlock_irqrestore(&ch->lock, flags);
76c75b25 1795
bb12588a
DD		 1796 if (!iu) {
1797 shost_printk(KERN_ERR, target->scsi_host, PFX
1798 "no IU available to send response\n");
76c75b25 1799 return 1;
bb12588a
DD		 1800 }
1801
1802 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1803 memcpy(iu->buf, rsp, len);
1804 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1805
509c07bc 1806 err = srp_post_send(ch, iu, len);
76c75b25 1807 if (err) {
bb12588a
DD		 1808 shost_printk(KERN_ERR, target->scsi_host, PFX
1809 "unable to post response: %d\n", err);
509c07bc 1810 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
76c75b25 1811 }
bb12588a 1812
bb12588a
DD		 1813 return err;
1814}
1815
509c07bc 1816static void srp_process_cred_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1817 struct srp_cred_req *req)
1818{
1819 struct srp_cred_rsp rsp = {
1820 .opcode = SRP_CRED_RSP,
1821 .tag = req->tag,
1822 };
1823 s32 delta = be32_to_cpu(req->req_lim_delta);
1824
509c07bc
BVA		 1825 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1826 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
bb12588a
DD		 1827 "problems processing SRP_CRED_REQ\n");
1828}
1829
509c07bc 1830static void srp_process_aer_req(struct srp_rdma_ch *ch,
bb12588a
DD		 1831 struct srp_aer_req *req)
1832{
509c07bc 1833 struct srp_target_port *target = ch->target;
bb12588a
DD		 1834 struct srp_aer_rsp rsp = {
1835 .opcode = SRP_AER_RSP,
1836 .tag = req->tag,
1837 };
1838 s32 delta = be32_to_cpu(req->req_lim_delta);
1839
1840 shost_printk(KERN_ERR, target->scsi_host, PFX
1841 "ignoring AER for LUN %llu\n", be64_to_cpu(req->lun));
1842
509c07bc 1843 if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
bb12588a
DD		 1844 shost_printk(KERN_ERR, target->scsi_host, PFX
1845 "problems processing SRP_AER_REQ\n");
1846}
1847
509c07bc 1848static void srp_handle_recv(struct srp_rdma_ch *ch, struct ib_wc *wc)
aef9ec39 1849{
509c07bc 1850 struct srp_target_port *target = ch->target;
dcb4cb85 1851 struct ib_device *dev = target->srp_host->srp_dev->dev;
737b94eb 1852 struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
c996bb47 1853 int res;
aef9ec39
RD		 1854 u8 opcode;
1855
509c07bc 1856 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1857 DMA_FROM_DEVICE);
aef9ec39
RD		 1858
1859 opcode = *(u8 *) iu->buf;
1860
1861 if (0) {
7aa54bd7
DD		 1862 shost_printk(KERN_ERR, target->scsi_host,
1863 PFX "recv completion, opcode 0x%02x\n", opcode);
7a700811
BVA		 1864 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
1865 iu->buf, wc->byte_len, true);
aef9ec39
RD		 1866 }
1867
1868 switch (opcode) {
1869 case SRP_RSP:
509c07bc 1870 srp_process_rsp(ch, iu->buf);
aef9ec39
RD		 1871 break;
1872
bb12588a 1873 case SRP_CRED_REQ:
509c07bc 1874 srp_process_cred_req(ch, iu->buf);
bb12588a
DD		 1875 break;
1876
1877 case SRP_AER_REQ:
509c07bc 1878 srp_process_aer_req(ch, iu->buf);
bb12588a
DD		 1879 break;
1880
aef9ec39
RD		 1881 case SRP_T_LOGOUT:
1882 /* XXX Handle target logout */
7aa54bd7
DD		 1883 shost_printk(KERN_WARNING, target->scsi_host,
1884 PFX "Got target logout request\n");
aef9ec39
RD		 1885 break;
1886
1887 default:
7aa54bd7
DD		 1888 shost_printk(KERN_WARNING, target->scsi_host,
1889 PFX "Unhandled SRP opcode 0x%02x\n", opcode);
aef9ec39
RD		 1890 break;
1891 }
1892
509c07bc 1893 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
85507bcc 1894 DMA_FROM_DEVICE);
c996bb47 1895
509c07bc 1896 res = srp_post_recv(ch, iu);
c996bb47
BVA		 1897 if (res != 0)
1898 shost_printk(KERN_ERR, target->scsi_host,
1899 PFX "Recv failed with error code %d\n", res);
aef9ec39
RD		 1900}
1901
c1120f89
BVA		 1902/**
1903 * srp_tl_err_work() - handle a transport layer error
af24663b 1904 * @work: Work structure embedded in an SRP target port.
c1120f89
BVA		 1905 *
1906 * Note: This function may get invoked before the rport has been created,
1907 * hence the target->rport test.
1908 */
1909static void srp_tl_err_work(struct work_struct *work)
1910{
1911 struct srp_target_port *target;
1912
1913 target = container_of(work, struct srp_target_port, tl_err_work);
1914 if (target->rport)
1915 srp_start_tl_fail_timers(target->rport);
1916}
1917
5cfb1782 1918static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status,
7dad6b2e 1919 bool send_err, struct srp_rdma_ch *ch)
948d1e88 1920{
7dad6b2e
BVA		 1921 struct srp_target_port *target = ch->target;
1922
1923 if (wr_id == SRP_LAST_WR_ID) {
1924 complete(&ch->done);
1925 return;
1926 }
1927
c014c8cd 1928 if (ch->connected && !target->qp_in_error) {
5cfb1782
BVA		 1929 if (wr_id & LOCAL_INV_WR_ID_MASK) {
1930 shost_printk(KERN_ERR, target->scsi_host, PFX
1931 "LOCAL_INV failed with status %d\n",
1932 wc_status);
1933 } else if (wr_id & FAST_REG_WR_ID_MASK) {
1934 shost_printk(KERN_ERR, target->scsi_host, PFX
1935 "FAST_REG_MR failed status %d\n",
1936 wc_status);
1937 } else {
1938 shost_printk(KERN_ERR, target->scsi_host,
1939 PFX "failed %s status %d for iu %p\n",
1940 send_err ? "send" : "receive",
1941 wc_status, (void *)(uintptr_t)wr_id);
1942 }
c1120f89 1943 queue_work(system_long_wq, &target->tl_err_work);
4f0af697 1944 }
948d1e88
BVA		 1945 target->qp_in_error = true;
1946}
1947
509c07bc 1948static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr)
aef9ec39 1949{
509c07bc 1950 struct srp_rdma_ch *ch = ch_ptr;
aef9ec39 1951 struct ib_wc wc;
aef9ec39
RD		 1952
1953 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1954 while (ib_poll_cq(cq, 1, &wc) > 0) {
948d1e88 1955 if (likely(wc.status == IB_WC_SUCCESS)) {
509c07bc 1956 srp_handle_recv(ch, &wc);
948d1e88 1957 } else {
7dad6b2e 1958 srp_handle_qp_err(wc.wr_id, wc.status, false, ch);
aef9ec39 1959 }
9c03dc9f
BVA		 1960 }
1961}
1962
509c07bc 1963static void srp_send_completion(struct ib_cq *cq, void *ch_ptr)
9c03dc9f 1964{
509c07bc 1965 struct srp_rdma_ch *ch = ch_ptr;
9c03dc9f 1966 struct ib_wc wc;
dcb4cb85 1967 struct srp_iu *iu;
9c03dc9f
BVA		 1968
1969 while (ib_poll_cq(cq, 1, &wc) > 0) {
948d1e88
BVA		 1970 if (likely(wc.status == IB_WC_SUCCESS)) {
1971 iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
509c07bc 1972 list_add(&iu->list, &ch->free_tx);
948d1e88 1973 } else {
7dad6b2e 1974 srp_handle_qp_err(wc.wr_id, wc.status, true, ch);
9c03dc9f 1975 }
aef9ec39
RD		 1976 }
1977}
1978
76c75b25 1979static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
aef9ec39 1980{
76c75b25 1981 struct srp_target_port *target = host_to_target(shost);
a95cadb9 1982 struct srp_rport *rport = target->rport;
509c07bc 1983 struct srp_rdma_ch *ch;
aef9ec39
RD		 1984 struct srp_request *req;
1985 struct srp_iu *iu;
1986 struct srp_cmd *cmd;
85507bcc 1987 struct ib_device *dev;
76c75b25 1988 unsigned long flags;
77f2c1a4
BVA		 1989 u32 tag;
1990 u16 idx;
d1b4289e 1991 int len, ret;
a95cadb9
BVA		 1992 const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
1993
1994 /*
1995 * The SCSI EH thread is the only context from which srp_queuecommand()
1996 * can get invoked for blocked devices (SDEV_BLOCK /
1997 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
1998 * locking the rport mutex if invoked from inside the SCSI EH.
1999 */
2000 if (in_scsi_eh)
2001 mutex_lock(&rport->mutex);
aef9ec39 2002
d1b4289e
BVA		 2003 scmnd->result = srp_chkready(target->rport);
2004 if (unlikely(scmnd->result))
2005 goto err;
2ce19e72 2006
77f2c1a4
BVA		 2007 WARN_ON_ONCE(scmnd->request->tag < 0);
2008 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7 2009 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
77f2c1a4
BVA		 2010 idx = blk_mq_unique_tag_to_tag(tag);
2011 WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2012 dev_name(&shost->shost_gendev), tag, idx,
2013 target->req_ring_size);
509c07bc
BVA		 2014
2015 spin_lock_irqsave(&ch->lock, flags);
2016 iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
509c07bc 2017 spin_unlock_irqrestore(&ch->lock, flags);
aef9ec39 2018
77f2c1a4
BVA		 2019 if (!iu)
2020 goto err;
2021
2022 req = &ch->req_ring[idx];
05321937 2023 dev = target->srp_host->srp_dev->dev;
49248644 2024 ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
85507bcc 2025 DMA_TO_DEVICE);
aef9ec39 2026
f8b6e31e 2027 scmnd->host_scribble = (void *) req;
aef9ec39
RD		 2028
2029 cmd = iu->buf;
2030 memset(cmd, 0, sizeof *cmd);
2031
2032 cmd->opcode = SRP_CMD;
2033 cmd->lun = cpu_to_be64((u64) scmnd->device->lun << 48);
77f2c1a4 2034 cmd->tag = tag;
aef9ec39
RD		 2035 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2036
aef9ec39
RD		 2037 req->scmnd = scmnd;
2038 req->cmd = iu;
aef9ec39 2039
509c07bc 2040 len = srp_map_data(scmnd, ch, req);
aef9ec39 2041 if (len < 0) {
7aa54bd7 2042 shost_printk(KERN_ERR, target->scsi_host,
d1b4289e
BVA		 2043 PFX "Failed to map data (%d)\n", len);
2044 /*
2045 * If we ran out of memory descriptors (-ENOMEM) because an
2046 * application is queuing many requests with more than
52ede08f 2047 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
d1b4289e
BVA		 2048 * to reduce queue depth temporarily.
2049 */
2050 scmnd->result = len == -ENOMEM ?
2051 DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
76c75b25 2052 goto err_iu;
aef9ec39
RD		 2053 }
2054
49248644 2055 ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
85507bcc 2056 DMA_TO_DEVICE);
aef9ec39 2057
509c07bc 2058 if (srp_post_send(ch, iu, len)) {
7aa54bd7 2059 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
aef9ec39
RD		 2060 goto err_unmap;
2061 }
2062
d1b4289e
BVA		 2063 ret = 0;
2064
a95cadb9
BVA		 2065unlock_rport:
2066 if (in_scsi_eh)
2067 mutex_unlock(&rport->mutex);
2068
d1b4289e 2069 return ret;
aef9ec39
RD		 2070
2071err_unmap:
509c07bc 2072 srp_unmap_data(scmnd, ch, req);
aef9ec39 2073
76c75b25 2074err_iu:
509c07bc 2075 srp_put_tx_iu(ch, iu, SRP_IU_CMD);
76c75b25 2076
024ca901
BVA		 2077 /*
2078 * Avoid that the loops that iterate over the request ring can
2079 * encounter a dangling SCSI command pointer.
2080 */
2081 req->scmnd = NULL;
2082
d1b4289e
BVA		 2083err:
2084 if (scmnd->result) {
2085 scmnd->scsi_done(scmnd);
2086 ret = 0;
2087 } else {
2088 ret = SCSI_MLQUEUE_HOST_BUSY;
2089 }
a95cadb9 2090
d1b4289e 2091 goto unlock_rport;
aef9ec39
RD		 2092}
2093
4d73f95f
BVA		 2094/*
2095 * Note: the resources allocated in this function are freed in
509c07bc 2096 * srp_free_ch_ib().
4d73f95f 2097 */
509c07bc 2098static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
aef9ec39 2099{
509c07bc 2100 struct srp_target_port *target = ch->target;
aef9ec39
RD		 2101 int i;
2102
509c07bc
BVA		 2103 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2104 GFP_KERNEL);
2105 if (!ch->rx_ring)
4d73f95f 2106 goto err_no_ring;
509c07bc
BVA		 2107 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2108 GFP_KERNEL);
2109 if (!ch->tx_ring)
4d73f95f
BVA		 2110 goto err_no_ring;
2111
2112 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2113 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2114 ch->max_ti_iu_len,
2115 GFP_KERNEL, DMA_FROM_DEVICE);
2116 if (!ch->rx_ring[i])
aef9ec39
RD		 2117 goto err;
2118 }
2119
4d73f95f 2120 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2121 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2122 target->max_iu_len,
2123 GFP_KERNEL, DMA_TO_DEVICE);
2124 if (!ch->tx_ring[i])
aef9ec39 2125 goto err;
dcb4cb85 2126
509c07bc 2127 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
aef9ec39
RD		 2128 }
2129
2130 return 0;
2131
2132err:
4d73f95f 2133 for (i = 0; i < target->queue_size; ++i) {
509c07bc
BVA		 2134 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2135 srp_free_iu(target->srp_host, ch->tx_ring[i]);
aef9ec39
RD		 2136 }
2137
4d73f95f
BVA		 2138
2139err_no_ring:
509c07bc
BVA		 2140 kfree(ch->tx_ring);
2141 ch->tx_ring = NULL;
2142 kfree(ch->rx_ring);
2143 ch->rx_ring = NULL;
4d73f95f 2144
aef9ec39
RD		 2145 return -ENOMEM;
2146}
2147
c9b03c1a
BVA		 2148static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2149{
2150 uint64_t T_tr_ns, max_compl_time_ms;
2151 uint32_t rq_tmo_jiffies;
2152
2153 /*
2154 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2155 * table 91), both the QP timeout and the retry count have to be set
2156 * for RC QP's during the RTR to RTS transition.
2157 */
2158 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2159 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2160
2161 /*
2162 * Set target->rq_tmo_jiffies to one second more than the largest time
2163 * it can take before an error completion is generated. See also
2164 * C9-140..142 in the IBTA spec for more information about how to
2165 * convert the QP Local ACK Timeout value to nanoseconds.
2166 */
2167 T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2168 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2169 do_div(max_compl_time_ms, NSEC_PER_MSEC);
2170 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2171
2172 return rq_tmo_jiffies;
2173}
2174
961e0be8
DD		 2175static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2176 struct srp_login_rsp *lrsp,
509c07bc 2177 struct srp_rdma_ch *ch)
961e0be8 2178{
509c07bc 2179 struct srp_target_port *target = ch->target;
961e0be8
DD		 2180 struct ib_qp_attr *qp_attr = NULL;
2181 int attr_mask = 0;
2182 int ret;
2183 int i;
2184
2185 if (lrsp->opcode == SRP_LOGIN_RSP) {
509c07bc
BVA		 2186 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2187 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta);
961e0be8
DD		 2188
2189 /*
2190 * Reserve credits for task management so we don't
2191 * bounce requests back to the SCSI mid-layer.
2192 */
2193 target->scsi_host->can_queue
509c07bc 2194 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
961e0be8 2195 target->scsi_host->can_queue);
4d73f95f
BVA		 2196 target->scsi_host->cmd_per_lun
2197 = min_t(int, target->scsi_host->can_queue,
2198 target->scsi_host->cmd_per_lun);
961e0be8
DD		 2199 } else {
2200 shost_printk(KERN_WARNING, target->scsi_host,
2201 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2202 ret = -ECONNRESET;
2203 goto error;
2204 }
2205
509c07bc
BVA		 2206 if (!ch->rx_ring) {
2207 ret = srp_alloc_iu_bufs(ch);
961e0be8
DD		 2208 if (ret)
2209 goto error;
2210 }
2211
2212 ret = -ENOMEM;
2213 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2214 if (!qp_attr)
2215 goto error;
2216
2217 qp_attr->qp_state = IB_QPS_RTR;
2218 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2219 if (ret)
2220 goto error_free;
2221
509c07bc 2222 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2223 if (ret)
2224 goto error_free;
2225
4d73f95f 2226 for (i = 0; i < target->queue_size; i++) {
509c07bc
BVA		 2227 struct srp_iu *iu = ch->rx_ring[i];
2228
2229 ret = srp_post_recv(ch, iu);
961e0be8
DD		 2230 if (ret)
2231 goto error_free;
2232 }
2233
2234 qp_attr->qp_state = IB_QPS_RTS;
2235 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2236 if (ret)
2237 goto error_free;
2238
c9b03c1a
BVA		 2239 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2240
509c07bc 2241 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
961e0be8
DD		 2242 if (ret)
2243 goto error_free;
2244
2245 ret = ib_send_cm_rtu(cm_id, NULL, 0);
2246
2247error_free:
2248 kfree(qp_attr);
2249
2250error:
509c07bc 2251 ch->status = ret;
961e0be8
DD		 2252}
2253
aef9ec39
RD		 2254static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2255 struct ib_cm_event *event,
509c07bc 2256 struct srp_rdma_ch *ch)
aef9ec39 2257{
509c07bc 2258 struct srp_target_port *target = ch->target;
7aa54bd7 2259 struct Scsi_Host *shost = target->scsi_host;
aef9ec39
RD		 2260 struct ib_class_port_info *cpi;
2261 int opcode;
2262
2263 switch (event->param.rej_rcvd.reason) {
2264 case IB_CM_REJ_PORT_CM_REDIRECT:
2265 cpi = event->param.rej_rcvd.ari;
509c07bc
BVA		 2266 ch->path.dlid = cpi->redirect_lid;
2267 ch->path.pkey = cpi->redirect_pkey;
aef9ec39 2268 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
509c07bc 2269 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
aef9ec39 2270
509c07bc 2271 ch->status = ch->path.dlid ?
aef9ec39
RD		 2272 SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2273 break;
2274
2275 case IB_CM_REJ_PORT_REDIRECT:
5d7cbfd6 2276 if (srp_target_is_topspin(target)) {
aef9ec39
RD		 2277 /*
2278 * Topspin/Cisco SRP gateways incorrectly send
2279 * reject reason code 25 when they mean 24
2280 * (port redirect).
2281 */
509c07bc 2282 memcpy(ch->path.dgid.raw,
aef9ec39
RD		 2283 event->param.rej_rcvd.ari, 16);
2284
7aa54bd7
DD		 2285 shost_printk(KERN_DEBUG, shost,
2286 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
509c07bc
BVA		 2287 be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2288 be64_to_cpu(ch->path.dgid.global.interface_id));
aef9ec39 2289
509c07bc 2290 ch->status = SRP_PORT_REDIRECT;
aef9ec39 2291 } else {
7aa54bd7
DD		 2292 shost_printk(KERN_WARNING, shost,
2293 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
509c07bc 2294 ch->status = -ECONNRESET;
aef9ec39
RD		 2295 }
2296 break;
2297
2298 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
7aa54bd7
DD		 2299 shost_printk(KERN_WARNING, shost,
2300 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
509c07bc 2301 ch->status = -ECONNRESET;
aef9ec39
RD		 2302 break;
2303
2304 case IB_CM_REJ_CONSUMER_DEFINED:
2305 opcode = *(u8 *) event->private_data;
2306 if (opcode == SRP_LOGIN_REJ) {
2307 struct srp_login_rej *rej = event->private_data;
2308 u32 reason = be32_to_cpu(rej->reason);
2309
2310 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
7aa54bd7
DD		 2311 shost_printk(KERN_WARNING, shost,
2312 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
aef9ec39 2313 else
e7ffde01
BVA		 2314 shost_printk(KERN_WARNING, shost, PFX
2315 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
747fe000
BVA		 2316 target->sgid.raw,
2317 target->orig_dgid.raw, reason);
aef9ec39 2318 } else
7aa54bd7
DD		 2319 shost_printk(KERN_WARNING, shost,
2320 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2321 " opcode 0x%02x\n", opcode);
509c07bc 2322 ch->status = -ECONNRESET;
aef9ec39
RD		 2323 break;
2324
9fe4bcf4
DD		 2325 case IB_CM_REJ_STALE_CONN:
2326 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n");
509c07bc 2327 ch->status = SRP_STALE_CONN;
9fe4bcf4
DD		 2328 break;
2329
aef9ec39 2330 default:
7aa54bd7
DD		 2331 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n",
2332 event->param.rej_rcvd.reason);
509c07bc 2333 ch->status = -ECONNRESET;
aef9ec39
RD		 2334 }
2335}
2336
2337static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2338{
509c07bc
BVA		 2339 struct srp_rdma_ch *ch = cm_id->context;
2340 struct srp_target_port *target = ch->target;
aef9ec39 2341 int comp = 0;
aef9ec39
RD		 2342
2343 switch (event->event) {
2344 case IB_CM_REQ_ERROR:
7aa54bd7
DD		 2345 shost_printk(KERN_DEBUG, target->scsi_host,
2346 PFX "Sending CM REQ failed\n");
aef9ec39 2347 comp = 1;
509c07bc 2348 ch->status = -ECONNRESET;
aef9ec39
RD		 2349 break;
2350
2351 case IB_CM_REP_RECEIVED:
2352 comp = 1;
509c07bc 2353 srp_cm_rep_handler(cm_id, event->private_data, ch);
aef9ec39
RD		 2354 break;
2355
2356 case IB_CM_REJ_RECEIVED:
7aa54bd7 2357 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
aef9ec39
RD		 2358 comp = 1;
2359
509c07bc 2360 srp_cm_rej_handler(cm_id, event, ch);
aef9ec39
RD		 2361 break;
2362
b7ac4ab4 2363 case IB_CM_DREQ_RECEIVED:
7aa54bd7
DD		 2364 shost_printk(KERN_WARNING, target->scsi_host,
2365 PFX "DREQ received - connection closed\n");
c014c8cd 2366 ch->connected = false;
b7ac4ab4 2367 if (ib_send_cm_drep(cm_id, NULL, 0))
7aa54bd7
DD		 2368 shost_printk(KERN_ERR, target->scsi_host,
2369 PFX "Sending CM DREP failed\n");
c1120f89 2370 queue_work(system_long_wq, &target->tl_err_work);
aef9ec39
RD		 2371 break;
2372
2373 case IB_CM_TIMEWAIT_EXIT:
7aa54bd7
DD		 2374 shost_printk(KERN_ERR, target->scsi_host,
2375 PFX "connection closed\n");
ac72d766 2376 comp = 1;
aef9ec39 2377
509c07bc 2378 ch->status = 0;
aef9ec39
RD		 2379 break;
2380
b7ac4ab4
IR		 2381 case IB_CM_MRA_RECEIVED:
2382 case IB_CM_DREQ_ERROR:
2383 case IB_CM_DREP_RECEIVED:
2384 break;
2385
aef9ec39 2386 default:
7aa54bd7
DD		 2387 shost_printk(KERN_WARNING, target->scsi_host,
2388 PFX "Unhandled CM event %d\n", event->event);
aef9ec39
RD		 2389 break;
2390 }
2391
2392 if (comp)
509c07bc 2393 complete(&ch->done);
aef9ec39 2394
aef9ec39
RD		 2395 return 0;
2396}
2397
71444b97
JW		 2398/**
2399 * srp_change_queue_depth - setting device queue depth
2400 * @sdev: scsi device struct
2401 * @qdepth: requested queue depth
71444b97
JW		 2402 *
2403 * Returns queue depth.
2404 */
2405static int
db5ed4df 2406srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
71444b97 2407{
c40ecc12 2408 if (!sdev->tagged_supported)
1e6f2416 2409 qdepth = 1;
db5ed4df 2410 return scsi_change_queue_depth(sdev, qdepth);
71444b97
JW		 2411}
2412
509c07bc
BVA		 2413static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag,
2414 unsigned int lun, u8 func)
aef9ec39 2415{
509c07bc 2416 struct srp_target_port *target = ch->target;
a95cadb9 2417 struct srp_rport *rport = target->rport;
19081f31 2418 struct ib_device *dev = target->srp_host->srp_dev->dev;
aef9ec39
RD		 2419 struct srp_iu *iu;
2420 struct srp_tsk_mgmt *tsk_mgmt;
aef9ec39 2421
c014c8cd 2422 if (!ch->connected || target->qp_in_error)
3780d1f0
BVA		 2423 return -1;
2424
509c07bc 2425 init_completion(&ch->tsk_mgmt_done);
aef9ec39 2426
a95cadb9 2427 /*
509c07bc 2428 * Lock the rport mutex to avoid that srp_create_ch_ib() is
a95cadb9
BVA		 2429 * invoked while a task management function is being sent.
2430 */
2431 mutex_lock(&rport->mutex);
509c07bc
BVA		 2432 spin_lock_irq(&ch->lock);
2433 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2434 spin_unlock_irq(&ch->lock);
76c75b25 2435
a95cadb9
BVA		 2436 if (!iu) {
2437 mutex_unlock(&rport->mutex);
2438
76c75b25 2439 return -1;
a95cadb9 2440 }
aef9ec39 2441
19081f31
DD		 2442 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2443 DMA_TO_DEVICE);
aef9ec39
RD		 2444 tsk_mgmt = iu->buf;
2445 memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2446
2447 tsk_mgmt->opcode = SRP_TSK_MGMT;
f8b6e31e
DD		 2448 tsk_mgmt->lun = cpu_to_be64((u64) lun << 48);
2449 tsk_mgmt->tag = req_tag | SRP_TAG_TSK_MGMT;
aef9ec39 2450 tsk_mgmt->tsk_mgmt_func = func;
f8b6e31e 2451 tsk_mgmt->task_tag = req_tag;
aef9ec39 2452
19081f31
DD		 2453 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2454 DMA_TO_DEVICE);
509c07bc
BVA		 2455 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2456 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
a95cadb9
BVA		 2457 mutex_unlock(&rport->mutex);
2458
76c75b25
BVA		 2459 return -1;
2460 }
a95cadb9 2461 mutex_unlock(&rport->mutex);
d945e1df 2462
509c07bc 2463 if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
aef9ec39 2464 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
d945e1df 2465 return -1;
aef9ec39 2466
d945e1df 2467 return 0;
d945e1df
RD		 2468}
2469
aef9ec39
RD		 2470static int srp_abort(struct scsi_cmnd *scmnd)
2471{
d945e1df 2472 struct srp_target_port *target = host_to_target(scmnd->device->host);
f8b6e31e 2473 struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
77f2c1a4 2474 u32 tag;
d92c0da7 2475 u16 ch_idx;
509c07bc 2476 struct srp_rdma_ch *ch;
086f44f5 2477 int ret;
d945e1df 2478
7aa54bd7 2479 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
aef9ec39 2480
d92c0da7 2481 if (!req)
99b6697a 2482 return SUCCESS;
77f2c1a4 2483 tag = blk_mq_unique_tag(scmnd->request);
d92c0da7
BVA		 2484 ch_idx = blk_mq_unique_tag_to_hwq(tag);
2485 if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2486 return SUCCESS;
2487 ch = &target->ch[ch_idx];
2488 if (!srp_claim_req(ch, req, NULL, scmnd))
2489 return SUCCESS;
2490 shost_printk(KERN_ERR, target->scsi_host,
2491 "Sending SRP abort for tag %#x\n", tag);
77f2c1a4 2492 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
80d5e8a2 2493 SRP_TSK_ABORT_TASK) == 0)
086f44f5 2494 ret = SUCCESS;
ed9b2264 2495 else if (target->rport->state == SRP_RPORT_LOST)
99e1c139 2496 ret = FAST_IO_FAIL;
086f44f5
BVA		 2497 else
2498 ret = FAILED;
509c07bc 2499 srp_free_req(ch, req, scmnd, 0);
22032991 2500 scmnd->result = DID_ABORT << 16;
d8536670 2501 scmnd->scsi_done(scmnd);
d945e1df 2502
086f44f5 2503 return ret;
aef9ec39
RD		 2504}
2505
2506static int srp_reset_device(struct scsi_cmnd *scmnd)
2507{
d945e1df 2508 struct srp_target_port *target = host_to_target(scmnd->device->host);
d92c0da7 2509 struct srp_rdma_ch *ch;
536ae14e 2510 int i;
d945e1df 2511
7aa54bd7 2512 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
aef9ec39 2513
d92c0da7 2514 ch = &target->ch[0];
509c07bc 2515 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
f8b6e31e 2516 SRP_TSK_LUN_RESET))
d945e1df 2517 return FAILED;
509c07bc 2518 if (ch->tsk_mgmt_status)
d945e1df
RD		 2519 return FAILED;
2520
d92c0da7
BVA		 2521 for (i = 0; i < target->ch_count; i++) {
2522 ch = &target->ch[i];
2523 for (i = 0; i < target->req_ring_size; ++i) {
2524 struct srp_request *req = &ch->req_ring[i];
509c07bc 2525
d92c0da7
BVA		 2526 srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2527 }
536ae14e 2528 }
d945e1df 2529
d945e1df 2530 return SUCCESS;
aef9ec39
RD		 2531}
2532
2533static int srp_reset_host(struct scsi_cmnd *scmnd)
2534{
2535 struct srp_target_port *target = host_to_target(scmnd->device->host);
aef9ec39 2536
7aa54bd7 2537 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
aef9ec39 2538
ed9b2264 2539 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
aef9ec39
RD		 2540}
2541
c9b03c1a
BVA		 2542static int srp_slave_configure(struct scsi_device *sdev)
2543{
2544 struct Scsi_Host *shost = sdev->host;
2545 struct srp_target_port *target = host_to_target(shost);
2546 struct request_queue *q = sdev->request_queue;
2547 unsigned long timeout;
2548
2549 if (sdev->type == TYPE_DISK) {
2550 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2551 blk_queue_rq_timeout(q, timeout);
2552 }
2553
2554 return 0;
2555}
2556
ee959b00
TJ		 2557static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2558 char *buf)
6ecb0c84 2559{
ee959b00 2560 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2561
45c37cad 2562 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
6ecb0c84
RD		 2563}
2564
ee959b00
TJ		 2565static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2566 char *buf)
6ecb0c84 2567{
ee959b00 2568 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2569
45c37cad 2570 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
6ecb0c84
RD		 2571}
2572
ee959b00
TJ		 2573static ssize_t show_service_id(struct device *dev,
2574 struct device_attribute *attr, char *buf)
6ecb0c84 2575{
ee959b00 2576 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2577
45c37cad 2578 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
6ecb0c84
RD		 2579}
2580
ee959b00
TJ		 2581static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2582 char *buf)
6ecb0c84 2583{
ee959b00 2584 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6ecb0c84 2585
747fe000 2586 return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
6ecb0c84
RD		 2587}
2588
848b3082
BVA		 2589static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2590 char *buf)
2591{
2592 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2593
747fe000 2594 return sprintf(buf, "%pI6\n", target->sgid.raw);
848b3082
BVA		 2595}
2596
ee959b00
TJ		 2597static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2598 char *buf)
6ecb0c84 2599{
ee959b00 2600 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7 2601 struct srp_rdma_ch *ch = &target->ch[0];
6ecb0c84 2602
509c07bc 2603 return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
6ecb0c84
RD		 2604}
2605
ee959b00
TJ		 2606static ssize_t show_orig_dgid(struct device *dev,
2607 struct device_attribute *attr, char *buf)
3633b3d0 2608{
ee959b00 2609 struct srp_target_port *target = host_to_target(class_to_shost(dev));
3633b3d0 2610
747fe000 2611 return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
3633b3d0
IR		 2612}
2613
89de7486
BVA		 2614static ssize_t show_req_lim(struct device *dev,
2615 struct device_attribute *attr, char *buf)
2616{
2617 struct srp_target_port *target = host_to_target(class_to_shost(dev));
d92c0da7
BVA		 2618 struct srp_rdma_ch *ch;
2619 int i, req_lim = INT_MAX;
89de7486 2620
d92c0da7
BVA		 2621 for (i = 0; i < target->ch_count; i++) {
2622 ch = &target->ch[i];
2623 req_lim = min(req_lim, ch->req_lim);
2624 }
2625 return sprintf(buf, "%d\n", req_lim);
89de7486
BVA		 2626}
2627
ee959b00
TJ		 2628static ssize_t show_zero_req_lim(struct device *dev,
2629 struct device_attribute *attr, char *buf)
6bfa24fa 2630{
ee959b00 2631 struct srp_target_port *target = host_to_target(class_to_shost(dev));
6bfa24fa 2632
6bfa24fa
RD		 2633 return sprintf(buf, "%d\n", target->zero_req_lim);
2634}
2635
ee959b00
TJ		 2636static ssize_t show_local_ib_port(struct device *dev,
2637 struct device_attribute *attr, char *buf)
ded7f1a1 2638{
ee959b00 2639 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1
IR		 2640
2641 return sprintf(buf, "%d\n", target->srp_host->port);
2642}
2643
ee959b00
TJ		 2644static ssize_t show_local_ib_device(struct device *dev,
2645 struct device_attribute *attr, char *buf)
ded7f1a1 2646{
ee959b00 2647 struct srp_target_port *target = host_to_target(class_to_shost(dev));
ded7f1a1 2648
05321937 2649 return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
ded7f1a1
IR		 2650}
2651
d92c0da7
BVA		 2652static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2653 char *buf)
2654{
2655 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2656
2657 return sprintf(buf, "%d\n", target->ch_count);
2658}
2659
4b5e5f41
BVA		 2660static ssize_t show_comp_vector(struct device *dev,
2661 struct device_attribute *attr, char *buf)
2662{
2663 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2664
2665 return sprintf(buf, "%d\n", target->comp_vector);
2666}
2667
7bb312e4
VP		 2668static ssize_t show_tl_retry_count(struct device *dev,
2669 struct device_attribute *attr, char *buf)
2670{
2671 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2672
2673 return sprintf(buf, "%d\n", target->tl_retry_count);
2674}
2675
49248644
DD		 2676static ssize_t show_cmd_sg_entries(struct device *dev,
2677 struct device_attribute *attr, char *buf)
2678{
2679 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2680
2681 return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2682}
2683
c07d424d
DD		 2684static ssize_t show_allow_ext_sg(struct device *dev,
2685 struct device_attribute *attr, char *buf)
2686{
2687 struct srp_target_port *target = host_to_target(class_to_shost(dev));
2688
2689 return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2690}
2691
ee959b00
TJ		 2692static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL);
2693static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL);
2694static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL);
2695static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL);
848b3082 2696static DEVICE_ATTR(sgid, S_IRUGO, show_sgid, NULL);
ee959b00
TJ		 2697static DEVICE_ATTR(dgid, S_IRUGO, show_dgid, NULL);
2698static DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL);
89de7486 2699static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL);
ee959b00
TJ		 2700static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
2701static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
2702static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
d92c0da7 2703static DEVICE_ATTR(ch_count, S_IRUGO, show_ch_count, NULL);
4b5e5f41 2704static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL);
7bb312e4 2705static DEVICE_ATTR(tl_retry_count, S_IRUGO, show_tl_retry_count, NULL);
49248644 2706static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL);
c07d424d 2707static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL);
ee959b00
TJ		 2708
2709static struct device_attribute *srp_host_attrs[] = {
2710 &dev_attr_id_ext,
2711 &dev_attr_ioc_guid,
2712 &dev_attr_service_id,
2713 &dev_attr_pkey,
848b3082 2714 &dev_attr_sgid,
ee959b00
TJ		 2715 &dev_attr_dgid,
2716 &dev_attr_orig_dgid,
89de7486 2717 &dev_attr_req_lim,
ee959b00
TJ		 2718 &dev_attr_zero_req_lim,
2719 &dev_attr_local_ib_port,
2720 &dev_attr_local_ib_device,
d92c0da7 2721 &dev_attr_ch_count,
4b5e5f41 2722 &dev_attr_comp_vector,
7bb312e4 2723 &dev_attr_tl_retry_count,
49248644 2724 &dev_attr_cmd_sg_entries,
c07d424d 2725 &dev_attr_allow_ext_sg,
6ecb0c84
RD		 2726 NULL
2727};
2728
aef9ec39
RD		 2729static struct scsi_host_template srp_template = {
2730 .module = THIS_MODULE,
b7f008fd
RD		 2731 .name = "InfiniBand SRP initiator",
2732 .proc_name = DRV_NAME,
c9b03c1a 2733 .slave_configure = srp_slave_configure,
aef9ec39
RD		 2734 .info = srp_target_info,
2735 .queuecommand = srp_queuecommand,
71444b97 2736 .change_queue_depth = srp_change_queue_depth,
aef9ec39
RD		 2737 .eh_abort_handler = srp_abort,
2738 .eh_device_reset_handler = srp_reset_device,
2739 .eh_host_reset_handler = srp_reset_host,
2742c1da 2740 .skip_settle_delay = true,
49248644 2741 .sg_tablesize = SRP_DEF_SG_TABLESIZE,
4d73f95f 2742 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE,
aef9ec39 2743 .this_id = -1,
4d73f95f 2744 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE,
6ecb0c84 2745 .use_clustering = ENABLE_CLUSTERING,
77f2c1a4
BVA		 2746 .shost_attrs = srp_host_attrs,
2747 .use_blk_tags = 1,
c40ecc12 2748 .track_queue_depth = 1,
aef9ec39
RD		 2749};
2750
34aa654e
BVA		 2751static int srp_sdev_count(struct Scsi_Host *host)
2752{
2753 struct scsi_device *sdev;
2754 int c = 0;
2755
2756 shost_for_each_device(sdev, host)
2757 c++;
2758
2759 return c;
2760}
2761
aef9ec39
RD		 2762static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2763{
3236822b
FT		 2764 struct srp_rport_identifiers ids;
2765 struct srp_rport *rport;
2766
34aa654e 2767 target->state = SRP_TARGET_SCANNING;
aef9ec39 2768 sprintf(target->target_name, "SRP.T10:%016llX",
45c37cad 2769 be64_to_cpu(target->id_ext));
aef9ec39 2770
05321937 2771 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
aef9ec39
RD		 2772 return -ENODEV;
2773
3236822b
FT		 2774 memcpy(ids.port_id, &target->id_ext, 8);
2775 memcpy(ids.port_id + 8, &target->ioc_guid, 8);
aebd5e47 2776 ids.roles = SRP_RPORT_ROLE_TARGET;
3236822b
FT		 2777 rport = srp_rport_add(target->scsi_host, &ids);
2778 if (IS_ERR(rport)) {
2779 scsi_remove_host(target->scsi_host);
2780 return PTR_ERR(rport);
2781 }
2782
dc1bdbd9 2783 rport->lld_data = target;
9dd69a60 2784 target->rport = rport;
dc1bdbd9 2785
b3589fd4 2786 spin_lock(&host->target_lock);
aef9ec39 2787 list_add_tail(&target->list, &host->target_list);
b3589fd4 2788 spin_unlock(&host->target_lock);
aef9ec39 2789
aef9ec39 2790 scsi_scan_target(&target->scsi_host->shost_gendev,
1962a4a1 2791 0, target->scsi_id, SCAN_WILD_CARD, 0);
aef9ec39 2792
c014c8cd
BVA		 2793 if (srp_connected_ch(target) < target->ch_count ||
2794 target->qp_in_error) {
34aa654e
BVA		 2795 shost_printk(KERN_INFO, target->scsi_host,
2796 PFX "SCSI scan failed - removing SCSI host\n");
2797 srp_queue_remove_work(target);
2798 goto out;
2799 }
2800
2801 pr_debug(PFX "%s: SCSI scan succeeded - detected %d LUNs\n",
2802 dev_name(&target->scsi_host->shost_gendev),
2803 srp_sdev_count(target->scsi_host));
2804
2805 spin_lock_irq(&target->lock);
2806 if (target->state == SRP_TARGET_SCANNING)
2807 target->state = SRP_TARGET_LIVE;
2808 spin_unlock_irq(&target->lock);
2809
2810out:
aef9ec39
RD		 2811 return 0;
2812}
2813
ee959b00 2814static void srp_release_dev(struct device *dev)
aef9ec39
RD		 2815{
2816 struct srp_host *host =
ee959b00 2817 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 2818
2819 complete(&host->released);
2820}
2821
2822static struct class srp_class = {
2823 .name = "infiniband_srp",
ee959b00 2824 .dev_release = srp_release_dev
aef9ec39
RD		 2825};
2826
96fc248a
BVA		 2827/**
2828 * srp_conn_unique() - check whether the connection to a target is unique
af24663b
BVA		 2829 * @host: SRP host.
2830 * @target: SRP target port.
96fc248a
BVA		 2831 */
2832static bool srp_conn_unique(struct srp_host *host,
2833 struct srp_target_port *target)
2834{
2835 struct srp_target_port *t;
2836 bool ret = false;
2837
2838 if (target->state == SRP_TARGET_REMOVED)
2839 goto out;
2840
2841 ret = true;
2842
2843 spin_lock(&host->target_lock);
2844 list_for_each_entry(t, &host->target_list, list) {
2845 if (t != target &&
2846 target->id_ext == t->id_ext &&
2847 target->ioc_guid == t->ioc_guid &&
2848 target->initiator_ext == t->initiator_ext) {
2849 ret = false;
2850 break;
2851 }
2852 }
2853 spin_unlock(&host->target_lock);
2854
2855out:
2856 return ret;
2857}
2858
aef9ec39
RD		 2859/*
2860 * Target ports are added by writing
2861 *
2862 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2863 * pkey=<P_Key>,service_id=<service ID>
2864 *
2865 * to the add_target sysfs attribute.
2866 */
2867enum {
2868 SRP_OPT_ERR = 0,
2869 SRP_OPT_ID_EXT = 1 << 0,
2870 SRP_OPT_IOC_GUID = 1 << 1,
2871 SRP_OPT_DGID = 1 << 2,
2872 SRP_OPT_PKEY = 1 << 3,
2873 SRP_OPT_SERVICE_ID = 1 << 4,
2874 SRP_OPT_MAX_SECT = 1 << 5,
52fb2b50 2875 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
0c0450db 2876 SRP_OPT_IO_CLASS = 1 << 7,
01cb9bcb 2877 SRP_OPT_INITIATOR_EXT = 1 << 8,
49248644 2878 SRP_OPT_CMD_SG_ENTRIES = 1 << 9,
c07d424d
DD		 2879 SRP_OPT_ALLOW_EXT_SG = 1 << 10,
2880 SRP_OPT_SG_TABLESIZE = 1 << 11,
4b5e5f41 2881 SRP_OPT_COMP_VECTOR = 1 << 12,
7bb312e4 2882 SRP_OPT_TL_RETRY_COUNT = 1 << 13,
4d73f95f 2883 SRP_OPT_QUEUE_SIZE = 1 << 14,
aef9ec39
RD		 2884 SRP_OPT_ALL = (SRP_OPT_ID_EXT |
2885 SRP_OPT_IOC_GUID |
2886 SRP_OPT_DGID |
2887 SRP_OPT_PKEY |
2888 SRP_OPT_SERVICE_ID),
2889};
2890
a447c093 2891static const match_table_t srp_opt_tokens = {
52fb2b50
VP		 2892 { SRP_OPT_ID_EXT, "id_ext=%s" },
2893 { SRP_OPT_IOC_GUID, "ioc_guid=%s" },
2894 { SRP_OPT_DGID, "dgid=%s" },
2895 { SRP_OPT_PKEY, "pkey=%x" },
2896 { SRP_OPT_SERVICE_ID, "service_id=%s" },
2897 { SRP_OPT_MAX_SECT, "max_sect=%d" },
2898 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" },
0c0450db 2899 { SRP_OPT_IO_CLASS, "io_class=%x" },
01cb9bcb 2900 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" },
49248644 2901 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" },
c07d424d
DD		 2902 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" },
2903 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" },
4b5e5f41 2904 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" },
7bb312e4 2905 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" },
4d73f95f 2906 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" },
52fb2b50 2907 { SRP_OPT_ERR, NULL }
aef9ec39
RD		 2908};
2909
2910static int srp_parse_options(const char *buf, struct srp_target_port *target)
2911{
2912 char *options, *sep_opt;
2913 char *p;
2914 char dgid[3];
2915 substring_t args[MAX_OPT_ARGS];
2916 int opt_mask = 0;
2917 int token;
2918 int ret = -EINVAL;
2919 int i;
2920
2921 options = kstrdup(buf, GFP_KERNEL);
2922 if (!options)
2923 return -ENOMEM;
2924
2925 sep_opt = options;
7dcf9c19 2926 while ((p = strsep(&sep_opt, ",\n")) != NULL) {
aef9ec39
RD		 2927 if (!*p)
2928 continue;
2929
2930 token = match_token(p, srp_opt_tokens, args);
2931 opt_mask |= token;
2932
2933 switch (token) {
2934 case SRP_OPT_ID_EXT:
2935 p = match_strdup(args);
a20f3a6d
IR		 2936 if (!p) {
2937 ret = -ENOMEM;
2938 goto out;
2939 }
aef9ec39
RD		 2940 target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2941 kfree(p);
2942 break;
2943
2944 case SRP_OPT_IOC_GUID:
2945 p = match_strdup(args);
a20f3a6d
IR		 2946 if (!p) {
2947 ret = -ENOMEM;
2948 goto out;
2949 }
aef9ec39
RD		 2950 target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
2951 kfree(p);
2952 break;
2953
2954 case SRP_OPT_DGID:
2955 p = match_strdup(args);
a20f3a6d
IR		 2956 if (!p) {
2957 ret = -ENOMEM;
2958 goto out;
2959 }
aef9ec39 2960 if (strlen(p) != 32) {
e0bda7d8 2961 pr_warn("bad dest GID parameter '%s'\n", p);
ce1823f0 2962 kfree(p);
aef9ec39
RD		 2963 goto out;
2964 }
2965
2966 for (i = 0; i < 16; ++i) {
747fe000
BVA		 2967 strlcpy(dgid, p + i * 2, sizeof(dgid));
2968 if (sscanf(dgid, "%hhx",
2969 &target->orig_dgid.raw[i]) < 1) {
2970 ret = -EINVAL;
2971 kfree(p);
2972 goto out;
2973 }
aef9ec39 2974 }
bf17c1c7 2975 kfree(p);
aef9ec39
RD		 2976 break;
2977
2978 case SRP_OPT_PKEY:
2979 if (match_hex(args, &token)) {
e0bda7d8 2980 pr_warn("bad P_Key parameter '%s'\n", p);
aef9ec39
RD		 2981 goto out;
2982 }
747fe000 2983 target->pkey = cpu_to_be16(token);
aef9ec39
RD		 2984 break;
2985
2986 case SRP_OPT_SERVICE_ID:
2987 p = match_strdup(args);
a20f3a6d
IR		 2988 if (!p) {
2989 ret = -ENOMEM;
2990 goto out;
2991 }
aef9ec39
RD		 2992 target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
2993 kfree(p);
2994 break;
2995
2996 case SRP_OPT_MAX_SECT:
2997 if (match_int(args, &token)) {
e0bda7d8 2998 pr_warn("bad max sect parameter '%s'\n", p);
aef9ec39
RD		 2999 goto out;
3000 }
3001 target->scsi_host->max_sectors = token;
3002 break;
3003
4d73f95f
BVA		 3004 case SRP_OPT_QUEUE_SIZE:
3005 if (match_int(args, &token) || token < 1) {
3006 pr_warn("bad queue_size parameter '%s'\n", p);
3007 goto out;
3008 }
3009 target->scsi_host->can_queue = token;
3010 target->queue_size = token + SRP_RSP_SQ_SIZE +
3011 SRP_TSK_MGMT_SQ_SIZE;
3012 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3013 target->scsi_host->cmd_per_lun = token;
3014 break;
3015
52fb2b50 3016 case SRP_OPT_MAX_CMD_PER_LUN:
4d73f95f 3017 if (match_int(args, &token) || token < 1) {
e0bda7d8
BVA		 3018 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3019 p);
52fb2b50
VP		 3020 goto out;
3021 }
4d73f95f 3022 target->scsi_host->cmd_per_lun = token;
52fb2b50
VP		 3023 break;
3024
0c0450db
R		 3025 case SRP_OPT_IO_CLASS:
3026 if (match_hex(args, &token)) {
e0bda7d8 3027 pr_warn("bad IO class parameter '%s'\n", p);
0c0450db
R		 3028 goto out;
3029 }
3030 if (token != SRP_REV10_IB_IO_CLASS &&
3031 token != SRP_REV16A_IB_IO_CLASS) {
e0bda7d8
BVA		 3032 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3033 token, SRP_REV10_IB_IO_CLASS,
3034 SRP_REV16A_IB_IO_CLASS);
0c0450db
R		 3035 goto out;
3036 }
3037 target->io_class = token;
3038 break;
3039
01cb9bcb
IR		 3040 case SRP_OPT_INITIATOR_EXT:
3041 p = match_strdup(args);
a20f3a6d
IR		 3042 if (!p) {
3043 ret = -ENOMEM;
3044 goto out;
3045 }
01cb9bcb
IR		 3046 target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3047 kfree(p);
3048 break;
3049
49248644
DD		 3050 case SRP_OPT_CMD_SG_ENTRIES:
3051 if (match_int(args, &token) || token < 1 || token > 255) {
e0bda7d8
BVA		 3052 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3053 p);
49248644
DD		 3054 goto out;
3055 }
3056 target->cmd_sg_cnt = token;
3057 break;
3058
c07d424d
DD		 3059 case SRP_OPT_ALLOW_EXT_SG:
3060 if (match_int(args, &token)) {
e0bda7d8 3061 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
c07d424d
DD		 3062 goto out;
3063 }
3064 target->allow_ext_sg = !!token;
3065 break;
3066
3067 case SRP_OPT_SG_TABLESIZE:
3068 if (match_int(args, &token) || token < 1 ||
3069 token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
e0bda7d8
BVA		 3070 pr_warn("bad max sg_tablesize parameter '%s'\n",
3071 p);
c07d424d
DD		 3072 goto out;
3073 }
3074 target->sg_tablesize = token;
3075 break;
3076
4b5e5f41
BVA		 3077 case SRP_OPT_COMP_VECTOR:
3078 if (match_int(args, &token) || token < 0) {
3079 pr_warn("bad comp_vector parameter '%s'\n", p);
3080 goto out;
3081 }
3082 target->comp_vector = token;
3083 break;
3084
7bb312e4
VP		 3085 case SRP_OPT_TL_RETRY_COUNT:
3086 if (match_int(args, &token) || token < 2 || token > 7) {
3087 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3088 p);
3089 goto out;
3090 }
3091 target->tl_retry_count = token;
3092 break;
3093
aef9ec39 3094 default:
e0bda7d8
BVA		 3095 pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3096 p);
aef9ec39
RD		 3097 goto out;
3098 }
3099 }
3100
3101 if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3102 ret = 0;
3103 else
3104 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3105 if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3106 !(srp_opt_tokens[i].token & opt_mask))
e0bda7d8
BVA		 3107 pr_warn("target creation request is missing parameter '%s'\n",
3108 srp_opt_tokens[i].pattern);
aef9ec39 3109
4d73f95f
BVA		 3110 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3111 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3112 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3113 target->scsi_host->cmd_per_lun,
3114 target->scsi_host->can_queue);
3115
aef9ec39
RD		 3116out:
3117 kfree(options);
3118 return ret;
3119}
3120
ee959b00
TJ		 3121static ssize_t srp_create_target(struct device *dev,
3122 struct device_attribute *attr,
aef9ec39
RD		 3123 const char *buf, size_t count)
3124{
3125 struct srp_host *host =
ee959b00 3126 container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3127 struct Scsi_Host *target_host;
3128 struct srp_target_port *target;
509c07bc 3129 struct srp_rdma_ch *ch;
d1b4289e
BVA		 3130 struct srp_device *srp_dev = host->srp_dev;
3131 struct ib_device *ibdev = srp_dev->dev;
d92c0da7
BVA		 3132 int ret, node_idx, node, cpu, i;
3133 bool multich = false;
aef9ec39
RD		 3134
3135 target_host = scsi_host_alloc(&srp_template,
3136 sizeof (struct srp_target_port));
3137 if (!target_host)
3138 return -ENOMEM;
3139
49248644 3140 target_host->transportt = ib_srp_transport_template;
fd1b6c4a
BVA		 3141 target_host->max_channel = 0;
3142 target_host->max_id = 1;
3c8edf0e
AR		 3143 target_host->max_lun = SRP_MAX_LUN;
3144 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
5f068992 3145
aef9ec39 3146 target = host_to_target(target_host);
aef9ec39 3147
49248644
DD		 3148 target->io_class = SRP_REV16A_IB_IO_CLASS;
3149 target->scsi_host = target_host;
3150 target->srp_host = host;
3151 target->lkey = host->srp_dev->mr->lkey;
3152 target->rkey = host->srp_dev->mr->rkey;
3153 target->cmd_sg_cnt = cmd_sg_entries;
c07d424d
DD		 3154 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries;
3155 target->allow_ext_sg = allow_ext_sg;
7bb312e4 3156 target->tl_retry_count = 7;
4d73f95f 3157 target->queue_size = SRP_DEFAULT_QUEUE_SIZE;
aef9ec39 3158
34aa654e
BVA		 3159 /*
3160 * Avoid that the SCSI host can be removed by srp_remove_target()
3161 * before this function returns.
3162 */
3163 scsi_host_get(target->scsi_host);
3164
2d7091bc
BVA		 3165 mutex_lock(&host->add_target_mutex);
3166
aef9ec39
RD		 3167 ret = srp_parse_options(buf, target);
3168 if (ret)
fb49c8bb 3169 goto out;
aef9ec39 3170
77f2c1a4
BVA		 3171 ret = scsi_init_shared_tag_map(target_host, target_host->can_queue);
3172 if (ret)
fb49c8bb 3173 goto out;
77f2c1a4 3174
4d73f95f
BVA		 3175 target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3176
96fc248a
BVA		 3177 if (!srp_conn_unique(target->srp_host, target)) {
3178 shost_printk(KERN_INFO, target->scsi_host,
3179 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3180 be64_to_cpu(target->id_ext),
3181 be64_to_cpu(target->ioc_guid),
3182 be64_to_cpu(target->initiator_ext));
3183 ret = -EEXIST;
fb49c8bb 3184 goto out;
96fc248a
BVA		 3185 }
3186
5cfb1782 3187 if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
d1b4289e 3188 target->cmd_sg_cnt < target->sg_tablesize) {
5cfb1782 3189 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
c07d424d
DD		 3190 target->sg_tablesize = target->cmd_sg_cnt;
3191 }
3192
3193 target_host->sg_tablesize = target->sg_tablesize;
3194 target->indirect_size = target->sg_tablesize *
3195 sizeof (struct srp_direct_buf);
49248644
DD		 3196 target->max_iu_len = sizeof (struct srp_cmd) +
3197 sizeof (struct srp_indirect_buf) +
3198 target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3199
c1120f89 3200 INIT_WORK(&target->tl_err_work, srp_tl_err_work);
ef6c49d8 3201 INIT_WORK(&target->remove_work, srp_remove_work);
8f26c9ff 3202 spin_lock_init(&target->lock);
747fe000 3203 ret = ib_query_gid(ibdev, host->port, 0, &target->sgid);
2088ca66 3204 if (ret)
fb49c8bb 3205 goto out;
aef9ec39 3206
d92c0da7
BVA		 3207 ret = -ENOMEM;
3208 target->ch_count = max_t(unsigned, num_online_nodes(),
3209 min(ch_count ? :
3210 min(4 * num_online_nodes(),
3211 ibdev->num_comp_vectors),
3212 num_online_cpus()));
3213 target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3214 GFP_KERNEL);
3215 if (!target->ch)
fb49c8bb 3216 goto out;
aef9ec39 3217
d92c0da7
BVA		 3218 node_idx = 0;
3219 for_each_online_node(node) {
3220 const int ch_start = (node_idx * target->ch_count /
3221 num_online_nodes());
3222 const int ch_end = ((node_idx + 1) * target->ch_count /
3223 num_online_nodes());
3224 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3225 num_online_nodes() + target->comp_vector)
3226 % ibdev->num_comp_vectors;
3227 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3228 num_online_nodes() + target->comp_vector)
3229 % ibdev->num_comp_vectors;
3230 int cpu_idx = 0;
3231
3232 for_each_online_cpu(cpu) {
3233 if (cpu_to_node(cpu) != node)
3234 continue;
3235 if (ch_start + cpu_idx >= ch_end)
3236 continue;
3237 ch = &target->ch[ch_start + cpu_idx];
3238 ch->target = target;
3239 ch->comp_vector = cv_start == cv_end ? cv_start :
3240 cv_start + cpu_idx % (cv_end - cv_start);
3241 spin_lock_init(&ch->lock);
3242 INIT_LIST_HEAD(&ch->free_tx);
3243 ret = srp_new_cm_id(ch);
3244 if (ret)
3245 goto err_disconnect;
aef9ec39 3246
d92c0da7
BVA		 3247 ret = srp_create_ch_ib(ch);
3248 if (ret)
3249 goto err_disconnect;
3250
3251 ret = srp_alloc_req_data(ch);
3252 if (ret)
3253 goto err_disconnect;
3254
3255 ret = srp_connect_ch(ch, multich);
3256 if (ret) {
3257 shost_printk(KERN_ERR, target->scsi_host,
3258 PFX "Connection %d/%d failed\n",
3259 ch_start + cpu_idx,
3260 target->ch_count);
3261 if (node_idx == 0 && cpu_idx == 0) {
3262 goto err_disconnect;
3263 } else {
3264 srp_free_ch_ib(target, ch);
3265 srp_free_req_data(target, ch);
3266 target->ch_count = ch - target->ch;
3267 break;
3268 }
3269 }
3270
3271 multich = true;
3272 cpu_idx++;
3273 }
3274 node_idx++;
aef9ec39
RD		 3275 }
3276
d92c0da7
BVA		 3277 target->scsi_host->nr_hw_queues = target->ch_count;
3278
aef9ec39
RD		 3279 ret = srp_add_target(host, target);
3280 if (ret)
3281 goto err_disconnect;
3282
34aa654e
BVA		 3283 if (target->state != SRP_TARGET_REMOVED) {
3284 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3285 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3286 be64_to_cpu(target->id_ext),
3287 be64_to_cpu(target->ioc_guid),
747fe000 3288 be16_to_cpu(target->pkey),
34aa654e 3289 be64_to_cpu(target->service_id),
747fe000 3290 target->sgid.raw, target->orig_dgid.raw);
34aa654e 3291 }
e7ffde01 3292
2d7091bc
BVA		 3293 ret = count;
3294
3295out:
3296 mutex_unlock(&host->add_target_mutex);
34aa654e
BVA		 3297
3298 scsi_host_put(target->scsi_host);
3299
2d7091bc 3300 return ret;
aef9ec39
RD		 3301
3302err_disconnect:
3303 srp_disconnect_target(target);
3304
d92c0da7
BVA		 3305 for (i = 0; i < target->ch_count; i++) {
3306 ch = &target->ch[i];
3307 srp_free_ch_ib(target, ch);
3308 srp_free_req_data(target, ch);
3309 }
aef9ec39 3310
d92c0da7 3311 kfree(target->ch);
2d7091bc 3312 goto out;
aef9ec39
RD		 3313}
3314
ee959b00 3315static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
aef9ec39 3316
ee959b00
TJ		 3317static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3318 char *buf)
aef9ec39 3319{
ee959b00 3320 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39 3321
05321937 3322 return sprintf(buf, "%s\n", host->srp_dev->dev->name);
aef9ec39
RD		 3323}
3324
ee959b00 3325static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
aef9ec39 3326
ee959b00
TJ		 3327static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3328 char *buf)
aef9ec39 3329{
ee959b00 3330 struct srp_host *host = container_of(dev, struct srp_host, dev);
aef9ec39
RD		 3331
3332 return sprintf(buf, "%d\n", host->port);
3333}
3334
ee959b00 3335static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
aef9ec39 3336
f5358a17 3337static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
aef9ec39
RD		 3338{
3339 struct srp_host *host;
3340
3341 host = kzalloc(sizeof *host, GFP_KERNEL);
3342 if (!host)
3343 return NULL;
3344
3345 INIT_LIST_HEAD(&host->target_list);
b3589fd4 3346 spin_lock_init(&host->target_lock);
aef9ec39 3347 init_completion(&host->released);
2d7091bc 3348 mutex_init(&host->add_target_mutex);
05321937 3349 host->srp_dev = device;
aef9ec39
RD		 3350 host->port = port;
3351
ee959b00
TJ		 3352 host->dev.class = &srp_class;
3353 host->dev.parent = device->dev->dma_device;
d927e38c 3354 dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
aef9ec39 3355
ee959b00 3356 if (device_register(&host->dev))
f5358a17 3357 goto free_host;
ee959b00 3358 if (device_create_file(&host->dev, &dev_attr_add_target))
aef9ec39 3359 goto err_class;
ee959b00 3360 if (device_create_file(&host->dev, &dev_attr_ibdev))
aef9ec39 3361 goto err_class;
ee959b00 3362 if (device_create_file(&host->dev, &dev_attr_port))
aef9ec39
RD		 3363 goto err_class;
3364
3365 return host;
3366
3367err_class:
ee959b00 3368 device_unregister(&host->dev);
aef9ec39 3369
f5358a17 3370free_host:
aef9ec39
RD		 3371 kfree(host);
3372
3373 return NULL;
3374}
3375
3376static void srp_add_one(struct ib_device *device)
3377{
f5358a17
RD		 3378 struct srp_device *srp_dev;
3379 struct ib_device_attr *dev_attr;
aef9ec39 3380 struct srp_host *host;
52ede08f
BVA		 3381 int mr_page_shift, s, e, p;
3382 u64 max_pages_per_mr;
aef9ec39 3383
f5358a17
RD		 3384 dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
3385 if (!dev_attr)
cf311cd4 3386 return;
aef9ec39 3387
f5358a17 3388 if (ib_query_device(device, dev_attr)) {
e0bda7d8 3389 pr_warn("Query device failed for %s\n", device->name);
f5358a17
RD		 3390 goto free_attr;
3391 }
3392
3393 srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
3394 if (!srp_dev)
3395 goto free_attr;
3396
d1b4289e
BVA		 3397 srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3398 device->map_phys_fmr && device->unmap_fmr);
5cfb1782
BVA		 3399 srp_dev->has_fr = (dev_attr->device_cap_flags &
3400 IB_DEVICE_MEM_MGT_EXTENSIONS);
3401 if (!srp_dev->has_fmr && !srp_dev->has_fr)
3402 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3403
3404 srp_dev->use_fast_reg = (srp_dev->has_fr &&
3405 (!srp_dev->has_fmr || prefer_fr));
d1b4289e 3406
f5358a17
RD		 3407 /*
3408 * Use the smallest page size supported by the HCA, down to a
8f26c9ff
DD		 3409 * minimum of 4096 bytes. We're unlikely to build large sglists
3410 * out of smaller entries.
f5358a17 3411 */
52ede08f
BVA		 3412 mr_page_shift = max(12, ffs(dev_attr->page_size_cap) - 1);
3413 srp_dev->mr_page_size = 1 << mr_page_shift;
3414 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1);
3415 max_pages_per_mr = dev_attr->max_mr_size;
3416 do_div(max_pages_per_mr, srp_dev->mr_page_size);
3417 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3418 max_pages_per_mr);
5cfb1782
BVA		 3419 if (srp_dev->use_fast_reg) {
3420 srp_dev->max_pages_per_mr =
3421 min_t(u32, srp_dev->max_pages_per_mr,
3422 dev_attr->max_fast_reg_page_list_len);
3423 }
52ede08f
BVA		 3424 srp_dev->mr_max_size = srp_dev->mr_page_size *
3425 srp_dev->max_pages_per_mr;
5cfb1782 3426 pr_debug("%s: mr_page_shift = %d, dev_attr->max_mr_size = %#llx, dev_attr->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
52ede08f 3427 device->name, mr_page_shift, dev_attr->max_mr_size,
5cfb1782 3428 dev_attr->max_fast_reg_page_list_len,
52ede08f 3429 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
f5358a17
RD		 3430
3431 INIT_LIST_HEAD(&srp_dev->dev_list);
3432
3433 srp_dev->dev = device;
3434 srp_dev->pd = ib_alloc_pd(device);
3435 if (IS_ERR(srp_dev->pd))
3436 goto free_dev;
3437
3438 srp_dev->mr = ib_get_dma_mr(srp_dev->pd,
3439 IB_ACCESS_LOCAL_WRITE |
3440 IB_ACCESS_REMOTE_READ |
3441 IB_ACCESS_REMOTE_WRITE);
3442 if (IS_ERR(srp_dev->mr))
3443 goto err_pd;
3444
07ebafba 3445 if (device->node_type == RDMA_NODE_IB_SWITCH) {
aef9ec39
RD		 3446 s = 0;
3447 e = 0;
3448 } else {
3449 s = 1;
3450 e = device->phys_port_cnt;
3451 }
3452
3453 for (p = s; p <= e; ++p) {
f5358a17 3454 host = srp_add_port(srp_dev, p);
aef9ec39 3455 if (host)
f5358a17 3456 list_add_tail(&host->list, &srp_dev->dev_list);
aef9ec39
RD		 3457 }
3458
f5358a17
RD		 3459 ib_set_client_data(device, &srp_client, srp_dev);
3460
3461 goto free_attr;
3462
3463err_pd:
3464 ib_dealloc_pd(srp_dev->pd);
3465
3466free_dev:
3467 kfree(srp_dev);
3468
3469free_attr:
3470 kfree(dev_attr);
aef9ec39
RD		 3471}
3472
3473static void srp_remove_one(struct ib_device *device)
3474{
f5358a17 3475 struct srp_device *srp_dev;
aef9ec39 3476 struct srp_host *host, *tmp_host;
ef6c49d8 3477 struct srp_target_port *target;
aef9ec39 3478
f5358a17 3479 srp_dev = ib_get_client_data(device, &srp_client);
1fe0cb84
DB		 3480 if (!srp_dev)
3481 return;
aef9ec39 3482
f5358a17 3483 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
ee959b00 3484 device_unregister(&host->dev);
aef9ec39
RD		 3485 /*
3486 * Wait for the sysfs entry to go away, so that no new
3487 * target ports can be created.
3488 */
3489 wait_for_completion(&host->released);
3490
3491 /*
ef6c49d8 3492 * Remove all target ports.
aef9ec39 3493 */
b3589fd4 3494 spin_lock(&host->target_lock);
ef6c49d8
BVA		 3495 list_for_each_entry(target, &host->target_list, list)
3496 srp_queue_remove_work(target);
b3589fd4 3497 spin_unlock(&host->target_lock);
aef9ec39
RD		 3498
3499 /*
bcc05910 3500 * Wait for tl_err and target port removal tasks.
aef9ec39 3501 */
ef6c49d8 3502 flush_workqueue(system_long_wq);
bcc05910 3503 flush_workqueue(srp_remove_wq);
aef9ec39 3504
aef9ec39
RD		 3505 kfree(host);
3506 }
3507
f5358a17
RD		 3508 ib_dereg_mr(srp_dev->mr);
3509 ib_dealloc_pd(srp_dev->pd);
3510
3511 kfree(srp_dev);
aef9ec39
RD		 3512}
3513
3236822b 3514static struct srp_function_template ib_srp_transport_functions = {
ed9b2264
BVA		 3515 .has_rport_state = true,
3516 .reset_timer_if_blocked = true,
a95cadb9 3517 .reconnect_delay = &srp_reconnect_delay,
ed9b2264
BVA		 3518 .fast_io_fail_tmo = &srp_fast_io_fail_tmo,
3519 .dev_loss_tmo = &srp_dev_loss_tmo,
3520 .reconnect = srp_rport_reconnect,
dc1bdbd9 3521 .rport_delete = srp_rport_delete,
ed9b2264 3522 .terminate_rport_io = srp_terminate_io,
3236822b
FT		 3523};
3524
aef9ec39
RD		 3525static int __init srp_init_module(void)
3526{
3527 int ret;
3528
dcb4cb85 3529 BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));
dd5e6e38 3530
49248644 3531 if (srp_sg_tablesize) {
e0bda7d8 3532 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
49248644
DD		 3533 if (!cmd_sg_entries)
3534 cmd_sg_entries = srp_sg_tablesize;
3535 }
3536
3537 if (!cmd_sg_entries)
3538 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3539
3540 if (cmd_sg_entries > 255) {
e0bda7d8 3541 pr_warn("Clamping cmd_sg_entries to 255\n");
49248644 3542 cmd_sg_entries = 255;
1e89a194
DD		 3543 }
3544
c07d424d
DD		 3545 if (!indirect_sg_entries)
3546 indirect_sg_entries = cmd_sg_entries;
3547 else if (indirect_sg_entries < cmd_sg_entries) {
e0bda7d8
BVA		 3548 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3549 cmd_sg_entries);
c07d424d
DD		 3550 indirect_sg_entries = cmd_sg_entries;
3551 }
3552
bcc05910 3553 srp_remove_wq = create_workqueue("srp_remove");
da05be29
WY		 3554 if (!srp_remove_wq) {
3555 ret = -ENOMEM;
bcc05910
BVA		 3556 goto out;
3557 }
3558
3559 ret = -ENOMEM;
3236822b
FT		 3560 ib_srp_transport_template =
3561 srp_attach_transport(&ib_srp_transport_functions);
3562 if (!ib_srp_transport_template)
bcc05910 3563 goto destroy_wq;
3236822b 3564
aef9ec39
RD		 3565 ret = class_register(&srp_class);
3566 if (ret) {
e0bda7d8 3567 pr_err("couldn't register class infiniband_srp\n");
bcc05910 3568 goto release_tr;
aef9ec39
RD		 3569 }
3570
c1a0b23b
MT		 3571 ib_sa_register_client(&srp_sa_client);
3572
aef9ec39
RD		 3573 ret = ib_register_client(&srp_client);
3574 if (ret) {
e0bda7d8 3575 pr_err("couldn't register IB client\n");
bcc05910 3576 goto unreg_sa;
aef9ec39
RD		 3577 }
3578
bcc05910
BVA		 3579out:
3580 return ret;
3581
3582unreg_sa:
3583 ib_sa_unregister_client(&srp_sa_client);
3584 class_unregister(&srp_class);
3585
3586release_tr:
3587 srp_release_transport(ib_srp_transport_template);
3588
3589destroy_wq:
3590 destroy_workqueue(srp_remove_wq);
3591 goto out;
aef9ec39
RD		 3592}
3593
3594static void __exit srp_cleanup_module(void)
3595{
3596 ib_unregister_client(&srp_client);
c1a0b23b 3597 ib_sa_unregister_client(&srp_sa_client);
aef9ec39 3598 class_unregister(&srp_class);
3236822b 3599 srp_release_transport(ib_srp_transport_template);
bcc05910 3600 destroy_workqueue(srp_remove_wq);
aef9ec39
RD		 3601}
3602
3603module_init(srp_init_module);
3604module_exit(srp_cleanup_module);
Linux kernel - Deliverables
This page took 1.01947 seconds and 5 git commands to generate.