projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
target: move the state and execute lists to the command
[deliverable/linux.git] / drivers / target / target_core_transport.c
CommitLineData
c66ac9db
NB		 1/*******************************************************************************
2 * Filename: target_core_transport.c
3 *
4 * This file contains the Generic Target Engine Core.
5 *
6 * Copyright (c) 2002, 2003, 2004, 2005 PyX Technologies, Inc.
7 * Copyright (c) 2005, 2006, 2007 SBE, Inc.
8 * Copyright (c) 2007-2010 Rising Tide Systems
9 * Copyright (c) 2008-2010 Linux-iSCSI.org
10 *
11 * Nicholas A. Bellinger <nab@kernel.org>
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
26 *
27 ******************************************************************************/
28
c66ac9db
NB		 29#include <linux/net.h>
30#include <linux/delay.h>
31#include <linux/string.h>
32#include <linux/timer.h>
33#include <linux/slab.h>
34#include <linux/blkdev.h>
35#include <linux/spinlock.h>
c66ac9db
NB		 36#include <linux/kthread.h>
37#include <linux/in.h>
38#include <linux/cdrom.h>
827509e3 39#include <linux/module.h>
015487b8 40#include <linux/ratelimit.h>
c66ac9db
NB		 41#include <asm/unaligned.h>
42#include <net/sock.h>
43#include <net/tcp.h>
44#include <scsi/scsi.h>
45#include <scsi/scsi_cmnd.h>
e66ecd50 46#include <scsi/scsi_tcq.h>
c66ac9db
NB		 47
48#include <target/target_core_base.h>
c4795fb2
CH		 49#include <target/target_core_backend.h>
50#include <target/target_core_fabric.h>
c66ac9db
NB		 51#include <target/target_core_configfs.h>
52
e26d99ae 53#include "target_core_internal.h"
c66ac9db 54#include "target_core_alua.h"
c66ac9db 55#include "target_core_pr.h"
c66ac9db
NB		 56#include "target_core_ua.h"
57
e3d6f909 58static int sub_api_initialized;
c66ac9db 59
35e0e757 60static struct workqueue_struct *target_completion_wq;
c66ac9db 61static struct kmem_cache *se_sess_cache;
c66ac9db 62struct kmem_cache *se_ua_cache;
c66ac9db
NB		 63struct kmem_cache *t10_pr_reg_cache;
64struct kmem_cache *t10_alua_lu_gp_cache;
65struct kmem_cache *t10_alua_lu_gp_mem_cache;
66struct kmem_cache *t10_alua_tg_pt_gp_cache;
67struct kmem_cache *t10_alua_tg_pt_gp_mem_cache;
68
c66ac9db 69static int transport_generic_write_pending(struct se_cmd *);
5951146d 70static int transport_processing_thread(void *param);
4d2300cc 71static int __transport_execute_tasks(struct se_device *dev, struct se_cmd *);
c66ac9db 72static void transport_complete_task_attr(struct se_cmd *cmd);
07bde79a 73static void transport_handle_queue_full(struct se_cmd *cmd,
e057f533 74 struct se_device *dev);
c66ac9db 75static void transport_free_dev_tasks(struct se_cmd *cmd);
05d1c7c0 76static int transport_generic_get_mem(struct se_cmd *cmd);
39c05f32 77static void transport_put_cmd(struct se_cmd *cmd);
3df8d40b 78static void transport_remove_cmd_from_queue(struct se_cmd *cmd);
c66ac9db 79static int transport_set_sense_codes(struct se_cmd *cmd, u8 asc, u8 ascq);
35e0e757 80static void target_complete_ok_work(struct work_struct *work);
c66ac9db 81
e3d6f909 82int init_se_kmem_caches(void)
c66ac9db 83{
c66ac9db
NB		 84 se_sess_cache = kmem_cache_create("se_sess_cache",
85 sizeof(struct se_session), __alignof__(struct se_session),
86 0, NULL);
6708bb27
AG		 87 if (!se_sess_cache) {
88 pr_err("kmem_cache_create() for struct se_session"
c66ac9db 89 " failed\n");
c8e31f26 90 goto out;
c66ac9db
NB		 91 }
92 se_ua_cache = kmem_cache_create("se_ua_cache",
93 sizeof(struct se_ua), __alignof__(struct se_ua),
94 0, NULL);
6708bb27
AG		 95 if (!se_ua_cache) {
96 pr_err("kmem_cache_create() for struct se_ua failed\n");
35e0e757 97 goto out_free_sess_cache;
c66ac9db 98 }
c66ac9db
NB		 99 t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
100 sizeof(struct t10_pr_registration),
101 __alignof__(struct t10_pr_registration), 0, NULL);
6708bb27
AG		 102 if (!t10_pr_reg_cache) {
103 pr_err("kmem_cache_create() for struct t10_pr_registration"
c66ac9db 104 " failed\n");
35e0e757 105 goto out_free_ua_cache;
c66ac9db
NB		 106 }
107 t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
108 sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
109 0, NULL);
6708bb27
AG		 110 if (!t10_alua_lu_gp_cache) {
111 pr_err("kmem_cache_create() for t10_alua_lu_gp_cache"
c66ac9db 112 " failed\n");
35e0e757 113 goto out_free_pr_reg_cache;
c66ac9db
NB		 114 }
115 t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
116 sizeof(struct t10_alua_lu_gp_member),
117 __alignof__(struct t10_alua_lu_gp_member), 0, NULL);
6708bb27
AG		 118 if (!t10_alua_lu_gp_mem_cache) {
119 pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_"
c66ac9db 120 "cache failed\n");
35e0e757 121 goto out_free_lu_gp_cache;
c66ac9db
NB		 122 }
123 t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
124 sizeof(struct t10_alua_tg_pt_gp),
125 __alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
6708bb27
AG		 126 if (!t10_alua_tg_pt_gp_cache) {
127 pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
c66ac9db 128 "cache failed\n");
35e0e757 129 goto out_free_lu_gp_mem_cache;
c66ac9db
NB		 130 }
131 t10_alua_tg_pt_gp_mem_cache = kmem_cache_create(
132 "t10_alua_tg_pt_gp_mem_cache",
133 sizeof(struct t10_alua_tg_pt_gp_member),
134 __alignof__(struct t10_alua_tg_pt_gp_member),
135 0, NULL);
6708bb27
AG		 136 if (!t10_alua_tg_pt_gp_mem_cache) {
137 pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
c66ac9db 138 "mem_t failed\n");
35e0e757 139 goto out_free_tg_pt_gp_cache;
c66ac9db
NB		 140 }
141
35e0e757
CH		 142 target_completion_wq = alloc_workqueue("target_completion",
143 WQ_MEM_RECLAIM, 0);
144 if (!target_completion_wq)
145 goto out_free_tg_pt_gp_mem_cache;
146
c66ac9db 147 return 0;
35e0e757
CH		 148
149out_free_tg_pt_gp_mem_cache:
150 kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
151out_free_tg_pt_gp_cache:
152 kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
153out_free_lu_gp_mem_cache:
154 kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
155out_free_lu_gp_cache:
156 kmem_cache_destroy(t10_alua_lu_gp_cache);
157out_free_pr_reg_cache:
158 kmem_cache_destroy(t10_pr_reg_cache);
159out_free_ua_cache:
160 kmem_cache_destroy(se_ua_cache);
161out_free_sess_cache:
162 kmem_cache_destroy(se_sess_cache);
c66ac9db 163out:
e3d6f909 164 return -ENOMEM;
c66ac9db
NB		 165}
166
e3d6f909 167void release_se_kmem_caches(void)
c66ac9db 168{
35e0e757 169 destroy_workqueue(target_completion_wq);
c66ac9db
NB		 170 kmem_cache_destroy(se_sess_cache);
171 kmem_cache_destroy(se_ua_cache);
c66ac9db
NB		 172 kmem_cache_destroy(t10_pr_reg_cache);
173 kmem_cache_destroy(t10_alua_lu_gp_cache);
174 kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
175 kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
176 kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
c66ac9db
NB		 177}
178
e3d6f909
AG		 179/* This code ensures unique mib indexes are handed out. */
180static DEFINE_SPINLOCK(scsi_mib_index_lock);
181static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX];
e89d15ee
NB		 182
183/*
184 * Allocate a new row index for the entry type specified
185 */
186u32 scsi_get_new_index(scsi_index_t type)
187{
188 u32 new_index;
189
e3d6f909 190 BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX));
e89d15ee 191
e3d6f909
AG		 192 spin_lock(&scsi_mib_index_lock);
193 new_index = ++scsi_mib_index[type];
194 spin_unlock(&scsi_mib_index_lock);
e89d15ee
NB		 195
196 return new_index;
197}
198
e26d99ae 199static void transport_init_queue_obj(struct se_queue_obj *qobj)
c66ac9db
NB		 200{
201 atomic_set(&qobj->queue_cnt, 0);
202 INIT_LIST_HEAD(&qobj->qobj_list);
203 init_waitqueue_head(&qobj->thread_wq);
204 spin_lock_init(&qobj->cmd_queue_lock);
205}
c66ac9db 206
dbc5623e 207void transport_subsystem_check_init(void)
c66ac9db
NB		 208{
209 int ret;
210
dbc5623e
NB		 211 if (sub_api_initialized)
212 return;
213
c66ac9db
NB		 214 ret = request_module("target_core_iblock");
215 if (ret != 0)
6708bb27 216 pr_err("Unable to load target_core_iblock\n");
c66ac9db
NB		 217
218 ret = request_module("target_core_file");
219 if (ret != 0)
6708bb27 220 pr_err("Unable to load target_core_file\n");
c66ac9db
NB		 221
222 ret = request_module("target_core_pscsi");
223 if (ret != 0)
6708bb27 224 pr_err("Unable to load target_core_pscsi\n");
c66ac9db
NB		 225
226 ret = request_module("target_core_stgt");
227 if (ret != 0)
6708bb27 228 pr_err("Unable to load target_core_stgt\n");
c66ac9db 229
e3d6f909 230 sub_api_initialized = 1;
dbc5623e 231 return;
c66ac9db
NB		 232}
233
234struct se_session *transport_init_session(void)
235{
236 struct se_session *se_sess;
237
238 se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
6708bb27
AG		 239 if (!se_sess) {
240 pr_err("Unable to allocate struct se_session from"
c66ac9db
NB		 241 " se_sess_cache\n");
242 return ERR_PTR(-ENOMEM);
243 }
244 INIT_LIST_HEAD(&se_sess->sess_list);
245 INIT_LIST_HEAD(&se_sess->sess_acl_list);
a17f091d
NB		 246 INIT_LIST_HEAD(&se_sess->sess_cmd_list);
247 INIT_LIST_HEAD(&se_sess->sess_wait_list);
248 spin_lock_init(&se_sess->sess_cmd_lock);
41ac82b6 249 kref_init(&se_sess->sess_kref);
c66ac9db
NB		 250
251 return se_sess;
252}
253EXPORT_SYMBOL(transport_init_session);
254
255/*
140854cb 256 * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called.
c66ac9db
NB		 257 */
258void __transport_register_session(
259 struct se_portal_group *se_tpg,
260 struct se_node_acl *se_nacl,
261 struct se_session *se_sess,
262 void *fabric_sess_ptr)
263{
264 unsigned char buf[PR_REG_ISID_LEN];
265
266 se_sess->se_tpg = se_tpg;
267 se_sess->fabric_sess_ptr = fabric_sess_ptr;
268 /*
269 * Used by struct se_node_acl's under ConfigFS to locate active se_session-t
270 *
271 * Only set for struct se_session's that will actually be moving I/O.
272 * eg: *NOT* discovery sessions.
273 */
274 if (se_nacl) {
275 /*
276 * If the fabric module supports an ISID based TransportID,
277 * save this value in binary from the fabric I_T Nexus now.
278 */
e3d6f909 279 if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
c66ac9db 280 memset(&buf[0], 0, PR_REG_ISID_LEN);
e3d6f909 281 se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess,
c66ac9db
NB		 282 &buf[0], PR_REG_ISID_LEN);
283 se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
284 }
afb999ff
NB		 285 kref_get(&se_nacl->acl_kref);
286
c66ac9db
NB		 287 spin_lock_irq(&se_nacl->nacl_sess_lock);
288 /*
289 * The se_nacl->nacl_sess pointer will be set to the
290 * last active I_T Nexus for each struct se_node_acl.
291 */
292 se_nacl->nacl_sess = se_sess;
293
294 list_add_tail(&se_sess->sess_acl_list,
295 &se_nacl->acl_sess_list);
296 spin_unlock_irq(&se_nacl->nacl_sess_lock);
297 }
298 list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);
299
6708bb27 300 pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
e3d6f909 301 se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr);
c66ac9db
NB		 302}
303EXPORT_SYMBOL(__transport_register_session);
304
305void transport_register_session(
306 struct se_portal_group *se_tpg,
307 struct se_node_acl *se_nacl,
308 struct se_session *se_sess,
309 void *fabric_sess_ptr)
310{
140854cb
NB		 311 unsigned long flags;
312
313 spin_lock_irqsave(&se_tpg->session_lock, flags);
c66ac9db 314 __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
140854cb 315 spin_unlock_irqrestore(&se_tpg->session_lock, flags);
c66ac9db
NB		 316}
317EXPORT_SYMBOL(transport_register_session);
318
41ac82b6
NB		 319static void target_release_session(struct kref *kref)
320{
321 struct se_session *se_sess = container_of(kref,
322 struct se_session, sess_kref);
323 struct se_portal_group *se_tpg = se_sess->se_tpg;
324
325 se_tpg->se_tpg_tfo->close_session(se_sess);
326}
327
328void target_get_session(struct se_session *se_sess)
329{
330 kref_get(&se_sess->sess_kref);
331}
332EXPORT_SYMBOL(target_get_session);
333
334int target_put_session(struct se_session *se_sess)
335{
336 return kref_put(&se_sess->sess_kref, target_release_session);
337}
338EXPORT_SYMBOL(target_put_session);
339
afb999ff
NB		 340static void target_complete_nacl(struct kref *kref)
341{
342 struct se_node_acl *nacl = container_of(kref,
343 struct se_node_acl, acl_kref);
344
345 complete(&nacl->acl_free_comp);
346}
347
348void target_put_nacl(struct se_node_acl *nacl)
349{
350 kref_put(&nacl->acl_kref, target_complete_nacl);
351}
352
c66ac9db
NB		 353void transport_deregister_session_configfs(struct se_session *se_sess)
354{
355 struct se_node_acl *se_nacl;
23388864 356 unsigned long flags;
c66ac9db
NB		 357 /*
358 * Used by struct se_node_acl's under ConfigFS to locate active struct se_session
359 */
360 se_nacl = se_sess->se_node_acl;
6708bb27 361 if (se_nacl) {
23388864 362 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
337c0607
NB		 363 if (se_nacl->acl_stop == 0)
364 list_del(&se_sess->sess_acl_list);
c66ac9db
NB		 365 /*
366 * If the session list is empty, then clear the pointer.
367 * Otherwise, set the struct se_session pointer from the tail
368 * element of the per struct se_node_acl active session list.
369 */
370 if (list_empty(&se_nacl->acl_sess_list))
371 se_nacl->nacl_sess = NULL;
372 else {
373 se_nacl->nacl_sess = container_of(
374 se_nacl->acl_sess_list.prev,
375 struct se_session, sess_acl_list);
376 }
23388864 377 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
c66ac9db
NB		 378 }
379}
380EXPORT_SYMBOL(transport_deregister_session_configfs);
381
382void transport_free_session(struct se_session *se_sess)
383{
384 kmem_cache_free(se_sess_cache, se_sess);
385}
386EXPORT_SYMBOL(transport_free_session);
387
388void transport_deregister_session(struct se_session *se_sess)
389{
390 struct se_portal_group *se_tpg = se_sess->se_tpg;
01468346 391 struct target_core_fabric_ops *se_tfo;
c66ac9db 392 struct se_node_acl *se_nacl;
e63a8e19 393 unsigned long flags;
01468346 394 bool comp_nacl = true;
c66ac9db 395
6708bb27 396 if (!se_tpg) {
c66ac9db
NB		 397 transport_free_session(se_sess);
398 return;
399 }
01468346 400 se_tfo = se_tpg->se_tpg_tfo;
c66ac9db 401
e63a8e19 402 spin_lock_irqsave(&se_tpg->session_lock, flags);
c66ac9db
NB		 403 list_del(&se_sess->sess_list);
404 se_sess->se_tpg = NULL;
405 se_sess->fabric_sess_ptr = NULL;
e63a8e19 406 spin_unlock_irqrestore(&se_tpg->session_lock, flags);
c66ac9db
NB		 407
408 /*
409 * Determine if we need to do extra work for this initiator node's
410 * struct se_node_acl if it had been previously dynamically generated.
411 */
412 se_nacl = se_sess->se_node_acl;
01468346
NB		 413
414 spin_lock_irqsave(&se_tpg->acl_node_lock, flags);
415 if (se_nacl && se_nacl->dynamic_node_acl) {
416 if (!se_tfo->tpg_check_demo_mode_cache(se_tpg)) {
417 list_del(&se_nacl->acl_list);
418 se_tpg->num_node_acls--;
419 spin_unlock_irqrestore(&se_tpg->acl_node_lock, flags);
420 core_tpg_wait_for_nacl_pr_ref(se_nacl);
421 core_free_device_list_for_node(se_nacl, se_tpg);
422 se_tfo->tpg_release_fabric_acl(se_tpg, se_nacl);
423
424 comp_nacl = false;
425 spin_lock_irqsave(&se_tpg->acl_node_lock, flags);
c66ac9db 426 }
c66ac9db 427 }
01468346 428 spin_unlock_irqrestore(&se_tpg->acl_node_lock, flags);
c66ac9db 429
6708bb27 430 pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
e3d6f909 431 se_tpg->se_tpg_tfo->get_fabric_name());
01468346 432 /*
afb999ff
NB		 433 * If last kref is dropping now for an explict NodeACL, awake sleeping
434 * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group
435 * removal context.
01468346
NB		 436 */
437 if (se_nacl && comp_nacl == true)
afb999ff 438 target_put_nacl(se_nacl);
01468346 439
afb999ff 440 transport_free_session(se_sess);
c66ac9db
NB		 441}
442EXPORT_SYMBOL(transport_deregister_session);
443
444/*
a1d8b49a 445 * Called with cmd->t_state_lock held.
c66ac9db 446 */
cf572a96 447static void target_remove_from_state_list(struct se_cmd *cmd)
c66ac9db 448{
42bf829e 449 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 450 unsigned long flags;
451
42bf829e
CH		 452 if (!dev)
453 return;
c66ac9db 454
cf572a96
CH		 455 if (cmd->transport_state & CMD_T_BUSY)
456 return;
c66ac9db 457
cf572a96
CH		 458 spin_lock_irqsave(&dev->execute_task_lock, flags);
459 if (cmd->state_active) {
460 list_del(&cmd->state_list);
461 atomic_dec(&cmd->t_task_cdbs_ex_left);
462 cmd->state_active = false;
c66ac9db 463 }
cf572a96 464 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
c66ac9db
NB		 465}
466
467/* transport_cmd_check_stop():
468 *
7d680f3b 469 * 'transport_off = 1' determines if CMD_T_ACTIVE should be cleared.
c66ac9db
NB		 470 * 'transport_off = 2' determines if task_dev_state should be removed.
471 *
472 * A non-zero u8 t_state sets cmd->t_state.
473 * Returns 1 when command is stopped, else 0.
474 */
475static int transport_cmd_check_stop(
476 struct se_cmd *cmd,
477 int transport_off,
478 u8 t_state)
479{
480 unsigned long flags;
481
a1d8b49a 482 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db
NB		 483 /*
484 * Determine if IOCTL context caller in requesting the stopping of this
485 * command for LUN shutdown purposes.
486 */
7d680f3b
CH		 487 if (cmd->transport_state & CMD_T_LUN_STOP) {
488 pr_debug("%s:%d CMD_T_LUN_STOP for ITT: 0x%08x\n",
489 __func__, __LINE__, cmd->se_tfo->get_task_tag(cmd));
c66ac9db 490
7d680f3b 491 cmd->transport_state &= ~CMD_T_ACTIVE;
c66ac9db 492 if (transport_off == 2)
cf572a96 493 target_remove_from_state_list(cmd);
a1d8b49a 494 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 495
a1d8b49a 496 complete(&cmd->transport_lun_stop_comp);
c66ac9db
NB		 497 return 1;
498 }
499 /*
500 * Determine if frontend context caller is requesting the stopping of
e3d6f909 501 * this command for frontend exceptions.
c66ac9db 502 */
7d680f3b
CH		 503 if (cmd->transport_state & CMD_T_STOP) {
504 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08x\n",
505 __func__, __LINE__,
e3d6f909 506 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 507
c66ac9db 508 if (transport_off == 2)
cf572a96 509 target_remove_from_state_list(cmd);
c66ac9db
NB		 510
511 /*
512 * Clear struct se_cmd->se_lun before the transport_off == 2 handoff
513 * to FE.
514 */
515 if (transport_off == 2)
516 cmd->se_lun = NULL;
a1d8b49a 517 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 518
a1d8b49a 519 complete(&cmd->t_transport_stop_comp);
c66ac9db
NB		 520 return 1;
521 }
522 if (transport_off) {
7d680f3b 523 cmd->transport_state &= ~CMD_T_ACTIVE;
c66ac9db 524 if (transport_off == 2) {
cf572a96 525 target_remove_from_state_list(cmd);
c66ac9db
NB		 526 /*
527 * Clear struct se_cmd->se_lun before the transport_off == 2
528 * handoff to fabric module.
529 */
530 cmd->se_lun = NULL;
531 /*
532 * Some fabric modules like tcm_loop can release
25985edc 533 * their internally allocated I/O reference now and
c66ac9db 534 * struct se_cmd now.
88dd9e26
NB		 535 *
536 * Fabric modules are expected to return '1' here if the
537 * se_cmd being passed is released at this point,
538 * or zero if not being released.
c66ac9db 539 */
e3d6f909 540 if (cmd->se_tfo->check_stop_free != NULL) {
c66ac9db 541 spin_unlock_irqrestore(
a1d8b49a 542 &cmd->t_state_lock, flags);
c66ac9db 543
88dd9e26 544 return cmd->se_tfo->check_stop_free(cmd);
c66ac9db
NB		 545 }
546 }
a1d8b49a 547 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 548
549 return 0;
550 } else if (t_state)
551 cmd->t_state = t_state;
a1d8b49a 552 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 553
554 return 0;
555}
556
557static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
558{
559 return transport_cmd_check_stop(cmd, 2, 0);
560}
561
562static void transport_lun_remove_cmd(struct se_cmd *cmd)
563{
e3d6f909 564 struct se_lun *lun = cmd->se_lun;
c66ac9db
NB		 565 unsigned long flags;
566
567 if (!lun)
568 return;
569
a1d8b49a 570 spin_lock_irqsave(&cmd->t_state_lock, flags);
7d680f3b
CH		 571 if (cmd->transport_state & CMD_T_DEV_ACTIVE) {
572 cmd->transport_state &= ~CMD_T_DEV_ACTIVE;
cf572a96 573 target_remove_from_state_list(cmd);
c66ac9db 574 }
a1d8b49a 575 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 576
c66ac9db 577 spin_lock_irqsave(&lun->lun_cmd_lock, flags);
3d26fea0
CH		 578 if (!list_empty(&cmd->se_lun_node))
579 list_del_init(&cmd->se_lun_node);
c66ac9db
NB		 580 spin_unlock_irqrestore(&lun->lun_cmd_lock, flags);
581}
582
583void transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
584{
c8e31f26 585 if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
8dc52b54 586 transport_lun_remove_cmd(cmd);
c66ac9db
NB		 587
588 if (transport_cmd_check_stop_to_fabric(cmd))
589 return;
77039d1e 590 if (remove) {
3df8d40b 591 transport_remove_cmd_from_queue(cmd);
e6a2573f 592 transport_put_cmd(cmd);
77039d1e 593 }
c66ac9db
NB		 594}
595
f7a5cc0b
CH		 596static void transport_add_cmd_to_queue(struct se_cmd *cmd, int t_state,
597 bool at_head)
c66ac9db
NB		 598{
599 struct se_device *dev = cmd->se_dev;
e3d6f909 600 struct se_queue_obj *qobj = &dev->dev_queue_obj;
c66ac9db
NB		 601 unsigned long flags;
602
c66ac9db 603 if (t_state) {
a1d8b49a 604 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db 605 cmd->t_state = t_state;
7d680f3b 606 cmd->transport_state |= CMD_T_ACTIVE;
a1d8b49a 607 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 608 }
609
610 spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
79a7fef2
RD		 611
612 /* If the cmd is already on the list, remove it before we add it */
613 if (!list_empty(&cmd->se_queue_node))
614 list_del(&cmd->se_queue_node);
615 else
616 atomic_inc(&qobj->queue_cnt);
617
f7a5cc0b 618 if (at_head)
07bde79a 619 list_add(&cmd->se_queue_node, &qobj->qobj_list);
f7a5cc0b 620 else
07bde79a 621 list_add_tail(&cmd->se_queue_node, &qobj->qobj_list);
7d680f3b 622 cmd->transport_state |= CMD_T_QUEUED;
c66ac9db
NB		 623 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
624
c66ac9db 625 wake_up_interruptible(&qobj->thread_wq);
c66ac9db
NB		 626}
627
5951146d
AG		 628static struct se_cmd *
629transport_get_cmd_from_queue(struct se_queue_obj *qobj)
c66ac9db 630{
5951146d 631 struct se_cmd *cmd;
c66ac9db
NB		 632 unsigned long flags;
633
634 spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
635 if (list_empty(&qobj->qobj_list)) {
636 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
637 return NULL;
638 }
5951146d 639 cmd = list_first_entry(&qobj->qobj_list, struct se_cmd, se_queue_node);
c66ac9db 640
7d680f3b 641 cmd->transport_state &= ~CMD_T_QUEUED;
79a7fef2 642 list_del_init(&cmd->se_queue_node);
c66ac9db
NB		 643 atomic_dec(&qobj->queue_cnt);
644 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
645
5951146d 646 return cmd;
c66ac9db
NB		 647}
648
3df8d40b 649static void transport_remove_cmd_from_queue(struct se_cmd *cmd)
c66ac9db 650{
3df8d40b 651 struct se_queue_obj *qobj = &cmd->se_dev->dev_queue_obj;
c66ac9db
NB		 652 unsigned long flags;
653
654 spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
7d680f3b 655 if (!(cmd->transport_state & CMD_T_QUEUED)) {
c66ac9db
NB		 656 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
657 return;
658 }
7d680f3b 659 cmd->transport_state &= ~CMD_T_QUEUED;
79a7fef2
RD		 660 atomic_dec(&qobj->queue_cnt);
661 list_del_init(&cmd->se_queue_node);
c66ac9db 662 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
c66ac9db
NB		 663}
664
665/*
666 * Completion function used by TCM subsystem plugins (such as FILEIO)
667 * for queueing up response from struct se_subsystem_api->do_task()
668 */
669void transport_complete_sync_cache(struct se_cmd *cmd, int good)
670{
785fdf70 671 struct se_task *task = cmd->t_task;
c66ac9db
NB		 672
673 if (good) {
674 cmd->scsi_status = SAM_STAT_GOOD;
675 task->task_scsi_status = GOOD;
676 } else {
677 task->task_scsi_status = SAM_STAT_CHECK_CONDITION;
03e98c9e
NB		 678 task->task_se_cmd->scsi_sense_reason =
679 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
680
c66ac9db
NB		 681 }
682
683 transport_complete_task(task, good);
684}
685EXPORT_SYMBOL(transport_complete_sync_cache);
686
35e0e757
CH		 687static void target_complete_failure_work(struct work_struct *work)
688{
689 struct se_cmd *cmd = container_of(work, struct se_cmd, work);
690
03e98c9e 691 transport_generic_request_failure(cmd);
35e0e757
CH		 692}
693
c66ac9db
NB		 694/* transport_complete_task():
695 *
696 * Called from interrupt and non interrupt context depending
697 * on the transport plugin.
698 */
699void transport_complete_task(struct se_task *task, int success)
700{
e3d6f909 701 struct se_cmd *cmd = task->task_se_cmd;
42bf829e 702 struct se_device *dev = cmd->se_dev;
c66ac9db 703 unsigned long flags;
c66ac9db 704
a1d8b49a 705 spin_lock_irqsave(&cmd->t_state_lock, flags);
cf572a96 706 cmd->transport_state &= ~CMD_T_BUSY;
c66ac9db
NB		 707
708 /*
709 * See if any sense data exists, if so set the TASK_SENSE flag.
710 * Also check for any other post completion work that needs to be
711 * done by the plugins.
712 */
713 if (dev && dev->transport->transport_complete) {
714 if (dev->transport->transport_complete(task) != 0) {
715 cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE;
c66ac9db
NB		 716 success = 1;
717 }
718 }
719
720 /*
721 * See if we are waiting for outstanding struct se_task
722 * to complete for an exception condition
723 */
cf572a96 724 if (cmd->transport_state & CMD_T_REQUEST_STOP) {
a1d8b49a 725 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
cf572a96 726 complete(&cmd->task_stop_comp);
c66ac9db
NB		 727 return;
728 }
2235007c
CH		 729
730 if (!success)
7d680f3b 731 cmd->transport_state |= CMD_T_FAILED;
2235007c 732
c66ac9db
NB		 733 /*
734 * Decrement the outstanding t_task_cdbs_left count. The last
735 * struct se_task from struct se_cmd will complete itself into the
736 * device queue depending upon int success.
737 */
6708bb27 738 if (!atomic_dec_and_test(&cmd->t_task_cdbs_left)) {
a1d8b49a 739 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 740 return;
741 }
3d28934a
NB		 742 /*
743 * Check for case where an explict ABORT_TASK has been received
744 * and transport_wait_for_tasks() will be waiting for completion..
745 */
746 if (cmd->transport_state & CMD_T_ABORTED &&
747 cmd->transport_state & CMD_T_STOP) {
748 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
749 complete(&cmd->t_transport_stop_comp);
750 return;
751 } else if (cmd->transport_state & CMD_T_FAILED) {
41e16e98 752 cmd->scsi_sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
35e0e757 753 INIT_WORK(&cmd->work, target_complete_failure_work);
c66ac9db 754 } else {
35e0e757 755 INIT_WORK(&cmd->work, target_complete_ok_work);
c66ac9db 756 }
35e0e757 757
35e0e757 758 cmd->t_state = TRANSPORT_COMPLETE;
3d28934a 759 cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE);
a1d8b49a 760 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 761
35e0e757 762 queue_work(target_completion_wq, &cmd->work);
c66ac9db
NB		 763}
764EXPORT_SYMBOL(transport_complete_task);
765
6bb35e00
CH		 766void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status)
767{
785fdf70 768 struct se_task *task = cmd->t_task;
6bb35e00
CH		 769
770 task->task_scsi_status = scsi_status;
771 transport_complete_task(task, scsi_status == GOOD);
772}
773EXPORT_SYMBOL(target_complete_cmd);
774
cf572a96 775static void target_add_to_state_list(struct se_cmd *cmd)
c66ac9db 776{
cf572a96
CH		 777 struct se_device *dev = cmd->se_dev;
778 unsigned long flags;
c66ac9db 779
cf572a96
CH		 780 spin_lock_irqsave(&dev->execute_task_lock, flags);
781 if (!cmd->state_active) {
782 list_add_tail(&cmd->state_list, &dev->state_list);
783 cmd->state_active = true;
c66ac9db 784 }
cf572a96 785 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
c66ac9db
NB		 786}
787
cf572a96 788static void __target_add_to_execute_list(struct se_cmd *cmd)
c66ac9db 789{
cf572a96
CH		 790 struct se_device *dev = cmd->se_dev;
791 bool head_of_queue = false;
c66ac9db 792
cf572a96 793 if (!list_empty(&cmd->execute_list))
c66ac9db 794 return;
c66ac9db 795
cf572a96
CH		 796 if (dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED &&
797 cmd->sam_task_attr == MSG_HEAD_TAG)
798 head_of_queue = true;
c66ac9db 799
cf572a96
CH		 800 if (head_of_queue)
801 list_add(&cmd->execute_list, &dev->execute_list);
802 else
803 list_add_tail(&cmd->execute_list, &dev->execute_list);
785fdf70 804
cf572a96 805 atomic_inc(&dev->execute_tasks);
1880807a 806
cf572a96
CH		 807 if (cmd->state_active)
808 return;
c66ac9db 809
cf572a96
CH		 810 if (head_of_queue)
811 list_add(&cmd->state_list, &dev->state_list);
812 else
813 list_add_tail(&cmd->state_list, &dev->state_list);
c66ac9db 814
cf572a96 815 cmd->state_active = true;
4d2300cc
NB		 816}
817
cf572a96 818static void target_add_to_execute_list(struct se_cmd *cmd)
4d2300cc
NB		 819{
820 unsigned long flags;
821 struct se_device *dev = cmd->se_dev;
822
823 spin_lock_irqsave(&dev->execute_task_lock, flags);
cf572a96 824 __target_add_to_execute_list(cmd);
c66ac9db 825 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
c66ac9db
NB		 826}
827
cf572a96 828void __target_remove_from_execute_list(struct se_cmd *cmd)
04629b7b 829{
cf572a96
CH		 830 list_del_init(&cmd->execute_list);
831 atomic_dec(&cmd->se_dev->execute_tasks);
04629b7b
CH		 832}
833
cf572a96 834static void target_remove_from_execute_list(struct se_cmd *cmd)
c66ac9db 835{
cf572a96 836 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 837 unsigned long flags;
838
cf572a96 839 if (WARN_ON(list_empty(&cmd->execute_list)))
af57c3ac 840 return;
af57c3ac 841
c66ac9db 842 spin_lock_irqsave(&dev->execute_task_lock, flags);
cf572a96 843 __target_remove_from_execute_list(cmd);
c66ac9db
NB		 844 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
845}
846
07bde79a 847/*
f147abb4 848 * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status
07bde79a
NB		 849 */
850
851static void target_qf_do_work(struct work_struct *work)
852{
853 struct se_device *dev = container_of(work, struct se_device,
854 qf_work_queue);
bcac364a 855 LIST_HEAD(qf_cmd_list);
07bde79a
NB		 856 struct se_cmd *cmd, *cmd_tmp;
857
858 spin_lock_irq(&dev->qf_cmd_lock);
bcac364a
RD		 859 list_splice_init(&dev->qf_cmd_list, &qf_cmd_list);
860 spin_unlock_irq(&dev->qf_cmd_lock);
07bde79a 861
bcac364a 862 list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) {
07bde79a
NB		 863 list_del(&cmd->se_qf_node);
864 atomic_dec(&dev->dev_qf_count);
865 smp_mb__after_atomic_dec();
07bde79a 866
6708bb27 867 pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
07bde79a 868 " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd,
e057f533 869 (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" :
07bde79a
NB		 870 (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING"
871 : "UNKNOWN");
f7a5cc0b
CH		 872
873 transport_add_cmd_to_queue(cmd, cmd->t_state, true);
07bde79a 874 }
07bde79a
NB		 875}
876
c66ac9db
NB		 877unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
878{
879 switch (cmd->data_direction) {
880 case DMA_NONE:
881 return "NONE";
882 case DMA_FROM_DEVICE:
883 return "READ";
884 case DMA_TO_DEVICE:
885 return "WRITE";
886 case DMA_BIDIRECTIONAL:
887 return "BIDI";
888 default:
889 break;
890 }
891
892 return "UNKNOWN";
893}
894
895void transport_dump_dev_state(
896 struct se_device *dev,
897 char *b,
898 int *bl)
899{
900 *bl += sprintf(b + *bl, "Status: ");
901 switch (dev->dev_status) {
902 case TRANSPORT_DEVICE_ACTIVATED:
903 *bl += sprintf(b + *bl, "ACTIVATED");
904 break;
905 case TRANSPORT_DEVICE_DEACTIVATED:
906 *bl += sprintf(b + *bl, "DEACTIVATED");
907 break;
908 case TRANSPORT_DEVICE_SHUTDOWN:
909 *bl += sprintf(b + *bl, "SHUTDOWN");
910 break;
911 case TRANSPORT_DEVICE_OFFLINE_ACTIVATED:
912 case TRANSPORT_DEVICE_OFFLINE_DEACTIVATED:
913 *bl += sprintf(b + *bl, "OFFLINE");
914 break;
915 default:
916 *bl += sprintf(b + *bl, "UNKNOWN=%d", dev->dev_status);
917 break;
918 }
919
65586d51
NB		 920 *bl += sprintf(b + *bl, " Execute/Max Queue Depth: %d/%d",
921 atomic_read(&dev->execute_tasks), dev->queue_depth);
c66ac9db 922 *bl += sprintf(b + *bl, " SectorSize: %u MaxSectors: %u\n",
e3d6f909 923 dev->se_sub_dev->se_dev_attrib.block_size, dev->se_sub_dev->se_dev_attrib.max_sectors);
c66ac9db
NB		 924 *bl += sprintf(b + *bl, " ");
925}
926
c66ac9db
NB		 927void transport_dump_vpd_proto_id(
928 struct t10_vpd *vpd,
929 unsigned char *p_buf,
930 int p_buf_len)
931{
932 unsigned char buf[VPD_TMP_BUF_SIZE];
933 int len;
934
935 memset(buf, 0, VPD_TMP_BUF_SIZE);
936 len = sprintf(buf, "T10 VPD Protocol Identifier: ");
937
938 switch (vpd->protocol_identifier) {
939 case 0x00:
940 sprintf(buf+len, "Fibre Channel\n");
941 break;
942 case 0x10:
943 sprintf(buf+len, "Parallel SCSI\n");
944 break;
945 case 0x20:
946 sprintf(buf+len, "SSA\n");
947 break;
948 case 0x30:
949 sprintf(buf+len, "IEEE 1394\n");
950 break;
951 case 0x40:
952 sprintf(buf+len, "SCSI Remote Direct Memory Access"
953 " Protocol\n");
954 break;
955 case 0x50:
956 sprintf(buf+len, "Internet SCSI (iSCSI)\n");
957 break;
958 case 0x60:
959 sprintf(buf+len, "SAS Serial SCSI Protocol\n");
960 break;
961 case 0x70:
962 sprintf(buf+len, "Automation/Drive Interface Transport"
963 " Protocol\n");
964 break;
965 case 0x80:
966 sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
967 break;
968 default:
969 sprintf(buf+len, "Unknown 0x%02x\n",
970 vpd->protocol_identifier);
971 break;
972 }
973
974 if (p_buf)
975 strncpy(p_buf, buf, p_buf_len);
976 else
6708bb27 977 pr_debug("%s", buf);
c66ac9db
NB		 978}
979
980void
981transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
982{
983 /*
984 * Check if the Protocol Identifier Valid (PIV) bit is set..
985 *
986 * from spc3r23.pdf section 7.5.1
987 */
988 if (page_83[1] & 0x80) {
989 vpd->protocol_identifier = (page_83[0] & 0xf0);
990 vpd->protocol_identifier_set = 1;
991 transport_dump_vpd_proto_id(vpd, NULL, 0);
992 }
993}
994EXPORT_SYMBOL(transport_set_vpd_proto_id);
995
996int transport_dump_vpd_assoc(
997 struct t10_vpd *vpd,
998 unsigned char *p_buf,
999 int p_buf_len)
1000{
1001 unsigned char buf[VPD_TMP_BUF_SIZE];
e3d6f909
AG		 1002 int ret = 0;
1003 int len;
c66ac9db
NB		 1004
1005 memset(buf, 0, VPD_TMP_BUF_SIZE);
1006 len = sprintf(buf, "T10 VPD Identifier Association: ");
1007
1008 switch (vpd->association) {
1009 case 0x00:
1010 sprintf(buf+len, "addressed logical unit\n");
1011 break;
1012 case 0x10:
1013 sprintf(buf+len, "target port\n");
1014 break;
1015 case 0x20:
1016 sprintf(buf+len, "SCSI target device\n");
1017 break;
1018 default:
1019 sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
e3d6f909 1020 ret = -EINVAL;
c66ac9db
NB		 1021 break;
1022 }
1023
1024 if (p_buf)
1025 strncpy(p_buf, buf, p_buf_len);
1026 else
6708bb27 1027 pr_debug("%s", buf);
c66ac9db
NB		 1028
1029 return ret;
1030}
1031
1032int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
1033{
1034 /*
1035 * The VPD identification association..
1036 *
1037 * from spc3r23.pdf Section 7.6.3.1 Table 297
1038 */
1039 vpd->association = (page_83[1] & 0x30);
1040 return transport_dump_vpd_assoc(vpd, NULL, 0);
1041}
1042EXPORT_SYMBOL(transport_set_vpd_assoc);
1043
1044int transport_dump_vpd_ident_type(
1045 struct t10_vpd *vpd,
1046 unsigned char *p_buf,
1047 int p_buf_len)
1048{
1049 unsigned char buf[VPD_TMP_BUF_SIZE];
e3d6f909
AG		 1050 int ret = 0;
1051 int len;
c66ac9db
NB		 1052
1053 memset(buf, 0, VPD_TMP_BUF_SIZE);
1054 len = sprintf(buf, "T10 VPD Identifier Type: ");
1055
1056 switch (vpd->device_identifier_type) {
1057 case 0x00:
1058 sprintf(buf+len, "Vendor specific\n");
1059 break;
1060 case 0x01:
1061 sprintf(buf+len, "T10 Vendor ID based\n");
1062 break;
1063 case 0x02:
1064 sprintf(buf+len, "EUI-64 based\n");
1065 break;
1066 case 0x03:
1067 sprintf(buf+len, "NAA\n");
1068 break;
1069 case 0x04:
1070 sprintf(buf+len, "Relative target port identifier\n");
1071 break;
1072 case 0x08:
1073 sprintf(buf+len, "SCSI name string\n");
1074 break;
1075 default:
1076 sprintf(buf+len, "Unsupported: 0x%02x\n",
1077 vpd->device_identifier_type);
e3d6f909 1078 ret = -EINVAL;
c66ac9db
NB		 1079 break;
1080 }
1081
e3d6f909
AG		 1082 if (p_buf) {
1083 if (p_buf_len < strlen(buf)+1)
1084 return -EINVAL;
c66ac9db 1085 strncpy(p_buf, buf, p_buf_len);
e3d6f909 1086 } else {
6708bb27 1087 pr_debug("%s", buf);
e3d6f909 1088 }
c66ac9db
NB		 1089
1090 return ret;
1091}
1092
1093int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
1094{
1095 /*
1096 * The VPD identifier type..
1097 *
1098 * from spc3r23.pdf Section 7.6.3.1 Table 298
1099 */
1100 vpd->device_identifier_type = (page_83[1] & 0x0f);
1101 return transport_dump_vpd_ident_type(vpd, NULL, 0);
1102}
1103EXPORT_SYMBOL(transport_set_vpd_ident_type);
1104
1105int transport_dump_vpd_ident(
1106 struct t10_vpd *vpd,
1107 unsigned char *p_buf,
1108 int p_buf_len)
1109{
1110 unsigned char buf[VPD_TMP_BUF_SIZE];
1111 int ret = 0;
1112
1113 memset(buf, 0, VPD_TMP_BUF_SIZE);
1114
1115 switch (vpd->device_identifier_code_set) {
1116 case 0x01: /* Binary */
1117 sprintf(buf, "T10 VPD Binary Device Identifier: %s\n",
1118 &vpd->device_identifier[0]);
1119 break;
1120 case 0x02: /* ASCII */
1121 sprintf(buf, "T10 VPD ASCII Device Identifier: %s\n",
1122 &vpd->device_identifier[0]);
1123 break;
1124 case 0x03: /* UTF-8 */
1125 sprintf(buf, "T10 VPD UTF-8 Device Identifier: %s\n",
1126 &vpd->device_identifier[0]);
1127 break;
1128 default:
1129 sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
1130 " 0x%02x", vpd->device_identifier_code_set);
e3d6f909 1131 ret = -EINVAL;
c66ac9db
NB		 1132 break;
1133 }
1134
1135 if (p_buf)
1136 strncpy(p_buf, buf, p_buf_len);
1137 else
6708bb27 1138 pr_debug("%s", buf);
c66ac9db
NB		 1139
1140 return ret;
1141}
1142
1143int
1144transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
1145{
1146 static const char hex_str[] = "0123456789abcdef";
1147 int j = 0, i = 4; /* offset to start of the identifer */
1148
1149 /*
1150 * The VPD Code Set (encoding)
1151 *
1152 * from spc3r23.pdf Section 7.6.3.1 Table 296
1153 */
1154 vpd->device_identifier_code_set = (page_83[0] & 0x0f);
1155 switch (vpd->device_identifier_code_set) {
1156 case 0x01: /* Binary */
1157 vpd->device_identifier[j++] =
1158 hex_str[vpd->device_identifier_type];
1159 while (i < (4 + page_83[3])) {
1160 vpd->device_identifier[j++] =
1161 hex_str[(page_83[i] & 0xf0) >> 4];
1162 vpd->device_identifier[j++] =
1163 hex_str[page_83[i] & 0x0f];
1164 i++;
1165 }
1166 break;
1167 case 0x02: /* ASCII */
1168 case 0x03: /* UTF-8 */
1169 while (i < (4 + page_83[3]))
1170 vpd->device_identifier[j++] = page_83[i++];
1171 break;
1172 default:
1173 break;
1174 }
1175
1176 return transport_dump_vpd_ident(vpd, NULL, 0);
1177}
1178EXPORT_SYMBOL(transport_set_vpd_ident);
1179
1180static void core_setup_task_attr_emulation(struct se_device *dev)
1181{
1182 /*
1183 * If this device is from Target_Core_Mod/pSCSI, disable the
1184 * SAM Task Attribute emulation.
1185 *
1186 * This is currently not available in upsream Linux/SCSI Target
1187 * mode code, and is assumed to be disabled while using TCM/pSCSI.
1188 */
e3d6f909 1189 if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV) {
c66ac9db
NB		 1190 dev->dev_task_attr_type = SAM_TASK_ATTR_PASSTHROUGH;
1191 return;
1192 }
1193
1194 dev->dev_task_attr_type = SAM_TASK_ATTR_EMULATED;
6708bb27 1195 pr_debug("%s: Using SAM_TASK_ATTR_EMULATED for SPC: 0x%02x"
e3d6f909
AG		 1196 " device\n", dev->transport->name,
1197 dev->transport->get_device_rev(dev));
c66ac9db
NB		 1198}
1199
1200static void scsi_dump_inquiry(struct se_device *dev)
1201{
e3d6f909 1202 struct t10_wwn *wwn = &dev->se_sub_dev->t10_wwn;
e59a41b6 1203 char buf[17];
c66ac9db
NB		 1204 int i, device_type;
1205 /*
1206 * Print Linux/SCSI style INQUIRY formatting to the kernel ring buffer
1207 */
c66ac9db
NB		 1208 for (i = 0; i < 8; i++)
1209 if (wwn->vendor[i] >= 0x20)
e59a41b6 1210 buf[i] = wwn->vendor[i];
c66ac9db 1211 else
e59a41b6
SAS		 1212 buf[i] = ' ';
1213 buf[i] = '\0';
1214 pr_debug(" Vendor: %s\n", buf);
c66ac9db 1215
c66ac9db
NB		 1216 for (i = 0; i < 16; i++)
1217 if (wwn->model[i] >= 0x20)
e59a41b6 1218 buf[i] = wwn->model[i];
c66ac9db 1219 else
e59a41b6
SAS		 1220 buf[i] = ' ';
1221 buf[i] = '\0';
1222 pr_debug(" Model: %s\n", buf);
c66ac9db 1223
c66ac9db
NB		 1224 for (i = 0; i < 4; i++)
1225 if (wwn->revision[i] >= 0x20)
e59a41b6 1226 buf[i] = wwn->revision[i];
c66ac9db 1227 else
e59a41b6
SAS		 1228 buf[i] = ' ';
1229 buf[i] = '\0';
1230 pr_debug(" Revision: %s\n", buf);
c66ac9db 1231
e3d6f909 1232 device_type = dev->transport->get_device_type(dev);
6708bb27
AG		 1233 pr_debug(" Type: %s ", scsi_device_type(device_type));
1234 pr_debug(" ANSI SCSI revision: %02x\n",
e3d6f909 1235 dev->transport->get_device_rev(dev));
c66ac9db
NB		 1236}
1237
1238struct se_device *transport_add_device_to_core_hba(
1239 struct se_hba *hba,
1240 struct se_subsystem_api *transport,
1241 struct se_subsystem_dev *se_dev,
1242 u32 device_flags,
1243 void *transport_dev,
1244 struct se_dev_limits *dev_limits,
1245 const char *inquiry_prod,
1246 const char *inquiry_rev)
1247{
12a18bdc 1248 int force_pt;
c66ac9db
NB		 1249 struct se_device *dev;
1250
1251 dev = kzalloc(sizeof(struct se_device), GFP_KERNEL);
6708bb27
AG		 1252 if (!dev) {
1253 pr_err("Unable to allocate memory for se_dev_t\n");
c66ac9db
NB		 1254 return NULL;
1255 }
c66ac9db 1256
e3d6f909 1257 transport_init_queue_obj(&dev->dev_queue_obj);
c66ac9db
NB		 1258 dev->dev_flags = device_flags;
1259 dev->dev_status |= TRANSPORT_DEVICE_DEACTIVATED;
5951146d 1260 dev->dev_ptr = transport_dev;
c66ac9db
NB		 1261 dev->se_hba = hba;
1262 dev->se_sub_dev = se_dev;
1263 dev->transport = transport;
c66ac9db
NB		 1264 INIT_LIST_HEAD(&dev->dev_list);
1265 INIT_LIST_HEAD(&dev->dev_sep_list);
1266 INIT_LIST_HEAD(&dev->dev_tmr_list);
cf572a96 1267 INIT_LIST_HEAD(&dev->execute_list);
c66ac9db 1268 INIT_LIST_HEAD(&dev->delayed_cmd_list);
cf572a96 1269 INIT_LIST_HEAD(&dev->state_list);
07bde79a 1270 INIT_LIST_HEAD(&dev->qf_cmd_list);
c66ac9db
NB		 1271 spin_lock_init(&dev->execute_task_lock);
1272 spin_lock_init(&dev->delayed_cmd_lock);
c66ac9db
NB		 1273 spin_lock_init(&dev->dev_reservation_lock);
1274 spin_lock_init(&dev->dev_status_lock);
c66ac9db
NB		 1275 spin_lock_init(&dev->se_port_lock);
1276 spin_lock_init(&dev->se_tmr_lock);
07bde79a 1277 spin_lock_init(&dev->qf_cmd_lock);
c66ac9db
NB		 1278 atomic_set(&dev->dev_ordered_id, 0);
1279
1280 se_dev_set_default_attribs(dev, dev_limits);
1281
1282 dev->dev_index = scsi_get_new_index(SCSI_DEVICE_INDEX);
1283 dev->creation_time = get_jiffies_64();
1284 spin_lock_init(&dev->stats_lock);
1285
1286 spin_lock(&hba->device_lock);
1287 list_add_tail(&dev->dev_list, &hba->hba_dev_list);
1288 hba->dev_count++;
1289 spin_unlock(&hba->device_lock);
1290 /*
1291 * Setup the SAM Task Attribute emulation for struct se_device
1292 */
1293 core_setup_task_attr_emulation(dev);
1294 /*
1295 * Force PR and ALUA passthrough emulation with internal object use.
1296 */
1297 force_pt = (hba->hba_flags & HBA_FLAGS_INTERNAL_USE);
1298 /*
1299 * Setup the Reservations infrastructure for struct se_device
1300 */
1301 core_setup_reservations(dev, force_pt);
1302 /*
1303 * Setup the Asymmetric Logical Unit Assignment for struct se_device
1304 */
1305 if (core_setup_alua(dev, force_pt) < 0)
1306 goto out;
1307
1308 /*
1309 * Startup the struct se_device processing thread
1310 */
1311 dev->process_thread = kthread_run(transport_processing_thread, dev,
e3d6f909 1312 "LIO_%s", dev->transport->name);
c66ac9db 1313 if (IS_ERR(dev->process_thread)) {
6708bb27 1314 pr_err("Unable to create kthread: LIO_%s\n",
e3d6f909 1315 dev->transport->name);
c66ac9db
NB		 1316 goto out;
1317 }
07bde79a
NB		 1318 /*
1319 * Setup work_queue for QUEUE_FULL
1320 */
1321 INIT_WORK(&dev->qf_work_queue, target_qf_do_work);
c66ac9db
NB		 1322 /*
1323 * Preload the initial INQUIRY const values if we are doing
1324 * anything virtual (IBLOCK, FILEIO, RAMDISK), but not for TCM/pSCSI
1325 * passthrough because this is being provided by the backend LLD.
1326 * This is required so that transport_get_inquiry() copies these
1327 * originals once back into DEV_T10_WWN(dev) for the virtual device
1328 * setup.
1329 */
e3d6f909 1330 if (dev->transport->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV) {
f22c1196 1331 if (!inquiry_prod || !inquiry_rev) {
6708bb27 1332 pr_err("All non TCM/pSCSI plugins require"
c66ac9db
NB		 1333 " INQUIRY consts\n");
1334 goto out;
1335 }
1336
e3d6f909
AG		 1337 strncpy(&dev->se_sub_dev->t10_wwn.vendor[0], "LIO-ORG", 8);
1338 strncpy(&dev->se_sub_dev->t10_wwn.model[0], inquiry_prod, 16);
1339 strncpy(&dev->se_sub_dev->t10_wwn.revision[0], inquiry_rev, 4);
c66ac9db
NB		 1340 }
1341 scsi_dump_inquiry(dev);
1342
12a18bdc 1343 return dev;
c66ac9db 1344out:
c66ac9db
NB		 1345 kthread_stop(dev->process_thread);
1346
1347 spin_lock(&hba->device_lock);
1348 list_del(&dev->dev_list);
1349 hba->dev_count--;
1350 spin_unlock(&hba->device_lock);
1351
1352 se_release_vpd_for_dev(dev);
1353
c66ac9db
NB		 1354 kfree(dev);
1355
1356 return NULL;
1357}
1358EXPORT_SYMBOL(transport_add_device_to_core_hba);
1359
1360/* transport_generic_prepare_cdb():
1361 *
1362 * Since the Initiator sees iSCSI devices as LUNs, the SCSI CDB will
1363 * contain the iSCSI LUN in bits 7-5 of byte 1 as per SAM-2.
1364 * The point of this is since we are mapping iSCSI LUNs to
1365 * SCSI Target IDs having a non-zero LUN in the CDB will throw the
1366 * devices and HBAs for a loop.
1367 */
1368static inline void transport_generic_prepare_cdb(
1369 unsigned char *cdb)
1370{
1371 switch (cdb[0]) {
1372 case READ_10: /* SBC - RDProtect */
1373 case READ_12: /* SBC - RDProtect */
1374 case READ_16: /* SBC - RDProtect */
1375 case SEND_DIAGNOSTIC: /* SPC - SELF-TEST Code */
1376 case VERIFY: /* SBC - VRProtect */
1377 case VERIFY_16: /* SBC - VRProtect */
1378 case WRITE_VERIFY: /* SBC - VRProtect */
1379 case WRITE_VERIFY_12: /* SBC - VRProtect */
1380 break;
1381 default:
1382 cdb[1] &= 0x1f; /* clear logical unit number */
1383 break;
1384 }
1385}
1386
c66ac9db
NB		 1387static int transport_generic_cmd_sequencer(struct se_cmd *, unsigned char *);
1388
c66ac9db
NB		 1389/*
1390 * Used by fabric modules containing a local struct se_cmd within their
1391 * fabric dependent per I/O descriptor.
1392 */
1393void transport_init_se_cmd(
1394 struct se_cmd *cmd,
1395 struct target_core_fabric_ops *tfo,
1396 struct se_session *se_sess,
1397 u32 data_length,
1398 int data_direction,
1399 int task_attr,
1400 unsigned char *sense_buffer)
1401{
5951146d
AG		 1402 INIT_LIST_HEAD(&cmd->se_lun_node);
1403 INIT_LIST_HEAD(&cmd->se_delayed_node);
07bde79a 1404 INIT_LIST_HEAD(&cmd->se_qf_node);
79a7fef2 1405 INIT_LIST_HEAD(&cmd->se_queue_node);
a17f091d 1406 INIT_LIST_HEAD(&cmd->se_cmd_list);
cf572a96
CH		 1407 INIT_LIST_HEAD(&cmd->execute_list);
1408 INIT_LIST_HEAD(&cmd->state_list);
a1d8b49a
AG		 1409 init_completion(&cmd->transport_lun_fe_stop_comp);
1410 init_completion(&cmd->transport_lun_stop_comp);
1411 init_completion(&cmd->t_transport_stop_comp);
a17f091d 1412 init_completion(&cmd->cmd_wait_comp);
cf572a96 1413 init_completion(&cmd->task_stop_comp);
a1d8b49a 1414 spin_lock_init(&cmd->t_state_lock);
7d680f3b 1415 cmd->transport_state = CMD_T_DEV_ACTIVE;
c66ac9db
NB		 1416
1417 cmd->se_tfo = tfo;
1418 cmd->se_sess = se_sess;
1419 cmd->data_length = data_length;
1420 cmd->data_direction = data_direction;
1421 cmd->sam_task_attr = task_attr;
1422 cmd->sense_buffer = sense_buffer;
cf572a96
CH		 1423
1424 cmd->state_active = false;
c66ac9db
NB		 1425}
1426EXPORT_SYMBOL(transport_init_se_cmd);
1427
1428static int transport_check_alloc_task_attr(struct se_cmd *cmd)
1429{
1430 /*
1431 * Check if SAM Task Attribute emulation is enabled for this
1432 * struct se_device storage object
1433 */
5951146d 1434 if (cmd->se_dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED)
c66ac9db
NB		 1435 return 0;
1436
e66ecd50 1437 if (cmd->sam_task_attr == MSG_ACA_TAG) {
6708bb27 1438 pr_debug("SAM Task Attribute ACA"
c66ac9db 1439 " emulation is not supported\n");
e3d6f909 1440 return -EINVAL;
c66ac9db
NB		 1441 }
1442 /*
1443 * Used to determine when ORDERED commands should go from
1444 * Dormant to Active status.
1445 */
5951146d 1446 cmd->se_ordered_id = atomic_inc_return(&cmd->se_dev->dev_ordered_id);
c66ac9db 1447 smp_mb__after_atomic_inc();
6708bb27 1448 pr_debug("Allocated se_ordered_id: %u for Task Attr: 0x%02x on %s\n",
c66ac9db 1449 cmd->se_ordered_id, cmd->sam_task_attr,
6708bb27 1450 cmd->se_dev->transport->name);
c66ac9db
NB		 1451 return 0;
1452}
1453
a12f41f8 1454/* target_setup_cmd_from_cdb():
c66ac9db
NB		 1455 *
1456 * Called from fabric RX Thread.
1457 */
a12f41f8 1458int target_setup_cmd_from_cdb(
c66ac9db
NB		 1459 struct se_cmd *cmd,
1460 unsigned char *cdb)
1461{
1462 int ret;
1463
1464 transport_generic_prepare_cdb(cdb);
c66ac9db
NB		 1465 /*
1466 * Ensure that the received CDB is less than the max (252 + 8) bytes
1467 * for VARIABLE_LENGTH_CMD
1468 */
1469 if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
6708bb27 1470 pr_err("Received SCSI CDB with command_size: %d that"
c66ac9db
NB		 1471 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1472 scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
03e98c9e
NB		 1473 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1474 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
e3d6f909 1475 return -EINVAL;
c66ac9db
NB		 1476 }
1477 /*
1478 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
1479 * allocate the additional extended CDB buffer now.. Otherwise
1480 * setup the pointer from __t_task_cdb to t_task_cdb.
1481 */
a1d8b49a
AG		 1482 if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) {
1483 cmd->t_task_cdb = kzalloc(scsi_command_size(cdb),
c66ac9db 1484 GFP_KERNEL);
6708bb27
AG		 1485 if (!cmd->t_task_cdb) {
1486 pr_err("Unable to allocate cmd->t_task_cdb"
a1d8b49a 1487 " %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
c66ac9db 1488 scsi_command_size(cdb),
a1d8b49a 1489 (unsigned long)sizeof(cmd->__t_task_cdb));
03e98c9e
NB		 1490 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1491 cmd->scsi_sense_reason =
1492 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
e3d6f909 1493 return -ENOMEM;
c66ac9db
NB		 1494 }
1495 } else
a1d8b49a 1496 cmd->t_task_cdb = &cmd->__t_task_cdb[0];
c66ac9db 1497 /*
a1d8b49a 1498 * Copy the original CDB into cmd->
c66ac9db 1499 */
a1d8b49a 1500 memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
c66ac9db
NB		 1501 /*
1502 * Setup the received CDB based on SCSI defined opcodes and
1503 * perform unit attention, persistent reservations and ALUA
a1d8b49a 1504 * checks for virtual device backends. The cmd->t_task_cdb
c66ac9db
NB		 1505 * pointer is expected to be setup before we reach this point.
1506 */
1507 ret = transport_generic_cmd_sequencer(cmd, cdb);
1508 if (ret < 0)
1509 return ret;
1510 /*
1511 * Check for SAM Task Attribute Emulation
1512 */
1513 if (transport_check_alloc_task_attr(cmd) < 0) {
1514 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1515 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
5951146d 1516 return -EINVAL;
c66ac9db
NB		 1517 }
1518 spin_lock(&cmd->se_lun->lun_sep_lock);
1519 if (cmd->se_lun->lun_sep)
1520 cmd->se_lun->lun_sep->sep_stats.cmd_pdus++;
1521 spin_unlock(&cmd->se_lun->lun_sep_lock);
1522 return 0;
1523}
a12f41f8 1524EXPORT_SYMBOL(target_setup_cmd_from_cdb);
c66ac9db 1525
695434e1
NB		 1526/*
1527 * Used by fabric module frontends to queue tasks directly.
1528 * Many only be used from process context only
1529 */
1530int transport_handle_cdb_direct(
1531 struct se_cmd *cmd)
1532{
dd8ae59d
NB		 1533 int ret;
1534
695434e1
NB		 1535 if (!cmd->se_lun) {
1536 dump_stack();
6708bb27 1537 pr_err("cmd->se_lun is NULL\n");
695434e1
NB		 1538 return -EINVAL;
1539 }
1540 if (in_interrupt()) {
1541 dump_stack();
6708bb27 1542 pr_err("transport_generic_handle_cdb cannot be called"
695434e1
NB		 1543 " from interrupt context\n");
1544 return -EINVAL;
1545 }
dd8ae59d 1546 /*
7d680f3b 1547 * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE following
dd8ae59d
NB		 1548 * transport_generic_handle_cdb*() -> transport_add_cmd_to_queue()
1549 * in existing usage to ensure that outstanding descriptors are handled
d14921d6 1550 * correctly during shutdown via transport_wait_for_tasks()
dd8ae59d
NB		 1551 *
1552 * Also, we don't take cmd->t_state_lock here as we only expect
1553 * this to be called for initial descriptor submission.
1554 */
1555 cmd->t_state = TRANSPORT_NEW_CMD;
7d680f3b
CH		 1556 cmd->transport_state |= CMD_T_ACTIVE;
1557
dd8ae59d
NB		 1558 /*
1559 * transport_generic_new_cmd() is already handling QUEUE_FULL,
1560 * so follow TRANSPORT_NEW_CMD processing thread context usage
1561 * and call transport_generic_request_failure() if necessary..
1562 */
1563 ret = transport_generic_new_cmd(cmd);
03e98c9e
NB		 1564 if (ret < 0)
1565 transport_generic_request_failure(cmd);
1566
dd8ae59d 1567 return 0;
695434e1
NB		 1568}
1569EXPORT_SYMBOL(transport_handle_cdb_direct);
1570
a6360785
NB		 1571/**
1572 * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd
1573 *
1574 * @se_cmd: command descriptor to submit
1575 * @se_sess: associated se_sess for endpoint
1576 * @cdb: pointer to SCSI CDB
1577 * @sense: pointer to SCSI sense buffer
1578 * @unpacked_lun: unpacked LUN to reference for struct se_lun
1579 * @data_length: fabric expected data transfer length
1580 * @task_addr: SAM task attribute
1581 * @data_dir: DMA data direction
1582 * @flags: flags for command submission from target_sc_flags_tables
1583 *
1584 * This may only be called from process context, and also currently
1585 * assumes internal allocation of fabric payload buffer by target-core.
1586 **/
1edcdb49 1587void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess,
a6360785
NB		 1588 unsigned char *cdb, unsigned char *sense, u32 unpacked_lun,
1589 u32 data_length, int task_attr, int data_dir, int flags)
1590{
1591 struct se_portal_group *se_tpg;
1592 int rc;
1593
1594 se_tpg = se_sess->se_tpg;
1595 BUG_ON(!se_tpg);
1596 BUG_ON(se_cmd->se_tfo || se_cmd->se_sess);
1597 BUG_ON(in_interrupt());
1598 /*
1599 * Initialize se_cmd for target operation. From this point
1600 * exceptions are handled by sending exception status via
1601 * target_core_fabric_ops->queue_status() callback
1602 */
1603 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
1604 data_length, data_dir, task_attr, sense);
b0d79946
SAS		 1605 if (flags & TARGET_SCF_UNKNOWN_SIZE)
1606 se_cmd->unknown_data_length = 1;
a6360785
NB		 1607 /*
1608 * Obtain struct se_cmd->cmd_kref reference and add new cmd to
1609 * se_sess->sess_cmd_list. A second kref_get here is necessary
1610 * for fabrics using TARGET_SCF_ACK_KREF that expect a second
1611 * kref_put() to happen during fabric packet acknowledgement.
1612 */
1613 target_get_sess_cmd(se_sess, se_cmd, (flags & TARGET_SCF_ACK_KREF));
1614 /*
1615 * Signal bidirectional data payloads to target-core
1616 */
1617 if (flags & TARGET_SCF_BIDI_OP)
1618 se_cmd->se_cmd_flags |= SCF_BIDI;
1619 /*
1620 * Locate se_lun pointer and attach it to struct se_cmd
1621 */
735703ca
NB		 1622 if (transport_lookup_cmd_lun(se_cmd, unpacked_lun) < 0) {
1623 transport_send_check_condition_and_sense(se_cmd,
1624 se_cmd->scsi_sense_reason, 0);
1625 target_put_sess_cmd(se_sess, se_cmd);
1626 return;
1627 }
a6360785
NB		 1628 /*
1629 * Sanitize CDBs via transport_generic_cmd_sequencer() and
1630 * allocate the necessary tasks to complete the received CDB+data
1631 */
a12f41f8 1632 rc = target_setup_cmd_from_cdb(se_cmd, cdb);
735703ca
NB		 1633 if (rc != 0) {
1634 transport_generic_request_failure(se_cmd);
1635 return;
1636 }
11e319ed
AG		 1637
1638 /*
1639 * Check if we need to delay processing because of ALUA
1640 * Active/NonOptimized primary access state..
1641 */
1642 core_alua_check_nonop_delay(se_cmd);
1643
a6360785
NB		 1644 /*
1645 * Dispatch se_cmd descriptor to se_lun->lun_se_dev backend
1646 * for immediate execution of READs, otherwise wait for
1647 * transport_generic_handle_data() to be called for WRITEs
1648 * when fabric has filled the incoming buffer.
1649 */
1650 transport_handle_cdb_direct(se_cmd);
1edcdb49 1651 return;
a6360785
NB		 1652}
1653EXPORT_SYMBOL(target_submit_cmd);
1654
9f0d05c2
NB		 1655static void target_complete_tmr_failure(struct work_struct *work)
1656{
1657 struct se_cmd *se_cmd = container_of(work, struct se_cmd, work);
1658
1659 se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST;
1660 se_cmd->se_tfo->queue_tm_rsp(se_cmd);
1661 transport_generic_free_cmd(se_cmd, 0);
1662}
1663
ea98d7f9
AG		 1664/**
1665 * target_submit_tmr - lookup unpacked lun and submit uninitialized se_cmd
1666 * for TMR CDBs
1667 *
1668 * @se_cmd: command descriptor to submit
1669 * @se_sess: associated se_sess for endpoint
1670 * @sense: pointer to SCSI sense buffer
1671 * @unpacked_lun: unpacked LUN to reference for struct se_lun
1672 * @fabric_context: fabric context for TMR req
1673 * @tm_type: Type of TM request
c0974f89
NB		 1674 * @gfp: gfp type for caller
1675 * @tag: referenced task tag for TMR_ABORT_TASK
c7042cae 1676 * @flags: submit cmd flags
ea98d7f9
AG		 1677 *
1678 * Callable from all contexts.
1679 **/
1680
c7042cae 1681int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
ea98d7f9 1682 unsigned char *sense, u32 unpacked_lun,
c0974f89
NB		 1683 void *fabric_tmr_ptr, unsigned char tm_type,
1684 gfp_t gfp, unsigned int tag, int flags)
ea98d7f9
AG		 1685{
1686 struct se_portal_group *se_tpg;
1687 int ret;
1688
1689 se_tpg = se_sess->se_tpg;
1690 BUG_ON(!se_tpg);
1691
1692 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
1693 0, DMA_NONE, MSG_SIMPLE_TAG, sense);
c7042cae
NB		 1694 /*
1695 * FIXME: Currently expect caller to handle se_cmd->se_tmr_req
1696 * allocation failure.
1697 */
c0974f89 1698 ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp);
c7042cae
NB		 1699 if (ret < 0)
1700 return -ENOMEM;
ea98d7f9 1701
c0974f89
NB		 1702 if (tm_type == TMR_ABORT_TASK)
1703 se_cmd->se_tmr_req->ref_task_tag = tag;
1704
ea98d7f9
AG		 1705 /* See target_submit_cmd for commentary */
1706 target_get_sess_cmd(se_sess, se_cmd, (flags & TARGET_SCF_ACK_KREF));
1707
ea98d7f9
AG		 1708 ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun);
1709 if (ret) {
9f0d05c2
NB		 1710 /*
1711 * For callback during failure handling, push this work off
1712 * to process context with TMR_LUN_DOES_NOT_EXIST status.
1713 */
1714 INIT_WORK(&se_cmd->work, target_complete_tmr_failure);
1715 schedule_work(&se_cmd->work);
c7042cae 1716 return 0;
ea98d7f9
AG		 1717 }
1718 transport_generic_handle_tmr(se_cmd);
c7042cae 1719 return 0;
ea98d7f9
AG		 1720}
1721EXPORT_SYMBOL(target_submit_tmr);
1722
c66ac9db
NB		 1723/*
1724 * Used by fabric module frontends defining a TFO->new_cmd_map() caller
1725 * to queue up a newly setup se_cmd w/ TRANSPORT_NEW_CMD_MAP in order to
1726 * complete setup in TCM process context w/ TFO->new_cmd_map().
1727 */
1728int transport_generic_handle_cdb_map(
1729 struct se_cmd *cmd)
1730{
e3d6f909 1731 if (!cmd->se_lun) {
c66ac9db 1732 dump_stack();
6708bb27 1733 pr_err("cmd->se_lun is NULL\n");
e3d6f909 1734 return -EINVAL;
c66ac9db
NB		 1735 }
1736
f7a5cc0b 1737 transport_add_cmd_to_queue(cmd, TRANSPORT_NEW_CMD_MAP, false);
c66ac9db
NB		 1738 return 0;
1739}
1740EXPORT_SYMBOL(transport_generic_handle_cdb_map);
1741
1742/* transport_generic_handle_data():
1743 *
1744 *
1745 */
1746int transport_generic_handle_data(
1747 struct se_cmd *cmd)
1748{
1749 /*
1750 * For the software fabric case, then we assume the nexus is being
1751 * failed/shutdown when signals are pending from the kthread context
1752 * caller, so we return a failure. For the HW target mode case running
1753 * in interrupt code, the signal_pending() check is skipped.
1754 */
1755 if (!in_interrupt() && signal_pending(current))
e3d6f909 1756 return -EPERM;
c66ac9db
NB		 1757 /*
1758 * If the received CDB has aleady been ABORTED by the generic
1759 * target engine, we now call transport_check_aborted_status()
1760 * to queue any delated TASK_ABORTED status for the received CDB to the
25985edc 1761 * fabric module as we are expecting no further incoming DATA OUT
c66ac9db
NB		 1762 * sequences at this point.
1763 */
1764 if (transport_check_aborted_status(cmd, 1) != 0)
1765 return 0;
1766
f7a5cc0b 1767 transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_WRITE, false);
c66ac9db
NB		 1768 return 0;
1769}
1770EXPORT_SYMBOL(transport_generic_handle_data);
1771
1772/* transport_generic_handle_tmr():
1773 *
1774 *
1775 */
1776int transport_generic_handle_tmr(
1777 struct se_cmd *cmd)
1778{
f7a5cc0b 1779 transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_TMR, false);
c66ac9db
NB		 1780 return 0;
1781}
1782EXPORT_SYMBOL(transport_generic_handle_tmr);
1783
cdbb70bb 1784/*
cf572a96 1785 * If the cmd is active, request it to be stopped and sleep until it
cdbb70bb
CH		 1786 * has completed.
1787 */
cf572a96 1788bool target_stop_cmd(struct se_cmd *cmd, unsigned long *flags)
cdbb70bb 1789{
cdbb70bb
CH		 1790 bool was_active = false;
1791
cf572a96
CH		 1792 if (cmd->transport_state & CMD_T_BUSY) {
1793 cmd->transport_state |= CMD_T_REQUEST_STOP;
cdbb70bb
CH		 1794 spin_unlock_irqrestore(&cmd->t_state_lock, *flags);
1795
cf572a96
CH		 1796 pr_debug("cmd %p waiting to complete\n", cmd);
1797 wait_for_completion(&cmd->task_stop_comp);
1798 pr_debug("cmd %p stopped successfully\n", cmd);
cdbb70bb
CH		 1799
1800 spin_lock_irqsave(&cmd->t_state_lock, *flags);
1801 atomic_dec(&cmd->t_task_cdbs_left);
cf572a96
CH		 1802 cmd->transport_state &= ~CMD_T_REQUEST_STOP;
1803 cmd->transport_state &= ~CMD_T_BUSY;
cdbb70bb
CH		 1804 was_active = true;
1805 }
1806
cdbb70bb
CH		 1807 return was_active;
1808}
1809
c66ac9db
NB		 1810/*
1811 * Handle SAM-esque emulation for generic transport request failures.
1812 */
2fbff127 1813void transport_generic_request_failure(struct se_cmd *cmd)
c66ac9db 1814{
07bde79a
NB		 1815 int ret = 0;
1816
6708bb27 1817 pr_debug("-----[ Storage Engine Exception for cmd: %p ITT: 0x%08x"
e3d6f909 1818 " CDB: 0x%02x\n", cmd, cmd->se_tfo->get_task_tag(cmd),
a1d8b49a 1819 cmd->t_task_cdb[0]);
03e98c9e 1820 pr_debug("-----[ i_state: %d t_state: %d scsi_sense_reason: %d\n",
e3d6f909 1821 cmd->se_tfo->get_cmd_state(cmd),
03e98c9e 1822 cmd->t_state, cmd->scsi_sense_reason);
785fdf70
CH		 1823 pr_debug("-----[ t_task_cdbs_left: %d"
1824 " t_task_cdbs_ex_left: %d --"
7d680f3b 1825 " CMD_T_ACTIVE: %d CMD_T_STOP: %d CMD_T_SENT: %d\n",
a1d8b49a 1826 atomic_read(&cmd->t_task_cdbs_left),
a1d8b49a 1827 atomic_read(&cmd->t_task_cdbs_ex_left),
7d680f3b
CH		 1828 (cmd->transport_state & CMD_T_ACTIVE) != 0,
1829 (cmd->transport_state & CMD_T_STOP) != 0,
1830 (cmd->transport_state & CMD_T_SENT) != 0);
c66ac9db 1831
c66ac9db
NB		 1832 /*
1833 * For SAM Task Attribute emulation for failed struct se_cmd
1834 */
1835 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
1836 transport_complete_task_attr(cmd);
1837
03e98c9e
NB		 1838 switch (cmd->scsi_sense_reason) {
1839 case TCM_NON_EXISTENT_LUN:
1840 case TCM_UNSUPPORTED_SCSI_OPCODE:
1841 case TCM_INVALID_CDB_FIELD:
1842 case TCM_INVALID_PARAMETER_LIST:
1843 case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
1844 case TCM_UNKNOWN_MODE_PAGE:
1845 case TCM_WRITE_PROTECTED:
1846 case TCM_CHECK_CONDITION_ABORT_CMD:
1847 case TCM_CHECK_CONDITION_UNIT_ATTENTION:
1848 case TCM_CHECK_CONDITION_NOT_READY:
c66ac9db 1849 break;
03e98c9e 1850 case TCM_RESERVATION_CONFLICT:
c66ac9db
NB		 1851 /*
1852 * No SENSE Data payload for this case, set SCSI Status
1853 * and queue the response to $FABRIC_MOD.
1854 *
1855 * Uses linux/include/scsi/scsi.h SAM status codes defs
1856 */
1857 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
1858 /*
1859 * For UA Interlock Code 11b, a RESERVATION CONFLICT will
1860 * establish a UNIT ATTENTION with PREVIOUS RESERVATION
1861 * CONFLICT STATUS.
1862 *
1863 * See spc4r17, section 7.4.6 Control Mode Page, Table 349
1864 */
e3d6f909
AG		 1865 if (cmd->se_sess &&
1866 cmd->se_dev->se_sub_dev->se_dev_attrib.emulate_ua_intlck_ctrl == 2)
1867 core_scsi3_ua_allocate(cmd->se_sess->se_node_acl,
c66ac9db
NB		 1868 cmd->orig_fe_lun, 0x2C,
1869 ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
1870
07bde79a 1871 ret = cmd->se_tfo->queue_status(cmd);
f147abb4 1872 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a 1873 goto queue_full;
c66ac9db 1874 goto check_stop;
c66ac9db 1875 default:
6708bb27 1876 pr_err("Unknown transport error for CDB 0x%02x: %d\n",
03e98c9e 1877 cmd->t_task_cdb[0], cmd->scsi_sense_reason);
c66ac9db
NB		 1878 cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
1879 break;
1880 }
16ab8e60
NB		 1881 /*
1882 * If a fabric does not define a cmd->se_tfo->new_cmd_map caller,
1883 * make the call to transport_send_check_condition_and_sense()
1884 * directly. Otherwise expect the fabric to make the call to
1885 * transport_send_check_condition_and_sense() after handling
1886 * possible unsoliticied write data payloads.
1887 */
03e98c9e
NB		 1888 ret = transport_send_check_condition_and_sense(cmd,
1889 cmd->scsi_sense_reason, 0);
1890 if (ret == -EAGAIN || ret == -ENOMEM)
1891 goto queue_full;
07bde79a 1892
c66ac9db
NB		 1893check_stop:
1894 transport_lun_remove_cmd(cmd);
6708bb27 1895 if (!transport_cmd_check_stop_to_fabric(cmd))
c66ac9db 1896 ;
07bde79a
NB		 1897 return;
1898
1899queue_full:
e057f533
CH		 1900 cmd->t_state = TRANSPORT_COMPLETE_QF_OK;
1901 transport_handle_queue_full(cmd, cmd->se_dev);
c66ac9db 1902}
2fbff127 1903EXPORT_SYMBOL(transport_generic_request_failure);
c66ac9db 1904
c66ac9db
NB		 1905static inline u32 transport_lba_21(unsigned char *cdb)
1906{
1907 return ((cdb[1] & 0x1f) << 16) | (cdb[2] << 8) | cdb[3];
1908}
1909
1910static inline u32 transport_lba_32(unsigned char *cdb)
1911{
1912 return (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5];
1913}
1914
1915static inline unsigned long long transport_lba_64(unsigned char *cdb)
1916{
1917 unsigned int __v1, __v2;
1918
1919 __v1 = (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5];
1920 __v2 = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
1921
1922 return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32;
1923}
1924
1925/*
1926 * For VARIABLE_LENGTH_CDB w/ 32 byte extended CDBs
1927 */
1928static inline unsigned long long transport_lba_64_ext(unsigned char *cdb)
1929{
1930 unsigned int __v1, __v2;
1931
1932 __v1 = (cdb[12] << 24) | (cdb[13] << 16) | (cdb[14] << 8) | cdb[15];
1933 __v2 = (cdb[16] << 24) | (cdb[17] << 16) | (cdb[18] << 8) | cdb[19];
1934
1935 return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32;
1936}
1937
1938static void transport_set_supported_SAM_opcode(struct se_cmd *se_cmd)
1939{
1940 unsigned long flags;
1941
a1d8b49a 1942 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
c66ac9db 1943 se_cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;
a1d8b49a 1944 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
c66ac9db
NB		 1945}
1946
c66ac9db
NB		 1947/*
1948 * Called from Fabric Module context from transport_execute_tasks()
1949 *
1950 * The return of this function determins if the tasks from struct se_cmd
1951 * get added to the execution queue in transport_execute_tasks(),
1952 * or are added to the delayed or ordered lists here.
1953 */
1954static inline int transport_execute_task_attr(struct se_cmd *cmd)
1955{
5951146d 1956 if (cmd->se_dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED)
c66ac9db
NB		 1957 return 1;
1958 /*
25985edc 1959 * Check for the existence of HEAD_OF_QUEUE, and if true return 1
c66ac9db
NB		 1960 * to allow the passed struct se_cmd list of tasks to the front of the list.
1961 */
e66ecd50 1962 if (cmd->sam_task_attr == MSG_HEAD_TAG) {
6708bb27 1963 pr_debug("Added HEAD_OF_QUEUE for CDB:"
c66ac9db 1964 " 0x%02x, se_ordered_id: %u\n",
6708bb27 1965 cmd->t_task_cdb[0],
c66ac9db
NB		 1966 cmd->se_ordered_id);
1967 return 1;
e66ecd50 1968 } else if (cmd->sam_task_attr == MSG_ORDERED_TAG) {
5951146d 1969 atomic_inc(&cmd->se_dev->dev_ordered_sync);
c66ac9db
NB		 1970 smp_mb__after_atomic_inc();
1971
6708bb27 1972 pr_debug("Added ORDERED for CDB: 0x%02x to ordered"
c66ac9db 1973 " list, se_ordered_id: %u\n",
a1d8b49a 1974 cmd->t_task_cdb[0],
c66ac9db
NB		 1975 cmd->se_ordered_id);
1976 /*
1977 * Add ORDERED command to tail of execution queue if
1978 * no other older commands exist that need to be
1979 * completed first.
1980 */
6708bb27 1981 if (!atomic_read(&cmd->se_dev->simple_cmds))
c66ac9db
NB		 1982 return 1;
1983 } else {
1984 /*
1985 * For SIMPLE and UNTAGGED Task Attribute commands
1986 */
5951146d 1987 atomic_inc(&cmd->se_dev->simple_cmds);
c66ac9db
NB		 1988 smp_mb__after_atomic_inc();
1989 }
1990 /*
1991 * Otherwise if one or more outstanding ORDERED task attribute exist,
1992 * add the dormant task(s) built for the passed struct se_cmd to the
1993 * execution queue and become in Active state for this struct se_device.
1994 */
5951146d 1995 if (atomic_read(&cmd->se_dev->dev_ordered_sync) != 0) {
c66ac9db
NB		 1996 /*
1997 * Otherwise, add cmd w/ tasks to delayed cmd queue that
25985edc 1998 * will be drained upon completion of HEAD_OF_QUEUE task.
c66ac9db 1999 */
5951146d 2000 spin_lock(&cmd->se_dev->delayed_cmd_lock);
c66ac9db 2001 cmd->se_cmd_flags |= SCF_DELAYED_CMD_FROM_SAM_ATTR;
5951146d
AG		 2002 list_add_tail(&cmd->se_delayed_node,
2003 &cmd->se_dev->delayed_cmd_list);
2004 spin_unlock(&cmd->se_dev->delayed_cmd_lock);
c66ac9db 2005
6708bb27 2006 pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to"
c66ac9db 2007 " delayed CMD list, se_ordered_id: %u\n",
a1d8b49a 2008 cmd->t_task_cdb[0], cmd->sam_task_attr,
c66ac9db
NB		 2009 cmd->se_ordered_id);
2010 /*
2011 * Return zero to let transport_execute_tasks() know
2012 * not to add the delayed tasks to the execution list.
2013 */
2014 return 0;
2015 }
2016 /*
2017 * Otherwise, no ORDERED task attributes exist..
2018 */
2019 return 1;
2020}
2021
2022/*
2023 * Called from fabric module context in transport_generic_new_cmd() and
2024 * transport_generic_process_write()
2025 */
2026static int transport_execute_tasks(struct se_cmd *cmd)
2027{
2028 int add_tasks;
40be67f4 2029 struct se_device *se_dev = cmd->se_dev;
c66ac9db
NB		 2030 /*
2031 * Call transport_cmd_check_stop() to see if a fabric exception
25985edc 2032 * has occurred that prevents execution.
c66ac9db 2033 */
6708bb27 2034 if (!transport_cmd_check_stop(cmd, 0, TRANSPORT_PROCESSING)) {
c66ac9db
NB		 2035 /*
2036 * Check for SAM Task Attribute emulation and HEAD_OF_QUEUE
2037 * attribute for the tasks of the received struct se_cmd CDB
2038 */
2039 add_tasks = transport_execute_task_attr(cmd);
e3d6f909 2040 if (!add_tasks)
c66ac9db 2041 goto execute_tasks;
cf572a96 2042
4d2300cc
NB		 2043 __transport_execute_tasks(se_dev, cmd);
2044 return 0;
c66ac9db 2045 }
4d2300cc 2046
c66ac9db 2047execute_tasks:
4d2300cc 2048 __transport_execute_tasks(se_dev, NULL);
c66ac9db
NB		 2049 return 0;
2050}
2051
4d2300cc 2052static int __transport_execute_tasks(struct se_device *dev, struct se_cmd *new_cmd)
c66ac9db
NB		 2053{
2054 int error;
2055 struct se_cmd *cmd = NULL;
c66ac9db
NB		 2056 unsigned long flags;
2057
c66ac9db 2058check_depth:
e3d6f909 2059 spin_lock_irq(&dev->execute_task_lock);
4d2300cc 2060 if (new_cmd != NULL)
cf572a96 2061 __target_add_to_execute_list(new_cmd);
4d2300cc 2062
cf572a96 2063 if (list_empty(&dev->execute_list)) {
e3d6f909 2064 spin_unlock_irq(&dev->execute_task_lock);
c66ac9db
NB		 2065 return 0;
2066 }
cf572a96
CH		 2067 cmd = list_first_entry(&dev->execute_list, struct se_cmd, execute_list);
2068 __target_remove_from_execute_list(cmd);
e3d6f909 2069 spin_unlock_irq(&dev->execute_task_lock);
c66ac9db 2070
a1d8b49a 2071 spin_lock_irqsave(&cmd->t_state_lock, flags);
cf572a96 2072 cmd->transport_state |= CMD_T_BUSY;
785fdf70 2073 cmd->transport_state |= CMD_T_SENT;
c66ac9db 2074
a1d8b49a 2075 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 2076
6bb35e00
CH		 2077 if (cmd->execute_cmd)
2078 error = cmd->execute_cmd(cmd);
5bda90c8 2079 else
cf572a96
CH		 2080 error = dev->transport->do_task(cmd->t_task);
2081
d29a5b6a 2082 if (error != 0) {
d29a5b6a 2083 spin_lock_irqsave(&cmd->t_state_lock, flags);
cf572a96 2084 cmd->transport_state &= ~CMD_T_BUSY;
7d680f3b 2085 cmd->transport_state &= ~CMD_T_SENT;
d29a5b6a 2086 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
7d680f3b 2087
03e98c9e 2088 transport_generic_request_failure(cmd);
c66ac9db
NB		 2089 }
2090
4d2300cc 2091 new_cmd = NULL;
c66ac9db
NB		 2092 goto check_depth;
2093
2094 return 0;
2095}
2096
c66ac9db
NB		 2097static inline u32 transport_get_sectors_6(
2098 unsigned char *cdb,
2099 struct se_cmd *cmd,
2100 int *ret)
2101{
5951146d 2102 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 2103
2104 /*
2105 * Assume TYPE_DISK for non struct se_device objects.
2106 * Use 8-bit sector value.
2107 */
2108 if (!dev)
2109 goto type_disk;
2110
2111 /*
2112 * Use 24-bit allocation length for TYPE_TAPE.
2113 */
e3d6f909 2114 if (dev->transport->get_device_type(dev) == TYPE_TAPE)
c66ac9db
NB		 2115 return (u32)(cdb[2] << 16) + (cdb[3] << 8) + cdb[4];
2116
2117 /*
2118 * Everything else assume TYPE_DISK Sector CDB location.
9b5cd7f3
RD		 2119 * Use 8-bit sector value. SBC-3 says:
2120 *
2121 * A TRANSFER LENGTH field set to zero specifies that 256
2122 * logical blocks shall be written. Any other value
2123 * specifies the number of logical blocks that shall be
2124 * written.
c66ac9db
NB		 2125 */
2126type_disk:
9b5cd7f3 2127 return cdb[4] ? : 256;
c66ac9db
NB		 2128}
2129
2130static inline u32 transport_get_sectors_10(
2131 unsigned char *cdb,
2132 struct se_cmd *cmd,
2133 int *ret)
2134{
5951146d 2135 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 2136
2137 /*
2138 * Assume TYPE_DISK for non struct se_device objects.
2139 * Use 16-bit sector value.
2140 */
2141 if (!dev)
2142 goto type_disk;
2143
2144 /*
2145 * XXX_10 is not defined in SSC, throw an exception
2146 */
e3d6f909
AG		 2147 if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
2148 *ret = -EINVAL;
c66ac9db
NB		 2149 return 0;
2150 }
2151
2152 /*
2153 * Everything else assume TYPE_DISK Sector CDB location.
2154 * Use 16-bit sector value.
2155 */
2156type_disk:
2157 return (u32)(cdb[7] << 8) + cdb[8];
2158}
2159
2160static inline u32 transport_get_sectors_12(
2161 unsigned char *cdb,
2162 struct se_cmd *cmd,
2163 int *ret)
2164{
5951146d 2165 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 2166
2167 /*
2168 * Assume TYPE_DISK for non struct se_device objects.
2169 * Use 32-bit sector value.
2170 */
2171 if (!dev)
2172 goto type_disk;
2173
2174 /*
2175 * XXX_12 is not defined in SSC, throw an exception
2176 */
e3d6f909
AG		 2177 if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
2178 *ret = -EINVAL;
c66ac9db
NB		 2179 return 0;
2180 }
2181
2182 /*
2183 * Everything else assume TYPE_DISK Sector CDB location.
2184 * Use 32-bit sector value.
2185 */
2186type_disk:
2187 return (u32)(cdb[6] << 24) + (cdb[7] << 16) + (cdb[8] << 8) + cdb[9];
2188}
2189
2190static inline u32 transport_get_sectors_16(
2191 unsigned char *cdb,
2192 struct se_cmd *cmd,
2193 int *ret)
2194{
5951146d 2195 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 2196
2197 /*
2198 * Assume TYPE_DISK for non struct se_device objects.
2199 * Use 32-bit sector value.
2200 */
2201 if (!dev)
2202 goto type_disk;
2203
2204 /*
2205 * Use 24-bit allocation length for TYPE_TAPE.
2206 */
e3d6f909 2207 if (dev->transport->get_device_type(dev) == TYPE_TAPE)
c66ac9db
NB		 2208 return (u32)(cdb[12] << 16) + (cdb[13] << 8) + cdb[14];
2209
2210type_disk:
2211 return (u32)(cdb[10] << 24) + (cdb[11] << 16) +
2212 (cdb[12] << 8) + cdb[13];
2213}
2214
2215/*
2216 * Used for VARIABLE_LENGTH_CDB WRITE_32 and READ_32 variants
2217 */
2218static inline u32 transport_get_sectors_32(
2219 unsigned char *cdb,
2220 struct se_cmd *cmd,
2221 int *ret)
2222{
2223 /*
2224 * Assume TYPE_DISK for non struct se_device objects.
2225 * Use 32-bit sector value.
2226 */
2227 return (u32)(cdb[28] << 24) + (cdb[29] << 16) +
2228 (cdb[30] << 8) + cdb[31];
2229
2230}
2231
2232static inline u32 transport_get_size(
2233 u32 sectors,
2234 unsigned char *cdb,
2235 struct se_cmd *cmd)
2236{
5951146d 2237 struct se_device *dev = cmd->se_dev;
c66ac9db 2238
e3d6f909 2239 if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
c66ac9db 2240 if (cdb[1] & 1) { /* sectors */
e3d6f909 2241 return dev->se_sub_dev->se_dev_attrib.block_size * sectors;
c66ac9db
NB		 2242 } else /* bytes */
2243 return sectors;
2244 }
8b1e1244 2245
6708bb27 2246 pr_debug("Returning block_size: %u, sectors: %u == %u for"
8b1e1244
AG		 2247 " %s object\n", dev->se_sub_dev->se_dev_attrib.block_size,
2248 sectors, dev->se_sub_dev->se_dev_attrib.block_size * sectors,
2249 dev->transport->name);
2250
e3d6f909 2251 return dev->se_sub_dev->se_dev_attrib.block_size * sectors;
c66ac9db
NB		 2252}
2253
c66ac9db
NB		 2254static void transport_xor_callback(struct se_cmd *cmd)
2255{
2256 unsigned char *buf, *addr;
ec98f782 2257 struct scatterlist *sg;
c66ac9db
NB		 2258 unsigned int offset;
2259 int i;
ec98f782 2260 int count;
c66ac9db
NB		 2261 /*
2262 * From sbc3r22.pdf section 5.48 XDWRITEREAD (10) command
2263 *
2264 * 1) read the specified logical block(s);
2265 * 2) transfer logical blocks from the data-out buffer;
2266 * 3) XOR the logical blocks transferred from the data-out buffer with
2267 * the logical blocks read, storing the resulting XOR data in a buffer;
2268 * 4) if the DISABLE WRITE bit is set to zero, then write the logical
2269 * blocks transferred from the data-out buffer; and
2270 * 5) transfer the resulting XOR data to the data-in buffer.
2271 */
2272 buf = kmalloc(cmd->data_length, GFP_KERNEL);
6708bb27
AG		 2273 if (!buf) {
2274 pr_err("Unable to allocate xor_callback buf\n");
c66ac9db
NB		 2275 return;
2276 }
2277 /*
ec98f782 2278 * Copy the scatterlist WRITE buffer located at cmd->t_data_sg
c66ac9db
NB		 2279 * into the locally allocated *buf
2280 */
ec98f782
AG		 2281 sg_copy_to_buffer(cmd->t_data_sg,
2282 cmd->t_data_nents,
2283 buf,
2284 cmd->data_length);
2285
c66ac9db
NB		 2286 /*
2287 * Now perform the XOR against the BIDI read memory located at
a1d8b49a 2288 * cmd->t_mem_bidi_list
c66ac9db
NB		 2289 */
2290
2291 offset = 0;
ec98f782 2292 for_each_sg(cmd->t_bidi_data_sg, sg, cmd->t_bidi_data_nents, count) {
ca747d61 2293 addr = kmap_atomic(sg_page(sg));
ec98f782 2294 if (!addr)
c66ac9db
NB		 2295 goto out;
2296
ec98f782
AG		 2297 for (i = 0; i < sg->length; i++)
2298 *(addr + sg->offset + i) ^= *(buf + offset + i);
c66ac9db 2299
ec98f782 2300 offset += sg->length;
ca747d61 2301 kunmap_atomic(addr);
c66ac9db 2302 }
ec98f782 2303
c66ac9db
NB		 2304out:
2305 kfree(buf);
2306}
2307
2308/*
2309 * Used to obtain Sense Data from underlying Linux/SCSI struct scsi_cmnd
2310 */
2311static int transport_get_sense_data(struct se_cmd *cmd)
2312{
2313 unsigned char *buffer = cmd->sense_buffer, *sense_buffer = NULL;
42bf829e 2314 struct se_device *dev = cmd->se_dev;
785fdf70 2315 struct se_task *task = NULL;
c66ac9db
NB		 2316 unsigned long flags;
2317 u32 offset = 0;
2318
e3d6f909
AG		 2319 WARN_ON(!cmd->se_lun);
2320
42bf829e
CH		 2321 if (!dev)
2322 return 0;
2323
a1d8b49a 2324 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db 2325 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
a1d8b49a 2326 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 2327 return 0;
2328 }
2329
cf572a96
CH		 2330 if (!cmd->t_task)
2331 goto out;
c66ac9db 2332
cf572a96
CH		 2333 if (!(cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE))
2334 goto out;
c66ac9db 2335
cf572a96
CH		 2336 if (!dev->transport->get_sense_buffer) {
2337 pr_err("dev->transport->get_sense_buffer is NULL\n");
2338 goto out;
2339 }
c66ac9db 2340
cf572a96
CH		 2341 sense_buffer = dev->transport->get_sense_buffer(task);
2342 if (!sense_buffer) {
2343 pr_err("ITT[0x%08x]_TASK[%p]: Unable to locate"
2344 " sense buffer for task with sense\n",
2345 cmd->se_tfo->get_task_tag(cmd), task);
2346 goto out;
c66ac9db 2347 }
cf572a96
CH		 2348 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2349
2350 offset = cmd->se_tfo->set_fabric_sense_len(cmd, TRANSPORT_SENSE_BUFFER);
2351
2352 memcpy(&buffer[offset], sense_buffer, TRANSPORT_SENSE_BUFFER);
2353 cmd->scsi_status = task->task_scsi_status;
2354
2355 /* Automatically padded */
2356 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER + offset;
2357
2358 pr_debug("HBA_[%u]_PLUG[%s]: Set SAM STATUS: 0x%02x and sense\n",
2359 dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status);
2360 return 0;
2361
785fdf70 2362out:
a1d8b49a 2363 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 2364 return -1;
2365}
2366
ec98f782
AG		 2367static inline long long transport_dev_end_lba(struct se_device *dev)
2368{
2369 return dev->transport->get_blocks(dev) + 1;
2370}
2371
2372static int transport_cmd_get_valid_sectors(struct se_cmd *cmd)
2373{
2374 struct se_device *dev = cmd->se_dev;
2375 u32 sectors;
2376
2377 if (dev->transport->get_device_type(dev) != TYPE_DISK)
2378 return 0;
2379
2380 sectors = (cmd->data_length / dev->se_sub_dev->se_dev_attrib.block_size);
2381
6708bb27
AG		 2382 if ((cmd->t_task_lba + sectors) > transport_dev_end_lba(dev)) {
2383 pr_err("LBA: %llu Sectors: %u exceeds"
ec98f782
AG		 2384 " transport_dev_end_lba(): %llu\n",
2385 cmd->t_task_lba, sectors,
2386 transport_dev_end_lba(dev));
7abbe7f3 2387 return -EINVAL;
ec98f782
AG		 2388 }
2389
7abbe7f3 2390 return 0;
ec98f782
AG		 2391}
2392
706d5860
NB		 2393static int target_check_write_same_discard(unsigned char *flags, struct se_device *dev)
2394{
2395 /*
2396 * Determine if the received WRITE_SAME is used to for direct
2397 * passthrough into Linux/SCSI with struct request via TCM/pSCSI
2398 * or we are signaling the use of internal WRITE_SAME + UNMAP=1
2399 * emulation for -> Linux/BLOCK disbard with TCM/IBLOCK code.
2400 */
2401 int passthrough = (dev->transport->transport_type ==
2402 TRANSPORT_PLUGIN_PHBA_PDEV);
2403
2404 if (!passthrough) {
2405 if ((flags[0] & 0x04) || (flags[0] & 0x02)) {
2406 pr_err("WRITE_SAME PBDATA and LBDATA"
2407 " bits not supported for Block Discard"
2408 " Emulation\n");
2409 return -ENOSYS;
2410 }
2411 /*
2412 * Currently for the emulated case we only accept
2413 * tpws with the UNMAP=1 bit set.
2414 */
2415 if (!(flags[0] & 0x08)) {
2416 pr_err("WRITE_SAME w/o UNMAP bit not"
2417 " supported for Block Discard Emulation\n");
2418 return -ENOSYS;
2419 }
2420 }
2421
2422 return 0;
2423}
2424
c66ac9db
NB		 2425/* transport_generic_cmd_sequencer():
2426 *
2427 * Generic Command Sequencer that should work for most DAS transport
2428 * drivers.
2429 *
a12f41f8 2430 * Called from target_setup_cmd_from_cdb() in the $FABRIC_MOD
c66ac9db
NB		 2431 * RX Thread.
2432 *
2433 * FIXME: Need to support other SCSI OPCODES where as well.
2434 */
2435static int transport_generic_cmd_sequencer(
2436 struct se_cmd *cmd,
2437 unsigned char *cdb)
2438{
5951146d 2439 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 2440 struct se_subsystem_dev *su_dev = dev->se_sub_dev;
2441 int ret = 0, sector_ret = 0, passthrough;
2442 u32 sectors = 0, size = 0, pr_reg_type = 0;
2443 u16 service_action;
2444 u8 alua_ascq = 0;
2445 /*
2446 * Check for an existing UNIT ATTENTION condition
2447 */
2448 if (core_scsi3_ua_check(cmd, cdb) < 0) {
c66ac9db
NB		 2449 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2450 cmd->scsi_sense_reason = TCM_CHECK_CONDITION_UNIT_ATTENTION;
5951146d 2451 return -EINVAL;
c66ac9db
NB		 2452 }
2453 /*
2454 * Check status of Asymmetric Logical Unit Assignment port
2455 */
e3d6f909 2456 ret = su_dev->t10_alua.alua_state_check(cmd, cdb, &alua_ascq);
c66ac9db 2457 if (ret != 0) {
c66ac9db 2458 /*
25985edc 2459 * Set SCSI additional sense code (ASC) to 'LUN Not Accessible';
c66ac9db
NB		 2460 * The ALUA additional sense code qualifier (ASCQ) is determined
2461 * by the ALUA primary or secondary access state..
2462 */
2463 if (ret > 0) {
6708bb27 2464 pr_debug("[%s]: ALUA TG Port not available,"
c66ac9db 2465 " SenseKey: NOT_READY, ASC/ASCQ: 0x04/0x%02x\n",
e3d6f909 2466 cmd->se_tfo->get_fabric_name(), alua_ascq);
8b1e1244 2467
c66ac9db
NB		 2468 transport_set_sense_codes(cmd, 0x04, alua_ascq);
2469 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2470 cmd->scsi_sense_reason = TCM_CHECK_CONDITION_NOT_READY;
5951146d 2471 return -EINVAL;
c66ac9db
NB		 2472 }
2473 goto out_invalid_cdb_field;
2474 }
2475 /*
2476 * Check status for SPC-3 Persistent Reservations
2477 */
e3d6f909
AG		 2478 if (su_dev->t10_pr.pr_ops.t10_reservation_check(cmd, &pr_reg_type) != 0) {
2479 if (su_dev->t10_pr.pr_ops.t10_seq_non_holder(
03e98c9e
NB		 2480 cmd, cdb, pr_reg_type) != 0) {
2481 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2482 cmd->se_cmd_flags |= SCF_SCSI_RESERVATION_CONFLICT;
087a03b3 2483 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
03e98c9e
NB		 2484 cmd->scsi_sense_reason = TCM_RESERVATION_CONFLICT;
2485 return -EBUSY;
2486 }
c66ac9db
NB		 2487 /*
2488 * This means the CDB is allowed for the SCSI Initiator port
2489 * when said port is *NOT* holding the legacy SPC-2 or
2490 * SPC-3 Persistent Reservation.
2491 */
2492 }
2493
5bda90c8
CH		 2494 /*
2495 * If we operate in passthrough mode we skip most CDB emulation and
2496 * instead hand the commands down to the physical SCSI device.
2497 */
2498 passthrough =
2499 (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV);
2500
c66ac9db
NB		 2501 switch (cdb[0]) {
2502 case READ_6:
2503 sectors = transport_get_sectors_6(cdb, cmd, &sector_ret);
2504 if (sector_ret)
2505 goto out_unsupported_cdb;
2506 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2507 cmd->t_task_lba = transport_lba_21(cdb);
c66ac9db
NB		 2508 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2509 break;
2510 case READ_10:
2511 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
2512 if (sector_ret)
2513 goto out_unsupported_cdb;
2514 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2515 cmd->t_task_lba = transport_lba_32(cdb);
c66ac9db
NB		 2516 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2517 break;
2518 case READ_12:
2519 sectors = transport_get_sectors_12(cdb, cmd, &sector_ret);
2520 if (sector_ret)
2521 goto out_unsupported_cdb;
2522 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2523 cmd->t_task_lba = transport_lba_32(cdb);
c66ac9db
NB		 2524 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2525 break;
2526 case READ_16:
2527 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
2528 if (sector_ret)
2529 goto out_unsupported_cdb;
2530 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2531 cmd->t_task_lba = transport_lba_64(cdb);
c66ac9db
NB		 2532 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2533 break;
2534 case WRITE_6:
2535 sectors = transport_get_sectors_6(cdb, cmd, &sector_ret);
2536 if (sector_ret)
2537 goto out_unsupported_cdb;
2538 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2539 cmd->t_task_lba = transport_lba_21(cdb);
c66ac9db
NB		 2540 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2541 break;
2542 case WRITE_10:
2543 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
2544 if (sector_ret)
2545 goto out_unsupported_cdb;
2546 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2547 cmd->t_task_lba = transport_lba_32(cdb);
2d3a4b51
CH		 2548 if (cdb[1] & 0x8)
2549 cmd->se_cmd_flags |= SCF_FUA;
c66ac9db
NB		 2550 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2551 break;
2552 case WRITE_12:
2553 sectors = transport_get_sectors_12(cdb, cmd, &sector_ret);
2554 if (sector_ret)
2555 goto out_unsupported_cdb;
2556 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2557 cmd->t_task_lba = transport_lba_32(cdb);
2d3a4b51
CH		 2558 if (cdb[1] & 0x8)
2559 cmd->se_cmd_flags |= SCF_FUA;
c66ac9db
NB		 2560 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2561 break;
2562 case WRITE_16:
2563 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
2564 if (sector_ret)
2565 goto out_unsupported_cdb;
2566 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2567 cmd->t_task_lba = transport_lba_64(cdb);
2d3a4b51
CH		 2568 if (cdb[1] & 0x8)
2569 cmd->se_cmd_flags |= SCF_FUA;
c66ac9db
NB		 2570 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2571 break;
2572 case XDWRITEREAD_10:
2573 if ((cmd->data_direction != DMA_TO_DEVICE) ||
33c3fafc 2574 !(cmd->se_cmd_flags & SCF_BIDI))
c66ac9db
NB		 2575 goto out_invalid_cdb_field;
2576 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
2577 if (sector_ret)
2578 goto out_unsupported_cdb;
2579 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2580 cmd->t_task_lba = transport_lba_32(cdb);
c66ac9db 2581 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
7c1c6af3 2582
5bda90c8
CH		 2583 /*
2584 * Do now allow BIDI commands for passthrough mode.
2585 */
2586 if (passthrough)
7c1c6af3 2587 goto out_unsupported_cdb;
5bda90c8 2588
c66ac9db 2589 /*
35e0e757 2590 * Setup BIDI XOR callback to be run after I/O completion.
c66ac9db
NB		 2591 */
2592 cmd->transport_complete_callback = &transport_xor_callback;
2d3a4b51
CH		 2593 if (cdb[1] & 0x8)
2594 cmd->se_cmd_flags |= SCF_FUA;
c66ac9db
NB		 2595 break;
2596 case VARIABLE_LENGTH_CMD:
2597 service_action = get_unaligned_be16(&cdb[8]);
c66ac9db
NB		 2598 switch (service_action) {
2599 case XDWRITEREAD_32:
2600 sectors = transport_get_sectors_32(cdb, cmd, &sector_ret);
2601 if (sector_ret)
2602 goto out_unsupported_cdb;
2603 size = transport_get_size(sectors, cdb, cmd);
2604 /*
2605 * Use WRITE_32 and READ_32 opcodes for the emulated
2606 * XDWRITE_READ_32 logic.
2607 */
a1d8b49a 2608 cmd->t_task_lba = transport_lba_64_ext(cdb);
c66ac9db
NB		 2609 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2610
5bda90c8
CH		 2611 /*
2612 * Do now allow BIDI commands for passthrough mode.
2613 */
c66ac9db 2614 if (passthrough)
7c1c6af3 2615 goto out_unsupported_cdb;
5bda90c8 2616
c66ac9db 2617 /*
35e0e757
CH		 2618 * Setup BIDI XOR callback to be run during after I/O
2619 * completion.
c66ac9db
NB		 2620 */
2621 cmd->transport_complete_callback = &transport_xor_callback;
2d3a4b51
CH		 2622 if (cdb[1] & 0x8)
2623 cmd->se_cmd_flags |= SCF_FUA;
c66ac9db
NB		 2624 break;
2625 case WRITE_SAME_32:
2626 sectors = transport_get_sectors_32(cdb, cmd, &sector_ret);
2627 if (sector_ret)
2628 goto out_unsupported_cdb;
dd3a5ad8 2629
6708bb27 2630 if (sectors)
12850626 2631 size = transport_get_size(1, cdb, cmd);
6708bb27
AG		 2632 else {
2633 pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not"
2634 " supported\n");
2635 goto out_invalid_cdb_field;
2636 }
dd3a5ad8 2637
a1d8b49a 2638 cmd->t_task_lba = get_unaligned_be64(&cdb[12]);
c66ac9db
NB		 2639 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2640
706d5860 2641 if (target_check_write_same_discard(&cdb[10], dev) < 0)
67236c44 2642 goto out_unsupported_cdb;
5bda90c8 2643 if (!passthrough)
6bb35e00 2644 cmd->execute_cmd = target_emulate_write_same;
c66ac9db
NB		 2645 break;
2646 default:
6708bb27 2647 pr_err("VARIABLE_LENGTH_CMD service action"
c66ac9db
NB		 2648 " 0x%04x not supported\n", service_action);
2649 goto out_unsupported_cdb;
2650 }
2651 break;
e434f1f1 2652 case MAINTENANCE_IN:
e3d6f909 2653 if (dev->transport->get_device_type(dev) != TYPE_ROM) {
c66ac9db
NB		 2654 /* MAINTENANCE_IN from SCC-2 */
2655 /*
2656 * Check for emulated MI_REPORT_TARGET_PGS.
2657 */
e76a35d6
CH		 2658 if (cdb[1] == MI_REPORT_TARGET_PGS &&
2659 su_dev->t10_alua.alua_type == SPC3_ALUA_EMULATED) {
6bb35e00 2660 cmd->execute_cmd =
e76a35d6 2661 target_emulate_report_target_port_groups;
c66ac9db
NB		 2662 }
2663 size = (cdb[6] << 24) | (cdb[7] << 16) |
2664 (cdb[8] << 8) | cdb[9];
2665 } else {
2666 /* GPCMD_SEND_KEY from multi media commands */
2667 size = (cdb[8] << 8) + cdb[9];
2668 }
05d1c7c0 2669 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2670 break;
2671 case MODE_SELECT:
2672 size = cdb[4];
2673 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2674 break;
2675 case MODE_SELECT_10:
2676 size = (cdb[7] << 8) + cdb[8];
2677 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2678 break;
2679 case MODE_SENSE:
2680 size = cdb[4];
05d1c7c0 2681 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
5bda90c8 2682 if (!passthrough)
6bb35e00 2683 cmd->execute_cmd = target_emulate_modesense;
c66ac9db
NB		 2684 break;
2685 case MODE_SENSE_10:
5bda90c8
CH		 2686 size = (cdb[7] << 8) + cdb[8];
2687 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2688 if (!passthrough)
6bb35e00 2689 cmd->execute_cmd = target_emulate_modesense;
5bda90c8 2690 break;
c66ac9db
NB		 2691 case GPCMD_READ_BUFFER_CAPACITY:
2692 case GPCMD_SEND_OPC:
2693 case LOG_SELECT:
2694 case LOG_SENSE:
2695 size = (cdb[7] << 8) + cdb[8];
05d1c7c0 2696 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2697 break;
2698 case READ_BLOCK_LIMITS:
2699 size = READ_BLOCK_LEN;
05d1c7c0 2700 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2701 break;
2702 case GPCMD_GET_CONFIGURATION:
2703 case GPCMD_READ_FORMAT_CAPACITIES:
2704 case GPCMD_READ_DISC_INFO:
2705 case GPCMD_READ_TRACK_RZONE_INFO:
2706 size = (cdb[7] << 8) + cdb[8];
2707 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2708 break;
2709 case PERSISTENT_RESERVE_IN:
617c0e06 2710 if (su_dev->t10_pr.res_type == SPC3_PERSISTENT_RESERVATIONS)
6bb35e00 2711 cmd->execute_cmd = target_scsi3_emulate_pr_in;
617c0e06
CH		 2712 size = (cdb[7] << 8) + cdb[8];
2713 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2714 break;
c66ac9db 2715 case PERSISTENT_RESERVE_OUT:
617c0e06 2716 if (su_dev->t10_pr.res_type == SPC3_PERSISTENT_RESERVATIONS)
6bb35e00 2717 cmd->execute_cmd = target_scsi3_emulate_pr_out;
c66ac9db 2718 size = (cdb[7] << 8) + cdb[8];
05d1c7c0 2719 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2720 break;
2721 case GPCMD_MECHANISM_STATUS:
2722 case GPCMD_READ_DVD_STRUCTURE:
2723 size = (cdb[8] << 8) + cdb[9];
2724 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2725 break;
2726 case READ_POSITION:
2727 size = READ_POSITION_LEN;
05d1c7c0 2728 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db 2729 break;
e434f1f1 2730 case MAINTENANCE_OUT:
e3d6f909 2731 if (dev->transport->get_device_type(dev) != TYPE_ROM) {
c66ac9db
NB		 2732 /* MAINTENANCE_OUT from SCC-2
2733 *
2734 * Check for emulated MO_SET_TARGET_PGS.
2735 */
e76a35d6
CH		 2736 if (cdb[1] == MO_SET_TARGET_PGS &&
2737 su_dev->t10_alua.alua_type == SPC3_ALUA_EMULATED) {
6bb35e00 2738 cmd->execute_cmd =
e76a35d6 2739 target_emulate_set_target_port_groups;
c66ac9db
NB		 2740 }
2741
2742 size = (cdb[6] << 24) | (cdb[7] << 16) |
2743 (cdb[8] << 8) | cdb[9];
2744 } else {
2745 /* GPCMD_REPORT_KEY from multi media commands */
2746 size = (cdb[8] << 8) + cdb[9];
2747 }
05d1c7c0 2748 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2749 break;
2750 case INQUIRY:
2751 size = (cdb[3] << 8) + cdb[4];
2752 /*
2753 * Do implict HEAD_OF_QUEUE processing for INQUIRY.
2754 * See spc4r17 section 5.3
2755 */
5951146d 2756 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
e66ecd50 2757 cmd->sam_task_attr = MSG_HEAD_TAG;
05d1c7c0 2758 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
5bda90c8 2759 if (!passthrough)
6bb35e00 2760 cmd->execute_cmd = target_emulate_inquiry;
c66ac9db
NB		 2761 break;
2762 case READ_BUFFER:
2763 size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
05d1c7c0 2764 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2765 break;
2766 case READ_CAPACITY:
2767 size = READ_CAP_LEN;
05d1c7c0 2768 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
5bda90c8 2769 if (!passthrough)
6bb35e00 2770 cmd->execute_cmd = target_emulate_readcapacity;
c66ac9db
NB		 2771 break;
2772 case READ_MEDIA_SERIAL_NUMBER:
2773 case SECURITY_PROTOCOL_IN:
2774 case SECURITY_PROTOCOL_OUT:
2775 size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
05d1c7c0 2776 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2777 break;
2778 case SERVICE_ACTION_IN:
5bda90c8
CH		 2779 switch (cmd->t_task_cdb[1] & 0x1f) {
2780 case SAI_READ_CAPACITY_16:
2781 if (!passthrough)
6bb35e00 2782 cmd->execute_cmd =
5bda90c8
CH		 2783 target_emulate_readcapacity_16;
2784 break;
2785 default:
2786 if (passthrough)
2787 break;
2788
2789 pr_err("Unsupported SA: 0x%02x\n",
2790 cmd->t_task_cdb[1] & 0x1f);
b168fe8c 2791 goto out_invalid_cdb_field;
5bda90c8
CH		 2792 }
2793 /*FALLTHROUGH*/
c66ac9db
NB		 2794 case ACCESS_CONTROL_IN:
2795 case ACCESS_CONTROL_OUT:
2796 case EXTENDED_COPY:
2797 case READ_ATTRIBUTE:
2798 case RECEIVE_COPY_RESULTS:
2799 case WRITE_ATTRIBUTE:
2800 size = (cdb[10] << 24) | (cdb[11] << 16) |
2801 (cdb[12] << 8) | cdb[13];
05d1c7c0 2802 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2803 break;
2804 case RECEIVE_DIAGNOSTIC:
2805 case SEND_DIAGNOSTIC:
2806 size = (cdb[3] << 8) | cdb[4];
05d1c7c0 2807 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2808 break;
2809/* #warning FIXME: Figure out correct GPCMD_READ_CD blocksize. */
2810#if 0
2811 case GPCMD_READ_CD:
2812 sectors = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
2813 size = (2336 * sectors);
05d1c7c0 2814 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2815 break;
2816#endif
2817 case READ_TOC:
2818 size = cdb[8];
05d1c7c0 2819 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2820 break;
2821 case REQUEST_SENSE:
2822 size = cdb[4];
05d1c7c0 2823 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
5bda90c8 2824 if (!passthrough)
6bb35e00 2825 cmd->execute_cmd = target_emulate_request_sense;
c66ac9db
NB		 2826 break;
2827 case READ_ELEMENT_STATUS:
2828 size = 65536 * cdb[7] + 256 * cdb[8] + cdb[9];
05d1c7c0 2829 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2830 break;
2831 case WRITE_BUFFER:
2832 size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
05d1c7c0 2833 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2834 break;
2835 case RESERVE:
2836 case RESERVE_10:
2837 /*
2838 * The SPC-2 RESERVE does not contain a size in the SCSI CDB.
2839 * Assume the passthrough or $FABRIC_MOD will tell us about it.
2840 */
2841 if (cdb[0] == RESERVE_10)
2842 size = (cdb[7] << 8) | cdb[8];
2843 else
2844 size = cmd->data_length;
2845
2846 /*
2847 * Setup the legacy emulated handler for SPC-2 and
2848 * >= SPC-3 compatible reservation handling (CRH=1)
2849 * Otherwise, we assume the underlying SCSI logic is
2850 * is running in SPC_PASSTHROUGH, and wants reservations
2851 * emulation disabled.
2852 */
e76a35d6 2853 if (su_dev->t10_pr.res_type != SPC_PASSTHROUGH)
6bb35e00 2854 cmd->execute_cmd = target_scsi2_reservation_reserve;
c66ac9db
NB		 2855 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
2856 break;
2857 case RELEASE:
2858 case RELEASE_10:
2859 /*
2860 * The SPC-2 RELEASE does not contain a size in the SCSI CDB.
2861 * Assume the passthrough or $FABRIC_MOD will tell us about it.
2862 */
2863 if (cdb[0] == RELEASE_10)
2864 size = (cdb[7] << 8) | cdb[8];
2865 else
2866 size = cmd->data_length;
2867
e76a35d6 2868 if (su_dev->t10_pr.res_type != SPC_PASSTHROUGH)
6bb35e00 2869 cmd->execute_cmd = target_scsi2_reservation_release;
c66ac9db
NB		 2870 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
2871 break;
2872 case SYNCHRONIZE_CACHE:
8e94b8db 2873 case SYNCHRONIZE_CACHE_16:
c66ac9db
NB		 2874 /*
2875 * Extract LBA and range to be flushed for emulated SYNCHRONIZE_CACHE
2876 */
2877 if (cdb[0] == SYNCHRONIZE_CACHE) {
2878 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
a1d8b49a 2879 cmd->t_task_lba = transport_lba_32(cdb);
c66ac9db
NB		 2880 } else {
2881 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
a1d8b49a 2882 cmd->t_task_lba = transport_lba_64(cdb);
c66ac9db
NB		 2883 }
2884 if (sector_ret)
2885 goto out_unsupported_cdb;
2886
2887 size = transport_get_size(sectors, cdb, cmd);
2888 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
2889
5bda90c8 2890 if (passthrough)
c66ac9db 2891 break;
5bda90c8 2892
c66ac9db
NB		 2893 /*
2894 * Check to ensure that LBA + Range does not exceed past end of
7abbe7f3 2895 * device for IBLOCK and FILEIO ->do_sync_cache() backend calls
c66ac9db 2896 */
7abbe7f3
NB		 2897 if ((cmd->t_task_lba != 0) || (sectors != 0)) {
2898 if (transport_cmd_get_valid_sectors(cmd) < 0)
2899 goto out_invalid_cdb_field;
2900 }
6bb35e00 2901 cmd->execute_cmd = target_emulate_synchronize_cache;
c66ac9db
NB		 2902 break;
2903 case UNMAP:
2904 size = get_unaligned_be16(&cdb[7]);
05d1c7c0 2905 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
5bda90c8 2906 if (!passthrough)
6bb35e00 2907 cmd->execute_cmd = target_emulate_unmap;
c66ac9db
NB		 2908 break;
2909 case WRITE_SAME_16:
2910 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
2911 if (sector_ret)
2912 goto out_unsupported_cdb;
dd3a5ad8 2913
6708bb27 2914 if (sectors)
12850626 2915 size = transport_get_size(1, cdb, cmd);
6708bb27
AG		 2916 else {
2917 pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not supported\n");
2918 goto out_invalid_cdb_field;
2919 }
dd3a5ad8 2920
5db0753b 2921 cmd->t_task_lba = get_unaligned_be64(&cdb[2]);
706d5860
NB		 2922 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2923
2924 if (target_check_write_same_discard(&cdb[1], dev) < 0)
67236c44 2925 goto out_unsupported_cdb;
5bda90c8 2926 if (!passthrough)
6bb35e00 2927 cmd->execute_cmd = target_emulate_write_same;
706d5860
NB		 2928 break;
2929 case WRITE_SAME:
2930 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
2931 if (sector_ret)
2932 goto out_unsupported_cdb;
2933
2934 if (sectors)
12850626 2935 size = transport_get_size(1, cdb, cmd);
706d5860
NB		 2936 else {
2937 pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not supported\n");
2938 goto out_invalid_cdb_field;
c66ac9db 2939 }
706d5860
NB		 2940
2941 cmd->t_task_lba = get_unaligned_be32(&cdb[2]);
c66ac9db 2942 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
706d5860
NB		 2943 /*
2944 * Follow sbcr26 with WRITE_SAME (10) and check for the existence
2945 * of byte 1 bit 3 UNMAP instead of original reserved field
2946 */
2947 if (target_check_write_same_discard(&cdb[1], dev) < 0)
67236c44 2948 goto out_unsupported_cdb;
5bda90c8 2949 if (!passthrough)
6bb35e00 2950 cmd->execute_cmd = target_emulate_write_same;
c66ac9db
NB		 2951 break;
2952 case ALLOW_MEDIUM_REMOVAL:
c66ac9db 2953 case ERASE:
c66ac9db
NB		 2954 case REZERO_UNIT:
2955 case SEEK_10:
c66ac9db
NB		 2956 case SPACE:
2957 case START_STOP:
2958 case TEST_UNIT_READY:
2959 case VERIFY:
2960 case WRITE_FILEMARKS:
5bda90c8
CH		 2961 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
2962 if (!passthrough)
6bb35e00 2963 cmd->execute_cmd = target_emulate_noop;
5bda90c8
CH		 2964 break;
2965 case GPCMD_CLOSE_TRACK:
2966 case INITIALIZE_ELEMENT_STATUS:
2967 case GPCMD_LOAD_UNLOAD:
2968 case GPCMD_SET_SPEED:
c66ac9db
NB		 2969 case MOVE_MEDIUM:
2970 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
2971 break;
2972 case REPORT_LUNS:
6bb35e00 2973 cmd->execute_cmd = target_report_luns;
c66ac9db
NB		 2974 size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
2975 /*
2976 * Do implict HEAD_OF_QUEUE processing for REPORT_LUNS
2977 * See spc4r17 section 5.3
2978 */
5951146d 2979 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
e66ecd50 2980 cmd->sam_task_attr = MSG_HEAD_TAG;
05d1c7c0 2981 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db 2982 break;
ed0b2144
SH		 2983 case GET_EVENT_STATUS_NOTIFICATION:
2984 size = (cdb[7] << 8) | cdb[8];
2985 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2986 break;
c66ac9db 2987 default:
6708bb27 2988 pr_warn("TARGET_CORE[%s]: Unsupported SCSI Opcode"
c66ac9db 2989 " 0x%02x, sending CHECK_CONDITION.\n",
e3d6f909 2990 cmd->se_tfo->get_fabric_name(), cdb[0]);
c66ac9db
NB		 2991 goto out_unsupported_cdb;
2992 }
2993
b0d79946
SAS		 2994 if (cmd->unknown_data_length)
2995 cmd->data_length = size;
2996
c66ac9db 2997 if (size != cmd->data_length) {
6708bb27 2998 pr_warn("TARGET_CORE[%s]: Expected Transfer Length:"
c66ac9db 2999 " %u does not match SCSI CDB Length: %u for SAM Opcode:"
e3d6f909 3000 " 0x%02x\n", cmd->se_tfo->get_fabric_name(),
c66ac9db
NB		 3001 cmd->data_length, size, cdb[0]);
3002
3003 cmd->cmd_spdtl = size;
3004
3005 if (cmd->data_direction == DMA_TO_DEVICE) {
6708bb27 3006 pr_err("Rejecting underflow/overflow"
c66ac9db
NB		 3007 " WRITE data\n");
3008 goto out_invalid_cdb_field;
3009 }
3010 /*
3011 * Reject READ_* or WRITE_* with overflow/underflow for
3012 * type SCF_SCSI_DATA_SG_IO_CDB.
3013 */
6708bb27
AG		 3014 if (!ret && (dev->se_sub_dev->se_dev_attrib.block_size != 512)) {
3015 pr_err("Failing OVERFLOW/UNDERFLOW for LBA op"
c66ac9db 3016 " CDB on non 512-byte sector setup subsystem"
e3d6f909 3017 " plugin: %s\n", dev->transport->name);
c66ac9db
NB		 3018 /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */
3019 goto out_invalid_cdb_field;
3020 }
3021
3022 if (size > cmd->data_length) {
3023 cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
3024 cmd->residual_count = (size - cmd->data_length);
3025 } else {
3026 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
3027 cmd->residual_count = (cmd->data_length - size);
3028 }
3029 cmd->data_length = size;
3030 }
3031
015487b8 3032 if (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB &&
bebe2fdc
CH		 3033 (sectors > dev->se_sub_dev->se_dev_attrib.fabric_max_sectors ||
3034 sectors > dev->se_sub_dev->se_dev_attrib.max_sectors)) {
015487b8
RD		 3035 printk_ratelimited(KERN_ERR "SCSI OP %02xh with too big sectors %u\n",
3036 cdb[0], sectors);
3037 goto out_invalid_cdb_field;
3038 }
3039
5bda90c8 3040 /* reject any command that we don't have a handler for */
6bb35e00 3041 if (!(passthrough || cmd->execute_cmd ||
5bda90c8
CH		 3042 (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB)))
3043 goto out_unsupported_cdb;
3044
c66ac9db
NB		 3045 transport_set_supported_SAM_opcode(cmd);
3046 return ret;
3047
3048out_unsupported_cdb:
3049 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3050 cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
5951146d 3051 return -EINVAL;
c66ac9db
NB		 3052out_invalid_cdb_field:
3053 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3054 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
5951146d 3055 return -EINVAL;
c66ac9db
NB		 3056}
3057
c66ac9db 3058/*
35e0e757 3059 * Called from I/O completion to determine which dormant/delayed
c66ac9db
NB		 3060 * and ordered cmds need to have their tasks added to the execution queue.
3061 */
3062static void transport_complete_task_attr(struct se_cmd *cmd)
3063{
5951146d 3064 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 3065 struct se_cmd *cmd_p, *cmd_tmp;
3066 int new_active_tasks = 0;
3067
e66ecd50 3068 if (cmd->sam_task_attr == MSG_SIMPLE_TAG) {
c66ac9db
NB		 3069 atomic_dec(&dev->simple_cmds);
3070 smp_mb__after_atomic_dec();
3071 dev->dev_cur_ordered_id++;
6708bb27 3072 pr_debug("Incremented dev->dev_cur_ordered_id: %u for"
c66ac9db
NB		 3073 " SIMPLE: %u\n", dev->dev_cur_ordered_id,
3074 cmd->se_ordered_id);
e66ecd50 3075 } else if (cmd->sam_task_attr == MSG_HEAD_TAG) {
c66ac9db 3076 dev->dev_cur_ordered_id++;
6708bb27 3077 pr_debug("Incremented dev_cur_ordered_id: %u for"
c66ac9db
NB		 3078 " HEAD_OF_QUEUE: %u\n", dev->dev_cur_ordered_id,
3079 cmd->se_ordered_id);
e66ecd50 3080 } else if (cmd->sam_task_attr == MSG_ORDERED_TAG) {
c66ac9db
NB		 3081 atomic_dec(&dev->dev_ordered_sync);
3082 smp_mb__after_atomic_dec();
c66ac9db
NB		 3083
3084 dev->dev_cur_ordered_id++;
6708bb27 3085 pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED:"
c66ac9db
NB		 3086 " %u\n", dev->dev_cur_ordered_id, cmd->se_ordered_id);
3087 }
3088 /*
3089 * Process all commands up to the last received
3090 * ORDERED task attribute which requires another blocking
3091 * boundary
3092 */
3093 spin_lock(&dev->delayed_cmd_lock);
3094 list_for_each_entry_safe(cmd_p, cmd_tmp,
5951146d 3095 &dev->delayed_cmd_list, se_delayed_node) {
c66ac9db 3096
5951146d 3097 list_del(&cmd_p->se_delayed_node);
c66ac9db
NB		 3098 spin_unlock(&dev->delayed_cmd_lock);
3099
6708bb27 3100 pr_debug("Calling add_tasks() for"
c66ac9db
NB		 3101 " cmd_p: 0x%02x Task Attr: 0x%02x"
3102 " Dormant -> Active, se_ordered_id: %u\n",
6708bb27 3103 cmd_p->t_task_cdb[0],
c66ac9db
NB		 3104 cmd_p->sam_task_attr, cmd_p->se_ordered_id);
3105
cf572a96 3106 target_add_to_execute_list(cmd_p);
c66ac9db
NB		 3107 new_active_tasks++;
3108
3109 spin_lock(&dev->delayed_cmd_lock);
e66ecd50 3110 if (cmd_p->sam_task_attr == MSG_ORDERED_TAG)
c66ac9db
NB		 3111 break;
3112 }
3113 spin_unlock(&dev->delayed_cmd_lock);
3114 /*
3115 * If new tasks have become active, wake up the transport thread
3116 * to do the processing of the Active tasks.
3117 */
3118 if (new_active_tasks != 0)
e3d6f909 3119 wake_up_interruptible(&dev->dev_queue_obj.thread_wq);
c66ac9db
NB		 3120}
3121
e057f533 3122static void transport_complete_qf(struct se_cmd *cmd)
07bde79a
NB		 3123{
3124 int ret = 0;
3125
e057f533
CH		 3126 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
3127 transport_complete_task_attr(cmd);
3128
3129 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
3130 ret = cmd->se_tfo->queue_status(cmd);
3131 if (ret)
3132 goto out;
3133 }
07bde79a
NB		 3134
3135 switch (cmd->data_direction) {
3136 case DMA_FROM_DEVICE:
3137 ret = cmd->se_tfo->queue_data_in(cmd);
3138 break;
3139 case DMA_TO_DEVICE:
ec98f782 3140 if (cmd->t_bidi_data_sg) {
07bde79a
NB		 3141 ret = cmd->se_tfo->queue_data_in(cmd);
3142 if (ret < 0)
e057f533 3143 break;
07bde79a
NB		 3144 }
3145 /* Fall through for DMA_TO_DEVICE */
3146 case DMA_NONE:
3147 ret = cmd->se_tfo->queue_status(cmd);
3148 break;
3149 default:
3150 break;
3151 }
3152
e057f533
CH		 3153out:
3154 if (ret < 0) {
3155 transport_handle_queue_full(cmd, cmd->se_dev);
3156 return;
3157 }
3158 transport_lun_remove_cmd(cmd);
3159 transport_cmd_check_stop_to_fabric(cmd);
07bde79a
NB		 3160}
3161
3162static void transport_handle_queue_full(
3163 struct se_cmd *cmd,
e057f533 3164 struct se_device *dev)
07bde79a
NB		 3165{
3166 spin_lock_irq(&dev->qf_cmd_lock);
07bde79a
NB		 3167 list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list);
3168 atomic_inc(&dev->dev_qf_count);
3169 smp_mb__after_atomic_inc();
3170 spin_unlock_irq(&cmd->se_dev->qf_cmd_lock);
3171
3172 schedule_work(&cmd->se_dev->qf_work_queue);
3173}
3174
35e0e757 3175static void target_complete_ok_work(struct work_struct *work)
c66ac9db 3176{
35e0e757 3177 struct se_cmd *cmd = container_of(work, struct se_cmd, work);
07bde79a 3178 int reason = 0, ret;
35e0e757 3179
c66ac9db
NB		 3180 /*
3181 * Check if we need to move delayed/dormant tasks from cmds on the
3182 * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task
3183 * Attribute.
3184 */
5951146d 3185 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
c66ac9db 3186 transport_complete_task_attr(cmd);
07bde79a
NB		 3187 /*
3188 * Check to schedule QUEUE_FULL work, or execute an existing
3189 * cmd->transport_qf_callback()
3190 */
3191 if (atomic_read(&cmd->se_dev->dev_qf_count) != 0)
3192 schedule_work(&cmd->se_dev->qf_work_queue);
3193
c66ac9db
NB		 3194 /*
3195 * Check if we need to retrieve a sense buffer from
3196 * the struct se_cmd in question.
3197 */
3198 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
3199 if (transport_get_sense_data(cmd) < 0)
3200 reason = TCM_NON_EXISTENT_LUN;
3201
3202 /*
3203 * Only set when an struct se_task->task_scsi_status returned
3204 * a non GOOD status.
3205 */
3206 if (cmd->scsi_status) {
07bde79a 3207 ret = transport_send_check_condition_and_sense(
c66ac9db 3208 cmd, reason, 1);
f147abb4 3209 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a
NB		 3210 goto queue_full;
3211
c66ac9db
NB		 3212 transport_lun_remove_cmd(cmd);
3213 transport_cmd_check_stop_to_fabric(cmd);
3214 return;
3215 }
3216 }
3217 /*
25985edc 3218 * Check for a callback, used by amongst other things
c66ac9db
NB		 3219 * XDWRITE_READ_10 emulation.
3220 */
3221 if (cmd->transport_complete_callback)
3222 cmd->transport_complete_callback(cmd);
3223
3224 switch (cmd->data_direction) {
3225 case DMA_FROM_DEVICE:
3226 spin_lock(&cmd->se_lun->lun_sep_lock);
e3d6f909
AG		 3227 if (cmd->se_lun->lun_sep) {
3228 cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
c66ac9db
NB		 3229 cmd->data_length;
3230 }
3231 spin_unlock(&cmd->se_lun->lun_sep_lock);
c66ac9db 3232
07bde79a 3233 ret = cmd->se_tfo->queue_data_in(cmd);
f147abb4 3234 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a 3235 goto queue_full;
c66ac9db
NB		 3236 break;
3237 case DMA_TO_DEVICE:
3238 spin_lock(&cmd->se_lun->lun_sep_lock);
e3d6f909
AG		 3239 if (cmd->se_lun->lun_sep) {
3240 cmd->se_lun->lun_sep->sep_stats.rx_data_octets +=
c66ac9db
NB		 3241 cmd->data_length;
3242 }
3243 spin_unlock(&cmd->se_lun->lun_sep_lock);
3244 /*
3245 * Check if we need to send READ payload for BIDI-COMMAND
3246 */
ec98f782 3247 if (cmd->t_bidi_data_sg) {
c66ac9db 3248 spin_lock(&cmd->se_lun->lun_sep_lock);
e3d6f909
AG		 3249 if (cmd->se_lun->lun_sep) {
3250 cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
c66ac9db
NB		 3251 cmd->data_length;
3252 }
3253 spin_unlock(&cmd->se_lun->lun_sep_lock);
07bde79a 3254 ret = cmd->se_tfo->queue_data_in(cmd);
f147abb4 3255 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a 3256 goto queue_full;
c66ac9db
NB		 3257 break;
3258 }
3259 /* Fall through for DMA_TO_DEVICE */
3260 case DMA_NONE:
07bde79a 3261 ret = cmd->se_tfo->queue_status(cmd);
f147abb4 3262 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a 3263 goto queue_full;
c66ac9db
NB		 3264 break;
3265 default:
3266 break;
3267 }
3268
3269 transport_lun_remove_cmd(cmd);
3270 transport_cmd_check_stop_to_fabric(cmd);
07bde79a
NB		 3271 return;
3272
3273queue_full:
6708bb27 3274 pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p,"
07bde79a 3275 " data_direction: %d\n", cmd, cmd->data_direction);
e057f533
CH		 3276 cmd->t_state = TRANSPORT_COMPLETE_QF_OK;
3277 transport_handle_queue_full(cmd, cmd->se_dev);
c66ac9db
NB		 3278}
3279
3280static void transport_free_dev_tasks(struct se_cmd *cmd)
3281{
785fdf70 3282 struct se_task *task;
c66ac9db 3283
785fdf70 3284 task = cmd->t_task;
cf572a96 3285 if (task && !(cmd->transport_state & CMD_T_BUSY))
42bf829e 3286 cmd->se_dev->transport->free_task(task);
c66ac9db
NB		 3287}
3288
6708bb27 3289static inline void transport_free_sgl(struct scatterlist *sgl, int nents)
c66ac9db 3290{
ec98f782 3291 struct scatterlist *sg;
ec98f782 3292 int count;
c66ac9db 3293
6708bb27
AG		 3294 for_each_sg(sgl, sg, nents, count)
3295 __free_page(sg_page(sg));
c66ac9db 3296
6708bb27
AG		 3297 kfree(sgl);
3298}
c66ac9db 3299
6708bb27
AG		 3300static inline void transport_free_pages(struct se_cmd *cmd)
3301{
3302 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC)
3303 return;
3304
3305 transport_free_sgl(cmd->t_data_sg, cmd->t_data_nents);
ec98f782
AG		 3306 cmd->t_data_sg = NULL;
3307 cmd->t_data_nents = 0;
c66ac9db 3308
6708bb27 3309 transport_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents);
ec98f782
AG		 3310 cmd->t_bidi_data_sg = NULL;
3311 cmd->t_bidi_data_nents = 0;
c66ac9db
NB		 3312}
3313
e26d99ae
CH		 3314/**
3315 * transport_release_cmd - free a command
3316 * @cmd: command to free
3317 *
3318 * This routine unconditionally frees a command, and reference counting
3319 * or list removal must be done in the caller.
3320 */
3321static void transport_release_cmd(struct se_cmd *cmd)
3322{
3323 BUG_ON(!cmd->se_tfo);
3324
c8e31f26 3325 if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
e26d99ae
CH		 3326 core_tmr_release_req(cmd->se_tmr_req);
3327 if (cmd->t_task_cdb != cmd->__t_task_cdb)
3328 kfree(cmd->t_task_cdb);
3329 /*
7481deb4
NB		 3330 * If this cmd has been setup with target_get_sess_cmd(), drop
3331 * the kref and call ->release_cmd() in kref callback.
e26d99ae 3332 */
7481deb4
NB		 3333 if (cmd->check_release != 0) {
3334 target_put_sess_cmd(cmd->se_sess, cmd);
3335 return;
3336 }
e26d99ae
CH		 3337 cmd->se_tfo->release_cmd(cmd);
3338}
3339
d3df7825
CH		 3340/**
3341 * transport_put_cmd - release a reference to a command
3342 * @cmd: command to release
3343 *
3344 * This routine releases our reference to the command and frees it if possible.
3345 */
39c05f32 3346static void transport_put_cmd(struct se_cmd *cmd)
c66ac9db
NB		 3347{
3348 unsigned long flags;
4911e3cc 3349 int free_tasks = 0;
c66ac9db 3350
a1d8b49a 3351 spin_lock_irqsave(&cmd->t_state_lock, flags);
4911e3cc
CH		 3352 if (atomic_read(&cmd->t_fe_count)) {
3353 if (!atomic_dec_and_test(&cmd->t_fe_count))
3354 goto out_busy;
3355 }
3356
3357 if (atomic_read(&cmd->t_se_count)) {
3358 if (!atomic_dec_and_test(&cmd->t_se_count))
3359 goto out_busy;
3360 }
3361
7d680f3b
CH		 3362 if (cmd->transport_state & CMD_T_DEV_ACTIVE) {
3363 cmd->transport_state &= ~CMD_T_DEV_ACTIVE;
cf572a96 3364 target_remove_from_state_list(cmd);
4911e3cc 3365 free_tasks = 1;
c66ac9db 3366 }
a1d8b49a 3367 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 3368
4911e3cc
CH		 3369 if (free_tasks != 0)
3370 transport_free_dev_tasks(cmd);
d3df7825 3371
c66ac9db 3372 transport_free_pages(cmd);
31afc39c 3373 transport_release_cmd(cmd);
39c05f32 3374 return;
4911e3cc
CH		 3375out_busy:
3376 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 3377}
3378
c66ac9db 3379/*
ec98f782
AG		 3380 * transport_generic_map_mem_to_cmd - Use fabric-alloced pages instead of
3381 * allocating in the core.
c66ac9db
NB		 3382 * @cmd: Associated se_cmd descriptor
3383 * @mem: SGL style memory for TCM WRITE / READ
3384 * @sg_mem_num: Number of SGL elements
3385 * @mem_bidi_in: SGL style memory for TCM BIDI READ
3386 * @sg_mem_bidi_num: Number of BIDI READ SGL elements
3387 *
3388 * Return: nonzero return cmd was rejected for -ENOMEM or inproper usage
3389 * of parameters.
3390 */
3391int transport_generic_map_mem_to_cmd(
3392 struct se_cmd *cmd,
5951146d
AG		 3393 struct scatterlist *sgl,
3394 u32 sgl_count,
3395 struct scatterlist *sgl_bidi,
3396 u32 sgl_bidi_count)
c66ac9db 3397{
5951146d 3398 if (!sgl || !sgl_count)
c66ac9db 3399 return 0;
c66ac9db 3400
c66ac9db
NB		 3401 if ((cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) ||
3402 (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB)) {
fef58a60
NB		 3403 /*
3404 * Reject SCSI data overflow with map_mem_to_cmd() as incoming
3405 * scatterlists already have been set to follow what the fabric
3406 * passes for the original expected data transfer length.
3407 */
3408 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
3409 pr_warn("Rejecting SCSI DATA overflow for fabric using"
3410 " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n");
3411 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3412 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
3413 return -EINVAL;
3414 }
c66ac9db 3415
ec98f782
AG		 3416 cmd->t_data_sg = sgl;
3417 cmd->t_data_nents = sgl_count;
c66ac9db 3418
ec98f782
AG		 3419 if (sgl_bidi && sgl_bidi_count) {
3420 cmd->t_bidi_data_sg = sgl_bidi;
3421 cmd->t_bidi_data_nents = sgl_bidi_count;
c66ac9db
NB		 3422 }
3423 cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
c66ac9db
NB		 3424 }
3425
3426 return 0;
3427}
3428EXPORT_SYMBOL(transport_generic_map_mem_to_cmd);
3429
4949314c 3430void *transport_kmap_data_sg(struct se_cmd *cmd)
05d1c7c0 3431{
ec98f782 3432 struct scatterlist *sg = cmd->t_data_sg;
4949314c
AG		 3433 struct page **pages;
3434 int i;
05d1c7c0 3435
ec98f782 3436 BUG_ON(!sg);
05d1c7c0 3437 /*
ec98f782
AG		 3438 * We need to take into account a possible offset here for fabrics like
3439 * tcm_loop who may be using a contig buffer from the SCSI midlayer for
3440 * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd()
05d1c7c0 3441 */
4949314c
AG		 3442 if (!cmd->t_data_nents)
3443 return NULL;
3444 else if (cmd->t_data_nents == 1)
3445 return kmap(sg_page(sg)) + sg->offset;
3446
3447 /* >1 page. use vmap */
3448 pages = kmalloc(sizeof(*pages) * cmd->t_data_nents, GFP_KERNEL);
3449 if (!pages)
3450 return NULL;
3451
3452 /* convert sg[] to pages[] */
3453 for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) {
3454 pages[i] = sg_page(sg);
3455 }
3456
3457 cmd->t_data_vmap = vmap(pages, cmd->t_data_nents, VM_MAP, PAGE_KERNEL);
3458 kfree(pages);
3459 if (!cmd->t_data_vmap)
3460 return NULL;
3461
3462 return cmd->t_data_vmap + cmd->t_data_sg[0].offset;
05d1c7c0 3463}
4949314c 3464EXPORT_SYMBOL(transport_kmap_data_sg);
05d1c7c0 3465
4949314c 3466void transport_kunmap_data_sg(struct se_cmd *cmd)
05d1c7c0 3467{
a1edf9cf 3468 if (!cmd->t_data_nents) {
4949314c 3469 return;
a1edf9cf 3470 } else if (cmd->t_data_nents == 1) {
4949314c 3471 kunmap(sg_page(cmd->t_data_sg));
a1edf9cf
AG		 3472 return;
3473 }
4949314c
AG		 3474
3475 vunmap(cmd->t_data_vmap);
3476 cmd->t_data_vmap = NULL;
05d1c7c0 3477}
4949314c 3478EXPORT_SYMBOL(transport_kunmap_data_sg);
05d1c7c0 3479
c66ac9db 3480static int
05d1c7c0 3481transport_generic_get_mem(struct se_cmd *cmd)
c66ac9db 3482{
ec98f782
AG		 3483 u32 length = cmd->data_length;
3484 unsigned int nents;
3485 struct page *page;
9db9da33 3486 gfp_t zero_flag;
ec98f782 3487 int i = 0;
c66ac9db 3488
ec98f782
AG		 3489 nents = DIV_ROUND_UP(length, PAGE_SIZE);
3490 cmd->t_data_sg = kmalloc(sizeof(struct scatterlist) * nents, GFP_KERNEL);
3491 if (!cmd->t_data_sg)
3492 return -ENOMEM;
c66ac9db 3493
ec98f782
AG		 3494 cmd->t_data_nents = nents;
3495 sg_init_table(cmd->t_data_sg, nents);
c66ac9db 3496
9db9da33 3497 zero_flag = cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB ? 0 : __GFP_ZERO;
3498
ec98f782
AG		 3499 while (length) {
3500 u32 page_len = min_t(u32, length, PAGE_SIZE);
9db9da33 3501 page = alloc_page(GFP_KERNEL | zero_flag);
ec98f782
AG		 3502 if (!page)
3503 goto out;
c66ac9db 3504
ec98f782
AG		 3505 sg_set_page(&cmd->t_data_sg[i], page, page_len, 0);
3506 length -= page_len;
3507 i++;
c66ac9db 3508 }
c66ac9db 3509 return 0;
c66ac9db 3510
ec98f782
AG		 3511out:
3512 while (i >= 0) {
3513 __free_page(sg_page(&cmd->t_data_sg[i]));
3514 i--;
c66ac9db 3515 }
ec98f782
AG		 3516 kfree(cmd->t_data_sg);
3517 cmd->t_data_sg = NULL;
3518 return -ENOMEM;
c66ac9db
NB		 3519}
3520
da0f7619 3521/*
b16a35b0
AG		 3522 * Allocate any required resources to execute the command. For writes we
3523 * might not have the payload yet, so notify the fabric via a call to
3524 * ->write_pending instead. Otherwise place it on the execution queue.
c66ac9db 3525 */
a1d8b49a 3526int transport_generic_new_cmd(struct se_cmd *cmd)
c66ac9db 3527{
da0f7619 3528 struct se_device *dev = cmd->se_dev;
4101f0a8 3529 struct se_task *task;
c66ac9db
NB		 3530 int ret = 0;
3531
3532 /*
3533 * Determine is the TCM fabric module has already allocated physical
3534 * memory, and is directly calling transport_generic_map_mem_to_cmd()
ec98f782 3535 * beforehand.
c66ac9db 3536 */
ec98f782
AG		 3537 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) &&
3538 cmd->data_length) {
05d1c7c0 3539 ret = transport_generic_get_mem(cmd);
c66ac9db 3540 if (ret < 0)
03e98c9e 3541 goto out_fail;
c66ac9db 3542 }
da0f7619 3543
4101f0a8
CH		 3544 /* Workaround for handling zero-length control CDBs */
3545 if ((cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB) &&
3546 !cmd->data_length) {
7d680f3b 3547 spin_lock_irq(&cmd->t_state_lock);
410f6702 3548 cmd->t_state = TRANSPORT_COMPLETE;
7d680f3b
CH		 3549 cmd->transport_state |= CMD_T_ACTIVE;
3550 spin_unlock_irq(&cmd->t_state_lock);
91ec1d35
NB		 3551
3552 if (cmd->t_task_cdb[0] == REQUEST_SENSE) {
3553 u8 ua_asc = 0, ua_ascq = 0;
3554
3555 core_scsi3_ua_clear_for_request_sense(cmd,
3556 &ua_asc, &ua_ascq);
3557 }
3558
410f6702
RD		 3559 INIT_WORK(&cmd->work, target_complete_ok_work);
3560 queue_work(target_completion_wq, &cmd->work);
3561 return 0;
3562 }
da0f7619 3563
4101f0a8
CH		 3564 if (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) {
3565 struct se_dev_attrib *attr = &dev->se_sub_dev->se_dev_attrib;
3566
3567 if (transport_cmd_get_valid_sectors(cmd) < 0)
3568 return -EINVAL;
3569
3570 BUG_ON(cmd->data_length % attr->block_size);
3571 BUG_ON(DIV_ROUND_UP(cmd->data_length, attr->block_size) >
3572 attr->max_sectors);
da0f7619
CH		 3573 }
3574
4101f0a8
CH		 3575 task = dev->transport->alloc_task(cmd->t_task_cdb);
3576 if (!task) {
3577 pr_err("Unable to allocate struct se_task\n");
3578 goto out_fail;
3579 }
3580
4101f0a8
CH		 3581 task->task_se_cmd = cmd;
3582 task->task_data_direction = cmd->data_direction;
3583 task->task_sg = cmd->t_data_sg;
3584 task->task_sg_nents = cmd->t_data_nents;
3585
785fdf70 3586 cmd->t_task = task;
4101f0a8
CH		 3587
3588 atomic_inc(&cmd->t_fe_count);
3589 atomic_inc(&cmd->t_se_count);
3590
785fdf70
CH		 3591 atomic_set(&cmd->t_task_cdbs_left, 1);
3592 atomic_set(&cmd->t_task_cdbs_ex_left, 1);
da0f7619 3593
c66ac9db 3594 /*
a1d8b49a 3595 * For WRITEs, let the fabric know its buffer is ready..
c66ac9db
NB		 3596 * This WRITE struct se_cmd (and all of its associated struct se_task's)
3597 * will be added to the struct se_device execution queue after its WRITE
3598 * data has arrived. (ie: It gets handled by the transport processing
3599 * thread a second time)
3600 */
3601 if (cmd->data_direction == DMA_TO_DEVICE) {
cf572a96 3602 target_add_to_state_list(cmd);
c66ac9db
NB		 3603 return transport_generic_write_pending(cmd);
3604 }
3605 /*
3606 * Everything else but a WRITE, add the struct se_cmd's struct se_task's
3607 * to the execution queue.
3608 */
3609 transport_execute_tasks(cmd);
3610 return 0;
da0f7619
CH		 3611
3612out_fail:
3613 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3614 cmd->scsi_sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
3615 return -EINVAL;
c66ac9db 3616}
a1d8b49a 3617EXPORT_SYMBOL(transport_generic_new_cmd);
c66ac9db
NB		 3618
3619/* transport_generic_process_write():
3620 *
3621 *
3622 */
3623void transport_generic_process_write(struct se_cmd *cmd)
3624{
c66ac9db
NB		 3625 transport_execute_tasks(cmd);
3626}
3627EXPORT_SYMBOL(transport_generic_process_write);
3628
e057f533 3629static void transport_write_pending_qf(struct se_cmd *cmd)
07bde79a 3630{
f147abb4
NB		 3631 int ret;
3632
3633 ret = cmd->se_tfo->write_pending(cmd);
3634 if (ret == -EAGAIN || ret == -ENOMEM) {
e057f533
CH		 3635 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n",
3636 cmd);
3637 transport_handle_queue_full(cmd, cmd->se_dev);
3638 }
07bde79a
NB		 3639}
3640
c66ac9db
NB		 3641static int transport_generic_write_pending(struct se_cmd *cmd)
3642{
3643 unsigned long flags;
3644 int ret;
3645
a1d8b49a 3646 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db 3647 cmd->t_state = TRANSPORT_WRITE_PENDING;
a1d8b49a 3648 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
07bde79a 3649
c66ac9db
NB		 3650 /*
3651 * Clear the se_cmd for WRITE_PENDING status in order to set
7d680f3b
CH		 3652 * CMD_T_ACTIVE so that transport_generic_handle_data can be called
3653 * from HW target mode interrupt code. This is safe to be called
3654 * with transport_off=1 before the cmd->se_tfo->write_pending
c66ac9db
NB		 3655 * because the se_cmd->se_lun pointer is not being cleared.
3656 */
3657 transport_cmd_check_stop(cmd, 1, 0);
3658
3659 /*
3660 * Call the fabric write_pending function here to let the
3661 * frontend know that WRITE buffers are ready.
3662 */
e3d6f909 3663 ret = cmd->se_tfo->write_pending(cmd);
f147abb4 3664 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a
NB		 3665 goto queue_full;
3666 else if (ret < 0)
c66ac9db
NB		 3667 return ret;
3668
03e98c9e 3669 return 1;
07bde79a
NB		 3670
3671queue_full:
6708bb27 3672 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd);
07bde79a 3673 cmd->t_state = TRANSPORT_COMPLETE_QF_WP;
e057f533 3674 transport_handle_queue_full(cmd, cmd->se_dev);
f147abb4 3675 return 0;
c66ac9db
NB		 3676}
3677
39c05f32 3678void transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
c66ac9db 3679{
d14921d6 3680 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) {
c8e31f26 3681 if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
d14921d6
NB		 3682 transport_wait_for_tasks(cmd);
3683
35462975 3684 transport_release_cmd(cmd);
d14921d6
NB		 3685 } else {
3686 if (wait_for_tasks)
3687 transport_wait_for_tasks(cmd);
3688
c66ac9db
NB		 3689 core_dec_lacl_count(cmd->se_sess->se_node_acl, cmd);
3690
82f1c8a4 3691 if (cmd->se_lun)
c66ac9db 3692 transport_lun_remove_cmd(cmd);
c66ac9db 3693
f4366772
NB		 3694 transport_free_dev_tasks(cmd);
3695
39c05f32 3696 transport_put_cmd(cmd);
c66ac9db
NB		 3697 }
3698}
3699EXPORT_SYMBOL(transport_generic_free_cmd);
3700
a17f091d
NB		 3701/* target_get_sess_cmd - Add command to active ->sess_cmd_list
3702 * @se_sess: session to reference
3703 * @se_cmd: command descriptor to add
a6360785 3704 * @ack_kref: Signal that fabric will perform an ack target_put_sess_cmd()
a17f091d 3705 */
a6360785
NB		 3706void target_get_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd,
3707 bool ack_kref)
a17f091d
NB		 3708{
3709 unsigned long flags;
3710
7481deb4 3711 kref_init(&se_cmd->cmd_kref);
a6360785
NB		 3712 /*
3713 * Add a second kref if the fabric caller is expecting to handle
3714 * fabric acknowledgement that requires two target_put_sess_cmd()
3715 * invocations before se_cmd descriptor release.
3716 */
86715569 3717 if (ack_kref == true) {
a6360785 3718 kref_get(&se_cmd->cmd_kref);
86715569
NB		 3719 se_cmd->se_cmd_flags |= SCF_ACK_KREF;
3720 }
7481deb4 3721
a17f091d
NB		 3722 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
3723 list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list);
3724 se_cmd->check_release = 1;
3725 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
3726}
3727EXPORT_SYMBOL(target_get_sess_cmd);
3728
7481deb4 3729static void target_release_cmd_kref(struct kref *kref)
a17f091d 3730{
7481deb4
NB		 3731 struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
3732 struct se_session *se_sess = se_cmd->se_sess;
a17f091d
NB		 3733 unsigned long flags;
3734
3735 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
3736 if (list_empty(&se_cmd->se_cmd_list)) {
3737 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
ffc32d52 3738 se_cmd->se_tfo->release_cmd(se_cmd);
7481deb4 3739 return;
a17f091d 3740 }
a17f091d
NB		 3741 if (se_sess->sess_tearing_down && se_cmd->cmd_wait_set) {
3742 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
3743 complete(&se_cmd->cmd_wait_comp);
7481deb4 3744 return;
a17f091d
NB		 3745 }
3746 list_del(&se_cmd->se_cmd_list);
3747 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
3748
7481deb4
NB		 3749 se_cmd->se_tfo->release_cmd(se_cmd);
3750}
3751
3752/* target_put_sess_cmd - Check for active I/O shutdown via kref_put
3753 * @se_sess: session to reference
3754 * @se_cmd: command descriptor to drop
3755 */
3756int target_put_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd)
3757{
3758 return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref);
a17f091d
NB		 3759}
3760EXPORT_SYMBOL(target_put_sess_cmd);
3761
3762/* target_splice_sess_cmd_list - Split active cmds into sess_wait_list
3763 * @se_sess: session to split
3764 */
3765void target_splice_sess_cmd_list(struct se_session *se_sess)
3766{
3767 struct se_cmd *se_cmd;
3768 unsigned long flags;
3769
3770 WARN_ON(!list_empty(&se_sess->sess_wait_list));
3771 INIT_LIST_HEAD(&se_sess->sess_wait_list);
3772
3773 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
3774 se_sess->sess_tearing_down = 1;
3775
3776 list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list);
3777
3778 list_for_each_entry(se_cmd, &se_sess->sess_wait_list, se_cmd_list)
3779 se_cmd->cmd_wait_set = 1;
3780
3781 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
3782}
3783EXPORT_SYMBOL(target_splice_sess_cmd_list);
3784
3785/* target_wait_for_sess_cmds - Wait for outstanding descriptors
3786 * @se_sess: session to wait for active I/O
3787 * @wait_for_tasks: Make extra transport_wait_for_tasks call
3788 */
3789void target_wait_for_sess_cmds(
3790 struct se_session *se_sess,
3791 int wait_for_tasks)
3792{
3793 struct se_cmd *se_cmd, *tmp_cmd;
3794 bool rc = false;
3795
3796 list_for_each_entry_safe(se_cmd, tmp_cmd,
3797 &se_sess->sess_wait_list, se_cmd_list) {
3798 list_del(&se_cmd->se_cmd_list);
3799
3800 pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:"
3801 " %d\n", se_cmd, se_cmd->t_state,
3802 se_cmd->se_tfo->get_cmd_state(se_cmd));
3803
3804 if (wait_for_tasks) {
3805 pr_debug("Calling transport_wait_for_tasks se_cmd: %p t_state: %d,"
3806 " fabric state: %d\n", se_cmd, se_cmd->t_state,
3807 se_cmd->se_tfo->get_cmd_state(se_cmd));
3808
3809 rc = transport_wait_for_tasks(se_cmd);
3810
3811 pr_debug("After transport_wait_for_tasks se_cmd: %p t_state: %d,"
3812 " fabric state: %d\n", se_cmd, se_cmd->t_state,
3813 se_cmd->se_tfo->get_cmd_state(se_cmd));
3814 }
3815
3816 if (!rc) {
3817 wait_for_completion(&se_cmd->cmd_wait_comp);
3818 pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d"
3819 " fabric state: %d\n", se_cmd, se_cmd->t_state,
3820 se_cmd->se_tfo->get_cmd_state(se_cmd));
3821 }
3822
3823 se_cmd->se_tfo->release_cmd(se_cmd);
3824 }
3825}
3826EXPORT_SYMBOL(target_wait_for_sess_cmds);
3827
c66ac9db
NB		 3828/* transport_lun_wait_for_tasks():
3829 *
3830 * Called from ConfigFS context to stop the passed struct se_cmd to allow
3831 * an struct se_lun to be successfully shutdown.
3832 */
3833static int transport_lun_wait_for_tasks(struct se_cmd *cmd, struct se_lun *lun)
3834{
cf572a96 3835 struct se_task *task = cmd->t_task;
c66ac9db 3836 unsigned long flags;
cf572a96
CH		 3837 int ret = 0;
3838
c66ac9db
NB		 3839 /*
3840 * If the frontend has already requested this struct se_cmd to
3841 * be stopped, we can safely ignore this struct se_cmd.
3842 */
a1d8b49a 3843 spin_lock_irqsave(&cmd->t_state_lock, flags);
7d680f3b
CH		 3844 if (cmd->transport_state & CMD_T_STOP) {
3845 cmd->transport_state &= ~CMD_T_LUN_STOP;
3846
3847 pr_debug("ConfigFS ITT[0x%08x] - CMD_T_STOP, skipping\n",
3848 cmd->se_tfo->get_task_tag(cmd));
a1d8b49a 3849 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 3850 transport_cmd_check_stop(cmd, 1, 0);
e3d6f909 3851 return -EPERM;
c66ac9db 3852 }
7d680f3b 3853 cmd->transport_state |= CMD_T_LUN_FE_STOP;
a1d8b49a 3854 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 3855
5951146d 3856 wake_up_interruptible(&cmd->se_dev->dev_queue_obj.thread_wq);
c66ac9db 3857
cf572a96
CH		 3858 // XXX: audit task_flags checks.
3859 spin_lock_irqsave(&cmd->t_state_lock, flags);
3860 if ((cmd->transport_state & CMD_T_BUSY) &&
3861 (cmd->transport_state & CMD_T_SENT)) {
3862 if (!target_stop_cmd(cmd, &flags))
3863 ret++;
3864 spin_lock_irqsave(&cmd->t_state_lock, flags);
3865 } else {
3866 spin_unlock_irqrestore(&cmd->t_state_lock,
3867 flags);
3868 target_remove_from_execute_list(cmd);
3869 }
c66ac9db 3870
785fdf70
CH		 3871 pr_debug("ConfigFS: cmd: %p stop tasks ret:"
3872 " %d\n", cmd, ret);
c66ac9db 3873 if (!ret) {
6708bb27 3874 pr_debug("ConfigFS: ITT[0x%08x] - stopping cmd....\n",
e3d6f909 3875 cmd->se_tfo->get_task_tag(cmd));
a1d8b49a 3876 wait_for_completion(&cmd->transport_lun_stop_comp);
6708bb27 3877 pr_debug("ConfigFS: ITT[0x%08x] - stopped cmd....\n",
e3d6f909 3878 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 3879 }
3df8d40b 3880 transport_remove_cmd_from_queue(cmd);
c66ac9db
NB		 3881
3882 return 0;
3883}
3884
c66ac9db
NB		 3885static void __transport_clear_lun_from_sessions(struct se_lun *lun)
3886{
3887 struct se_cmd *cmd = NULL;
3888 unsigned long lun_flags, cmd_flags;
3889 /*
3890 * Do exception processing and return CHECK_CONDITION status to the
3891 * Initiator Port.
3892 */
3893 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
5951146d
AG		 3894 while (!list_empty(&lun->lun_cmd_list)) {
3895 cmd = list_first_entry(&lun->lun_cmd_list,
3896 struct se_cmd, se_lun_node);
3d26fea0 3897 list_del_init(&cmd->se_lun_node);
5951146d 3898
c66ac9db
NB		 3899 /*
3900 * This will notify iscsi_target_transport.c:
3901 * transport_cmd_check_stop() that a LUN shutdown is in
3902 * progress for the iscsi_cmd_t.
3903 */
a1d8b49a 3904 spin_lock(&cmd->t_state_lock);
6708bb27 3905 pr_debug("SE_LUN[%d] - Setting cmd->transport"
c66ac9db 3906 "_lun_stop for ITT: 0x%08x\n",
e3d6f909
AG		 3907 cmd->se_lun->unpacked_lun,
3908 cmd->se_tfo->get_task_tag(cmd));
7d680f3b 3909 cmd->transport_state |= CMD_T_LUN_STOP;
a1d8b49a 3910 spin_unlock(&cmd->t_state_lock);
c66ac9db
NB		 3911
3912 spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
3913
6708bb27
AG		 3914 if (!cmd->se_lun) {
3915 pr_err("ITT: 0x%08x, [i,t]_state: %u/%u\n",
e3d6f909
AG		 3916 cmd->se_tfo->get_task_tag(cmd),
3917 cmd->se_tfo->get_cmd_state(cmd), cmd->t_state);
c66ac9db
NB		 3918 BUG();
3919 }
3920 /*
3921 * If the Storage engine still owns the iscsi_cmd_t, determine
3922 * and/or stop its context.
3923 */
6708bb27 3924 pr_debug("SE_LUN[%d] - ITT: 0x%08x before transport"
e3d6f909
AG		 3925 "_lun_wait_for_tasks()\n", cmd->se_lun->unpacked_lun,
3926 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 3927
e3d6f909 3928 if (transport_lun_wait_for_tasks(cmd, cmd->se_lun) < 0) {
c66ac9db
NB		 3929 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
3930 continue;
3931 }
3932
6708bb27 3933 pr_debug("SE_LUN[%d] - ITT: 0x%08x after transport_lun"
c66ac9db 3934 "_wait_for_tasks(): SUCCESS\n",
e3d6f909
AG		 3935 cmd->se_lun->unpacked_lun,
3936 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 3937
a1d8b49a 3938 spin_lock_irqsave(&cmd->t_state_lock, cmd_flags);
7d680f3b 3939 if (!(cmd->transport_state & CMD_T_DEV_ACTIVE)) {
a1d8b49a 3940 spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
c66ac9db
NB		 3941 goto check_cond;
3942 }
7d680f3b 3943 cmd->transport_state &= ~CMD_T_DEV_ACTIVE;
cf572a96 3944 target_remove_from_state_list(cmd);
a1d8b49a 3945 spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
c66ac9db
NB		 3946
3947 transport_free_dev_tasks(cmd);
3948 /*
3949 * The Storage engine stopped this struct se_cmd before it was
3950 * send to the fabric frontend for delivery back to the
3951 * Initiator Node. Return this SCSI CDB back with an
3952 * CHECK_CONDITION status.
3953 */
3954check_cond:
3955 transport_send_check_condition_and_sense(cmd,
3956 TCM_NON_EXISTENT_LUN, 0);
3957 /*
3958 * If the fabric frontend is waiting for this iscsi_cmd_t to
3959 * be released, notify the waiting thread now that LU has
3960 * finished accessing it.
3961 */
a1d8b49a 3962 spin_lock_irqsave(&cmd->t_state_lock, cmd_flags);
7d680f3b 3963 if (cmd->transport_state & CMD_T_LUN_FE_STOP) {
6708bb27 3964 pr_debug("SE_LUN[%d] - Detected FE stop for"
c66ac9db
NB		 3965 " struct se_cmd: %p ITT: 0x%08x\n",
3966 lun->unpacked_lun,
e3d6f909 3967 cmd, cmd->se_tfo->get_task_tag(cmd));
c66ac9db 3968
a1d8b49a 3969 spin_unlock_irqrestore(&cmd->t_state_lock,
c66ac9db
NB		 3970 cmd_flags);
3971 transport_cmd_check_stop(cmd, 1, 0);
a1d8b49a 3972 complete(&cmd->transport_lun_fe_stop_comp);
c66ac9db
NB		 3973 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
3974 continue;
3975 }
6708bb27 3976 pr_debug("SE_LUN[%d] - ITT: 0x%08x finished processing\n",
e3d6f909 3977 lun->unpacked_lun, cmd->se_tfo->get_task_tag(cmd));
c66ac9db 3978
a1d8b49a 3979 spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
c66ac9db
NB		 3980 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
3981 }
3982 spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
3983}
3984
3985static int transport_clear_lun_thread(void *p)
3986{
8359cf43 3987 struct se_lun *lun = p;
c66ac9db
NB		 3988
3989 __transport_clear_lun_from_sessions(lun);
3990 complete(&lun->lun_shutdown_comp);
3991
3992 return 0;
3993}
3994
3995int transport_clear_lun_from_sessions(struct se_lun *lun)
3996{
3997 struct task_struct *kt;
3998
5951146d 3999 kt = kthread_run(transport_clear_lun_thread, lun,
c66ac9db
NB		 4000 "tcm_cl_%u", lun->unpacked_lun);
4001 if (IS_ERR(kt)) {
6708bb27 4002 pr_err("Unable to start clear_lun thread\n");
e3d6f909 4003 return PTR_ERR(kt);
c66ac9db
NB		 4004 }
4005 wait_for_completion(&lun->lun_shutdown_comp);
4006
4007 return 0;
4008}
4009
d14921d6
NB		 4010/**
4011 * transport_wait_for_tasks - wait for completion to occur
4012 * @cmd: command to wait
c66ac9db 4013 *
d14921d6
NB		 4014 * Called from frontend fabric context to wait for storage engine
4015 * to pause and/or release frontend generated struct se_cmd.
c66ac9db 4016 */
a17f091d 4017bool transport_wait_for_tasks(struct se_cmd *cmd)
c66ac9db
NB		 4018{
4019 unsigned long flags;
4020
a1d8b49a 4021 spin_lock_irqsave(&cmd->t_state_lock, flags);
c8e31f26
AG		 4022 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) &&
4023 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
d14921d6 4024 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
a17f091d 4025 return false;
d14921d6
NB		 4026 }
4027 /*
4028 * Only perform a possible wait_for_tasks if SCF_SUPPORTED_SAM_OPCODE
4029 * has been set in transport_set_supported_SAM_opcode().
4030 */
c8e31f26
AG		 4031 if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) &&
4032 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
d14921d6 4033 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
a17f091d 4034 return false;
d14921d6 4035 }
c66ac9db
NB		 4036 /*
4037 * If we are already stopped due to an external event (ie: LUN shutdown)
4038 * sleep until the connection can have the passed struct se_cmd back.
a1d8b49a 4039 * The cmd->transport_lun_stopped_sem will be upped by
c66ac9db
NB		 4040 * transport_clear_lun_from_sessions() once the ConfigFS context caller
4041 * has completed its operation on the struct se_cmd.
4042 */
7d680f3b 4043 if (cmd->transport_state & CMD_T_LUN_STOP) {
6708bb27 4044 pr_debug("wait_for_tasks: Stopping"
e3d6f909 4045 " wait_for_completion(&cmd->t_tasktransport_lun_fe"
c66ac9db 4046 "_stop_comp); for ITT: 0x%08x\n",
e3d6f909 4047 cmd->se_tfo->get_task_tag(cmd));
c66ac9db
NB		 4048 /*
4049 * There is a special case for WRITES where a FE exception +
4050 * LUN shutdown means ConfigFS context is still sleeping on
4051 * transport_lun_stop_comp in transport_lun_wait_for_tasks().
4052 * We go ahead and up transport_lun_stop_comp just to be sure
4053 * here.
4054 */
a1d8b49a
AG		 4055 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4056 complete(&cmd->transport_lun_stop_comp);
4057 wait_for_completion(&cmd->transport_lun_fe_stop_comp);
4058 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db 4059
cf572a96 4060 target_remove_from_state_list(cmd);
c66ac9db
NB		 4061 /*
4062 * At this point, the frontend who was the originator of this
4063 * struct se_cmd, now owns the structure and can be released through
4064 * normal means below.
4065 */
6708bb27 4066 pr_debug("wait_for_tasks: Stopped"
e3d6f909 4067 " wait_for_completion(&cmd->t_tasktransport_lun_fe_"
c66ac9db 4068 "stop_comp); for ITT: 0x%08x\n",
e3d6f909 4069 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 4070
7d680f3b 4071 cmd->transport_state &= ~CMD_T_LUN_STOP;
c66ac9db 4072 }
7d680f3b 4073
3d28934a 4074 if (!(cmd->transport_state & CMD_T_ACTIVE)) {
d14921d6 4075 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
a17f091d 4076 return false;
d14921d6 4077 }
c66ac9db 4078
7d680f3b 4079 cmd->transport_state |= CMD_T_STOP;
c66ac9db 4080
6708bb27 4081 pr_debug("wait_for_tasks: Stopping %p ITT: 0x%08x"
7d680f3b 4082 " i_state: %d, t_state: %d, CMD_T_STOP\n",
f2da9dbd
CH		 4083 cmd, cmd->se_tfo->get_task_tag(cmd),
4084 cmd->se_tfo->get_cmd_state(cmd), cmd->t_state);
c66ac9db 4085
a1d8b49a 4086 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 4087
5951146d 4088 wake_up_interruptible(&cmd->se_dev->dev_queue_obj.thread_wq);
c66ac9db 4089
a1d8b49a 4090 wait_for_completion(&cmd->t_transport_stop_comp);
c66ac9db 4091
a1d8b49a 4092 spin_lock_irqsave(&cmd->t_state_lock, flags);
7d680f3b 4093 cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP);
c66ac9db 4094
6708bb27 4095 pr_debug("wait_for_tasks: Stopped wait_for_compltion("
a1d8b49a 4096 "&cmd->t_transport_stop_comp) for ITT: 0x%08x\n",
e3d6f909 4097 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 4098
d14921d6 4099 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
a17f091d
NB		 4100
4101 return true;
c66ac9db 4102}
d14921d6 4103EXPORT_SYMBOL(transport_wait_for_tasks);
c66ac9db
NB		 4104
4105static int transport_get_sense_codes(
4106 struct se_cmd *cmd,
4107 u8 *asc,
4108 u8 *ascq)
4109{
4110 *asc = cmd->scsi_asc;
4111 *ascq = cmd->scsi_ascq;
4112
4113 return 0;
4114}
4115
4116static int transport_set_sense_codes(
4117 struct se_cmd *cmd,
4118 u8 asc,
4119 u8 ascq)
4120{
4121 cmd->scsi_asc = asc;
4122 cmd->scsi_ascq = ascq;
4123
4124 return 0;
4125}
4126
4127int transport_send_check_condition_and_sense(
4128 struct se_cmd *cmd,
4129 u8 reason,
4130 int from_transport)
4131{
4132 unsigned char *buffer = cmd->sense_buffer;
4133 unsigned long flags;
4134 int offset;
4135 u8 asc = 0, ascq = 0;
4136
a1d8b49a 4137 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db 4138 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
a1d8b49a 4139 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 4140 return 0;
4141 }
4142 cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
a1d8b49a 4143 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 4144
4145 if (!reason && from_transport)
4146 goto after_reason;
4147
4148 if (!from_transport)
4149 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
4150 /*
4151 * Data Segment and SenseLength of the fabric response PDU.
4152 *
4153 * TRANSPORT_SENSE_BUFFER is now set to SCSI_SENSE_BUFFERSIZE
4154 * from include/scsi/scsi_cmnd.h
4155 */
e3d6f909 4156 offset = cmd->se_tfo->set_fabric_sense_len(cmd,
c66ac9db
NB		 4157 TRANSPORT_SENSE_BUFFER);
4158 /*
4159 * Actual SENSE DATA, see SPC-3 7.23.2 SPC_SENSE_KEY_OFFSET uses
4160 * SENSE KEY values from include/scsi/scsi.h
4161 */
4162 switch (reason) {
4163 case TCM_NON_EXISTENT_LUN:
eb39d340
NB		 4164 /* CURRENT ERROR */
4165 buffer[offset] = 0x70;
895f3022 4166 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
eb39d340
NB		 4167 /* ILLEGAL REQUEST */
4168 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
4169 /* LOGICAL UNIT NOT SUPPORTED */
4170 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x25;
4171 break;
c66ac9db
NB		 4172 case TCM_UNSUPPORTED_SCSI_OPCODE:
4173 case TCM_SECTOR_COUNT_TOO_MANY:
4174 /* CURRENT ERROR */
4175 buffer[offset] = 0x70;
895f3022 4176 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4177 /* ILLEGAL REQUEST */
4178 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
4179 /* INVALID COMMAND OPERATION CODE */
4180 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x20;
4181 break;
4182 case TCM_UNKNOWN_MODE_PAGE:
4183 /* CURRENT ERROR */
4184 buffer[offset] = 0x70;
895f3022 4185 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4186 /* ILLEGAL REQUEST */
4187 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
4188 /* INVALID FIELD IN CDB */
4189 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24;
4190 break;
4191 case TCM_CHECK_CONDITION_ABORT_CMD:
4192 /* CURRENT ERROR */
4193 buffer[offset] = 0x70;
895f3022 4194 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4195 /* ABORTED COMMAND */
4196 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
4197 /* BUS DEVICE RESET FUNCTION OCCURRED */
4198 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x29;
4199 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x03;
4200 break;
4201 case TCM_INCORRECT_AMOUNT_OF_DATA:
4202 /* CURRENT ERROR */
4203 buffer[offset] = 0x70;
895f3022 4204 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4205 /* ABORTED COMMAND */
4206 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
4207 /* WRITE ERROR */
4208 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c;
4209 /* NOT ENOUGH UNSOLICITED DATA */
4210 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0d;
4211 break;
4212 case TCM_INVALID_CDB_FIELD:
4213 /* CURRENT ERROR */
4214 buffer[offset] = 0x70;
895f3022 4215 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
9fbc8909
RD		 4216 /* ILLEGAL REQUEST */
4217 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
c66ac9db
NB		 4218 /* INVALID FIELD IN CDB */
4219 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24;
4220 break;
4221 case TCM_INVALID_PARAMETER_LIST:
4222 /* CURRENT ERROR */
4223 buffer[offset] = 0x70;
895f3022 4224 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
9fbc8909
RD		 4225 /* ILLEGAL REQUEST */
4226 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
c66ac9db
NB		 4227 /* INVALID FIELD IN PARAMETER LIST */
4228 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x26;
4229 break;
4230 case TCM_UNEXPECTED_UNSOLICITED_DATA:
4231 /* CURRENT ERROR */
4232 buffer[offset] = 0x70;
895f3022 4233 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4234 /* ABORTED COMMAND */
4235 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
4236 /* WRITE ERROR */
4237 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c;
4238 /* UNEXPECTED_UNSOLICITED_DATA */
4239 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0c;
4240 break;
4241 case TCM_SERVICE_CRC_ERROR:
4242 /* CURRENT ERROR */
4243 buffer[offset] = 0x70;
895f3022 4244 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4245 /* ABORTED COMMAND */
4246 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
4247 /* PROTOCOL SERVICE CRC ERROR */
4248 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x47;
4249 /* N/A */
4250 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x05;
4251 break;
4252 case TCM_SNACK_REJECTED:
4253 /* CURRENT ERROR */
4254 buffer[offset] = 0x70;
895f3022 4255 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4256 /* ABORTED COMMAND */
4257 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
4258 /* READ ERROR */
4259 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x11;
4260 /* FAILED RETRANSMISSION REQUEST */
4261 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x13;
4262 break;
4263 case TCM_WRITE_PROTECTED:
4264 /* CURRENT ERROR */
4265 buffer[offset] = 0x70;
895f3022 4266 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4267 /* DATA PROTECT */
4268 buffer[offset+SPC_SENSE_KEY_OFFSET] = DATA_PROTECT;
4269 /* WRITE PROTECTED */
4270 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x27;
4271 break;
4272 case TCM_CHECK_CONDITION_UNIT_ATTENTION:
4273 /* CURRENT ERROR */
4274 buffer[offset] = 0x70;
895f3022 4275 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4276 /* UNIT ATTENTION */
4277 buffer[offset+SPC_SENSE_KEY_OFFSET] = UNIT_ATTENTION;
4278 core_scsi3_ua_for_check_condition(cmd, &asc, &ascq);
4279 buffer[offset+SPC_ASC_KEY_OFFSET] = asc;
4280 buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq;
4281 break;
4282 case TCM_CHECK_CONDITION_NOT_READY:
4283 /* CURRENT ERROR */
4284 buffer[offset] = 0x70;
895f3022 4285 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4286 /* Not Ready */
4287 buffer[offset+SPC_SENSE_KEY_OFFSET] = NOT_READY;
4288 transport_get_sense_codes(cmd, &asc, &ascq);
4289 buffer[offset+SPC_ASC_KEY_OFFSET] = asc;
4290 buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq;
4291 break;
4292 case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
4293 default:
4294 /* CURRENT ERROR */
4295 buffer[offset] = 0x70;
895f3022 4296 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4297 /* ILLEGAL REQUEST */
4298 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
4299 /* LOGICAL UNIT COMMUNICATION FAILURE */
4300 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x80;
4301 break;
4302 }
4303 /*
4304 * This code uses linux/include/scsi/scsi.h SAM status codes!
4305 */
4306 cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
4307 /*
4308 * Automatically padded, this value is encoded in the fabric's
4309 * data_length response PDU containing the SCSI defined sense data.
4310 */
4311 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER + offset;
4312
4313after_reason:
07bde79a 4314 return cmd->se_tfo->queue_status(cmd);
c66ac9db
NB		 4315}
4316EXPORT_SYMBOL(transport_send_check_condition_and_sense);
4317
4318int transport_check_aborted_status(struct se_cmd *cmd, int send_status)
4319{
4320 int ret = 0;
4321
7d680f3b 4322 if (cmd->transport_state & CMD_T_ABORTED) {
6708bb27 4323 if (!send_status ||
c66ac9db
NB		 4324 (cmd->se_cmd_flags & SCF_SENT_DELAYED_TAS))
4325 return 1;
8b1e1244 4326
6708bb27 4327 pr_debug("Sending delayed SAM_STAT_TASK_ABORTED"
c66ac9db 4328 " status for CDB: 0x%02x ITT: 0x%08x\n",
a1d8b49a 4329 cmd->t_task_cdb[0],
e3d6f909 4330 cmd->se_tfo->get_task_tag(cmd));
8b1e1244 4331
c66ac9db 4332 cmd->se_cmd_flags |= SCF_SENT_DELAYED_TAS;
e3d6f909 4333 cmd->se_tfo->queue_status(cmd);
c66ac9db
NB		 4334 ret = 1;
4335 }
4336 return ret;
4337}
4338EXPORT_SYMBOL(transport_check_aborted_status);
4339
4340void transport_send_task_abort(struct se_cmd *cmd)
4341{
c252f003
NB		 4342 unsigned long flags;
4343
4344 spin_lock_irqsave(&cmd->t_state_lock, flags);
4345 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
4346 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4347 return;
4348 }
4349 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4350
c66ac9db
NB		 4351 /*
4352 * If there are still expected incoming fabric WRITEs, we wait
4353 * until until they have completed before sending a TASK_ABORTED
4354 * response. This response with TASK_ABORTED status will be
4355 * queued back to fabric module by transport_check_aborted_status().
4356 */
4357 if (cmd->data_direction == DMA_TO_DEVICE) {
e3d6f909 4358 if (cmd->se_tfo->write_pending_status(cmd) != 0) {
7d680f3b 4359 cmd->transport_state |= CMD_T_ABORTED;
c66ac9db 4360 smp_mb__after_atomic_inc();
c66ac9db
NB		 4361 }
4362 }
4363 cmd->scsi_status = SAM_STAT_TASK_ABORTED;
8b1e1244 4364
6708bb27 4365 pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x,"
a1d8b49a 4366 " ITT: 0x%08x\n", cmd->t_task_cdb[0],
e3d6f909 4367 cmd->se_tfo->get_task_tag(cmd));
8b1e1244 4368
e3d6f909 4369 cmd->se_tfo->queue_status(cmd);
c66ac9db
NB		 4370}
4371
e26d99ae 4372static int transport_generic_do_tmr(struct se_cmd *cmd)
c66ac9db 4373{
5951146d 4374 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 4375 struct se_tmr_req *tmr = cmd->se_tmr_req;
4376 int ret;
4377
4378 switch (tmr->function) {
5c6cd613 4379 case TMR_ABORT_TASK:
3d28934a 4380 core_tmr_abort_task(dev, tmr, cmd->se_sess);
c66ac9db 4381 break;
5c6cd613
NB		 4382 case TMR_ABORT_TASK_SET:
4383 case TMR_CLEAR_ACA:
4384 case TMR_CLEAR_TASK_SET:
c66ac9db
NB		 4385 tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
4386 break;
5c6cd613 4387 case TMR_LUN_RESET:
c66ac9db
NB		 4388 ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
4389 tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
4390 TMR_FUNCTION_REJECTED;
4391 break;
5c6cd613 4392 case TMR_TARGET_WARM_RESET:
c66ac9db
NB		 4393 tmr->response = TMR_FUNCTION_REJECTED;
4394 break;
5c6cd613 4395 case TMR_TARGET_COLD_RESET:
c66ac9db
NB		 4396 tmr->response = TMR_FUNCTION_REJECTED;
4397 break;
c66ac9db 4398 default:
6708bb27 4399 pr_err("Uknown TMR function: 0x%02x.\n",
c66ac9db
NB		 4400 tmr->function);
4401 tmr->response = TMR_FUNCTION_REJECTED;
4402 break;
4403 }
4404
4405 cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
e3d6f909 4406 cmd->se_tfo->queue_tm_rsp(cmd);
c66ac9db 4407
b7b8bef7 4408 transport_cmd_check_stop_to_fabric(cmd);
c66ac9db
NB		 4409 return 0;
4410}
4411
c66ac9db
NB		 4412/* transport_processing_thread():
4413 *
4414 *
4415 */
4416static int transport_processing_thread(void *param)
4417{
5951146d 4418 int ret;
c66ac9db 4419 struct se_cmd *cmd;
8359cf43 4420 struct se_device *dev = param;
c66ac9db 4421
c66ac9db 4422 while (!kthread_should_stop()) {
e3d6f909
AG		 4423 ret = wait_event_interruptible(dev->dev_queue_obj.thread_wq,
4424 atomic_read(&dev->dev_queue_obj.queue_cnt) ||
c66ac9db
NB		 4425 kthread_should_stop());
4426 if (ret < 0)
4427 goto out;
4428
c66ac9db 4429get_cmd:
5951146d
AG		 4430 cmd = transport_get_cmd_from_queue(&dev->dev_queue_obj);
4431 if (!cmd)
c66ac9db
NB		 4432 continue;
4433
5951146d 4434 switch (cmd->t_state) {
680b73c5
CH		 4435 case TRANSPORT_NEW_CMD:
4436 BUG();
4437 break;
c66ac9db 4438 case TRANSPORT_NEW_CMD_MAP:
6708bb27
AG		 4439 if (!cmd->se_tfo->new_cmd_map) {
4440 pr_err("cmd->se_tfo->new_cmd_map is"
c66ac9db
NB		 4441 " NULL for TRANSPORT_NEW_CMD_MAP\n");
4442 BUG();
4443 }
e3d6f909 4444 ret = cmd->se_tfo->new_cmd_map(cmd);
c66ac9db 4445 if (ret < 0) {
03e98c9e 4446 transport_generic_request_failure(cmd);
c66ac9db
NB		 4447 break;
4448 }
c66ac9db 4449 ret = transport_generic_new_cmd(cmd);
f147abb4 4450 if (ret < 0) {
03e98c9e
NB		 4451 transport_generic_request_failure(cmd);
4452 break;
c66ac9db
NB		 4453 }
4454 break;
4455 case TRANSPORT_PROCESS_WRITE:
4456 transport_generic_process_write(cmd);
4457 break;
c66ac9db
NB		 4458 case TRANSPORT_PROCESS_TMR:
4459 transport_generic_do_tmr(cmd);
4460 break;
07bde79a 4461 case TRANSPORT_COMPLETE_QF_WP:
e057f533
CH		 4462 transport_write_pending_qf(cmd);
4463 break;
4464 case TRANSPORT_COMPLETE_QF_OK:
4465 transport_complete_qf(cmd);
07bde79a 4466 break;
c66ac9db 4467 default:
f2da9dbd
CH		 4468 pr_err("Unknown t_state: %d for ITT: 0x%08x "
4469 "i_state: %d on SE LUN: %u\n",
4470 cmd->t_state,
e3d6f909
AG		 4471 cmd->se_tfo->get_task_tag(cmd),
4472 cmd->se_tfo->get_cmd_state(cmd),
4473 cmd->se_lun->unpacked_lun);
c66ac9db
NB		 4474 BUG();
4475 }
4476
4477 goto get_cmd;
4478 }
4479
4480out:
cf572a96 4481 WARN_ON(!list_empty(&dev->state_list));
ce8762f6 4482 WARN_ON(!list_empty(&dev->dev_queue_obj.qobj_list));
c66ac9db
NB		 4483 dev->process_thread = NULL;
4484 return 0;
4485}
Linux kernel - Deliverables
This page took 0.371742 seconds and 5 git commands to generate.