projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[PATCH] sched: add new SCHED_BATCH policy
[deliverable/linux.git] / drivers / scsi / iscsi_tcp.c
CommitLineData
7ba24713
AA		 1/*
2 * iSCSI Initiator over TCP/IP Data-Path
3 *
4 * Copyright (C) 2004 Dmitry Yusupov
5 * Copyright (C) 2004 Alex Aizman
6 * Copyright (C) 2005 Mike Christie
7 * maintained by open-iscsi@googlegroups.com
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published
11 * by the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * General Public License for more details.
18 *
19 * See the file COPYING included with this distribution for more details.
20 *
21 * Credits:
22 * Christoph Hellwig
23 * FUJITA Tomonori
24 * Arne Redlich
25 * Zhenyu Wang
26 */
27
28#include <linux/types.h>
29#include <linux/list.h>
30#include <linux/inet.h>
31#include <linux/blkdev.h>
32#include <linux/crypto.h>
33#include <linux/delay.h>
34#include <linux/kfifo.h>
35#include <linux/scatterlist.h>
0b950672 36#include <linux/mutex.h>
7ba24713
AA		 37#include <net/tcp.h>
38#include <scsi/scsi_cmnd.h>
39#include <scsi/scsi_device.h>
40#include <scsi/scsi_eh.h>
41#include <scsi/scsi_request.h>
42#include <scsi/scsi_tcq.h>
43#include <scsi/scsi_host.h>
44#include <scsi/scsi.h>
45#include <scsi/scsi_transport_iscsi.h>
46
47#include "iscsi_tcp.h"
48
49MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
50 "Alex Aizman <itn780@yahoo.com>");
51MODULE_DESCRIPTION("iSCSI/TCP data-path");
52MODULE_LICENSE("GPL");
4d841d6b 53MODULE_VERSION("0:4.445");
7ba24713
AA		 54/* #define DEBUG_TCP */
55/* #define DEBUG_SCSI */
56#define DEBUG_ASSERT
57
58#ifdef DEBUG_TCP
59#define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt)
60#else
61#define debug_tcp(fmt...)
62#endif
63
64#ifdef DEBUG_SCSI
65#define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt)
66#else
67#define debug_scsi(fmt...)
68#endif
69
70#ifndef DEBUG_ASSERT
71#ifdef BUG_ON
72#undef BUG_ON
73#endif
74#define BUG_ON(expr)
75#endif
76
77#define INVALID_SN_DELTA 0xffff
78
79static unsigned int iscsi_max_lun = 512;
80module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
81
82/* global data */
83static kmem_cache_t *taskcache;
84
85static inline void
86iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size)
87{
88 sg_init_one(&ibuf->sg, (u8 *)vbuf, size);
89 ibuf->sent = 0;
7cae5159 90 ibuf->use_sendmsg = 0;
7ba24713
AA		 91}
92
93static inline void
94iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
95{
7cae5159
MC		 96 ibuf->sg.page = virt_to_page(vbuf);
97 ibuf->sg.offset = offset_in_page(vbuf);
7ba24713
AA		 98 ibuf->sg.length = size;
99 ibuf->sent = 0;
7cae5159 100 ibuf->use_sendmsg = 1;
7ba24713
AA		 101}
102
103static inline void
104iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
105{
7cae5159
MC		 106 ibuf->sg.page = sg->page;
107 ibuf->sg.offset = sg->offset;
108 ibuf->sg.length = sg->length;
7ba24713
AA		 109 /*
110 * Fastpath: sg element fits into single page
111 */
a1e80c20 112 if (sg->length + sg->offset <= PAGE_SIZE && !PageSlab(sg->page))
7cae5159
MC		 113 ibuf->use_sendmsg = 0;
114 else
115 ibuf->use_sendmsg = 1;
7ba24713
AA		 116 ibuf->sent = 0;
117}
118
119static inline int
120iscsi_buf_left(struct iscsi_buf *ibuf)
121{
122 int rc;
123
124 rc = ibuf->sg.length - ibuf->sent;
125 BUG_ON(rc < 0);
126 return rc;
127}
128
129static inline void
af973481
MC		 130iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
131 u8* crc)
7ba24713 132{
af973481
MC		 133 crypto_digest_digest(conn->tx_tfm, &buf->sg, 1, crc);
134 buf->sg.length += sizeof(uint32_t);
7ba24713
AA		 135}
136
137static void
138iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err)
139{
140 struct iscsi_session *session = conn->session;
141 unsigned long flags;
142
143 spin_lock_irqsave(&session->lock, flags);
144 if (session->conn_cnt == 1 || session->leadconn == conn)
145 session->state = ISCSI_STATE_FAILED;
146 spin_unlock_irqrestore(&session->lock, flags);
147 set_bit(SUSPEND_BIT, &conn->suspend_tx);
148 set_bit(SUSPEND_BIT, &conn->suspend_rx);
149 iscsi_conn_error(iscsi_handle(conn), err);
150}
151
152static inline int
153iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr)
154{
155 uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn);
156 uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);
157
158 if (max_cmdsn < exp_cmdsn -1 &&
159 max_cmdsn > exp_cmdsn - INVALID_SN_DELTA)
160 return ISCSI_ERR_MAX_CMDSN;
161 if (max_cmdsn > session->max_cmdsn ||
162 max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA)
163 session->max_cmdsn = max_cmdsn;
164 if (exp_cmdsn > session->exp_cmdsn ||
165 exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA)
166 session->exp_cmdsn = exp_cmdsn;
167
168 return 0;
169}
170
171static inline int
172iscsi_hdr_extract(struct iscsi_conn *conn)
173{
174 struct sk_buff *skb = conn->in.skb;
175
176 if (conn->in.copy >= conn->hdr_size &&
177 conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
178 /*
179 * Zero-copy PDU Header: using connection context
180 * to store header pointer.
181 */
182 if (skb_shinfo(skb)->frag_list == NULL &&
183 !skb_shinfo(skb)->nr_frags)
184 conn->in.hdr = (struct iscsi_hdr *)
185 ((char*)skb->data + conn->in.offset);
186 else {
187 /* ignoring return code since we checked
188 * in.copy before */
189 skb_copy_bits(skb, conn->in.offset,
190 &conn->hdr, conn->hdr_size);
191 conn->in.hdr = &conn->hdr;
192 }
193 conn->in.offset += conn->hdr_size;
194 conn->in.copy -= conn->hdr_size;
195 } else {
196 int hdr_remains;
197 int copylen;
198
199 /*
200 * PDU header scattered across SKB's,
201 * copying it... This'll happen quite rarely.
202 */
203
204 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER)
205 conn->in.hdr_offset = 0;
206
207 hdr_remains = conn->hdr_size - conn->in.hdr_offset;
208 BUG_ON(hdr_remains <= 0);
209
210 copylen = min(conn->in.copy, hdr_remains);
211 skb_copy_bits(skb, conn->in.offset,
212 (char*)&conn->hdr + conn->in.hdr_offset, copylen);
213
214 debug_tcp("PDU gather offset %d bytes %d in.offset %d "
215 "in.copy %d\n", conn->in.hdr_offset, copylen,
216 conn->in.offset, conn->in.copy);
217
218 conn->in.offset += copylen;
219 conn->in.copy -= copylen;
220 if (copylen < hdr_remains) {
221 conn->in_progress = IN_PROGRESS_HEADER_GATHER;
222 conn->in.hdr_offset += copylen;
223 return -EAGAIN;
224 }
225 conn->in.hdr = &conn->hdr;
226 conn->discontiguous_hdr_cnt++;
227 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
228 }
229
230 return 0;
231}
232
233static inline void
234iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
235{
236 struct scsi_cmnd *sc = ctask->sc;
237 struct iscsi_session *session = conn->session;
238
239 spin_lock(&session->lock);
240 if (unlikely(!sc)) {
241 spin_unlock(&session->lock);
242 return;
243 }
244 if (sc->sc_data_direction == DMA_TO_DEVICE) {
245 struct iscsi_data_task *dtask, *n;
246 /* WRITE: cleanup Data-Out's if any */
247 spin_lock(&conn->lock);
248 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
249 list_del(&dtask->item);
250 mempool_free(dtask, ctask->datapool);
251 }
252 spin_unlock(&conn->lock);
253 }
254 ctask->xmstate = XMSTATE_IDLE;
255 ctask->r2t = NULL;
256 ctask->sc = NULL;
257 __kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
258 spin_unlock(&session->lock);
259}
260
261/**
262 * iscsi_cmd_rsp - SCSI Command Response processing
263 * @conn: iscsi connection
264 * @ctask: scsi command task
265 **/
266static int
267iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
268{
269 int rc;
270 struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr;
271 struct iscsi_session *session = conn->session;
272 struct scsi_cmnd *sc = ctask->sc;
273
274 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
275 if (rc) {
276 sc->result = (DID_ERROR << 16);
277 goto out;
278 }
279
280 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
281
282 sc->result = (DID_OK << 16) | rhdr->cmd_status;
283
284 if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
285 sc->result = (DID_ERROR << 16);
286 goto out;
287 }
288
289 if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) {
290 int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE);
291
292 memcpy(sc->sense_buffer, conn->data + 2, sensecopy);
293 debug_scsi("copied %d bytes of sense\n", sensecopy);
294 }
295
296 if (sc->sc_data_direction == DMA_TO_DEVICE)
297 goto out;
298
299 if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
300 int res_count = be32_to_cpu(rhdr->residual_count);
301
302 if (res_count > 0 && res_count <= sc->request_bufflen)
303 sc->resid = res_count;
304 else
305 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
306 } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
307 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
308 else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW)
309 sc->resid = be32_to_cpu(rhdr->residual_count);
310
311out:
312 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
313 (long)sc, sc->result, ctask->itt);
314 conn->scsirsp_pdus_cnt++;
315 iscsi_ctask_cleanup(conn, ctask);
316 sc->scsi_done(sc);
317 return rc;
318}
319
320/**
321 * iscsi_data_rsp - SCSI Data-In Response processing
322 * @conn: iscsi connection
323 * @ctask: scsi command task
324 **/
325static int
326iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
327{
328 int rc;
329 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr;
330 struct iscsi_session *session = conn->session;
331 int datasn = be32_to_cpu(rhdr->datasn);
332
333 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
334 if (rc)
335 return rc;
336 /*
337 * setup Data-In byte counter (gets decremented..)
338 */
339 ctask->data_count = conn->in.datalen;
340
341 if (conn->in.datalen == 0)
342 return 0;
343
344 if (ctask->datasn != datasn)
345 return ISCSI_ERR_DATASN;
346
347 ctask->datasn++;
348
349 ctask->data_offset = be32_to_cpu(rhdr->offset);
350 if (ctask->data_offset + conn->in.datalen > ctask->total_length)
351 return ISCSI_ERR_DATA_OFFSET;
352
353 if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
354 struct scsi_cmnd *sc = ctask->sc;
355
356 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
bf310b8f 357 if (rhdr->flags & ISCSI_FLAG_DATA_UNDERFLOW) {
7ba24713
AA		 358 int res_count = be32_to_cpu(rhdr->residual_count);
359
360 if (res_count > 0 &&
361 res_count <= sc->request_bufflen) {
362 sc->resid = res_count;
363 sc->result = (DID_OK << 16) | rhdr->cmd_status;
364 } else
365 sc->result = (DID_BAD_TARGET << 16) |
366 rhdr->cmd_status;
bf310b8f 367 } else if (rhdr->flags & ISCSI_FLAG_DATA_OVERFLOW) {
7ba24713
AA		 368 sc->resid = be32_to_cpu(rhdr->residual_count);
369 sc->result = (DID_OK << 16) | rhdr->cmd_status;
370 } else
371 sc->result = (DID_OK << 16) | rhdr->cmd_status;
372 }
373
374 conn->datain_pdus_cnt++;
375 return 0;
376}
377
378/**
379 * iscsi_solicit_data_init - initialize first Data-Out
380 * @conn: iscsi connection
381 * @ctask: scsi command task
382 * @r2t: R2T info
383 *
384 * Notes:
385 * Initialize first Data-Out within this R2T sequence and finds
386 * proper data_offset within this SCSI command.
387 *
388 * This function is called with connection lock taken.
389 **/
390static void
391iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
392 struct iscsi_r2t_info *r2t)
393{
394 struct iscsi_data *hdr;
395 struct iscsi_data_task *dtask;
396 struct scsi_cmnd *sc = ctask->sc;
397
398 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
399 BUG_ON(!dtask);
400 hdr = &dtask->hdr;
401 memset(hdr, 0, sizeof(struct iscsi_data));
402 hdr->ttt = r2t->ttt;
403 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
404 r2t->solicit_datasn++;
405 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
406 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
407 hdr->itt = ctask->hdr.itt;
408 hdr->exp_statsn = r2t->exp_statsn;
409 hdr->offset = cpu_to_be32(r2t->data_offset);
410 if (r2t->data_length > conn->max_xmit_dlength) {
411 hton24(hdr->dlength, conn->max_xmit_dlength);
412 r2t->data_count = conn->max_xmit_dlength;
413 hdr->flags = 0;
414 } else {
415 hton24(hdr->dlength, r2t->data_length);
416 r2t->data_count = r2t->data_length;
417 hdr->flags = ISCSI_FLAG_CMD_FINAL;
418 }
419 conn->dataout_pdus_cnt++;
420
421 r2t->sent = 0;
422
af973481
MC		 423 iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr,
424 sizeof(struct iscsi_hdr));
7ba24713
AA		 425
426 r2t->dtask = dtask;
427
428 if (sc->use_sg) {
429 int i, sg_count = 0;
430 struct scatterlist *sg = sc->request_buffer;
431
432 r2t->sg = NULL;
433 for (i = 0; i < sc->use_sg; i++, sg += 1) {
434 /* FIXME: prefetch ? */
435 if (sg_count + sg->length > r2t->data_offset) {
436 int page_offset;
437
438 /* sg page found! */
439
440 /* offset within this page */
441 page_offset = r2t->data_offset - sg_count;
442
443 /* fill in this buffer */
444 iscsi_buf_init_sg(&r2t->sendbuf, sg);
445 r2t->sendbuf.sg.offset += page_offset;
446 r2t->sendbuf.sg.length -= page_offset;
447
448 /* xmit logic will continue with next one */
449 r2t->sg = sg + 1;
450 break;
451 }
452 sg_count += sg->length;
453 }
454 BUG_ON(r2t->sg == NULL);
455 } else
456 iscsi_buf_init_iov(&ctask->sendbuf,
457 (char*)sc->request_buffer + r2t->data_offset,
458 r2t->data_count);
459
460 list_add(&dtask->item, &ctask->dataqueue);
461}
462
463/**
464 * iscsi_r2t_rsp - iSCSI R2T Response processing
465 * @conn: iscsi connection
466 * @ctask: scsi command task
467 **/
468static int
469iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
470{
471 struct iscsi_r2t_info *r2t;
472 struct iscsi_session *session = conn->session;
473 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr;
474 int r2tsn = be32_to_cpu(rhdr->r2tsn);
475 int rc;
476
477 if (conn->in.ahslen)
478 return ISCSI_ERR_AHSLEN;
479
480 if (conn->in.datalen)
481 return ISCSI_ERR_DATALEN;
482
483 if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn)
484 return ISCSI_ERR_R2TSN;
485
486 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
487 if (rc)
488 return rc;
489
490 /* FIXME: use R2TSN to detect missing R2T */
491
492 /* fill-in new R2T associated with the task */
493 spin_lock(&session->lock);
494 if (!ctask->sc || ctask->mtask ||
495 session->state != ISCSI_STATE_LOGGED_IN) {
496 printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
497 "recovery...\n", ctask->itt);
498 spin_unlock(&session->lock);
499 return 0;
500 }
501 rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
502 BUG_ON(!rc);
503
504 r2t->exp_statsn = rhdr->statsn;
505 r2t->data_length = be32_to_cpu(rhdr->data_length);
506 if (r2t->data_length == 0 ||
507 r2t->data_length > session->max_burst) {
508 spin_unlock(&session->lock);
509 return ISCSI_ERR_DATALEN;
510 }
511
512 r2t->data_offset = be32_to_cpu(rhdr->data_offset);
513 if (r2t->data_offset + r2t->data_length > ctask->total_length) {
514 spin_unlock(&session->lock);
515 return ISCSI_ERR_DATALEN;
516 }
517
518 r2t->ttt = rhdr->ttt; /* no flip */
519 r2t->solicit_datasn = 0;
520
521 iscsi_solicit_data_init(conn, ctask, r2t);
522
523 ctask->exp_r2tsn = r2tsn + 1;
524 ctask->xmstate |= XMSTATE_SOL_HDR;
525 __kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*));
526 __kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*));
527
55e3299d 528 scsi_queue_work(session->host, &conn->xmitwork);
7ba24713
AA		 529 conn->r2t_pdus_cnt++;
530 spin_unlock(&session->lock);
531
532 return 0;
533}
534
535static int
536iscsi_hdr_recv(struct iscsi_conn *conn)
537{
538 int rc = 0;
539 struct iscsi_hdr *hdr;
540 struct iscsi_cmd_task *ctask;
541 struct iscsi_session *session = conn->session;
542 uint32_t cdgst, rdgst = 0;
543
544 hdr = conn->in.hdr;
545
546 /* verify PDU length */
547 conn->in.datalen = ntoh24(hdr->dlength);
548 if (conn->in.datalen > conn->max_recv_dlength) {
549 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
550 conn->in.datalen, conn->max_recv_dlength);
551 return ISCSI_ERR_DATALEN;
552 }
553 conn->data_copied = 0;
554
555 /* read AHS */
02cf9311 556 conn->in.ahslen = hdr->hlength * 4;
7ba24713
AA		 557 conn->in.offset += conn->in.ahslen;
558 conn->in.copy -= conn->in.ahslen;
559 if (conn->in.copy < 0) {
560 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
561 "%d bytes\n", conn->in.ahslen);
562 return ISCSI_ERR_AHSLEN;
563 }
564
565 /* calculate read padding */
566 conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1);
567 if (conn->in.padding) {
568 conn->in.padding = ISCSI_PAD_LEN - conn->in.padding;
569 debug_scsi("read padding %d bytes\n", conn->in.padding);
570 }
571
572 if (conn->hdrdgst_en) {
573 struct scatterlist sg;
574
575 sg_init_one(&sg, (u8 *)hdr,
576 sizeof(struct iscsi_hdr) + conn->in.ahslen);
577 crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
578 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
579 conn->in.ahslen);
8a47cd34
MC		 580 if (cdgst != rdgst) {
581 printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error "
582 "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst,
583 cdgst);
584 return ISCSI_ERR_HDR_DGST;
585 }
7ba24713
AA		 586 }
587
588 /* save opcode for later */
0d2f1655 589 conn->in.opcode = hdr->opcode & ISCSI_OPCODE_MASK;
7ba24713
AA		 590
591 /* verify itt (itt encoding: age+cid+itt) */
592 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
593 if ((hdr->itt & AGE_MASK) !=
594 (session->age << AGE_SHIFT)) {
595 printk(KERN_ERR "iscsi_tcp: received itt %x expected "
596 "session age (%x)\n", hdr->itt,
597 session->age & AGE_MASK);
598 return ISCSI_ERR_BAD_ITT;
599 }
600
601 if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) {
602 printk(KERN_ERR "iscsi_tcp: received itt %x, expected "
603 "CID (%x)\n", hdr->itt, conn->id);
604 return ISCSI_ERR_BAD_ITT;
605 }
606 conn->in.itt = hdr->itt & ITT_MASK;
607 } else
608 conn->in.itt = hdr->itt;
609
610 debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
611 hdr->opcode, conn->in.offset, conn->in.copy,
612 conn->in.ahslen, conn->in.datalen);
613
614 if (conn->in.itt < session->cmds_max) {
7ba24713
AA		 615 ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt];
616
617 if (!ctask->sc) {
618 printk(KERN_INFO "iscsi_tcp: dropping ctask with "
619 "itt 0x%x\n", ctask->itt);
620 conn->in.datalen = 0; /* force drop */
621 return 0;
622 }
623
624 if (ctask->sc->SCp.phase != session->age) {
625 printk(KERN_ERR "iscsi_tcp: ctask's session age %d, "
626 "expected %d\n", ctask->sc->SCp.phase,
627 session->age);
628 return ISCSI_ERR_SESSION_FAILED;
629 }
630
631 conn->in.ctask = ctask;
632
633 debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n",
634 hdr->opcode, conn->id, (long)ctask->sc,
635 ctask->itt, conn->in.datalen);
636
637 switch(conn->in.opcode) {
638 case ISCSI_OP_SCSI_CMD_RSP:
639 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
733bb6a7 640 if (!conn->in.datalen)
7ba24713
AA		 641 rc = iscsi_cmd_rsp(conn, ctask);
642 else
643 /*
644 * got sense or response data; copying PDU
645 * Header to the connection's header
646 * placeholder
647 */
648 memcpy(&conn->hdr, hdr,
649 sizeof(struct iscsi_hdr));
650 break;
651 case ISCSI_OP_SCSI_DATA_IN:
652 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
653 /* save flags for non-exceptional status */
654 conn->in.flags = hdr->flags;
655 /* save cmd_status for sense data */
656 conn->in.cmd_status =
657 ((struct iscsi_data_rsp*)hdr)->cmd_status;
658 rc = iscsi_data_rsp(conn, ctask);
659 break;
660 case ISCSI_OP_R2T:
661 BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
733bb6a7 662 if (ctask->sc->sc_data_direction == DMA_TO_DEVICE)
7ba24713
AA		 663 rc = iscsi_r2t_rsp(conn, ctask);
664 else
665 rc = ISCSI_ERR_PROTO;
666 break;
7ba24713
AA		 667 default:
668 rc = ISCSI_ERR_BAD_OPCODE;
669 break;
670 }
671 } else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET &&
672 conn->in.itt < ISCSI_MGMT_ITT_OFFSET +
673 session->mgmtpool_max) {
674 struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *)
675 session->mgmt_cmds[conn->in.itt -
676 ISCSI_MGMT_ITT_OFFSET];
677
678 debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n",
679 conn->in.opcode, conn->id, mtask->itt,
680 conn->in.datalen);
681
682 switch(conn->in.opcode) {
683 case ISCSI_OP_LOGIN_RSP:
684 case ISCSI_OP_TEXT_RSP:
42f72aa9 685 case ISCSI_OP_LOGOUT_RSP:
7ba24713
AA		 686 rc = iscsi_check_assign_cmdsn(session,
687 (struct iscsi_nopin*)hdr);
688 if (rc)
689 break;
690
691 if (!conn->in.datalen) {
692 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
693 NULL, 0);
694 if (conn->login_mtask != mtask) {
695 spin_lock(&session->lock);
696 __kfifo_put(session->mgmtpool.queue,
697 (void*)&mtask, sizeof(void*));
698 spin_unlock(&session->lock);
699 }
700 }
701 break;
702 case ISCSI_OP_SCSI_TMFUNC_RSP:
703 rc = iscsi_check_assign_cmdsn(session,
704 (struct iscsi_nopin*)hdr);
705 if (rc)
706 break;
707
708 if (conn->in.datalen || conn->in.ahslen) {
709 rc = ISCSI_ERR_PROTO;
710 break;
711 }
712 conn->tmfrsp_pdus_cnt++;
713 spin_lock(&session->lock);
714 if (conn->tmabort_state == TMABORT_INITIAL) {
715 __kfifo_put(session->mgmtpool.queue,
716 (void*)&mtask, sizeof(void*));
717 conn->tmabort_state =
718 ((struct iscsi_tm_rsp *)hdr)->
baebc497 719 response == ISCSI_TMF_RSP_COMPLETE ?
7ba24713
AA		 720 TMABORT_SUCCESS:TMABORT_FAILED;
721 /* unblock eh_abort() */
722 wake_up(&conn->ehwait);
723 }
724 spin_unlock(&session->lock);
725 break;
42f72aa9 726 case ISCSI_OP_NOOP_IN:
99744878
MC		 727 if (hdr->ttt != ISCSI_RESERVED_TAG) {
728 rc = ISCSI_ERR_PROTO;
729 break;
730 }
42f72aa9 731 rc = iscsi_check_assign_cmdsn(session,
99744878
MC		 732 (struct iscsi_nopin*)hdr);
733 if (rc)
734 break;
735 conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
736
737 if (!conn->in.datalen) {
738 struct iscsi_mgmt_task *mtask;
739
740 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
741 NULL, 0);
742 mtask = (struct iscsi_mgmt_task *)
743 session->mgmt_cmds[conn->in.itt -
744 ISCSI_MGMT_ITT_OFFSET];
745 if (conn->login_mtask != mtask) {
746 spin_lock(&session->lock);
747 __kfifo_put(session->mgmtpool.queue,
748 (void*)&mtask, sizeof(void*));
749 spin_unlock(&session->lock);
750 }
751 }
752 break;
7ba24713
AA		 753 default:
754 rc = ISCSI_ERR_BAD_OPCODE;
755 break;
756 }
757 } else if (conn->in.itt == ISCSI_RESERVED_TAG) {
99744878
MC		 758 switch(conn->in.opcode) {
759 case ISCSI_OP_NOOP_IN:
760 if (!conn->in.datalen) {
761 rc = iscsi_check_assign_cmdsn(session,
7ba24713 762 (struct iscsi_nopin*)hdr);
99744878
MC		 763 if (!rc && hdr->ttt != ISCSI_RESERVED_TAG)
764 rc = iscsi_recv_pdu(iscsi_handle(conn),
765 hdr, NULL, 0);
42f72aa9 766 } else
99744878
MC		 767 rc = ISCSI_ERR_PROTO;
768 break;
769 case ISCSI_OP_REJECT:
770 /* we need sth like iscsi_reject_rsp()*/
771 case ISCSI_OP_ASYNC_EVENT:
772 /* we need sth like iscsi_async_event_rsp() */
7ba24713 773 rc = ISCSI_ERR_BAD_OPCODE;
99744878
MC		 774 break;
775 default:
776 rc = ISCSI_ERR_BAD_OPCODE;
777 break;
778 }
7ba24713
AA		 779 } else
780 rc = ISCSI_ERR_BAD_ITT;
781
782 return rc;
783}
784
785/**
786 * iscsi_ctask_copy - copy skb bits to the destanation cmd task
787 * @conn: iscsi connection
788 * @ctask: scsi command task
789 * @buf: buffer to copy to
790 * @buf_size: size of buffer
791 * @offset: offset within the buffer
792 *
793 * Notes:
794 * The function calls skb_copy_bits() and updates per-connection and
795 * per-cmd byte counters.
796 *
797 * Read counters (in bytes):
798 *
799 * conn->in.offset offset within in progress SKB
800 * conn->in.copy left to copy from in progress SKB
801 * including padding
802 * conn->in.copied copied already from in progress SKB
803 * conn->data_copied copied already from in progress buffer
804 * ctask->sent total bytes sent up to the MidLayer
805 * ctask->data_count left to copy from in progress Data-In
806 * buf_left left to copy from in progress buffer
807 **/
808static inline int
809iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
810 void *buf, int buf_size, int offset)
811{
812 int buf_left = buf_size - (conn->data_copied + offset);
813 int size = min(conn->in.copy, buf_left);
814 int rc;
815
816 size = min(size, ctask->data_count);
817
818 debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
819 size, conn->in.offset, conn->in.copied);
820
821 BUG_ON(size <= 0);
822 BUG_ON(ctask->sent + size > ctask->total_length);
823
824 rc = skb_copy_bits(conn->in.skb, conn->in.offset,
825 (char*)buf + (offset + conn->data_copied), size);
826 /* must fit into skb->len */
827 BUG_ON(rc);
828
829 conn->in.offset += size;
830 conn->in.copy -= size;
831 conn->in.copied += size;
832 conn->data_copied += size;
833 ctask->sent += size;
834 ctask->data_count -= size;
835
836 BUG_ON(conn->in.copy < 0);
837 BUG_ON(ctask->data_count < 0);
838
839 if (buf_size != (conn->data_copied + offset)) {
840 if (!ctask->data_count) {
841 BUG_ON(buf_size - conn->data_copied < 0);
842 /* done with this PDU */
843 return buf_size - conn->data_copied;
844 }
845 return -EAGAIN;
846 }
847
848 /* done with this buffer or with both - PDU and buffer */
849 conn->data_copied = 0;
850 return 0;
851}
852
853/**
854 * iscsi_tcp_copy - copy skb bits to the destanation buffer
855 * @conn: iscsi connection
856 * @buf: buffer to copy to
857 * @buf_size: number of bytes to copy
858 *
859 * Notes:
860 * The function calls skb_copy_bits() and updates per-connection
861 * byte counters.
862 **/
863static inline int
864iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size)
865{
866 int buf_left = buf_size - conn->data_copied;
867 int size = min(conn->in.copy, buf_left);
868 int rc;
869
870 debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
871 size, conn->in.offset, conn->data_copied);
872 BUG_ON(size <= 0);
873
874 rc = skb_copy_bits(conn->in.skb, conn->in.offset,
875 (char*)buf + conn->data_copied, size);
876 BUG_ON(rc);
877
878 conn->in.offset += size;
879 conn->in.copy -= size;
880 conn->in.copied += size;
881 conn->data_copied += size;
882
883 if (buf_size != conn->data_copied)
884 return -EAGAIN;
885
886 return 0;
887}
888
889static inline void
890partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg,
891 int offset, int length)
892{
893 struct scatterlist temp;
894
895 memcpy(&temp, sg, sizeof(struct scatterlist));
896 temp.offset = offset;
897 temp.length = length;
898 crypto_digest_update(conn->data_rx_tfm, &temp, 1);
899}
900
f6cfba1d
MC		 901static void
902iscsi_recv_digest_update(struct iscsi_conn *conn, char* buf, int len)
903{
904 struct scatterlist tmp;
905
906 sg_init_one(&tmp, buf, len);
907 crypto_digest_update(conn->data_rx_tfm, &tmp, 1);
908}
909
7ba24713
AA		 910static int iscsi_scsi_data_in(struct iscsi_conn *conn)
911{
912 struct iscsi_cmd_task *ctask = conn->in.ctask;
913 struct scsi_cmnd *sc = ctask->sc;
f6cfba1d 914 struct scatterlist *sg;
7ba24713
AA		 915 int i, offset, rc = 0;
916
917 BUG_ON((void*)ctask != sc->SCp.ptr);
918
919 /*
920 * copying Data-In into the Scsi_Cmnd
921 */
922 if (!sc->use_sg) {
923 i = ctask->data_count;
924 rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer,
925 sc->request_bufflen, ctask->data_offset);
926 if (rc == -EAGAIN)
927 return rc;
42f72aa9 928 if (conn->datadgst_en)
f6cfba1d 929 iscsi_recv_digest_update(conn, sc->request_buffer, i);
7ba24713
AA		 930 rc = 0;
931 goto done;
932 }
933
934 offset = ctask->data_offset;
935 sg = sc->request_buffer;
936
937 if (ctask->data_offset)
938 for (i = 0; i < ctask->sg_count; i++)
939 offset -= sg[i].length;
940 /* we've passed through partial sg*/
941 if (offset < 0)
942 offset = 0;
943
944 for (i = ctask->sg_count; i < sc->use_sg; i++) {
945 char *dest;
946
947 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
948 rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset,
949 sg[i].length, offset);
950 kunmap_atomic(dest, KM_SOFTIRQ0);
951 if (rc == -EAGAIN)
952 /* continue with the next SKB/PDU */
953 return rc;
954 if (!rc) {
955 if (conn->datadgst_en) {
956 if (!offset)
957 crypto_digest_update(conn->data_rx_tfm,
958 &sg[i], 1);
959 else
960 partial_sg_digest_update(conn, &sg[i],
961 sg[i].offset + offset,
962 sg[i].length - offset);
963 }
964 offset = 0;
965 ctask->sg_count++;
966 }
967
968 if (!ctask->data_count) {
969 if (rc && conn->datadgst_en)
970 /*
971 * data-in is complete, but buffer not...
972 */
973 partial_sg_digest_update(conn, &sg[i],
974 sg[i].offset, sg[i].length-rc);
975 rc = 0;
976 break;
977 }
978
979 if (!conn->in.copy)
980 return -EAGAIN;
981 }
982 BUG_ON(ctask->data_count);
983
984done:
985 /* check for non-exceptional status */
986 if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) {
987 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
988 (long)sc, sc->result, ctask->itt);
989 conn->scsirsp_pdus_cnt++;
990 iscsi_ctask_cleanup(conn, ctask);
991 sc->scsi_done(sc);
992 }
993
994 return rc;
995}
996
997static int
998iscsi_data_recv(struct iscsi_conn *conn)
999{
1000 struct iscsi_session *session = conn->session;
1001 int rc = 0;
1002
1003 switch(conn->in.opcode) {
1004 case ISCSI_OP_SCSI_DATA_IN:
1005 rc = iscsi_scsi_data_in(conn);
1006 break;
1007 case ISCSI_OP_SCSI_CMD_RSP: {
1008 /*
1009 * SCSI Sense Data:
1010 * copying the entire Data Segment.
1011 */
1012 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1013 rc = -EAGAIN;
1014 goto exit;
1015 }
1016
1017 /*
1018 * check for sense
1019 */
1020 conn->in.hdr = &conn->hdr;
1021 conn->senselen = (conn->data[0] << 8) | conn->data[1];
1022 rc = iscsi_cmd_rsp(conn, conn->in.ctask);
42f72aa9 1023 if (!rc && conn->datadgst_en)
f6cfba1d
MC		 1024 iscsi_recv_digest_update(conn, conn->data,
1025 conn->in.datalen);
7ba24713
AA		 1026 }
1027 break;
1028 case ISCSI_OP_TEXT_RSP:
1029 case ISCSI_OP_LOGIN_RSP:
1030 case ISCSI_OP_NOOP_IN: {
1031 struct iscsi_mgmt_task *mtask = NULL;
1032
1033 if (conn->in.itt != ISCSI_RESERVED_TAG)
1034 mtask = (struct iscsi_mgmt_task *)
1035 session->mgmt_cmds[conn->in.itt -
1036 ISCSI_MGMT_ITT_OFFSET];
1037
1038 /*
1039 * Collect data segment to the connection's data
1040 * placeholder
1041 */
1042 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1043 rc = -EAGAIN;
1044 goto exit;
1045 }
1046
1047 rc = iscsi_recv_pdu(iscsi_handle(conn), conn->in.hdr,
1048 conn->data, conn->in.datalen);
1049
42f72aa9 1050 if (!rc && conn->datadgst_en &&
f6cfba1d
MC		 1051 conn->in.opcode != ISCSI_OP_LOGIN_RSP)
1052 iscsi_recv_digest_update(conn, conn->data,
1053 conn->in.datalen);
1054
7ba24713
AA		 1055 if (mtask && conn->login_mtask != mtask) {
1056 spin_lock(&session->lock);
1057 __kfifo_put(session->mgmtpool.queue, (void*)&mtask,
1058 sizeof(void*));
1059 spin_unlock(&session->lock);
1060 }
1061 }
1062 break;
f6cfba1d
MC		 1063 case ISCSI_OP_ASYNC_EVENT:
1064 case ISCSI_OP_REJECT:
7ba24713
AA		 1065 default:
1066 BUG_ON(1);
1067 }
1068exit:
1069 return rc;
1070}
1071
1072/**
1073 * iscsi_tcp_data_recv - TCP receive in sendfile fashion
1074 * @rd_desc: read descriptor
1075 * @skb: socket buffer
1076 * @offset: offset in skb
1077 * @len: skb->len - offset
1078 **/
1079static int
1080iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
1081 unsigned int offset, size_t len)
1082{
1083 int rc;
1084 struct iscsi_conn *conn = rd_desc->arg.data;
7ba24713
AA		 1085 int processed;
1086 char pad[ISCSI_PAD_LEN];
1087 struct scatterlist sg;
1088
1089 /*
1090 * Save current SKB and its offset in the corresponding
1091 * connection context.
1092 */
bb052349 1093 conn->in.copy = skb->len - offset;
7ba24713
AA		 1094 conn->in.offset = offset;
1095 conn->in.skb = skb;
1096 conn->in.len = conn->in.copy;
1097 BUG_ON(conn->in.copy <= 0);
1098 debug_tcp("in %d bytes\n", conn->in.copy);
1099
1100more:
1101 conn->in.copied = 0;
1102 rc = 0;
1103
1104 if (unlikely(conn->suspend_rx)) {
1105 debug_tcp("conn %d Rx suspended!\n", conn->id);
1106 return 0;
1107 }
1108
1109 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
1110 conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
1111 rc = iscsi_hdr_extract(conn);
1112 if (rc) {
1113 if (rc == -EAGAIN)
1114 goto nomore;
1115 else {
1116 iscsi_conn_failure(conn, rc);
1117 return 0;
1118 }
1119 }
1120
1121 /*
1122 * Verify and process incoming PDU header.
1123 */
1124 rc = iscsi_hdr_recv(conn);
1125 if (!rc && conn->in.datalen) {
8a47cd34 1126 if (conn->datadgst_en) {
7ba24713
AA		 1127 BUG_ON(!conn->data_rx_tfm);
1128 crypto_digest_init(conn->data_rx_tfm);
1129 }
1130 conn->in_progress = IN_PROGRESS_DATA_RECV;
1131 } else if (rc) {
1132 iscsi_conn_failure(conn, rc);
1133 return 0;
1134 }
1135 }
1136
1137 if (conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
f6cfba1d 1138 uint32_t recv_digest;
7ba24713
AA		 1139 debug_tcp("extra data_recv offset %d copy %d\n",
1140 conn->in.offset, conn->in.copy);
f6cfba1d
MC		 1141 skb_copy_bits(conn->in.skb, conn->in.offset,
1142 &recv_digest, 4);
1143 conn->in.offset += 4;
1144 conn->in.copy -= 4;
1145 if (recv_digest != conn->in.datadgst) {
1146 debug_tcp("iscsi_tcp: data digest error!"
1147 "0x%x != 0x%x\n", recv_digest,
1148 conn->in.datadgst);
1149 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
1150 return 0;
1151 } else {
1152 debug_tcp("iscsi_tcp: data digest match!"
1153 "0x%x == 0x%x\n", recv_digest,
1154 conn->in.datadgst);
1155 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
7ba24713
AA		 1156 }
1157 }
1158
1159 if (conn->in_progress == IN_PROGRESS_DATA_RECV && conn->in.copy) {
1160
1161 debug_tcp("data_recv offset %d copy %d\n",
1162 conn->in.offset, conn->in.copy);
1163
1164 rc = iscsi_data_recv(conn);
1165 if (rc) {
1166 if (rc == -EAGAIN) {
1167 rd_desc->count = conn->in.datalen -
1168 conn->in.ctask->data_count;
1169 goto again;
1170 }
1171 iscsi_conn_failure(conn, rc);
1172 return 0;
1173 }
1174 conn->in.copy -= conn->in.padding;
1175 conn->in.offset += conn->in.padding;
8a47cd34 1176 if (conn->datadgst_en) {
7ba24713
AA		 1177 if (conn->in.padding) {
1178 debug_tcp("padding -> %d\n", conn->in.padding);
1179 memset(pad, 0, conn->in.padding);
1180 sg_init_one(&sg, pad, conn->in.padding);
1181 crypto_digest_update(conn->data_rx_tfm, &sg, 1);
1182 }
1183 crypto_digest_final(conn->data_rx_tfm,
1184 (u8 *) & conn->in.datadgst);
1185 debug_tcp("rx digest 0x%x\n", conn->in.datadgst);
1186 conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
1187 } else
1188 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1189 }
1190
1191 debug_tcp("f, processed %d from out of %d padding %d\n",
1192 conn->in.offset - offset, (int)len, conn->in.padding);
1193 BUG_ON(conn->in.offset - offset > len);
1194
1195 if (conn->in.offset - offset != len) {
1196 debug_tcp("continue to process %d bytes\n",
1197 (int)len - (conn->in.offset - offset));
1198 goto more;
1199 }
1200
1201nomore:
1202 processed = conn->in.offset - offset;
1203 BUG_ON(processed == 0);
1204 return processed;
1205
1206again:
1207 processed = conn->in.offset - offset;
1208 debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
1209 processed, (int)len, (int)rd_desc->count);
1210 BUG_ON(processed == 0);
1211 BUG_ON(processed > len);
1212
1213 conn->rxdata_octets += processed;
1214 return processed;
1215}
1216
1217static void
1218iscsi_tcp_data_ready(struct sock *sk, int flag)
1219{
1220 struct iscsi_conn *conn = sk->sk_user_data;
1221 read_descriptor_t rd_desc;
1222
1223 read_lock(&sk->sk_callback_lock);
1224
1225 /* use rd_desc to pass 'conn' to iscsi_tcp_data_recv */
1226 rd_desc.arg.data = conn;
1227 rd_desc.count = 0;
1228 tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);
1229
1230 read_unlock(&sk->sk_callback_lock);
1231}
1232
1233static void
1234iscsi_tcp_state_change(struct sock *sk)
1235{
1236 struct iscsi_conn *conn;
1237 struct iscsi_session *session;
1238 void (*old_state_change)(struct sock *);
1239
1240 read_lock(&sk->sk_callback_lock);
1241
1242 conn = (struct iscsi_conn*)sk->sk_user_data;
1243 session = conn->session;
1244
e6273993
MC		 1245 if ((sk->sk_state == TCP_CLOSE_WAIT ||
1246 sk->sk_state == TCP_CLOSE) &&
1247 !atomic_read(&sk->sk_rmem_alloc)) {
7ba24713
AA		 1248 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1249 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1250 }
1251
1252 old_state_change = conn->old_state_change;
1253
1254 read_unlock(&sk->sk_callback_lock);
1255
1256 old_state_change(sk);
1257}
1258
1259/**
1260 * iscsi_write_space - Called when more output buffer space is available
1261 * @sk: socket space is available for
1262 **/
1263static void
1264iscsi_write_space(struct sock *sk)
1265{
1266 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1267 conn->old_write_space(sk);
1268 debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1269 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
55e3299d 1270 scsi_queue_work(conn->session->host, &conn->xmitwork);
7ba24713
AA		 1271}
1272
1273static void
1274iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1275{
1276 struct sock *sk = conn->sock->sk;
1277
1278 /* assign new callbacks */
1279 write_lock_bh(&sk->sk_callback_lock);
1280 sk->sk_user_data = conn;
1281 conn->old_data_ready = sk->sk_data_ready;
1282 conn->old_state_change = sk->sk_state_change;
1283 conn->old_write_space = sk->sk_write_space;
1284 sk->sk_data_ready = iscsi_tcp_data_ready;
1285 sk->sk_state_change = iscsi_tcp_state_change;
1286 sk->sk_write_space = iscsi_write_space;
1287 write_unlock_bh(&sk->sk_callback_lock);
1288}
1289
1290static void
1291iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1292{
1293 struct sock *sk = conn->sock->sk;
1294
1295 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1296 write_lock_bh(&sk->sk_callback_lock);
1297 sk->sk_user_data = NULL;
1298 sk->sk_data_ready = conn->old_data_ready;
1299 sk->sk_state_change = conn->old_state_change;
1300 sk->sk_write_space = conn->old_write_space;
1301 sk->sk_no_check = 0;
1302 write_unlock_bh(&sk->sk_callback_lock);
1303}
1304
1305/**
1306 * iscsi_send - generic send routine
1307 * @sk: kernel's socket
1308 * @buf: buffer to write from
1309 * @size: actual size to write
1310 * @flags: socket's flags
7ba24713
AA		 1311 */
1312static inline int
56851698 1313iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
7ba24713 1314{
56851698 1315 struct socket *sk = conn->sock;
7cae5159 1316 int offset = buf->sg.offset + buf->sent;
7ba24713 1317
7cae5159
MC		 1318 /*
1319 * if we got use_sg=0 or are sending something we kmallocd
1320 * then we did not have to do kmap (kmap returns page_address)
1321 *
1322 * if we got use_sg > 0, but had to drop down, we do not
1323 * set clustering so this should only happen for that
1324 * slab case.
1325 */
1326 if (buf->use_sendmsg)
1327 return sock_no_sendpage(sk, buf->sg.page, offset, size, flags);
1328 else
1329 return conn->sendpage(sk, buf->sg.page, offset, size, flags);
7ba24713
AA		 1330}
1331
1332/**
1333 * iscsi_sendhdr - send PDU Header via tcp_sendpage()
1334 * @conn: iscsi connection
1335 * @buf: buffer to write from
1336 * @datalen: lenght of data to be sent after the header
1337 *
1338 * Notes:
1339 * (Tx, Fast Path)
1340 **/
1341static inline int
1342iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
1343{
7ba24713
AA		 1344 int flags = 0; /* MSG_DONTWAIT; */
1345 int res, size;
1346
1347 size = buf->sg.length - buf->sent;
1348 BUG_ON(buf->sent + size > buf->sg.length);
1349 if (buf->sent + size != buf->sg.length || datalen)
1350 flags |= MSG_MORE;
1351
56851698 1352 res = iscsi_send(conn, buf, size, flags);
7ba24713
AA		 1353 debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
1354 if (res >= 0) {
1355 conn->txdata_octets += res;
1356 buf->sent += res;
1357 if (size != res)
1358 return -EAGAIN;
1359 return 0;
1360 } else if (res == -EAGAIN) {
1361 conn->sendpage_failures_cnt++;
1362 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1363 } else if (res == -EPIPE)
1364 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1365
1366 return res;
1367}
1368
1369/**
1370 * iscsi_sendpage - send one page of iSCSI Data-Out.
1371 * @conn: iscsi connection
1372 * @buf: buffer to write from
1373 * @count: remaining data
1374 * @sent: number of bytes sent
1375 *
1376 * Notes:
1377 * (Tx, Fast Path)
1378 **/
1379static inline int
1380iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1381 int *count, int *sent)
1382{
7ba24713
AA		 1383 int flags = 0; /* MSG_DONTWAIT; */
1384 int res, size;
1385
1386 size = buf->sg.length - buf->sent;
1387 BUG_ON(buf->sent + size > buf->sg.length);
1388 if (size > *count)
1389 size = *count;
b13941f6 1390 if (buf->sent + size != buf->sg.length || *count != size)
7ba24713
AA		 1391 flags |= MSG_MORE;
1392
56851698 1393 res = iscsi_send(conn, buf, size, flags);
7ba24713
AA		 1394 debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
1395 size, buf->sent, *count, *sent, res);
1396 if (res >= 0) {
1397 conn->txdata_octets += res;
1398 buf->sent += res;
1399 *count -= res;
1400 *sent += res;
1401 if (size != res)
1402 return -EAGAIN;
1403 return 0;
1404 } else if (res == -EAGAIN) {
1405 conn->sendpage_failures_cnt++;
1406 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1407 } else if (res == -EPIPE)
1408 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1409
1410 return res;
1411}
1412
1413static inline void
1414iscsi_data_digest_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1415{
1416 BUG_ON(!conn->data_tx_tfm);
1417 crypto_digest_init(conn->data_tx_tfm);
1418 ctask->digest_count = 4;
1419}
1420
7ba24713
AA		 1421static inline int
1422iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1423 struct iscsi_buf *buf, uint32_t *digest, int final)
1424{
1425 int rc = 0;
1426 int sent = 0;
1427
1428 if (final)
1429 crypto_digest_final(conn->data_tx_tfm, (u8*)digest);
1430
1431 iscsi_buf_init_virt(buf, (char*)digest, 4);
1432 rc = iscsi_sendpage(conn, buf, &ctask->digest_count, &sent);
1433 if (rc) {
1434 ctask->datadigest = *digest;
1435 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1436 } else
1437 ctask->digest_count = 4;
1438 return rc;
1439}
1440
1441/**
1442 * iscsi_solicit_data_cont - initialize next Data-Out
1443 * @conn: iscsi connection
1444 * @ctask: scsi command task
1445 * @r2t: R2T info
1446 * @left: bytes left to transfer
1447 *
1448 * Notes:
1449 * Initialize next Data-Out within this R2T sequence and continue
1450 * to process next Scatter-Gather element(if any) of this SCSI command.
1451 *
1452 * Called under connection lock.
1453 **/
1454static void
1455iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1456 struct iscsi_r2t_info *r2t, int left)
1457{
1458 struct iscsi_data *hdr;
1459 struct iscsi_data_task *dtask;
1460 struct scsi_cmnd *sc = ctask->sc;
1461 int new_offset;
1462
1463 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1464 BUG_ON(!dtask);
1465 hdr = &dtask->hdr;
1466 memset(hdr, 0, sizeof(struct iscsi_data));
1467 hdr->ttt = r2t->ttt;
1468 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1469 r2t->solicit_datasn++;
1470 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1471 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1472 hdr->itt = ctask->hdr.itt;
1473 hdr->exp_statsn = r2t->exp_statsn;
1474 new_offset = r2t->data_offset + r2t->sent;
1475 hdr->offset = cpu_to_be32(new_offset);
1476 if (left > conn->max_xmit_dlength) {
1477 hton24(hdr->dlength, conn->max_xmit_dlength);
1478 r2t->data_count = conn->max_xmit_dlength;
1479 } else {
1480 hton24(hdr->dlength, left);
1481 r2t->data_count = left;
1482 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1483 }
1484 conn->dataout_pdus_cnt++;
1485
af973481
MC		 1486 iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr,
1487 sizeof(struct iscsi_hdr));
7ba24713
AA		 1488
1489 r2t->dtask = dtask;
1490
1491 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) {
1492 BUG_ON(ctask->bad_sg == r2t->sg);
1493 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1494 r2t->sg += 1;
1495 } else
1496 iscsi_buf_init_iov(&ctask->sendbuf,
1497 (char*)sc->request_buffer + new_offset,
1498 r2t->data_count);
1499
1500 list_add(&dtask->item, &ctask->dataqueue);
1501}
1502
1503static void
1504iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1505{
1506 struct iscsi_data *hdr;
1507 struct iscsi_data_task *dtask;
1508
1509 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1510 BUG_ON(!dtask);
1511 hdr = &dtask->hdr;
1512 memset(hdr, 0, sizeof(struct iscsi_data));
1513 hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1514 hdr->datasn = cpu_to_be32(ctask->unsol_datasn);
1515 ctask->unsol_datasn++;
1516 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1517 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1518 hdr->itt = ctask->hdr.itt;
1519 hdr->exp_statsn = cpu_to_be32(conn->exp_statsn);
1520 hdr->offset = cpu_to_be32(ctask->total_length -
1521 ctask->r2t_data_count -
1522 ctask->unsol_count);
1523 if (ctask->unsol_count > conn->max_xmit_dlength) {
1524 hton24(hdr->dlength, conn->max_xmit_dlength);
1525 ctask->data_count = conn->max_xmit_dlength;
1526 hdr->flags = 0;
1527 } else {
1528 hton24(hdr->dlength, ctask->unsol_count);
1529 ctask->data_count = ctask->unsol_count;
1530 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1531 }
1532
af973481
MC		 1533 iscsi_buf_init_virt(&ctask->headbuf, (char*)hdr,
1534 sizeof(struct iscsi_hdr));
7ba24713
AA		 1535
1536 list_add(&dtask->item, &ctask->dataqueue);
1537
1538 ctask->dtask = dtask;
1539}
1540
1541/**
1542 * iscsi_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1543 * @conn: iscsi connection
1544 * @ctask: scsi command task
1545 * @sc: scsi command
1546 **/
1547static void
1548iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1549 struct scsi_cmnd *sc)
1550{
1551 struct iscsi_session *session = conn->session;
1552
1553 BUG_ON(__kfifo_len(ctask->r2tqueue));
1554
1555 ctask->sc = sc;
1556 ctask->conn = conn;
1557 ctask->hdr.opcode = ISCSI_OP_SCSI_CMD;
1558 ctask->hdr.flags = ISCSI_ATTR_SIMPLE;
1559 int_to_scsilun(sc->device->lun, (struct scsi_lun *)ctask->hdr.lun);
1560 ctask->hdr.itt = ctask->itt | (conn->id << CID_SHIFT) |
1561 (session->age << AGE_SHIFT);
1562 ctask->hdr.data_length = cpu_to_be32(sc->request_bufflen);
1563 ctask->hdr.cmdsn = cpu_to_be32(session->cmdsn); session->cmdsn++;
1564 ctask->hdr.exp_statsn = cpu_to_be32(conn->exp_statsn);
1565 memcpy(ctask->hdr.cdb, sc->cmnd, sc->cmd_len);
1566 memset(&ctask->hdr.cdb[sc->cmd_len], 0, MAX_COMMAND_SIZE - sc->cmd_len);
1567
1568 ctask->mtask = NULL;
1569 ctask->sent = 0;
1570 ctask->sg_count = 0;
1571
1572 ctask->total_length = sc->request_bufflen;
1573
1574 if (sc->sc_data_direction == DMA_TO_DEVICE) {
1575 ctask->exp_r2tsn = 0;
1576 ctask->hdr.flags |= ISCSI_FLAG_CMD_WRITE;
1577 BUG_ON(ctask->total_length == 0);
1578 if (sc->use_sg) {
1579 struct scatterlist *sg = sc->request_buffer;
1580
1581 iscsi_buf_init_sg(&ctask->sendbuf,
1582 &sg[ctask->sg_count++]);
1583 ctask->sg = sg;
1584 ctask->bad_sg = sg + sc->use_sg;
1585 } else {
1586 iscsi_buf_init_iov(&ctask->sendbuf, sc->request_buffer,
1587 sc->request_bufflen);
1588 }
1589
1590 /*
1591 * Write counters:
1592 *
1593 * imm_count bytes to be sent right after
1594 * SCSI PDU Header
1595 *
1596 * unsol_count bytes(as Data-Out) to be sent
1597 * without R2T ack right after
1598 * immediate data
1599 *
1600 * r2t_data_count bytes to be sent via R2T ack's
1601 *
1602 * pad_count bytes to be sent as zero-padding
1603 */
1604 ctask->imm_count = 0;
1605 ctask->unsol_count = 0;
1606 ctask->unsol_datasn = 0;
1607 ctask->xmstate = XMSTATE_W_HDR;
1608 /* calculate write padding */
1609 ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1610 if (ctask->pad_count) {
1611 ctask->pad_count = ISCSI_PAD_LEN - ctask->pad_count;
1612 debug_scsi("write padding %d bytes\n",
1613 ctask->pad_count);
1614 ctask->xmstate |= XMSTATE_W_PAD;
1615 }
1616 if (session->imm_data_en) {
1617 if (ctask->total_length >= session->first_burst)
1618 ctask->imm_count = min(session->first_burst,
1619 conn->max_xmit_dlength);
1620 else
1621 ctask->imm_count = min(ctask->total_length,
1622 conn->max_xmit_dlength);
1623 hton24(ctask->hdr.dlength, ctask->imm_count);
1624 ctask->xmstate |= XMSTATE_IMM_DATA;
1625 } else
1626 zero_data(ctask->hdr.dlength);
1627
1628 if (!session->initial_r2t_en)
1629 ctask->unsol_count = min(session->first_burst,
1630 ctask->total_length) - ctask->imm_count;
1631 if (!ctask->unsol_count)
1632 /* No unsolicit Data-Out's */
1633 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1634 else
1635 ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
1636
1637 ctask->r2t_data_count = ctask->total_length -
1638 ctask->imm_count -
1639 ctask->unsol_count;
1640
1641 debug_scsi("cmd [itt %x total %d imm %d imm_data %d "
1642 "r2t_data %d]\n",
1643 ctask->itt, ctask->total_length, ctask->imm_count,
1644 ctask->unsol_count, ctask->r2t_data_count);
1645 } else {
1646 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1647 if (sc->sc_data_direction == DMA_FROM_DEVICE)
1648 ctask->hdr.flags |= ISCSI_FLAG_CMD_READ;
1649 ctask->datasn = 0;
1650 ctask->xmstate = XMSTATE_R_HDR;
1651 zero_data(ctask->hdr.dlength);
1652 }
1653
42f72aa9 1654 iscsi_buf_init_virt(&ctask->headbuf, (char*)&ctask->hdr,
af973481 1655 sizeof(struct iscsi_hdr));
7ba24713
AA		 1656 conn->scsicmd_pdus_cnt++;
1657}
1658
1659/**
1660 * iscsi_mtask_xmit - xmit management(immediate) task
1661 * @conn: iscsi connection
1662 * @mtask: task management task
1663 *
1664 * Notes:
1665 * The function can return -EAGAIN in which case caller must
1666 * call it again later, or recover. '0' return code means successful
1667 * xmit.
1668 *
1669 * Management xmit state machine consists of two states:
1670 * IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress
1671 * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
1672 **/
1673static int
1674iscsi_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1675{
1676
1677 debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
1678 conn->id, mtask->xmstate, mtask->itt);
1679
1680 if (mtask->xmstate & XMSTATE_IMM_HDR) {
1681 mtask->xmstate &= ~XMSTATE_IMM_HDR;
1682 if (mtask->data_count)
1683 mtask->xmstate |= XMSTATE_IMM_DATA;
af973481
MC		 1684 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
1685 conn->stop_stage != STOP_CONN_RECOVER &&
1686 conn->hdrdgst_en)
1687 iscsi_hdr_digest(conn, &mtask->headbuf,
1688 (u8*)mtask->hdrext);
7ba24713
AA		 1689 if (iscsi_sendhdr(conn, &mtask->headbuf, mtask->data_count)) {
1690 mtask->xmstate |= XMSTATE_IMM_HDR;
1691 if (mtask->data_count)
1692 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1693 return -EAGAIN;
1694 }
1695 }
1696
1697 if (mtask->xmstate & XMSTATE_IMM_DATA) {
1698 BUG_ON(!mtask->data_count);
1699 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1700 /* FIXME: implement.
1701 * Virtual buffer could be spreaded across multiple pages...
1702 */
1703 do {
1704 if (iscsi_sendpage(conn, &mtask->sendbuf,
1705 &mtask->data_count, &mtask->sent)) {
1706 mtask->xmstate |= XMSTATE_IMM_DATA;
1707 return -EAGAIN;
1708 }
1709 } while (mtask->data_count);
1710 }
1711
1712 BUG_ON(mtask->xmstate != XMSTATE_IDLE);
1713 return 0;
1714}
1715
1716static inline int
1717handle_xmstate_r_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1718{
1719 ctask->xmstate &= ~XMSTATE_R_HDR;
42f72aa9 1720 if (conn->hdrdgst_en)
af973481 1721 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext);
7ba24713
AA		 1722 if (!iscsi_sendhdr(conn, &ctask->headbuf, 0)) {
1723 BUG_ON(ctask->xmstate != XMSTATE_IDLE);
1724 return 0; /* wait for Data-In */
1725 }
1726 ctask->xmstate |= XMSTATE_R_HDR;
1727 return -EAGAIN;
1728}
1729
1730static inline int
1731handle_xmstate_w_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1732{
1733 ctask->xmstate &= ~XMSTATE_W_HDR;
42f72aa9 1734 if (conn->hdrdgst_en)
af973481 1735 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext);
7ba24713
AA		 1736 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->imm_count)) {
1737 ctask->xmstate |= XMSTATE_W_HDR;
1738 return -EAGAIN;
1739 }
1740 return 0;
1741}
1742
1743static inline int
1744handle_xmstate_data_digest(struct iscsi_conn *conn,
1745 struct iscsi_cmd_task *ctask)
1746{
1747 ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
1748 debug_tcp("resent data digest 0x%x\n", ctask->datadigest);
1749 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1750 &ctask->datadigest, 0)) {
1751 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1752 debug_tcp("resent data digest 0x%x fail!\n",
1753 ctask->datadigest);
1754 return -EAGAIN;
1755 }
1756 return 0;
1757}
1758
1759static inline int
1760handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1761{
1762 BUG_ON(!ctask->imm_count);
1763 ctask->xmstate &= ~XMSTATE_IMM_DATA;
1764
1765 if (conn->datadgst_en) {
1766 iscsi_data_digest_init(conn, ctask);
1767 ctask->immdigest = 0;
1768 }
1769
1770 for (;;) {
1771 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->imm_count,
1772 &ctask->sent)) {
1773 ctask->xmstate |= XMSTATE_IMM_DATA;
1774 if (conn->datadgst_en) {
1775 crypto_digest_final(conn->data_tx_tfm,
1776 (u8*)&ctask->immdigest);
1777 debug_tcp("tx imm sendpage fail 0x%x\n",
1778 ctask->datadigest);
1779 }
1780 return -EAGAIN;
1781 }
1782 if (conn->datadgst_en)
7cae5159
MC		 1783 crypto_digest_update(conn->data_tx_tfm,
1784 &ctask->sendbuf.sg, 1);
7ba24713
AA		 1785
1786 if (!ctask->imm_count)
1787 break;
1788 iscsi_buf_init_sg(&ctask->sendbuf,
1789 &ctask->sg[ctask->sg_count++]);
1790 }
1791
1792 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1793 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1794 &ctask->immdigest, 1)) {
1795 debug_tcp("sending imm digest 0x%x fail!\n",
1796 ctask->immdigest);
1797 return -EAGAIN;
1798 }
1799 debug_tcp("sending imm digest 0x%x\n", ctask->immdigest);
1800 }
1801
1802 return 0;
1803}
1804
1805static inline int
1806handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1807{
1808 struct iscsi_data_task *dtask;
1809
1810 ctask->xmstate |= XMSTATE_UNS_DATA;
1811 if (ctask->xmstate & XMSTATE_UNS_INIT) {
1812 iscsi_unsolicit_data_init(conn, ctask);
1813 BUG_ON(!ctask->dtask);
1814 dtask = ctask->dtask;
af973481
MC		 1815 if (conn->hdrdgst_en)
1816 iscsi_hdr_digest(conn, &ctask->headbuf,
1817 (u8*)dtask->hdrext);
7ba24713
AA		 1818 ctask->xmstate &= ~XMSTATE_UNS_INIT;
1819 }
1820 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->data_count)) {
1821 ctask->xmstate &= ~XMSTATE_UNS_DATA;
1822 ctask->xmstate |= XMSTATE_UNS_HDR;
1823 return -EAGAIN;
1824 }
1825
1826 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1827 ctask->itt, ctask->unsol_count, ctask->sent);
1828 return 0;
1829}
1830
1831static inline int
1832handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1833{
1834 struct iscsi_data_task *dtask = ctask->dtask;
1835
1836 BUG_ON(!ctask->data_count);
1837 ctask->xmstate &= ~XMSTATE_UNS_DATA;
1838
1839 if (conn->datadgst_en) {
1840 iscsi_data_digest_init(conn, ctask);
1841 dtask->digest = 0;
1842 }
1843
1844 for (;;) {
1845 int start = ctask->sent;
1846
1847 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->data_count,
1848 &ctask->sent)) {
1849 ctask->unsol_count -= ctask->sent - start;
1850 ctask->xmstate |= XMSTATE_UNS_DATA;
1851 /* will continue with this ctask later.. */
1852 if (conn->datadgst_en) {
1853 crypto_digest_final(conn->data_tx_tfm,
1854 (u8 *)&dtask->digest);
1855 debug_tcp("tx uns data fail 0x%x\n",
1856 dtask->digest);
1857 }
1858 return -EAGAIN;
1859 }
1860
1861 BUG_ON(ctask->sent > ctask->total_length);
1862 ctask->unsol_count -= ctask->sent - start;
1863
1864 /*
1865 * XXX:we may run here with un-initial sendbuf.
1866 * so pass it
1867 */
1868 if (conn->datadgst_en && ctask->sent - start > 0)
7cae5159
MC		 1869 crypto_digest_update(conn->data_tx_tfm,
1870 &ctask->sendbuf.sg, 1);
7ba24713
AA		 1871
1872 if (!ctask->data_count)
1873 break;
1874 iscsi_buf_init_sg(&ctask->sendbuf,
1875 &ctask->sg[ctask->sg_count++]);
1876 }
1877 BUG_ON(ctask->unsol_count < 0);
1878
1879 /*
1880 * Done with the Data-Out. Next, check if we need
1881 * to send another unsolicited Data-Out.
1882 */
1883 if (ctask->unsol_count) {
1884 if (conn->datadgst_en) {
1885 if (iscsi_digest_final_send(conn, ctask,
1886 &dtask->digestbuf,
1887 &dtask->digest, 1)) {
1888 debug_tcp("send uns digest 0x%x fail\n",
1889 dtask->digest);
1890 return -EAGAIN;
1891 }
1892 debug_tcp("sending uns digest 0x%x, more uns\n",
1893 dtask->digest);
1894 }
1895 ctask->xmstate |= XMSTATE_UNS_INIT;
1896 return 1;
1897 }
1898
1899 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1900 if (iscsi_digest_final_send(conn, ctask,
1901 &dtask->digestbuf,
1902 &dtask->digest, 1)) {
1903 debug_tcp("send last uns digest 0x%x fail\n",
1904 dtask->digest);
1905 return -EAGAIN;
1906 }
1907 debug_tcp("sending uns digest 0x%x\n",dtask->digest);
1908 }
1909
1910 return 0;
1911}
1912
1913static inline int
1914handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1915{
1916 struct iscsi_session *session = conn->session;
1917 struct iscsi_r2t_info *r2t = ctask->r2t;
1918 struct iscsi_data_task *dtask = r2t->dtask;
1919 int left;
1920
1921 ctask->xmstate &= ~XMSTATE_SOL_DATA;
1922 ctask->dtask = dtask;
1923
1924 if (conn->datadgst_en) {
1925 iscsi_data_digest_init(conn, ctask);
1926 dtask->digest = 0;
1927 }
1928solicit_again:
1929 /*
1930 * send Data-Out whitnin this R2T sequence.
1931 */
1932 if (!r2t->data_count)
1933 goto data_out_done;
1934
1935 if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) {
1936 ctask->xmstate |= XMSTATE_SOL_DATA;
1937 /* will continue with this ctask later.. */
1938 if (conn->datadgst_en) {
1939 crypto_digest_final(conn->data_tx_tfm,
1940 (u8 *)&dtask->digest);
1941 debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
1942 }
1943 return -EAGAIN;
1944 }
1945
1946 BUG_ON(r2t->data_count < 0);
1947 if (conn->datadgst_en)
7cae5159 1948 crypto_digest_update(conn->data_tx_tfm, &r2t->sendbuf.sg, 1);
7ba24713
AA		 1949
1950 if (r2t->data_count) {
1951 BUG_ON(ctask->sc->use_sg == 0);
1952 if (!iscsi_buf_left(&r2t->sendbuf)) {
1953 BUG_ON(ctask->bad_sg == r2t->sg);
1954 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1955 r2t->sg += 1;
1956 }
1957 goto solicit_again;
1958 }
1959
1960data_out_done:
1961 /*
1962 * Done with this Data-Out. Next, check if we have
1963 * to send another Data-Out for this R2T.
1964 */
1965 BUG_ON(r2t->data_length - r2t->sent < 0);
1966 left = r2t->data_length - r2t->sent;
1967 if (left) {
1968 if (conn->datadgst_en) {
1969 if (iscsi_digest_final_send(conn, ctask,
1970 &dtask->digestbuf,
1971 &dtask->digest, 1)) {
1972 debug_tcp("send r2t data digest 0x%x"
1973 "fail\n", dtask->digest);
1974 return -EAGAIN;
1975 }
1976 debug_tcp("r2t data send digest 0x%x\n",
1977 dtask->digest);
1978 }
1979 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1980 ctask->xmstate |= XMSTATE_SOL_DATA;
1981 ctask->xmstate &= ~XMSTATE_SOL_HDR;
1982 return 1;
1983 }
1984
1985 /*
1986 * Done with this R2T. Check if there are more
1987 * outstanding R2Ts ready to be processed.
1988 */
1989 BUG_ON(ctask->r2t_data_count - r2t->data_length < 0);
1990 if (conn->datadgst_en) {
1991 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
1992 &dtask->digest, 1)) {
1993 debug_tcp("send last r2t data digest 0x%x"
1994 "fail\n", dtask->digest);
1995 return -EAGAIN;
1996 }
1997 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
1998 }
1999
2000 ctask->r2t_data_count -= r2t->data_length;
2001 ctask->r2t = NULL;
2002 spin_lock_bh(&session->lock);
2003 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
2004 spin_unlock_bh(&session->lock);
2005 if (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
2006 ctask->r2t = r2t;
2007 ctask->xmstate |= XMSTATE_SOL_DATA;
2008 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2009 return 1;
2010 }
2011
2012 return 0;
2013}
2014
2015static inline int
2016handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2017{
2018 struct iscsi_data_task *dtask = ctask->dtask;
2019 int sent;
2020
2021 ctask->xmstate &= ~XMSTATE_W_PAD;
2022 iscsi_buf_init_virt(&ctask->sendbuf, (char*)&ctask->pad,
2023 ctask->pad_count);
2024 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->pad_count, &sent)) {
2025 ctask->xmstate |= XMSTATE_W_PAD;
2026 return -EAGAIN;
2027 }
2028
2029 if (conn->datadgst_en) {
7cae5159 2030 crypto_digest_update(conn->data_tx_tfm, &ctask->sendbuf.sg, 1);
7ba24713
AA		 2031 /* imm data? */
2032 if (!dtask) {
2033 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
2034 &ctask->immdigest, 1)) {
2035 debug_tcp("send padding digest 0x%x"
2036 "fail!\n", ctask->immdigest);
2037 return -EAGAIN;
2038 }
2039 debug_tcp("done with padding, digest 0x%x\n",
2040 ctask->datadigest);
2041 } else {
2042 if (iscsi_digest_final_send(conn, ctask,
2043 &dtask->digestbuf,
2044 &dtask->digest, 1)) {
2045 debug_tcp("send padding digest 0x%x"
2046 "fail\n", dtask->digest);
2047 return -EAGAIN;
2048 }
2049 debug_tcp("done with padding, digest 0x%x\n",
2050 dtask->digest);
2051 }
2052 }
2053
2054 return 0;
2055}
2056
2057static int
2058iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2059{
2060 int rc = 0;
2061
2062 debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
2063 conn->id, ctask->xmstate, ctask->itt);
2064
2065 /*
2066 * serialize with TMF AbortTask
2067 */
2068 if (ctask->mtask)
2069 return rc;
2070
2071 if (ctask->xmstate & XMSTATE_R_HDR) {
2072 rc = handle_xmstate_r_hdr(conn, ctask);
2073 return rc;
2074 }
2075
2076 if (ctask->xmstate & XMSTATE_W_HDR) {
2077 rc = handle_xmstate_w_hdr(conn, ctask);
2078 if (rc)
2079 return rc;
2080 }
2081
2082 /* XXX: for data digest xmit recover */
2083 if (ctask->xmstate & XMSTATE_DATA_DIGEST) {
2084 rc = handle_xmstate_data_digest(conn, ctask);
2085 if (rc)
2086 return rc;
2087 }
2088
2089 if (ctask->xmstate & XMSTATE_IMM_DATA) {
2090 rc = handle_xmstate_imm_data(conn, ctask);
2091 if (rc)
2092 return rc;
2093 }
2094
2095 if (ctask->xmstate & XMSTATE_UNS_HDR) {
2096 BUG_ON(!ctask->unsol_count);
2097 ctask->xmstate &= ~XMSTATE_UNS_HDR;
2098unsolicit_head_again:
2099 rc = handle_xmstate_uns_hdr(conn, ctask);
2100 if (rc)
2101 return rc;
2102 }
2103
2104 if (ctask->xmstate & XMSTATE_UNS_DATA) {
2105 rc = handle_xmstate_uns_data(conn, ctask);
2106 if (rc == 1)
2107 goto unsolicit_head_again;
2108 else if (rc)
2109 return rc;
2110 goto done;
2111 }
2112
2113 if (ctask->xmstate & XMSTATE_SOL_HDR) {
2114 struct iscsi_r2t_info *r2t;
2115
2116 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2117 ctask->xmstate |= XMSTATE_SOL_DATA;
2118 if (!ctask->r2t)
2119 __kfifo_get(ctask->r2tqueue, (void*)&ctask->r2t,
2120 sizeof(void*));
2121solicit_head_again:
2122 r2t = ctask->r2t;
af973481 2123 if (conn->hdrdgst_en)
42f72aa9 2124 iscsi_hdr_digest(conn, &r2t->headbuf,
af973481 2125 (u8*)r2t->dtask->hdrext);
7ba24713
AA		 2126 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) {
2127 ctask->xmstate &= ~XMSTATE_SOL_DATA;
2128 ctask->xmstate |= XMSTATE_SOL_HDR;
2129 return -EAGAIN;
2130 }
2131
2132 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
2133 r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
2134 r2t->sent);
2135 }
2136
2137 if (ctask->xmstate & XMSTATE_SOL_DATA) {
2138 rc = handle_xmstate_sol_data(conn, ctask);
2139 if (rc == 1)
2140 goto solicit_head_again;
2141 if (rc)
2142 return rc;
2143 }
2144
2145done:
2146 /*
2147 * Last thing to check is whether we need to send write
2148 * padding. Note that we check for xmstate equality, not just the bit.
2149 */
2150 if (ctask->xmstate == XMSTATE_W_PAD)
2151 rc = handle_xmstate_w_pad(conn, ctask);
2152
2153 return rc;
2154}
2155
2156/**
2157 * iscsi_data_xmit - xmit any command into the scheduled connection
2158 * @conn: iscsi connection
2159 *
2160 * Notes:
2161 * The function can return -EAGAIN in which case the caller must
2162 * re-schedule it again later or recover. '0' return code means
2163 * successful xmit.
2164 **/
2165static int
2166iscsi_data_xmit(struct iscsi_conn *conn)
2167{
2168 if (unlikely(conn->suspend_tx)) {
2169 debug_tcp("conn %d Tx suspended!\n", conn->id);
2170 return 0;
2171 }
2172
2173 /*
2174 * Transmit in the following order:
2175 *
2176 * 1) un-finished xmit (ctask or mtask)
2177 * 2) immediate control PDUs
2178 * 3) write data
2179 * 4) SCSI commands
2180 * 5) non-immediate control PDUs
2181 *
2182 * No need to lock around __kfifo_get as long as
2183 * there's one producer and one consumer.
2184 */
2185
2186 BUG_ON(conn->ctask && conn->mtask);
2187
2188 if (conn->ctask) {
2189 if (iscsi_ctask_xmit(conn, conn->ctask))
2190 goto again;
2191 /* done with this in-progress ctask */
2192 conn->ctask = NULL;
2193 }
2194 if (conn->mtask) {
2195 if (iscsi_mtask_xmit(conn, conn->mtask))
2196 goto again;
2197 /* done with this in-progress mtask */
2198 conn->mtask = NULL;
2199 }
2200
2201 /* process immediate first */
2202 if (unlikely(__kfifo_len(conn->immqueue))) {
2203 struct iscsi_session *session = conn->session;
2204 while (__kfifo_get(conn->immqueue, (void*)&conn->mtask,
2205 sizeof(void*))) {
2206 if (iscsi_mtask_xmit(conn, conn->mtask))
2207 goto again;
2208
2209 if (conn->mtask->hdr.itt ==
2210 cpu_to_be32(ISCSI_RESERVED_TAG)) {
2211 spin_lock_bh(&session->lock);
2212 __kfifo_put(session->mgmtpool.queue,
2213 (void*)&conn->mtask, sizeof(void*));
2214 spin_unlock_bh(&session->lock);
2215 }
2216 }
2217 /* done with this mtask */
2218 conn->mtask = NULL;
2219 }
2220
2221 /* process write queue */
2222 while (__kfifo_get(conn->writequeue, (void*)&conn->ctask,
2223 sizeof(void*))) {
2224 if (iscsi_ctask_xmit(conn, conn->ctask))
2225 goto again;
2226 }
2227
2228 /* process command queue */
2229 while (__kfifo_get(conn->xmitqueue, (void*)&conn->ctask,
2230 sizeof(void*))) {
2231 if (iscsi_ctask_xmit(conn, conn->ctask))
2232 goto again;
2233 }
2234 /* done with this ctask */
2235 conn->ctask = NULL;
2236
2237 /* process the rest control plane PDUs, if any */
2238 if (unlikely(__kfifo_len(conn->mgmtqueue))) {
2239 struct iscsi_session *session = conn->session;
2240
2241 while (__kfifo_get(conn->mgmtqueue, (void*)&conn->mtask,
2242 sizeof(void*))) {
2243 if (iscsi_mtask_xmit(conn, conn->mtask))
2244 goto again;
2245
2246 if (conn->mtask->hdr.itt ==
2247 cpu_to_be32(ISCSI_RESERVED_TAG)) {
2248 spin_lock_bh(&session->lock);
2249 __kfifo_put(session->mgmtpool.queue,
2250 (void*)&conn->mtask,
2251 sizeof(void*));
2252 spin_unlock_bh(&session->lock);
2253 }
2254 }
2255 /* done with this mtask */
2256 conn->mtask = NULL;
2257 }
2258
2259 return 0;
2260
2261again:
2262 if (unlikely(conn->suspend_tx))
2263 return 0;
2264
2265 return -EAGAIN;
2266}
2267
2268static void
2269iscsi_xmitworker(void *data)
2270{
2271 struct iscsi_conn *conn = data;
2272
2273 /*
2274 * serialize Xmit worker on a per-connection basis.
2275 */
0b950672 2276 mutex_lock(&conn->xmitmutex);
7ba24713 2277 if (iscsi_data_xmit(conn))
55e3299d 2278 scsi_queue_work(conn->session->host, &conn->xmitwork);
0b950672 2279 mutex_unlock(&conn->xmitmutex);
7ba24713
AA		 2280}
2281
2282#define FAILURE_BAD_HOST 1
2283#define FAILURE_SESSION_FAILED 2
2284#define FAILURE_SESSION_FREED 3
2285#define FAILURE_WINDOW_CLOSED 4
2286#define FAILURE_SESSION_TERMINATE 5
2287
2288static int
2289iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
2290{
2291 struct Scsi_Host *host;
2292 int reason = 0;
2293 struct iscsi_session *session;
2294 struct iscsi_conn *conn = NULL;
2295 struct iscsi_cmd_task *ctask = NULL;
2296
2297 sc->scsi_done = done;
2298 sc->result = 0;
2299
2300 host = sc->device->host;
2301 session = iscsi_hostdata(host->hostdata);
2302 BUG_ON(host != session->host);
2303
2304 spin_lock(&session->lock);
2305
2306 if (session->state != ISCSI_STATE_LOGGED_IN) {
2307 if (session->state == ISCSI_STATE_FAILED) {
2308 reason = FAILURE_SESSION_FAILED;
2309 goto reject;
2310 } else if (session->state == ISCSI_STATE_TERMINATE) {
2311 reason = FAILURE_SESSION_TERMINATE;
2312 goto fault;
2313 }
2314 reason = FAILURE_SESSION_FREED;
2315 goto fault;
2316 }
2317
2318 /*
2319 * Check for iSCSI window and take care of CmdSN wrap-around
2320 */
2321 if ((int)(session->max_cmdsn - session->cmdsn) < 0) {
2322 reason = FAILURE_WINDOW_CLOSED;
2323 goto reject;
2324 }
2325
2326 conn = session->leadconn;
2327
2328 __kfifo_get(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
2329 BUG_ON(ctask->sc);
2330
2331 sc->SCp.phase = session->age;
2332 sc->SCp.ptr = (char*)ctask;
2333 iscsi_cmd_init(conn, ctask, sc);
2334
2335 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
2336 debug_scsi(
2337 "ctask enq [%s cid %d sc %lx itt 0x%x len %d cmdsn %d win %d]\n",
2338 sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read",
2339 conn->id, (long)sc, ctask->itt, sc->request_bufflen,
2340 session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1);
2341 spin_unlock(&session->lock);
2342
55e3299d 2343 scsi_queue_work(host, &conn->xmitwork);
7ba24713
AA		 2344 return 0;
2345
2346reject:
2347 spin_unlock(&session->lock);
2348 debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason);
2349 return SCSI_MLQUEUE_HOST_BUSY;
2350
2351fault:
2352 spin_unlock(&session->lock);
2353 printk(KERN_ERR "iscsi_tcp: cmd 0x%x is not queued (%d)\n",
2354 sc->cmnd[0], reason);
2355 sc->sense_buffer[0] = 0x70;
2356 sc->sense_buffer[2] = NOT_READY;
2357 sc->sense_buffer[7] = 0x6;
2358 sc->sense_buffer[12] = 0x08;
2359 sc->sense_buffer[13] = 0x00;
2360 sc->result = (DID_NO_CONNECT << 16);
2361 sc->resid = sc->request_bufflen;
2362 sc->scsi_done(sc);
2363 return 0;
2364}
2365
9e3961ba
MC		 2366static int
2367iscsi_change_queue_depth(struct scsi_device *sdev, int depth)
2368{
2369 if (depth > ISCSI_MAX_CMD_PER_LUN)
2370 depth = ISCSI_MAX_CMD_PER_LUN;
2371 scsi_adjust_queue_depth(sdev, scsi_get_tag_type(sdev), depth);
2372 return sdev->queue_depth;
2373}
2374
7ba24713
AA		 2375static int
2376iscsi_pool_init(struct iscsi_queue *q, int max, void ***items, int item_size)
2377{
2378 int i;
2379
2380 *items = kmalloc(max * sizeof(void*), GFP_KERNEL);
2381 if (*items == NULL)
2382 return -ENOMEM;
2383
2384 q->max = max;
2385 q->pool = kmalloc(max * sizeof(void*), GFP_KERNEL);
2386 if (q->pool == NULL) {
2387 kfree(*items);
2388 return -ENOMEM;
2389 }
2390
2391 q->queue = kfifo_init((void*)q->pool, max * sizeof(void*),
2392 GFP_KERNEL, NULL);
2393 if (q->queue == ERR_PTR(-ENOMEM)) {
2394 kfree(q->pool);
2395 kfree(*items);
2396 return -ENOMEM;
2397 }
2398
2399 for (i = 0; i < max; i++) {
2400 q->pool[i] = kmalloc(item_size, GFP_KERNEL);
2401 if (q->pool[i] == NULL) {
2402 int j;
2403
2404 for (j = 0; j < i; j++)
2405 kfree(q->pool[j]);
2406
2407 kfifo_free(q->queue);
2408 kfree(q->pool);
2409 kfree(*items);
2410 return -ENOMEM;
2411 }
2412 memset(q->pool[i], 0, item_size);
2413 (*items)[i] = q->pool[i];
2414 __kfifo_put(q->queue, (void*)&q->pool[i], sizeof(void*));
2415 }
2416 return 0;
2417}
2418
2419static void
2420iscsi_pool_free(struct iscsi_queue *q, void **items)
2421{
2422 int i;
2423
2424 for (i = 0; i < q->max; i++)
2425 kfree(items[i]);
2426 kfree(q->pool);
2427 kfree(items);
2428}
2429
7b8631b5
MC		 2430static struct iscsi_cls_conn *
2431iscsi_conn_create(struct Scsi_Host *shost, uint32_t conn_idx)
7ba24713 2432{
7b8631b5
MC		 2433 struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
2434 struct iscsi_conn *conn;
2435 struct iscsi_cls_conn *cls_conn;
7ba24713 2436
7b8631b5
MC		 2437 cls_conn = iscsi_create_conn(hostdata_session(shost->hostdata),
2438 conn_idx);
2439 if (!cls_conn)
2440 return NULL;
2441 conn = cls_conn->dd_data;
7ba24713 2442
7b8631b5 2443 memset(conn, 0, sizeof(struct iscsi_conn));
7ba24713
AA		 2444 conn->c_stage = ISCSI_CONN_INITIAL_STAGE;
2445 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2446 conn->id = conn_idx;
2447 conn->exp_statsn = 0;
2448 conn->tmabort_state = TMABORT_INITIAL;
2449
2450 /* initial operational parameters */
2451 conn->hdr_size = sizeof(struct iscsi_hdr);
2452 conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2453 conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2454
2455 spin_lock_init(&conn->lock);
2456
2457 /* initialize general xmit PDU commands queue */
2458 conn->xmitqueue = kfifo_alloc(session->cmds_max * sizeof(void*),
2459 GFP_KERNEL, NULL);
2460 if (conn->xmitqueue == ERR_PTR(-ENOMEM))
2461 goto xmitqueue_alloc_fail;
2462
2463 /* initialize write response PDU commands queue */
2464 conn->writequeue = kfifo_alloc(session->cmds_max * sizeof(void*),
2465 GFP_KERNEL, NULL);
2466 if (conn->writequeue == ERR_PTR(-ENOMEM))
2467 goto writequeue_alloc_fail;
2468
2469 /* initialize general immediate & non-immediate PDU commands queue */
2470 conn->immqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2471 GFP_KERNEL, NULL);
2472 if (conn->immqueue == ERR_PTR(-ENOMEM))
2473 goto immqueue_alloc_fail;
2474
2475 conn->mgmtqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2476 GFP_KERNEL, NULL);
2477 if (conn->mgmtqueue == ERR_PTR(-ENOMEM))
2478 goto mgmtqueue_alloc_fail;
2479
2480 INIT_WORK(&conn->xmitwork, iscsi_xmitworker, conn);
2481
2482 /* allocate login_mtask used for the login/text sequences */
2483 spin_lock_bh(&session->lock);
2484 if (!__kfifo_get(session->mgmtpool.queue,
2485 (void*)&conn->login_mtask,
2486 sizeof(void*))) {
2487 spin_unlock_bh(&session->lock);
2488 goto login_mtask_alloc_fail;
2489 }
2490 spin_unlock_bh(&session->lock);
2491
2492 /* allocate initial PDU receive place holder */
2493 if (conn->data_size <= PAGE_SIZE)
2494 conn->data = kmalloc(conn->data_size, GFP_KERNEL);
2495 else
2496 conn->data = (void*)__get_free_pages(GFP_KERNEL,
2497 get_order(conn->data_size));
2498 if (!conn->data)
2499 goto max_recv_dlenght_alloc_fail;
2500
2501 init_timer(&conn->tmabort_timer);
0b950672 2502 mutex_init(&conn->xmitmutex);
7ba24713
AA		 2503 init_waitqueue_head(&conn->ehwait);
2504
7b8631b5 2505 return cls_conn;
7ba24713
AA		 2506
2507max_recv_dlenght_alloc_fail:
2508 spin_lock_bh(&session->lock);
2509 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2510 sizeof(void*));
2511 spin_unlock_bh(&session->lock);
2512login_mtask_alloc_fail:
2513 kfifo_free(conn->mgmtqueue);
2514mgmtqueue_alloc_fail:
2515 kfifo_free(conn->immqueue);
2516immqueue_alloc_fail:
2517 kfifo_free(conn->writequeue);
2518writequeue_alloc_fail:
2519 kfifo_free(conn->xmitqueue);
2520xmitqueue_alloc_fail:
7b8631b5
MC		 2521 iscsi_destroy_conn(cls_conn);
2522 return NULL;
7ba24713
AA		 2523}
2524
2525static void
7b8631b5 2526iscsi_conn_destroy(struct iscsi_cls_conn *cls_conn)
7ba24713 2527{
7b8631b5 2528 struct iscsi_conn *conn = cls_conn->dd_data;
7ba24713 2529 struct iscsi_session *session = conn->session;
3e97c7e6 2530 unsigned long flags;
7ba24713 2531
0b950672 2532 mutex_lock(&conn->xmitmutex);
7ba24713
AA		 2533 set_bit(SUSPEND_BIT, &conn->suspend_tx);
2534 if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE && conn->sock) {
2535 struct sock *sk = conn->sock->sk;
2536
2537 /*
2538 * conn_start() has never been called!
2539 * need to cleanup the socket.
2540 */
2541 write_lock_bh(&sk->sk_callback_lock);
2542 set_bit(SUSPEND_BIT, &conn->suspend_rx);
2543 write_unlock_bh(&sk->sk_callback_lock);
2544
2545 sock_hold(conn->sock->sk);
2546 iscsi_conn_restore_callbacks(conn);
2547 sock_put(conn->sock->sk);
2548 sock_release(conn->sock);
2549 conn->sock = NULL;
2550 }
2551
2552 spin_lock_bh(&session->lock);
2553 conn->c_stage = ISCSI_CONN_CLEANUP_WAIT;
2554 if (session->leadconn == conn) {
2555 /*
2556 * leading connection? then give up on recovery.
2557 */
2558 session->state = ISCSI_STATE_TERMINATE;
2559 wake_up(&conn->ehwait);
2560 }
2561 spin_unlock_bh(&session->lock);
2562
0b950672 2563 mutex_unlock(&conn->xmitmutex);
7ba24713
AA		 2564
2565 /*
2566 * Block until all in-progress commands for this connection
2567 * time out or fail.
2568 */
2569 for (;;) {
3e97c7e6 2570 spin_lock_irqsave(session->host->host_lock, flags);
7ba24713 2571 if (!session->host->host_busy) { /* OK for ERL == 0 */
3e97c7e6 2572 spin_unlock_irqrestore(session->host->host_lock, flags);
7ba24713
AA		 2573 break;
2574 }
3e97c7e6 2575 spin_unlock_irqrestore(session->host->host_lock, flags);
7ba24713
AA		 2576 msleep_interruptible(500);
2577 printk("conn_destroy(): host_busy %d host_failed %d\n",
2578 session->host->host_busy, session->host->host_failed);
2579 /*
2580 * force eh_abort() to unblock
2581 */
2582 wake_up(&conn->ehwait);
2583 }
2584
2585 /* now free crypto */
2586 if (conn->hdrdgst_en || conn->datadgst_en) {
2587 if (conn->tx_tfm)
2588 crypto_free_tfm(conn->tx_tfm);
2589 if (conn->rx_tfm)
2590 crypto_free_tfm(conn->rx_tfm);
2591 if (conn->data_tx_tfm)
2592 crypto_free_tfm(conn->data_tx_tfm);
2593 if (conn->data_rx_tfm)
2594 crypto_free_tfm(conn->data_rx_tfm);
2595 }
2596
2597 /* free conn->data, size = MaxRecvDataSegmentLength */
2598 if (conn->data_size <= PAGE_SIZE)
2599 kfree(conn->data);
2600 else
2601 free_pages((unsigned long)conn->data,
2602 get_order(conn->data_size));
2603
2604 spin_lock_bh(&session->lock);
2605 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2606 sizeof(void*));
2607 list_del(&conn->item);
2608 if (list_empty(&session->connections))
2609 session->leadconn = NULL;
2610 if (session->leadconn && session->leadconn == conn)
2611 session->leadconn = container_of(session->connections.next,
2612 struct iscsi_conn, item);
2613
2614 if (session->leadconn == NULL)
2615 /* none connections exits.. reset sequencing */
2616 session->cmdsn = session->max_cmdsn = session->exp_cmdsn = 1;
2617 spin_unlock_bh(&session->lock);
2618
2619 kfifo_free(conn->xmitqueue);
2620 kfifo_free(conn->writequeue);
2621 kfifo_free(conn->immqueue);
2622 kfifo_free(conn->mgmtqueue);
7b8631b5
MC		 2623
2624 iscsi_destroy_conn(cls_conn);
7ba24713
AA		 2625}
2626
2627static int
2628iscsi_conn_bind(iscsi_sessionh_t sessionh, iscsi_connh_t connh,
2629 uint32_t transport_fd, int is_leading)
2630{
2631 struct iscsi_session *session = iscsi_ptr(sessionh);
2632 struct iscsi_conn *tmp = ERR_PTR(-EEXIST), *conn = iscsi_ptr(connh);
2633 struct sock *sk;
2634 struct socket *sock;
2635 int err;
2636
2637 /* lookup for existing socket */
2638 sock = sockfd_lookup(transport_fd, &err);
2639 if (!sock) {
2640 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
2641 return -EEXIST;
2642 }
2643
2644 /* lookup for existing connection */
2645 spin_lock_bh(&session->lock);
2646 list_for_each_entry(tmp, &session->connections, item) {
2647 if (tmp == conn) {
2648 if (conn->c_stage != ISCSI_CONN_STOPPED ||
2649 conn->stop_stage == STOP_CONN_TERM) {
2650 printk(KERN_ERR "iscsi_tcp: can't bind "
2651 "non-stopped connection (%d:%d)\n",
2652 conn->c_stage, conn->stop_stage);
2653 spin_unlock_bh(&session->lock);
2654 return -EIO;
2655 }
2656 break;
2657 }
2658 }
2659 if (tmp != conn) {
2660 /* bind new iSCSI connection to session */
2661 conn->session = session;
2662
2663 list_add(&conn->item, &session->connections);
2664 }
2665 spin_unlock_bh(&session->lock);
2666
2667 if (conn->stop_stage != STOP_CONN_SUSPEND) {
2668 /* bind iSCSI connection and socket */
2669 conn->sock = sock;
2670
2671 /* setup Socket parameters */
2672 sk = sock->sk;
2673 sk->sk_reuse = 1;
2674 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
2675 sk->sk_allocation = GFP_ATOMIC;
2676
2677 /* FIXME: disable Nagle's algorithm */
2678
2679 /*
2680 * Intercept TCP callbacks for sendfile like receive
2681 * processing.
2682 */
2683 iscsi_conn_set_callbacks(conn);
2684
56851698
FT		 2685 conn->sendpage = conn->sock->ops->sendpage;
2686
7ba24713
AA		 2687 /*
2688 * set receive state machine into initial state
2689 */
2690 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2691 }
2692
2693 if (is_leading)
2694 session->leadconn = conn;
2695
2696 /*
2697 * Unblock xmitworker(), Login Phase will pass through.
2698 */
2699 clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2700 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2701
2702 return 0;
2703}
2704
2705static int
2706iscsi_conn_start(iscsi_connh_t connh)
2707{
2708 struct iscsi_conn *conn = iscsi_ptr(connh);
2709 struct iscsi_session *session = conn->session;
2710 struct sock *sk;
2711
2712 /* FF phase warming up... */
2713
2714 if (session == NULL) {
2715 printk(KERN_ERR "iscsi_tcp: can't start unbound connection\n");
2716 return -EPERM;
2717 }
2718
2719 sk = conn->sock->sk;
2720
2721 write_lock_bh(&sk->sk_callback_lock);
2722 spin_lock_bh(&session->lock);
2723 conn->c_stage = ISCSI_CONN_STARTED;
2724 session->state = ISCSI_STATE_LOGGED_IN;
2725
2726 switch(conn->stop_stage) {
2727 case STOP_CONN_RECOVER:
2728 /*
2729 * unblock eh_abort() if it is blocked. re-try all
2730 * commands after successful recovery
2731 */
2732 session->conn_cnt++;
2733 conn->stop_stage = 0;
2734 conn->tmabort_state = TMABORT_INITIAL;
2735 session->age++;
2736 wake_up(&conn->ehwait);
2737 break;
2738 case STOP_CONN_TERM:
2739 session->conn_cnt++;
2740 conn->stop_stage = 0;
2741 break;
2742 case STOP_CONN_SUSPEND:
2743 conn->stop_stage = 0;
2744 clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2745 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2746 break;
2747 default:
2748 break;
2749 }
2750 spin_unlock_bh(&session->lock);
2751 write_unlock_bh(&sk->sk_callback_lock);
2752
2753 return 0;
2754}
2755
2756static void
2757iscsi_conn_stop(iscsi_connh_t connh, int flag)
2758{
2759 struct iscsi_conn *conn = iscsi_ptr(connh);
2760 struct iscsi_session *session = conn->session;
2761 struct sock *sk;
2762 unsigned long flags;
2763
2764 BUG_ON(!conn->sock);
2765 sk = conn->sock->sk;
2766 write_lock_bh(&sk->sk_callback_lock);
2767 set_bit(SUSPEND_BIT, &conn->suspend_rx);
2768 write_unlock_bh(&sk->sk_callback_lock);
2769
0b950672 2770 mutex_lock(&conn->xmitmutex);
7ba24713
AA		 2771
2772 spin_lock_irqsave(session->host->host_lock, flags);
2773 spin_lock(&session->lock);
2774 conn->stop_stage = flag;
2775 conn->c_stage = ISCSI_CONN_STOPPED;
2776 set_bit(SUSPEND_BIT, &conn->suspend_tx);
2777
2778 if (flag != STOP_CONN_SUSPEND)
2779 session->conn_cnt--;
2780
2781 if (session->conn_cnt == 0 || session->leadconn == conn)
2782 session->state = ISCSI_STATE_FAILED;
2783
2784 spin_unlock(&session->lock);
2785 spin_unlock_irqrestore(session->host->host_lock, flags);
2786
2787 if (flag == STOP_CONN_TERM || flag == STOP_CONN_RECOVER) {
2788 struct iscsi_cmd_task *ctask;
2789 struct iscsi_mgmt_task *mtask;
2790
2791 /*
2792 * Socket must go now.
2793 */
2794 sock_hold(conn->sock->sk);
2795 iscsi_conn_restore_callbacks(conn);
2796 sock_put(conn->sock->sk);
2797
2798 /*
2799 * flush xmit queues.
2800 */
2801 spin_lock_bh(&session->lock);
2802 while (__kfifo_get(conn->writequeue, (void*)&ctask,
2803 sizeof(void*)) ||
2804 __kfifo_get(conn->xmitqueue, (void*)&ctask,
2805 sizeof(void*))) {
2806 struct iscsi_r2t_info *r2t;
2807
2808 /*
2809 * flush ctask's r2t queues
2810 */
2811 while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
2812 sizeof(void*)))
2813 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
2814 sizeof(void*));
2815
2816 spin_unlock_bh(&session->lock);
2817 local_bh_disable();
2818 iscsi_ctask_cleanup(conn, ctask);
2819 local_bh_enable();
2820 spin_lock_bh(&session->lock);
2821 }
2822 conn->ctask = NULL;
2823 while (__kfifo_get(conn->immqueue, (void*)&mtask,
2824 sizeof(void*)) ||
2825 __kfifo_get(conn->mgmtqueue, (void*)&mtask,
2826 sizeof(void*))) {
2827 __kfifo_put(session->mgmtpool.queue,
2828 (void*)&mtask, sizeof(void*));
2829 }
2830 conn->mtask = NULL;
2831 spin_unlock_bh(&session->lock);
2832
2833 /*
2834 * release socket only after we stopped data_xmit()
2835 * activity and flushed all outstandings
2836 */
2837 sock_release(conn->sock);
2838 conn->sock = NULL;
2839
2840 /*
2841 * for connection level recovery we should not calculate
2842 * header digest. conn->hdr_size used for optimization
2843 * in hdr_extract() and will be re-negotiated at
2844 * set_param() time.
2845 */
8a47cd34 2846 if (flag == STOP_CONN_RECOVER) {
7ba24713 2847 conn->hdr_size = sizeof(struct iscsi_hdr);
8a47cd34
MC		 2848 conn->hdrdgst_en = 0;
2849 conn->datadgst_en = 0;
2850 }
7ba24713 2851 }
0b950672 2852 mutex_unlock(&conn->xmitmutex);
7ba24713
AA		 2853}
2854
2855static int
2856iscsi_conn_send_generic(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
2857 char *data, uint32_t data_size)
2858{
2859 struct iscsi_session *session = conn->session;
2860 struct iscsi_nopout *nop = (struct iscsi_nopout *)hdr;
2861 struct iscsi_mgmt_task *mtask;
2862
2863 spin_lock_bh(&session->lock);
2864 if (session->state == ISCSI_STATE_TERMINATE) {
2865 spin_unlock_bh(&session->lock);
2866 return -EPERM;
2867 }
2868 if (hdr->opcode == (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) ||
2869 hdr->opcode == (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE))
2870 /*
2871 * Login and Text are sent serially, in
2872 * request-followed-by-response sequence.
2873 * Same mtask can be used. Same ITT must be used.
2874 * Note that login_mtask is preallocated at conn_create().
2875 */
2876 mtask = conn->login_mtask;
2877 else {
2878 BUG_ON(conn->c_stage == ISCSI_CONN_INITIAL_STAGE);
2879 BUG_ON(conn->c_stage == ISCSI_CONN_STOPPED);
2880
2881 if (!__kfifo_get(session->mgmtpool.queue,
2882 (void*)&mtask, sizeof(void*))) {
2883 spin_unlock_bh(&session->lock);
2884 return -ENOSPC;
2885 }
2886 }
2887
2888 /*
2889 * pre-format CmdSN and ExpStatSN for outgoing PDU.
2890 */
2891 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2892 hdr->itt = mtask->itt | (conn->id << CID_SHIFT) |
2893 (session->age << AGE_SHIFT);
2894 nop->cmdsn = cpu_to_be32(session->cmdsn);
2895 if (conn->c_stage == ISCSI_CONN_STARTED &&
2896 !(hdr->opcode & ISCSI_OP_IMMEDIATE))
2897 session->cmdsn++;
2898 } else
2899 /* do not advance CmdSN */
2900 nop->cmdsn = cpu_to_be32(session->cmdsn);
2901
2902 nop->exp_statsn = cpu_to_be32(conn->exp_statsn);
2903
2904 memcpy(&mtask->hdr, hdr, sizeof(struct iscsi_hdr));
2905
af973481 2906 iscsi_buf_init_virt(&mtask->headbuf, (char*)&mtask->hdr,
7ba24713 2907 sizeof(struct iscsi_hdr));
7ba24713
AA		 2908
2909 spin_unlock_bh(&session->lock);
2910
2911 if (data_size) {
2912 memcpy(mtask->data, data, data_size);
2913 mtask->data_count = data_size;
2914 } else
2915 mtask->data_count = 0;
2916
2917 mtask->xmstate = XMSTATE_IMM_HDR;
2918
2919 if (mtask->data_count) {
2920 iscsi_buf_init_iov(&mtask->sendbuf, (char*)mtask->data,
2921 mtask->data_count);
2922 }
2923
2924 debug_scsi("mgmtpdu [op 0x%x hdr->itt 0x%x datalen %d]\n",
2925 hdr->opcode, hdr->itt, data_size);
2926
2927 /*
2928 * since send_pdu() could be called at least from two contexts,
2929 * we need to serialize __kfifo_put, so we don't have to take
2930 * additional lock on fast data-path
2931 */
2932 if (hdr->opcode & ISCSI_OP_IMMEDIATE)
2933 __kfifo_put(conn->immqueue, (void*)&mtask, sizeof(void*));
2934 else
2935 __kfifo_put(conn->mgmtqueue, (void*)&mtask, sizeof(void*));
2936
55e3299d 2937 scsi_queue_work(session->host, &conn->xmitwork);
7ba24713
AA		 2938 return 0;
2939}
2940
2941static int
2942iscsi_eh_host_reset(struct scsi_cmnd *sc)
2943{
2944 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2945 struct iscsi_conn *conn = ctask->conn;
2946 struct iscsi_session *session = conn->session;
2947
2948 spin_lock_bh(&session->lock);
2949 if (session->state == ISCSI_STATE_TERMINATE) {
2950 debug_scsi("failing host reset: session terminated "
2951 "[CID %d age %d]", conn->id, session->age);
2952 spin_unlock_bh(&session->lock);
2953 return FAILED;
2954 }
2955 spin_unlock_bh(&session->lock);
2956
2957 debug_scsi("failing connection CID %d due to SCSI host reset "
2958 "[itt 0x%x age %d]", conn->id, ctask->itt,
2959 session->age);
2960 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
2961
2962 return SUCCESS;
2963}
2964
2965static void
2966iscsi_tmabort_timedout(unsigned long data)
2967{
2968 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)data;
2969 struct iscsi_conn *conn = ctask->conn;
2970 struct iscsi_session *session = conn->session;
2971
2972 spin_lock(&session->lock);
2973 if (conn->tmabort_state == TMABORT_INITIAL) {
2974 __kfifo_put(session->mgmtpool.queue,
2975 (void*)&ctask->mtask, sizeof(void*));
2976 conn->tmabort_state = TMABORT_TIMEDOUT;
2977 debug_scsi("tmabort timedout [sc %lx itt 0x%x]\n",
2978 (long)ctask->sc, ctask->itt);
2979 /* unblock eh_abort() */
2980 wake_up(&conn->ehwait);
2981 }
2982 spin_unlock(&session->lock);
2983}
2984
2985static int
2986iscsi_eh_abort(struct scsi_cmnd *sc)
2987{
2988 int rc;
2989 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2990 struct iscsi_conn *conn = ctask->conn;
2991 struct iscsi_session *session = conn->session;
2992
2993 conn->eh_abort_cnt++;
2994 debug_scsi("aborting [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
2995
2996 /*
2997 * two cases for ERL=0 here:
2998 *
2999 * 1) connection-level failure;
3000 * 2) recovery due protocol error;
3001 */
0b950672 3002 mutex_lock(&conn->xmitmutex);
7ba24713
AA		 3003 spin_lock_bh(&session->lock);
3004 if (session->state != ISCSI_STATE_LOGGED_IN) {
3005 if (session->state == ISCSI_STATE_TERMINATE) {
3006 spin_unlock_bh(&session->lock);
0b950672 3007 mutex_unlock(&conn->xmitmutex);
7ba24713
AA		 3008 goto failed;
3009 }
3010 spin_unlock_bh(&session->lock);
3011 } else {
3012 struct iscsi_tm *hdr = &conn->tmhdr;
3013
3014 /*
3015 * Still LOGGED_IN...
3016 */
3017
3018 if (!ctask->sc || sc->SCp.phase != session->age) {
3019 /*
3020 * 1) ctask completed before time out. But session
3021 * is still ok => Happy Retry.
3022 * 2) session was re-open during time out of ctask.
3023 */
3024 spin_unlock_bh(&session->lock);
0b950672 3025 mutex_unlock(&conn->xmitmutex);
7ba24713
AA		 3026 goto success;
3027 }
3028 conn->tmabort_state = TMABORT_INITIAL;
3029 spin_unlock_bh(&session->lock);
3030
3031 /*
3032 * ctask timed out but session is OK
3033 * ERL=0 requires task mgmt abort to be issued on each
3034 * failed command. requests must be serialized.
3035 */
3036 memset(hdr, 0, sizeof(struct iscsi_tm));
3037 hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE;
3038 hdr->flags = ISCSI_TM_FUNC_ABORT_TASK;
3039 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3040 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
3041 hdr->rtt = ctask->hdr.itt;
3042 hdr->refcmdsn = ctask->hdr.cmdsn;
3043
3044 rc = iscsi_conn_send_generic(conn, (struct iscsi_hdr *)hdr,
3045 NULL, 0);
3046 if (rc) {
3047 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3048 debug_scsi("abort sent failure [itt 0x%x]", ctask->itt);
3049 } else {
3050 struct iscsi_r2t_info *r2t;
3051
3052 /*
3053 * TMF abort vs. TMF response race logic
3054 */
3055 spin_lock_bh(&session->lock);
3056 ctask->mtask = (struct iscsi_mgmt_task *)
3057 session->mgmt_cmds[(hdr->itt & ITT_MASK) -
3058 ISCSI_MGMT_ITT_OFFSET];
3059 /*
3060 * have to flush r2tqueue to avoid r2t leaks
3061 */
3062 while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
3063 sizeof(void*))) {
3064 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
3065 sizeof(void*));
3066 }
3067 if (conn->tmabort_state == TMABORT_INITIAL) {
3068 conn->tmfcmd_pdus_cnt++;
3069 conn->tmabort_timer.expires = 3*HZ + jiffies;
3070 conn->tmabort_timer.function =
3071 iscsi_tmabort_timedout;
3072 conn->tmabort_timer.data = (unsigned long)ctask;
3073 add_timer(&conn->tmabort_timer);
3074 debug_scsi("abort sent [itt 0x%x]", ctask->itt);
3075 } else {
3076 if (!ctask->sc ||
3077 conn->tmabort_state == TMABORT_SUCCESS) {
3078 conn->tmabort_state = TMABORT_INITIAL;
3079 spin_unlock_bh(&session->lock);
0b950672 3080 mutex_unlock(&conn->xmitmutex);
7ba24713
AA		 3081 goto success;
3082 }
3083 conn->tmabort_state = TMABORT_INITIAL;
3084 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3085 }
3086 spin_unlock_bh(&session->lock);
3087 }
3088 }
0b950672 3089 mutex_unlock(&conn->xmitmutex);
7ba24713
AA		 3090
3091
3092 /*
3093 * block eh thread until:
3094 *
3095 * 1) abort response;
3096 * 2) abort timeout;
3097 * 3) session re-opened;
3098 * 4) session terminated;
3099 */
3100 for (;;) {
3101 int p_state = session->state;
3102
3103 rc = wait_event_interruptible(conn->ehwait,
3104 (p_state == ISCSI_STATE_LOGGED_IN ?
3105 (session->state == ISCSI_STATE_TERMINATE ||
3106 conn->tmabort_state != TMABORT_INITIAL) :
3107 (session->state == ISCSI_STATE_TERMINATE ||
3108 session->state == ISCSI_STATE_LOGGED_IN)));
3109 if (rc) {
3110 /* shutdown.. */
3111 session->state = ISCSI_STATE_TERMINATE;
3112 goto failed;
3113 }
3114
3115 if (signal_pending(current))
3116 flush_signals(current);
3117
3118 if (session->state == ISCSI_STATE_TERMINATE)
3119 goto failed;
3120
3121 spin_lock_bh(&session->lock);
3122 if (sc->SCp.phase == session->age &&
3123 (conn->tmabort_state == TMABORT_TIMEDOUT ||
3124 conn->tmabort_state == TMABORT_FAILED)) {
3125 conn->tmabort_state = TMABORT_INITIAL;
3126 if (!ctask->sc) {
3127 /*
3128 * ctask completed before tmf abort response or
3129 * time out.
3130 * But session is still ok => Happy Retry.
3131 */
3132 spin_unlock_bh(&session->lock);
3133 break;
3134 }
3135 spin_unlock_bh(&session->lock);
3136 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3137 continue;
3138 }
3139 spin_unlock_bh(&session->lock);
3140 break;
3141 }
3142
3143success:
3144 debug_scsi("abort success [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3145 rc = SUCCESS;
3146 goto exit;
3147
3148failed:
3149 debug_scsi("abort failed [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3150 rc = FAILED;
3151
3152exit:
3153 del_timer_sync(&conn->tmabort_timer);
3154
0b950672 3155 mutex_lock(&conn->xmitmutex);
7ba24713
AA		 3156 if (conn->sock) {
3157 struct sock *sk = conn->sock->sk;
3158
3159 write_lock_bh(&sk->sk_callback_lock);
3160 iscsi_ctask_cleanup(conn, ctask);
3161 write_unlock_bh(&sk->sk_callback_lock);
3162 }
0b950672 3163 mutex_unlock(&conn->xmitmutex);
7ba24713
AA		 3164 return rc;
3165}
3166
3167static int
3168iscsi_r2tpool_alloc(struct iscsi_session *session)
3169{
3170 int i;
3171 int cmd_i;
3172
3173 /*
3174 * initialize per-task: R2T pool and xmit queue
3175 */
3176 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3177 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3178
3179 /*
3180 * pre-allocated x4 as much r2ts to handle race when
3181 * target acks DataOut faster than we data_xmit() queues
3182 * could replenish r2tqueue.
3183 */
3184
3185 /* R2T pool */
3186 if (iscsi_pool_init(&ctask->r2tpool, session->max_r2t * 4,
3187 (void***)&ctask->r2ts, sizeof(struct iscsi_r2t_info))) {
3188 goto r2t_alloc_fail;
3189 }
3190
3191 /* R2T xmit queue */
3192 ctask->r2tqueue = kfifo_alloc(
3193 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
3194 if (ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
3195 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts);
3196 goto r2t_alloc_fail;
3197 }
3198
3199 /*
3200 * number of
3201 * Data-Out PDU's within R2T-sequence can be quite big;
3202 * using mempool
3203 */
3204 ctask->datapool = mempool_create(ISCSI_DTASK_DEFAULT_MAX,
3205 mempool_alloc_slab, mempool_free_slab, taskcache);
3206 if (ctask->datapool == NULL) {
3207 kfifo_free(ctask->r2tqueue);
3208 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts);
3209 goto r2t_alloc_fail;
3210 }
3211 INIT_LIST_HEAD(&ctask->dataqueue);
3212 }
3213
3214 return 0;
3215
3216r2t_alloc_fail:
3217 for (i = 0; i < cmd_i; i++) {
3218 mempool_destroy(session->cmds[i]->datapool);
3219 kfifo_free(session->cmds[i]->r2tqueue);
3220 iscsi_pool_free(&session->cmds[i]->r2tpool,
3221 (void**)session->cmds[i]->r2ts);
3222 }
3223 return -ENOMEM;
3224}
3225
3226static void
3227iscsi_r2tpool_free(struct iscsi_session *session)
3228{
3229 int i;
3230
3231 for (i = 0; i < session->cmds_max; i++) {
3232 mempool_destroy(session->cmds[i]->datapool);
3233 kfifo_free(session->cmds[i]->r2tqueue);
3234 iscsi_pool_free(&session->cmds[i]->r2tpool,
3235 (void**)session->cmds[i]->r2ts);
3236 }
3237}
3238
3239static struct scsi_host_template iscsi_sht = {
3240 .name = "iSCSI Initiator over TCP/IP, v."
3241 ISCSI_VERSION_STR,
9e3961ba
MC		 3242 .queuecommand = iscsi_queuecommand,
3243 .change_queue_depth = iscsi_change_queue_depth,
7ba24713
AA		 3244 .can_queue = ISCSI_XMIT_CMDS_MAX - 1,
3245 .sg_tablesize = ISCSI_SG_TABLESIZE,
9e3961ba
MC		 3246 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
3247 .eh_abort_handler = iscsi_eh_abort,
3248 .eh_host_reset_handler = iscsi_eh_host_reset,
3249 .use_clustering = DISABLE_CLUSTERING,
7ba24713
AA		 3250 .proc_name = "iscsi_tcp",
3251 .this_id = -1,
3252};
3253
7b8631b5
MC		 3254static struct iscsi_transport iscsi_tcp_transport;
3255
3256static struct Scsi_Host *
3257iscsi_session_create(struct scsi_transport_template *scsit,
3258 uint32_t initial_cmdsn)
7ba24713 3259{
7b8631b5 3260 struct Scsi_Host *shost;
7ba24713 3261 struct iscsi_session *session;
7b8631b5 3262 int cmd_i;
7ba24713 3263
7b8631b5
MC		 3264 shost = iscsi_transport_create_session(scsit, &iscsi_tcp_transport);
3265 if (!shost)
3266 return NULL;
7ba24713 3267
7b8631b5
MC		 3268 session = iscsi_hostdata(shost->hostdata);
3269 memset(session, 0, sizeof(struct iscsi_session));
3270 session->host = shost;
7ba24713
AA		 3271 session->state = ISCSI_STATE_LOGGED_IN;
3272 session->mgmtpool_max = ISCSI_MGMT_CMDS_MAX;
3273 session->cmds_max = ISCSI_XMIT_CMDS_MAX;
3274 session->cmdsn = initial_cmdsn;
3275 session->exp_cmdsn = initial_cmdsn + 1;
3276 session->max_cmdsn = initial_cmdsn + 1;
3277 session->max_r2t = 1;
3278
3279 /* initialize SCSI PDU commands pool */
3280 if (iscsi_pool_init(&session->cmdpool, session->cmds_max,
3281 (void***)&session->cmds, sizeof(struct iscsi_cmd_task)))
3282 goto cmdpool_alloc_fail;
3283
3284 /* pre-format cmds pool with ITT */
3285 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++)
3286 session->cmds[cmd_i]->itt = cmd_i;
3287
3288 spin_lock_init(&session->lock);
3289 INIT_LIST_HEAD(&session->connections);
3290
3291 /* initialize immediate command pool */
3292 if (iscsi_pool_init(&session->mgmtpool, session->mgmtpool_max,
3293 (void***)&session->mgmt_cmds, sizeof(struct iscsi_mgmt_task)))
3294 goto mgmtpool_alloc_fail;
3295
3296
3297 /* pre-format immediate cmds pool with ITT */
3298 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
3299 session->mgmt_cmds[cmd_i]->itt = ISCSI_MGMT_ITT_OFFSET + cmd_i;
3300 session->mgmt_cmds[cmd_i]->data = kmalloc(
3301 DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH, GFP_KERNEL);
3302 if (!session->mgmt_cmds[cmd_i]->data) {
3303 int j;
3304
3305 for (j = 0; j < cmd_i; j++)
3306 kfree(session->mgmt_cmds[j]->data);
3307 goto immdata_alloc_fail;
3308 }
3309 }
3310
3311 if (iscsi_r2tpool_alloc(session))
3312 goto r2tpool_alloc_fail;
3313
7b8631b5 3314 return shost;
7ba24713
AA		 3315
3316r2tpool_alloc_fail:
3317 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3318 kfree(session->mgmt_cmds[cmd_i]->data);
3319 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3320immdata_alloc_fail:
3321mgmtpool_alloc_fail:
3322 iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
3323cmdpool_alloc_fail:
7b8631b5 3324 return NULL;
7ba24713
AA		 3325}
3326
3327static void
7b8631b5 3328iscsi_session_destroy(struct Scsi_Host *shost)
7ba24713 3329{
7b8631b5 3330 struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
7ba24713
AA		 3331 int cmd_i;
3332 struct iscsi_data_task *dtask, *n;
7ba24713
AA		 3333
3334 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3335 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3336 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
3337 list_del(&dtask->item);
3338 mempool_free(dtask, ctask->datapool);
3339 }
3340 }
3341
3342 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3343 kfree(session->mgmt_cmds[cmd_i]->data);
3344
3345 iscsi_r2tpool_free(session);
3346 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3347 iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
7b8631b5
MC		 3348
3349 iscsi_transport_destroy_session(shost);
7ba24713
AA		 3350}
3351
3352static int
3353iscsi_conn_set_param(iscsi_connh_t connh, enum iscsi_param param,
3354 uint32_t value)
3355{
3356 struct iscsi_conn *conn = iscsi_ptr(connh);
3357 struct iscsi_session *session = conn->session;
3358
3359 spin_lock_bh(&session->lock);
3360 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
3361 conn->stop_stage != STOP_CONN_RECOVER) {
3362 printk(KERN_ERR "iscsi_tcp: can not change parameter [%d]\n",
3363 param);
3364 spin_unlock_bh(&session->lock);
3365 return 0;
3366 }
3367 spin_unlock_bh(&session->lock);
3368
3369 switch(param) {
3370 case ISCSI_PARAM_MAX_RECV_DLENGTH: {
3371 char *saveptr = conn->data;
b53cb2a4 3372 gfp_t flags = GFP_KERNEL;
7ba24713
AA		 3373
3374 if (conn->data_size >= value) {
3375 conn->max_recv_dlength = value;
3376 break;
3377 }
3378
3379 spin_lock_bh(&session->lock);
3380 if (conn->stop_stage == STOP_CONN_RECOVER)
3381 flags = GFP_ATOMIC;
3382 spin_unlock_bh(&session->lock);
3383
3384 if (value <= PAGE_SIZE)
3385 conn->data = kmalloc(value, flags);
3386 else
3387 conn->data = (void*)__get_free_pages(flags,
3388 get_order(value));
3389 if (conn->data == NULL) {
3390 conn->data = saveptr;
3391 return -ENOMEM;
3392 }
3393 if (conn->data_size <= PAGE_SIZE)
3394 kfree(saveptr);
3395 else
3396 free_pages((unsigned long)saveptr,
3397 get_order(conn->data_size));
3398 conn->max_recv_dlength = value;
3399 conn->data_size = value;
3400 }
3401 break;
3402 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
3403 conn->max_xmit_dlength = value;
3404 break;
3405 case ISCSI_PARAM_HDRDGST_EN:
3406 conn->hdrdgst_en = value;
3407 conn->hdr_size = sizeof(struct iscsi_hdr);
3408 if (conn->hdrdgst_en) {
3409 conn->hdr_size += sizeof(__u32);
3410 if (!conn->tx_tfm)
3411 conn->tx_tfm = crypto_alloc_tfm("crc32c", 0);
3412 if (!conn->tx_tfm)
3413 return -ENOMEM;
3414 if (!conn->rx_tfm)
3415 conn->rx_tfm = crypto_alloc_tfm("crc32c", 0);
3416 if (!conn->rx_tfm) {
3417 crypto_free_tfm(conn->tx_tfm);
3418 return -ENOMEM;
3419 }
3420 } else {
3421 if (conn->tx_tfm)
3422 crypto_free_tfm(conn->tx_tfm);
3423 if (conn->rx_tfm)
3424 crypto_free_tfm(conn->rx_tfm);
3425 }
3426 break;
3427 case ISCSI_PARAM_DATADGST_EN:
3428 conn->datadgst_en = value;
3429 if (conn->datadgst_en) {
3430 if (!conn->data_tx_tfm)
3431 conn->data_tx_tfm =
3432 crypto_alloc_tfm("crc32c", 0);
3433 if (!conn->data_tx_tfm)
3434 return -ENOMEM;
3435 if (!conn->data_rx_tfm)
3436 conn->data_rx_tfm =
3437 crypto_alloc_tfm("crc32c", 0);
3438 if (!conn->data_rx_tfm) {
3439 crypto_free_tfm(conn->data_tx_tfm);
3440 return -ENOMEM;
3441 }
3442 } else {
3443 if (conn->data_tx_tfm)
3444 crypto_free_tfm(conn->data_tx_tfm);
3445 if (conn->data_rx_tfm)
3446 crypto_free_tfm(conn->data_rx_tfm);
3447 }
56851698
FT		 3448 conn->sendpage = conn->datadgst_en ?
3449 sock_no_sendpage : conn->sock->ops->sendpage;
7ba24713
AA		 3450 break;
3451 case ISCSI_PARAM_INITIAL_R2T_EN:
3452 session->initial_r2t_en = value;
3453 break;
3454 case ISCSI_PARAM_MAX_R2T:
3455 if (session->max_r2t == roundup_pow_of_two(value))
3456 break;
3457 iscsi_r2tpool_free(session);
3458 session->max_r2t = value;
3459 if (session->max_r2t & (session->max_r2t - 1))
3460 session->max_r2t = roundup_pow_of_two(session->max_r2t);
3461 if (iscsi_r2tpool_alloc(session))
3462 return -ENOMEM;
3463 break;
3464 case ISCSI_PARAM_IMM_DATA_EN:
3465 session->imm_data_en = value;
3466 break;
3467 case ISCSI_PARAM_FIRST_BURST:
3468 session->first_burst = value;
3469 break;
3470 case ISCSI_PARAM_MAX_BURST:
3471 session->max_burst = value;
3472 break;
3473 case ISCSI_PARAM_PDU_INORDER_EN:
3474 session->pdu_inorder_en = value;
3475 break;
3476 case ISCSI_PARAM_DATASEQ_INORDER_EN:
3477 session->dataseq_inorder_en = value;
3478 break;
3479 case ISCSI_PARAM_ERL:
3480 session->erl = value;
3481 break;
3482 case ISCSI_PARAM_IFMARKER_EN:
3483 BUG_ON(value);
3484 session->ifmarker_en = value;
3485 break;
3486 case ISCSI_PARAM_OFMARKER_EN:
3487 BUG_ON(value);
3488 session->ofmarker_en = value;
3489 break;
3490 default:
3491 break;
3492 }
3493
3494 return 0;
3495}
3496
3497static int
7b8631b5
MC		 3498iscsi_session_get_param(struct Scsi_Host *shost,
3499 enum iscsi_param param, uint32_t *value)
7ba24713 3500{
7b8631b5 3501 struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
7ba24713
AA		 3502
3503 switch(param) {
7ba24713
AA		 3504 case ISCSI_PARAM_INITIAL_R2T_EN:
3505 *value = session->initial_r2t_en;
3506 break;
3507 case ISCSI_PARAM_MAX_R2T:
3508 *value = session->max_r2t;
3509 break;
3510 case ISCSI_PARAM_IMM_DATA_EN:
3511 *value = session->imm_data_en;
3512 break;
3513 case ISCSI_PARAM_FIRST_BURST:
3514 *value = session->first_burst;
3515 break;
3516 case ISCSI_PARAM_MAX_BURST:
3517 *value = session->max_burst;
3518 break;
3519 case ISCSI_PARAM_PDU_INORDER_EN:
3520 *value = session->pdu_inorder_en;
3521 break;
3522 case ISCSI_PARAM_DATASEQ_INORDER_EN:
3523 *value = session->dataseq_inorder_en;
3524 break;
3525 case ISCSI_PARAM_ERL:
3526 *value = session->erl;
3527 break;
3528 case ISCSI_PARAM_IFMARKER_EN:
3529 *value = session->ifmarker_en;
3530 break;
3531 case ISCSI_PARAM_OFMARKER_EN:
3532 *value = session->ofmarker_en;
3533 break;
3534 default:
3535 return ISCSI_ERR_PARAM_NOT_FOUND;
3536 }
3537
3538 return 0;
3539}
3540
7b8631b5
MC		 3541static int
3542iscsi_conn_get_param(void *data, enum iscsi_param param, uint32_t *value)
3543{
3544 struct iscsi_conn *conn = data;
3545
3546 switch(param) {
3547 case ISCSI_PARAM_MAX_RECV_DLENGTH:
3548 *value = conn->max_recv_dlength;
3549 break;
3550 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
3551 *value = conn->max_xmit_dlength;
3552 break;
3553 case ISCSI_PARAM_HDRDGST_EN:
3554 *value = conn->hdrdgst_en;
3555 break;
3556 case ISCSI_PARAM_DATADGST_EN:
3557 *value = conn->datadgst_en;
3558 break;
3559 default:
3560 return ISCSI_ERR_PARAM_NOT_FOUND;
3561 }
3562
3563 return 0;
3564}
3565
7ba24713
AA		 3566static void
3567iscsi_conn_get_stats(iscsi_connh_t connh, struct iscsi_stats *stats)
3568{
3569 struct iscsi_conn *conn = iscsi_ptr(connh);
3570
3571 stats->txdata_octets = conn->txdata_octets;
3572 stats->rxdata_octets = conn->rxdata_octets;
3573 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
3574 stats->dataout_pdus = conn->dataout_pdus_cnt;
3575 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
3576 stats->datain_pdus = conn->datain_pdus_cnt;
3577 stats->r2t_pdus = conn->r2t_pdus_cnt;
3578 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
3579 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
3580 stats->custom_length = 3;
3581 strcpy(stats->custom[0].desc, "tx_sendpage_failures");
3582 stats->custom[0].value = conn->sendpage_failures_cnt;
3583 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
3584 stats->custom[1].value = conn->discontiguous_hdr_cnt;
3585 strcpy(stats->custom[2].desc, "eh_abort_cnt");
3586 stats->custom[2].value = conn->eh_abort_cnt;
3587}
3588
3589static int
3590iscsi_conn_send_pdu(iscsi_connh_t connh, struct iscsi_hdr *hdr, char *data,
3591 uint32_t data_size)
3592{
3593 struct iscsi_conn *conn = iscsi_ptr(connh);
3594 int rc;
3595
0b950672 3596 mutex_lock(&conn->xmitmutex);
7ba24713 3597 rc = iscsi_conn_send_generic(conn, hdr, data, data_size);
0b950672 3598 mutex_unlock(&conn->xmitmutex);
7ba24713
AA		 3599
3600 return rc;
3601}
3602
3603static struct iscsi_transport iscsi_tcp_transport = {
3604 .owner = THIS_MODULE,
3605 .name = "tcp",
3606 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
3607 | CAP_DATADGST,
3608 .host_template = &iscsi_sht,
3609 .hostdata_size = sizeof(struct iscsi_session),
7b8631b5 3610 .conndata_size = sizeof(struct iscsi_conn),
7ba24713
AA		 3611 .max_conn = 1,
3612 .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN,
3613 .create_session = iscsi_session_create,
3614 .destroy_session = iscsi_session_destroy,
3615 .create_conn = iscsi_conn_create,
3616 .bind_conn = iscsi_conn_bind,
3617 .destroy_conn = iscsi_conn_destroy,
3618 .set_param = iscsi_conn_set_param,
7b8631b5
MC		 3619 .get_conn_param = iscsi_conn_get_param,
3620 .get_session_param = iscsi_session_get_param,
7ba24713
AA		 3621 .start_conn = iscsi_conn_start,
3622 .stop_conn = iscsi_conn_stop,
3623 .send_pdu = iscsi_conn_send_pdu,
3624 .get_stats = iscsi_conn_get_stats,
3625};
3626
3627static int __init
3628iscsi_tcp_init(void)
3629{
7ba24713
AA		 3630 if (iscsi_max_lun < 1) {
3631 printk(KERN_ERR "Invalid max_lun value of %u\n", iscsi_max_lun);
3632 return -EINVAL;
3633 }
3634 iscsi_tcp_transport.max_lun = iscsi_max_lun;
3635
3636 taskcache = kmem_cache_create("iscsi_taskcache",
3637 sizeof(struct iscsi_data_task), 0,
3638 SLAB_HWCACHE_ALIGN | SLAB_NO_REAP, NULL, NULL);
3639 if (!taskcache)
3640 return -ENOMEM;
3641
7b8631b5 3642 if (!iscsi_register_transport(&iscsi_tcp_transport))
7ba24713
AA		 3643 kmem_cache_destroy(taskcache);
3644
7b8631b5 3645 return 0;
7ba24713
AA		 3646}
3647
3648static void __exit
3649iscsi_tcp_exit(void)
3650{
3651 iscsi_unregister_transport(&iscsi_tcp_transport);
3652 kmem_cache_destroy(taskcache);
3653}
3654
3655module_init(iscsi_tcp_init);
3656module_exit(iscsi_tcp_exit);
Linux kernel - Deliverables
This page took 0.206073 seconds and 5 git commands to generate.