projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[SCSI] iscsi: only preallocate login buffer
[deliverable/linux.git] / drivers / scsi / iscsi_tcp.c
CommitLineData
7ba24713
AA		 1/*
2 * iSCSI Initiator over TCP/IP Data-Path
3 *
4 * Copyright (C) 2004 Dmitry Yusupov
5 * Copyright (C) 2004 Alex Aizman
5bb0b55a
MC		 6 * Copyright (C) 2005 - 2006 Mike Christie
7 * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
7ba24713
AA		 8 * maintained by open-iscsi@googlegroups.com
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published
12 * by the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * See the file COPYING included with this distribution for more details.
21 *
22 * Credits:
23 * Christoph Hellwig
24 * FUJITA Tomonori
25 * Arne Redlich
26 * Zhenyu Wang
27 */
28
29#include <linux/types.h>
30#include <linux/list.h>
31#include <linux/inet.h>
32#include <linux/blkdev.h>
33#include <linux/crypto.h>
34#include <linux/delay.h>
35#include <linux/kfifo.h>
36#include <linux/scatterlist.h>
0b950672 37#include <linux/mutex.h>
7ba24713
AA		 38#include <net/tcp.h>
39#include <scsi/scsi_cmnd.h>
7ba24713
AA		 40#include <scsi/scsi_host.h>
41#include <scsi/scsi.h>
42#include <scsi/scsi_transport_iscsi.h>
43
44#include "iscsi_tcp.h"
45
46MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
47 "Alex Aizman <itn780@yahoo.com>");
48MODULE_DESCRIPTION("iSCSI/TCP data-path");
49MODULE_LICENSE("GPL");
4d841d6b 50MODULE_VERSION("0:4.445");
7ba24713 51/* #define DEBUG_TCP */
7ba24713
AA		 52#define DEBUG_ASSERT
53
54#ifdef DEBUG_TCP
5bb0b55a 55#define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
7ba24713
AA		 56#else
57#define debug_tcp(fmt...)
58#endif
59
7ba24713
AA		 60#ifndef DEBUG_ASSERT
61#ifdef BUG_ON
62#undef BUG_ON
63#endif
64#define BUG_ON(expr)
65#endif
66
7ba24713
AA		 67static unsigned int iscsi_max_lun = 512;
68module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
69
70/* global data */
71static kmem_cache_t *taskcache;
72
7ba24713
AA		 73static inline void
74iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
75{
7cae5159
MC		 76 ibuf->sg.page = virt_to_page(vbuf);
77 ibuf->sg.offset = offset_in_page(vbuf);
7ba24713
AA		 78 ibuf->sg.length = size;
79 ibuf->sent = 0;
7cae5159 80 ibuf->use_sendmsg = 1;
7ba24713
AA		 81}
82
83static inline void
84iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
85{
7cae5159
MC		 86 ibuf->sg.page = sg->page;
87 ibuf->sg.offset = sg->offset;
88 ibuf->sg.length = sg->length;
7ba24713
AA		 89 /*
90 * Fastpath: sg element fits into single page
91 */
a1e80c20 92 if (sg->length + sg->offset <= PAGE_SIZE && !PageSlab(sg->page))
7cae5159
MC		 93 ibuf->use_sendmsg = 0;
94 else
95 ibuf->use_sendmsg = 1;
7ba24713
AA		 96 ibuf->sent = 0;
97}
98
99static inline int
100iscsi_buf_left(struct iscsi_buf *ibuf)
101{
102 int rc;
103
104 rc = ibuf->sg.length - ibuf->sent;
105 BUG_ON(rc < 0);
106 return rc;
107}
108
109static inline void
af973481
MC		 110iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
111 u8* crc)
7ba24713 112{
5bb0b55a 113 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
7ba24713 114
5bb0b55a
MC		 115 crypto_digest_digest(tcp_conn->tx_tfm, &buf->sg, 1, crc);
116 buf->sg.length += sizeof(uint32_t);
7ba24713
AA		 117}
118
119static inline int
5bb0b55a 120iscsi_hdr_extract(struct iscsi_tcp_conn *tcp_conn)
7ba24713 121{
5bb0b55a 122 struct sk_buff *skb = tcp_conn->in.skb;
7ba24713 123
5bb0b55a 124 tcp_conn->in.zero_copy_hdr = 0;
7ba24713 125
5bb0b55a
MC		 126 if (tcp_conn->in.copy >= tcp_conn->hdr_size &&
127 tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
7ba24713
AA		 128 /*
129 * Zero-copy PDU Header: using connection context
130 * to store header pointer.
131 */
132 if (skb_shinfo(skb)->frag_list == NULL &&
5bb0b55a
MC		 133 !skb_shinfo(skb)->nr_frags) {
134 tcp_conn->in.hdr = (struct iscsi_hdr *)
135 ((char*)skb->data + tcp_conn->in.offset);
136 tcp_conn->in.zero_copy_hdr = 1;
137 } else {
7ba24713
AA		 138 /* ignoring return code since we checked
139 * in.copy before */
5bb0b55a
MC		 140 skb_copy_bits(skb, tcp_conn->in.offset,
141 &tcp_conn->hdr, tcp_conn->hdr_size);
142 tcp_conn->in.hdr = &tcp_conn->hdr;
7ba24713 143 }
5bb0b55a
MC		 144 tcp_conn->in.offset += tcp_conn->hdr_size;
145 tcp_conn->in.copy -= tcp_conn->hdr_size;
7ba24713
AA		 146 } else {
147 int hdr_remains;
148 int copylen;
149
150 /*
151 * PDU header scattered across SKB's,
152 * copying it... This'll happen quite rarely.
153 */
154
5bb0b55a
MC		 155 if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER)
156 tcp_conn->in.hdr_offset = 0;
7ba24713 157
5bb0b55a 158 hdr_remains = tcp_conn->hdr_size - tcp_conn->in.hdr_offset;
7ba24713
AA		 159 BUG_ON(hdr_remains <= 0);
160
5bb0b55a
MC		 161 copylen = min(tcp_conn->in.copy, hdr_remains);
162 skb_copy_bits(skb, tcp_conn->in.offset,
163 (char*)&tcp_conn->hdr + tcp_conn->in.hdr_offset,
164 copylen);
7ba24713
AA		 165
166 debug_tcp("PDU gather offset %d bytes %d in.offset %d "
5bb0b55a
MC		 167 "in.copy %d\n", tcp_conn->in.hdr_offset, copylen,
168 tcp_conn->in.offset, tcp_conn->in.copy);
7ba24713 169
5bb0b55a
MC		 170 tcp_conn->in.offset += copylen;
171 tcp_conn->in.copy -= copylen;
7ba24713 172 if (copylen < hdr_remains) {
5bb0b55a
MC		 173 tcp_conn->in_progress = IN_PROGRESS_HEADER_GATHER;
174 tcp_conn->in.hdr_offset += copylen;
7ba24713
AA		 175 return -EAGAIN;
176 }
5bb0b55a
MC		 177 tcp_conn->in.hdr = &tcp_conn->hdr;
178 tcp_conn->discontiguous_hdr_cnt++;
179 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
7ba24713
AA		 180 }
181
182 return 0;
183}
184
30a6c652
MC		 185/*
186 * must be called with session lock
187 */
188static void
189__iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
7ba24713 190{
5bb0b55a 191 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
30a6c652 192 struct scsi_cmnd *sc;
7ba24713 193
30a6c652
MC		 194 sc = ctask->sc;
195 if (unlikely(!sc))
7ba24713 196 return;
30a6c652 197
7ba24713
AA		 198 if (sc->sc_data_direction == DMA_TO_DEVICE) {
199 struct iscsi_data_task *dtask, *n;
5bb0b55a 200
7ba24713 201 /* WRITE: cleanup Data-Out's if any */
5bb0b55a
MC		 202 list_for_each_entry_safe(dtask, n, &tcp_ctask->dataqueue,
203 item) {
7ba24713 204 list_del(&dtask->item);
5bb0b55a 205 mempool_free(dtask, tcp_ctask->datapool);
7ba24713 206 }
7ba24713 207 }
5bb0b55a
MC		 208 tcp_ctask->xmstate = XMSTATE_IDLE;
209 tcp_ctask->r2t = NULL;
7ba24713
AA		 210}
211
212/**
213 * iscsi_data_rsp - SCSI Data-In Response processing
214 * @conn: iscsi connection
215 * @ctask: scsi command task
216 **/
217static int
218iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
219{
220 int rc;
5bb0b55a
MC		 221 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
222 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
223 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
7ba24713
AA		 224 struct iscsi_session *session = conn->session;
225 int datasn = be32_to_cpu(rhdr->datasn);
226
227 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
228 if (rc)
229 return rc;
230 /*
231 * setup Data-In byte counter (gets decremented..)
232 */
5bb0b55a 233 ctask->data_count = tcp_conn->in.datalen;
7ba24713 234
5bb0b55a 235 if (tcp_conn->in.datalen == 0)
7ba24713
AA		 236 return 0;
237
238 if (ctask->datasn != datasn)
239 return ISCSI_ERR_DATASN;
240
241 ctask->datasn++;
242
5bb0b55a
MC		 243 tcp_ctask->data_offset = be32_to_cpu(rhdr->offset);
244 if (tcp_ctask->data_offset + tcp_conn->in.datalen > ctask->total_length)
7ba24713
AA		 245 return ISCSI_ERR_DATA_OFFSET;
246
247 if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
248 struct scsi_cmnd *sc = ctask->sc;
249
250 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
bf310b8f 251 if (rhdr->flags & ISCSI_FLAG_DATA_UNDERFLOW) {
7ba24713
AA		 252 int res_count = be32_to_cpu(rhdr->residual_count);
253
254 if (res_count > 0 &&
255 res_count <= sc->request_bufflen) {
256 sc->resid = res_count;
257 sc->result = (DID_OK << 16) | rhdr->cmd_status;
258 } else
259 sc->result = (DID_BAD_TARGET << 16) |
260 rhdr->cmd_status;
bf310b8f 261 } else if (rhdr->flags & ISCSI_FLAG_DATA_OVERFLOW) {
7ba24713
AA		 262 sc->resid = be32_to_cpu(rhdr->residual_count);
263 sc->result = (DID_OK << 16) | rhdr->cmd_status;
264 } else
265 sc->result = (DID_OK << 16) | rhdr->cmd_status;
266 }
267
268 conn->datain_pdus_cnt++;
269 return 0;
270}
271
272/**
273 * iscsi_solicit_data_init - initialize first Data-Out
274 * @conn: iscsi connection
275 * @ctask: scsi command task
276 * @r2t: R2T info
277 *
278 * Notes:
279 * Initialize first Data-Out within this R2T sequence and finds
280 * proper data_offset within this SCSI command.
281 *
282 * This function is called with connection lock taken.
283 **/
284static void
285iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
286 struct iscsi_r2t_info *r2t)
287{
288 struct iscsi_data *hdr;
289 struct iscsi_data_task *dtask;
290 struct scsi_cmnd *sc = ctask->sc;
5bb0b55a 291 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
7ba24713 292
5bb0b55a 293 dtask = mempool_alloc(tcp_ctask->datapool, GFP_ATOMIC);
7ba24713 294 BUG_ON(!dtask);
30a6c652 295 INIT_LIST_HEAD(&dtask->item);
7ba24713
AA		 296 hdr = &dtask->hdr;
297 memset(hdr, 0, sizeof(struct iscsi_data));
298 hdr->ttt = r2t->ttt;
299 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
300 r2t->solicit_datasn++;
301 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
5bb0b55a
MC		 302 memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
303 hdr->itt = ctask->hdr->itt;
7ba24713
AA		 304 hdr->exp_statsn = r2t->exp_statsn;
305 hdr->offset = cpu_to_be32(r2t->data_offset);
306 if (r2t->data_length > conn->max_xmit_dlength) {
307 hton24(hdr->dlength, conn->max_xmit_dlength);
308 r2t->data_count = conn->max_xmit_dlength;
309 hdr->flags = 0;
310 } else {
311 hton24(hdr->dlength, r2t->data_length);
312 r2t->data_count = r2t->data_length;
313 hdr->flags = ISCSI_FLAG_CMD_FINAL;
314 }
315 conn->dataout_pdus_cnt++;
316
317 r2t->sent = 0;
318
6e458cc9 319 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
af973481 320 sizeof(struct iscsi_hdr));
7ba24713
AA		 321
322 r2t->dtask = dtask;
323
324 if (sc->use_sg) {
325 int i, sg_count = 0;
326 struct scatterlist *sg = sc->request_buffer;
327
328 r2t->sg = NULL;
329 for (i = 0; i < sc->use_sg; i++, sg += 1) {
330 /* FIXME: prefetch ? */
331 if (sg_count + sg->length > r2t->data_offset) {
332 int page_offset;
333
334 /* sg page found! */
335
336 /* offset within this page */
337 page_offset = r2t->data_offset - sg_count;
338
339 /* fill in this buffer */
340 iscsi_buf_init_sg(&r2t->sendbuf, sg);
341 r2t->sendbuf.sg.offset += page_offset;
342 r2t->sendbuf.sg.length -= page_offset;
343
344 /* xmit logic will continue with next one */
345 r2t->sg = sg + 1;
346 break;
347 }
348 sg_count += sg->length;
349 }
350 BUG_ON(r2t->sg == NULL);
351 } else
5bb0b55a 352 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
7ba24713
AA		 353 (char*)sc->request_buffer + r2t->data_offset,
354 r2t->data_count);
355
5bb0b55a 356 list_add(&dtask->item, &tcp_ctask->dataqueue);
7ba24713
AA		 357}
358
359/**
360 * iscsi_r2t_rsp - iSCSI R2T Response processing
361 * @conn: iscsi connection
362 * @ctask: scsi command task
363 **/
364static int
365iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
366{
367 struct iscsi_r2t_info *r2t;
368 struct iscsi_session *session = conn->session;
5bb0b55a
MC		 369 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
370 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
371 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
7ba24713
AA		 372 int r2tsn = be32_to_cpu(rhdr->r2tsn);
373 int rc;
374
5bb0b55a 375 if (tcp_conn->in.datalen)
7ba24713
AA		 376 return ISCSI_ERR_DATALEN;
377
5bb0b55a 378 if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn)
7ba24713
AA		 379 return ISCSI_ERR_R2TSN;
380
381 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
382 if (rc)
383 return rc;
384
385 /* FIXME: use R2TSN to detect missing R2T */
386
387 /* fill-in new R2T associated with the task */
388 spin_lock(&session->lock);
389 if (!ctask->sc || ctask->mtask ||
390 session->state != ISCSI_STATE_LOGGED_IN) {
391 printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
392 "recovery...\n", ctask->itt);
393 spin_unlock(&session->lock);
394 return 0;
395 }
5bb0b55a 396 rc = __kfifo_get(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
7ba24713
AA		 397 BUG_ON(!rc);
398
399 r2t->exp_statsn = rhdr->statsn;
400 r2t->data_length = be32_to_cpu(rhdr->data_length);
401 if (r2t->data_length == 0 ||
402 r2t->data_length > session->max_burst) {
403 spin_unlock(&session->lock);
404 return ISCSI_ERR_DATALEN;
405 }
406
407 r2t->data_offset = be32_to_cpu(rhdr->data_offset);
408 if (r2t->data_offset + r2t->data_length > ctask->total_length) {
409 spin_unlock(&session->lock);
410 return ISCSI_ERR_DATALEN;
411 }
412
413 r2t->ttt = rhdr->ttt; /* no flip */
414 r2t->solicit_datasn = 0;
415
416 iscsi_solicit_data_init(conn, ctask, r2t);
417
5bb0b55a
MC		 418 tcp_ctask->exp_r2tsn = r2tsn + 1;
419 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
420 __kfifo_put(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*));
421 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
7ba24713 422
55e3299d 423 scsi_queue_work(session->host, &conn->xmitwork);
7ba24713
AA		 424 conn->r2t_pdus_cnt++;
425 spin_unlock(&session->lock);
426
427 return 0;
428}
429
430static int
5bb0b55a 431iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
7ba24713 432{
5bb0b55a 433 int rc = 0, opcode, ahslen;
7ba24713 434 struct iscsi_hdr *hdr;
7ba24713 435 struct iscsi_session *session = conn->session;
5bb0b55a
MC		 436 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
437 uint32_t cdgst, rdgst = 0, itt;
7ba24713 438
5bb0b55a 439 hdr = tcp_conn->in.hdr;
7ba24713
AA		 440
441 /* verify PDU length */
5bb0b55a
MC		 442 tcp_conn->in.datalen = ntoh24(hdr->dlength);
443 if (tcp_conn->in.datalen > conn->max_recv_dlength) {
7ba24713 444 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
5bb0b55a 445 tcp_conn->in.datalen, conn->max_recv_dlength);
7ba24713
AA		 446 return ISCSI_ERR_DATALEN;
447 }
5bb0b55a 448 tcp_conn->data_copied = 0;
7ba24713
AA		 449
450 /* read AHS */
5bb0b55a
MC		 451 ahslen = hdr->hlength << 2;
452 tcp_conn->in.offset += ahslen;
453 tcp_conn->in.copy -= ahslen;
454 if (tcp_conn->in.copy < 0) {
7ba24713 455 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
5bb0b55a 456 "%d bytes\n", ahslen);
7ba24713
AA		 457 return ISCSI_ERR_AHSLEN;
458 }
459
460 /* calculate read padding */
5bb0b55a
MC		 461 tcp_conn->in.padding = tcp_conn->in.datalen & (ISCSI_PAD_LEN-1);
462 if (tcp_conn->in.padding) {
463 tcp_conn->in.padding = ISCSI_PAD_LEN - tcp_conn->in.padding;
464 debug_scsi("read padding %d bytes\n", tcp_conn->in.padding);
7ba24713
AA		 465 }
466
467 if (conn->hdrdgst_en) {
468 struct scatterlist sg;
469
470 sg_init_one(&sg, (u8 *)hdr,
5bb0b55a
MC		 471 sizeof(struct iscsi_hdr) + ahslen);
472 crypto_digest_digest(tcp_conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
7ba24713 473 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
5bb0b55a 474 ahslen);
8a47cd34 475 if (cdgst != rdgst) {
5bb0b55a
MC		 476 printk(KERN_ERR "iscsi_tcp: hdrdgst error "
477 "recv 0x%x calc 0x%x\n", rdgst, cdgst);
8a47cd34
MC		 478 return ISCSI_ERR_HDR_DGST;
479 }
7ba24713
AA		 480 }
481
5bb0b55a 482 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
7ba24713 483 /* verify itt (itt encoding: age+cid+itt) */
5bb0b55a
MC		 484 rc = iscsi_verify_itt(conn, hdr, &itt);
485 if (rc == ISCSI_ERR_NO_SCSI_CMD) {
486 tcp_conn->in.datalen = 0; /* force drop */
487 return 0;
488 } else if (rc)
489 return rc;
7ba24713
AA		 490
491 debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
5bb0b55a
MC		 492 opcode, tcp_conn->in.offset, tcp_conn->in.copy,
493 ahslen, tcp_conn->in.datalen);
7ba24713 494
5bb0b55a
MC		 495 switch(opcode) {
496 case ISCSI_OP_SCSI_DATA_IN:
497 tcp_conn->in.ctask = session->cmds[itt];
498 rc = iscsi_data_rsp(conn, tcp_conn->in.ctask);
499 /* fall through */
500 case ISCSI_OP_SCSI_CMD_RSP:
501 tcp_conn->in.ctask = session->cmds[itt];
502 if (tcp_conn->in.datalen)
503 goto copy_hdr;
7ba24713 504
5bb0b55a
MC		 505 spin_lock(&session->lock);
506 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask);
507 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
508 spin_unlock(&session->lock);
509 break;
510 case ISCSI_OP_R2T:
511 tcp_conn->in.ctask = session->cmds[itt];
512 if (ahslen)
513 rc = ISCSI_ERR_AHSLEN;
514 else if (tcp_conn->in.ctask->sc->sc_data_direction ==
515 DMA_TO_DEVICE)
516 rc = iscsi_r2t_rsp(conn, tcp_conn->in.ctask);
517 else
518 rc = ISCSI_ERR_PROTO;
519 break;
520 case ISCSI_OP_LOGIN_RSP:
521 case ISCSI_OP_TEXT_RSP:
522 case ISCSI_OP_LOGOUT_RSP:
523 case ISCSI_OP_NOOP_IN:
524 case ISCSI_OP_REJECT:
525 case ISCSI_OP_ASYNC_EVENT:
526 if (tcp_conn->in.datalen)
527 goto copy_hdr;
528 /* fall through */
529 case ISCSI_OP_SCSI_TMFUNC_RSP:
530 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
531 break;
532 default:
533 rc = ISCSI_ERR_BAD_OPCODE;
534 break;
535 }
7ba24713
AA		 536
537 return rc;
5bb0b55a
MC		 538
539copy_hdr:
540 /*
541 * if we did zero copy for the header but we will need multiple
542 * skbs to complete the command then we have to copy the header
543 * for later use
544 */
545 if (tcp_conn->in.zero_copy_hdr && tcp_conn->in.copy <
546 (tcp_conn->in.datalen + tcp_conn->in.padding +
547 (conn->datadgst_en ? 4 : 0))) {
548 debug_tcp("Copying header for later use. in.copy %d in.datalen"
549 " %d\n", tcp_conn->in.copy, tcp_conn->in.datalen);
550 memcpy(&tcp_conn->hdr, tcp_conn->in.hdr,
551 sizeof(struct iscsi_hdr));
552 tcp_conn->in.hdr = &tcp_conn->hdr;
553 tcp_conn->in.zero_copy_hdr = 0;
554 }
555 return 0;
7ba24713
AA		 556}
557
558/**
559 * iscsi_ctask_copy - copy skb bits to the destanation cmd task
5bb0b55a 560 * @conn: iscsi tcp connection
7ba24713
AA		 561 * @ctask: scsi command task
562 * @buf: buffer to copy to
563 * @buf_size: size of buffer
564 * @offset: offset within the buffer
565 *
566 * Notes:
567 * The function calls skb_copy_bits() and updates per-connection and
568 * per-cmd byte counters.
569 *
570 * Read counters (in bytes):
571 *
572 * conn->in.offset offset within in progress SKB
573 * conn->in.copy left to copy from in progress SKB
574 * including padding
575 * conn->in.copied copied already from in progress SKB
576 * conn->data_copied copied already from in progress buffer
577 * ctask->sent total bytes sent up to the MidLayer
578 * ctask->data_count left to copy from in progress Data-In
579 * buf_left left to copy from in progress buffer
580 **/
581static inline int
5bb0b55a 582iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask,
7ba24713
AA		 583 void *buf, int buf_size, int offset)
584{
5bb0b55a
MC		 585 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
586 int buf_left = buf_size - (tcp_conn->data_copied + offset);
587 int size = min(tcp_conn->in.copy, buf_left);
7ba24713
AA		 588 int rc;
589
590 size = min(size, ctask->data_count);
591
592 debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
5bb0b55a 593 size, tcp_conn->in.offset, tcp_conn->in.copied);
7ba24713
AA		 594
595 BUG_ON(size <= 0);
5bb0b55a 596 BUG_ON(tcp_ctask->sent + size > ctask->total_length);
7ba24713 597
5bb0b55a
MC		 598 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
599 (char*)buf + (offset + tcp_conn->data_copied), size);
7ba24713
AA		 600 /* must fit into skb->len */
601 BUG_ON(rc);
602
5bb0b55a
MC		 603 tcp_conn->in.offset += size;
604 tcp_conn->in.copy -= size;
605 tcp_conn->in.copied += size;
606 tcp_conn->data_copied += size;
607 tcp_ctask->sent += size;
7ba24713
AA		 608 ctask->data_count -= size;
609
5bb0b55a 610 BUG_ON(tcp_conn->in.copy < 0);
7ba24713
AA		 611 BUG_ON(ctask->data_count < 0);
612
5bb0b55a 613 if (buf_size != (tcp_conn->data_copied + offset)) {
7ba24713 614 if (!ctask->data_count) {
5bb0b55a 615 BUG_ON(buf_size - tcp_conn->data_copied < 0);
7ba24713 616 /* done with this PDU */
5bb0b55a 617 return buf_size - tcp_conn->data_copied;
7ba24713
AA		 618 }
619 return -EAGAIN;
620 }
621
622 /* done with this buffer or with both - PDU and buffer */
5bb0b55a 623 tcp_conn->data_copied = 0;
7ba24713
AA		 624 return 0;
625}
626
627/**
628 * iscsi_tcp_copy - copy skb bits to the destanation buffer
5bb0b55a 629 * @conn: iscsi tcp connection
7ba24713
AA		 630 *
631 * Notes:
632 * The function calls skb_copy_bits() and updates per-connection
633 * byte counters.
634 **/
635static inline int
5bb0b55a 636iscsi_tcp_copy(struct iscsi_tcp_conn *tcp_conn)
7ba24713 637{
5bb0b55a
MC		 638 void *buf = tcp_conn->data;
639 int buf_size = tcp_conn->in.datalen;
640 int buf_left = buf_size - tcp_conn->data_copied;
641 int size = min(tcp_conn->in.copy, buf_left);
7ba24713
AA		 642 int rc;
643
644 debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
5bb0b55a 645 size, tcp_conn->in.offset, tcp_conn->data_copied);
7ba24713
AA		 646 BUG_ON(size <= 0);
647
5bb0b55a
MC		 648 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
649 (char*)buf + tcp_conn->data_copied, size);
7ba24713
AA		 650 BUG_ON(rc);
651
5bb0b55a
MC		 652 tcp_conn->in.offset += size;
653 tcp_conn->in.copy -= size;
654 tcp_conn->in.copied += size;
655 tcp_conn->data_copied += size;
7ba24713 656
5bb0b55a 657 if (buf_size != tcp_conn->data_copied)
7ba24713
AA		 658 return -EAGAIN;
659
660 return 0;
661}
662
663static inline void
5bb0b55a
MC		 664partial_sg_digest_update(struct iscsi_tcp_conn *tcp_conn,
665 struct scatterlist *sg, int offset, int length)
7ba24713
AA		 666{
667 struct scatterlist temp;
668
669 memcpy(&temp, sg, sizeof(struct scatterlist));
670 temp.offset = offset;
671 temp.length = length;
5bb0b55a 672 crypto_digest_update(tcp_conn->data_rx_tfm, &temp, 1);
7ba24713
AA		 673}
674
f6cfba1d 675static void
5bb0b55a 676iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len)
f6cfba1d
MC		 677{
678 struct scatterlist tmp;
679
680 sg_init_one(&tmp, buf, len);
5bb0b55a 681 crypto_digest_update(tcp_conn->data_rx_tfm, &tmp, 1);
f6cfba1d
MC		 682}
683
7ba24713
AA		 684static int iscsi_scsi_data_in(struct iscsi_conn *conn)
685{
5bb0b55a
MC		 686 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
687 struct iscsi_cmd_task *ctask = tcp_conn->in.ctask;
688 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
7ba24713 689 struct scsi_cmnd *sc = ctask->sc;
f6cfba1d 690 struct scatterlist *sg;
7ba24713
AA		 691 int i, offset, rc = 0;
692
693 BUG_ON((void*)ctask != sc->SCp.ptr);
694
695 /*
696 * copying Data-In into the Scsi_Cmnd
697 */
698 if (!sc->use_sg) {
699 i = ctask->data_count;
5bb0b55a
MC		 700 rc = iscsi_ctask_copy(tcp_conn, ctask, sc->request_buffer,
701 sc->request_bufflen,
702 tcp_ctask->data_offset);
7ba24713
AA		 703 if (rc == -EAGAIN)
704 return rc;
42f72aa9 705 if (conn->datadgst_en)
5bb0b55a
MC		 706 iscsi_recv_digest_update(tcp_conn, sc->request_buffer,
707 i);
7ba24713
AA		 708 rc = 0;
709 goto done;
710 }
711
5bb0b55a 712 offset = tcp_ctask->data_offset;
7ba24713
AA		 713 sg = sc->request_buffer;
714
5bb0b55a
MC		 715 if (tcp_ctask->data_offset)
716 for (i = 0; i < tcp_ctask->sg_count; i++)
7ba24713
AA		 717 offset -= sg[i].length;
718 /* we've passed through partial sg*/
719 if (offset < 0)
720 offset = 0;
721
5bb0b55a 722 for (i = tcp_ctask->sg_count; i < sc->use_sg; i++) {
7ba24713
AA		 723 char *dest;
724
725 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
5bb0b55a 726 rc = iscsi_ctask_copy(tcp_conn, ctask, dest + sg[i].offset,
7ba24713
AA		 727 sg[i].length, offset);
728 kunmap_atomic(dest, KM_SOFTIRQ0);
729 if (rc == -EAGAIN)
730 /* continue with the next SKB/PDU */
731 return rc;
732 if (!rc) {
733 if (conn->datadgst_en) {
734 if (!offset)
5bb0b55a
MC		 735 crypto_digest_update(
736 tcp_conn->data_rx_tfm,
737 &sg[i], 1);
7ba24713 738 else
5bb0b55a
MC		 739 partial_sg_digest_update(tcp_conn,
740 &sg[i],
7ba24713
AA		 741 sg[i].offset + offset,
742 sg[i].length - offset);
743 }
744 offset = 0;
5bb0b55a 745 tcp_ctask->sg_count++;
7ba24713
AA		 746 }
747
748 if (!ctask->data_count) {
749 if (rc && conn->datadgst_en)
750 /*
751 * data-in is complete, but buffer not...
752 */
5bb0b55a 753 partial_sg_digest_update(tcp_conn, &sg[i],
7ba24713
AA		 754 sg[i].offset, sg[i].length-rc);
755 rc = 0;
756 break;
757 }
758
5bb0b55a 759 if (!tcp_conn->in.copy)
7ba24713
AA		 760 return -EAGAIN;
761 }
762 BUG_ON(ctask->data_count);
763
764done:
765 /* check for non-exceptional status */
5bb0b55a 766 if (tcp_conn->in.hdr->flags & ISCSI_FLAG_DATA_STATUS) {
7ba24713
AA		 767 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
768 (long)sc, sc->result, ctask->itt);
5bb0b55a
MC		 769 spin_lock(&conn->session->lock);
770 __iscsi_ctask_cleanup(conn, ctask);
771 __iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
772 spin_unlock(&conn->session->lock);
7ba24713
AA		 773 }
774
775 return rc;
776}
777
778static int
779iscsi_data_recv(struct iscsi_conn *conn)
780{
5bb0b55a
MC		 781 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
782 int rc = 0, opcode;
7ba24713 783
5bb0b55a
MC		 784 opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
785 switch (opcode) {
7ba24713
AA		 786 case ISCSI_OP_SCSI_DATA_IN:
787 rc = iscsi_scsi_data_in(conn);
788 break;
5bb0b55a
MC		 789 case ISCSI_OP_SCSI_CMD_RSP:
790 spin_lock(&conn->session->lock);
791 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask);
792 spin_unlock(&conn->session->lock);
7ba24713
AA		 793 case ISCSI_OP_TEXT_RSP:
794 case ISCSI_OP_LOGIN_RSP:
5bb0b55a
MC		 795 case ISCSI_OP_NOOP_IN:
796 case ISCSI_OP_ASYNC_EVENT:
797 case ISCSI_OP_REJECT:
7ba24713
AA		 798 /*
799 * Collect data segment to the connection's data
800 * placeholder
801 */
5bb0b55a 802 if (iscsi_tcp_copy(tcp_conn)) {
7ba24713
AA		 803 rc = -EAGAIN;
804 goto exit;
805 }
806
5bb0b55a
MC		 807 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, tcp_conn->data,
808 tcp_conn->in.datalen);
809 if (!rc && conn->datadgst_en && opcode != ISCSI_OP_LOGIN_RSP)
810 iscsi_recv_digest_update(tcp_conn, tcp_conn->data,
811 tcp_conn->in.datalen);
812 break;
7ba24713
AA		 813 default:
814 BUG_ON(1);
815 }
816exit:
817 return rc;
818}
819
820/**
821 * iscsi_tcp_data_recv - TCP receive in sendfile fashion
822 * @rd_desc: read descriptor
823 * @skb: socket buffer
824 * @offset: offset in skb
825 * @len: skb->len - offset
826 **/
827static int
828iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
829 unsigned int offset, size_t len)
830{
831 int rc;
832 struct iscsi_conn *conn = rd_desc->arg.data;
5bb0b55a 833 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
7ba24713
AA		 834 int processed;
835 char pad[ISCSI_PAD_LEN];
836 struct scatterlist sg;
837
838 /*
839 * Save current SKB and its offset in the corresponding
840 * connection context.
841 */
5bb0b55a
MC		 842 tcp_conn->in.copy = skb->len - offset;
843 tcp_conn->in.offset = offset;
844 tcp_conn->in.skb = skb;
845 tcp_conn->in.len = tcp_conn->in.copy;
846 BUG_ON(tcp_conn->in.copy <= 0);
847 debug_tcp("in %d bytes\n", tcp_conn->in.copy);
7ba24713
AA		 848
849more:
5bb0b55a 850 tcp_conn->in.copied = 0;
7ba24713
AA		 851 rc = 0;
852
853 if (unlikely(conn->suspend_rx)) {
854 debug_tcp("conn %d Rx suspended!\n", conn->id);
855 return 0;
856 }
857
5bb0b55a
MC		 858 if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
859 tcp_conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
860 rc = iscsi_hdr_extract(tcp_conn);
7ba24713
AA		 861 if (rc) {
862 if (rc == -EAGAIN)
863 goto nomore;
864 else {
865 iscsi_conn_failure(conn, rc);
866 return 0;
867 }
868 }
869
870 /*
871 * Verify and process incoming PDU header.
872 */
5bb0b55a
MC		 873 rc = iscsi_tcp_hdr_recv(conn);
874 if (!rc && tcp_conn->in.datalen) {
8a47cd34 875 if (conn->datadgst_en) {
5bb0b55a
MC		 876 BUG_ON(!tcp_conn->data_rx_tfm);
877 crypto_digest_init(tcp_conn->data_rx_tfm);
7ba24713 878 }
5bb0b55a 879 tcp_conn->in_progress = IN_PROGRESS_DATA_RECV;
7ba24713
AA		 880 } else if (rc) {
881 iscsi_conn_failure(conn, rc);
882 return 0;
883 }
884 }
885
5bb0b55a 886 if (tcp_conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
f6cfba1d 887 uint32_t recv_digest;
5bb0b55a 888
7ba24713 889 debug_tcp("extra data_recv offset %d copy %d\n",
5bb0b55a
MC		 890 tcp_conn->in.offset, tcp_conn->in.copy);
891 skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
f6cfba1d 892 &recv_digest, 4);
5bb0b55a
MC		 893 tcp_conn->in.offset += 4;
894 tcp_conn->in.copy -= 4;
895 if (recv_digest != tcp_conn->in.datadgst) {
f6cfba1d
MC		 896 debug_tcp("iscsi_tcp: data digest error!"
897 "0x%x != 0x%x\n", recv_digest,
5bb0b55a 898 tcp_conn->in.datadgst);
f6cfba1d
MC		 899 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
900 return 0;
901 } else {
902 debug_tcp("iscsi_tcp: data digest match!"
903 "0x%x == 0x%x\n", recv_digest,
5bb0b55a
MC		 904 tcp_conn->in.datadgst);
905 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
7ba24713
AA		 906 }
907 }
908
5bb0b55a
MC		 909 if (tcp_conn->in_progress == IN_PROGRESS_DATA_RECV &&
910 tcp_conn->in.copy) {
7ba24713
AA		 911
912 debug_tcp("data_recv offset %d copy %d\n",
5bb0b55a 913 tcp_conn->in.offset, tcp_conn->in.copy);
7ba24713
AA		 914
915 rc = iscsi_data_recv(conn);
916 if (rc) {
665b44ae 917 if (rc == -EAGAIN)
7ba24713 918 goto again;
7ba24713
AA		 919 iscsi_conn_failure(conn, rc);
920 return 0;
921 }
5bb0b55a
MC		 922 tcp_conn->in.copy -= tcp_conn->in.padding;
923 tcp_conn->in.offset += tcp_conn->in.padding;
8a47cd34 924 if (conn->datadgst_en) {
5bb0b55a
MC		 925 if (tcp_conn->in.padding) {
926 debug_tcp("padding -> %d\n",
927 tcp_conn->in.padding);
928 memset(pad, 0, tcp_conn->in.padding);
929 sg_init_one(&sg, pad, tcp_conn->in.padding);
930 crypto_digest_update(tcp_conn->data_rx_tfm,
931 &sg, 1);
7ba24713 932 }
5bb0b55a
MC		 933 crypto_digest_final(tcp_conn->data_rx_tfm,
934 (u8 *) & tcp_conn->in.datadgst);
935 debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst);
936 tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
7ba24713 937 } else
5bb0b55a 938 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
7ba24713
AA		 939 }
940
941 debug_tcp("f, processed %d from out of %d padding %d\n",
5bb0b55a
MC		 942 tcp_conn->in.offset - offset, (int)len, tcp_conn->in.padding);
943 BUG_ON(tcp_conn->in.offset - offset > len);
7ba24713 944
5bb0b55a 945 if (tcp_conn->in.offset - offset != len) {
7ba24713 946 debug_tcp("continue to process %d bytes\n",
5bb0b55a 947 (int)len - (tcp_conn->in.offset - offset));
7ba24713
AA		 948 goto more;
949 }
950
951nomore:
5bb0b55a 952 processed = tcp_conn->in.offset - offset;
7ba24713
AA		 953 BUG_ON(processed == 0);
954 return processed;
955
956again:
5bb0b55a 957 processed = tcp_conn->in.offset - offset;
7ba24713
AA		 958 debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
959 processed, (int)len, (int)rd_desc->count);
960 BUG_ON(processed == 0);
961 BUG_ON(processed > len);
962
963 conn->rxdata_octets += processed;
964 return processed;
965}
966
967static void
968iscsi_tcp_data_ready(struct sock *sk, int flag)
969{
970 struct iscsi_conn *conn = sk->sk_user_data;
971 read_descriptor_t rd_desc;
972
973 read_lock(&sk->sk_callback_lock);
974
665b44ae
MC		 975 /*
976 * Use rd_desc to pass 'conn' to iscsi_tcp_data_recv.
977 * We set count to 1 because we want the network layer to
978 * hand us all the skbs that are available. iscsi_tcp_data_recv
979 * handled pdus that cross buffers or pdus that still need data.
980 */
7ba24713 981 rd_desc.arg.data = conn;
665b44ae 982 rd_desc.count = 1;
7ba24713
AA		 983 tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);
984
985 read_unlock(&sk->sk_callback_lock);
986}
987
988static void
989iscsi_tcp_state_change(struct sock *sk)
990{
5bb0b55a 991 struct iscsi_tcp_conn *tcp_conn;
7ba24713
AA		 992 struct iscsi_conn *conn;
993 struct iscsi_session *session;
994 void (*old_state_change)(struct sock *);
995
996 read_lock(&sk->sk_callback_lock);
997
998 conn = (struct iscsi_conn*)sk->sk_user_data;
999 session = conn->session;
1000
e6273993
MC		 1001 if ((sk->sk_state == TCP_CLOSE_WAIT ||
1002 sk->sk_state == TCP_CLOSE) &&
1003 !atomic_read(&sk->sk_rmem_alloc)) {
7ba24713
AA		 1004 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1005 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1006 }
1007
5bb0b55a
MC		 1008 tcp_conn = conn->dd_data;
1009 old_state_change = tcp_conn->old_state_change;
7ba24713
AA		 1010
1011 read_unlock(&sk->sk_callback_lock);
1012
1013 old_state_change(sk);
1014}
1015
1016/**
1017 * iscsi_write_space - Called when more output buffer space is available
1018 * @sk: socket space is available for
1019 **/
1020static void
1021iscsi_write_space(struct sock *sk)
1022{
1023 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
5bb0b55a
MC		 1024 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1025
1026 tcp_conn->old_write_space(sk);
7ba24713 1027 debug_tcp("iscsi_write_space: cid %d\n", conn->id);
5bb0b55a 1028 clear_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
55e3299d 1029 scsi_queue_work(conn->session->host, &conn->xmitwork);
7ba24713
AA		 1030}
1031
1032static void
1033iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1034{
5bb0b55a
MC		 1035 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1036 struct sock *sk = tcp_conn->sock->sk;
7ba24713
AA		 1037
1038 /* assign new callbacks */
1039 write_lock_bh(&sk->sk_callback_lock);
1040 sk->sk_user_data = conn;
5bb0b55a
MC		 1041 tcp_conn->old_data_ready = sk->sk_data_ready;
1042 tcp_conn->old_state_change = sk->sk_state_change;
1043 tcp_conn->old_write_space = sk->sk_write_space;
7ba24713
AA		 1044 sk->sk_data_ready = iscsi_tcp_data_ready;
1045 sk->sk_state_change = iscsi_tcp_state_change;
1046 sk->sk_write_space = iscsi_write_space;
1047 write_unlock_bh(&sk->sk_callback_lock);
1048}
1049
1050static void
1051iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1052{
5bb0b55a
MC		 1053 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1054 struct sock *sk = tcp_conn->sock->sk;
7ba24713
AA		 1055
1056 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1057 write_lock_bh(&sk->sk_callback_lock);
1058 sk->sk_user_data = NULL;
5bb0b55a
MC		 1059 sk->sk_data_ready = tcp_conn->old_data_ready;
1060 sk->sk_state_change = tcp_conn->old_state_change;
1061 sk->sk_write_space = tcp_conn->old_write_space;
7ba24713
AA		 1062 sk->sk_no_check = 0;
1063 write_unlock_bh(&sk->sk_callback_lock);
1064}
1065
1066/**
1067 * iscsi_send - generic send routine
1068 * @sk: kernel's socket
1069 * @buf: buffer to write from
1070 * @size: actual size to write
1071 * @flags: socket's flags
7ba24713
AA		 1072 */
1073static inline int
56851698 1074iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
7ba24713 1075{
5bb0b55a
MC		 1076 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1077 struct socket *sk = tcp_conn->sock;
7cae5159 1078 int offset = buf->sg.offset + buf->sent;
7ba24713 1079
7cae5159
MC		 1080 /*
1081 * if we got use_sg=0 or are sending something we kmallocd
1082 * then we did not have to do kmap (kmap returns page_address)
1083 *
1084 * if we got use_sg > 0, but had to drop down, we do not
1085 * set clustering so this should only happen for that
1086 * slab case.
1087 */
1088 if (buf->use_sendmsg)
1089 return sock_no_sendpage(sk, buf->sg.page, offset, size, flags);
1090 else
5bb0b55a
MC		 1091 return tcp_conn->sendpage(sk, buf->sg.page, offset, size,
1092 flags);
7ba24713
AA		 1093}
1094
1095/**
1096 * iscsi_sendhdr - send PDU Header via tcp_sendpage()
1097 * @conn: iscsi connection
1098 * @buf: buffer to write from
1099 * @datalen: lenght of data to be sent after the header
1100 *
1101 * Notes:
1102 * (Tx, Fast Path)
1103 **/
1104static inline int
1105iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
1106{
5bb0b55a 1107 struct iscsi_tcp_conn *tcp_conn;
7ba24713
AA		 1108 int flags = 0; /* MSG_DONTWAIT; */
1109 int res, size;
1110
1111 size = buf->sg.length - buf->sent;
1112 BUG_ON(buf->sent + size > buf->sg.length);
1113 if (buf->sent + size != buf->sg.length || datalen)
1114 flags |= MSG_MORE;
1115
56851698 1116 res = iscsi_send(conn, buf, size, flags);
7ba24713
AA		 1117 debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
1118 if (res >= 0) {
1119 conn->txdata_octets += res;
1120 buf->sent += res;
1121 if (size != res)
1122 return -EAGAIN;
1123 return 0;
1124 } else if (res == -EAGAIN) {
5bb0b55a
MC		 1125 tcp_conn = conn->dd_data;
1126 tcp_conn->sendpage_failures_cnt++;
1127 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
7ba24713
AA		 1128 } else if (res == -EPIPE)
1129 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1130
1131 return res;
1132}
1133
1134/**
1135 * iscsi_sendpage - send one page of iSCSI Data-Out.
1136 * @conn: iscsi connection
1137 * @buf: buffer to write from
1138 * @count: remaining data
1139 * @sent: number of bytes sent
1140 *
1141 * Notes:
1142 * (Tx, Fast Path)
1143 **/
1144static inline int
1145iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1146 int *count, int *sent)
1147{
5bb0b55a 1148 struct iscsi_tcp_conn *tcp_conn;
7ba24713
AA		 1149 int flags = 0; /* MSG_DONTWAIT; */
1150 int res, size;
1151
1152 size = buf->sg.length - buf->sent;
1153 BUG_ON(buf->sent + size > buf->sg.length);
1154 if (size > *count)
1155 size = *count;
b13941f6 1156 if (buf->sent + size != buf->sg.length || *count != size)
7ba24713
AA		 1157 flags |= MSG_MORE;
1158
56851698 1159 res = iscsi_send(conn, buf, size, flags);
7ba24713
AA		 1160 debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
1161 size, buf->sent, *count, *sent, res);
1162 if (res >= 0) {
1163 conn->txdata_octets += res;
1164 buf->sent += res;
1165 *count -= res;
1166 *sent += res;
1167 if (size != res)
1168 return -EAGAIN;
1169 return 0;
1170 } else if (res == -EAGAIN) {
5bb0b55a
MC		 1171 tcp_conn = conn->dd_data;
1172 tcp_conn->sendpage_failures_cnt++;
1173 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
7ba24713
AA		 1174 } else if (res == -EPIPE)
1175 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1176
1177 return res;
1178}
1179
1180static inline void
5bb0b55a
MC		 1181iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn,
1182 struct iscsi_cmd_task *ctask)
7ba24713 1183{
5bb0b55a
MC		 1184 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1185
1186 BUG_ON(!tcp_conn->data_tx_tfm);
1187 crypto_digest_init(tcp_conn->data_tx_tfm);
1188 tcp_ctask->digest_count = 4;
7ba24713
AA		 1189}
1190
858119e1 1191static int
7ba24713
AA		 1192iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1193 struct iscsi_buf *buf, uint32_t *digest, int final)
1194{
5bb0b55a
MC		 1195 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1196 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
7ba24713
AA		 1197 int rc = 0;
1198 int sent = 0;
1199
1200 if (final)
5bb0b55a 1201 crypto_digest_final(tcp_conn->data_tx_tfm, (u8*)digest);
7ba24713 1202
6e458cc9 1203 iscsi_buf_init_iov(buf, (char*)digest, 4);
5bb0b55a 1204 rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
7ba24713 1205 if (rc) {
5bb0b55a
MC		 1206 tcp_ctask->datadigest = *digest;
1207 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
7ba24713 1208 } else
5bb0b55a 1209 tcp_ctask->digest_count = 4;
7ba24713
AA		 1210 return rc;
1211}
1212
1213/**
1214 * iscsi_solicit_data_cont - initialize next Data-Out
1215 * @conn: iscsi connection
1216 * @ctask: scsi command task
1217 * @r2t: R2T info
1218 * @left: bytes left to transfer
1219 *
1220 * Notes:
1221 * Initialize next Data-Out within this R2T sequence and continue
1222 * to process next Scatter-Gather element(if any) of this SCSI command.
1223 *
1224 * Called under connection lock.
1225 **/
1226static void
1227iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1228 struct iscsi_r2t_info *r2t, int left)
1229{
5bb0b55a 1230 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
7ba24713
AA		 1231 struct iscsi_data *hdr;
1232 struct iscsi_data_task *dtask;
1233 struct scsi_cmnd *sc = ctask->sc;
1234 int new_offset;
1235
5bb0b55a 1236 dtask = mempool_alloc(tcp_ctask->datapool, GFP_ATOMIC);
7ba24713 1237 BUG_ON(!dtask);
30a6c652 1238 INIT_LIST_HEAD(&dtask->item);
7ba24713
AA		 1239 hdr = &dtask->hdr;
1240 memset(hdr, 0, sizeof(struct iscsi_data));
1241 hdr->ttt = r2t->ttt;
1242 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1243 r2t->solicit_datasn++;
1244 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
5bb0b55a
MC		 1245 memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
1246 hdr->itt = ctask->hdr->itt;
7ba24713
AA		 1247 hdr->exp_statsn = r2t->exp_statsn;
1248 new_offset = r2t->data_offset + r2t->sent;
1249 hdr->offset = cpu_to_be32(new_offset);
1250 if (left > conn->max_xmit_dlength) {
1251 hton24(hdr->dlength, conn->max_xmit_dlength);
1252 r2t->data_count = conn->max_xmit_dlength;
1253 } else {
1254 hton24(hdr->dlength, left);
1255 r2t->data_count = left;
1256 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1257 }
1258 conn->dataout_pdus_cnt++;
1259
6e458cc9 1260 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
af973481 1261 sizeof(struct iscsi_hdr));
7ba24713
AA		 1262
1263 r2t->dtask = dtask;
1264
1265 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) {
5bb0b55a 1266 BUG_ON(tcp_ctask->bad_sg == r2t->sg);
7ba24713
AA		 1267 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1268 r2t->sg += 1;
1269 } else
5bb0b55a 1270 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
7ba24713
AA		 1271 (char*)sc->request_buffer + new_offset,
1272 r2t->data_count);
1273
5bb0b55a 1274 list_add(&dtask->item, &tcp_ctask->dataqueue);
7ba24713
AA		 1275}
1276
1277static void
1278iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1279{
5bb0b55a 1280 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
7ba24713
AA		 1281 struct iscsi_data_task *dtask;
1282
5bb0b55a 1283 dtask = mempool_alloc(tcp_ctask->datapool, GFP_ATOMIC);
7ba24713 1284 BUG_ON(!dtask);
30a6c652 1285 INIT_LIST_HEAD(&dtask->item);
7ba24713 1286
5bb0b55a
MC		 1287 iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr,
1288 tcp_ctask->r2t_data_count);
6e458cc9 1289 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
af973481 1290 sizeof(struct iscsi_hdr));
7ba24713 1291
5bb0b55a
MC		 1292 list_add(&dtask->item, &tcp_ctask->dataqueue);
1293 tcp_ctask->dtask = dtask;
7ba24713
AA		 1294}
1295
1296/**
5bb0b55a 1297 * iscsi_tcp_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
7ba24713
AA		 1298 * @conn: iscsi connection
1299 * @ctask: scsi command task
1300 * @sc: scsi command
1301 **/
1302static void
5bb0b55a 1303iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask)
7ba24713 1304{
5bb0b55a
MC		 1305 struct scsi_cmnd *sc = ctask->sc;
1306 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
7ba24713 1307
5bb0b55a 1308 BUG_ON(__kfifo_len(tcp_ctask->r2tqueue));
7ba24713 1309
5bb0b55a
MC		 1310 tcp_ctask->sent = 0;
1311 tcp_ctask->sg_count = 0;
7ba24713
AA		 1312
1313 if (sc->sc_data_direction == DMA_TO_DEVICE) {
5bb0b55a
MC		 1314 tcp_ctask->xmstate = XMSTATE_W_HDR;
1315 tcp_ctask->exp_r2tsn = 0;
7ba24713 1316 BUG_ON(ctask->total_length == 0);
5bb0b55a 1317
7ba24713
AA		 1318 if (sc->use_sg) {
1319 struct scatterlist *sg = sc->request_buffer;
1320
5bb0b55a
MC		 1321 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1322 &sg[tcp_ctask->sg_count++]);
1323 tcp_ctask->sg = sg;
1324 tcp_ctask->bad_sg = sg + sc->use_sg;
1325 } else
1326 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
1327 sc->request_buffer,
1328 sc->request_bufflen);
7ba24713 1329
5bb0b55a
MC		 1330 if (ctask->imm_count)
1331 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
1332
1333 tcp_ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1334 if (tcp_ctask->pad_count) {
1335 tcp_ctask->pad_count = ISCSI_PAD_LEN -
1336 tcp_ctask->pad_count;
7ba24713 1337 debug_scsi("write padding %d bytes\n",
5bb0b55a
MC		 1338 tcp_ctask->pad_count);
1339 tcp_ctask->xmstate |= XMSTATE_W_PAD;
7ba24713 1340 }
7ba24713 1341
5bb0b55a
MC		 1342 if (ctask->unsol_count)
1343 tcp_ctask->xmstate |= XMSTATE_UNS_HDR |
1344 XMSTATE_UNS_INIT;
1345 tcp_ctask->r2t_data_count = ctask->total_length -
7ba24713
AA		 1346 ctask->imm_count -
1347 ctask->unsol_count;
1348
1349 debug_scsi("cmd [itt %x total %d imm %d imm_data %d "
1350 "r2t_data %d]\n",
1351 ctask->itt, ctask->total_length, ctask->imm_count,
5bb0b55a
MC		 1352 ctask->unsol_count, tcp_ctask->r2t_data_count);
1353 } else
1354 tcp_ctask->xmstate = XMSTATE_R_HDR;
7ba24713 1355
6e458cc9 1356 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)ctask->hdr,
af973481 1357 sizeof(struct iscsi_hdr));
7ba24713
AA		 1358}
1359
1360/**
5bb0b55a 1361 * iscsi_tcp_mtask_xmit - xmit management(immediate) task
7ba24713
AA		 1362 * @conn: iscsi connection
1363 * @mtask: task management task
1364 *
1365 * Notes:
1366 * The function can return -EAGAIN in which case caller must
1367 * call it again later, or recover. '0' return code means successful
1368 * xmit.
1369 *
1370 * Management xmit state machine consists of two states:
1371 * IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress
1372 * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
1373 **/
1374static int
5bb0b55a 1375iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
7ba24713 1376{
5bb0b55a 1377 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
7ba24713
AA		 1378
1379 debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
5bb0b55a 1380 conn->id, tcp_mtask->xmstate, mtask->itt);
7ba24713 1381
5bb0b55a
MC		 1382 if (tcp_mtask->xmstate & XMSTATE_IMM_HDR) {
1383 tcp_mtask->xmstate &= ~XMSTATE_IMM_HDR;
7ba24713 1384 if (mtask->data_count)
5bb0b55a 1385 tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
af973481 1386 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
30a6c652 1387 conn->stop_stage != STOP_CONN_RECOVER &&
af973481 1388 conn->hdrdgst_en)
5bb0b55a
MC		 1389 iscsi_hdr_digest(conn, &tcp_mtask->headbuf,
1390 (u8*)tcp_mtask->hdrext);
1391 if (iscsi_sendhdr(conn, &tcp_mtask->headbuf,
1392 mtask->data_count)) {
1393 tcp_mtask->xmstate |= XMSTATE_IMM_HDR;
7ba24713 1394 if (mtask->data_count)
5bb0b55a 1395 tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA;
7ba24713
AA		 1396 return -EAGAIN;
1397 }
1398 }
1399
5bb0b55a 1400 if (tcp_mtask->xmstate & XMSTATE_IMM_DATA) {
7ba24713 1401 BUG_ON(!mtask->data_count);
5bb0b55a 1402 tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA;
7ba24713
AA		 1403 /* FIXME: implement.
1404 * Virtual buffer could be spreaded across multiple pages...
1405 */
1406 do {
5bb0b55a
MC		 1407 if (iscsi_sendpage(conn, &tcp_mtask->sendbuf,
1408 &mtask->data_count, &tcp_mtask->sent)) {
1409 tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
7ba24713
AA		 1410 return -EAGAIN;
1411 }
1412 } while (mtask->data_count);
1413 }
1414
5bb0b55a
MC		 1415 BUG_ON(tcp_mtask->xmstate != XMSTATE_IDLE);
1416 if (mtask->hdr->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1417 struct iscsi_session *session = conn->session;
1418
1419 spin_lock_bh(&session->lock);
1420 list_del(&conn->mtask->running);
1421 __kfifo_put(session->mgmtpool.queue, (void*)&conn->mtask,
1422 sizeof(void*));
1423 spin_unlock_bh(&session->lock);
1424 }
7ba24713
AA		 1425 return 0;
1426}
1427
1428static inline int
5bb0b55a
MC		 1429handle_xmstate_r_hdr(struct iscsi_conn *conn,
1430 struct iscsi_tcp_cmd_task *tcp_ctask)
7ba24713 1431{
5bb0b55a 1432 tcp_ctask->xmstate &= ~XMSTATE_R_HDR;
42f72aa9 1433 if (conn->hdrdgst_en)
5bb0b55a
MC		 1434 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1435 (u8*)tcp_ctask->hdrext);
1436 if (!iscsi_sendhdr(conn, &tcp_ctask->headbuf, 0)) {
1437 BUG_ON(tcp_ctask->xmstate != XMSTATE_IDLE);
7ba24713
AA		 1438 return 0; /* wait for Data-In */
1439 }
5bb0b55a 1440 tcp_ctask->xmstate |= XMSTATE_R_HDR;
7ba24713
AA		 1441 return -EAGAIN;
1442}
1443
1444static inline int
5bb0b55a
MC		 1445handle_xmstate_w_hdr(struct iscsi_conn *conn,
1446 struct iscsi_cmd_task *ctask)
7ba24713 1447{
5bb0b55a
MC		 1448 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1449
1450 tcp_ctask->xmstate &= ~XMSTATE_W_HDR;
42f72aa9 1451 if (conn->hdrdgst_en)
5bb0b55a
MC		 1452 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1453 (u8*)tcp_ctask->hdrext);
1454 if (iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count)) {
1455 tcp_ctask->xmstate |= XMSTATE_W_HDR;
7ba24713
AA		 1456 return -EAGAIN;
1457 }
1458 return 0;
1459}
1460
1461static inline int
1462handle_xmstate_data_digest(struct iscsi_conn *conn,
1463 struct iscsi_cmd_task *ctask)
1464{
5bb0b55a
MC		 1465 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1466
1467 tcp_ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
1468 debug_tcp("resent data digest 0x%x\n", tcp_ctask->datadigest);
1469 if (iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf,
1470 &tcp_ctask->datadigest, 0)) {
1471 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
7ba24713 1472 debug_tcp("resent data digest 0x%x fail!\n",
5bb0b55a 1473 tcp_ctask->datadigest);
7ba24713
AA		 1474 return -EAGAIN;
1475 }
1476 return 0;
1477}
1478
1479static inline int
1480handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1481{
5bb0b55a
MC		 1482 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1483 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1484
7ba24713 1485 BUG_ON(!ctask->imm_count);
5bb0b55a 1486 tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA;
7ba24713
AA		 1487
1488 if (conn->datadgst_en) {
5bb0b55a
MC		 1489 iscsi_data_digest_init(tcp_conn, ctask);
1490 tcp_ctask->immdigest = 0;
7ba24713
AA		 1491 }
1492
1493 for (;;) {
5bb0b55a
MC		 1494 if (iscsi_sendpage(conn, &tcp_ctask->sendbuf, &ctask->imm_count,
1495 &tcp_ctask->sent)) {
1496 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
7ba24713 1497 if (conn->datadgst_en) {
5bb0b55a
MC		 1498 crypto_digest_final(tcp_conn->data_tx_tfm,
1499 (u8*)&tcp_ctask->immdigest);
7ba24713 1500 debug_tcp("tx imm sendpage fail 0x%x\n",
5bb0b55a 1501 tcp_ctask->datadigest);
7ba24713
AA		 1502 }
1503 return -EAGAIN;
1504 }
1505 if (conn->datadgst_en)
5bb0b55a
MC		 1506 crypto_digest_update(tcp_conn->data_tx_tfm,
1507 &tcp_ctask->sendbuf.sg, 1);
7ba24713
AA		 1508
1509 if (!ctask->imm_count)
1510 break;
5bb0b55a
MC		 1511 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1512 &tcp_ctask->sg[tcp_ctask->sg_count++]);
7ba24713
AA		 1513 }
1514
5bb0b55a
MC		 1515 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
1516 if (iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf,
1517 &tcp_ctask->immdigest, 1)) {
7ba24713 1518 debug_tcp("sending imm digest 0x%x fail!\n",
5bb0b55a 1519 tcp_ctask->immdigest);
7ba24713
AA		 1520 return -EAGAIN;
1521 }
5bb0b55a 1522 debug_tcp("sending imm digest 0x%x\n", tcp_ctask->immdigest);
7ba24713
AA		 1523 }
1524
1525 return 0;
1526}
1527
1528static inline int
1529handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1530{
5bb0b55a 1531 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
7ba24713
AA		 1532 struct iscsi_data_task *dtask;
1533
5bb0b55a
MC		 1534 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
1535 if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) {
7ba24713 1536 iscsi_unsolicit_data_init(conn, ctask);
5bb0b55a
MC		 1537 BUG_ON(!tcp_ctask->dtask);
1538 dtask = tcp_ctask->dtask;
af973481 1539 if (conn->hdrdgst_en)
5bb0b55a 1540 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
af973481 1541 (u8*)dtask->hdrext);
5bb0b55a 1542 tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT;
7ba24713 1543 }
5bb0b55a
MC		 1544 if (iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count)) {
1545 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
1546 tcp_ctask->xmstate |= XMSTATE_UNS_HDR;
7ba24713
AA		 1547 return -EAGAIN;
1548 }
1549
1550 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
5bb0b55a 1551 ctask->itt, ctask->unsol_count, tcp_ctask->sent);
7ba24713
AA		 1552 return 0;
1553}
1554
1555static inline int
1556handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1557{
5bb0b55a
MC		 1558 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1559 struct iscsi_data_task *dtask = tcp_ctask->dtask;
1560 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
7ba24713
AA		 1561
1562 BUG_ON(!ctask->data_count);
5bb0b55a 1563 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
7ba24713
AA		 1564
1565 if (conn->datadgst_en) {
5bb0b55a 1566 iscsi_data_digest_init(tcp_conn, ctask);
7ba24713
AA		 1567 dtask->digest = 0;
1568 }
1569
1570 for (;;) {
5bb0b55a 1571 int start = tcp_ctask->sent;
7ba24713 1572
5bb0b55a
MC		 1573 if (iscsi_sendpage(conn, &tcp_ctask->sendbuf,
1574 &ctask->data_count, &tcp_ctask->sent)) {
1575 ctask->unsol_count -= tcp_ctask->sent - start;
1576 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
7ba24713
AA		 1577 /* will continue with this ctask later.. */
1578 if (conn->datadgst_en) {
5bb0b55a 1579 crypto_digest_final(tcp_conn->data_tx_tfm,
7ba24713
AA		 1580 (u8 *)&dtask->digest);
1581 debug_tcp("tx uns data fail 0x%x\n",
1582 dtask->digest);
1583 }
1584 return -EAGAIN;
1585 }
1586
5bb0b55a
MC		 1587 BUG_ON(tcp_ctask->sent > ctask->total_length);
1588 ctask->unsol_count -= tcp_ctask->sent - start;
7ba24713
AA		 1589
1590 /*
1591 * XXX:we may run here with un-initial sendbuf.
1592 * so pass it
1593 */
5bb0b55a
MC		 1594 if (conn->datadgst_en && tcp_ctask->sent - start > 0)
1595 crypto_digest_update(tcp_conn->data_tx_tfm,
1596 &tcp_ctask->sendbuf.sg, 1);
7ba24713
AA		 1597
1598 if (!ctask->data_count)
1599 break;
5bb0b55a
MC		 1600 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1601 &tcp_ctask->sg[tcp_ctask->sg_count++]);
7ba24713
AA		 1602 }
1603 BUG_ON(ctask->unsol_count < 0);
1604
1605 /*
1606 * Done with the Data-Out. Next, check if we need
1607 * to send another unsolicited Data-Out.
1608 */
1609 if (ctask->unsol_count) {
1610 if (conn->datadgst_en) {
1611 if (iscsi_digest_final_send(conn, ctask,
1612 &dtask->digestbuf,
1613 &dtask->digest, 1)) {
1614 debug_tcp("send uns digest 0x%x fail\n",
1615 dtask->digest);
1616 return -EAGAIN;
1617 }
1618 debug_tcp("sending uns digest 0x%x, more uns\n",
1619 dtask->digest);
1620 }
5bb0b55a 1621 tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
7ba24713
AA		 1622 return 1;
1623 }
1624
5bb0b55a 1625 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
7ba24713
AA		 1626 if (iscsi_digest_final_send(conn, ctask,
1627 &dtask->digestbuf,
1628 &dtask->digest, 1)) {
1629 debug_tcp("send last uns digest 0x%x fail\n",
1630 dtask->digest);
1631 return -EAGAIN;
1632 }
1633 debug_tcp("sending uns digest 0x%x\n",dtask->digest);
1634 }
1635
1636 return 0;
1637}
1638
1639static inline int
1640handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1641{
1642 struct iscsi_session *session = conn->session;
5bb0b55a
MC		 1643 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1644 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1645 struct iscsi_r2t_info *r2t = tcp_ctask->r2t;
7ba24713
AA		 1646 struct iscsi_data_task *dtask = r2t->dtask;
1647 int left;
1648
5bb0b55a
MC		 1649 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1650 tcp_ctask->dtask = dtask;
7ba24713
AA		 1651
1652 if (conn->datadgst_en) {
5bb0b55a 1653 iscsi_data_digest_init(tcp_conn, ctask);
7ba24713
AA		 1654 dtask->digest = 0;
1655 }
1656solicit_again:
1657 /*
1658 * send Data-Out whitnin this R2T sequence.
1659 */
1660 if (!r2t->data_count)
1661 goto data_out_done;
1662
1663 if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) {
5bb0b55a 1664 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
7ba24713
AA		 1665 /* will continue with this ctask later.. */
1666 if (conn->datadgst_en) {
5bb0b55a 1667 crypto_digest_final(tcp_conn->data_tx_tfm,
7ba24713
AA		 1668 (u8 *)&dtask->digest);
1669 debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
1670 }
1671 return -EAGAIN;
1672 }
1673
1674 BUG_ON(r2t->data_count < 0);
1675 if (conn->datadgst_en)
5bb0b55a
MC		 1676 crypto_digest_update(tcp_conn->data_tx_tfm, &r2t->sendbuf.sg,
1677 1);
7ba24713
AA		 1678
1679 if (r2t->data_count) {
1680 BUG_ON(ctask->sc->use_sg == 0);
1681 if (!iscsi_buf_left(&r2t->sendbuf)) {
5bb0b55a 1682 BUG_ON(tcp_ctask->bad_sg == r2t->sg);
7ba24713
AA		 1683 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1684 r2t->sg += 1;
1685 }
1686 goto solicit_again;
1687 }
1688
1689data_out_done:
1690 /*
1691 * Done with this Data-Out. Next, check if we have
1692 * to send another Data-Out for this R2T.
1693 */
1694 BUG_ON(r2t->data_length - r2t->sent < 0);
1695 left = r2t->data_length - r2t->sent;
1696 if (left) {
1697 if (conn->datadgst_en) {
1698 if (iscsi_digest_final_send(conn, ctask,
1699 &dtask->digestbuf,
1700 &dtask->digest, 1)) {
1701 debug_tcp("send r2t data digest 0x%x"
1702 "fail\n", dtask->digest);
1703 return -EAGAIN;
1704 }
1705 debug_tcp("r2t data send digest 0x%x\n",
1706 dtask->digest);
1707 }
1708 iscsi_solicit_data_cont(conn, ctask, r2t, left);
5bb0b55a
MC		 1709 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1710 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
7ba24713
AA		 1711 return 1;
1712 }
1713
1714 /*
1715 * Done with this R2T. Check if there are more
1716 * outstanding R2Ts ready to be processed.
1717 */
5bb0b55a 1718 BUG_ON(tcp_ctask->r2t_data_count - r2t->data_length < 0);
7ba24713
AA		 1719 if (conn->datadgst_en) {
1720 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
1721 &dtask->digest, 1)) {
1722 debug_tcp("send last r2t data digest 0x%x"
1723 "fail\n", dtask->digest);
1724 return -EAGAIN;
1725 }
1726 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
1727 }
1728
5bb0b55a
MC		 1729 tcp_ctask->r2t_data_count -= r2t->data_length;
1730 tcp_ctask->r2t = NULL;
7ba24713 1731 spin_lock_bh(&session->lock);
5bb0b55a 1732 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
7ba24713 1733 spin_unlock_bh(&session->lock);
5bb0b55a
MC		 1734 if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
1735 tcp_ctask->r2t = r2t;
1736 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1737 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
7ba24713
AA		 1738 return 1;
1739 }
1740
1741 return 0;
1742}
1743
1744static inline int
1745handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1746{
5bb0b55a
MC		 1747 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1748 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1749 struct iscsi_data_task *dtask = tcp_ctask->dtask;
7ba24713
AA		 1750 int sent;
1751
5bb0b55a 1752 tcp_ctask->xmstate &= ~XMSTATE_W_PAD;
6e458cc9 1753 iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,
5bb0b55a
MC		 1754 tcp_ctask->pad_count);
1755 if (iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,
1756 &sent)) {
1757 tcp_ctask->xmstate |= XMSTATE_W_PAD;
7ba24713
AA		 1758 return -EAGAIN;
1759 }
1760
1761 if (conn->datadgst_en) {
5bb0b55a
MC		 1762 crypto_digest_update(tcp_conn->data_tx_tfm,
1763 &tcp_ctask->sendbuf.sg, 1);
7ba24713
AA		 1764 /* imm data? */
1765 if (!dtask) {
5bb0b55a
MC		 1766 if (iscsi_digest_final_send(conn, ctask,
1767 &tcp_ctask->immbuf,
1768 &tcp_ctask->immdigest, 1)) {
7ba24713 1769 debug_tcp("send padding digest 0x%x"
5bb0b55a 1770 "fail!\n", tcp_ctask->immdigest);
7ba24713
AA		 1771 return -EAGAIN;
1772 }
1773 debug_tcp("done with padding, digest 0x%x\n",
5bb0b55a 1774 tcp_ctask->datadigest);
7ba24713
AA		 1775 } else {
1776 if (iscsi_digest_final_send(conn, ctask,
1777 &dtask->digestbuf,
1778 &dtask->digest, 1)) {
1779 debug_tcp("send padding digest 0x%x"
1780 "fail\n", dtask->digest);
1781 return -EAGAIN;
1782 }
1783 debug_tcp("done with padding, digest 0x%x\n",
1784 dtask->digest);
1785 }
1786 }
1787
1788 return 0;
1789}
1790
1791static int
5bb0b55a 1792iscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
7ba24713 1793{
5bb0b55a 1794 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
7ba24713
AA		 1795 int rc = 0;
1796
1797 debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
5bb0b55a 1798 conn->id, tcp_ctask->xmstate, ctask->itt);
7ba24713
AA		 1799
1800 /*
1801 * serialize with TMF AbortTask
1802 */
1803 if (ctask->mtask)
1804 return rc;
1805
5bb0b55a
MC		 1806 if (tcp_ctask->xmstate & XMSTATE_R_HDR) {
1807 rc = handle_xmstate_r_hdr(conn, tcp_ctask);
7ba24713
AA		 1808 return rc;
1809 }
1810
5bb0b55a 1811 if (tcp_ctask->xmstate & XMSTATE_W_HDR) {
7ba24713
AA		 1812 rc = handle_xmstate_w_hdr(conn, ctask);
1813 if (rc)
1814 return rc;
1815 }
1816
1817 /* XXX: for data digest xmit recover */
5bb0b55a 1818 if (tcp_ctask->xmstate & XMSTATE_DATA_DIGEST) {
7ba24713
AA		 1819 rc = handle_xmstate_data_digest(conn, ctask);
1820 if (rc)
1821 return rc;
1822 }
1823
5bb0b55a 1824 if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) {
7ba24713
AA		 1825 rc = handle_xmstate_imm_data(conn, ctask);
1826 if (rc)
1827 return rc;
1828 }
1829
5bb0b55a 1830 if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) {
7ba24713 1831 BUG_ON(!ctask->unsol_count);
5bb0b55a 1832 tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR;
7ba24713
AA		 1833unsolicit_head_again:
1834 rc = handle_xmstate_uns_hdr(conn, ctask);
1835 if (rc)
1836 return rc;
1837 }
1838
5bb0b55a 1839 if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
7ba24713
AA		 1840 rc = handle_xmstate_uns_data(conn, ctask);
1841 if (rc == 1)
1842 goto unsolicit_head_again;
1843 else if (rc)
1844 return rc;
1845 goto done;
1846 }
1847
5bb0b55a 1848 if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
7ba24713
AA		 1849 struct iscsi_r2t_info *r2t;
1850
5bb0b55a
MC		 1851 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1852 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1853 if (!tcp_ctask->r2t)
1854 __kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
7ba24713
AA		 1855 sizeof(void*));
1856solicit_head_again:
5bb0b55a 1857 r2t = tcp_ctask->r2t;
af973481 1858 if (conn->hdrdgst_en)
42f72aa9 1859 iscsi_hdr_digest(conn, &r2t->headbuf,
af973481 1860 (u8*)r2t->dtask->hdrext);
7ba24713 1861 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) {
5bb0b55a
MC		 1862 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1863 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
7ba24713
AA		 1864 return -EAGAIN;
1865 }
1866
1867 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
1868 r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
1869 r2t->sent);
1870 }
1871
5bb0b55a 1872 if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
7ba24713
AA		 1873 rc = handle_xmstate_sol_data(conn, ctask);
1874 if (rc == 1)
1875 goto solicit_head_again;
1876 if (rc)
1877 return rc;
1878 }
1879
1880done:
1881 /*
1882 * Last thing to check is whether we need to send write
1883 * padding. Note that we check for xmstate equality, not just the bit.
1884 */
5bb0b55a 1885 if (tcp_ctask->xmstate == XMSTATE_W_PAD)
7ba24713
AA		 1886 rc = handle_xmstate_w_pad(conn, ctask);
1887
1888 return rc;
1889}
1890
5bb0b55a
MC		 1891static struct iscsi_cls_conn *
1892iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
7ba24713 1893{
5bb0b55a
MC		 1894 struct iscsi_conn *conn;
1895 struct iscsi_cls_conn *cls_conn;
1896 struct iscsi_tcp_conn *tcp_conn;
7ba24713 1897
5bb0b55a
MC		 1898 cls_conn = iscsi_conn_setup(cls_session, conn_idx);
1899 if (!cls_conn)
1900 return NULL;
1901 conn = cls_conn->dd_data;
7ba24713 1902 /*
5bb0b55a
MC		 1903 * due to strange issues with iser these are not set
1904 * in iscsi_conn_setup
7ba24713 1905 */
5bb0b55a 1906 conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
7ba24713 1907
5bb0b55a
MC		 1908 tcp_conn = kzalloc(sizeof(*tcp_conn), GFP_KERNEL);
1909 if (!tcp_conn)
1910 goto tcp_conn_alloc_fail;
7ba24713 1911
5bb0b55a
MC		 1912 conn->dd_data = tcp_conn;
1913 tcp_conn->iscsi_conn = conn;
1914 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1915 /* initial operational parameters */
1916 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
1917 tcp_conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
7ba24713 1918
5bb0b55a
MC		 1919 /* allocate initial PDU receive place holder */
1920 if (tcp_conn->data_size <= PAGE_SIZE)
1921 tcp_conn->data = kmalloc(tcp_conn->data_size, GFP_KERNEL);
1922 else
1923 tcp_conn->data = (void*)__get_free_pages(GFP_KERNEL,
1924 get_order(tcp_conn->data_size));
1925 if (!tcp_conn->data)
1926 goto max_recv_dlenght_alloc_fail;
7ba24713 1927
5bb0b55a 1928 return cls_conn;
7ba24713 1929
5bb0b55a
MC		 1930max_recv_dlenght_alloc_fail:
1931 kfree(tcp_conn);
1932tcp_conn_alloc_fail:
1933 iscsi_conn_teardown(cls_conn);
1934 return NULL;
7ba24713
AA		 1935}
1936
1937static void
5bb0b55a 1938iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
7ba24713 1939{
5bb0b55a
MC		 1940 struct iscsi_conn *conn = cls_conn->dd_data;
1941 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1942 int digest = 0;
7ba24713 1943
5bb0b55a
MC		 1944 if (conn->hdrdgst_en || conn->datadgst_en)
1945 digest = 1;
7ba24713 1946
5bb0b55a 1947 iscsi_conn_teardown(cls_conn);
7ba24713 1948
5bb0b55a
MC		 1949 /* now free tcp_conn */
1950 if (digest) {
1951 if (tcp_conn->tx_tfm)
1952 crypto_free_tfm(tcp_conn->tx_tfm);
1953 if (tcp_conn->rx_tfm)
1954 crypto_free_tfm(tcp_conn->rx_tfm);
1955 if (tcp_conn->data_tx_tfm)
1956 crypto_free_tfm(tcp_conn->data_tx_tfm);
1957 if (tcp_conn->data_rx_tfm)
1958 crypto_free_tfm(tcp_conn->data_rx_tfm);
1959 }
7ba24713 1960
5bb0b55a
MC		 1961 /* free conn->data, size = MaxRecvDataSegmentLength */
1962 if (tcp_conn->data_size <= PAGE_SIZE)
1963 kfree(tcp_conn->data);
1964 else
1965 free_pages((unsigned long)tcp_conn->data,
1966 get_order(tcp_conn->data_size));
1967 kfree(tcp_conn);
1968}
7ba24713 1969
5bb0b55a
MC		 1970static int
1971iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
264faaaa 1972 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
5bb0b55a
MC		 1973 int is_leading)
1974{
1975 struct iscsi_conn *conn = cls_conn->dd_data;
1976 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1977 struct sock *sk;
1978 struct socket *sock;
1979 int err;
7ba24713 1980
5bb0b55a 1981 /* lookup for existing socket */
264faaaa 1982 sock = sockfd_lookup((int)transport_eph, &err);
5bb0b55a
MC		 1983 if (!sock) {
1984 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
1985 return -EEXIST;
7ba24713
AA		 1986 }
1987
5bb0b55a
MC		 1988 err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
1989 if (err)
1990 return err;
7ba24713 1991
5bb0b55a
MC		 1992 if (conn->stop_stage != STOP_CONN_SUSPEND) {
1993 /* bind iSCSI connection and socket */
1994 tcp_conn->sock = sock;
7ba24713
AA		 1995
1996 /* setup Socket parameters */
1997 sk = sock->sk;
1998 sk->sk_reuse = 1;
1999 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
2000 sk->sk_allocation = GFP_ATOMIC;
2001
2002 /* FIXME: disable Nagle's algorithm */
2003
2004 /*
2005 * Intercept TCP callbacks for sendfile like receive
2006 * processing.
2007 */
5bb0b55a 2008 conn->recv_lock = &sk->sk_callback_lock;
7ba24713 2009 iscsi_conn_set_callbacks(conn);
5bb0b55a 2010 tcp_conn->sendpage = tcp_conn->sock->ops->sendpage;
7ba24713
AA		 2011 /*
2012 * set receive state machine into initial state
2013 */
5bb0b55a 2014 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
7ba24713
AA		 2015 }
2016
7ba24713
AA		 2017 return 0;
2018}
2019
30a6c652 2020static void
5bb0b55a 2021iscsi_tcp_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
30a6c652 2022{
5bb0b55a 2023 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
30a6c652 2024 struct iscsi_r2t_info *r2t;
30a6c652
MC		 2025
2026 /* flush ctask's r2t queues */
5bb0b55a
MC		 2027 while (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*)))
2028 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
30a6c652 2029 sizeof(void*));
30a6c652 2030
5bb0b55a 2031 __iscsi_ctask_cleanup(conn, ctask);
30a6c652
MC		 2032}
2033
2034static void
5bb0b55a 2035iscsi_tcp_suspend_conn_rx(struct iscsi_conn *conn)
30a6c652 2036{
5bb0b55a 2037 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
7ba24713 2038 struct sock *sk;
7ba24713 2039
5bb0b55a
MC		 2040 if (!tcp_conn->sock)
2041 return;
2042
2043 sk = tcp_conn->sock->sk;
7ba24713 2044 write_lock_bh(&sk->sk_callback_lock);
5bb0b55a 2045 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
7ba24713 2046 write_unlock_bh(&sk->sk_callback_lock);
30a6c652 2047}
7ba24713 2048
30a6c652 2049static void
5bb0b55a 2050iscsi_tcp_terminate_conn(struct iscsi_conn *conn)
30a6c652 2051{
5bb0b55a 2052 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
7ba24713 2053
5bb0b55a
MC		 2054 if (!tcp_conn->sock)
2055 return;
7ba24713 2056
5bb0b55a 2057 sock_hold(tcp_conn->sock->sk);
30a6c652 2058 iscsi_conn_restore_callbacks(conn);
5bb0b55a 2059 sock_put(tcp_conn->sock->sk);
7ba24713 2060
5bb0b55a
MC		 2061 sock_release(tcp_conn->sock);
2062 tcp_conn->sock = NULL;
2063 conn->recv_lock = NULL;
7ba24713
AA		 2064}
2065
5bb0b55a 2066/* called with host lock */
30a6c652 2067static void
5bb0b55a
MC		 2068iscsi_tcp_mgmt_init(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask,
2069 char *data, uint32_t data_size)
7ba24713 2070{
5bb0b55a 2071 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
7ba24713 2072
6e458cc9
MC		 2073 iscsi_buf_init_iov(&tcp_mtask->headbuf, (char*)mtask->hdr,
2074 sizeof(struct iscsi_hdr));
5bb0b55a 2075 tcp_mtask->xmstate = XMSTATE_IMM_HDR;
7ba24713 2076
5bb0b55a
MC		 2077 if (mtask->data_count)
2078 iscsi_buf_init_iov(&tcp_mtask->sendbuf, (char*)mtask->data,
7ba24713 2079 mtask->data_count);
7ba24713
AA		 2080}
2081
2082static int
2083iscsi_r2tpool_alloc(struct iscsi_session *session)
2084{
2085 int i;
2086 int cmd_i;
2087
2088 /*
2089 * initialize per-task: R2T pool and xmit queue
2090 */
2091 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
2092 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
5bb0b55a 2093 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
7ba24713
AA		 2094
2095 /*
2096 * pre-allocated x4 as much r2ts to handle race when
2097 * target acks DataOut faster than we data_xmit() queues
2098 * could replenish r2tqueue.
2099 */
2100
2101 /* R2T pool */
5bb0b55a
MC		 2102 if (iscsi_pool_init(&tcp_ctask->r2tpool, session->max_r2t * 4,
2103 (void***)&tcp_ctask->r2ts,
2104 sizeof(struct iscsi_r2t_info))) {
7ba24713
AA		 2105 goto r2t_alloc_fail;
2106 }
2107
2108 /* R2T xmit queue */
5bb0b55a 2109 tcp_ctask->r2tqueue = kfifo_alloc(
7ba24713 2110 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
5bb0b55a
MC		 2111 if (tcp_ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
2112 iscsi_pool_free(&tcp_ctask->r2tpool,
2113 (void**)tcp_ctask->r2ts);
7ba24713
AA		 2114 goto r2t_alloc_fail;
2115 }
2116
2117 /*
2118 * number of
2119 * Data-Out PDU's within R2T-sequence can be quite big;
2120 * using mempool
2121 */
5bb0b55a
MC		 2122 tcp_ctask->datapool = mempool_create_slab_pool(ISCSI_DTASK_DEFAULT_MAX,
2123 taskcache);
2124 if (tcp_ctask->datapool == NULL) {
2125 kfifo_free(tcp_ctask->r2tqueue);
2126 iscsi_pool_free(&tcp_ctask->r2tpool,
2127 (void**)tcp_ctask->r2ts);
7ba24713
AA		 2128 goto r2t_alloc_fail;
2129 }
5bb0b55a 2130 INIT_LIST_HEAD(&tcp_ctask->dataqueue);
7ba24713
AA		 2131 }
2132
2133 return 0;
2134
2135r2t_alloc_fail:
2136 for (i = 0; i < cmd_i; i++) {
5bb0b55a
MC		 2137 struct iscsi_cmd_task *ctask = session->cmds[i];
2138 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2139
2140 mempool_destroy(tcp_ctask->datapool);
2141 kfifo_free(tcp_ctask->r2tqueue);
2142 iscsi_pool_free(&tcp_ctask->r2tpool,
2143 (void**)tcp_ctask->r2ts);
7ba24713
AA		 2144 }
2145 return -ENOMEM;
2146}
2147
2148static void
2149iscsi_r2tpool_free(struct iscsi_session *session)
2150{
2151 int i;
2152
2153 for (i = 0; i < session->cmds_max; i++) {
5bb0b55a
MC		 2154 struct iscsi_cmd_task *ctask = session->cmds[i];
2155 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
7ba24713 2156
5bb0b55a
MC		 2157 mempool_destroy(tcp_ctask->datapool);
2158 kfifo_free(tcp_ctask->r2tqueue);
2159 iscsi_pool_free(&tcp_ctask->r2tpool,
2160 (void**)tcp_ctask->r2ts);
7ba24713 2161 }
7ba24713
AA		 2162}
2163
2164static int
7b7232f3 2165iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
7ba24713
AA		 2166 uint32_t value)
2167{
7b7232f3 2168 struct iscsi_conn *conn = cls_conn->dd_data;
7ba24713 2169 struct iscsi_session *session = conn->session;
5bb0b55a 2170 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
7ba24713 2171
7ba24713
AA		 2172 switch(param) {
2173 case ISCSI_PARAM_MAX_RECV_DLENGTH: {
5bb0b55a 2174 char *saveptr = tcp_conn->data;
b53cb2a4 2175 gfp_t flags = GFP_KERNEL;
7ba24713 2176
5bb0b55a 2177 if (tcp_conn->data_size >= value) {
7ba24713
AA		 2178 conn->max_recv_dlength = value;
2179 break;
2180 }
2181
2182 spin_lock_bh(&session->lock);
2183 if (conn->stop_stage == STOP_CONN_RECOVER)
2184 flags = GFP_ATOMIC;
2185 spin_unlock_bh(&session->lock);
2186
2187 if (value <= PAGE_SIZE)
5bb0b55a 2188 tcp_conn->data = kmalloc(value, flags);
7ba24713 2189 else
5bb0b55a 2190 tcp_conn->data = (void*)__get_free_pages(flags,
7ba24713 2191 get_order(value));
5bb0b55a
MC		 2192 if (tcp_conn->data == NULL) {
2193 tcp_conn->data = saveptr;
7ba24713
AA		 2194 return -ENOMEM;
2195 }
5bb0b55a 2196 if (tcp_conn->data_size <= PAGE_SIZE)
7ba24713
AA		 2197 kfree(saveptr);
2198 else
2199 free_pages((unsigned long)saveptr,
5bb0b55a 2200 get_order(tcp_conn->data_size));
7ba24713 2201 conn->max_recv_dlength = value;
5bb0b55a 2202 tcp_conn->data_size = value;
7ba24713
AA		 2203 }
2204 break;
2205 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2206 conn->max_xmit_dlength = value;
2207 break;
2208 case ISCSI_PARAM_HDRDGST_EN:
2209 conn->hdrdgst_en = value;
5bb0b55a 2210 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
7ba24713 2211 if (conn->hdrdgst_en) {
5bb0b55a
MC		 2212 tcp_conn->hdr_size += sizeof(__u32);
2213 if (!tcp_conn->tx_tfm)
2214 tcp_conn->tx_tfm = crypto_alloc_tfm("crc32c",
2215 0);
2216 if (!tcp_conn->tx_tfm)
7ba24713 2217 return -ENOMEM;
5bb0b55a
MC		 2218 if (!tcp_conn->rx_tfm)
2219 tcp_conn->rx_tfm = crypto_alloc_tfm("crc32c",
2220 0);
2221 if (!tcp_conn->rx_tfm) {
2222 crypto_free_tfm(tcp_conn->tx_tfm);
7ba24713
AA		 2223 return -ENOMEM;
2224 }
2225 } else {
5bb0b55a
MC		 2226 if (tcp_conn->tx_tfm)
2227 crypto_free_tfm(tcp_conn->tx_tfm);
2228 if (tcp_conn->rx_tfm)
2229 crypto_free_tfm(tcp_conn->rx_tfm);
7ba24713
AA		 2230 }
2231 break;
2232 case ISCSI_PARAM_DATADGST_EN:
2233 conn->datadgst_en = value;
2234 if (conn->datadgst_en) {
5bb0b55a
MC		 2235 if (!tcp_conn->data_tx_tfm)
2236 tcp_conn->data_tx_tfm =
7ba24713 2237 crypto_alloc_tfm("crc32c", 0);
5bb0b55a 2238 if (!tcp_conn->data_tx_tfm)
7ba24713 2239 return -ENOMEM;
5bb0b55a
MC		 2240 if (!tcp_conn->data_rx_tfm)
2241 tcp_conn->data_rx_tfm =
7ba24713 2242 crypto_alloc_tfm("crc32c", 0);
5bb0b55a
MC		 2243 if (!tcp_conn->data_rx_tfm) {
2244 crypto_free_tfm(tcp_conn->data_tx_tfm);
7ba24713
AA		 2245 return -ENOMEM;
2246 }
2247 } else {
5bb0b55a
MC		 2248 if (tcp_conn->data_tx_tfm)
2249 crypto_free_tfm(tcp_conn->data_tx_tfm);
2250 if (tcp_conn->data_rx_tfm)
2251 crypto_free_tfm(tcp_conn->data_rx_tfm);
7ba24713 2252 }
5bb0b55a
MC		 2253 tcp_conn->sendpage = conn->datadgst_en ?
2254 sock_no_sendpage : tcp_conn->sock->ops->sendpage;
7ba24713
AA		 2255 break;
2256 case ISCSI_PARAM_INITIAL_R2T_EN:
2257 session->initial_r2t_en = value;
2258 break;
2259 case ISCSI_PARAM_MAX_R2T:
2260 if (session->max_r2t == roundup_pow_of_two(value))
2261 break;
2262 iscsi_r2tpool_free(session);
2263 session->max_r2t = value;
2264 if (session->max_r2t & (session->max_r2t - 1))
2265 session->max_r2t = roundup_pow_of_two(session->max_r2t);
2266 if (iscsi_r2tpool_alloc(session))
2267 return -ENOMEM;
2268 break;
2269 case ISCSI_PARAM_IMM_DATA_EN:
2270 session->imm_data_en = value;
2271 break;
2272 case ISCSI_PARAM_FIRST_BURST:
2273 session->first_burst = value;
2274 break;
2275 case ISCSI_PARAM_MAX_BURST:
2276 session->max_burst = value;
2277 break;
2278 case ISCSI_PARAM_PDU_INORDER_EN:
2279 session->pdu_inorder_en = value;
2280 break;
2281 case ISCSI_PARAM_DATASEQ_INORDER_EN:
2282 session->dataseq_inorder_en = value;
2283 break;
2284 case ISCSI_PARAM_ERL:
2285 session->erl = value;
2286 break;
2287 case ISCSI_PARAM_IFMARKER_EN:
2288 BUG_ON(value);
2289 session->ifmarker_en = value;
2290 break;
2291 case ISCSI_PARAM_OFMARKER_EN:
2292 BUG_ON(value);
2293 session->ofmarker_en = value;
2294 break;
8d2860b3
MC		 2295 case ISCSI_PARAM_EXP_STATSN:
2296 conn->exp_statsn = value;
2297 break;
7ba24713
AA		 2298 default:
2299 break;
2300 }
2301
2302 return 0;
2303}
2304
2305static int
7b7232f3 2306iscsi_session_get_param(struct iscsi_cls_session *cls_session,
7b8631b5 2307 enum iscsi_param param, uint32_t *value)
7ba24713 2308{
7b7232f3 2309 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
7b8631b5 2310 struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
7ba24713
AA		 2311
2312 switch(param) {
7ba24713
AA		 2313 case ISCSI_PARAM_INITIAL_R2T_EN:
2314 *value = session->initial_r2t_en;
2315 break;
2316 case ISCSI_PARAM_MAX_R2T:
2317 *value = session->max_r2t;
2318 break;
2319 case ISCSI_PARAM_IMM_DATA_EN:
2320 *value = session->imm_data_en;
2321 break;
2322 case ISCSI_PARAM_FIRST_BURST:
2323 *value = session->first_burst;
2324 break;
2325 case ISCSI_PARAM_MAX_BURST:
2326 *value = session->max_burst;
2327 break;
2328 case ISCSI_PARAM_PDU_INORDER_EN:
2329 *value = session->pdu_inorder_en;
2330 break;
2331 case ISCSI_PARAM_DATASEQ_INORDER_EN:
2332 *value = session->dataseq_inorder_en;
2333 break;
2334 case ISCSI_PARAM_ERL:
2335 *value = session->erl;
2336 break;
2337 case ISCSI_PARAM_IFMARKER_EN:
2338 *value = session->ifmarker_en;
2339 break;
2340 case ISCSI_PARAM_OFMARKER_EN:
2341 *value = session->ofmarker_en;
2342 break;
2343 default:
fd7255f5 2344 return -EINVAL;
7ba24713
AA		 2345 }
2346
2347 return 0;
2348}
2349
7b8631b5 2350static int
7b7232f3
MC		 2351iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn,
2352 enum iscsi_param param, uint32_t *value)
7b8631b5 2353{
7b7232f3 2354 struct iscsi_conn *conn = cls_conn->dd_data;
5bb0b55a 2355 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
fd7255f5 2356 struct inet_sock *inet;
7b8631b5
MC		 2357
2358 switch(param) {
2359 case ISCSI_PARAM_MAX_RECV_DLENGTH:
2360 *value = conn->max_recv_dlength;
2361 break;
2362 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2363 *value = conn->max_xmit_dlength;
2364 break;
2365 case ISCSI_PARAM_HDRDGST_EN:
2366 *value = conn->hdrdgst_en;
2367 break;
2368 case ISCSI_PARAM_DATADGST_EN:
2369 *value = conn->datadgst_en;
2370 break;
fd7255f5
MC		 2371 case ISCSI_PARAM_CONN_PORT:
2372 mutex_lock(&conn->xmitmutex);
5bb0b55a 2373 if (!tcp_conn->sock) {
fd7255f5
MC		 2374 mutex_unlock(&conn->xmitmutex);
2375 return -EINVAL;
2376 }
2377
5bb0b55a 2378 inet = inet_sk(tcp_conn->sock->sk);
fd7255f5
MC		 2379 *value = be16_to_cpu(inet->dport);
2380 mutex_unlock(&conn->xmitmutex);
8d2860b3
MC		 2381 case ISCSI_PARAM_EXP_STATSN:
2382 *value = conn->exp_statsn;
2383 break;
7b8631b5 2384 default:
fd7255f5 2385 return -EINVAL;
7b8631b5
MC		 2386 }
2387
2388 return 0;
2389}
2390
fd7255f5
MC		 2391static int
2392iscsi_conn_get_str_param(struct iscsi_cls_conn *cls_conn,
2393 enum iscsi_param param, char *buf)
2394{
2395 struct iscsi_conn *conn = cls_conn->dd_data;
5bb0b55a 2396 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
fd7255f5
MC		 2397 struct sock *sk;
2398 struct inet_sock *inet;
2399 struct ipv6_pinfo *np;
2400 int len = 0;
2401
2402 switch (param) {
2403 case ISCSI_PARAM_CONN_ADDRESS:
2404 mutex_lock(&conn->xmitmutex);
5bb0b55a 2405 if (!tcp_conn->sock) {
fd7255f5
MC		 2406 mutex_unlock(&conn->xmitmutex);
2407 return -EINVAL;
2408 }
2409
5bb0b55a 2410 sk = tcp_conn->sock->sk;
fd7255f5
MC		 2411 if (sk->sk_family == PF_INET) {
2412 inet = inet_sk(sk);
2413 len = sprintf(buf, "%u.%u.%u.%u\n",
2414 NIPQUAD(inet->daddr));
2415 } else {
2416 np = inet6_sk(sk);
2417 len = sprintf(buf,
2418 "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
2419 NIP6(np->daddr));
2420 }
2421 mutex_unlock(&conn->xmitmutex);
2422 break;
2423 default:
2424 return -EINVAL;
2425 }
2426
2427 return len;
2428}
2429
7ba24713 2430static void
7b7232f3 2431iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
7ba24713 2432{
7b7232f3 2433 struct iscsi_conn *conn = cls_conn->dd_data;
5bb0b55a 2434 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
7ba24713
AA		 2435
2436 stats->txdata_octets = conn->txdata_octets;
2437 stats->rxdata_octets = conn->rxdata_octets;
2438 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
2439 stats->dataout_pdus = conn->dataout_pdus_cnt;
2440 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
2441 stats->datain_pdus = conn->datain_pdus_cnt;
2442 stats->r2t_pdus = conn->r2t_pdus_cnt;
2443 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
2444 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
2445 stats->custom_length = 3;
2446 strcpy(stats->custom[0].desc, "tx_sendpage_failures");
5bb0b55a 2447 stats->custom[0].value = tcp_conn->sendpage_failures_cnt;
7ba24713 2448 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
5bb0b55a 2449 stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt;
7ba24713
AA		 2450 strcpy(stats->custom[2].desc, "eh_abort_cnt");
2451 stats->custom[2].value = conn->eh_abort_cnt;
2452}
2453
5bb0b55a
MC		 2454static struct iscsi_cls_session *
2455iscsi_tcp_session_create(struct iscsi_transport *iscsit,
2456 struct scsi_transport_template *scsit,
2457 uint32_t initial_cmdsn, uint32_t *hostno)
7ba24713 2458{
5bb0b55a
MC		 2459 struct iscsi_cls_session *cls_session;
2460 struct iscsi_session *session;
2461 uint32_t hn;
2462 int cmd_i;
7ba24713 2463
5bb0b55a
MC		 2464 cls_session = iscsi_session_setup(iscsit, scsit,
2465 sizeof(struct iscsi_tcp_cmd_task),
2466 sizeof(struct iscsi_tcp_mgmt_task),
2467 initial_cmdsn, &hn);
2468 if (!cls_session)
2469 return NULL;
2470 *hostno = hn;
7ba24713 2471
5bb0b55a
MC		 2472 session = class_to_transport_session(cls_session);
2473 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
2474 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
2475 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2476
2477 ctask->hdr = &tcp_ctask->hdr;
2478 }
2479
2480 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
2481 struct iscsi_mgmt_task *mtask = session->mgmt_cmds[cmd_i];
2482 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
2483
2484 mtask->hdr = &tcp_mtask->hdr;
2485 }
2486
2487 if (iscsi_r2tpool_alloc(class_to_transport_session(cls_session)))
2488 goto r2tpool_alloc_fail;
2489
2490 return cls_session;
2491
2492r2tpool_alloc_fail:
2493 iscsi_session_teardown(cls_session);
2494 return NULL;
2495}
2496
2497static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session)
2498{
2499 struct iscsi_session *session = class_to_transport_session(cls_session);
2500 struct iscsi_data_task *dtask, *n;
2501 int cmd_i;
2502
2503 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
2504 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
2505 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2506
2507 list_for_each_entry_safe(dtask, n, &tcp_ctask->dataqueue,
2508 item) {
2509 list_del(&dtask->item);
2510 mempool_free(dtask, tcp_ctask->datapool);
2511 }
2512 }
2513
2514 iscsi_r2tpool_free(class_to_transport_session(cls_session));
2515 iscsi_session_teardown(cls_session);
7ba24713
AA		 2516}
2517
5bb0b55a
MC		 2518static struct scsi_host_template iscsi_sht = {
2519 .name = "iSCSI Initiator over TCP/IP, v."
2520 ISCSI_VERSION_STR,
2521 .queuecommand = iscsi_queuecommand,
2522 .change_queue_depth = iscsi_change_queue_depth,
2523 .can_queue = ISCSI_XMIT_CMDS_MAX - 1,
2524 .sg_tablesize = ISCSI_SG_TABLESIZE,
2525 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
2526 .eh_abort_handler = iscsi_eh_abort,
2527 .eh_host_reset_handler = iscsi_eh_host_reset,
2528 .use_clustering = DISABLE_CLUSTERING,
2529 .proc_name = "iscsi_tcp",
2530 .this_id = -1,
2531};
2532
7ba24713
AA		 2533static struct iscsi_transport iscsi_tcp_transport = {
2534 .owner = THIS_MODULE,
2535 .name = "tcp",
2536 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
2537 | CAP_DATADGST,
fd7255f5
MC		 2538 .param_mask = ISCSI_MAX_RECV_DLENGTH |
2539 ISCSI_MAX_XMIT_DLENGTH |
2540 ISCSI_HDRDGST_EN |
2541 ISCSI_DATADGST_EN |
2542 ISCSI_INITIAL_R2T_EN |
2543 ISCSI_MAX_R2T |
2544 ISCSI_IMM_DATA_EN |
2545 ISCSI_FIRST_BURST |
2546 ISCSI_MAX_BURST |
2547 ISCSI_PDU_INORDER_EN |
2548 ISCSI_DATASEQ_INORDER_EN |
2549 ISCSI_ERL |
2550 ISCSI_CONN_PORT |
8d2860b3
MC		 2551 ISCSI_CONN_ADDRESS |
2552 ISCSI_EXP_STATSN,
7ba24713 2553 .host_template = &iscsi_sht,
7b8631b5 2554 .conndata_size = sizeof(struct iscsi_conn),
7ba24713
AA		 2555 .max_conn = 1,
2556 .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN,
5bb0b55a
MC		 2557 /* session management */
2558 .create_session = iscsi_tcp_session_create,
2559 .destroy_session = iscsi_tcp_session_destroy,
2560 /* connection management */
2561 .create_conn = iscsi_tcp_conn_create,
2562 .bind_conn = iscsi_tcp_conn_bind,
2563 .destroy_conn = iscsi_tcp_conn_destroy,
7ba24713 2564 .set_param = iscsi_conn_set_param,
7b8631b5 2565 .get_conn_param = iscsi_conn_get_param,
fd7255f5 2566 .get_conn_str_param = iscsi_conn_get_str_param,
7b8631b5 2567 .get_session_param = iscsi_session_get_param,
7ba24713
AA		 2568 .start_conn = iscsi_conn_start,
2569 .stop_conn = iscsi_conn_stop,
5bb0b55a
MC		 2570 /* these are called as part of conn recovery */
2571 .suspend_conn_recv = iscsi_tcp_suspend_conn_rx,
2572 .terminate_conn = iscsi_tcp_terminate_conn,
2573 /* IO */
7ba24713
AA		 2574 .send_pdu = iscsi_conn_send_pdu,
2575 .get_stats = iscsi_conn_get_stats,
5bb0b55a
MC		 2576 .init_cmd_task = iscsi_tcp_cmd_init,
2577 .init_mgmt_task = iscsi_tcp_mgmt_init,
2578 .xmit_cmd_task = iscsi_tcp_ctask_xmit,
2579 .xmit_mgmt_task = iscsi_tcp_mtask_xmit,
2580 .cleanup_cmd_task = iscsi_tcp_cleanup_ctask,
2581 /* recovery */
30a6c652 2582 .session_recovery_timedout = iscsi_session_recovery_timedout,
7ba24713
AA		 2583};
2584
2585static int __init
2586iscsi_tcp_init(void)
2587{
7ba24713 2588 if (iscsi_max_lun < 1) {
be2df72e
OG		 2589 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
2590 iscsi_max_lun);
7ba24713
AA		 2591 return -EINVAL;
2592 }
2593 iscsi_tcp_transport.max_lun = iscsi_max_lun;
2594
2595 taskcache = kmem_cache_create("iscsi_taskcache",
2596 sizeof(struct iscsi_data_task), 0,
ac2b898c 2597 SLAB_HWCACHE_ALIGN, NULL, NULL);
7ba24713
AA		 2598 if (!taskcache)
2599 return -ENOMEM;
2600
7b8631b5 2601 if (!iscsi_register_transport(&iscsi_tcp_transport))
7ba24713
AA		 2602 kmem_cache_destroy(taskcache);
2603
7b8631b5 2604 return 0;
7ba24713
AA		 2605}
2606
2607static void __exit
2608iscsi_tcp_exit(void)
2609{
2610 iscsi_unregister_transport(&iscsi_tcp_transport);
2611 kmem_cache_destroy(taskcache);
2612}
2613
2614module_init(iscsi_tcp_init);
2615module_exit(iscsi_tcp_exit);
Linux kernel - Deliverables
This page took 0.260626 seconds and 5 git commands to generate.