projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branch 'asoc/topic/ac97' into asoc-fsl
[deliverable/linux.git] / drivers / block / aoe / aoecmd.c
1 /* Copyright (c) 2013 Coraid, Inc. See COPYING for GPL terms. */
2 /*
3 * aoecmd.c
4 * Filesystem request handling methods
5 */
6
7 #include <linux/ata.h>
8 #include <linux/slab.h>
9 #include <linux/hdreg.h>
10 #include <linux/blkdev.h>
11 #include <linux/skbuff.h>
12 #include <linux/netdevice.h>
13 #include <linux/genhd.h>
14 #include <linux/moduleparam.h>
15 #include <linux/workqueue.h>
16 #include <linux/kthread.h>
17 #include <net/net_namespace.h>
18 #include <asm/unaligned.h>
19 #include <linux/uio.h>
20 #include "aoe.h"
21
22 #define MAXIOC (8192) /* default meant to avoid most soft lockups */
23
24 static void ktcomplete(struct frame *, struct sk_buff *);
25 static int count_targets(struct aoedev *d, int *untainted);
26
27 static struct buf *nextbuf(struct aoedev *);
28
29 static int aoe_deadsecs = 60 * 3;
30 module_param(aoe_deadsecs, int, 0644);
31 MODULE_PARM_DESC(aoe_deadsecs, "After aoe_deadsecs seconds, give up and fail dev.");
32
33 static int aoe_maxout = 64;
34 module_param(aoe_maxout, int, 0644);
35 MODULE_PARM_DESC(aoe_maxout,
36 "Only aoe_maxout outstanding packets for every MAC on eX.Y.");
37
38 /* The number of online cpus during module initialization gives us a
39 * convenient heuristic cap on the parallelism used for ktio threads
40 * doing I/O completion. It is not important that the cap equal the
41 * actual number of running CPUs at any given time, but because of CPU
42 * hotplug, we take care to use ncpus instead of using
43 * num_online_cpus() after module initialization.
44 */
45 static int ncpus;
46
47 /* mutex lock used for synchronization while thread spawning */
48 static DEFINE_MUTEX(ktio_spawn_lock);
49
50 static wait_queue_head_t *ktiowq;
51 static struct ktstate *kts;
52
53 /* io completion queue */
54 struct iocq_ktio {
55 struct list_head head;
56 spinlock_t lock;
57 };
58 static struct iocq_ktio *iocq;
59
60 static struct page *empty_page;
61
62 static struct sk_buff *
63 new_skb(ulong len)
64 {
65 struct sk_buff *skb;
66
67 skb = alloc_skb(len + MAX_HEADER, GFP_ATOMIC);
68 if (skb) {
69 skb_reserve(skb, MAX_HEADER);
70 skb_reset_mac_header(skb);
71 skb_reset_network_header(skb);
72 skb->protocol = __constant_htons(ETH_P_AOE);
73 skb_checksum_none_assert(skb);
74 }
75 return skb;
76 }
77
78 static struct frame *
79 getframe_deferred(struct aoedev *d, u32 tag)
80 {
81 struct list_head *head, *pos, *nx;
82 struct frame *f;
83
84 head = &d->rexmitq;
85 list_for_each_safe(pos, nx, head) {
86 f = list_entry(pos, struct frame, head);
87 if (f->tag == tag) {
88 list_del(pos);
89 return f;
90 }
91 }
92 return NULL;
93 }
94
95 static struct frame *
96 getframe(struct aoedev *d, u32 tag)
97 {
98 struct frame *f;
99 struct list_head *head, *pos, *nx;
100 u32 n;
101
102 n = tag % NFACTIVE;
103 head = &d->factive[n];
104 list_for_each_safe(pos, nx, head) {
105 f = list_entry(pos, struct frame, head);
106 if (f->tag == tag) {
107 list_del(pos);
108 return f;
109 }
110 }
111 return NULL;
112 }
113
114 /*
115 * Leave the top bit clear so we have tagspace for userland.
116 * The bottom 16 bits are the xmit tick for rexmit/rttavg processing.
117 * This driver reserves tag -1 to mean "unused frame."
118 */
119 static int
120 newtag(struct aoedev *d)
121 {
122 register ulong n;
123
124 n = jiffies & 0xffff;
125 return n |= (++d->lasttag & 0x7fff) << 16;
126 }
127
128 static u32
129 aoehdr_atainit(struct aoedev *d, struct aoetgt *t, struct aoe_hdr *h)
130 {
131 u32 host_tag = newtag(d);
132
133 memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
134 memcpy(h->dst, t->addr, sizeof h->dst);
135 h->type = __constant_cpu_to_be16(ETH_P_AOE);
136 h->verfl = AOE_HVER;
137 h->major = cpu_to_be16(d->aoemajor);
138 h->minor = d->aoeminor;
139 h->cmd = AOECMD_ATA;
140 h->tag = cpu_to_be32(host_tag);
141
142 return host_tag;
143 }
144
145 static inline void
146 put_lba(struct aoe_atahdr *ah, sector_t lba)
147 {
148 ah->lba0 = lba;
149 ah->lba1 = lba >>= 8;
150 ah->lba2 = lba >>= 8;
151 ah->lba3 = lba >>= 8;
152 ah->lba4 = lba >>= 8;
153 ah->lba5 = lba >>= 8;
154 }
155
156 static struct aoeif *
157 ifrotate(struct aoetgt *t)
158 {
159 struct aoeif *ifp;
160
161 ifp = t->ifp;
162 ifp++;
163 if (ifp >= &t->ifs[NAOEIFS] || ifp->nd == NULL)
164 ifp = t->ifs;
165 if (ifp->nd == NULL)
166 return NULL;
167 return t->ifp = ifp;
168 }
169
170 static void
171 skb_pool_put(struct aoedev *d, struct sk_buff *skb)
172 {
173 __skb_queue_tail(&d->skbpool, skb);
174 }
175
176 static struct sk_buff *
177 skb_pool_get(struct aoedev *d)
178 {
179 struct sk_buff *skb = skb_peek(&d->skbpool);
180
181 if (skb && atomic_read(&skb_shinfo(skb)->dataref) == 1) {
182 __skb_unlink(skb, &d->skbpool);
183 return skb;
184 }
185 if (skb_queue_len(&d->skbpool) < NSKBPOOLMAX &&
186 (skb = new_skb(ETH_ZLEN)))
187 return skb;
188
189 return NULL;
190 }
191
192 void
193 aoe_freetframe(struct frame *f)
194 {
195 struct aoetgt *t;
196
197 t = f->t;
198 f->buf = NULL;
199 f->lba = 0;
200 f->bv = NULL;
201 f->r_skb = NULL;
202 f->flags = 0;
203 list_add(&f->head, &t->ffree);
204 }
205
206 static struct frame *
207 newtframe(struct aoedev *d, struct aoetgt *t)
208 {
209 struct frame *f;
210 struct sk_buff *skb;
211 struct list_head *pos;
212
213 if (list_empty(&t->ffree)) {
214 if (t->falloc >= NSKBPOOLMAX*2)
215 return NULL;
216 f = kcalloc(1, sizeof(*f), GFP_ATOMIC);
217 if (f == NULL)
218 return NULL;
219 t->falloc++;
220 f->t = t;
221 } else {
222 pos = t->ffree.next;
223 list_del(pos);
224 f = list_entry(pos, struct frame, head);
225 }
226
227 skb = f->skb;
228 if (skb == NULL) {
229 f->skb = skb = new_skb(ETH_ZLEN);
230 if (!skb) {
231 bail: aoe_freetframe(f);
232 return NULL;
233 }
234 }
235
236 if (atomic_read(&skb_shinfo(skb)->dataref) != 1) {
237 skb = skb_pool_get(d);
238 if (skb == NULL)
239 goto bail;
240 skb_pool_put(d, f->skb);
241 f->skb = skb;
242 }
243
244 skb->truesize -= skb->data_len;
245 skb_shinfo(skb)->nr_frags = skb->data_len = 0;
246 skb_trim(skb, 0);
247 return f;
248 }
249
250 static struct frame *
251 newframe(struct aoedev *d)
252 {
253 struct frame *f;
254 struct aoetgt *t, **tt;
255 int totout = 0;
256 int use_tainted;
257 int has_untainted;
258
259 if (!d->targets || !d->targets[0]) {
260 printk(KERN_ERR "aoe: NULL TARGETS!\n");
261 return NULL;
262 }
263 tt = d->tgt; /* last used target */
264 for (use_tainted = 0, has_untainted = 0;;) {
265 tt++;
266 if (tt >= &d->targets[d->ntargets] || !*tt)
267 tt = d->targets;
268 t = *tt;
269 if (!t->taint) {
270 has_untainted = 1;
271 totout += t->nout;
272 }
273 if (t->nout < t->maxout
274 && (use_tainted || !t->taint)
275 && t->ifp->nd) {
276 f = newtframe(d, t);
277 if (f) {
278 ifrotate(t);
279 d->tgt = tt;
280 return f;
281 }
282 }
283 if (tt == d->tgt) { /* we've looped and found nada */
284 if (!use_tainted && !has_untainted)
285 use_tainted = 1;
286 else
287 break;
288 }
289 }
290 if (totout == 0) {
291 d->kicked++;
292 d->flags |= DEVFL_KICKME;
293 }
294 return NULL;
295 }
296
297 static void
298 skb_fillup(struct sk_buff *skb, struct bio_vec *bv, ulong off, ulong cnt)
299 {
300 int frag = 0;
301 ulong fcnt;
302 loop:
303 fcnt = bv->bv_len - (off - bv->bv_offset);
304 if (fcnt > cnt)
305 fcnt = cnt;
306 skb_fill_page_desc(skb, frag++, bv->bv_page, off, fcnt);
307 cnt -= fcnt;
308 if (cnt <= 0)
309 return;
310 bv++;
311 off = bv->bv_offset;
312 goto loop;
313 }
314
315 static void
316 fhash(struct frame *f)
317 {
318 struct aoedev *d = f->t->d;
319 u32 n;
320
321 n = f->tag % NFACTIVE;
322 list_add_tail(&f->head, &d->factive[n]);
323 }
324
325 static void
326 ata_rw_frameinit(struct frame *f)
327 {
328 struct aoetgt *t;
329 struct aoe_hdr *h;
330 struct aoe_atahdr *ah;
331 struct sk_buff *skb;
332 char writebit, extbit;
333
334 skb = f->skb;
335 h = (struct aoe_hdr *) skb_mac_header(skb);
336 ah = (struct aoe_atahdr *) (h + 1);
337 skb_put(skb, sizeof(*h) + sizeof(*ah));
338 memset(h, 0, skb->len);
339
340 writebit = 0x10;
341 extbit = 0x4;
342
343 t = f->t;
344 f->tag = aoehdr_atainit(t->d, t, h);
345 fhash(f);
346 t->nout++;
347 f->waited = 0;
348 f->waited_total = 0;
349 if (f->buf)
350 f->lba = f->buf->sector;
351
352 /* set up ata header */
353 ah->scnt = f->bcnt >> 9;
354 put_lba(ah, f->lba);
355 if (t->d->flags & DEVFL_EXT) {
356 ah->aflags |= AOEAFL_EXT;
357 } else {
358 extbit = 0;
359 ah->lba3 &= 0x0f;
360 ah->lba3 |= 0xe0; /* LBA bit + obsolete 0xa0 */
361 }
362 if (f->buf && bio_data_dir(f->buf->bio) == WRITE) {
363 skb_fillup(skb, f->bv, f->bv_off, f->bcnt);
364 ah->aflags |= AOEAFL_WRITE;
365 skb->len += f->bcnt;
366 skb->data_len = f->bcnt;
367 skb->truesize += f->bcnt;
368 t->wpkts++;
369 } else {
370 t->rpkts++;
371 writebit = 0;
372 }
373
374 ah->cmdstat = ATA_CMD_PIO_READ | writebit | extbit;
375 skb->dev = t->ifp->nd;
376 }
377
378 static int
379 aoecmd_ata_rw(struct aoedev *d)
380 {
381 struct frame *f;
382 struct buf *buf;
383 struct aoetgt *t;
384 struct sk_buff *skb;
385 struct sk_buff_head queue;
386 ulong bcnt, fbcnt;
387
388 buf = nextbuf(d);
389 if (buf == NULL)
390 return 0;
391 f = newframe(d);
392 if (f == NULL)
393 return 0;
394 t = *d->tgt;
395 bcnt = d->maxbcnt;
396 if (bcnt == 0)
397 bcnt = DEFAULTBCNT;
398 if (bcnt > buf->resid)
399 bcnt = buf->resid;
400 fbcnt = bcnt;
401 f->bv = buf->bv;
402 f->bv_off = f->bv->bv_offset + (f->bv->bv_len - buf->bv_resid);
403 do {
404 if (fbcnt < buf->bv_resid) {
405 buf->bv_resid -= fbcnt;
406 buf->resid -= fbcnt;
407 break;
408 }
409 fbcnt -= buf->bv_resid;
410 buf->resid -= buf->bv_resid;
411 if (buf->resid == 0) {
412 d->ip.buf = NULL;
413 break;
414 }
415 buf->bv++;
416 buf->bv_resid = buf->bv->bv_len;
417 WARN_ON(buf->bv_resid == 0);
418 } while (fbcnt);
419
420 /* initialize the headers & frame */
421 f->buf = buf;
422 f->bcnt = bcnt;
423 ata_rw_frameinit(f);
424
425 /* mark all tracking fields and load out */
426 buf->nframesout += 1;
427 buf->sector += bcnt >> 9;
428
429 skb = skb_clone(f->skb, GFP_ATOMIC);
430 if (skb) {
431 do_gettimeofday(&f->sent);
432 f->sent_jiffs = (u32) jiffies;
433 __skb_queue_head_init(&queue);
434 __skb_queue_tail(&queue, skb);
435 aoenet_xmit(&queue);
436 }
437 return 1;
438 }
439
440 /* some callers cannot sleep, and they can call this function,
441 * transmitting the packets later, when interrupts are on
442 */
443 static void
444 aoecmd_cfg_pkts(ushort aoemajor, unsigned char aoeminor, struct sk_buff_head *queue)
445 {
446 struct aoe_hdr *h;
447 struct aoe_cfghdr *ch;
448 struct sk_buff *skb;
449 struct net_device *ifp;
450
451 rcu_read_lock();
452 for_each_netdev_rcu(&init_net, ifp) {
453 dev_hold(ifp);
454 if (!is_aoe_netif(ifp))
455 goto cont;
456
457 skb = new_skb(sizeof *h + sizeof *ch);
458 if (skb == NULL) {
459 printk(KERN_INFO "aoe: skb alloc failure\n");
460 goto cont;
461 }
462 skb_put(skb, sizeof *h + sizeof *ch);
463 skb->dev = ifp;
464 __skb_queue_tail(queue, skb);
465 h = (struct aoe_hdr *) skb_mac_header(skb);
466 memset(h, 0, sizeof *h + sizeof *ch);
467
468 memset(h->dst, 0xff, sizeof h->dst);
469 memcpy(h->src, ifp->dev_addr, sizeof h->src);
470 h->type = __constant_cpu_to_be16(ETH_P_AOE);
471 h->verfl = AOE_HVER;
472 h->major = cpu_to_be16(aoemajor);
473 h->minor = aoeminor;
474 h->cmd = AOECMD_CFG;
475
476 cont:
477 dev_put(ifp);
478 }
479 rcu_read_unlock();
480 }
481
482 static void
483 resend(struct aoedev *d, struct frame *f)
484 {
485 struct sk_buff *skb;
486 struct sk_buff_head queue;
487 struct aoe_hdr *h;
488 struct aoe_atahdr *ah;
489 struct aoetgt *t;
490 char buf[128];
491 u32 n;
492
493 t = f->t;
494 n = newtag(d);
495 skb = f->skb;
496 if (ifrotate(t) == NULL) {
497 /* probably can't happen, but set it up to fail anyway */
498 pr_info("aoe: resend: no interfaces to rotate to.\n");
499 ktcomplete(f, NULL);
500 return;
501 }
502 h = (struct aoe_hdr *) skb_mac_header(skb);
503 ah = (struct aoe_atahdr *) (h+1);
504
505 if (!(f->flags & FFL_PROBE)) {
506 snprintf(buf, sizeof(buf),
507 "%15s e%ld.%d oldtag=%08x@%08lx newtag=%08x s=%pm d=%pm nout=%d\n",
508 "retransmit", d->aoemajor, d->aoeminor,
509 f->tag, jiffies, n,
510 h->src, h->dst, t->nout);
511 aoechr_error(buf);
512 }
513
514 f->tag = n;
515 fhash(f);
516 h->tag = cpu_to_be32(n);
517 memcpy(h->dst, t->addr, sizeof h->dst);
518 memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
519
520 skb->dev = t->ifp->nd;
521 skb = skb_clone(skb, GFP_ATOMIC);
522 if (skb == NULL)
523 return;
524 do_gettimeofday(&f->sent);
525 f->sent_jiffs = (u32) jiffies;
526 __skb_queue_head_init(&queue);
527 __skb_queue_tail(&queue, skb);
528 aoenet_xmit(&queue);
529 }
530
531 static int
532 tsince_hr(struct frame *f)
533 {
534 struct timeval now;
535 int n;
536
537 do_gettimeofday(&now);
538 n = now.tv_usec - f->sent.tv_usec;
539 n += (now.tv_sec - f->sent.tv_sec) * USEC_PER_SEC;
540
541 if (n < 0)
542 n = -n;
543
544 /* For relatively long periods, use jiffies to avoid
545 * discrepancies caused by updates to the system time.
546 *
547 * On system with HZ of 1000, 32-bits is over 49 days
548 * worth of jiffies, or over 71 minutes worth of usecs.
549 *
550 * Jiffies overflow is handled by subtraction of unsigned ints:
551 * (gdb) print (unsigned) 2 - (unsigned) 0xfffffffe
552 * $3 = 4
553 * (gdb)
554 */
555 if (n > USEC_PER_SEC / 4) {
556 n = ((u32) jiffies) - f->sent_jiffs;
557 n *= USEC_PER_SEC / HZ;
558 }
559
560 return n;
561 }
562
563 static int
564 tsince(u32 tag)
565 {
566 int n;
567
568 n = jiffies & 0xffff;
569 n -= tag & 0xffff;
570 if (n < 0)
571 n += 1<<16;
572 return jiffies_to_usecs(n + 1);
573 }
574
575 static struct aoeif *
576 getif(struct aoetgt *t, struct net_device *nd)
577 {
578 struct aoeif *p, *e;
579
580 p = t->ifs;
581 e = p + NAOEIFS;
582 for (; p < e; p++)
583 if (p->nd == nd)
584 return p;
585 return NULL;
586 }
587
588 static void
589 ejectif(struct aoetgt *t, struct aoeif *ifp)
590 {
591 struct aoeif *e;
592 struct net_device *nd;
593 ulong n;
594
595 nd = ifp->nd;
596 e = t->ifs + NAOEIFS - 1;
597 n = (e - ifp) * sizeof *ifp;
598 memmove(ifp, ifp+1, n);
599 e->nd = NULL;
600 dev_put(nd);
601 }
602
603 static struct frame *
604 reassign_frame(struct frame *f)
605 {
606 struct frame *nf;
607 struct sk_buff *skb;
608
609 nf = newframe(f->t->d);
610 if (!nf)
611 return NULL;
612 if (nf->t == f->t) {
613 aoe_freetframe(nf);
614 return NULL;
615 }
616
617 skb = nf->skb;
618 nf->skb = f->skb;
619 nf->buf = f->buf;
620 nf->bcnt = f->bcnt;
621 nf->lba = f->lba;
622 nf->bv = f->bv;
623 nf->bv_off = f->bv_off;
624 nf->waited = 0;
625 nf->waited_total = f->waited_total;
626 nf->sent = f->sent;
627 nf->sent_jiffs = f->sent_jiffs;
628 f->skb = skb;
629
630 return nf;
631 }
632
633 static void
634 probe(struct aoetgt *t)
635 {
636 struct aoedev *d;
637 struct frame *f;
638 struct sk_buff *skb;
639 struct sk_buff_head queue;
640 size_t n, m;
641 int frag;
642
643 d = t->d;
644 f = newtframe(d, t);
645 if (!f) {
646 pr_err("%s %pm for e%ld.%d: %s\n",
647 "aoe: cannot probe remote address",
648 t->addr,
649 (long) d->aoemajor, d->aoeminor,
650 "no frame available");
651 return;
652 }
653 f->flags |= FFL_PROBE;
654 ifrotate(t);
655 f->bcnt = t->d->maxbcnt ? t->d->maxbcnt : DEFAULTBCNT;
656 ata_rw_frameinit(f);
657 skb = f->skb;
658 for (frag = 0, n = f->bcnt; n > 0; ++frag, n -= m) {
659 if (n < PAGE_SIZE)
660 m = n;
661 else
662 m = PAGE_SIZE;
663 skb_fill_page_desc(skb, frag, empty_page, 0, m);
664 }
665 skb->len += f->bcnt;
666 skb->data_len = f->bcnt;
667 skb->truesize += f->bcnt;
668
669 skb = skb_clone(f->skb, GFP_ATOMIC);
670 if (skb) {
671 do_gettimeofday(&f->sent);
672 f->sent_jiffs = (u32) jiffies;
673 __skb_queue_head_init(&queue);
674 __skb_queue_tail(&queue, skb);
675 aoenet_xmit(&queue);
676 }
677 }
678
679 static long
680 rto(struct aoedev *d)
681 {
682 long t;
683
684 t = 2 * d->rttavg >> RTTSCALE;
685 t += 8 * d->rttdev >> RTTDSCALE;
686 if (t == 0)
687 t = 1;
688
689 return t;
690 }
691
692 static void
693 rexmit_deferred(struct aoedev *d)
694 {
695 struct aoetgt *t;
696 struct frame *f;
697 struct frame *nf;
698 struct list_head *pos, *nx, *head;
699 int since;
700 int untainted;
701
702 count_targets(d, &untainted);
703
704 head = &d->rexmitq;
705 list_for_each_safe(pos, nx, head) {
706 f = list_entry(pos, struct frame, head);
707 t = f->t;
708 if (t->taint) {
709 if (!(f->flags & FFL_PROBE)) {
710 nf = reassign_frame(f);
711 if (nf) {
712 if (t->nout_probes == 0
713 && untainted > 0) {
714 probe(t);
715 t->nout_probes++;
716 }
717 list_replace(&f->head, &nf->head);
718 pos = &nf->head;
719 aoe_freetframe(f);
720 f = nf;
721 t = f->t;
722 }
723 } else if (untainted < 1) {
724 /* don't probe w/o other untainted aoetgts */
725 goto stop_probe;
726 } else if (tsince_hr(f) < t->taint * rto(d)) {
727 /* reprobe slowly when taint is high */
728 continue;
729 }
730 } else if (f->flags & FFL_PROBE) {
731 stop_probe: /* don't probe untainted aoetgts */
732 list_del(pos);
733 aoe_freetframe(f);
734 /* leaving d->kicked, because this is routine */
735 f->t->d->flags |= DEVFL_KICKME;
736 continue;
737 }
738 if (t->nout >= t->maxout)
739 continue;
740 list_del(pos);
741 t->nout++;
742 if (f->flags & FFL_PROBE)
743 t->nout_probes++;
744 since = tsince_hr(f);
745 f->waited += since;
746 f->waited_total += since;
747 resend(d, f);
748 }
749 }
750
751 /* An aoetgt accumulates demerits quickly, and successful
752 * probing redeems the aoetgt slowly.
753 */
754 static void
755 scorn(struct aoetgt *t)
756 {
757 int n;
758
759 n = t->taint++;
760 t->taint += t->taint * 2;
761 if (n > t->taint)
762 t->taint = n;
763 if (t->taint > MAX_TAINT)
764 t->taint = MAX_TAINT;
765 }
766
767 static int
768 count_targets(struct aoedev *d, int *untainted)
769 {
770 int i, good;
771
772 for (i = good = 0; i < d->ntargets && d->targets[i]; ++i)
773 if (d->targets[i]->taint == 0)
774 good++;
775
776 if (untainted)
777 *untainted = good;
778 return i;
779 }
780
781 static void
782 rexmit_timer(ulong vp)
783 {
784 struct aoedev *d;
785 struct aoetgt *t;
786 struct aoeif *ifp;
787 struct frame *f;
788 struct list_head *head, *pos, *nx;
789 LIST_HEAD(flist);
790 register long timeout;
791 ulong flags, n;
792 int i;
793 int utgts; /* number of aoetgt descriptors (not slots) */
794 int since;
795
796 d = (struct aoedev *) vp;
797
798 spin_lock_irqsave(&d->lock, flags);
799
800 /* timeout based on observed timings and variations */
801 timeout = rto(d);
802
803 utgts = count_targets(d, NULL);
804
805 if (d->flags & DEVFL_TKILL) {
806 spin_unlock_irqrestore(&d->lock, flags);
807 return;
808 }
809
810 /* collect all frames to rexmit into flist */
811 for (i = 0; i < NFACTIVE; i++) {
812 head = &d->factive[i];
813 list_for_each_safe(pos, nx, head) {
814 f = list_entry(pos, struct frame, head);
815 if (tsince_hr(f) < timeout)
816 break; /* end of expired frames */
817 /* move to flist for later processing */
818 list_move_tail(pos, &flist);
819 }
820 }
821
822 /* process expired frames */
823 while (!list_empty(&flist)) {
824 pos = flist.next;
825 f = list_entry(pos, struct frame, head);
826 since = tsince_hr(f);
827 n = f->waited_total + since;
828 n /= USEC_PER_SEC;
829 if (aoe_deadsecs
830 && n > aoe_deadsecs
831 && !(f->flags & FFL_PROBE)) {
832 /* Waited too long. Device failure.
833 * Hang all frames on first hash bucket for downdev
834 * to clean up.
835 */
836 list_splice(&flist, &d->factive[0]);
837 aoedev_downdev(d);
838 goto out;
839 }
840
841 t = f->t;
842 n = f->waited + since;
843 n /= USEC_PER_SEC;
844 if (aoe_deadsecs && utgts > 0
845 && (n > aoe_deadsecs / utgts || n > HARD_SCORN_SECS))
846 scorn(t); /* avoid this target */
847
848 if (t->maxout != 1) {
849 t->ssthresh = t->maxout / 2;
850 t->maxout = 1;
851 }
852
853 if (f->flags & FFL_PROBE) {
854 t->nout_probes--;
855 } else {
856 ifp = getif(t, f->skb->dev);
857 if (ifp && ++ifp->lost > (t->nframes << 1)
858 && (ifp != t->ifs || t->ifs[1].nd)) {
859 ejectif(t, ifp);
860 ifp = NULL;
861 }
862 }
863 list_move_tail(pos, &d->rexmitq);
864 t->nout--;
865 }
866 rexmit_deferred(d);
867
868 out:
869 if ((d->flags & DEVFL_KICKME) && d->blkq) {
870 d->flags &= ~DEVFL_KICKME;
871 d->blkq->request_fn(d->blkq);
872 }
873
874 d->timer.expires = jiffies + TIMERTICK;
875 add_timer(&d->timer);
876
877 spin_unlock_irqrestore(&d->lock, flags);
878 }
879
880 static unsigned long
881 rqbiocnt(struct request *r)
882 {
883 struct bio *bio;
884 unsigned long n = 0;
885
886 __rq_for_each_bio(bio, r)
887 n++;
888 return n;
889 }
890
891 /* This can be removed if we are certain that no users of the block
892 * layer will ever use zero-count pages in bios. Otherwise we have to
893 * protect against the put_page sometimes done by the network layer.
894 *
895 * See http://oss.sgi.com/archives/xfs/2007-01/msg00594.html for
896 * discussion.
897 *
898 * We cannot use get_page in the workaround, because it insists on a
899 * positive page count as a precondition. So we use _count directly.
900 */
901 static void
902 bio_pageinc(struct bio *bio)
903 {
904 struct bio_vec *bv;
905 struct page *page;
906 int i;
907
908 bio_for_each_segment(bv, bio, i) {
909 /* Non-zero page count for non-head members of
910 * compound pages is no longer allowed by the kernel.
911 */
912 page = compound_trans_head(bv->bv_page);
913 atomic_inc(&page->_count);
914 }
915 }
916
917 static void
918 bio_pagedec(struct bio *bio)
919 {
920 struct bio_vec *bv;
921 struct page *page;
922 int i;
923
924 bio_for_each_segment(bv, bio, i) {
925 page = compound_trans_head(bv->bv_page);
926 atomic_dec(&page->_count);
927 }
928 }
929
930 static void
931 bufinit(struct buf *buf, struct request *rq, struct bio *bio)
932 {
933 memset(buf, 0, sizeof(*buf));
934 buf->rq = rq;
935 buf->bio = bio;
936 buf->resid = bio->bi_size;
937 buf->sector = bio->bi_sector;
938 bio_pageinc(bio);
939 buf->bv = bio_iovec(bio);
940 buf->bv_resid = buf->bv->bv_len;
941 WARN_ON(buf->bv_resid == 0);
942 }
943
944 static struct buf *
945 nextbuf(struct aoedev *d)
946 {
947 struct request *rq;
948 struct request_queue *q;
949 struct buf *buf;
950 struct bio *bio;
951
952 q = d->blkq;
953 if (q == NULL)
954 return NULL; /* initializing */
955 if (d->ip.buf)
956 return d->ip.buf;
957 rq = d->ip.rq;
958 if (rq == NULL) {
959 rq = blk_peek_request(q);
960 if (rq == NULL)
961 return NULL;
962 blk_start_request(rq);
963 d->ip.rq = rq;
964 d->ip.nxbio = rq->bio;
965 rq->special = (void *) rqbiocnt(rq);
966 }
967 buf = mempool_alloc(d->bufpool, GFP_ATOMIC);
968 if (buf == NULL) {
969 pr_err("aoe: nextbuf: unable to mempool_alloc!\n");
970 return NULL;
971 }
972 bio = d->ip.nxbio;
973 bufinit(buf, rq, bio);
974 bio = bio->bi_next;
975 d->ip.nxbio = bio;
976 if (bio == NULL)
977 d->ip.rq = NULL;
978 return d->ip.buf = buf;
979 }
980
981 /* enters with d->lock held */
982 void
983 aoecmd_work(struct aoedev *d)
984 {
985 rexmit_deferred(d);
986 while (aoecmd_ata_rw(d))
987 ;
988 }
989
990 /* this function performs work that has been deferred until sleeping is OK
991 */
992 void
993 aoecmd_sleepwork(struct work_struct *work)
994 {
995 struct aoedev *d = container_of(work, struct aoedev, work);
996 struct block_device *bd;
997 u64 ssize;
998
999 if (d->flags & DEVFL_GDALLOC)
1000 aoeblk_gdalloc(d);
1001
1002 if (d->flags & DEVFL_NEWSIZE) {
1003 ssize = get_capacity(d->gd);
1004 bd = bdget_disk(d->gd, 0);
1005 if (bd) {
1006 mutex_lock(&bd->bd_inode->i_mutex);
1007 i_size_write(bd->bd_inode, (loff_t)ssize<<9);
1008 mutex_unlock(&bd->bd_inode->i_mutex);
1009 bdput(bd);
1010 }
1011 spin_lock_irq(&d->lock);
1012 d->flags |= DEVFL_UP;
1013 d->flags &= ~DEVFL_NEWSIZE;
1014 spin_unlock_irq(&d->lock);
1015 }
1016 }
1017
1018 static void
1019 ata_ident_fixstring(u16 *id, int ns)
1020 {
1021 u16 s;
1022
1023 while (ns-- > 0) {
1024 s = *id;
1025 *id++ = s >> 8 | s << 8;
1026 }
1027 }
1028
1029 static void
1030 ataid_complete(struct aoedev *d, struct aoetgt *t, unsigned char *id)
1031 {
1032 u64 ssize;
1033 u16 n;
1034
1035 /* word 83: command set supported */
1036 n = get_unaligned_le16(&id[83 << 1]);
1037
1038 /* word 86: command set/feature enabled */
1039 n |= get_unaligned_le16(&id[86 << 1]);
1040
1041 if (n & (1<<10)) { /* bit 10: LBA 48 */
1042 d->flags |= DEVFL_EXT;
1043
1044 /* word 100: number lba48 sectors */
1045 ssize = get_unaligned_le64(&id[100 << 1]);
1046
1047 /* set as in ide-disk.c:init_idedisk_capacity */
1048 d->geo.cylinders = ssize;
1049 d->geo.cylinders /= (255 * 63);
1050 d->geo.heads = 255;
1051 d->geo.sectors = 63;
1052 } else {
1053 d->flags &= ~DEVFL_EXT;
1054
1055 /* number lba28 sectors */
1056 ssize = get_unaligned_le32(&id[60 << 1]);
1057
1058 /* NOTE: obsolete in ATA 6 */
1059 d->geo.cylinders = get_unaligned_le16(&id[54 << 1]);
1060 d->geo.heads = get_unaligned_le16(&id[55 << 1]);
1061 d->geo.sectors = get_unaligned_le16(&id[56 << 1]);
1062 }
1063
1064 ata_ident_fixstring((u16 *) &id[10<<1], 10); /* serial */
1065 ata_ident_fixstring((u16 *) &id[23<<1], 4); /* firmware */
1066 ata_ident_fixstring((u16 *) &id[27<<1], 20); /* model */
1067 memcpy(d->ident, id, sizeof(d->ident));
1068
1069 if (d->ssize != ssize)
1070 printk(KERN_INFO
1071 "aoe: %pm e%ld.%d v%04x has %llu sectors\n",
1072 t->addr,
1073 d->aoemajor, d->aoeminor,
1074 d->fw_ver, (long long)ssize);
1075 d->ssize = ssize;
1076 d->geo.start = 0;
1077 if (d->flags & (DEVFL_GDALLOC|DEVFL_NEWSIZE))
1078 return;
1079 if (d->gd != NULL) {
1080 set_capacity(d->gd, ssize);
1081 d->flags |= DEVFL_NEWSIZE;
1082 } else
1083 d->flags |= DEVFL_GDALLOC;
1084 schedule_work(&d->work);
1085 }
1086
1087 static void
1088 calc_rttavg(struct aoedev *d, struct aoetgt *t, int rtt)
1089 {
1090 register long n;
1091
1092 n = rtt;
1093
1094 /* cf. Congestion Avoidance and Control, Jacobson & Karels, 1988 */
1095 n -= d->rttavg >> RTTSCALE;
1096 d->rttavg += n;
1097 if (n < 0)
1098 n = -n;
1099 n -= d->rttdev >> RTTDSCALE;
1100 d->rttdev += n;
1101
1102 if (!t || t->maxout >= t->nframes)
1103 return;
1104 if (t->maxout < t->ssthresh)
1105 t->maxout += 1;
1106 else if (t->nout == t->maxout && t->next_cwnd-- == 0) {
1107 t->maxout += 1;
1108 t->next_cwnd = t->maxout;
1109 }
1110 }
1111
1112 static struct aoetgt *
1113 gettgt(struct aoedev *d, char *addr)
1114 {
1115 struct aoetgt **t, **e;
1116
1117 t = d->targets;
1118 e = t + d->ntargets;
1119 for (; t < e && *t; t++)
1120 if (memcmp((*t)->addr, addr, sizeof((*t)->addr)) == 0)
1121 return *t;
1122 return NULL;
1123 }
1124
1125 static void
1126 bvcpy(struct bio_vec *bv, ulong off, struct sk_buff *skb, long cnt)
1127 {
1128 ulong fcnt;
1129 char *p;
1130 int soff = 0;
1131 loop:
1132 fcnt = bv->bv_len - (off - bv->bv_offset);
1133 if (fcnt > cnt)
1134 fcnt = cnt;
1135 p = page_address(bv->bv_page) + off;
1136 skb_copy_bits(skb, soff, p, fcnt);
1137 soff += fcnt;
1138 cnt -= fcnt;
1139 if (cnt <= 0)
1140 return;
1141 bv++;
1142 off = bv->bv_offset;
1143 goto loop;
1144 }
1145
1146 void
1147 aoe_end_request(struct aoedev *d, struct request *rq, int fastfail)
1148 {
1149 struct bio *bio;
1150 int bok;
1151 struct request_queue *q;
1152
1153 q = d->blkq;
1154 if (rq == d->ip.rq)
1155 d->ip.rq = NULL;
1156 do {
1157 bio = rq->bio;
1158 bok = !fastfail && test_bit(BIO_UPTODATE, &bio->bi_flags);
1159 } while (__blk_end_request(rq, bok ? 0 : -EIO, bio->bi_size));
1160
1161 /* cf. http://lkml.org/lkml/2006/10/31/28 */
1162 if (!fastfail)
1163 __blk_run_queue(q);
1164 }
1165
1166 static void
1167 aoe_end_buf(struct aoedev *d, struct buf *buf)
1168 {
1169 struct request *rq;
1170 unsigned long n;
1171
1172 if (buf == d->ip.buf)
1173 d->ip.buf = NULL;
1174 rq = buf->rq;
1175 bio_pagedec(buf->bio);
1176 mempool_free(buf, d->bufpool);
1177 n = (unsigned long) rq->special;
1178 rq->special = (void *) --n;
1179 if (n == 0)
1180 aoe_end_request(d, rq, 0);
1181 }
1182
1183 static void
1184 ktiocomplete(struct frame *f)
1185 {
1186 struct aoe_hdr *hin, *hout;
1187 struct aoe_atahdr *ahin, *ahout;
1188 struct buf *buf;
1189 struct sk_buff *skb;
1190 struct aoetgt *t;
1191 struct aoeif *ifp;
1192 struct aoedev *d;
1193 long n;
1194 int untainted;
1195
1196 if (f == NULL)
1197 return;
1198
1199 t = f->t;
1200 d = t->d;
1201 skb = f->r_skb;
1202 buf = f->buf;
1203 if (f->flags & FFL_PROBE)
1204 goto out;
1205 if (!skb) /* just fail the buf. */
1206 goto noskb;
1207
1208 hout = (struct aoe_hdr *) skb_mac_header(f->skb);
1209 ahout = (struct aoe_atahdr *) (hout+1);
1210
1211 hin = (struct aoe_hdr *) skb->data;
1212 skb_pull(skb, sizeof(*hin));
1213 ahin = (struct aoe_atahdr *) skb->data;
1214 skb_pull(skb, sizeof(*ahin));
1215 if (ahin->cmdstat & 0xa9) { /* these bits cleared on success */
1216 pr_err("aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n",
1217 ahout->cmdstat, ahin->cmdstat,
1218 d->aoemajor, d->aoeminor);
1219 noskb: if (buf)
1220 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags);
1221 goto out;
1222 }
1223
1224 n = ahout->scnt << 9;
1225 switch (ahout->cmdstat) {
1226 case ATA_CMD_PIO_READ:
1227 case ATA_CMD_PIO_READ_EXT:
1228 if (skb->len < n) {
1229 pr_err("%s e%ld.%d. skb->len=%d need=%ld\n",
1230 "aoe: runt data size in read from",
1231 (long) d->aoemajor, d->aoeminor,
1232 skb->len, n);
1233 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags);
1234 break;
1235 }
1236 bvcpy(f->bv, f->bv_off, skb, n);
1237 case ATA_CMD_PIO_WRITE:
1238 case ATA_CMD_PIO_WRITE_EXT:
1239 spin_lock_irq(&d->lock);
1240 ifp = getif(t, skb->dev);
1241 if (ifp)
1242 ifp->lost = 0;
1243 spin_unlock_irq(&d->lock);
1244 break;
1245 case ATA_CMD_ID_ATA:
1246 if (skb->len < 512) {
1247 pr_info("%s e%ld.%d. skb->len=%d need=512\n",
1248 "aoe: runt data size in ataid from",
1249 (long) d->aoemajor, d->aoeminor,
1250 skb->len);
1251 break;
1252 }
1253 if (skb_linearize(skb))
1254 break;
1255 spin_lock_irq(&d->lock);
1256 ataid_complete(d, t, skb->data);
1257 spin_unlock_irq(&d->lock);
1258 break;
1259 default:
1260 pr_info("aoe: unrecognized ata command %2.2Xh for %d.%d\n",
1261 ahout->cmdstat,
1262 be16_to_cpu(get_unaligned(&hin->major)),
1263 hin->minor);
1264 }
1265 out:
1266 spin_lock_irq(&d->lock);
1267 if (t->taint > 0
1268 && --t->taint > 0
1269 && t->nout_probes == 0) {
1270 count_targets(d, &untainted);
1271 if (untainted > 0) {
1272 probe(t);
1273 t->nout_probes++;
1274 }
1275 }
1276
1277 aoe_freetframe(f);
1278
1279 if (buf && --buf->nframesout == 0 && buf->resid == 0)
1280 aoe_end_buf(d, buf);
1281
1282 spin_unlock_irq(&d->lock);
1283 aoedev_put(d);
1284 dev_kfree_skb(skb);
1285 }
1286
1287 /* Enters with iocq.lock held.
1288 * Returns true iff responses needing processing remain.
1289 */
1290 static int
1291 ktio(int id)
1292 {
1293 struct frame *f;
1294 struct list_head *pos;
1295 int i;
1296 int actual_id;
1297
1298 for (i = 0; ; ++i) {
1299 if (i == MAXIOC)
1300 return 1;
1301 if (list_empty(&iocq[id].head))
1302 return 0;
1303 pos = iocq[id].head.next;
1304 list_del(pos);
1305 f = list_entry(pos, struct frame, head);
1306 spin_unlock_irq(&iocq[id].lock);
1307 ktiocomplete(f);
1308
1309 /* Figure out if extra threads are required. */
1310 actual_id = f->t->d->aoeminor % ncpus;
1311
1312 if (!kts[actual_id].active) {
1313 BUG_ON(id != 0);
1314 mutex_lock(&ktio_spawn_lock);
1315 if (!kts[actual_id].active
1316 && aoe_ktstart(&kts[actual_id]) == 0)
1317 kts[actual_id].active = 1;
1318 mutex_unlock(&ktio_spawn_lock);
1319 }
1320 spin_lock_irq(&iocq[id].lock);
1321 }
1322 }
1323
1324 static int
1325 kthread(void *vp)
1326 {
1327 struct ktstate *k;
1328 DECLARE_WAITQUEUE(wait, current);
1329 int more;
1330
1331 k = vp;
1332 current->flags |= PF_NOFREEZE;
1333 set_user_nice(current, -10);
1334 complete(&k->rendez); /* tell spawner we're running */
1335 do {
1336 spin_lock_irq(k->lock);
1337 more = k->fn(k->id);
1338 if (!more) {
1339 add_wait_queue(k->waitq, &wait);
1340 __set_current_state(TASK_INTERRUPTIBLE);
1341 }
1342 spin_unlock_irq(k->lock);
1343 if (!more) {
1344 schedule();
1345 remove_wait_queue(k->waitq, &wait);
1346 } else
1347 cond_resched();
1348 } while (!kthread_should_stop());
1349 complete(&k->rendez); /* tell spawner we're stopping */
1350 return 0;
1351 }
1352
1353 void
1354 aoe_ktstop(struct ktstate *k)
1355 {
1356 kthread_stop(k->task);
1357 wait_for_completion(&k->rendez);
1358 }
1359
1360 int
1361 aoe_ktstart(struct ktstate *k)
1362 {
1363 struct task_struct *task;
1364
1365 init_completion(&k->rendez);
1366 task = kthread_run(kthread, k, "%s", k->name);
1367 if (task == NULL || IS_ERR(task))
1368 return -ENOMEM;
1369 k->task = task;
1370 wait_for_completion(&k->rendez); /* allow kthread to start */
1371 init_completion(&k->rendez); /* for waiting for exit later */
1372 return 0;
1373 }
1374
1375 /* pass it off to kthreads for processing */
1376 static void
1377 ktcomplete(struct frame *f, struct sk_buff *skb)
1378 {
1379 int id;
1380 ulong flags;
1381
1382 f->r_skb = skb;
1383 id = f->t->d->aoeminor % ncpus;
1384 spin_lock_irqsave(&iocq[id].lock, flags);
1385 if (!kts[id].active) {
1386 spin_unlock_irqrestore(&iocq[id].lock, flags);
1387 /* The thread with id has not been spawned yet,
1388 * so delegate the work to the main thread and
1389 * try spawning a new thread.
1390 */
1391 id = 0;
1392 spin_lock_irqsave(&iocq[id].lock, flags);
1393 }
1394 list_add_tail(&f->head, &iocq[id].head);
1395 spin_unlock_irqrestore(&iocq[id].lock, flags);
1396 wake_up(&ktiowq[id]);
1397 }
1398
1399 struct sk_buff *
1400 aoecmd_ata_rsp(struct sk_buff *skb)
1401 {
1402 struct aoedev *d;
1403 struct aoe_hdr *h;
1404 struct frame *f;
1405 u32 n;
1406 ulong flags;
1407 char ebuf[128];
1408 u16 aoemajor;
1409
1410 h = (struct aoe_hdr *) skb->data;
1411 aoemajor = be16_to_cpu(get_unaligned(&h->major));
1412 d = aoedev_by_aoeaddr(aoemajor, h->minor, 0);
1413 if (d == NULL) {
1414 snprintf(ebuf, sizeof ebuf, "aoecmd_ata_rsp: ata response "
1415 "for unknown device %d.%d\n",
1416 aoemajor, h->minor);
1417 aoechr_error(ebuf);
1418 return skb;
1419 }
1420
1421 spin_lock_irqsave(&d->lock, flags);
1422
1423 n = be32_to_cpu(get_unaligned(&h->tag));
1424 f = getframe(d, n);
1425 if (f) {
1426 calc_rttavg(d, f->t, tsince_hr(f));
1427 f->t->nout--;
1428 if (f->flags & FFL_PROBE)
1429 f->t->nout_probes--;
1430 } else {
1431 f = getframe_deferred(d, n);
1432 if (f) {
1433 calc_rttavg(d, NULL, tsince_hr(f));
1434 } else {
1435 calc_rttavg(d, NULL, tsince(n));
1436 spin_unlock_irqrestore(&d->lock, flags);
1437 aoedev_put(d);
1438 snprintf(ebuf, sizeof(ebuf),
1439 "%15s e%d.%d tag=%08x@%08lx s=%pm d=%pm\n",
1440 "unexpected rsp",
1441 get_unaligned_be16(&h->major),
1442 h->minor,
1443 get_unaligned_be32(&h->tag),
1444 jiffies,
1445 h->src,
1446 h->dst);
1447 aoechr_error(ebuf);
1448 return skb;
1449 }
1450 }
1451 aoecmd_work(d);
1452
1453 spin_unlock_irqrestore(&d->lock, flags);
1454
1455 ktcomplete(f, skb);
1456
1457 /*
1458 * Note here that we do not perform an aoedev_put, as we are
1459 * leaving this reference for the ktio to release.
1460 */
1461 return NULL;
1462 }
1463
1464 void
1465 aoecmd_cfg(ushort aoemajor, unsigned char aoeminor)
1466 {
1467 struct sk_buff_head queue;
1468
1469 __skb_queue_head_init(&queue);
1470 aoecmd_cfg_pkts(aoemajor, aoeminor, &queue);
1471 aoenet_xmit(&queue);
1472 }
1473
1474 struct sk_buff *
1475 aoecmd_ata_id(struct aoedev *d)
1476 {
1477 struct aoe_hdr *h;
1478 struct aoe_atahdr *ah;
1479 struct frame *f;
1480 struct sk_buff *skb;
1481 struct aoetgt *t;
1482
1483 f = newframe(d);
1484 if (f == NULL)
1485 return NULL;
1486
1487 t = *d->tgt;
1488
1489 /* initialize the headers & frame */
1490 skb = f->skb;
1491 h = (struct aoe_hdr *) skb_mac_header(skb);
1492 ah = (struct aoe_atahdr *) (h+1);
1493 skb_put(skb, sizeof *h + sizeof *ah);
1494 memset(h, 0, skb->len);
1495 f->tag = aoehdr_atainit(d, t, h);
1496 fhash(f);
1497 t->nout++;
1498 f->waited = 0;
1499 f->waited_total = 0;
1500
1501 /* set up ata header */
1502 ah->scnt = 1;
1503 ah->cmdstat = ATA_CMD_ID_ATA;
1504 ah->lba3 = 0xa0;
1505
1506 skb->dev = t->ifp->nd;
1507
1508 d->rttavg = RTTAVG_INIT;
1509 d->rttdev = RTTDEV_INIT;
1510 d->timer.function = rexmit_timer;
1511
1512 skb = skb_clone(skb, GFP_ATOMIC);
1513 if (skb) {
1514 do_gettimeofday(&f->sent);
1515 f->sent_jiffs = (u32) jiffies;
1516 }
1517
1518 return skb;
1519 }
1520
1521 static struct aoetgt **
1522 grow_targets(struct aoedev *d)
1523 {
1524 ulong oldn, newn;
1525 struct aoetgt **tt;
1526
1527 oldn = d->ntargets;
1528 newn = oldn * 2;
1529 tt = kcalloc(newn, sizeof(*d->targets), GFP_ATOMIC);
1530 if (!tt)
1531 return NULL;
1532 memmove(tt, d->targets, sizeof(*d->targets) * oldn);
1533 d->tgt = tt + (d->tgt - d->targets);
1534 kfree(d->targets);
1535 d->targets = tt;
1536 d->ntargets = newn;
1537
1538 return &d->targets[oldn];
1539 }
1540
1541 static struct aoetgt *
1542 addtgt(struct aoedev *d, char *addr, ulong nframes)
1543 {
1544 struct aoetgt *t, **tt, **te;
1545
1546 tt = d->targets;
1547 te = tt + d->ntargets;
1548 for (; tt < te && *tt; tt++)
1549 ;
1550
1551 if (tt == te) {
1552 tt = grow_targets(d);
1553 if (!tt)
1554 goto nomem;
1555 }
1556 t = kzalloc(sizeof(*t), GFP_ATOMIC);
1557 if (!t)
1558 goto nomem;
1559 t->nframes = nframes;
1560 t->d = d;
1561 memcpy(t->addr, addr, sizeof t->addr);
1562 t->ifp = t->ifs;
1563 aoecmd_wreset(t);
1564 t->maxout = t->nframes / 2;
1565 INIT_LIST_HEAD(&t->ffree);
1566 return *tt = t;
1567
1568 nomem:
1569 pr_info("aoe: cannot allocate memory to add target\n");
1570 return NULL;
1571 }
1572
1573 static void
1574 setdbcnt(struct aoedev *d)
1575 {
1576 struct aoetgt **t, **e;
1577 int bcnt = 0;
1578
1579 t = d->targets;
1580 e = t + d->ntargets;
1581 for (; t < e && *t; t++)
1582 if (bcnt == 0 || bcnt > (*t)->minbcnt)
1583 bcnt = (*t)->minbcnt;
1584 if (bcnt != d->maxbcnt) {
1585 d->maxbcnt = bcnt;
1586 pr_info("aoe: e%ld.%d: setting %d byte data frames\n",
1587 d->aoemajor, d->aoeminor, bcnt);
1588 }
1589 }
1590
1591 static void
1592 setifbcnt(struct aoetgt *t, struct net_device *nd, int bcnt)
1593 {
1594 struct aoedev *d;
1595 struct aoeif *p, *e;
1596 int minbcnt;
1597
1598 d = t->d;
1599 minbcnt = bcnt;
1600 p = t->ifs;
1601 e = p + NAOEIFS;
1602 for (; p < e; p++) {
1603 if (p->nd == NULL)
1604 break; /* end of the valid interfaces */
1605 if (p->nd == nd) {
1606 p->bcnt = bcnt; /* we're updating */
1607 nd = NULL;
1608 } else if (minbcnt > p->bcnt)
1609 minbcnt = p->bcnt; /* find the min interface */
1610 }
1611 if (nd) {
1612 if (p == e) {
1613 pr_err("aoe: device setifbcnt failure; too many interfaces.\n");
1614 return;
1615 }
1616 dev_hold(nd);
1617 p->nd = nd;
1618 p->bcnt = bcnt;
1619 }
1620 t->minbcnt = minbcnt;
1621 setdbcnt(d);
1622 }
1623
1624 void
1625 aoecmd_cfg_rsp(struct sk_buff *skb)
1626 {
1627 struct aoedev *d;
1628 struct aoe_hdr *h;
1629 struct aoe_cfghdr *ch;
1630 struct aoetgt *t;
1631 ulong flags, aoemajor;
1632 struct sk_buff *sl;
1633 struct sk_buff_head queue;
1634 u16 n;
1635
1636 sl = NULL;
1637 h = (struct aoe_hdr *) skb_mac_header(skb);
1638 ch = (struct aoe_cfghdr *) (h+1);
1639
1640 /*
1641 * Enough people have their dip switches set backwards to
1642 * warrant a loud message for this special case.
1643 */
1644 aoemajor = get_unaligned_be16(&h->major);
1645 if (aoemajor == 0xfff) {
1646 printk(KERN_ERR "aoe: Warning: shelf address is all ones. "
1647 "Check shelf dip switches.\n");
1648 return;
1649 }
1650 if (aoemajor == 0xffff) {
1651 pr_info("aoe: e%ld.%d: broadcast shelf number invalid\n",
1652 aoemajor, (int) h->minor);
1653 return;
1654 }
1655 if (h->minor == 0xff) {
1656 pr_info("aoe: e%ld.%d: broadcast slot number invalid\n",
1657 aoemajor, (int) h->minor);
1658 return;
1659 }
1660
1661 n = be16_to_cpu(ch->bufcnt);
1662 if (n > aoe_maxout) /* keep it reasonable */
1663 n = aoe_maxout;
1664
1665 d = aoedev_by_aoeaddr(aoemajor, h->minor, 1);
1666 if (d == NULL) {
1667 pr_info("aoe: device allocation failure\n");
1668 return;
1669 }
1670
1671 spin_lock_irqsave(&d->lock, flags);
1672
1673 t = gettgt(d, h->src);
1674 if (t) {
1675 t->nframes = n;
1676 if (n < t->maxout)
1677 aoecmd_wreset(t);
1678 } else {
1679 t = addtgt(d, h->src, n);
1680 if (!t)
1681 goto bail;
1682 }
1683 n = skb->dev->mtu;
1684 n -= sizeof(struct aoe_hdr) + sizeof(struct aoe_atahdr);
1685 n /= 512;
1686 if (n > ch->scnt)
1687 n = ch->scnt;
1688 n = n ? n * 512 : DEFAULTBCNT;
1689 setifbcnt(t, skb->dev, n);
1690
1691 /* don't change users' perspective */
1692 if (d->nopen == 0) {
1693 d->fw_ver = be16_to_cpu(ch->fwver);
1694 sl = aoecmd_ata_id(d);
1695 }
1696 bail:
1697 spin_unlock_irqrestore(&d->lock, flags);
1698 aoedev_put(d);
1699 if (sl) {
1700 __skb_queue_head_init(&queue);
1701 __skb_queue_tail(&queue, sl);
1702 aoenet_xmit(&queue);
1703 }
1704 }
1705
1706 void
1707 aoecmd_wreset(struct aoetgt *t)
1708 {
1709 t->maxout = 1;
1710 t->ssthresh = t->nframes / 2;
1711 t->next_cwnd = t->nframes;
1712 }
1713
1714 void
1715 aoecmd_cleanslate(struct aoedev *d)
1716 {
1717 struct aoetgt **t, **te;
1718
1719 d->rttavg = RTTAVG_INIT;
1720 d->rttdev = RTTDEV_INIT;
1721 d->maxbcnt = 0;
1722
1723 t = d->targets;
1724 te = t + d->ntargets;
1725 for (; t < te && *t; t++)
1726 aoecmd_wreset(*t);
1727 }
1728
1729 void
1730 aoe_failbuf(struct aoedev *d, struct buf *buf)
1731 {
1732 if (buf == NULL)
1733 return;
1734 buf->resid = 0;
1735 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags);
1736 if (buf->nframesout == 0)
1737 aoe_end_buf(d, buf);
1738 }
1739
1740 void
1741 aoe_flush_iocq(void)
1742 {
1743 int i;
1744
1745 for (i = 0; i < ncpus; i++) {
1746 if (kts[i].active)
1747 aoe_flush_iocq_by_index(i);
1748 }
1749 }
1750
1751 void
1752 aoe_flush_iocq_by_index(int id)
1753 {
1754 struct frame *f;
1755 struct aoedev *d;
1756 LIST_HEAD(flist);
1757 struct list_head *pos;
1758 struct sk_buff *skb;
1759 ulong flags;
1760
1761 spin_lock_irqsave(&iocq[id].lock, flags);
1762 list_splice_init(&iocq[id].head, &flist);
1763 spin_unlock_irqrestore(&iocq[id].lock, flags);
1764 while (!list_empty(&flist)) {
1765 pos = flist.next;
1766 list_del(pos);
1767 f = list_entry(pos, struct frame, head);
1768 d = f->t->d;
1769 skb = f->r_skb;
1770 spin_lock_irqsave(&d->lock, flags);
1771 if (f->buf) {
1772 f->buf->nframesout--;
1773 aoe_failbuf(d, f->buf);
1774 }
1775 aoe_freetframe(f);
1776 spin_unlock_irqrestore(&d->lock, flags);
1777 dev_kfree_skb(skb);
1778 aoedev_put(d);
1779 }
1780 }
1781
1782 int __init
1783 aoecmd_init(void)
1784 {
1785 void *p;
1786 int i;
1787 int ret;
1788
1789 /* get_zeroed_page returns page with ref count 1 */
1790 p = (void *) get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
1791 if (!p)
1792 return -ENOMEM;
1793 empty_page = virt_to_page(p);
1794
1795 ncpus = num_online_cpus();
1796
1797 iocq = kcalloc(ncpus, sizeof(struct iocq_ktio), GFP_KERNEL);
1798 if (!iocq)
1799 return -ENOMEM;
1800
1801 kts = kcalloc(ncpus, sizeof(struct ktstate), GFP_KERNEL);
1802 if (!kts) {
1803 ret = -ENOMEM;
1804 goto kts_fail;
1805 }
1806
1807 ktiowq = kcalloc(ncpus, sizeof(wait_queue_head_t), GFP_KERNEL);
1808 if (!ktiowq) {
1809 ret = -ENOMEM;
1810 goto ktiowq_fail;
1811 }
1812
1813 mutex_init(&ktio_spawn_lock);
1814
1815 for (i = 0; i < ncpus; i++) {
1816 INIT_LIST_HEAD(&iocq[i].head);
1817 spin_lock_init(&iocq[i].lock);
1818 init_waitqueue_head(&ktiowq[i]);
1819 snprintf(kts[i].name, sizeof(kts[i].name), "aoe_ktio%d", i);
1820 kts[i].fn = ktio;
1821 kts[i].waitq = &ktiowq[i];
1822 kts[i].lock = &iocq[i].lock;
1823 kts[i].id = i;
1824 kts[i].active = 0;
1825 }
1826 kts[0].active = 1;
1827 if (aoe_ktstart(&kts[0])) {
1828 ret = -ENOMEM;
1829 goto ktstart_fail;
1830 }
1831 return 0;
1832
1833 ktstart_fail:
1834 kfree(ktiowq);
1835 ktiowq_fail:
1836 kfree(kts);
1837 kts_fail:
1838 kfree(iocq);
1839
1840 return ret;
1841 }
1842
1843 void
1844 aoecmd_exit(void)
1845 {
1846 int i;
1847
1848 for (i = 0; i < ncpus; i++)
1849 if (kts[i].active)
1850 aoe_ktstop(&kts[i]);
1851
1852 aoe_flush_iocq();
1853
1854 /* Free up the iocq and thread speicific configuration
1855 * allocated during startup.
1856 */
1857 kfree(iocq);
1858 kfree(kts);
1859 kfree(ktiowq);
1860
1861 free_page((unsigned long) page_address(empty_page));
1862 empty_page = NULL;
1863 }
Linux kernel - Deliverables
This page took 0.070048 seconds and 5 git commands to generate.