Commit | Line | Data |
---|---|---|
ca47bbd9 | 1 | /* Copyright (c) 2013 Coraid, Inc. See COPYING for GPL terms. */ |
1da177e4 LT |
2 | /* |
3 | * aoecmd.c | |
4 | * Filesystem request handling methods | |
5 | */ | |
6 | ||
04b3ab52 | 7 | #include <linux/ata.h> |
5a0e3ad6 | 8 | #include <linux/slab.h> |
1da177e4 LT |
9 | #include <linux/hdreg.h> |
10 | #include <linux/blkdev.h> | |
11 | #include <linux/skbuff.h> | |
12 | #include <linux/netdevice.h> | |
3ae1c24e | 13 | #include <linux/genhd.h> |
68e0d42f | 14 | #include <linux/moduleparam.h> |
896831f5 EC |
15 | #include <linux/workqueue.h> |
16 | #include <linux/kthread.h> | |
881d966b | 17 | #include <net/net_namespace.h> |
475172fb | 18 | #include <asm/unaligned.h> |
896831f5 | 19 | #include <linux/uio.h> |
1da177e4 LT |
20 | #include "aoe.h" |
21 | ||
896831f5 EC |
22 | #define MAXIOC (8192) /* default meant to avoid most soft lockups */ |
23 | ||
24 | static void ktcomplete(struct frame *, struct sk_buff *); | |
bbb44e30 | 25 | static int count_targets(struct aoedev *d, int *untainted); |
896831f5 | 26 | |
69cf2d85 EC |
27 | static struct buf *nextbuf(struct aoedev *); |
28 | ||
b751e8b6 EC |
29 | static int aoe_deadsecs = 60 * 3; |
30 | module_param(aoe_deadsecs, int, 0644); | |
31 | MODULE_PARM_DESC(aoe_deadsecs, "After aoe_deadsecs seconds, give up and fail dev."); | |
1da177e4 | 32 | |
7b6ccc5f | 33 | static int aoe_maxout = 64; |
7df620d8 EC |
34 | module_param(aoe_maxout, int, 0644); |
35 | MODULE_PARM_DESC(aoe_maxout, | |
36 | "Only aoe_maxout outstanding packets for every MAC on eX.Y."); | |
37 | ||
8030d343 EC |
38 | /* The number of online cpus during module initialization gives us a |
39 | * convenient heuristic cap on the parallelism used for ktio threads | |
40 | * doing I/O completion. It is not important that the cap equal the | |
41 | * actual number of running CPUs at any given time, but because of CPU | |
42 | * hotplug, we take care to use ncpus instead of using | |
43 | * num_online_cpus() after module initialization. | |
44 | */ | |
45 | static int ncpus; | |
46 | ||
47 | /* mutex lock used for synchronization while thread spawning */ | |
48 | static DEFINE_MUTEX(ktio_spawn_lock); | |
49 | ||
50 | static wait_queue_head_t *ktiowq; | |
51 | static struct ktstate *kts; | |
896831f5 EC |
52 | |
53 | /* io completion queue */ | |
8030d343 | 54 | struct iocq_ktio { |
896831f5 EC |
55 | struct list_head head; |
56 | spinlock_t lock; | |
8030d343 EC |
57 | }; |
58 | static struct iocq_ktio *iocq; | |
896831f5 | 59 | |
bbb44e30 EC |
60 | static struct page *empty_page; |
61 | ||
68e0d42f | 62 | static struct sk_buff * |
e407a7f6 | 63 | new_skb(ulong len) |
1da177e4 LT |
64 | { |
65 | struct sk_buff *skb; | |
66 | ||
91c57464 | 67 | skb = alloc_skb(len + MAX_HEADER, GFP_ATOMIC); |
1da177e4 | 68 | if (skb) { |
91c57464 | 69 | skb_reserve(skb, MAX_HEADER); |
459a98ed | 70 | skb_reset_mac_header(skb); |
c1d2bbe1 | 71 | skb_reset_network_header(skb); |
1da177e4 | 72 | skb->protocol = __constant_htons(ETH_P_AOE); |
8babe8cc | 73 | skb_checksum_none_assert(skb); |
1da177e4 LT |
74 | } |
75 | return skb; | |
76 | } | |
77 | ||
3a0c40d2 EC |
78 | static struct frame * |
79 | getframe_deferred(struct aoedev *d, u32 tag) | |
80 | { | |
81 | struct list_head *head, *pos, *nx; | |
82 | struct frame *f; | |
83 | ||
84 | head = &d->rexmitq; | |
85 | list_for_each_safe(pos, nx, head) { | |
86 | f = list_entry(pos, struct frame, head); | |
87 | if (f->tag == tag) { | |
88 | list_del(pos); | |
89 | return f; | |
90 | } | |
91 | } | |
92 | return NULL; | |
93 | } | |
94 | ||
1da177e4 | 95 | static struct frame * |
64a80f5a | 96 | getframe(struct aoedev *d, u32 tag) |
1da177e4 | 97 | { |
896831f5 EC |
98 | struct frame *f; |
99 | struct list_head *head, *pos, *nx; | |
100 | u32 n; | |
1da177e4 | 101 | |
896831f5 | 102 | n = tag % NFACTIVE; |
64a80f5a | 103 | head = &d->factive[n]; |
896831f5 EC |
104 | list_for_each_safe(pos, nx, head) { |
105 | f = list_entry(pos, struct frame, head); | |
106 | if (f->tag == tag) { | |
107 | list_del(pos); | |
1da177e4 | 108 | return f; |
896831f5 EC |
109 | } |
110 | } | |
1da177e4 LT |
111 | return NULL; |
112 | } | |
113 | ||
114 | /* | |
115 | * Leave the top bit clear so we have tagspace for userland. | |
116 | * The bottom 16 bits are the xmit tick for rexmit/rttavg processing. | |
117 | * This driver reserves tag -1 to mean "unused frame." | |
118 | */ | |
119 | static int | |
64a80f5a | 120 | newtag(struct aoedev *d) |
1da177e4 LT |
121 | { |
122 | register ulong n; | |
123 | ||
124 | n = jiffies & 0xffff; | |
64a80f5a | 125 | return n |= (++d->lasttag & 0x7fff) << 16; |
1da177e4 LT |
126 | } |
127 | ||
896831f5 | 128 | static u32 |
68e0d42f | 129 | aoehdr_atainit(struct aoedev *d, struct aoetgt *t, struct aoe_hdr *h) |
1da177e4 | 130 | { |
64a80f5a | 131 | u32 host_tag = newtag(d); |
1da177e4 | 132 | |
68e0d42f EC |
133 | memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src); |
134 | memcpy(h->dst, t->addr, sizeof h->dst); | |
63e9cc5d | 135 | h->type = __constant_cpu_to_be16(ETH_P_AOE); |
1da177e4 | 136 | h->verfl = AOE_HVER; |
63e9cc5d | 137 | h->major = cpu_to_be16(d->aoemajor); |
1da177e4 LT |
138 | h->minor = d->aoeminor; |
139 | h->cmd = AOECMD_ATA; | |
63e9cc5d | 140 | h->tag = cpu_to_be32(host_tag); |
1da177e4 LT |
141 | |
142 | return host_tag; | |
143 | } | |
144 | ||
19bf2635 EC |
145 | static inline void |
146 | put_lba(struct aoe_atahdr *ah, sector_t lba) | |
147 | { | |
148 | ah->lba0 = lba; | |
149 | ah->lba1 = lba >>= 8; | |
150 | ah->lba2 = lba >>= 8; | |
151 | ah->lba3 = lba >>= 8; | |
152 | ah->lba4 = lba >>= 8; | |
153 | ah->lba5 = lba >>= 8; | |
154 | } | |
155 | ||
3f0f0133 | 156 | static struct aoeif * |
68e0d42f EC |
157 | ifrotate(struct aoetgt *t) |
158 | { | |
3f0f0133 EC |
159 | struct aoeif *ifp; |
160 | ||
161 | ifp = t->ifp; | |
162 | ifp++; | |
163 | if (ifp >= &t->ifs[NAOEIFS] || ifp->nd == NULL) | |
164 | ifp = t->ifs; | |
165 | if (ifp->nd == NULL) | |
166 | return NULL; | |
167 | return t->ifp = ifp; | |
68e0d42f EC |
168 | } |
169 | ||
9bb237b6 EC |
170 | static void |
171 | skb_pool_put(struct aoedev *d, struct sk_buff *skb) | |
172 | { | |
e9bb8fb0 | 173 | __skb_queue_tail(&d->skbpool, skb); |
9bb237b6 EC |
174 | } |
175 | ||
176 | static struct sk_buff * | |
177 | skb_pool_get(struct aoedev *d) | |
178 | { | |
e9bb8fb0 | 179 | struct sk_buff *skb = skb_peek(&d->skbpool); |
9bb237b6 | 180 | |
9bb237b6 | 181 | if (skb && atomic_read(&skb_shinfo(skb)->dataref) == 1) { |
e9bb8fb0 | 182 | __skb_unlink(skb, &d->skbpool); |
9bb237b6 EC |
183 | return skb; |
184 | } | |
e9bb8fb0 DM |
185 | if (skb_queue_len(&d->skbpool) < NSKBPOOLMAX && |
186 | (skb = new_skb(ETH_ZLEN))) | |
9bb237b6 | 187 | return skb; |
e9bb8fb0 | 188 | |
9bb237b6 EC |
189 | return NULL; |
190 | } | |
191 | ||
896831f5 EC |
192 | void |
193 | aoe_freetframe(struct frame *f) | |
194 | { | |
195 | struct aoetgt *t; | |
196 | ||
197 | t = f->t; | |
198 | f->buf = NULL; | |
bbb44e30 | 199 | f->lba = 0; |
896831f5 EC |
200 | f->bv = NULL; |
201 | f->r_skb = NULL; | |
bbb44e30 | 202 | f->flags = 0; |
896831f5 EC |
203 | list_add(&f->head, &t->ffree); |
204 | } | |
205 | ||
68e0d42f | 206 | static struct frame * |
896831f5 | 207 | newtframe(struct aoedev *d, struct aoetgt *t) |
1da177e4 | 208 | { |
896831f5 | 209 | struct frame *f; |
9bb237b6 | 210 | struct sk_buff *skb; |
896831f5 EC |
211 | struct list_head *pos; |
212 | ||
213 | if (list_empty(&t->ffree)) { | |
214 | if (t->falloc >= NSKBPOOLMAX*2) | |
215 | return NULL; | |
216 | f = kcalloc(1, sizeof(*f), GFP_ATOMIC); | |
217 | if (f == NULL) | |
218 | return NULL; | |
219 | t->falloc++; | |
220 | f->t = t; | |
221 | } else { | |
222 | pos = t->ffree.next; | |
223 | list_del(pos); | |
224 | f = list_entry(pos, struct frame, head); | |
225 | } | |
226 | ||
227 | skb = f->skb; | |
228 | if (skb == NULL) { | |
229 | f->skb = skb = new_skb(ETH_ZLEN); | |
230 | if (!skb) { | |
231 | bail: aoe_freetframe(f); | |
232 | return NULL; | |
233 | } | |
234 | } | |
235 | ||
236 | if (atomic_read(&skb_shinfo(skb)->dataref) != 1) { | |
237 | skb = skb_pool_get(d); | |
238 | if (skb == NULL) | |
239 | goto bail; | |
240 | skb_pool_put(d, f->skb); | |
241 | f->skb = skb; | |
242 | } | |
243 | ||
244 | skb->truesize -= skb->data_len; | |
245 | skb_shinfo(skb)->nr_frags = skb->data_len = 0; | |
246 | skb_trim(skb, 0); | |
247 | return f; | |
248 | } | |
249 | ||
250 | static struct frame * | |
251 | newframe(struct aoedev *d) | |
252 | { | |
253 | struct frame *f; | |
254 | struct aoetgt *t, **tt; | |
255 | int totout = 0; | |
bbb44e30 EC |
256 | int use_tainted; |
257 | int has_untainted; | |
68e0d42f | 258 | |
71114ec4 | 259 | if (!d->targets || !d->targets[0]) { |
68e0d42f EC |
260 | printk(KERN_ERR "aoe: NULL TARGETS!\n"); |
261 | return NULL; | |
262 | } | |
896831f5 | 263 | tt = d->tgt; /* last used target */ |
bbb44e30 | 264 | for (use_tainted = 0, has_untainted = 0;;) { |
896831f5 | 265 | tt++; |
71114ec4 | 266 | if (tt >= &d->targets[d->ntargets] || !*tt) |
896831f5 EC |
267 | tt = d->targets; |
268 | t = *tt; | |
bbb44e30 EC |
269 | if (!t->taint) { |
270 | has_untainted = 1; | |
271 | totout += t->nout; | |
272 | } | |
896831f5 | 273 | if (t->nout < t->maxout |
bbb44e30 | 274 | && (use_tainted || !t->taint) |
896831f5 EC |
275 | && t->ifp->nd) { |
276 | f = newtframe(d, t); | |
277 | if (f) { | |
896831f5 | 278 | ifrotate(t); |
3f0f0133 | 279 | d->tgt = tt; |
68e0d42f EC |
280 | return f; |
281 | } | |
68e0d42f | 282 | } |
bbb44e30 EC |
283 | if (tt == d->tgt) { /* we've looped and found nada */ |
284 | if (!use_tainted && !has_untainted) | |
285 | use_tainted = 1; | |
286 | else | |
287 | break; | |
288 | } | |
896831f5 EC |
289 | } |
290 | if (totout == 0) { | |
291 | d->kicked++; | |
292 | d->flags |= DEVFL_KICKME; | |
9bb237b6 | 293 | } |
68e0d42f EC |
294 | return NULL; |
295 | } | |
296 | ||
3d5b0605 EC |
297 | static void |
298 | skb_fillup(struct sk_buff *skb, struct bio_vec *bv, ulong off, ulong cnt) | |
299 | { | |
300 | int frag = 0; | |
301 | ulong fcnt; | |
302 | loop: | |
303 | fcnt = bv->bv_len - (off - bv->bv_offset); | |
304 | if (fcnt > cnt) | |
305 | fcnt = cnt; | |
306 | skb_fill_page_desc(skb, frag++, bv->bv_page, off, fcnt); | |
307 | cnt -= fcnt; | |
308 | if (cnt <= 0) | |
309 | return; | |
310 | bv++; | |
311 | off = bv->bv_offset; | |
312 | goto loop; | |
313 | } | |
314 | ||
896831f5 EC |
315 | static void |
316 | fhash(struct frame *f) | |
317 | { | |
64a80f5a | 318 | struct aoedev *d = f->t->d; |
896831f5 EC |
319 | u32 n; |
320 | ||
321 | n = f->tag % NFACTIVE; | |
64a80f5a | 322 | list_add_tail(&f->head, &d->factive[n]); |
896831f5 EC |
323 | } |
324 | ||
bbb44e30 EC |
325 | static void |
326 | ata_rw_frameinit(struct frame *f) | |
327 | { | |
328 | struct aoetgt *t; | |
329 | struct aoe_hdr *h; | |
330 | struct aoe_atahdr *ah; | |
331 | struct sk_buff *skb; | |
332 | char writebit, extbit; | |
333 | ||
334 | skb = f->skb; | |
335 | h = (struct aoe_hdr *) skb_mac_header(skb); | |
336 | ah = (struct aoe_atahdr *) (h + 1); | |
337 | skb_put(skb, sizeof(*h) + sizeof(*ah)); | |
338 | memset(h, 0, skb->len); | |
339 | ||
340 | writebit = 0x10; | |
341 | extbit = 0x4; | |
342 | ||
343 | t = f->t; | |
344 | f->tag = aoehdr_atainit(t->d, t, h); | |
345 | fhash(f); | |
346 | t->nout++; | |
347 | f->waited = 0; | |
348 | f->waited_total = 0; | |
349 | if (f->buf) | |
350 | f->lba = f->buf->sector; | |
351 | ||
352 | /* set up ata header */ | |
353 | ah->scnt = f->bcnt >> 9; | |
354 | put_lba(ah, f->lba); | |
355 | if (t->d->flags & DEVFL_EXT) { | |
356 | ah->aflags |= AOEAFL_EXT; | |
357 | } else { | |
358 | extbit = 0; | |
359 | ah->lba3 &= 0x0f; | |
360 | ah->lba3 |= 0xe0; /* LBA bit + obsolete 0xa0 */ | |
361 | } | |
362 | if (f->buf && bio_data_dir(f->buf->bio) == WRITE) { | |
363 | skb_fillup(skb, f->bv, f->bv_off, f->bcnt); | |
364 | ah->aflags |= AOEAFL_WRITE; | |
365 | skb->len += f->bcnt; | |
366 | skb->data_len = f->bcnt; | |
367 | skb->truesize += f->bcnt; | |
368 | t->wpkts++; | |
369 | } else { | |
370 | t->rpkts++; | |
371 | writebit = 0; | |
372 | } | |
373 | ||
374 | ah->cmdstat = ATA_CMD_PIO_READ | writebit | extbit; | |
375 | skb->dev = t->ifp->nd; | |
376 | } | |
377 | ||
68e0d42f EC |
378 | static int |
379 | aoecmd_ata_rw(struct aoedev *d) | |
380 | { | |
381 | struct frame *f; | |
1da177e4 | 382 | struct buf *buf; |
68e0d42f | 383 | struct aoetgt *t; |
1da177e4 | 384 | struct sk_buff *skb; |
69cf2d85 | 385 | struct sk_buff_head queue; |
3d5b0605 | 386 | ulong bcnt, fbcnt; |
1da177e4 | 387 | |
69cf2d85 EC |
388 | buf = nextbuf(d); |
389 | if (buf == NULL) | |
390 | return 0; | |
896831f5 | 391 | f = newframe(d); |
68e0d42f EC |
392 | if (f == NULL) |
393 | return 0; | |
394 | t = *d->tgt; | |
3f0f0133 | 395 | bcnt = d->maxbcnt; |
68e0d42f EC |
396 | if (bcnt == 0) |
397 | bcnt = DEFAULTBCNT; | |
3d5b0605 EC |
398 | if (bcnt > buf->resid) |
399 | bcnt = buf->resid; | |
400 | fbcnt = bcnt; | |
401 | f->bv = buf->bv; | |
402 | f->bv_off = f->bv->bv_offset + (f->bv->bv_len - buf->bv_resid); | |
403 | do { | |
404 | if (fbcnt < buf->bv_resid) { | |
405 | buf->bv_resid -= fbcnt; | |
406 | buf->resid -= fbcnt; | |
407 | break; | |
408 | } | |
409 | fbcnt -= buf->bv_resid; | |
410 | buf->resid -= buf->bv_resid; | |
411 | if (buf->resid == 0) { | |
69cf2d85 | 412 | d->ip.buf = NULL; |
3d5b0605 EC |
413 | break; |
414 | } | |
415 | buf->bv++; | |
416 | buf->bv_resid = buf->bv->bv_len; | |
417 | WARN_ON(buf->bv_resid == 0); | |
418 | } while (fbcnt); | |
419 | ||
1da177e4 | 420 | /* initialize the headers & frame */ |
1da177e4 | 421 | f->buf = buf; |
19bf2635 | 422 | f->bcnt = bcnt; |
bbb44e30 | 423 | ata_rw_frameinit(f); |
1da177e4 LT |
424 | |
425 | /* mark all tracking fields and load out */ | |
426 | buf->nframesout += 1; | |
1da177e4 | 427 | buf->sector += bcnt >> 9; |
1da177e4 | 428 | |
bbb44e30 | 429 | skb = skb_clone(f->skb, GFP_ATOMIC); |
69cf2d85 | 430 | if (skb) { |
5f0c9c48 EC |
431 | do_gettimeofday(&f->sent); |
432 | f->sent_jiffs = (u32) jiffies; | |
69cf2d85 EC |
433 | __skb_queue_head_init(&queue); |
434 | __skb_queue_tail(&queue, skb); | |
435 | aoenet_xmit(&queue); | |
436 | } | |
68e0d42f | 437 | return 1; |
1da177e4 LT |
438 | } |
439 | ||
3ae1c24e EC |
440 | /* some callers cannot sleep, and they can call this function, |
441 | * transmitting the packets later, when interrupts are on | |
442 | */ | |
e9bb8fb0 DM |
443 | static void |
444 | aoecmd_cfg_pkts(ushort aoemajor, unsigned char aoeminor, struct sk_buff_head *queue) | |
3ae1c24e EC |
445 | { |
446 | struct aoe_hdr *h; | |
447 | struct aoe_cfghdr *ch; | |
e9bb8fb0 | 448 | struct sk_buff *skb; |
3ae1c24e EC |
449 | struct net_device *ifp; |
450 | ||
840a185d ED |
451 | rcu_read_lock(); |
452 | for_each_netdev_rcu(&init_net, ifp) { | |
3ae1c24e EC |
453 | dev_hold(ifp); |
454 | if (!is_aoe_netif(ifp)) | |
7562f876 | 455 | goto cont; |
3ae1c24e | 456 | |
e407a7f6 | 457 | skb = new_skb(sizeof *h + sizeof *ch); |
3ae1c24e | 458 | if (skb == NULL) { |
a12c93f0 | 459 | printk(KERN_INFO "aoe: skb alloc failure\n"); |
7562f876 | 460 | goto cont; |
3ae1c24e | 461 | } |
19900cde | 462 | skb_put(skb, sizeof *h + sizeof *ch); |
e407a7f6 | 463 | skb->dev = ifp; |
e9bb8fb0 | 464 | __skb_queue_tail(queue, skb); |
abdbf94d | 465 | h = (struct aoe_hdr *) skb_mac_header(skb); |
3ae1c24e EC |
466 | memset(h, 0, sizeof *h + sizeof *ch); |
467 | ||
468 | memset(h->dst, 0xff, sizeof h->dst); | |
469 | memcpy(h->src, ifp->dev_addr, sizeof h->src); | |
470 | h->type = __constant_cpu_to_be16(ETH_P_AOE); | |
471 | h->verfl = AOE_HVER; | |
472 | h->major = cpu_to_be16(aoemajor); | |
473 | h->minor = aoeminor; | |
474 | h->cmd = AOECMD_CFG; | |
475 | ||
7562f876 PE |
476 | cont: |
477 | dev_put(ifp); | |
3ae1c24e | 478 | } |
840a185d | 479 | rcu_read_unlock(); |
3ae1c24e EC |
480 | } |
481 | ||
1da177e4 | 482 | static void |
896831f5 | 483 | resend(struct aoedev *d, struct frame *f) |
1da177e4 LT |
484 | { |
485 | struct sk_buff *skb; | |
69cf2d85 | 486 | struct sk_buff_head queue; |
1da177e4 | 487 | struct aoe_hdr *h; |
19bf2635 | 488 | struct aoe_atahdr *ah; |
896831f5 | 489 | struct aoetgt *t; |
1da177e4 LT |
490 | char buf[128]; |
491 | u32 n; | |
1da177e4 | 492 | |
896831f5 | 493 | t = f->t; |
64a80f5a | 494 | n = newtag(d); |
68e0d42f | 495 | skb = f->skb; |
3f0f0133 EC |
496 | if (ifrotate(t) == NULL) { |
497 | /* probably can't happen, but set it up to fail anyway */ | |
498 | pr_info("aoe: resend: no interfaces to rotate to.\n"); | |
499 | ktcomplete(f, NULL); | |
500 | return; | |
501 | } | |
68e0d42f EC |
502 | h = (struct aoe_hdr *) skb_mac_header(skb); |
503 | ah = (struct aoe_atahdr *) (h+1); | |
1da177e4 | 504 | |
bbb44e30 EC |
505 | if (!(f->flags & FFL_PROBE)) { |
506 | snprintf(buf, sizeof(buf), | |
507 | "%15s e%ld.%d oldtag=%08x@%08lx newtag=%08x s=%pm d=%pm nout=%d\n", | |
508 | "retransmit", d->aoemajor, d->aoeminor, | |
509 | f->tag, jiffies, n, | |
510 | h->src, h->dst, t->nout); | |
511 | aoechr_error(buf); | |
512 | } | |
1da177e4 | 513 | |
1da177e4 | 514 | f->tag = n; |
896831f5 | 515 | fhash(f); |
63e9cc5d | 516 | h->tag = cpu_to_be32(n); |
68e0d42f EC |
517 | memcpy(h->dst, t->addr, sizeof h->dst); |
518 | memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src); | |
519 | ||
68e0d42f | 520 | skb->dev = t->ifp->nd; |
4f51dc5e EC |
521 | skb = skb_clone(skb, GFP_ATOMIC); |
522 | if (skb == NULL) | |
523 | return; | |
5f0c9c48 EC |
524 | do_gettimeofday(&f->sent); |
525 | f->sent_jiffs = (u32) jiffies; | |
69cf2d85 EC |
526 | __skb_queue_head_init(&queue); |
527 | __skb_queue_tail(&queue, skb); | |
528 | aoenet_xmit(&queue); | |
1da177e4 LT |
529 | } |
530 | ||
5f0c9c48 EC |
531 | static int |
532 | tsince_hr(struct frame *f) | |
533 | { | |
534 | struct timeval now; | |
535 | int n; | |
536 | ||
537 | do_gettimeofday(&now); | |
538 | n = now.tv_usec - f->sent.tv_usec; | |
539 | n += (now.tv_sec - f->sent.tv_sec) * USEC_PER_SEC; | |
540 | ||
541 | if (n < 0) | |
542 | n = -n; | |
543 | ||
544 | /* For relatively long periods, use jiffies to avoid | |
545 | * discrepancies caused by updates to the system time. | |
546 | * | |
547 | * On system with HZ of 1000, 32-bits is over 49 days | |
548 | * worth of jiffies, or over 71 minutes worth of usecs. | |
549 | * | |
550 | * Jiffies overflow is handled by subtraction of unsigned ints: | |
551 | * (gdb) print (unsigned) 2 - (unsigned) 0xfffffffe | |
552 | * $3 = 4 | |
553 | * (gdb) | |
554 | */ | |
555 | if (n > USEC_PER_SEC / 4) { | |
556 | n = ((u32) jiffies) - f->sent_jiffs; | |
557 | n *= USEC_PER_SEC / HZ; | |
558 | } | |
559 | ||
560 | return n; | |
561 | } | |
562 | ||
1da177e4 | 563 | static int |
896831f5 | 564 | tsince(u32 tag) |
1da177e4 LT |
565 | { |
566 | int n; | |
567 | ||
568 | n = jiffies & 0xffff; | |
569 | n -= tag & 0xffff; | |
570 | if (n < 0) | |
571 | n += 1<<16; | |
5f0c9c48 | 572 | return jiffies_to_usecs(n + 1); |
1da177e4 LT |
573 | } |
574 | ||
68e0d42f EC |
575 | static struct aoeif * |
576 | getif(struct aoetgt *t, struct net_device *nd) | |
577 | { | |
578 | struct aoeif *p, *e; | |
579 | ||
580 | p = t->ifs; | |
581 | e = p + NAOEIFS; | |
582 | for (; p < e; p++) | |
583 | if (p->nd == nd) | |
584 | return p; | |
585 | return NULL; | |
586 | } | |
587 | ||
68e0d42f EC |
588 | static void |
589 | ejectif(struct aoetgt *t, struct aoeif *ifp) | |
590 | { | |
591 | struct aoeif *e; | |
1b86fda9 | 592 | struct net_device *nd; |
68e0d42f EC |
593 | ulong n; |
594 | ||
1b86fda9 | 595 | nd = ifp->nd; |
68e0d42f EC |
596 | e = t->ifs + NAOEIFS - 1; |
597 | n = (e - ifp) * sizeof *ifp; | |
598 | memmove(ifp, ifp+1, n); | |
599 | e->nd = NULL; | |
1b86fda9 | 600 | dev_put(nd); |
68e0d42f EC |
601 | } |
602 | ||
3fc9b032 | 603 | static struct frame * |
bbb44e30 | 604 | reassign_frame(struct frame *f) |
3fc9b032 | 605 | { |
3fc9b032 EC |
606 | struct frame *nf; |
607 | struct sk_buff *skb; | |
608 | ||
3fc9b032 EC |
609 | nf = newframe(f->t->d); |
610 | if (!nf) | |
611 | return NULL; | |
bbb44e30 EC |
612 | if (nf->t == f->t) { |
613 | aoe_freetframe(nf); | |
614 | return NULL; | |
615 | } | |
3fc9b032 EC |
616 | |
617 | skb = nf->skb; | |
618 | nf->skb = f->skb; | |
619 | nf->buf = f->buf; | |
620 | nf->bcnt = f->bcnt; | |
621 | nf->lba = f->lba; | |
622 | nf->bv = f->bv; | |
623 | nf->bv_off = f->bv_off; | |
624 | nf->waited = 0; | |
625 | nf->waited_total = f->waited_total; | |
626 | nf->sent = f->sent; | |
fe7252bf | 627 | nf->sent_jiffs = f->sent_jiffs; |
3fc9b032 | 628 | f->skb = skb; |
3fc9b032 EC |
629 | |
630 | return nf; | |
631 | } | |
632 | ||
bbb44e30 EC |
633 | static void |
634 | probe(struct aoetgt *t) | |
68e0d42f | 635 | { |
bbb44e30 EC |
636 | struct aoedev *d; |
637 | struct frame *f; | |
638 | struct sk_buff *skb; | |
639 | struct sk_buff_head queue; | |
640 | size_t n, m; | |
641 | int frag; | |
896831f5 | 642 | |
bbb44e30 EC |
643 | d = t->d; |
644 | f = newtframe(d, t); | |
645 | if (!f) { | |
646 | pr_err("%s %pm for e%ld.%d: %s\n", | |
647 | "aoe: cannot probe remote address", | |
648 | t->addr, | |
649 | (long) d->aoemajor, d->aoeminor, | |
650 | "no frame available"); | |
651 | return; | |
68e0d42f | 652 | } |
bbb44e30 EC |
653 | f->flags |= FFL_PROBE; |
654 | ifrotate(t); | |
655 | f->bcnt = t->d->maxbcnt ? t->d->maxbcnt : DEFAULTBCNT; | |
656 | ata_rw_frameinit(f); | |
657 | skb = f->skb; | |
658 | for (frag = 0, n = f->bcnt; n > 0; ++frag, n -= m) { | |
659 | if (n < PAGE_SIZE) | |
660 | m = n; | |
661 | else | |
662 | m = PAGE_SIZE; | |
663 | skb_fill_page_desc(skb, frag, empty_page, 0, m); | |
3fc9b032 | 664 | } |
bbb44e30 EC |
665 | skb->len += f->bcnt; |
666 | skb->data_len = f->bcnt; | |
667 | skb->truesize += f->bcnt; | |
668 | ||
669 | skb = skb_clone(f->skb, GFP_ATOMIC); | |
670 | if (skb) { | |
671 | do_gettimeofday(&f->sent); | |
672 | f->sent_jiffs = (u32) jiffies; | |
673 | __skb_queue_head_init(&queue); | |
674 | __skb_queue_tail(&queue, skb); | |
675 | aoenet_xmit(&queue); | |
676 | } | |
677 | } | |
678 | ||
679 | static long | |
680 | rto(struct aoedev *d) | |
681 | { | |
682 | long t; | |
683 | ||
684 | t = 2 * d->rttavg >> RTTSCALE; | |
685 | t += 8 * d->rttdev >> RTTDSCALE; | |
686 | if (t == 0) | |
687 | t = 1; | |
688 | ||
689 | return t; | |
68e0d42f EC |
690 | } |
691 | ||
3a0c40d2 EC |
692 | static void |
693 | rexmit_deferred(struct aoedev *d) | |
694 | { | |
695 | struct aoetgt *t; | |
696 | struct frame *f; | |
bbb44e30 | 697 | struct frame *nf; |
3a0c40d2 | 698 | struct list_head *pos, *nx, *head; |
3fc9b032 | 699 | int since; |
bbb44e30 EC |
700 | int untainted; |
701 | ||
702 | count_targets(d, &untainted); | |
3a0c40d2 EC |
703 | |
704 | head = &d->rexmitq; | |
705 | list_for_each_safe(pos, nx, head) { | |
706 | f = list_entry(pos, struct frame, head); | |
707 | t = f->t; | |
bbb44e30 EC |
708 | if (t->taint) { |
709 | if (!(f->flags & FFL_PROBE)) { | |
710 | nf = reassign_frame(f); | |
711 | if (nf) { | |
712 | if (t->nout_probes == 0 | |
713 | && untainted > 0) { | |
714 | probe(t); | |
715 | t->nout_probes++; | |
716 | } | |
717 | list_replace(&f->head, &nf->head); | |
718 | pos = &nf->head; | |
719 | aoe_freetframe(f); | |
720 | f = nf; | |
721 | t = f->t; | |
722 | } | |
723 | } else if (untainted < 1) { | |
724 | /* don't probe w/o other untainted aoetgts */ | |
725 | goto stop_probe; | |
726 | } else if (tsince_hr(f) < t->taint * rto(d)) { | |
727 | /* reprobe slowly when taint is high */ | |
728 | continue; | |
729 | } | |
730 | } else if (f->flags & FFL_PROBE) { | |
731 | stop_probe: /* don't probe untainted aoetgts */ | |
732 | list_del(pos); | |
733 | aoe_freetframe(f); | |
734 | /* leaving d->kicked, because this is routine */ | |
735 | f->t->d->flags |= DEVFL_KICKME; | |
736 | continue; | |
737 | } | |
3a0c40d2 EC |
738 | if (t->nout >= t->maxout) |
739 | continue; | |
740 | list_del(pos); | |
741 | t->nout++; | |
bbb44e30 EC |
742 | if (f->flags & FFL_PROBE) |
743 | t->nout_probes++; | |
3fc9b032 EC |
744 | since = tsince_hr(f); |
745 | f->waited += since; | |
746 | f->waited_total += since; | |
3a0c40d2 EC |
747 | resend(d, f); |
748 | } | |
749 | } | |
750 | ||
bbb44e30 EC |
751 | /* An aoetgt accumulates demerits quickly, and successful |
752 | * probing redeems the aoetgt slowly. | |
753 | */ | |
754 | static void | |
755 | scorn(struct aoetgt *t) | |
756 | { | |
757 | int n; | |
758 | ||
759 | n = t->taint++; | |
760 | t->taint += t->taint * 2; | |
761 | if (n > t->taint) | |
762 | t->taint = n; | |
763 | if (t->taint > MAX_TAINT) | |
764 | t->taint = MAX_TAINT; | |
765 | } | |
766 | ||
767 | static int | |
768 | count_targets(struct aoedev *d, int *untainted) | |
769 | { | |
770 | int i, good; | |
771 | ||
772 | for (i = good = 0; i < d->ntargets && d->targets[i]; ++i) | |
773 | if (d->targets[i]->taint == 0) | |
774 | good++; | |
775 | ||
776 | if (untainted) | |
777 | *untainted = good; | |
778 | return i; | |
779 | } | |
780 | ||
1da177e4 LT |
781 | static void |
782 | rexmit_timer(ulong vp) | |
783 | { | |
784 | struct aoedev *d; | |
3a0c40d2 | 785 | struct aoetgt *t; |
68e0d42f | 786 | struct aoeif *ifp; |
896831f5 EC |
787 | struct frame *f; |
788 | struct list_head *head, *pos, *nx; | |
789 | LIST_HEAD(flist); | |
1da177e4 LT |
790 | register long timeout; |
791 | ulong flags, n; | |
896831f5 | 792 | int i; |
bbb44e30 | 793 | int utgts; /* number of aoetgt descriptors (not slots) */ |
3fc9b032 | 794 | int since; |
1da177e4 LT |
795 | |
796 | d = (struct aoedev *) vp; | |
1da177e4 | 797 | |
0d555ecf EC |
798 | spin_lock_irqsave(&d->lock, flags); |
799 | ||
3a0c40d2 | 800 | /* timeout based on observed timings and variations */ |
bbb44e30 EC |
801 | timeout = rto(d); |
802 | ||
803 | utgts = count_targets(d, NULL); | |
1da177e4 | 804 | |
1da177e4 | 805 | if (d->flags & DEVFL_TKILL) { |
1c6f3fca | 806 | spin_unlock_irqrestore(&d->lock, flags); |
1da177e4 LT |
807 | return; |
808 | } | |
896831f5 EC |
809 | |
810 | /* collect all frames to rexmit into flist */ | |
64a80f5a EC |
811 | for (i = 0; i < NFACTIVE; i++) { |
812 | head = &d->factive[i]; | |
813 | list_for_each_safe(pos, nx, head) { | |
814 | f = list_entry(pos, struct frame, head); | |
5f0c9c48 | 815 | if (tsince_hr(f) < timeout) |
64a80f5a EC |
816 | break; /* end of expired frames */ |
817 | /* move to flist for later processing */ | |
818 | list_move_tail(pos, &flist); | |
68e0d42f | 819 | } |
64a80f5a | 820 | } |
69cf2d85 | 821 | |
896831f5 EC |
822 | /* process expired frames */ |
823 | while (!list_empty(&flist)) { | |
824 | pos = flist.next; | |
825 | f = list_entry(pos, struct frame, head); | |
3fc9b032 EC |
826 | since = tsince_hr(f); |
827 | n = f->waited_total + since; | |
5f0c9c48 | 828 | n /= USEC_PER_SEC; |
c450ba0f EC |
829 | if (aoe_deadsecs |
830 | && n > aoe_deadsecs | |
831 | && !(f->flags & FFL_PROBE)) { | |
896831f5 EC |
832 | /* Waited too long. Device failure. |
833 | * Hang all frames on first hash bucket for downdev | |
834 | * to clean up. | |
835 | */ | |
64a80f5a | 836 | list_splice(&flist, &d->factive[0]); |
896831f5 | 837 | aoedev_downdev(d); |
3a0c40d2 | 838 | goto out; |
896831f5 | 839 | } |
896831f5 EC |
840 | |
841 | t = f->t; | |
bbb44e30 EC |
842 | n = f->waited + since; |
843 | n /= USEC_PER_SEC; | |
844 | if (aoe_deadsecs && utgts > 0 | |
845 | && (n > aoe_deadsecs / utgts || n > HARD_SCORN_SECS)) | |
846 | scorn(t); /* avoid this target */ | |
d54d35ac | 847 | |
3a0c40d2 EC |
848 | if (t->maxout != 1) { |
849 | t->ssthresh = t->maxout / 2; | |
850 | t->maxout = 1; | |
896831f5 EC |
851 | } |
852 | ||
bbb44e30 EC |
853 | if (f->flags & FFL_PROBE) { |
854 | t->nout_probes--; | |
855 | } else { | |
856 | ifp = getif(t, f->skb->dev); | |
857 | if (ifp && ++ifp->lost > (t->nframes << 1) | |
858 | && (ifp != t->ifs || t->ifs[1].nd)) { | |
859 | ejectif(t, ifp); | |
860 | ifp = NULL; | |
861 | } | |
896831f5 | 862 | } |
3a0c40d2 EC |
863 | list_move_tail(pos, &d->rexmitq); |
864 | t->nout--; | |
896831f5 | 865 | } |
3a0c40d2 | 866 | rexmit_deferred(d); |
896831f5 | 867 | |
3a0c40d2 | 868 | out: |
bbb44e30 | 869 | if ((d->flags & DEVFL_KICKME) && d->blkq) { |
4f51dc5e | 870 | d->flags &= ~DEVFL_KICKME; |
69cf2d85 | 871 | d->blkq->request_fn(d->blkq); |
4f51dc5e | 872 | } |
1da177e4 | 873 | |
1da177e4 LT |
874 | d->timer.expires = jiffies + TIMERTICK; |
875 | add_timer(&d->timer); | |
876 | ||
877 | spin_unlock_irqrestore(&d->lock, flags); | |
69cf2d85 | 878 | } |
1da177e4 | 879 | |
69cf2d85 EC |
880 | static unsigned long |
881 | rqbiocnt(struct request *r) | |
882 | { | |
883 | struct bio *bio; | |
884 | unsigned long n = 0; | |
885 | ||
886 | __rq_for_each_bio(bio, r) | |
887 | n++; | |
888 | return n; | |
889 | } | |
890 | ||
891 | /* This can be removed if we are certain that no users of the block | |
892 | * layer will ever use zero-count pages in bios. Otherwise we have to | |
893 | * protect against the put_page sometimes done by the network layer. | |
894 | * | |
895 | * See http://oss.sgi.com/archives/xfs/2007-01/msg00594.html for | |
896 | * discussion. | |
897 | * | |
898 | * We cannot use get_page in the workaround, because it insists on a | |
899 | * positive page count as a precondition. So we use _count directly. | |
900 | */ | |
901 | static void | |
902 | bio_pageinc(struct bio *bio) | |
903 | { | |
904 | struct bio_vec *bv; | |
905 | struct page *page; | |
906 | int i; | |
907 | ||
908 | bio_for_each_segment(bv, bio, i) { | |
69cf2d85 | 909 | /* Non-zero page count for non-head members of |
fb32975d | 910 | * compound pages is no longer allowed by the kernel. |
69cf2d85 | 911 | */ |
fb32975d | 912 | page = compound_trans_head(bv->bv_page); |
69cf2d85 EC |
913 | atomic_inc(&page->_count); |
914 | } | |
915 | } | |
916 | ||
917 | static void | |
918 | bio_pagedec(struct bio *bio) | |
919 | { | |
920 | struct bio_vec *bv; | |
fb32975d | 921 | struct page *page; |
69cf2d85 EC |
922 | int i; |
923 | ||
fb32975d EC |
924 | bio_for_each_segment(bv, bio, i) { |
925 | page = compound_trans_head(bv->bv_page); | |
926 | atomic_dec(&page->_count); | |
927 | } | |
69cf2d85 EC |
928 | } |
929 | ||
930 | static void | |
931 | bufinit(struct buf *buf, struct request *rq, struct bio *bio) | |
932 | { | |
69cf2d85 EC |
933 | memset(buf, 0, sizeof(*buf)); |
934 | buf->rq = rq; | |
935 | buf->bio = bio; | |
936 | buf->resid = bio->bi_size; | |
937 | buf->sector = bio->bi_sector; | |
938 | bio_pageinc(bio); | |
ebb37277 | 939 | buf->bv = bio_iovec(bio); |
2124469e | 940 | buf->bv_resid = buf->bv->bv_len; |
69cf2d85 EC |
941 | WARN_ON(buf->bv_resid == 0); |
942 | } | |
943 | ||
944 | static struct buf * | |
945 | nextbuf(struct aoedev *d) | |
946 | { | |
947 | struct request *rq; | |
948 | struct request_queue *q; | |
949 | struct buf *buf; | |
950 | struct bio *bio; | |
951 | ||
952 | q = d->blkq; | |
953 | if (q == NULL) | |
954 | return NULL; /* initializing */ | |
955 | if (d->ip.buf) | |
956 | return d->ip.buf; | |
957 | rq = d->ip.rq; | |
958 | if (rq == NULL) { | |
959 | rq = blk_peek_request(q); | |
960 | if (rq == NULL) | |
961 | return NULL; | |
962 | blk_start_request(rq); | |
963 | d->ip.rq = rq; | |
964 | d->ip.nxbio = rq->bio; | |
965 | rq->special = (void *) rqbiocnt(rq); | |
966 | } | |
967 | buf = mempool_alloc(d->bufpool, GFP_ATOMIC); | |
968 | if (buf == NULL) { | |
969 | pr_err("aoe: nextbuf: unable to mempool_alloc!\n"); | |
970 | return NULL; | |
971 | } | |
972 | bio = d->ip.nxbio; | |
973 | bufinit(buf, rq, bio); | |
974 | bio = bio->bi_next; | |
975 | d->ip.nxbio = bio; | |
976 | if (bio == NULL) | |
977 | d->ip.rq = NULL; | |
978 | return d->ip.buf = buf; | |
1da177e4 LT |
979 | } |
980 | ||
68e0d42f EC |
981 | /* enters with d->lock held */ |
982 | void | |
983 | aoecmd_work(struct aoedev *d) | |
984 | { | |
3a0c40d2 | 985 | rexmit_deferred(d); |
69cf2d85 EC |
986 | while (aoecmd_ata_rw(d)) |
987 | ; | |
68e0d42f EC |
988 | } |
989 | ||
3ae1c24e EC |
990 | /* this function performs work that has been deferred until sleeping is OK |
991 | */ | |
992 | void | |
c4028958 | 993 | aoecmd_sleepwork(struct work_struct *work) |
3ae1c24e | 994 | { |
c4028958 | 995 | struct aoedev *d = container_of(work, struct aoedev, work); |
b21faa25 EC |
996 | struct block_device *bd; |
997 | u64 ssize; | |
3ae1c24e EC |
998 | |
999 | if (d->flags & DEVFL_GDALLOC) | |
1000 | aoeblk_gdalloc(d); | |
1001 | ||
1002 | if (d->flags & DEVFL_NEWSIZE) { | |
80795aef | 1003 | ssize = get_capacity(d->gd); |
3ae1c24e | 1004 | bd = bdget_disk(d->gd, 0); |
3ae1c24e EC |
1005 | if (bd) { |
1006 | mutex_lock(&bd->bd_inode->i_mutex); | |
1007 | i_size_write(bd->bd_inode, (loff_t)ssize<<9); | |
1008 | mutex_unlock(&bd->bd_inode->i_mutex); | |
1009 | bdput(bd); | |
1010 | } | |
b21faa25 | 1011 | spin_lock_irq(&d->lock); |
3ae1c24e EC |
1012 | d->flags |= DEVFL_UP; |
1013 | d->flags &= ~DEVFL_NEWSIZE; | |
b21faa25 | 1014 | spin_unlock_irq(&d->lock); |
3ae1c24e EC |
1015 | } |
1016 | } | |
1017 | ||
667be1e7 EC |
1018 | static void |
1019 | ata_ident_fixstring(u16 *id, int ns) | |
1020 | { | |
1021 | u16 s; | |
1022 | ||
1023 | while (ns-- > 0) { | |
1024 | s = *id; | |
1025 | *id++ = s >> 8 | s << 8; | |
1026 | } | |
1027 | } | |
1028 | ||
1da177e4 | 1029 | static void |
68e0d42f | 1030 | ataid_complete(struct aoedev *d, struct aoetgt *t, unsigned char *id) |
1da177e4 LT |
1031 | { |
1032 | u64 ssize; | |
1033 | u16 n; | |
1034 | ||
1035 | /* word 83: command set supported */ | |
f885f8d1 | 1036 | n = get_unaligned_le16(&id[83 << 1]); |
1da177e4 LT |
1037 | |
1038 | /* word 86: command set/feature enabled */ | |
f885f8d1 | 1039 | n |= get_unaligned_le16(&id[86 << 1]); |
1da177e4 LT |
1040 | |
1041 | if (n & (1<<10)) { /* bit 10: LBA 48 */ | |
1042 | d->flags |= DEVFL_EXT; | |
1043 | ||
1044 | /* word 100: number lba48 sectors */ | |
f885f8d1 | 1045 | ssize = get_unaligned_le64(&id[100 << 1]); |
1da177e4 LT |
1046 | |
1047 | /* set as in ide-disk.c:init_idedisk_capacity */ | |
1048 | d->geo.cylinders = ssize; | |
1049 | d->geo.cylinders /= (255 * 63); | |
1050 | d->geo.heads = 255; | |
1051 | d->geo.sectors = 63; | |
1052 | } else { | |
1053 | d->flags &= ~DEVFL_EXT; | |
1054 | ||
1055 | /* number lba28 sectors */ | |
f885f8d1 | 1056 | ssize = get_unaligned_le32(&id[60 << 1]); |
1da177e4 LT |
1057 | |
1058 | /* NOTE: obsolete in ATA 6 */ | |
f885f8d1 HH |
1059 | d->geo.cylinders = get_unaligned_le16(&id[54 << 1]); |
1060 | d->geo.heads = get_unaligned_le16(&id[55 << 1]); | |
1061 | d->geo.sectors = get_unaligned_le16(&id[56 << 1]); | |
1da177e4 | 1062 | } |
3ae1c24e | 1063 | |
667be1e7 EC |
1064 | ata_ident_fixstring((u16 *) &id[10<<1], 10); /* serial */ |
1065 | ata_ident_fixstring((u16 *) &id[23<<1], 4); /* firmware */ | |
1066 | ata_ident_fixstring((u16 *) &id[27<<1], 20); /* model */ | |
1067 | memcpy(d->ident, id, sizeof(d->ident)); | |
1068 | ||
3ae1c24e | 1069 | if (d->ssize != ssize) |
1d75981a | 1070 | printk(KERN_INFO |
411c41ee HH |
1071 | "aoe: %pm e%ld.%d v%04x has %llu sectors\n", |
1072 | t->addr, | |
3ae1c24e EC |
1073 | d->aoemajor, d->aoeminor, |
1074 | d->fw_ver, (long long)ssize); | |
1da177e4 LT |
1075 | d->ssize = ssize; |
1076 | d->geo.start = 0; | |
6b9699bb EC |
1077 | if (d->flags & (DEVFL_GDALLOC|DEVFL_NEWSIZE)) |
1078 | return; | |
1da177e4 | 1079 | if (d->gd != NULL) { |
80795aef | 1080 | set_capacity(d->gd, ssize); |
3ae1c24e | 1081 | d->flags |= DEVFL_NEWSIZE; |
68e0d42f | 1082 | } else |
3ae1c24e | 1083 | d->flags |= DEVFL_GDALLOC; |
1da177e4 | 1084 | schedule_work(&d->work); |
1da177e4 LT |
1085 | } |
1086 | ||
1087 | static void | |
3a0c40d2 | 1088 | calc_rttavg(struct aoedev *d, struct aoetgt *t, int rtt) |
1da177e4 LT |
1089 | { |
1090 | register long n; | |
1091 | ||
1092 | n = rtt; | |
3a0c40d2 EC |
1093 | |
1094 | /* cf. Congestion Avoidance and Control, Jacobson & Karels, 1988 */ | |
1095 | n -= d->rttavg >> RTTSCALE; | |
1096 | d->rttavg += n; | |
1097 | if (n < 0) | |
1098 | n = -n; | |
1099 | n -= d->rttdev >> RTTDSCALE; | |
1100 | d->rttdev += n; | |
1101 | ||
1102 | if (!t || t->maxout >= t->nframes) | |
1103 | return; | |
1104 | if (t->maxout < t->ssthresh) | |
1105 | t->maxout += 1; | |
1106 | else if (t->nout == t->maxout && t->next_cwnd-- == 0) { | |
1107 | t->maxout += 1; | |
1108 | t->next_cwnd = t->maxout; | |
1109 | } | |
1da177e4 LT |
1110 | } |
1111 | ||
68e0d42f EC |
1112 | static struct aoetgt * |
1113 | gettgt(struct aoedev *d, char *addr) | |
1114 | { | |
1115 | struct aoetgt **t, **e; | |
1116 | ||
1117 | t = d->targets; | |
71114ec4 | 1118 | e = t + d->ntargets; |
68e0d42f EC |
1119 | for (; t < e && *t; t++) |
1120 | if (memcmp((*t)->addr, addr, sizeof((*t)->addr)) == 0) | |
1121 | return *t; | |
1122 | return NULL; | |
1123 | } | |
1124 | ||
3d5b0605 | 1125 | static void |
896831f5 | 1126 | bvcpy(struct bio_vec *bv, ulong off, struct sk_buff *skb, long cnt) |
3d5b0605 EC |
1127 | { |
1128 | ulong fcnt; | |
1129 | char *p; | |
1130 | int soff = 0; | |
1131 | loop: | |
1132 | fcnt = bv->bv_len - (off - bv->bv_offset); | |
1133 | if (fcnt > cnt) | |
1134 | fcnt = cnt; | |
1135 | p = page_address(bv->bv_page) + off; | |
1136 | skb_copy_bits(skb, soff, p, fcnt); | |
1137 | soff += fcnt; | |
1138 | cnt -= fcnt; | |
1139 | if (cnt <= 0) | |
1140 | return; | |
1141 | bv++; | |
1142 | off = bv->bv_offset; | |
1143 | goto loop; | |
1144 | } | |
1145 | ||
69cf2d85 EC |
1146 | void |
1147 | aoe_end_request(struct aoedev *d, struct request *rq, int fastfail) | |
1148 | { | |
1149 | struct bio *bio; | |
1150 | int bok; | |
1151 | struct request_queue *q; | |
1152 | ||
1153 | q = d->blkq; | |
1154 | if (rq == d->ip.rq) | |
1155 | d->ip.rq = NULL; | |
1156 | do { | |
1157 | bio = rq->bio; | |
1158 | bok = !fastfail && test_bit(BIO_UPTODATE, &bio->bi_flags); | |
1159 | } while (__blk_end_request(rq, bok ? 0 : -EIO, bio->bi_size)); | |
1160 | ||
1161 | /* cf. http://lkml.org/lkml/2006/10/31/28 */ | |
1162 | if (!fastfail) | |
11cfb6ff | 1163 | __blk_run_queue(q); |
69cf2d85 EC |
1164 | } |
1165 | ||
1166 | static void | |
1167 | aoe_end_buf(struct aoedev *d, struct buf *buf) | |
1168 | { | |
1169 | struct request *rq; | |
1170 | unsigned long n; | |
1171 | ||
1172 | if (buf == d->ip.buf) | |
1173 | d->ip.buf = NULL; | |
1174 | rq = buf->rq; | |
1175 | bio_pagedec(buf->bio); | |
1176 | mempool_free(buf, d->bufpool); | |
1177 | n = (unsigned long) rq->special; | |
1178 | rq->special = (void *) --n; | |
1179 | if (n == 0) | |
1180 | aoe_end_request(d, rq, 0); | |
1181 | } | |
1182 | ||
3d5b0605 | 1183 | static void |
896831f5 | 1184 | ktiocomplete(struct frame *f) |
3d5b0605 | 1185 | { |
896831f5 EC |
1186 | struct aoe_hdr *hin, *hout; |
1187 | struct aoe_atahdr *ahin, *ahout; | |
1188 | struct buf *buf; | |
1189 | struct sk_buff *skb; | |
1190 | struct aoetgt *t; | |
1191 | struct aoeif *ifp; | |
1192 | struct aoedev *d; | |
1193 | long n; | |
bbb44e30 | 1194 | int untainted; |
3d5b0605 | 1195 | |
896831f5 | 1196 | if (f == NULL) |
3d5b0605 | 1197 | return; |
896831f5 EC |
1198 | |
1199 | t = f->t; | |
1200 | d = t->d; | |
bbb44e30 EC |
1201 | skb = f->r_skb; |
1202 | buf = f->buf; | |
1203 | if (f->flags & FFL_PROBE) | |
1204 | goto out; | |
1205 | if (!skb) /* just fail the buf. */ | |
1206 | goto noskb; | |
896831f5 EC |
1207 | |
1208 | hout = (struct aoe_hdr *) skb_mac_header(f->skb); | |
1209 | ahout = (struct aoe_atahdr *) (hout+1); | |
896831f5 EC |
1210 | |
1211 | hin = (struct aoe_hdr *) skb->data; | |
1212 | skb_pull(skb, sizeof(*hin)); | |
1213 | ahin = (struct aoe_atahdr *) skb->data; | |
1214 | skb_pull(skb, sizeof(*ahin)); | |
1215 | if (ahin->cmdstat & 0xa9) { /* these bits cleared on success */ | |
1216 | pr_err("aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n", | |
1217 | ahout->cmdstat, ahin->cmdstat, | |
1218 | d->aoemajor, d->aoeminor); | |
a04b41cd | 1219 | noskb: if (buf) |
69cf2d85 | 1220 | clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); |
bbb44e30 | 1221 | goto out; |
3d5b0605 | 1222 | } |
896831f5 EC |
1223 | |
1224 | n = ahout->scnt << 9; | |
1225 | switch (ahout->cmdstat) { | |
1226 | case ATA_CMD_PIO_READ: | |
1227 | case ATA_CMD_PIO_READ_EXT: | |
1228 | if (skb->len < n) { | |
bf29754a EC |
1229 | pr_err("%s e%ld.%d. skb->len=%d need=%ld\n", |
1230 | "aoe: runt data size in read from", | |
1231 | (long) d->aoemajor, d->aoeminor, | |
1232 | skb->len, n); | |
69cf2d85 | 1233 | clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); |
896831f5 EC |
1234 | break; |
1235 | } | |
1236 | bvcpy(f->bv, f->bv_off, skb, n); | |
1237 | case ATA_CMD_PIO_WRITE: | |
1238 | case ATA_CMD_PIO_WRITE_EXT: | |
1239 | spin_lock_irq(&d->lock); | |
1240 | ifp = getif(t, skb->dev); | |
3f0f0133 | 1241 | if (ifp) |
896831f5 | 1242 | ifp->lost = 0; |
896831f5 EC |
1243 | spin_unlock_irq(&d->lock); |
1244 | break; | |
1245 | case ATA_CMD_ID_ATA: | |
1246 | if (skb->len < 512) { | |
bf29754a EC |
1247 | pr_info("%s e%ld.%d. skb->len=%d need=512\n", |
1248 | "aoe: runt data size in ataid from", | |
1249 | (long) d->aoemajor, d->aoeminor, | |
896831f5 EC |
1250 | skb->len); |
1251 | break; | |
1252 | } | |
1253 | if (skb_linearize(skb)) | |
1254 | break; | |
1255 | spin_lock_irq(&d->lock); | |
1256 | ataid_complete(d, t, skb->data); | |
1257 | spin_unlock_irq(&d->lock); | |
1258 | break; | |
1259 | default: | |
1260 | pr_info("aoe: unrecognized ata command %2.2Xh for %d.%d\n", | |
1261 | ahout->cmdstat, | |
1262 | be16_to_cpu(get_unaligned(&hin->major)), | |
1263 | hin->minor); | |
1264 | } | |
bbb44e30 | 1265 | out: |
896831f5 | 1266 | spin_lock_irq(&d->lock); |
bbb44e30 EC |
1267 | if (t->taint > 0 |
1268 | && --t->taint > 0 | |
1269 | && t->nout_probes == 0) { | |
1270 | count_targets(d, &untainted); | |
1271 | if (untainted > 0) { | |
1272 | probe(t); | |
1273 | t->nout_probes++; | |
1274 | } | |
1275 | } | |
896831f5 EC |
1276 | |
1277 | aoe_freetframe(f); | |
1278 | ||
69cf2d85 EC |
1279 | if (buf && --buf->nframesout == 0 && buf->resid == 0) |
1280 | aoe_end_buf(d, buf); | |
896831f5 | 1281 | |
69cf2d85 EC |
1282 | spin_unlock_irq(&d->lock); |
1283 | aoedev_put(d); | |
896831f5 | 1284 | dev_kfree_skb(skb); |
3d5b0605 EC |
1285 | } |
1286 | ||
896831f5 EC |
1287 | /* Enters with iocq.lock held. |
1288 | * Returns true iff responses needing processing remain. | |
1289 | */ | |
1290 | static int | |
8030d343 | 1291 | ktio(int id) |
896831f5 EC |
1292 | { |
1293 | struct frame *f; | |
1294 | struct list_head *pos; | |
1295 | int i; | |
8030d343 | 1296 | int actual_id; |
896831f5 EC |
1297 | |
1298 | for (i = 0; ; ++i) { | |
1299 | if (i == MAXIOC) | |
1300 | return 1; | |
8030d343 | 1301 | if (list_empty(&iocq[id].head)) |
896831f5 | 1302 | return 0; |
8030d343 | 1303 | pos = iocq[id].head.next; |
896831f5 | 1304 | list_del(pos); |
896831f5 | 1305 | f = list_entry(pos, struct frame, head); |
8030d343 | 1306 | spin_unlock_irq(&iocq[id].lock); |
896831f5 | 1307 | ktiocomplete(f); |
8030d343 EC |
1308 | |
1309 | /* Figure out if extra threads are required. */ | |
1310 | actual_id = f->t->d->aoeminor % ncpus; | |
1311 | ||
1312 | if (!kts[actual_id].active) { | |
1313 | BUG_ON(id != 0); | |
1314 | mutex_lock(&ktio_spawn_lock); | |
1315 | if (!kts[actual_id].active | |
1316 | && aoe_ktstart(&kts[actual_id]) == 0) | |
1317 | kts[actual_id].active = 1; | |
1318 | mutex_unlock(&ktio_spawn_lock); | |
1319 | } | |
1320 | spin_lock_irq(&iocq[id].lock); | |
896831f5 EC |
1321 | } |
1322 | } | |
1323 | ||
1324 | static int | |
1325 | kthread(void *vp) | |
1326 | { | |
1327 | struct ktstate *k; | |
1328 | DECLARE_WAITQUEUE(wait, current); | |
1329 | int more; | |
1330 | ||
1331 | k = vp; | |
1332 | current->flags |= PF_NOFREEZE; | |
1333 | set_user_nice(current, -10); | |
1334 | complete(&k->rendez); /* tell spawner we're running */ | |
1335 | do { | |
1336 | spin_lock_irq(k->lock); | |
8030d343 | 1337 | more = k->fn(k->id); |
896831f5 EC |
1338 | if (!more) { |
1339 | add_wait_queue(k->waitq, &wait); | |
1340 | __set_current_state(TASK_INTERRUPTIBLE); | |
1341 | } | |
1342 | spin_unlock_irq(k->lock); | |
1343 | if (!more) { | |
1344 | schedule(); | |
1345 | remove_wait_queue(k->waitq, &wait); | |
1346 | } else | |
1347 | cond_resched(); | |
1348 | } while (!kthread_should_stop()); | |
1349 | complete(&k->rendez); /* tell spawner we're stopping */ | |
1350 | return 0; | |
1351 | } | |
1352 | ||
eb086ec5 | 1353 | void |
896831f5 EC |
1354 | aoe_ktstop(struct ktstate *k) |
1355 | { | |
1356 | kthread_stop(k->task); | |
1357 | wait_for_completion(&k->rendez); | |
1358 | } | |
1359 | ||
eb086ec5 | 1360 | int |
896831f5 EC |
1361 | aoe_ktstart(struct ktstate *k) |
1362 | { | |
1363 | struct task_struct *task; | |
1364 | ||
1365 | init_completion(&k->rendez); | |
f170168b | 1366 | task = kthread_run(kthread, k, "%s", k->name); |
896831f5 EC |
1367 | if (task == NULL || IS_ERR(task)) |
1368 | return -ENOMEM; | |
1369 | k->task = task; | |
1370 | wait_for_completion(&k->rendez); /* allow kthread to start */ | |
1371 | init_completion(&k->rendez); /* for waiting for exit later */ | |
1372 | return 0; | |
1373 | } | |
1374 | ||
1375 | /* pass it off to kthreads for processing */ | |
1376 | static void | |
1377 | ktcomplete(struct frame *f, struct sk_buff *skb) | |
1378 | { | |
8030d343 | 1379 | int id; |
896831f5 EC |
1380 | ulong flags; |
1381 | ||
1382 | f->r_skb = skb; | |
8030d343 EC |
1383 | id = f->t->d->aoeminor % ncpus; |
1384 | spin_lock_irqsave(&iocq[id].lock, flags); | |
1385 | if (!kts[id].active) { | |
1386 | spin_unlock_irqrestore(&iocq[id].lock, flags); | |
1387 | /* The thread with id has not been spawned yet, | |
1388 | * so delegate the work to the main thread and | |
1389 | * try spawning a new thread. | |
1390 | */ | |
1391 | id = 0; | |
1392 | spin_lock_irqsave(&iocq[id].lock, flags); | |
1393 | } | |
1394 | list_add_tail(&f->head, &iocq[id].head); | |
1395 | spin_unlock_irqrestore(&iocq[id].lock, flags); | |
1396 | wake_up(&ktiowq[id]); | |
896831f5 EC |
1397 | } |
1398 | ||
1399 | struct sk_buff * | |
1da177e4 LT |
1400 | aoecmd_ata_rsp(struct sk_buff *skb) |
1401 | { | |
1402 | struct aoedev *d; | |
896831f5 | 1403 | struct aoe_hdr *h; |
1da177e4 | 1404 | struct frame *f; |
896831f5 | 1405 | u32 n; |
1da177e4 LT |
1406 | ulong flags; |
1407 | char ebuf[128]; | |
32465c65 | 1408 | u16 aoemajor; |
1409 | ||
896831f5 EC |
1410 | h = (struct aoe_hdr *) skb->data; |
1411 | aoemajor = be16_to_cpu(get_unaligned(&h->major)); | |
0c966214 | 1412 | d = aoedev_by_aoeaddr(aoemajor, h->minor, 0); |
1da177e4 LT |
1413 | if (d == NULL) { |
1414 | snprintf(ebuf, sizeof ebuf, "aoecmd_ata_rsp: ata response " | |
1415 | "for unknown device %d.%d\n", | |
896831f5 | 1416 | aoemajor, h->minor); |
1da177e4 | 1417 | aoechr_error(ebuf); |
896831f5 | 1418 | return skb; |
1da177e4 LT |
1419 | } |
1420 | ||
1421 | spin_lock_irqsave(&d->lock, flags); | |
1422 | ||
896831f5 | 1423 | n = be32_to_cpu(get_unaligned(&h->tag)); |
64a80f5a | 1424 | f = getframe(d, n); |
3a0c40d2 | 1425 | if (f) { |
5f0c9c48 | 1426 | calc_rttavg(d, f->t, tsince_hr(f)); |
3a0c40d2 | 1427 | f->t->nout--; |
bbb44e30 EC |
1428 | if (f->flags & FFL_PROBE) |
1429 | f->t->nout_probes--; | |
3a0c40d2 EC |
1430 | } else { |
1431 | f = getframe_deferred(d, n); | |
1432 | if (f) { | |
5f0c9c48 | 1433 | calc_rttavg(d, NULL, tsince_hr(f)); |
3a0c40d2 EC |
1434 | } else { |
1435 | calc_rttavg(d, NULL, tsince(n)); | |
1436 | spin_unlock_irqrestore(&d->lock, flags); | |
1437 | aoedev_put(d); | |
1438 | snprintf(ebuf, sizeof(ebuf), | |
2292a7e1 | 1439 | "%15s e%d.%d tag=%08x@%08lx s=%pm d=%pm\n", |
3a0c40d2 EC |
1440 | "unexpected rsp", |
1441 | get_unaligned_be16(&h->major), | |
1442 | h->minor, | |
1443 | get_unaligned_be32(&h->tag), | |
2292a7e1 EC |
1444 | jiffies, |
1445 | h->src, | |
1446 | h->dst); | |
3a0c40d2 EC |
1447 | aoechr_error(ebuf); |
1448 | return skb; | |
1449 | } | |
1da177e4 | 1450 | } |
1da177e4 | 1451 | aoecmd_work(d); |
1da177e4 LT |
1452 | |
1453 | spin_unlock_irqrestore(&d->lock, flags); | |
896831f5 EC |
1454 | |
1455 | ktcomplete(f, skb); | |
1456 | ||
1457 | /* | |
1458 | * Note here that we do not perform an aoedev_put, as we are | |
1459 | * leaving this reference for the ktio to release. | |
1460 | */ | |
1461 | return NULL; | |
1da177e4 LT |
1462 | } |
1463 | ||
1464 | void | |
1465 | aoecmd_cfg(ushort aoemajor, unsigned char aoeminor) | |
1466 | { | |
e9bb8fb0 | 1467 | struct sk_buff_head queue; |
1da177e4 | 1468 | |
e9bb8fb0 DM |
1469 | __skb_queue_head_init(&queue); |
1470 | aoecmd_cfg_pkts(aoemajor, aoeminor, &queue); | |
1471 | aoenet_xmit(&queue); | |
1da177e4 | 1472 | } |
a04b41cd | 1473 | |
68e0d42f | 1474 | struct sk_buff * |
1da177e4 LT |
1475 | aoecmd_ata_id(struct aoedev *d) |
1476 | { | |
1477 | struct aoe_hdr *h; | |
1478 | struct aoe_atahdr *ah; | |
1479 | struct frame *f; | |
1480 | struct sk_buff *skb; | |
68e0d42f | 1481 | struct aoetgt *t; |
1da177e4 | 1482 | |
896831f5 | 1483 | f = newframe(d); |
68e0d42f | 1484 | if (f == NULL) |
1da177e4 | 1485 | return NULL; |
68e0d42f EC |
1486 | |
1487 | t = *d->tgt; | |
1da177e4 LT |
1488 | |
1489 | /* initialize the headers & frame */ | |
e407a7f6 | 1490 | skb = f->skb; |
abdbf94d | 1491 | h = (struct aoe_hdr *) skb_mac_header(skb); |
1da177e4 | 1492 | ah = (struct aoe_atahdr *) (h+1); |
19900cde EC |
1493 | skb_put(skb, sizeof *h + sizeof *ah); |
1494 | memset(h, 0, skb->len); | |
68e0d42f | 1495 | f->tag = aoehdr_atainit(d, t, h); |
896831f5 | 1496 | fhash(f); |
68e0d42f | 1497 | t->nout++; |
1da177e4 | 1498 | f->waited = 0; |
3fc9b032 | 1499 | f->waited_total = 0; |
1da177e4 | 1500 | |
1da177e4 LT |
1501 | /* set up ata header */ |
1502 | ah->scnt = 1; | |
04b3ab52 | 1503 | ah->cmdstat = ATA_CMD_ID_ATA; |
1da177e4 LT |
1504 | ah->lba3 = 0xa0; |
1505 | ||
68e0d42f | 1506 | skb->dev = t->ifp->nd; |
1da177e4 | 1507 | |
3a0c40d2 EC |
1508 | d->rttavg = RTTAVG_INIT; |
1509 | d->rttdev = RTTDEV_INIT; | |
1da177e4 | 1510 | d->timer.function = rexmit_timer; |
1da177e4 | 1511 | |
5f0c9c48 EC |
1512 | skb = skb_clone(skb, GFP_ATOMIC); |
1513 | if (skb) { | |
1514 | do_gettimeofday(&f->sent); | |
1515 | f->sent_jiffs = (u32) jiffies; | |
1516 | } | |
1517 | ||
1518 | return skb; | |
1da177e4 | 1519 | } |
a04b41cd | 1520 | |
71114ec4 EC |
1521 | static struct aoetgt ** |
1522 | grow_targets(struct aoedev *d) | |
1523 | { | |
1524 | ulong oldn, newn; | |
1525 | struct aoetgt **tt; | |
1526 | ||
1527 | oldn = d->ntargets; | |
1528 | newn = oldn * 2; | |
1529 | tt = kcalloc(newn, sizeof(*d->targets), GFP_ATOMIC); | |
1530 | if (!tt) | |
1531 | return NULL; | |
1532 | memmove(tt, d->targets, sizeof(*d->targets) * oldn); | |
1533 | d->tgt = tt + (d->tgt - d->targets); | |
1534 | kfree(d->targets); | |
1535 | d->targets = tt; | |
1536 | d->ntargets = newn; | |
1537 | ||
1538 | return &d->targets[oldn]; | |
1539 | } | |
1540 | ||
68e0d42f EC |
1541 | static struct aoetgt * |
1542 | addtgt(struct aoedev *d, char *addr, ulong nframes) | |
1543 | { | |
1544 | struct aoetgt *t, **tt, **te; | |
68e0d42f EC |
1545 | |
1546 | tt = d->targets; | |
71114ec4 | 1547 | te = tt + d->ntargets; |
68e0d42f EC |
1548 | for (; tt < te && *tt; tt++) |
1549 | ; | |
1550 | ||
578c4aa0 | 1551 | if (tt == te) { |
71114ec4 EC |
1552 | tt = grow_targets(d); |
1553 | if (!tt) | |
1554 | goto nomem; | |
578c4aa0 | 1555 | } |
896831f5 | 1556 | t = kzalloc(sizeof(*t), GFP_ATOMIC); |
71114ec4 EC |
1557 | if (!t) |
1558 | goto nomem; | |
68e0d42f | 1559 | t->nframes = nframes; |
896831f5 | 1560 | t->d = d; |
68e0d42f EC |
1561 | memcpy(t->addr, addr, sizeof t->addr); |
1562 | t->ifp = t->ifs; | |
3a0c40d2 | 1563 | aoecmd_wreset(t); |
bbb44e30 | 1564 | t->maxout = t->nframes / 2; |
896831f5 | 1565 | INIT_LIST_HEAD(&t->ffree); |
68e0d42f | 1566 | return *tt = t; |
71114ec4 EC |
1567 | |
1568 | nomem: | |
1569 | pr_info("aoe: cannot allocate memory to add target\n"); | |
1570 | return NULL; | |
68e0d42f EC |
1571 | } |
1572 | ||
3f0f0133 EC |
1573 | static void |
1574 | setdbcnt(struct aoedev *d) | |
1575 | { | |
1576 | struct aoetgt **t, **e; | |
1577 | int bcnt = 0; | |
1578 | ||
1579 | t = d->targets; | |
71114ec4 | 1580 | e = t + d->ntargets; |
3f0f0133 EC |
1581 | for (; t < e && *t; t++) |
1582 | if (bcnt == 0 || bcnt > (*t)->minbcnt) | |
1583 | bcnt = (*t)->minbcnt; | |
1584 | if (bcnt != d->maxbcnt) { | |
1585 | d->maxbcnt = bcnt; | |
1586 | pr_info("aoe: e%ld.%d: setting %d byte data frames\n", | |
1587 | d->aoemajor, d->aoeminor, bcnt); | |
1588 | } | |
1589 | } | |
1590 | ||
1591 | static void | |
1592 | setifbcnt(struct aoetgt *t, struct net_device *nd, int bcnt) | |
1593 | { | |
1594 | struct aoedev *d; | |
1595 | struct aoeif *p, *e; | |
1596 | int minbcnt; | |
1597 | ||
1598 | d = t->d; | |
1599 | minbcnt = bcnt; | |
1600 | p = t->ifs; | |
1601 | e = p + NAOEIFS; | |
1602 | for (; p < e; p++) { | |
1603 | if (p->nd == NULL) | |
1604 | break; /* end of the valid interfaces */ | |
1605 | if (p->nd == nd) { | |
1606 | p->bcnt = bcnt; /* we're updating */ | |
1607 | nd = NULL; | |
1608 | } else if (minbcnt > p->bcnt) | |
1609 | minbcnt = p->bcnt; /* find the min interface */ | |
1610 | } | |
1611 | if (nd) { | |
1612 | if (p == e) { | |
1613 | pr_err("aoe: device setifbcnt failure; too many interfaces.\n"); | |
1614 | return; | |
1615 | } | |
1b86fda9 | 1616 | dev_hold(nd); |
3f0f0133 EC |
1617 | p->nd = nd; |
1618 | p->bcnt = bcnt; | |
1619 | } | |
1620 | t->minbcnt = minbcnt; | |
1621 | setdbcnt(d); | |
1622 | } | |
1623 | ||
1da177e4 LT |
1624 | void |
1625 | aoecmd_cfg_rsp(struct sk_buff *skb) | |
1626 | { | |
1627 | struct aoedev *d; | |
1628 | struct aoe_hdr *h; | |
1629 | struct aoe_cfghdr *ch; | |
68e0d42f | 1630 | struct aoetgt *t; |
0c966214 | 1631 | ulong flags, aoemajor; |
1da177e4 | 1632 | struct sk_buff *sl; |
69cf2d85 | 1633 | struct sk_buff_head queue; |
19bf2635 | 1634 | u16 n; |
1da177e4 | 1635 | |
69cf2d85 | 1636 | sl = NULL; |
abdbf94d | 1637 | h = (struct aoe_hdr *) skb_mac_header(skb); |
1da177e4 LT |
1638 | ch = (struct aoe_cfghdr *) (h+1); |
1639 | ||
1640 | /* | |
1641 | * Enough people have their dip switches set backwards to | |
1642 | * warrant a loud message for this special case. | |
1643 | */ | |
823ed72e | 1644 | aoemajor = get_unaligned_be16(&h->major); |
1da177e4 | 1645 | if (aoemajor == 0xfff) { |
a12c93f0 | 1646 | printk(KERN_ERR "aoe: Warning: shelf address is all ones. " |
6bb6285f | 1647 | "Check shelf dip switches.\n"); |
1da177e4 LT |
1648 | return; |
1649 | } | |
7159e969 EC |
1650 | if (aoemajor == 0xffff) { |
1651 | pr_info("aoe: e%ld.%d: broadcast shelf number invalid\n", | |
0c966214 | 1652 | aoemajor, (int) h->minor); |
6583303c EC |
1653 | return; |
1654 | } | |
7159e969 EC |
1655 | if (h->minor == 0xff) { |
1656 | pr_info("aoe: e%ld.%d: broadcast slot number invalid\n", | |
1657 | aoemajor, (int) h->minor); | |
1da177e4 LT |
1658 | return; |
1659 | } | |
1660 | ||
19bf2635 | 1661 | n = be16_to_cpu(ch->bufcnt); |
7df620d8 EC |
1662 | if (n > aoe_maxout) /* keep it reasonable */ |
1663 | n = aoe_maxout; | |
1da177e4 | 1664 | |
7159e969 EC |
1665 | d = aoedev_by_aoeaddr(aoemajor, h->minor, 1); |
1666 | if (d == NULL) { | |
1667 | pr_info("aoe: device allocation failure\n"); | |
1668 | return; | |
1669 | } | |
1670 | ||
1da177e4 LT |
1671 | spin_lock_irqsave(&d->lock, flags); |
1672 | ||
68e0d42f | 1673 | t = gettgt(d, h->src); |
1b8a1636 EC |
1674 | if (t) { |
1675 | t->nframes = n; | |
1676 | if (n < t->maxout) | |
3a0c40d2 | 1677 | aoecmd_wreset(t); |
1b8a1636 | 1678 | } else { |
68e0d42f | 1679 | t = addtgt(d, h->src, n); |
69cf2d85 EC |
1680 | if (!t) |
1681 | goto bail; | |
68e0d42f | 1682 | } |
3f0f0133 EC |
1683 | n = skb->dev->mtu; |
1684 | n -= sizeof(struct aoe_hdr) + sizeof(struct aoe_atahdr); | |
1685 | n /= 512; | |
1686 | if (n > ch->scnt) | |
1687 | n = ch->scnt; | |
1688 | n = n ? n * 512 : DEFAULTBCNT; | |
1689 | setifbcnt(t, skb->dev, n); | |
3ae1c24e EC |
1690 | |
1691 | /* don't change users' perspective */ | |
69cf2d85 EC |
1692 | if (d->nopen == 0) { |
1693 | d->fw_ver = be16_to_cpu(ch->fwver); | |
1694 | sl = aoecmd_ata_id(d); | |
1da177e4 | 1695 | } |
69cf2d85 | 1696 | bail: |
1da177e4 | 1697 | spin_unlock_irqrestore(&d->lock, flags); |
69cf2d85 | 1698 | aoedev_put(d); |
e9bb8fb0 | 1699 | if (sl) { |
e9bb8fb0 DM |
1700 | __skb_queue_head_init(&queue); |
1701 | __skb_queue_tail(&queue, sl); | |
1702 | aoenet_xmit(&queue); | |
1703 | } | |
1da177e4 LT |
1704 | } |
1705 | ||
3a0c40d2 EC |
1706 | void |
1707 | aoecmd_wreset(struct aoetgt *t) | |
1708 | { | |
1709 | t->maxout = 1; | |
1710 | t->ssthresh = t->nframes / 2; | |
1711 | t->next_cwnd = t->nframes; | |
1712 | } | |
1713 | ||
68e0d42f EC |
1714 | void |
1715 | aoecmd_cleanslate(struct aoedev *d) | |
1716 | { | |
1717 | struct aoetgt **t, **te; | |
68e0d42f | 1718 | |
3a0c40d2 EC |
1719 | d->rttavg = RTTAVG_INIT; |
1720 | d->rttdev = RTTDEV_INIT; | |
3f0f0133 | 1721 | d->maxbcnt = 0; |
68e0d42f EC |
1722 | |
1723 | t = d->targets; | |
71114ec4 | 1724 | te = t + d->ntargets; |
3f0f0133 | 1725 | for (; t < te && *t; t++) |
3a0c40d2 | 1726 | aoecmd_wreset(*t); |
68e0d42f | 1727 | } |
896831f5 | 1728 | |
69cf2d85 EC |
1729 | void |
1730 | aoe_failbuf(struct aoedev *d, struct buf *buf) | |
1731 | { | |
1732 | if (buf == NULL) | |
1733 | return; | |
1734 | buf->resid = 0; | |
1735 | clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); | |
1736 | if (buf->nframesout == 0) | |
1737 | aoe_end_buf(d, buf); | |
1738 | } | |
1739 | ||
1740 | void | |
1741 | aoe_flush_iocq(void) | |
8030d343 EC |
1742 | { |
1743 | int i; | |
1744 | ||
1745 | for (i = 0; i < ncpus; i++) { | |
1746 | if (kts[i].active) | |
1747 | aoe_flush_iocq_by_index(i); | |
1748 | } | |
1749 | } | |
1750 | ||
1751 | void | |
1752 | aoe_flush_iocq_by_index(int id) | |
896831f5 EC |
1753 | { |
1754 | struct frame *f; | |
1755 | struct aoedev *d; | |
1756 | LIST_HEAD(flist); | |
1757 | struct list_head *pos; | |
1758 | struct sk_buff *skb; | |
1759 | ulong flags; | |
1760 | ||
8030d343 EC |
1761 | spin_lock_irqsave(&iocq[id].lock, flags); |
1762 | list_splice_init(&iocq[id].head, &flist); | |
1763 | spin_unlock_irqrestore(&iocq[id].lock, flags); | |
896831f5 EC |
1764 | while (!list_empty(&flist)) { |
1765 | pos = flist.next; | |
1766 | list_del(pos); | |
1767 | f = list_entry(pos, struct frame, head); | |
1768 | d = f->t->d; | |
1769 | skb = f->r_skb; | |
1770 | spin_lock_irqsave(&d->lock, flags); | |
1771 | if (f->buf) { | |
1772 | f->buf->nframesout--; | |
1773 | aoe_failbuf(d, f->buf); | |
1774 | } | |
1775 | aoe_freetframe(f); | |
1776 | spin_unlock_irqrestore(&d->lock, flags); | |
1777 | dev_kfree_skb(skb); | |
69cf2d85 | 1778 | aoedev_put(d); |
896831f5 EC |
1779 | } |
1780 | } | |
1781 | ||
1782 | int __init | |
1783 | aoecmd_init(void) | |
1784 | { | |
bbb44e30 | 1785 | void *p; |
8030d343 EC |
1786 | int i; |
1787 | int ret; | |
bbb44e30 EC |
1788 | |
1789 | /* get_zeroed_page returns page with ref count 1 */ | |
1790 | p = (void *) get_zeroed_page(GFP_KERNEL | __GFP_REPEAT); | |
1791 | if (!p) | |
1792 | return -ENOMEM; | |
1793 | empty_page = virt_to_page(p); | |
1794 | ||
8030d343 EC |
1795 | ncpus = num_online_cpus(); |
1796 | ||
1797 | iocq = kcalloc(ncpus, sizeof(struct iocq_ktio), GFP_KERNEL); | |
1798 | if (!iocq) | |
1799 | return -ENOMEM; | |
1800 | ||
1801 | kts = kcalloc(ncpus, sizeof(struct ktstate), GFP_KERNEL); | |
1802 | if (!kts) { | |
1803 | ret = -ENOMEM; | |
1804 | goto kts_fail; | |
1805 | } | |
1806 | ||
1807 | ktiowq = kcalloc(ncpus, sizeof(wait_queue_head_t), GFP_KERNEL); | |
1808 | if (!ktiowq) { | |
1809 | ret = -ENOMEM; | |
1810 | goto ktiowq_fail; | |
1811 | } | |
1812 | ||
1813 | mutex_init(&ktio_spawn_lock); | |
1814 | ||
1815 | for (i = 0; i < ncpus; i++) { | |
1816 | INIT_LIST_HEAD(&iocq[i].head); | |
1817 | spin_lock_init(&iocq[i].lock); | |
1818 | init_waitqueue_head(&ktiowq[i]); | |
1819 | snprintf(kts[i].name, sizeof(kts[i].name), "aoe_ktio%d", i); | |
1820 | kts[i].fn = ktio; | |
1821 | kts[i].waitq = &ktiowq[i]; | |
1822 | kts[i].lock = &iocq[i].lock; | |
1823 | kts[i].id = i; | |
1824 | kts[i].active = 0; | |
1825 | } | |
1826 | kts[0].active = 1; | |
1827 | if (aoe_ktstart(&kts[0])) { | |
1828 | ret = -ENOMEM; | |
1829 | goto ktstart_fail; | |
1830 | } | |
1831 | return 0; | |
1832 | ||
1833 | ktstart_fail: | |
1834 | kfree(ktiowq); | |
1835 | ktiowq_fail: | |
1836 | kfree(kts); | |
1837 | kts_fail: | |
1838 | kfree(iocq); | |
1839 | ||
1840 | return ret; | |
896831f5 EC |
1841 | } |
1842 | ||
1843 | void | |
1844 | aoecmd_exit(void) | |
1845 | { | |
8030d343 EC |
1846 | int i; |
1847 | ||
1848 | for (i = 0; i < ncpus; i++) | |
1849 | if (kts[i].active) | |
1850 | aoe_ktstop(&kts[i]); | |
1851 | ||
69cf2d85 | 1852 | aoe_flush_iocq(); |
bbb44e30 | 1853 | |
8030d343 EC |
1854 | /* Free up the iocq and thread speicific configuration |
1855 | * allocated during startup. | |
1856 | */ | |
1857 | kfree(iocq); | |
1858 | kfree(kts); | |
1859 | kfree(ktiowq); | |
1860 | ||
bbb44e30 EC |
1861 | free_page((unsigned long) page_address(empty_page)); |
1862 | empty_page = NULL; | |
896831f5 | 1863 | } |