projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge ../bleed-2.6
[deliverable/linux.git] / drivers / usb / host / ehci-sched.c
CommitLineData
1da177e4
LT		 1/*
2 * Copyright (c) 2001-2004 by David Brownell
3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the
7 * Free Software Foundation; either version 2 of the License, or (at your
8 * option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software Foundation,
17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 */
19
20/* this file is part of ehci-hcd.c */
21
22/*-------------------------------------------------------------------------*/
23
24/*
25 * EHCI scheduled transaction support: interrupt, iso, split iso
26 * These are called "periodic" transactions in the EHCI spec.
27 *
28 * Note that for interrupt transfers, the QH/QTD manipulation is shared
29 * with the "asynchronous" transaction support (control/bulk transfers).
30 * The only real difference is in how interrupt transfers are scheduled.
31 *
32 * For ISO, we make an "iso_stream" head to serve the same role as a QH.
33 * It keeps track of every ITD (or SITD) that's linked, and holds enough
34 * pre-calculated schedule data to make appending to the queue be quick.
35 */
36
37static int ehci_get_frame (struct usb_hcd *hcd);
38
39/*-------------------------------------------------------------------------*/
40
41/*
42 * periodic_next_shadow - return "next" pointer on shadow list
43 * @periodic: host pointer to qh/itd/sitd
44 * @tag: hardware tag for type of this record
45 */
46static union ehci_shadow *
47periodic_next_shadow (union ehci_shadow *periodic, __le32 tag)
48{
49 switch (tag) {
50 case Q_TYPE_QH:
51 return &periodic->qh->qh_next;
52 case Q_TYPE_FSTN:
53 return &periodic->fstn->fstn_next;
54 case Q_TYPE_ITD:
55 return &periodic->itd->itd_next;
56 // case Q_TYPE_SITD:
57 default:
58 return &periodic->sitd->sitd_next;
59 }
60}
61
62/* caller must hold ehci->lock */
63static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr)
64{
65 union ehci_shadow *prev_p = &ehci->pshadow [frame];
66 __le32 *hw_p = &ehci->periodic [frame];
67 union ehci_shadow here = *prev_p;
68
69 /* find predecessor of "ptr"; hw and shadow lists are in sync */
70 while (here.ptr && here.ptr != ptr) {
71 prev_p = periodic_next_shadow (prev_p, Q_NEXT_TYPE (*hw_p));
72 hw_p = here.hw_next;
73 here = *prev_p;
74 }
75 /* an interrupt entry (at list end) could have been shared */
76 if (!here.ptr)
77 return;
78
79 /* update shadow and hardware lists ... the old "next" pointers
80 * from ptr may still be in use, the caller updates them.
81 */
82 *prev_p = *periodic_next_shadow (&here, Q_NEXT_TYPE (*hw_p));
83 *hw_p = *here.hw_next;
84}
85
86/* how many of the uframe's 125 usecs are allocated? */
87static unsigned short
88periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe)
89{
90 __le32 *hw_p = &ehci->periodic [frame];
91 union ehci_shadow *q = &ehci->pshadow [frame];
92 unsigned usecs = 0;
93
94 while (q->ptr) {
95 switch (Q_NEXT_TYPE (*hw_p)) {
96 case Q_TYPE_QH:
97 /* is it in the S-mask? */
98 if (q->qh->hw_info2 & cpu_to_le32 (1 << uframe))
99 usecs += q->qh->usecs;
100 /* ... or C-mask? */
101 if (q->qh->hw_info2 & cpu_to_le32 (1 << (8 + uframe)))
102 usecs += q->qh->c_usecs;
103 hw_p = &q->qh->hw_next;
104 q = &q->qh->qh_next;
105 break;
106 // case Q_TYPE_FSTN:
107 default:
108 /* for "save place" FSTNs, count the relevant INTR
109 * bandwidth from the previous frame
110 */
111 if (q->fstn->hw_prev != EHCI_LIST_END) {
112 ehci_dbg (ehci, "ignoring FSTN cost ...\n");
113 }
114 hw_p = &q->fstn->hw_next;
115 q = &q->fstn->fstn_next;
116 break;
117 case Q_TYPE_ITD:
118 usecs += q->itd->usecs [uframe];
119 hw_p = &q->itd->hw_next;
120 q = &q->itd->itd_next;
121 break;
122 case Q_TYPE_SITD:
123 /* is it in the S-mask? (count SPLIT, DATA) */
124 if (q->sitd->hw_uframe & cpu_to_le32 (1 << uframe)) {
125 if (q->sitd->hw_fullspeed_ep &
126 __constant_cpu_to_le32 (1<<31))
127 usecs += q->sitd->stream->usecs;
128 else /* worst case for OUT start-split */
129 usecs += HS_USECS_ISO (188);
130 }
131
132 /* ... C-mask? (count CSPLIT, DATA) */
133 if (q->sitd->hw_uframe &
134 cpu_to_le32 (1 << (8 + uframe))) {
135 /* worst case for IN complete-split */
136 usecs += q->sitd->stream->c_usecs;
137 }
138
139 hw_p = &q->sitd->hw_next;
140 q = &q->sitd->sitd_next;
141 break;
142 }
143 }
144#ifdef DEBUG
145 if (usecs > 100)
146 ehci_err (ehci, "uframe %d sched overrun: %d usecs\n",
147 frame * 8 + uframe, usecs);
148#endif
149 return usecs;
150}
151
152/*-------------------------------------------------------------------------*/
153
154static int same_tt (struct usb_device *dev1, struct usb_device *dev2)
155{
156 if (!dev1->tt || !dev2->tt)
157 return 0;
158 if (dev1->tt != dev2->tt)
159 return 0;
160 if (dev1->tt->multi)
161 return dev1->ttport == dev2->ttport;
162 else
163 return 1;
164}
165
166/* return true iff the device's transaction translator is available
167 * for a periodic transfer starting at the specified frame, using
168 * all the uframes in the mask.
169 */
170static int tt_no_collision (
171 struct ehci_hcd *ehci,
172 unsigned period,
173 struct usb_device *dev,
174 unsigned frame,
175 u32 uf_mask
176)
177{
178 if (period == 0) /* error */
179 return 0;
180
181 /* note bandwidth wastage: split never follows csplit
182 * (different dev or endpoint) until the next uframe.
183 * calling convention doesn't make that distinction.
184 */
185 for (; frame < ehci->periodic_size; frame += period) {
186 union ehci_shadow here;
187 __le32 type;
188
189 here = ehci->pshadow [frame];
190 type = Q_NEXT_TYPE (ehci->periodic [frame]);
191 while (here.ptr) {
192 switch (type) {
193 case Q_TYPE_ITD:
194 type = Q_NEXT_TYPE (here.itd->hw_next);
195 here = here.itd->itd_next;
196 continue;
197 case Q_TYPE_QH:
198 if (same_tt (dev, here.qh->dev)) {
199 u32 mask;
200
201 mask = le32_to_cpu (here.qh->hw_info2);
202 /* "knows" no gap is needed */
203 mask |= mask >> 8;
204 if (mask & uf_mask)
205 break;
206 }
207 type = Q_NEXT_TYPE (here.qh->hw_next);
208 here = here.qh->qh_next;
209 continue;
210 case Q_TYPE_SITD:
211 if (same_tt (dev, here.sitd->urb->dev)) {
212 u16 mask;
213
214 mask = le32_to_cpu (here.sitd
215 ->hw_uframe);
216 /* FIXME assumes no gap for IN! */
217 mask |= mask >> 8;
218 if (mask & uf_mask)
219 break;
220 }
221 type = Q_NEXT_TYPE (here.sitd->hw_next);
222 here = here.sitd->sitd_next;
223 continue;
224 // case Q_TYPE_FSTN:
225 default:
226 ehci_dbg (ehci,
227 "periodic frame %d bogus type %d\n",
228 frame, type);
229 }
230
231 /* collision or error */
232 return 0;
233 }
234 }
235
236 /* no collision */
237 return 1;
238}
239
240/*-------------------------------------------------------------------------*/
241
242static int enable_periodic (struct ehci_hcd *ehci)
243{
244 u32 cmd;
245 int status;
246
247 /* did clearing PSE did take effect yet?
248 * takes effect only at frame boundaries...
249 */
250 status = handshake (&ehci->regs->status, STS_PSS, 0, 9 * 125);
251 if (status != 0) {
252 ehci_to_hcd(ehci)->state = HC_STATE_HALT;
253 return status;
254 }
255
256 cmd = readl (&ehci->regs->command) | CMD_PSE;
257 writel (cmd, &ehci->regs->command);
258 /* posted write ... PSS happens later */
259 ehci_to_hcd(ehci)->state = HC_STATE_RUNNING;
260
261 /* make sure ehci_work scans these */
262 ehci->next_uframe = readl (&ehci->regs->frame_index)
263 % (ehci->periodic_size << 3);
264 return 0;
265}
266
267static int disable_periodic (struct ehci_hcd *ehci)
268{
269 u32 cmd;
270 int status;
271
272 /* did setting PSE not take effect yet?
273 * takes effect only at frame boundaries...
274 */
275 status = handshake (&ehci->regs->status, STS_PSS, STS_PSS, 9 * 125);
276 if (status != 0) {
277 ehci_to_hcd(ehci)->state = HC_STATE_HALT;
278 return status;
279 }
280
281 cmd = readl (&ehci->regs->command) & ~CMD_PSE;
282 writel (cmd, &ehci->regs->command);
283 /* posted write ... */
284
285 ehci->next_uframe = -1;
286 return 0;
287}
288
289/*-------------------------------------------------------------------------*/
290
291/* periodic schedule slots have iso tds (normal or split) first, then a
292 * sparse tree for active interrupt transfers.
293 *
294 * this just links in a qh; caller guarantees uframe masks are set right.
295 * no FSTN support (yet; ehci 0.96+)
296 */
297static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
298{
299 unsigned i;
300 unsigned period = qh->period;
301
302 dev_dbg (&qh->dev->dev,
303 "link qh%d-%04x/%p start %d [%d/%d us]\n",
7dedacf4 304 period, le32_to_cpup (&qh->hw_info2) & (QH_CMASK | QH_SMASK),
1da177e4
LT		 305 qh, qh->start, qh->usecs, qh->c_usecs);
306
307 /* high bandwidth, or otherwise every microframe */
308 if (period == 0)
309 period = 1;
310
311 for (i = qh->start; i < ehci->periodic_size; i += period) {
312 union ehci_shadow *prev = &ehci->pshadow [i];
9a5d3e98 313 __le32 *hw_p = &ehci->periodic [i];
1da177e4 314 union ehci_shadow here = *prev;
9a5d3e98 315 __le32 type = 0;
1da177e4
LT		 316
317 /* skip the iso nodes at list head */
318 while (here.ptr) {
319 type = Q_NEXT_TYPE (*hw_p);
320 if (type == Q_TYPE_QH)
321 break;
322 prev = periodic_next_shadow (prev, type);
323 hw_p = &here.qh->hw_next;
324 here = *prev;
325 }
326
327 /* sorting each branch by period (slow-->fast)
328 * enables sharing interior tree nodes
329 */
330 while (here.ptr && qh != here.qh) {
331 if (qh->period > here.qh->period)
332 break;
333 prev = &here.qh->qh_next;
334 hw_p = &here.qh->hw_next;
335 here = *prev;
336 }
337 /* link in this qh, unless some earlier pass did that */
338 if (qh != here.qh) {
339 qh->qh_next = here;
340 if (here.qh)
341 qh->hw_next = *hw_p;
342 wmb ();
343 prev->qh = qh;
344 *hw_p = QH_NEXT (qh->qh_dma);
345 }
346 }
347 qh->qh_state = QH_STATE_LINKED;
348 qh_get (qh);
349
350 /* update per-qh bandwidth for usbfs */
351 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period
352 ? ((qh->usecs + qh->c_usecs) / qh->period)
353 : (qh->usecs * 8);
354
355 /* maybe enable periodic schedule processing */
356 if (!ehci->periodic_sched++)
357 return enable_periodic (ehci);
358
359 return 0;
360}
361
362static void qh_unlink_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
363{
364 unsigned i;
365 unsigned period;
366
367 // FIXME:
368 // IF this isn't high speed
369 // and this qh is active in the current uframe
370 // (and overlay token SplitXstate is false?)
371 // THEN
372 // qh->hw_info1 |= __constant_cpu_to_le32 (1 << 7 /* "ignore" */);
373
374 /* high bandwidth, or otherwise part of every microframe */
375 if ((period = qh->period) == 0)
376 period = 1;
377
378 for (i = qh->start; i < ehci->periodic_size; i += period)
379 periodic_unlink (ehci, i, qh);
380
381 /* update per-qh bandwidth for usbfs */
382 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period
383 ? ((qh->usecs + qh->c_usecs) / qh->period)
384 : (qh->usecs * 8);
385
386 dev_dbg (&qh->dev->dev,
387 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
7dedacf4
DB		 388 qh->period,
389 le32_to_cpup (&qh->hw_info2) & (QH_CMASK | QH_SMASK),
1da177e4
LT		 390 qh, qh->start, qh->usecs, qh->c_usecs);
391
392 /* qh->qh_next still "live" to HC */
393 qh->qh_state = QH_STATE_UNLINK;
394 qh->qh_next.ptr = NULL;
395 qh_put (qh);
396
397 /* maybe turn off periodic schedule */
398 ehci->periodic_sched--;
399 if (!ehci->periodic_sched)
400 (void) disable_periodic (ehci);
401}
402
403static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
404{
405 unsigned wait;
406
407 qh_unlink_periodic (ehci, qh);
408
409 /* simple/paranoid: always delay, expecting the HC needs to read
410 * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and
411 * expect khubd to clean up after any CSPLITs we won't issue.
412 * active high speed queues may need bigger delays...
413 */
414 if (list_empty (&qh->qtd_list)
7dedacf4 415 || (__constant_cpu_to_le32 (QH_CMASK)
1da177e4
LT		 416 & qh->hw_info2) != 0)
417 wait = 2;
418 else
419 wait = 55; /* worst case: 3 * 1024 */
420
421 udelay (wait);
422 qh->qh_state = QH_STATE_IDLE;
423 qh->hw_next = EHCI_LIST_END;
424 wmb ();
425}
426
427/*-------------------------------------------------------------------------*/
428
429static int check_period (
430 struct ehci_hcd *ehci,
431 unsigned frame,
432 unsigned uframe,
433 unsigned period,
434 unsigned usecs
435) {
436 int claimed;
437
438 /* complete split running into next frame?
439 * given FSTN support, we could sometimes check...
440 */
441 if (uframe >= 8)
442 return 0;
443
444 /*
445 * 80% periodic == 100 usec/uframe available
446 * convert "usecs we need" to "max already claimed"
447 */
448 usecs = 100 - usecs;
449
450 /* we "know" 2 and 4 uframe intervals were rejected; so
451 * for period 0, check _every_ microframe in the schedule.
452 */
453 if (unlikely (period == 0)) {
454 do {
455 for (uframe = 0; uframe < 7; uframe++) {
456 claimed = periodic_usecs (ehci, frame, uframe);
457 if (claimed > usecs)
458 return 0;
459 }
460 } while ((frame += 1) < ehci->periodic_size);
461
462 /* just check the specified uframe, at that period */
463 } else {
464 do {
465 claimed = periodic_usecs (ehci, frame, uframe);
466 if (claimed > usecs)
467 return 0;
468 } while ((frame += period) < ehci->periodic_size);
469 }
470
471 // success!
472 return 1;
473}
474
475static int check_intr_schedule (
476 struct ehci_hcd *ehci,
477 unsigned frame,
478 unsigned uframe,
479 const struct ehci_qh *qh,
480 __le32 *c_maskp
481)
482{
483 int retval = -ENOSPC;
484 u8 mask;
485
486 if (qh->c_usecs && uframe >= 6) /* FSTN territory? */
487 goto done;
488
489 if (!check_period (ehci, frame, uframe, qh->period, qh->usecs))
490 goto done;
491 if (!qh->c_usecs) {
492 retval = 0;
493 *c_maskp = 0;
494 goto done;
495 }
496
497 /* Make sure this tt's buffer is also available for CSPLITs.
498 * We pessimize a bit; probably the typical full speed case
499 * doesn't need the second CSPLIT.
500 *
501 * NOTE: both SPLIT and CSPLIT could be checked in just
502 * one smart pass...
503 */
504 mask = 0x03 << (uframe + qh->gap_uf);
505 *c_maskp = cpu_to_le32 (mask << 8);
506
507 mask |= 1 << uframe;
508 if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) {
509 if (!check_period (ehci, frame, uframe + qh->gap_uf + 1,
510 qh->period, qh->c_usecs))
511 goto done;
512 if (!check_period (ehci, frame, uframe + qh->gap_uf,
513 qh->period, qh->c_usecs))
514 goto done;
515 retval = 0;
516 }
517done:
518 return retval;
519}
520
521/* "first fit" scheduling policy used the first time through,
522 * or when the previous schedule slot can't be re-used.
523 */
524static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
525{
526 int status;
527 unsigned uframe;
528 __le32 c_mask;
529 unsigned frame; /* 0..(qh->period - 1), or NO_FRAME */
530
531 qh_refresh(ehci, qh);
532 qh->hw_next = EHCI_LIST_END;
533 frame = qh->start;
534
535 /* reuse the previous schedule slots, if we can */
536 if (frame < qh->period) {
7dedacf4 537 uframe = ffs (le32_to_cpup (&qh->hw_info2) & QH_SMASK);
1da177e4
LT		 538 status = check_intr_schedule (ehci, frame, --uframe,
539 qh, &c_mask);
540 } else {
541 uframe = 0;
542 c_mask = 0;
543 status = -ENOSPC;
544 }
545
546 /* else scan the schedule to find a group of slots such that all
547 * uframes have enough periodic bandwidth available.
548 */
549 if (status) {
550 /* "normal" case, uframing flexible except with splits */
551 if (qh->period) {
552 frame = qh->period - 1;
553 do {
554 for (uframe = 0; uframe < 8; uframe++) {
555 status = check_intr_schedule (ehci,
556 frame, uframe, qh,
557 &c_mask);
558 if (status == 0)
559 break;
560 }
561 } while (status && frame--);
562
563 /* qh->period == 0 means every uframe */
564 } else {
565 frame = 0;
566 status = check_intr_schedule (ehci, 0, 0, qh, &c_mask);
567 }
568 if (status)
569 goto done;
570 qh->start = frame;
571
572 /* reset S-frame and (maybe) C-frame masks */
7dedacf4 573 qh->hw_info2 &= __constant_cpu_to_le32(~(QH_CMASK | QH_SMASK));
1da177e4
LT		 574 qh->hw_info2 |= qh->period
575 ? cpu_to_le32 (1 << uframe)
7dedacf4 576 : __constant_cpu_to_le32 (QH_SMASK);
1da177e4
LT		 577 qh->hw_info2 |= c_mask;
578 } else
579 ehci_dbg (ehci, "reused qh %p schedule\n", qh);
580
581 /* stuff into the periodic schedule */
582 status = qh_link_periodic (ehci, qh);
583done:
584 return status;
585}
586
587static int intr_submit (
588 struct ehci_hcd *ehci,
589 struct usb_host_endpoint *ep,
590 struct urb *urb,
591 struct list_head *qtd_list,
55016f10 592 gfp_t mem_flags
1da177e4
LT		 593) {
594 unsigned epnum;
595 unsigned long flags;
596 struct ehci_qh *qh;
597 int status = 0;
598 struct list_head empty;
599
600 /* get endpoint and transfer/schedule data */
601 epnum = ep->desc.bEndpointAddress;
602
603 spin_lock_irqsave (&ehci->lock, flags);
604
605 /* get qh and force any scheduling errors */
606 INIT_LIST_HEAD (&empty);
607 qh = qh_append_tds (ehci, urb, &empty, epnum, &ep->hcpriv);
608 if (qh == NULL) {
609 status = -ENOMEM;
610 goto done;
611 }
612 if (qh->qh_state == QH_STATE_IDLE) {
613 if ((status = qh_schedule (ehci, qh)) != 0)
614 goto done;
615 }
616
617 /* then queue the urb's tds to the qh */
618 qh = qh_append_tds (ehci, urb, qtd_list, epnum, &ep->hcpriv);
619 BUG_ON (qh == NULL);
620
621 /* ... update usbfs periodic stats */
622 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
623
624done:
625 spin_unlock_irqrestore (&ehci->lock, flags);
626 if (status)
627 qtd_list_free (ehci, urb, qtd_list);
628
629 return status;
630}
631
632/*-------------------------------------------------------------------------*/
633
634/* ehci_iso_stream ops work with both ITD and SITD */
635
636static struct ehci_iso_stream *
55016f10 637iso_stream_alloc (gfp_t mem_flags)
1da177e4
LT		 638{
639 struct ehci_iso_stream *stream;
640
7b842b6e 641 stream = kzalloc(sizeof *stream, mem_flags);
1da177e4 642 if (likely (stream != NULL)) {
1da177e4
LT		 643 INIT_LIST_HEAD(&stream->td_list);
644 INIT_LIST_HEAD(&stream->free_list);
645 stream->next_uframe = -1;
646 stream->refcount = 1;
647 }
648 return stream;
649}
650
651static void
652iso_stream_init (
653 struct ehci_hcd *ehci,
654 struct ehci_iso_stream *stream,
655 struct usb_device *dev,
656 int pipe,
657 unsigned interval
658)
659{
660 static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
661
662 u32 buf1;
663 unsigned epnum, maxp;
664 int is_input;
665 long bandwidth;
666
667 /*
668 * this might be a "high bandwidth" highspeed endpoint,
669 * as encoded in the ep descriptor's wMaxPacket field
670 */
671 epnum = usb_pipeendpoint (pipe);
672 is_input = usb_pipein (pipe) ? USB_DIR_IN : 0;
673 maxp = usb_maxpacket(dev, pipe, !is_input);
674 if (is_input) {
675 buf1 = (1 << 11);
676 } else {
677 buf1 = 0;
678 }
679
680 /* knows about ITD vs SITD */
681 if (dev->speed == USB_SPEED_HIGH) {
682 unsigned multi = hb_mult(maxp);
683
684 stream->highspeed = 1;
685
686 maxp = max_packet(maxp);
687 buf1 |= maxp;
688 maxp *= multi;
689
690 stream->buf0 = cpu_to_le32 ((epnum << 8) | dev->devnum);
691 stream->buf1 = cpu_to_le32 (buf1);
692 stream->buf2 = cpu_to_le32 (multi);
693
694 /* usbfs wants to report the average usecs per frame tied up
695 * when transfers on this endpoint are scheduled ...
696 */
697 stream->usecs = HS_USECS_ISO (maxp);
698 bandwidth = stream->usecs * 8;
699 bandwidth /= 1 << (interval - 1);
700
701 } else {
702 u32 addr;
d0384200 703 int think_time;
1da177e4
LT		 704
705 addr = dev->ttport << 24;
706 if (!ehci_is_TDI(ehci)
707 || (dev->tt->hub !=
708 ehci_to_hcd(ehci)->self.root_hub))
709 addr |= dev->tt->hub->devnum << 16;
710 addr |= epnum << 8;
711 addr |= dev->devnum;
712 stream->usecs = HS_USECS_ISO (maxp);
d0384200 713 think_time = dev->tt ? dev->tt->think_time : 0;
714 stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time (
715 dev->speed, is_input, 1, maxp));
1da177e4
LT		 716 if (is_input) {
717 u32 tmp;
718
719 addr |= 1 << 31;
720 stream->c_usecs = stream->usecs;
721 stream->usecs = HS_USECS_ISO (1);
722 stream->raw_mask = 1;
723
724 /* pessimistic c-mask */
725 tmp = usb_calc_bus_time (USB_SPEED_FULL, 1, 0, maxp)
726 / (125 * 1000);
727 stream->raw_mask |= 3 << (tmp + 9);
728 } else
729 stream->raw_mask = smask_out [maxp / 188];
730 bandwidth = stream->usecs + stream->c_usecs;
731 bandwidth /= 1 << (interval + 2);
732
733 /* stream->splits gets created from raw_mask later */
734 stream->address = cpu_to_le32 (addr);
735 }
736 stream->bandwidth = bandwidth;
737
738 stream->udev = dev;
739
740 stream->bEndpointAddress = is_input | epnum;
741 stream->interval = interval;
742 stream->maxp = maxp;
743}
744
745static void
746iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream)
747{
748 stream->refcount--;
749
750 /* free whenever just a dev->ep reference remains.
751 * not like a QH -- no persistent state (toggle, halt)
752 */
753 if (stream->refcount == 1) {
754 int is_in;
755
756 // BUG_ON (!list_empty(&stream->td_list));
757
758 while (!list_empty (&stream->free_list)) {
759 struct list_head *entry;
760
761 entry = stream->free_list.next;
762 list_del (entry);
763
764 /* knows about ITD vs SITD */
765 if (stream->highspeed) {
766 struct ehci_itd *itd;
767
768 itd = list_entry (entry, struct ehci_itd,
769 itd_list);
770 dma_pool_free (ehci->itd_pool, itd,
771 itd->itd_dma);
772 } else {
773 struct ehci_sitd *sitd;
774
775 sitd = list_entry (entry, struct ehci_sitd,
776 sitd_list);
777 dma_pool_free (ehci->sitd_pool, sitd,
778 sitd->sitd_dma);
779 }
780 }
781
782 is_in = (stream->bEndpointAddress & USB_DIR_IN) ? 0x10 : 0;
783 stream->bEndpointAddress &= 0x0f;
784 stream->ep->hcpriv = NULL;
785
786 if (stream->rescheduled) {
787 ehci_info (ehci, "ep%d%s-iso rescheduled "
788 "%lu times in %lu seconds\n",
789 stream->bEndpointAddress, is_in ? "in" : "out",
790 stream->rescheduled,
791 ((jiffies - stream->start)/HZ)
792 );
793 }
794
795 kfree(stream);
796 }
797}
798
799static inline struct ehci_iso_stream *
800iso_stream_get (struct ehci_iso_stream *stream)
801{
802 if (likely (stream != NULL))
803 stream->refcount++;
804 return stream;
805}
806
807static struct ehci_iso_stream *
808iso_stream_find (struct ehci_hcd *ehci, struct urb *urb)
809{
810 unsigned epnum;
811 struct ehci_iso_stream *stream;
812 struct usb_host_endpoint *ep;
813 unsigned long flags;
814
815 epnum = usb_pipeendpoint (urb->pipe);
816 if (usb_pipein(urb->pipe))
817 ep = urb->dev->ep_in[epnum];
818 else
819 ep = urb->dev->ep_out[epnum];
820
821 spin_lock_irqsave (&ehci->lock, flags);
822 stream = ep->hcpriv;
823
824 if (unlikely (stream == NULL)) {
825 stream = iso_stream_alloc(GFP_ATOMIC);
826 if (likely (stream != NULL)) {
827 /* dev->ep owns the initial refcount */
828 ep->hcpriv = stream;
829 stream->ep = ep;
830 iso_stream_init(ehci, stream, urb->dev, urb->pipe,
831 urb->interval);
832 }
833
834 /* if dev->ep [epnum] is a QH, info1.maxpacket is nonzero */
835 } else if (unlikely (stream->hw_info1 != 0)) {
836 ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n",
837 urb->dev->devpath, epnum,
838 usb_pipein(urb->pipe) ? "in" : "out");
839 stream = NULL;
840 }
841
842 /* caller guarantees an eventual matching iso_stream_put */
843 stream = iso_stream_get (stream);
844
845 spin_unlock_irqrestore (&ehci->lock, flags);
846 return stream;
847}
848
849/*-------------------------------------------------------------------------*/
850
851/* ehci_iso_sched ops can be ITD-only or SITD-only */
852
853static struct ehci_iso_sched *
55016f10 854iso_sched_alloc (unsigned packets, gfp_t mem_flags)
1da177e4
LT		 855{
856 struct ehci_iso_sched *iso_sched;
857 int size = sizeof *iso_sched;
858
859 size += packets * sizeof (struct ehci_iso_packet);
860 iso_sched = kmalloc (size, mem_flags);
861 if (likely (iso_sched != NULL)) {
862 memset(iso_sched, 0, size);
863 INIT_LIST_HEAD (&iso_sched->td_list);
864 }
865 return iso_sched;
866}
867
868static inline void
869itd_sched_init (
870 struct ehci_iso_sched *iso_sched,
871 struct ehci_iso_stream *stream,
872 struct urb *urb
873)
874{
875 unsigned i;
876 dma_addr_t dma = urb->transfer_dma;
877
878 /* how many uframes are needed for these transfers */
879 iso_sched->span = urb->number_of_packets * stream->interval;
880
881 /* figure out per-uframe itd fields that we'll need later
882 * when we fit new itds into the schedule.
883 */
884 for (i = 0; i < urb->number_of_packets; i++) {
885 struct ehci_iso_packet *uframe = &iso_sched->packet [i];
886 unsigned length;
887 dma_addr_t buf;
888 u32 trans;
889
890 length = urb->iso_frame_desc [i].length;
891 buf = dma + urb->iso_frame_desc [i].offset;
892
893 trans = EHCI_ISOC_ACTIVE;
894 trans |= buf & 0x0fff;
895 if (unlikely (((i + 1) == urb->number_of_packets))
896 && !(urb->transfer_flags & URB_NO_INTERRUPT))
897 trans |= EHCI_ITD_IOC;
898 trans |= length << 16;
899 uframe->transaction = cpu_to_le32 (trans);
900
77078570 901 /* might need to cross a buffer page within a uframe */
1da177e4
LT		 902 uframe->bufp = (buf & ~(u64)0x0fff);
903 buf += length;
904 if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff))))
905 uframe->cross = 1;
906 }
907}
908
909static void
910iso_sched_free (
911 struct ehci_iso_stream *stream,
912 struct ehci_iso_sched *iso_sched
913)
914{
915 if (!iso_sched)
916 return;
917 // caller must hold ehci->lock!
918 list_splice (&iso_sched->td_list, &stream->free_list);
919 kfree (iso_sched);
920}
921
922static int
923itd_urb_transaction (
924 struct ehci_iso_stream *stream,
925 struct ehci_hcd *ehci,
926 struct urb *urb,
55016f10 927 gfp_t mem_flags
1da177e4
LT		 928)
929{
930 struct ehci_itd *itd;
931 dma_addr_t itd_dma;
932 int i;
933 unsigned num_itds;
934 struct ehci_iso_sched *sched;
935 unsigned long flags;
936
937 sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
938 if (unlikely (sched == NULL))
939 return -ENOMEM;
940
941 itd_sched_init (sched, stream, urb);
942
943 if (urb->interval < 8)
944 num_itds = 1 + (sched->span + 7) / 8;
945 else
946 num_itds = urb->number_of_packets;
947
948 /* allocate/init ITDs */
949 spin_lock_irqsave (&ehci->lock, flags);
950 for (i = 0; i < num_itds; i++) {
951
952 /* free_list.next might be cache-hot ... but maybe
953 * the HC caches it too. avoid that issue for now.
954 */
955
956 /* prefer previously-allocated itds */
957 if (likely (!list_empty(&stream->free_list))) {
958 itd = list_entry (stream->free_list.prev,
959 struct ehci_itd, itd_list);
960 list_del (&itd->itd_list);
961 itd_dma = itd->itd_dma;
962 } else
963 itd = NULL;
964
965 if (!itd) {
966 spin_unlock_irqrestore (&ehci->lock, flags);
967 itd = dma_pool_alloc (ehci->itd_pool, mem_flags,
968 &itd_dma);
969 spin_lock_irqsave (&ehci->lock, flags);
970 }
971
972 if (unlikely (NULL == itd)) {
973 iso_sched_free (stream, sched);
974 spin_unlock_irqrestore (&ehci->lock, flags);
975 return -ENOMEM;
976 }
977 memset (itd, 0, sizeof *itd);
978 itd->itd_dma = itd_dma;
979 list_add (&itd->itd_list, &sched->td_list);
980 }
981 spin_unlock_irqrestore (&ehci->lock, flags);
982
983 /* temporarily store schedule info in hcpriv */
984 urb->hcpriv = sched;
985 urb->error_count = 0;
986 return 0;
987}
988
989/*-------------------------------------------------------------------------*/
990
991static inline int
992itd_slot_ok (
993 struct ehci_hcd *ehci,
994 u32 mod,
995 u32 uframe,
996 u8 usecs,
997 u32 period
998)
999{
1000 uframe %= period;
1001 do {
1002 /* can't commit more than 80% periodic == 100 usec */
1003 if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7)
1004 > (100 - usecs))
1005 return 0;
1006
1007 /* we know urb->interval is 2^N uframes */
1008 uframe += period;
1009 } while (uframe < mod);
1010 return 1;
1011}
1012
1013static inline int
1014sitd_slot_ok (
1015 struct ehci_hcd *ehci,
1016 u32 mod,
1017 struct ehci_iso_stream *stream,
1018 u32 uframe,
1019 struct ehci_iso_sched *sched,
1020 u32 period_uframes
1021)
1022{
1023 u32 mask, tmp;
1024 u32 frame, uf;
1025
1026 mask = stream->raw_mask << (uframe & 7);
1027
1028 /* for IN, don't wrap CSPLIT into the next frame */
1029 if (mask & ~0xffff)
1030 return 0;
1031
1032 /* this multi-pass logic is simple, but performance may
1033 * suffer when the schedule data isn't cached.
1034 */
1035
1036 /* check bandwidth */
1037 uframe %= period_uframes;
1038 do {
1039 u32 max_used;
1040
1041 frame = uframe >> 3;
1042 uf = uframe & 7;
1043
1044 /* tt must be idle for start(s), any gap, and csplit.
1045 * assume scheduling slop leaves 10+% for control/bulk.
1046 */
1047 if (!tt_no_collision (ehci, period_uframes << 3,
1048 stream->udev, frame, mask))
1049 return 0;
1050
1051 /* check starts (OUT uses more than one) */
1052 max_used = 100 - stream->usecs;
1053 for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) {
1054 if (periodic_usecs (ehci, frame, uf) > max_used)
1055 return 0;
1056 }
1057
1058 /* for IN, check CSPLIT */
1059 if (stream->c_usecs) {
1060 max_used = 100 - stream->c_usecs;
1061 do {
1062 tmp = 1 << uf;
1063 tmp <<= 8;
1064 if ((stream->raw_mask & tmp) == 0)
1065 continue;
1066 if (periodic_usecs (ehci, frame, uf)
1067 > max_used)
1068 return 0;
1069 } while (++uf < 8);
1070 }
1071
1072 /* we know urb->interval is 2^N uframes */
1073 uframe += period_uframes;
1074 } while (uframe < mod);
1075
1076 stream->splits = cpu_to_le32(stream->raw_mask << (uframe & 7));
1077 return 1;
1078}
1079
1080/*
1081 * This scheduler plans almost as far into the future as it has actual
1082 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to
1083 * "as small as possible" to be cache-friendlier.) That limits the size
1084 * transfers you can stream reliably; avoid more than 64 msec per urb.
1085 * Also avoid queue depths of less than ehci's worst irq latency (affected
1086 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1087 * and other factors); or more than about 230 msec total (for portability,
1088 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler!
1089 */
1090
1091#define SCHEDULE_SLOP 10 /* frames */
1092
1093static int
1094iso_stream_schedule (
1095 struct ehci_hcd *ehci,
1096 struct urb *urb,
1097 struct ehci_iso_stream *stream
1098)
1099{
1100 u32 now, start, max, period;
1101 int status;
1102 unsigned mod = ehci->periodic_size << 3;
1103 struct ehci_iso_sched *sched = urb->hcpriv;
1104
1105 if (sched->span > (mod - 8 * SCHEDULE_SLOP)) {
1106 ehci_dbg (ehci, "iso request %p too long\n", urb);
1107 status = -EFBIG;
1108 goto fail;
1109 }
1110
1111 if ((stream->depth + sched->span) > mod) {
1112 ehci_dbg (ehci, "request %p would overflow (%d+%d>%d)\n",
1113 urb, stream->depth, sched->span, mod);
1114 status = -EFBIG;
1115 goto fail;
1116 }
1117
1118 now = readl (&ehci->regs->frame_index) % mod;
1119
1120 /* when's the last uframe this urb could start? */
1121 max = now + mod;
1122
1123 /* typical case: reuse current schedule. stream is still active,
1124 * and no gaps from host falling behind (irq delays etc)
1125 */
1126 if (likely (!list_empty (&stream->td_list))) {
1127 start = stream->next_uframe;
1128 if (start < now)
1129 start += mod;
1130 if (likely ((start + sched->span) < max))
1131 goto ready;
1132 /* else fell behind; someday, try to reschedule */
1133 status = -EL2NSYNC;
1134 goto fail;
1135 }
1136
1137 /* need to schedule; when's the next (u)frame we could start?
1138 * this is bigger than ehci->i_thresh allows; scheduling itself
1139 * isn't free, the slop should handle reasonably slow cpus. it
1140 * can also help high bandwidth if the dma and irq loads don't
1141 * jump until after the queue is primed.
1142 */
1143 start = SCHEDULE_SLOP * 8 + (now & ~0x07);
1144 start %= mod;
1145 stream->next_uframe = start;
1146
1147 /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */
1148
1149 period = urb->interval;
1150 if (!stream->highspeed)
1151 period <<= 3;
1152
1153 /* find a uframe slot with enough bandwidth */
1154 for (; start < (stream->next_uframe + period); start++) {
1155 int enough_space;
1156
1157 /* check schedule: enough space? */
1158 if (stream->highspeed)
1159 enough_space = itd_slot_ok (ehci, mod, start,
1160 stream->usecs, period);
1161 else {
1162 if ((start % 8) >= 6)
1163 continue;
1164 enough_space = sitd_slot_ok (ehci, mod, stream,
1165 start, sched, period);
1166 }
1167
1168 /* schedule it here if there's enough bandwidth */
1169 if (enough_space) {
1170 stream->next_uframe = start % mod;
1171 goto ready;
1172 }
1173 }
1174
1175 /* no room in the schedule */
1176 ehci_dbg (ehci, "iso %ssched full %p (now %d max %d)\n",
1177 list_empty (&stream->td_list) ? "" : "re",
1178 urb, now, max);
1179 status = -ENOSPC;
1180
1181fail:
1182 iso_sched_free (stream, sched);
1183 urb->hcpriv = NULL;
1184 return status;
1185
1186ready:
1187 /* report high speed start in uframes; full speed, in frames */
1188 urb->start_frame = stream->next_uframe;
1189 if (!stream->highspeed)
1190 urb->start_frame >>= 3;
1191 return 0;
1192}
1193
1194/*-------------------------------------------------------------------------*/
1195
1196static inline void
1197itd_init (struct ehci_iso_stream *stream, struct ehci_itd *itd)
1198{
1199 int i;
1200
77078570 1201 /* it's been recently zeroed */
1da177e4
LT		 1202 itd->hw_next = EHCI_LIST_END;
1203 itd->hw_bufp [0] = stream->buf0;
1204 itd->hw_bufp [1] = stream->buf1;
1205 itd->hw_bufp [2] = stream->buf2;
1206
1207 for (i = 0; i < 8; i++)
1208 itd->index[i] = -1;
1209
1210 /* All other fields are filled when scheduling */
1211}
1212
1213static inline void
1214itd_patch (
1215 struct ehci_itd *itd,
1216 struct ehci_iso_sched *iso_sched,
1217 unsigned index,
77078570 1218 u16 uframe
1da177e4
LT		 1219)
1220{
1221 struct ehci_iso_packet *uf = &iso_sched->packet [index];
1222 unsigned pg = itd->pg;
1223
1224 // BUG_ON (pg == 6 && uf->cross);
1225
1226 uframe &= 0x07;
1227 itd->index [uframe] = index;
1228
1229 itd->hw_transaction [uframe] = uf->transaction;
1230 itd->hw_transaction [uframe] |= cpu_to_le32 (pg << 12);
1231 itd->hw_bufp [pg] |= cpu_to_le32 (uf->bufp & ~(u32)0);
1232 itd->hw_bufp_hi [pg] |= cpu_to_le32 ((u32)(uf->bufp >> 32));
1233
1234 /* iso_frame_desc[].offset must be strictly increasing */
77078570 1235 if (unlikely (uf->cross)) {
1da177e4
LT		 1236 u64 bufp = uf->bufp + 4096;
1237 itd->pg = ++pg;
1238 itd->hw_bufp [pg] |= cpu_to_le32 (bufp & ~(u32)0);
1239 itd->hw_bufp_hi [pg] |= cpu_to_le32 ((u32)(bufp >> 32));
1240 }
1241}
1242
1243static inline void
1244itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1245{
1246 /* always prepend ITD/SITD ... only QH tree is order-sensitive */
1247 itd->itd_next = ehci->pshadow [frame];
1248 itd->hw_next = ehci->periodic [frame];
1249 ehci->pshadow [frame].itd = itd;
1250 itd->frame = frame;
1251 wmb ();
1252 ehci->periodic [frame] = cpu_to_le32 (itd->itd_dma) | Q_TYPE_ITD;
1253}
1254
1255/* fit urb's itds into the selected schedule slot; activate as needed */
1256static int
1257itd_link_urb (
1258 struct ehci_hcd *ehci,
1259 struct urb *urb,
1260 unsigned mod,
1261 struct ehci_iso_stream *stream
1262)
1263{
77078570 1264 int packet;
1da177e4
LT		 1265 unsigned next_uframe, uframe, frame;
1266 struct ehci_iso_sched *iso_sched = urb->hcpriv;
1267 struct ehci_itd *itd;
1268
1269 next_uframe = stream->next_uframe % mod;
1270
1271 if (unlikely (list_empty(&stream->td_list))) {
1272 ehci_to_hcd(ehci)->self.bandwidth_allocated
1273 += stream->bandwidth;
1274 ehci_vdbg (ehci,
1275 "schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1276 urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1277 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1278 urb->interval,
1279 next_uframe >> 3, next_uframe & 0x7);
1280 stream->start = jiffies;
1281 }
1282 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1283
1284 /* fill iTDs uframe by uframe */
1285 for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) {
1286 if (itd == NULL) {
1287 /* ASSERT: we have all necessary itds */
1288 // BUG_ON (list_empty (&iso_sched->td_list));
1289
1290 /* ASSERT: no itds for this endpoint in this uframe */
1291
1292 itd = list_entry (iso_sched->td_list.next,
1293 struct ehci_itd, itd_list);
1294 list_move_tail (&itd->itd_list, &stream->td_list);
1295 itd->stream = iso_stream_get (stream);
1296 itd->urb = usb_get_urb (urb);
1da177e4
LT		 1297 itd_init (stream, itd);
1298 }
1299
1300 uframe = next_uframe & 0x07;
1301 frame = next_uframe >> 3;
1302
1303 itd->usecs [uframe] = stream->usecs;
77078570 1304 itd_patch (itd, iso_sched, packet, uframe);
1da177e4
LT		 1305
1306 next_uframe += stream->interval;
1307 stream->depth += stream->interval;
1308 next_uframe %= mod;
1309 packet++;
1310
1311 /* link completed itds into the schedule */
1312 if (((next_uframe >> 3) != frame)
1313 || packet == urb->number_of_packets) {
1314 itd_link (ehci, frame % ehci->periodic_size, itd);
1315 itd = NULL;
1316 }
1317 }
1318 stream->next_uframe = next_uframe;
1319
1320 /* don't need that schedule data any more */
1321 iso_sched_free (stream, iso_sched);
1322 urb->hcpriv = NULL;
1323
1324 timer_action (ehci, TIMER_IO_WATCHDOG);
1325 if (unlikely (!ehci->periodic_sched++))
1326 return enable_periodic (ehci);
1327 return 0;
1328}
1329
1330#define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1331
1332static unsigned
1333itd_complete (
1334 struct ehci_hcd *ehci,
1335 struct ehci_itd *itd,
1336 struct pt_regs *regs
1337) {
1338 struct urb *urb = itd->urb;
1339 struct usb_iso_packet_descriptor *desc;
1340 u32 t;
1341 unsigned uframe;
1342 int urb_index = -1;
1343 struct ehci_iso_stream *stream = itd->stream;
1344 struct usb_device *dev;
1345
1346 /* for each uframe with a packet */
1347 for (uframe = 0; uframe < 8; uframe++) {
1348 if (likely (itd->index[uframe] == -1))
1349 continue;
1350 urb_index = itd->index[uframe];
1351 desc = &urb->iso_frame_desc [urb_index];
1352
1353 t = le32_to_cpup (&itd->hw_transaction [uframe]);
1354 itd->hw_transaction [uframe] = 0;
1355 stream->depth -= stream->interval;
1356
1357 /* report transfer status */
1358 if (unlikely (t & ISO_ERRS)) {
1359 urb->error_count++;
1360 if (t & EHCI_ISOC_BUF_ERR)
1361 desc->status = usb_pipein (urb->pipe)
1362 ? -ENOSR /* hc couldn't read */
1363 : -ECOMM; /* hc couldn't write */
1364 else if (t & EHCI_ISOC_BABBLE)
1365 desc->status = -EOVERFLOW;
1366 else /* (t & EHCI_ISOC_XACTERR) */
1367 desc->status = -EPROTO;
1368
1369 /* HC need not update length with this error */
1370 if (!(t & EHCI_ISOC_BABBLE))
1371 desc->actual_length = EHCI_ITD_LENGTH (t);
1372 } else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) {
1373 desc->status = 0;
1374 desc->actual_length = EHCI_ITD_LENGTH (t);
1375 }
1376 }
1377
1378 usb_put_urb (urb);
1379 itd->urb = NULL;
1380 itd->stream = NULL;
1381 list_move (&itd->itd_list, &stream->free_list);
1382 iso_stream_put (ehci, stream);
1383
1384 /* handle completion now? */
1385 if (likely ((urb_index + 1) != urb->number_of_packets))
1386 return 0;
1387
1388 /* ASSERT: it's really the last itd for this urb
1389 list_for_each_entry (itd, &stream->td_list, itd_list)
1390 BUG_ON (itd->urb == urb);
1391 */
1392
1393 /* give urb back to the driver ... can be out-of-order */
1394 dev = usb_get_dev (urb->dev);
1395 ehci_urb_done (ehci, urb, regs);
1396 urb = NULL;
1397
1398 /* defer stopping schedule; completion can submit */
1399 ehci->periodic_sched--;
1400 if (unlikely (!ehci->periodic_sched))
1401 (void) disable_periodic (ehci);
1402 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1403
1404 if (unlikely (list_empty (&stream->td_list))) {
1405 ehci_to_hcd(ehci)->self.bandwidth_allocated
1406 -= stream->bandwidth;
1407 ehci_vdbg (ehci,
1408 "deschedule devp %s ep%d%s-iso\n",
1409 dev->devpath, stream->bEndpointAddress & 0x0f,
1410 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
1411 }
1412 iso_stream_put (ehci, stream);
1413 usb_put_dev (dev);
1414
1415 return 1;
1416}
1417
1418/*-------------------------------------------------------------------------*/
1419
5db539e4 1420static int itd_submit (struct ehci_hcd *ehci, struct urb *urb,
55016f10 1421 gfp_t mem_flags)
1da177e4
LT		 1422{
1423 int status = -EINVAL;
1424 unsigned long flags;
1425 struct ehci_iso_stream *stream;
1426
1427 /* Get iso_stream head */
1428 stream = iso_stream_find (ehci, urb);
1429 if (unlikely (stream == NULL)) {
1430 ehci_dbg (ehci, "can't get iso stream\n");
1431 return -ENOMEM;
1432 }
1433 if (unlikely (urb->interval != stream->interval)) {
1434 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
1435 stream->interval, urb->interval);
1436 goto done;
1437 }
1438
1439#ifdef EHCI_URB_TRACE
1440 ehci_dbg (ehci,
1441 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1442 __FUNCTION__, urb->dev->devpath, urb,
1443 usb_pipeendpoint (urb->pipe),
1444 usb_pipein (urb->pipe) ? "in" : "out",
1445 urb->transfer_buffer_length,
1446 urb->number_of_packets, urb->interval,
1447 stream);
1448#endif
1449
1450 /* allocate ITDs w/o locking anything */
1451 status = itd_urb_transaction (stream, ehci, urb, mem_flags);
1452 if (unlikely (status < 0)) {
1453 ehci_dbg (ehci, "can't init itds\n");
1454 goto done;
1455 }
1456
1457 /* schedule ... need to lock */
1458 spin_lock_irqsave (&ehci->lock, flags);
1459 status = iso_stream_schedule (ehci, urb, stream);
1460 if (likely (status == 0))
1461 itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
1462 spin_unlock_irqrestore (&ehci->lock, flags);
1463
1464done:
1465 if (unlikely (status < 0))
1466 iso_stream_put (ehci, stream);
1467 return status;
1468}
1469
1470#ifdef CONFIG_USB_EHCI_SPLIT_ISO
1471
1472/*-------------------------------------------------------------------------*/
1473
1474/*
1475 * "Split ISO TDs" ... used for USB 1.1 devices going through the
1476 * TTs in USB 2.0 hubs. These need microframe scheduling.
1477 */
1478
1479static inline void
1480sitd_sched_init (
1481 struct ehci_iso_sched *iso_sched,
1482 struct ehci_iso_stream *stream,
1483 struct urb *urb
1484)
1485{
1486 unsigned i;
1487 dma_addr_t dma = urb->transfer_dma;
1488
1489 /* how many frames are needed for these transfers */
1490 iso_sched->span = urb->number_of_packets * stream->interval;
1491
1492 /* figure out per-frame sitd fields that we'll need later
1493 * when we fit new sitds into the schedule.
1494 */
1495 for (i = 0; i < urb->number_of_packets; i++) {
1496 struct ehci_iso_packet *packet = &iso_sched->packet [i];
1497 unsigned length;
1498 dma_addr_t buf;
1499 u32 trans;
1500
1501 length = urb->iso_frame_desc [i].length & 0x03ff;
1502 buf = dma + urb->iso_frame_desc [i].offset;
1503
1504 trans = SITD_STS_ACTIVE;
1505 if (((i + 1) == urb->number_of_packets)
1506 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1507 trans |= SITD_IOC;
1508 trans |= length << 16;
1509 packet->transaction = cpu_to_le32 (trans);
1510
1511 /* might need to cross a buffer page within a td */
1512 packet->bufp = buf;
1513 packet->buf1 = (buf + length) & ~0x0fff;
1514 if (packet->buf1 != (buf & ~(u64)0x0fff))
1515 packet->cross = 1;
1516
1517 /* OUT uses multiple start-splits */
1518 if (stream->bEndpointAddress & USB_DIR_IN)
1519 continue;
1520 length = (length + 187) / 188;
1521 if (length > 1) /* BEGIN vs ALL */
1522 length |= 1 << 3;
1523 packet->buf1 |= length;
1524 }
1525}
1526
1527static int
1528sitd_urb_transaction (
1529 struct ehci_iso_stream *stream,
1530 struct ehci_hcd *ehci,
1531 struct urb *urb,
55016f10 1532 gfp_t mem_flags
1da177e4
LT		 1533)
1534{
1535 struct ehci_sitd *sitd;
1536 dma_addr_t sitd_dma;
1537 int i;
1538 struct ehci_iso_sched *iso_sched;
1539 unsigned long flags;
1540
1541 iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1542 if (iso_sched == NULL)
1543 return -ENOMEM;
1544
1545 sitd_sched_init (iso_sched, stream, urb);
1546
1547 /* allocate/init sITDs */
1548 spin_lock_irqsave (&ehci->lock, flags);
1549 for (i = 0; i < urb->number_of_packets; i++) {
1550
1551 /* NOTE: for now, we don't try to handle wraparound cases
1552 * for IN (using sitd->hw_backpointer, like a FSTN), which
1553 * means we never need two sitds for full speed packets.
1554 */
1555
1556 /* free_list.next might be cache-hot ... but maybe
1557 * the HC caches it too. avoid that issue for now.
1558 */
1559
1560 /* prefer previously-allocated sitds */
1561 if (!list_empty(&stream->free_list)) {
1562 sitd = list_entry (stream->free_list.prev,
1563 struct ehci_sitd, sitd_list);
1564 list_del (&sitd->sitd_list);
1565 sitd_dma = sitd->sitd_dma;
1566 } else
1567 sitd = NULL;
1568
1569 if (!sitd) {
1570 spin_unlock_irqrestore (&ehci->lock, flags);
1571 sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags,
1572 &sitd_dma);
1573 spin_lock_irqsave (&ehci->lock, flags);
1574 }
1575
1576 if (!sitd) {
1577 iso_sched_free (stream, iso_sched);
1578 spin_unlock_irqrestore (&ehci->lock, flags);
1579 return -ENOMEM;
1580 }
1581 memset (sitd, 0, sizeof *sitd);
1582 sitd->sitd_dma = sitd_dma;
1583 list_add (&sitd->sitd_list, &iso_sched->td_list);
1584 }
1585
1586 /* temporarily store schedule info in hcpriv */
1587 urb->hcpriv = iso_sched;
1588 urb->error_count = 0;
1589
1590 spin_unlock_irqrestore (&ehci->lock, flags);
1591 return 0;
1592}
1593
1594/*-------------------------------------------------------------------------*/
1595
1596static inline void
1597sitd_patch (
1598 struct ehci_iso_stream *stream,
1599 struct ehci_sitd *sitd,
1600 struct ehci_iso_sched *iso_sched,
1601 unsigned index
1602)
1603{
1604 struct ehci_iso_packet *uf = &iso_sched->packet [index];
1605 u64 bufp = uf->bufp;
1606
1607 sitd->hw_next = EHCI_LIST_END;
1608 sitd->hw_fullspeed_ep = stream->address;
1609 sitd->hw_uframe = stream->splits;
1610 sitd->hw_results = uf->transaction;
1611 sitd->hw_backpointer = EHCI_LIST_END;
1612
1613 bufp = uf->bufp;
1614 sitd->hw_buf [0] = cpu_to_le32 (bufp);
1615 sitd->hw_buf_hi [0] = cpu_to_le32 (bufp >> 32);
1616
1617 sitd->hw_buf [1] = cpu_to_le32 (uf->buf1);
1618 if (uf->cross)
1619 bufp += 4096;
1620 sitd->hw_buf_hi [1] = cpu_to_le32 (bufp >> 32);
1621 sitd->index = index;
1622}
1623
1624static inline void
1625sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
1626{
1627 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */
1628 sitd->sitd_next = ehci->pshadow [frame];
1629 sitd->hw_next = ehci->periodic [frame];
1630 ehci->pshadow [frame].sitd = sitd;
1631 sitd->frame = frame;
1632 wmb ();
1633 ehci->periodic [frame] = cpu_to_le32 (sitd->sitd_dma) | Q_TYPE_SITD;
1634}
1635
1636/* fit urb's sitds into the selected schedule slot; activate as needed */
1637static int
1638sitd_link_urb (
1639 struct ehci_hcd *ehci,
1640 struct urb *urb,
1641 unsigned mod,
1642 struct ehci_iso_stream *stream
1643)
1644{
1645 int packet;
1646 unsigned next_uframe;
1647 struct ehci_iso_sched *sched = urb->hcpriv;
1648 struct ehci_sitd *sitd;
1649
1650 next_uframe = stream->next_uframe;
1651
1652 if (list_empty(&stream->td_list)) {
1653 /* usbfs ignores TT bandwidth */
1654 ehci_to_hcd(ehci)->self.bandwidth_allocated
1655 += stream->bandwidth;
1656 ehci_vdbg (ehci,
1657 "sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
1658 urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1659 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1660 (next_uframe >> 3) % ehci->periodic_size,
1661 stream->interval, le32_to_cpu (stream->splits));
1662 stream->start = jiffies;
1663 }
1664 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1665
1666 /* fill sITDs frame by frame */
1667 for (packet = 0, sitd = NULL;
1668 packet < urb->number_of_packets;
1669 packet++) {
1670
1671 /* ASSERT: we have all necessary sitds */
1672 BUG_ON (list_empty (&sched->td_list));
1673
1674 /* ASSERT: no itds for this endpoint in this frame */
1675
1676 sitd = list_entry (sched->td_list.next,
1677 struct ehci_sitd, sitd_list);
1678 list_move_tail (&sitd->sitd_list, &stream->td_list);
1679 sitd->stream = iso_stream_get (stream);
1680 sitd->urb = usb_get_urb (urb);
1681
1682 sitd_patch (stream, sitd, sched, packet);
1683 sitd_link (ehci, (next_uframe >> 3) % ehci->periodic_size,
1684 sitd);
1685
1686 next_uframe += stream->interval << 3;
1687 stream->depth += stream->interval << 3;
1688 }
1689 stream->next_uframe = next_uframe % mod;
1690
1691 /* don't need that schedule data any more */
1692 iso_sched_free (stream, sched);
1693 urb->hcpriv = NULL;
1694
1695 timer_action (ehci, TIMER_IO_WATCHDOG);
1696 if (!ehci->periodic_sched++)
1697 return enable_periodic (ehci);
1698 return 0;
1699}
1700
1701/*-------------------------------------------------------------------------*/
1702
1703#define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
1704 | SITD_STS_XACT | SITD_STS_MMF)
1705
1706static unsigned
1707sitd_complete (
1708 struct ehci_hcd *ehci,
1709 struct ehci_sitd *sitd,
1710 struct pt_regs *regs
1711) {
1712 struct urb *urb = sitd->urb;
1713 struct usb_iso_packet_descriptor *desc;
1714 u32 t;
1715 int urb_index = -1;
1716 struct ehci_iso_stream *stream = sitd->stream;
1717 struct usb_device *dev;
1718
1719 urb_index = sitd->index;
1720 desc = &urb->iso_frame_desc [urb_index];
1721 t = le32_to_cpup (&sitd->hw_results);
1722
1723 /* report transfer status */
1724 if (t & SITD_ERRS) {
1725 urb->error_count++;
1726 if (t & SITD_STS_DBE)
1727 desc->status = usb_pipein (urb->pipe)
1728 ? -ENOSR /* hc couldn't read */
1729 : -ECOMM; /* hc couldn't write */
1730 else if (t & SITD_STS_BABBLE)
1731 desc->status = -EOVERFLOW;
1732 else /* XACT, MMF, etc */
1733 desc->status = -EPROTO;
1734 } else {
1735 desc->status = 0;
1736 desc->actual_length = desc->length - SITD_LENGTH (t);
1737 }
1738
1739 usb_put_urb (urb);
1740 sitd->urb = NULL;
1741 sitd->stream = NULL;
1742 list_move (&sitd->sitd_list, &stream->free_list);
1743 stream->depth -= stream->interval << 3;
1744 iso_stream_put (ehci, stream);
1745
1746 /* handle completion now? */
1747 if ((urb_index + 1) != urb->number_of_packets)
1748 return 0;
1749
1750 /* ASSERT: it's really the last sitd for this urb
1751 list_for_each_entry (sitd, &stream->td_list, sitd_list)
1752 BUG_ON (sitd->urb == urb);
1753 */
1754
1755 /* give urb back to the driver */
1756 dev = usb_get_dev (urb->dev);
1757 ehci_urb_done (ehci, urb, regs);
1758 urb = NULL;
1759
1760 /* defer stopping schedule; completion can submit */
1761 ehci->periodic_sched--;
1762 if (!ehci->periodic_sched)
1763 (void) disable_periodic (ehci);
1764 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1765
1766 if (list_empty (&stream->td_list)) {
1767 ehci_to_hcd(ehci)->self.bandwidth_allocated
1768 -= stream->bandwidth;
1769 ehci_vdbg (ehci,
1770 "deschedule devp %s ep%d%s-iso\n",
1771 dev->devpath, stream->bEndpointAddress & 0x0f,
1772 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
1773 }
1774 iso_stream_put (ehci, stream);
1775 usb_put_dev (dev);
1776
1777 return 1;
1778}
1779
1780
5db539e4 1781static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
55016f10 1782 gfp_t mem_flags)
1da177e4
LT		 1783{
1784 int status = -EINVAL;
1785 unsigned long flags;
1786 struct ehci_iso_stream *stream;
1787
1788 /* Get iso_stream head */
1789 stream = iso_stream_find (ehci, urb);
1790 if (stream == NULL) {
1791 ehci_dbg (ehci, "can't get iso stream\n");
1792 return -ENOMEM;
1793 }
1794 if (urb->interval != stream->interval) {
1795 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
1796 stream->interval, urb->interval);
1797 goto done;
1798 }
1799
1800#ifdef EHCI_URB_TRACE
1801 ehci_dbg (ehci,
1802 "submit %p dev%s ep%d%s-iso len %d\n",
1803 urb, urb->dev->devpath,
1804 usb_pipeendpoint (urb->pipe),
1805 usb_pipein (urb->pipe) ? "in" : "out",
1806 urb->transfer_buffer_length);
1807#endif
1808
1809 /* allocate SITDs */
1810 status = sitd_urb_transaction (stream, ehci, urb, mem_flags);
1811 if (status < 0) {
1812 ehci_dbg (ehci, "can't init sitds\n");
1813 goto done;
1814 }
1815
1816 /* schedule ... need to lock */
1817 spin_lock_irqsave (&ehci->lock, flags);
1818 status = iso_stream_schedule (ehci, urb, stream);
1819 if (status == 0)
1820 sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
1821 spin_unlock_irqrestore (&ehci->lock, flags);
1822
1823done:
1824 if (status < 0)
1825 iso_stream_put (ehci, stream);
1826 return status;
1827}
1828
1829#else
1830
1831static inline int
5db539e4
OK		 1832sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
1833 unsigned mem_flags)
1da177e4
LT		 1834{
1835 ehci_dbg (ehci, "split iso support is disabled\n");
1836 return -ENOSYS;
1837}
1838
1839static inline unsigned
1840sitd_complete (
1841 struct ehci_hcd *ehci,
1842 struct ehci_sitd *sitd,
1843 struct pt_regs *regs
1844) {
1845 ehci_err (ehci, "sitd_complete %p?\n", sitd);
1846 return 0;
1847}
1848
1849#endif /* USB_EHCI_SPLIT_ISO */
1850
1851/*-------------------------------------------------------------------------*/
1852
1853static void
1854scan_periodic (struct ehci_hcd *ehci, struct pt_regs *regs)
1855{
1856 unsigned frame, clock, now_uframe, mod;
1857 unsigned modified;
1858
1859 mod = ehci->periodic_size << 3;
1860
1861 /*
1862 * When running, scan from last scan point up to "now"
1863 * else clean up by scanning everything that's left.
1864 * Touches as few pages as possible: cache-friendly.
1865 */
1866 now_uframe = ehci->next_uframe;
1867 if (HC_IS_RUNNING (ehci_to_hcd(ehci)->state))
1868 clock = readl (&ehci->regs->frame_index);
1869 else
1870 clock = now_uframe + mod - 1;
1871 clock %= mod;
1872
1873 for (;;) {
1874 union ehci_shadow q, *q_p;
1875 __le32 type, *hw_p;
1876 unsigned uframes;
1877
1878 /* don't scan past the live uframe */
1879 frame = now_uframe >> 3;
1880 if (frame == (clock >> 3))
1881 uframes = now_uframe & 0x07;
1882 else {
1883 /* safe to scan the whole frame at once */
1884 now_uframe |= 0x07;
1885 uframes = 8;
1886 }
1887
1888restart:
1889 /* scan each element in frame's queue for completions */
1890 q_p = &ehci->pshadow [frame];
1891 hw_p = &ehci->periodic [frame];
1892 q.ptr = q_p->ptr;
1893 type = Q_NEXT_TYPE (*hw_p);
1894 modified = 0;
1895
1896 while (q.ptr != NULL) {
1897 unsigned uf;
1898 union ehci_shadow temp;
1899 int live;
1900
1901 live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state);
1902 switch (type) {
1903 case Q_TYPE_QH:
1904 /* handle any completions */
1905 temp.qh = qh_get (q.qh);
1906 type = Q_NEXT_TYPE (q.qh->hw_next);
1907 q = q.qh->qh_next;
1908 modified = qh_completions (ehci, temp.qh, regs);
1909 if (unlikely (list_empty (&temp.qh->qtd_list)))
1910 intr_deschedule (ehci, temp.qh);
1911 qh_put (temp.qh);
1912 break;
1913 case Q_TYPE_FSTN:
1914 /* for "save place" FSTNs, look at QH entries
1915 * in the previous frame for completions.
1916 */
1917 if (q.fstn->hw_prev != EHCI_LIST_END) {
1918 dbg ("ignoring completions from FSTNs");
1919 }
1920 type = Q_NEXT_TYPE (q.fstn->hw_next);
1921 q = q.fstn->fstn_next;
1922 break;
1923 case Q_TYPE_ITD:
1924 /* skip itds for later in the frame */
1925 rmb ();
1926 for (uf = live ? uframes : 8; uf < 8; uf++) {
1927 if (0 == (q.itd->hw_transaction [uf]
1928 & ITD_ACTIVE))
1929 continue;
1930 q_p = &q.itd->itd_next;
1931 hw_p = &q.itd->hw_next;
1932 type = Q_NEXT_TYPE (q.itd->hw_next);
1933 q = *q_p;
1934 break;
1935 }
1936 if (uf != 8)
1937 break;
1938
1939 /* this one's ready ... HC won't cache the
1940 * pointer for much longer, if at all.
1941 */
1942 *q_p = q.itd->itd_next;
1943 *hw_p = q.itd->hw_next;
1944 type = Q_NEXT_TYPE (q.itd->hw_next);
1945 wmb();
1946 modified = itd_complete (ehci, q.itd, regs);
1947 q = *q_p;
1948 break;
1949 case Q_TYPE_SITD:
1950 if ((q.sitd->hw_results & SITD_ACTIVE)
1951 && live) {
1952 q_p = &q.sitd->sitd_next;
1953 hw_p = &q.sitd->hw_next;
1954 type = Q_NEXT_TYPE (q.sitd->hw_next);
1955 q = *q_p;
1956 break;
1957 }
1958 *q_p = q.sitd->sitd_next;
1959 *hw_p = q.sitd->hw_next;
1960 type = Q_NEXT_TYPE (q.sitd->hw_next);
1961 wmb();
1962 modified = sitd_complete (ehci, q.sitd, regs);
1963 q = *q_p;
1964 break;
1965 default:
1966 dbg ("corrupt type %d frame %d shadow %p",
1967 type, frame, q.ptr);
1968 // BUG ();
1969 q.ptr = NULL;
1970 }
1971
1972 /* assume completion callbacks modify the queue */
1973 if (unlikely (modified))
1974 goto restart;
1975 }
1976
1977 /* stop when we catch up to the HC */
1978
1979 // FIXME: this assumes we won't get lapped when
1980 // latencies climb; that should be rare, but...
1981 // detect it, and just go all the way around.
1982 // FLR might help detect this case, so long as latencies
1983 // don't exceed periodic_size msec (default 1.024 sec).
1984
1985 // FIXME: likewise assumes HC doesn't halt mid-scan
1986
1987 if (now_uframe == clock) {
1988 unsigned now;
1989
1990 if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state))
1991 break;
1992 ehci->next_uframe = now_uframe;
1993 now = readl (&ehci->regs->frame_index) % mod;
1994 if (now_uframe == now)
1995 break;
1996
1997 /* rescan the rest of this frame, then ... */
1998 clock = now;
1999 } else {
2000 now_uframe++;
2001 now_uframe %= mod;
2002 }
2003 }
2004}
Linux kernel - Deliverables
This page took 0.141218 seconds and 5 git commands to generate.