projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[PATCH] USB: ub documentation update
[deliverable/linux.git] / drivers / usb / host / ehci-sched.c
CommitLineData
1da177e4
LT		 1/*
2 * Copyright (c) 2001-2004 by David Brownell
3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the
7 * Free Software Foundation; either version 2 of the License, or (at your
8 * option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software Foundation,
17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 */
19
20/* this file is part of ehci-hcd.c */
21
22/*-------------------------------------------------------------------------*/
23
24/*
25 * EHCI scheduled transaction support: interrupt, iso, split iso
26 * These are called "periodic" transactions in the EHCI spec.
27 *
28 * Note that for interrupt transfers, the QH/QTD manipulation is shared
29 * with the "asynchronous" transaction support (control/bulk transfers).
30 * The only real difference is in how interrupt transfers are scheduled.
31 *
32 * For ISO, we make an "iso_stream" head to serve the same role as a QH.
33 * It keeps track of every ITD (or SITD) that's linked, and holds enough
34 * pre-calculated schedule data to make appending to the queue be quick.
35 */
36
37static int ehci_get_frame (struct usb_hcd *hcd);
38
39/*-------------------------------------------------------------------------*/
40
41/*
42 * periodic_next_shadow - return "next" pointer on shadow list
43 * @periodic: host pointer to qh/itd/sitd
44 * @tag: hardware tag for type of this record
45 */
46static union ehci_shadow *
47periodic_next_shadow (union ehci_shadow *periodic, __le32 tag)
48{
49 switch (tag) {
50 case Q_TYPE_QH:
51 return &periodic->qh->qh_next;
52 case Q_TYPE_FSTN:
53 return &periodic->fstn->fstn_next;
54 case Q_TYPE_ITD:
55 return &periodic->itd->itd_next;
56 // case Q_TYPE_SITD:
57 default:
58 return &periodic->sitd->sitd_next;
59 }
60}
61
62/* caller must hold ehci->lock */
63static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr)
64{
65 union ehci_shadow *prev_p = &ehci->pshadow [frame];
66 __le32 *hw_p = &ehci->periodic [frame];
67 union ehci_shadow here = *prev_p;
68
69 /* find predecessor of "ptr"; hw and shadow lists are in sync */
70 while (here.ptr && here.ptr != ptr) {
71 prev_p = periodic_next_shadow (prev_p, Q_NEXT_TYPE (*hw_p));
72 hw_p = here.hw_next;
73 here = *prev_p;
74 }
75 /* an interrupt entry (at list end) could have been shared */
76 if (!here.ptr)
77 return;
78
79 /* update shadow and hardware lists ... the old "next" pointers
80 * from ptr may still be in use, the caller updates them.
81 */
82 *prev_p = *periodic_next_shadow (&here, Q_NEXT_TYPE (*hw_p));
83 *hw_p = *here.hw_next;
84}
85
86/* how many of the uframe's 125 usecs are allocated? */
87static unsigned short
88periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe)
89{
90 __le32 *hw_p = &ehci->periodic [frame];
91 union ehci_shadow *q = &ehci->pshadow [frame];
92 unsigned usecs = 0;
93
94 while (q->ptr) {
95 switch (Q_NEXT_TYPE (*hw_p)) {
96 case Q_TYPE_QH:
97 /* is it in the S-mask? */
98 if (q->qh->hw_info2 & cpu_to_le32 (1 << uframe))
99 usecs += q->qh->usecs;
100 /* ... or C-mask? */
101 if (q->qh->hw_info2 & cpu_to_le32 (1 << (8 + uframe)))
102 usecs += q->qh->c_usecs;
103 hw_p = &q->qh->hw_next;
104 q = &q->qh->qh_next;
105 break;
106 // case Q_TYPE_FSTN:
107 default:
108 /* for "save place" FSTNs, count the relevant INTR
109 * bandwidth from the previous frame
110 */
111 if (q->fstn->hw_prev != EHCI_LIST_END) {
112 ehci_dbg (ehci, "ignoring FSTN cost ...\n");
113 }
114 hw_p = &q->fstn->hw_next;
115 q = &q->fstn->fstn_next;
116 break;
117 case Q_TYPE_ITD:
118 usecs += q->itd->usecs [uframe];
119 hw_p = &q->itd->hw_next;
120 q = &q->itd->itd_next;
121 break;
122 case Q_TYPE_SITD:
123 /* is it in the S-mask? (count SPLIT, DATA) */
124 if (q->sitd->hw_uframe & cpu_to_le32 (1 << uframe)) {
125 if (q->sitd->hw_fullspeed_ep &
126 __constant_cpu_to_le32 (1<<31))
127 usecs += q->sitd->stream->usecs;
128 else /* worst case for OUT start-split */
129 usecs += HS_USECS_ISO (188);
130 }
131
132 /* ... C-mask? (count CSPLIT, DATA) */
133 if (q->sitd->hw_uframe &
134 cpu_to_le32 (1 << (8 + uframe))) {
135 /* worst case for IN complete-split */
136 usecs += q->sitd->stream->c_usecs;
137 }
138
139 hw_p = &q->sitd->hw_next;
140 q = &q->sitd->sitd_next;
141 break;
142 }
143 }
144#ifdef DEBUG
145 if (usecs > 100)
146 ehci_err (ehci, "uframe %d sched overrun: %d usecs\n",
147 frame * 8 + uframe, usecs);
148#endif
149 return usecs;
150}
151
152/*-------------------------------------------------------------------------*/
153
154static int same_tt (struct usb_device *dev1, struct usb_device *dev2)
155{
156 if (!dev1->tt || !dev2->tt)
157 return 0;
158 if (dev1->tt != dev2->tt)
159 return 0;
160 if (dev1->tt->multi)
161 return dev1->ttport == dev2->ttport;
162 else
163 return 1;
164}
165
166/* return true iff the device's transaction translator is available
167 * for a periodic transfer starting at the specified frame, using
168 * all the uframes in the mask.
169 */
170static int tt_no_collision (
171 struct ehci_hcd *ehci,
172 unsigned period,
173 struct usb_device *dev,
174 unsigned frame,
175 u32 uf_mask
176)
177{
178 if (period == 0) /* error */
179 return 0;
180
181 /* note bandwidth wastage: split never follows csplit
182 * (different dev or endpoint) until the next uframe.
183 * calling convention doesn't make that distinction.
184 */
185 for (; frame < ehci->periodic_size; frame += period) {
186 union ehci_shadow here;
187 __le32 type;
188
189 here = ehci->pshadow [frame];
190 type = Q_NEXT_TYPE (ehci->periodic [frame]);
191 while (here.ptr) {
192 switch (type) {
193 case Q_TYPE_ITD:
194 type = Q_NEXT_TYPE (here.itd->hw_next);
195 here = here.itd->itd_next;
196 continue;
197 case Q_TYPE_QH:
198 if (same_tt (dev, here.qh->dev)) {
199 u32 mask;
200
201 mask = le32_to_cpu (here.qh->hw_info2);
202 /* "knows" no gap is needed */
203 mask |= mask >> 8;
204 if (mask & uf_mask)
205 break;
206 }
207 type = Q_NEXT_TYPE (here.qh->hw_next);
208 here = here.qh->qh_next;
209 continue;
210 case Q_TYPE_SITD:
211 if (same_tt (dev, here.sitd->urb->dev)) {
212 u16 mask;
213
214 mask = le32_to_cpu (here.sitd
215 ->hw_uframe);
216 /* FIXME assumes no gap for IN! */
217 mask |= mask >> 8;
218 if (mask & uf_mask)
219 break;
220 }
221 type = Q_NEXT_TYPE (here.sitd->hw_next);
222 here = here.sitd->sitd_next;
223 continue;
224 // case Q_TYPE_FSTN:
225 default:
226 ehci_dbg (ehci,
227 "periodic frame %d bogus type %d\n",
228 frame, type);
229 }
230
231 /* collision or error */
232 return 0;
233 }
234 }
235
236 /* no collision */
237 return 1;
238}
239
240/*-------------------------------------------------------------------------*/
241
242static int enable_periodic (struct ehci_hcd *ehci)
243{
244 u32 cmd;
245 int status;
246
247 /* did clearing PSE did take effect yet?
248 * takes effect only at frame boundaries...
249 */
250 status = handshake (&ehci->regs->status, STS_PSS, 0, 9 * 125);
251 if (status != 0) {
252 ehci_to_hcd(ehci)->state = HC_STATE_HALT;
253 return status;
254 }
255
256 cmd = readl (&ehci->regs->command) | CMD_PSE;
257 writel (cmd, &ehci->regs->command);
258 /* posted write ... PSS happens later */
259 ehci_to_hcd(ehci)->state = HC_STATE_RUNNING;
260
261 /* make sure ehci_work scans these */
262 ehci->next_uframe = readl (&ehci->regs->frame_index)
263 % (ehci->periodic_size << 3);
264 return 0;
265}
266
267static int disable_periodic (struct ehci_hcd *ehci)
268{
269 u32 cmd;
270 int status;
271
272 /* did setting PSE not take effect yet?
273 * takes effect only at frame boundaries...
274 */
275 status = handshake (&ehci->regs->status, STS_PSS, STS_PSS, 9 * 125);
276 if (status != 0) {
277 ehci_to_hcd(ehci)->state = HC_STATE_HALT;
278 return status;
279 }
280
281 cmd = readl (&ehci->regs->command) & ~CMD_PSE;
282 writel (cmd, &ehci->regs->command);
283 /* posted write ... */
284
285 ehci->next_uframe = -1;
286 return 0;
287}
288
289/*-------------------------------------------------------------------------*/
290
291/* periodic schedule slots have iso tds (normal or split) first, then a
292 * sparse tree for active interrupt transfers.
293 *
294 * this just links in a qh; caller guarantees uframe masks are set right.
295 * no FSTN support (yet; ehci 0.96+)
296 */
297static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
298{
299 unsigned i;
300 unsigned period = qh->period;
301
302 dev_dbg (&qh->dev->dev,
303 "link qh%d-%04x/%p start %d [%d/%d us]\n",
304 period, le32_to_cpup (&qh->hw_info2) & 0xffff,
305 qh, qh->start, qh->usecs, qh->c_usecs);
306
307 /* high bandwidth, or otherwise every microframe */
308 if (period == 0)
309 period = 1;
310
311 for (i = qh->start; i < ehci->periodic_size; i += period) {
312 union ehci_shadow *prev = &ehci->pshadow [i];
9a5d3e98 313 __le32 *hw_p = &ehci->periodic [i];
1da177e4 314 union ehci_shadow here = *prev;
9a5d3e98 315 __le32 type = 0;
1da177e4
LT		 316
317 /* skip the iso nodes at list head */
318 while (here.ptr) {
319 type = Q_NEXT_TYPE (*hw_p);
320 if (type == Q_TYPE_QH)
321 break;
322 prev = periodic_next_shadow (prev, type);
323 hw_p = &here.qh->hw_next;
324 here = *prev;
325 }
326
327 /* sorting each branch by period (slow-->fast)
328 * enables sharing interior tree nodes
329 */
330 while (here.ptr && qh != here.qh) {
331 if (qh->period > here.qh->period)
332 break;
333 prev = &here.qh->qh_next;
334 hw_p = &here.qh->hw_next;
335 here = *prev;
336 }
337 /* link in this qh, unless some earlier pass did that */
338 if (qh != here.qh) {
339 qh->qh_next = here;
340 if (here.qh)
341 qh->hw_next = *hw_p;
342 wmb ();
343 prev->qh = qh;
344 *hw_p = QH_NEXT (qh->qh_dma);
345 }
346 }
347 qh->qh_state = QH_STATE_LINKED;
348 qh_get (qh);
349
350 /* update per-qh bandwidth for usbfs */
351 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period
352 ? ((qh->usecs + qh->c_usecs) / qh->period)
353 : (qh->usecs * 8);
354
355 /* maybe enable periodic schedule processing */
356 if (!ehci->periodic_sched++)
357 return enable_periodic (ehci);
358
359 return 0;
360}
361
362static void qh_unlink_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
363{
364 unsigned i;
365 unsigned period;
366
367 // FIXME:
368 // IF this isn't high speed
369 // and this qh is active in the current uframe
370 // (and overlay token SplitXstate is false?)
371 // THEN
372 // qh->hw_info1 |= __constant_cpu_to_le32 (1 << 7 /* "ignore" */);
373
374 /* high bandwidth, or otherwise part of every microframe */
375 if ((period = qh->period) == 0)
376 period = 1;
377
378 for (i = qh->start; i < ehci->periodic_size; i += period)
379 periodic_unlink (ehci, i, qh);
380
381 /* update per-qh bandwidth for usbfs */
382 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period
383 ? ((qh->usecs + qh->c_usecs) / qh->period)
384 : (qh->usecs * 8);
385
386 dev_dbg (&qh->dev->dev,
387 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
388 qh->period, le32_to_cpup (&qh->hw_info2) & 0xffff,
389 qh, qh->start, qh->usecs, qh->c_usecs);
390
391 /* qh->qh_next still "live" to HC */
392 qh->qh_state = QH_STATE_UNLINK;
393 qh->qh_next.ptr = NULL;
394 qh_put (qh);
395
396 /* maybe turn off periodic schedule */
397 ehci->periodic_sched--;
398 if (!ehci->periodic_sched)
399 (void) disable_periodic (ehci);
400}
401
402static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
403{
404 unsigned wait;
405
406 qh_unlink_periodic (ehci, qh);
407
408 /* simple/paranoid: always delay, expecting the HC needs to read
409 * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and
410 * expect khubd to clean up after any CSPLITs we won't issue.
411 * active high speed queues may need bigger delays...
412 */
413 if (list_empty (&qh->qtd_list)
414 || (__constant_cpu_to_le32 (0x0ff << 8)
415 & qh->hw_info2) != 0)
416 wait = 2;
417 else
418 wait = 55; /* worst case: 3 * 1024 */
419
420 udelay (wait);
421 qh->qh_state = QH_STATE_IDLE;
422 qh->hw_next = EHCI_LIST_END;
423 wmb ();
424}
425
426/*-------------------------------------------------------------------------*/
427
428static int check_period (
429 struct ehci_hcd *ehci,
430 unsigned frame,
431 unsigned uframe,
432 unsigned period,
433 unsigned usecs
434) {
435 int claimed;
436
437 /* complete split running into next frame?
438 * given FSTN support, we could sometimes check...
439 */
440 if (uframe >= 8)
441 return 0;
442
443 /*
444 * 80% periodic == 100 usec/uframe available
445 * convert "usecs we need" to "max already claimed"
446 */
447 usecs = 100 - usecs;
448
449 /* we "know" 2 and 4 uframe intervals were rejected; so
450 * for period 0, check _every_ microframe in the schedule.
451 */
452 if (unlikely (period == 0)) {
453 do {
454 for (uframe = 0; uframe < 7; uframe++) {
455 claimed = periodic_usecs (ehci, frame, uframe);
456 if (claimed > usecs)
457 return 0;
458 }
459 } while ((frame += 1) < ehci->periodic_size);
460
461 /* just check the specified uframe, at that period */
462 } else {
463 do {
464 claimed = periodic_usecs (ehci, frame, uframe);
465 if (claimed > usecs)
466 return 0;
467 } while ((frame += period) < ehci->periodic_size);
468 }
469
470 // success!
471 return 1;
472}
473
474static int check_intr_schedule (
475 struct ehci_hcd *ehci,
476 unsigned frame,
477 unsigned uframe,
478 const struct ehci_qh *qh,
479 __le32 *c_maskp
480)
481{
482 int retval = -ENOSPC;
483 u8 mask;
484
485 if (qh->c_usecs && uframe >= 6) /* FSTN territory? */
486 goto done;
487
488 if (!check_period (ehci, frame, uframe, qh->period, qh->usecs))
489 goto done;
490 if (!qh->c_usecs) {
491 retval = 0;
492 *c_maskp = 0;
493 goto done;
494 }
495
496 /* Make sure this tt's buffer is also available for CSPLITs.
497 * We pessimize a bit; probably the typical full speed case
498 * doesn't need the second CSPLIT.
499 *
500 * NOTE: both SPLIT and CSPLIT could be checked in just
501 * one smart pass...
502 */
503 mask = 0x03 << (uframe + qh->gap_uf);
504 *c_maskp = cpu_to_le32 (mask << 8);
505
506 mask |= 1 << uframe;
507 if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) {
508 if (!check_period (ehci, frame, uframe + qh->gap_uf + 1,
509 qh->period, qh->c_usecs))
510 goto done;
511 if (!check_period (ehci, frame, uframe + qh->gap_uf,
512 qh->period, qh->c_usecs))
513 goto done;
514 retval = 0;
515 }
516done:
517 return retval;
518}
519
520/* "first fit" scheduling policy used the first time through,
521 * or when the previous schedule slot can't be re-used.
522 */
523static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
524{
525 int status;
526 unsigned uframe;
527 __le32 c_mask;
528 unsigned frame; /* 0..(qh->period - 1), or NO_FRAME */
529
530 qh_refresh(ehci, qh);
531 qh->hw_next = EHCI_LIST_END;
532 frame = qh->start;
533
534 /* reuse the previous schedule slots, if we can */
535 if (frame < qh->period) {
536 uframe = ffs (le32_to_cpup (&qh->hw_info2) & 0x00ff);
537 status = check_intr_schedule (ehci, frame, --uframe,
538 qh, &c_mask);
539 } else {
540 uframe = 0;
541 c_mask = 0;
542 status = -ENOSPC;
543 }
544
545 /* else scan the schedule to find a group of slots such that all
546 * uframes have enough periodic bandwidth available.
547 */
548 if (status) {
549 /* "normal" case, uframing flexible except with splits */
550 if (qh->period) {
551 frame = qh->period - 1;
552 do {
553 for (uframe = 0; uframe < 8; uframe++) {
554 status = check_intr_schedule (ehci,
555 frame, uframe, qh,
556 &c_mask);
557 if (status == 0)
558 break;
559 }
560 } while (status && frame--);
561
562 /* qh->period == 0 means every uframe */
563 } else {
564 frame = 0;
565 status = check_intr_schedule (ehci, 0, 0, qh, &c_mask);
566 }
567 if (status)
568 goto done;
569 qh->start = frame;
570
571 /* reset S-frame and (maybe) C-frame masks */
572 qh->hw_info2 &= __constant_cpu_to_le32 (~0xffff);
573 qh->hw_info2 |= qh->period
574 ? cpu_to_le32 (1 << uframe)
575 : __constant_cpu_to_le32 (0xff);
576 qh->hw_info2 |= c_mask;
577 } else
578 ehci_dbg (ehci, "reused qh %p schedule\n", qh);
579
580 /* stuff into the periodic schedule */
581 status = qh_link_periodic (ehci, qh);
582done:
583 return status;
584}
585
586static int intr_submit (
587 struct ehci_hcd *ehci,
588 struct usb_host_endpoint *ep,
589 struct urb *urb,
590 struct list_head *qtd_list,
5db539e4 591 unsigned mem_flags
1da177e4
LT		 592) {
593 unsigned epnum;
594 unsigned long flags;
595 struct ehci_qh *qh;
596 int status = 0;
597 struct list_head empty;
598
599 /* get endpoint and transfer/schedule data */
600 epnum = ep->desc.bEndpointAddress;
601
602 spin_lock_irqsave (&ehci->lock, flags);
603
604 /* get qh and force any scheduling errors */
605 INIT_LIST_HEAD (&empty);
606 qh = qh_append_tds (ehci, urb, &empty, epnum, &ep->hcpriv);
607 if (qh == NULL) {
608 status = -ENOMEM;
609 goto done;
610 }
611 if (qh->qh_state == QH_STATE_IDLE) {
612 if ((status = qh_schedule (ehci, qh)) != 0)
613 goto done;
614 }
615
616 /* then queue the urb's tds to the qh */
617 qh = qh_append_tds (ehci, urb, qtd_list, epnum, &ep->hcpriv);
618 BUG_ON (qh == NULL);
619
620 /* ... update usbfs periodic stats */
621 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
622
623done:
624 spin_unlock_irqrestore (&ehci->lock, flags);
625 if (status)
626 qtd_list_free (ehci, urb, qtd_list);
627
628 return status;
629}
630
631/*-------------------------------------------------------------------------*/
632
633/* ehci_iso_stream ops work with both ITD and SITD */
634
635static struct ehci_iso_stream *
5db539e4 636iso_stream_alloc (unsigned mem_flags)
1da177e4
LT		 637{
638 struct ehci_iso_stream *stream;
639
77078570 640 stream = kcalloc(1, sizeof *stream, mem_flags);
1da177e4 641 if (likely (stream != NULL)) {
1da177e4
LT		 642 INIT_LIST_HEAD(&stream->td_list);
643 INIT_LIST_HEAD(&stream->free_list);
644 stream->next_uframe = -1;
645 stream->refcount = 1;
646 }
647 return stream;
648}
649
650static void
651iso_stream_init (
652 struct ehci_hcd *ehci,
653 struct ehci_iso_stream *stream,
654 struct usb_device *dev,
655 int pipe,
656 unsigned interval
657)
658{
659 static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
660
661 u32 buf1;
662 unsigned epnum, maxp;
663 int is_input;
664 long bandwidth;
665
666 /*
667 * this might be a "high bandwidth" highspeed endpoint,
668 * as encoded in the ep descriptor's wMaxPacket field
669 */
670 epnum = usb_pipeendpoint (pipe);
671 is_input = usb_pipein (pipe) ? USB_DIR_IN : 0;
672 maxp = usb_maxpacket(dev, pipe, !is_input);
673 if (is_input) {
674 buf1 = (1 << 11);
675 } else {
676 buf1 = 0;
677 }
678
679 /* knows about ITD vs SITD */
680 if (dev->speed == USB_SPEED_HIGH) {
681 unsigned multi = hb_mult(maxp);
682
683 stream->highspeed = 1;
684
685 maxp = max_packet(maxp);
686 buf1 |= maxp;
687 maxp *= multi;
688
689 stream->buf0 = cpu_to_le32 ((epnum << 8) | dev->devnum);
690 stream->buf1 = cpu_to_le32 (buf1);
691 stream->buf2 = cpu_to_le32 (multi);
692
693 /* usbfs wants to report the average usecs per frame tied up
694 * when transfers on this endpoint are scheduled ...
695 */
696 stream->usecs = HS_USECS_ISO (maxp);
697 bandwidth = stream->usecs * 8;
698 bandwidth /= 1 << (interval - 1);
699
700 } else {
701 u32 addr;
702
703 addr = dev->ttport << 24;
704 if (!ehci_is_TDI(ehci)
705 || (dev->tt->hub !=
706 ehci_to_hcd(ehci)->self.root_hub))
707 addr |= dev->tt->hub->devnum << 16;
708 addr |= epnum << 8;
709 addr |= dev->devnum;
710 stream->usecs = HS_USECS_ISO (maxp);
711 if (is_input) {
712 u32 tmp;
713
714 addr |= 1 << 31;
715 stream->c_usecs = stream->usecs;
716 stream->usecs = HS_USECS_ISO (1);
717 stream->raw_mask = 1;
718
719 /* pessimistic c-mask */
720 tmp = usb_calc_bus_time (USB_SPEED_FULL, 1, 0, maxp)
721 / (125 * 1000);
722 stream->raw_mask |= 3 << (tmp + 9);
723 } else
724 stream->raw_mask = smask_out [maxp / 188];
725 bandwidth = stream->usecs + stream->c_usecs;
726 bandwidth /= 1 << (interval + 2);
727
728 /* stream->splits gets created from raw_mask later */
729 stream->address = cpu_to_le32 (addr);
730 }
731 stream->bandwidth = bandwidth;
732
733 stream->udev = dev;
734
735 stream->bEndpointAddress = is_input | epnum;
736 stream->interval = interval;
737 stream->maxp = maxp;
738}
739
740static void
741iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream)
742{
743 stream->refcount--;
744
745 /* free whenever just a dev->ep reference remains.
746 * not like a QH -- no persistent state (toggle, halt)
747 */
748 if (stream->refcount == 1) {
749 int is_in;
750
751 // BUG_ON (!list_empty(&stream->td_list));
752
753 while (!list_empty (&stream->free_list)) {
754 struct list_head *entry;
755
756 entry = stream->free_list.next;
757 list_del (entry);
758
759 /* knows about ITD vs SITD */
760 if (stream->highspeed) {
761 struct ehci_itd *itd;
762
763 itd = list_entry (entry, struct ehci_itd,
764 itd_list);
765 dma_pool_free (ehci->itd_pool, itd,
766 itd->itd_dma);
767 } else {
768 struct ehci_sitd *sitd;
769
770 sitd = list_entry (entry, struct ehci_sitd,
771 sitd_list);
772 dma_pool_free (ehci->sitd_pool, sitd,
773 sitd->sitd_dma);
774 }
775 }
776
777 is_in = (stream->bEndpointAddress & USB_DIR_IN) ? 0x10 : 0;
778 stream->bEndpointAddress &= 0x0f;
779 stream->ep->hcpriv = NULL;
780
781 if (stream->rescheduled) {
782 ehci_info (ehci, "ep%d%s-iso rescheduled "
783 "%lu times in %lu seconds\n",
784 stream->bEndpointAddress, is_in ? "in" : "out",
785 stream->rescheduled,
786 ((jiffies - stream->start)/HZ)
787 );
788 }
789
790 kfree(stream);
791 }
792}
793
794static inline struct ehci_iso_stream *
795iso_stream_get (struct ehci_iso_stream *stream)
796{
797 if (likely (stream != NULL))
798 stream->refcount++;
799 return stream;
800}
801
802static struct ehci_iso_stream *
803iso_stream_find (struct ehci_hcd *ehci, struct urb *urb)
804{
805 unsigned epnum;
806 struct ehci_iso_stream *stream;
807 struct usb_host_endpoint *ep;
808 unsigned long flags;
809
810 epnum = usb_pipeendpoint (urb->pipe);
811 if (usb_pipein(urb->pipe))
812 ep = urb->dev->ep_in[epnum];
813 else
814 ep = urb->dev->ep_out[epnum];
815
816 spin_lock_irqsave (&ehci->lock, flags);
817 stream = ep->hcpriv;
818
819 if (unlikely (stream == NULL)) {
820 stream = iso_stream_alloc(GFP_ATOMIC);
821 if (likely (stream != NULL)) {
822 /* dev->ep owns the initial refcount */
823 ep->hcpriv = stream;
824 stream->ep = ep;
825 iso_stream_init(ehci, stream, urb->dev, urb->pipe,
826 urb->interval);
827 }
828
829 /* if dev->ep [epnum] is a QH, info1.maxpacket is nonzero */
830 } else if (unlikely (stream->hw_info1 != 0)) {
831 ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n",
832 urb->dev->devpath, epnum,
833 usb_pipein(urb->pipe) ? "in" : "out");
834 stream = NULL;
835 }
836
837 /* caller guarantees an eventual matching iso_stream_put */
838 stream = iso_stream_get (stream);
839
840 spin_unlock_irqrestore (&ehci->lock, flags);
841 return stream;
842}
843
844/*-------------------------------------------------------------------------*/
845
846/* ehci_iso_sched ops can be ITD-only or SITD-only */
847
848static struct ehci_iso_sched *
5db539e4 849iso_sched_alloc (unsigned packets, unsigned mem_flags)
1da177e4
LT		 850{
851 struct ehci_iso_sched *iso_sched;
852 int size = sizeof *iso_sched;
853
854 size += packets * sizeof (struct ehci_iso_packet);
855 iso_sched = kmalloc (size, mem_flags);
856 if (likely (iso_sched != NULL)) {
857 memset(iso_sched, 0, size);
858 INIT_LIST_HEAD (&iso_sched->td_list);
859 }
860 return iso_sched;
861}
862
863static inline void
864itd_sched_init (
865 struct ehci_iso_sched *iso_sched,
866 struct ehci_iso_stream *stream,
867 struct urb *urb
868)
869{
870 unsigned i;
871 dma_addr_t dma = urb->transfer_dma;
872
873 /* how many uframes are needed for these transfers */
874 iso_sched->span = urb->number_of_packets * stream->interval;
875
876 /* figure out per-uframe itd fields that we'll need later
877 * when we fit new itds into the schedule.
878 */
879 for (i = 0; i < urb->number_of_packets; i++) {
880 struct ehci_iso_packet *uframe = &iso_sched->packet [i];
881 unsigned length;
882 dma_addr_t buf;
883 u32 trans;
884
885 length = urb->iso_frame_desc [i].length;
886 buf = dma + urb->iso_frame_desc [i].offset;
887
888 trans = EHCI_ISOC_ACTIVE;
889 trans |= buf & 0x0fff;
890 if (unlikely (((i + 1) == urb->number_of_packets))
891 && !(urb->transfer_flags & URB_NO_INTERRUPT))
892 trans |= EHCI_ITD_IOC;
893 trans |= length << 16;
894 uframe->transaction = cpu_to_le32 (trans);
895
77078570 896 /* might need to cross a buffer page within a uframe */
1da177e4
LT		 897 uframe->bufp = (buf & ~(u64)0x0fff);
898 buf += length;
899 if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff))))
900 uframe->cross = 1;
901 }
902}
903
904static void
905iso_sched_free (
906 struct ehci_iso_stream *stream,
907 struct ehci_iso_sched *iso_sched
908)
909{
910 if (!iso_sched)
911 return;
912 // caller must hold ehci->lock!
913 list_splice (&iso_sched->td_list, &stream->free_list);
914 kfree (iso_sched);
915}
916
917static int
918itd_urb_transaction (
919 struct ehci_iso_stream *stream,
920 struct ehci_hcd *ehci,
921 struct urb *urb,
5db539e4 922 unsigned mem_flags
1da177e4
LT		 923)
924{
925 struct ehci_itd *itd;
926 dma_addr_t itd_dma;
927 int i;
928 unsigned num_itds;
929 struct ehci_iso_sched *sched;
930 unsigned long flags;
931
932 sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
933 if (unlikely (sched == NULL))
934 return -ENOMEM;
935
936 itd_sched_init (sched, stream, urb);
937
938 if (urb->interval < 8)
939 num_itds = 1 + (sched->span + 7) / 8;
940 else
941 num_itds = urb->number_of_packets;
942
943 /* allocate/init ITDs */
944 spin_lock_irqsave (&ehci->lock, flags);
945 for (i = 0; i < num_itds; i++) {
946
947 /* free_list.next might be cache-hot ... but maybe
948 * the HC caches it too. avoid that issue for now.
949 */
950
951 /* prefer previously-allocated itds */
952 if (likely (!list_empty(&stream->free_list))) {
953 itd = list_entry (stream->free_list.prev,
954 struct ehci_itd, itd_list);
955 list_del (&itd->itd_list);
956 itd_dma = itd->itd_dma;
957 } else
958 itd = NULL;
959
960 if (!itd) {
961 spin_unlock_irqrestore (&ehci->lock, flags);
962 itd = dma_pool_alloc (ehci->itd_pool, mem_flags,
963 &itd_dma);
964 spin_lock_irqsave (&ehci->lock, flags);
965 }
966
967 if (unlikely (NULL == itd)) {
968 iso_sched_free (stream, sched);
969 spin_unlock_irqrestore (&ehci->lock, flags);
970 return -ENOMEM;
971 }
972 memset (itd, 0, sizeof *itd);
973 itd->itd_dma = itd_dma;
974 list_add (&itd->itd_list, &sched->td_list);
975 }
976 spin_unlock_irqrestore (&ehci->lock, flags);
977
978 /* temporarily store schedule info in hcpriv */
979 urb->hcpriv = sched;
980 urb->error_count = 0;
981 return 0;
982}
983
984/*-------------------------------------------------------------------------*/
985
986static inline int
987itd_slot_ok (
988 struct ehci_hcd *ehci,
989 u32 mod,
990 u32 uframe,
991 u8 usecs,
992 u32 period
993)
994{
995 uframe %= period;
996 do {
997 /* can't commit more than 80% periodic == 100 usec */
998 if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7)
999 > (100 - usecs))
1000 return 0;
1001
1002 /* we know urb->interval is 2^N uframes */
1003 uframe += period;
1004 } while (uframe < mod);
1005 return 1;
1006}
1007
1008static inline int
1009sitd_slot_ok (
1010 struct ehci_hcd *ehci,
1011 u32 mod,
1012 struct ehci_iso_stream *stream,
1013 u32 uframe,
1014 struct ehci_iso_sched *sched,
1015 u32 period_uframes
1016)
1017{
1018 u32 mask, tmp;
1019 u32 frame, uf;
1020
1021 mask = stream->raw_mask << (uframe & 7);
1022
1023 /* for IN, don't wrap CSPLIT into the next frame */
1024 if (mask & ~0xffff)
1025 return 0;
1026
1027 /* this multi-pass logic is simple, but performance may
1028 * suffer when the schedule data isn't cached.
1029 */
1030
1031 /* check bandwidth */
1032 uframe %= period_uframes;
1033 do {
1034 u32 max_used;
1035
1036 frame = uframe >> 3;
1037 uf = uframe & 7;
1038
1039 /* tt must be idle for start(s), any gap, and csplit.
1040 * assume scheduling slop leaves 10+% for control/bulk.
1041 */
1042 if (!tt_no_collision (ehci, period_uframes << 3,
1043 stream->udev, frame, mask))
1044 return 0;
1045
1046 /* check starts (OUT uses more than one) */
1047 max_used = 100 - stream->usecs;
1048 for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) {
1049 if (periodic_usecs (ehci, frame, uf) > max_used)
1050 return 0;
1051 }
1052
1053 /* for IN, check CSPLIT */
1054 if (stream->c_usecs) {
1055 max_used = 100 - stream->c_usecs;
1056 do {
1057 tmp = 1 << uf;
1058 tmp <<= 8;
1059 if ((stream->raw_mask & tmp) == 0)
1060 continue;
1061 if (periodic_usecs (ehci, frame, uf)
1062 > max_used)
1063 return 0;
1064 } while (++uf < 8);
1065 }
1066
1067 /* we know urb->interval is 2^N uframes */
1068 uframe += period_uframes;
1069 } while (uframe < mod);
1070
1071 stream->splits = cpu_to_le32(stream->raw_mask << (uframe & 7));
1072 return 1;
1073}
1074
1075/*
1076 * This scheduler plans almost as far into the future as it has actual
1077 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to
1078 * "as small as possible" to be cache-friendlier.) That limits the size
1079 * transfers you can stream reliably; avoid more than 64 msec per urb.
1080 * Also avoid queue depths of less than ehci's worst irq latency (affected
1081 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1082 * and other factors); or more than about 230 msec total (for portability,
1083 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler!
1084 */
1085
1086#define SCHEDULE_SLOP 10 /* frames */
1087
1088static int
1089iso_stream_schedule (
1090 struct ehci_hcd *ehci,
1091 struct urb *urb,
1092 struct ehci_iso_stream *stream
1093)
1094{
1095 u32 now, start, max, period;
1096 int status;
1097 unsigned mod = ehci->periodic_size << 3;
1098 struct ehci_iso_sched *sched = urb->hcpriv;
1099
1100 if (sched->span > (mod - 8 * SCHEDULE_SLOP)) {
1101 ehci_dbg (ehci, "iso request %p too long\n", urb);
1102 status = -EFBIG;
1103 goto fail;
1104 }
1105
1106 if ((stream->depth + sched->span) > mod) {
1107 ehci_dbg (ehci, "request %p would overflow (%d+%d>%d)\n",
1108 urb, stream->depth, sched->span, mod);
1109 status = -EFBIG;
1110 goto fail;
1111 }
1112
1113 now = readl (&ehci->regs->frame_index) % mod;
1114
1115 /* when's the last uframe this urb could start? */
1116 max = now + mod;
1117
1118 /* typical case: reuse current schedule. stream is still active,
1119 * and no gaps from host falling behind (irq delays etc)
1120 */
1121 if (likely (!list_empty (&stream->td_list))) {
1122 start = stream->next_uframe;
1123 if (start < now)
1124 start += mod;
1125 if (likely ((start + sched->span) < max))
1126 goto ready;
1127 /* else fell behind; someday, try to reschedule */
1128 status = -EL2NSYNC;
1129 goto fail;
1130 }
1131
1132 /* need to schedule; when's the next (u)frame we could start?
1133 * this is bigger than ehci->i_thresh allows; scheduling itself
1134 * isn't free, the slop should handle reasonably slow cpus. it
1135 * can also help high bandwidth if the dma and irq loads don't
1136 * jump until after the queue is primed.
1137 */
1138 start = SCHEDULE_SLOP * 8 + (now & ~0x07);
1139 start %= mod;
1140 stream->next_uframe = start;
1141
1142 /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */
1143
1144 period = urb->interval;
1145 if (!stream->highspeed)
1146 period <<= 3;
1147
1148 /* find a uframe slot with enough bandwidth */
1149 for (; start < (stream->next_uframe + period); start++) {
1150 int enough_space;
1151
1152 /* check schedule: enough space? */
1153 if (stream->highspeed)
1154 enough_space = itd_slot_ok (ehci, mod, start,
1155 stream->usecs, period);
1156 else {
1157 if ((start % 8) >= 6)
1158 continue;
1159 enough_space = sitd_slot_ok (ehci, mod, stream,
1160 start, sched, period);
1161 }
1162
1163 /* schedule it here if there's enough bandwidth */
1164 if (enough_space) {
1165 stream->next_uframe = start % mod;
1166 goto ready;
1167 }
1168 }
1169
1170 /* no room in the schedule */
1171 ehci_dbg (ehci, "iso %ssched full %p (now %d max %d)\n",
1172 list_empty (&stream->td_list) ? "" : "re",
1173 urb, now, max);
1174 status = -ENOSPC;
1175
1176fail:
1177 iso_sched_free (stream, sched);
1178 urb->hcpriv = NULL;
1179 return status;
1180
1181ready:
1182 /* report high speed start in uframes; full speed, in frames */
1183 urb->start_frame = stream->next_uframe;
1184 if (!stream->highspeed)
1185 urb->start_frame >>= 3;
1186 return 0;
1187}
1188
1189/*-------------------------------------------------------------------------*/
1190
1191static inline void
1192itd_init (struct ehci_iso_stream *stream, struct ehci_itd *itd)
1193{
1194 int i;
1195
77078570 1196 /* it's been recently zeroed */
1da177e4
LT		 1197 itd->hw_next = EHCI_LIST_END;
1198 itd->hw_bufp [0] = stream->buf0;
1199 itd->hw_bufp [1] = stream->buf1;
1200 itd->hw_bufp [2] = stream->buf2;
1201
1202 for (i = 0; i < 8; i++)
1203 itd->index[i] = -1;
1204
1205 /* All other fields are filled when scheduling */
1206}
1207
1208static inline void
1209itd_patch (
1210 struct ehci_itd *itd,
1211 struct ehci_iso_sched *iso_sched,
1212 unsigned index,
77078570 1213 u16 uframe
1da177e4
LT		 1214)
1215{
1216 struct ehci_iso_packet *uf = &iso_sched->packet [index];
1217 unsigned pg = itd->pg;
1218
1219 // BUG_ON (pg == 6 && uf->cross);
1220
1221 uframe &= 0x07;
1222 itd->index [uframe] = index;
1223
1224 itd->hw_transaction [uframe] = uf->transaction;
1225 itd->hw_transaction [uframe] |= cpu_to_le32 (pg << 12);
1226 itd->hw_bufp [pg] |= cpu_to_le32 (uf->bufp & ~(u32)0);
1227 itd->hw_bufp_hi [pg] |= cpu_to_le32 ((u32)(uf->bufp >> 32));
1228
1229 /* iso_frame_desc[].offset must be strictly increasing */
77078570 1230 if (unlikely (uf->cross)) {
1da177e4
LT		 1231 u64 bufp = uf->bufp + 4096;
1232 itd->pg = ++pg;
1233 itd->hw_bufp [pg] |= cpu_to_le32 (bufp & ~(u32)0);
1234 itd->hw_bufp_hi [pg] |= cpu_to_le32 ((u32)(bufp >> 32));
1235 }
1236}
1237
1238static inline void
1239itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1240{
1241 /* always prepend ITD/SITD ... only QH tree is order-sensitive */
1242 itd->itd_next = ehci->pshadow [frame];
1243 itd->hw_next = ehci->periodic [frame];
1244 ehci->pshadow [frame].itd = itd;
1245 itd->frame = frame;
1246 wmb ();
1247 ehci->periodic [frame] = cpu_to_le32 (itd->itd_dma) | Q_TYPE_ITD;
1248}
1249
1250/* fit urb's itds into the selected schedule slot; activate as needed */
1251static int
1252itd_link_urb (
1253 struct ehci_hcd *ehci,
1254 struct urb *urb,
1255 unsigned mod,
1256 struct ehci_iso_stream *stream
1257)
1258{
77078570 1259 int packet;
1da177e4
LT		 1260 unsigned next_uframe, uframe, frame;
1261 struct ehci_iso_sched *iso_sched = urb->hcpriv;
1262 struct ehci_itd *itd;
1263
1264 next_uframe = stream->next_uframe % mod;
1265
1266 if (unlikely (list_empty(&stream->td_list))) {
1267 ehci_to_hcd(ehci)->self.bandwidth_allocated
1268 += stream->bandwidth;
1269 ehci_vdbg (ehci,
1270 "schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1271 urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1272 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1273 urb->interval,
1274 next_uframe >> 3, next_uframe & 0x7);
1275 stream->start = jiffies;
1276 }
1277 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1278
1279 /* fill iTDs uframe by uframe */
1280 for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) {
1281 if (itd == NULL) {
1282 /* ASSERT: we have all necessary itds */
1283 // BUG_ON (list_empty (&iso_sched->td_list));
1284
1285 /* ASSERT: no itds for this endpoint in this uframe */
1286
1287 itd = list_entry (iso_sched->td_list.next,
1288 struct ehci_itd, itd_list);
1289 list_move_tail (&itd->itd_list, &stream->td_list);
1290 itd->stream = iso_stream_get (stream);
1291 itd->urb = usb_get_urb (urb);
1da177e4
LT		 1292 itd_init (stream, itd);
1293 }
1294
1295 uframe = next_uframe & 0x07;
1296 frame = next_uframe >> 3;
1297
1298 itd->usecs [uframe] = stream->usecs;
77078570 1299 itd_patch (itd, iso_sched, packet, uframe);
1da177e4
LT		 1300
1301 next_uframe += stream->interval;
1302 stream->depth += stream->interval;
1303 next_uframe %= mod;
1304 packet++;
1305
1306 /* link completed itds into the schedule */
1307 if (((next_uframe >> 3) != frame)
1308 || packet == urb->number_of_packets) {
1309 itd_link (ehci, frame % ehci->periodic_size, itd);
1310 itd = NULL;
1311 }
1312 }
1313 stream->next_uframe = next_uframe;
1314
1315 /* don't need that schedule data any more */
1316 iso_sched_free (stream, iso_sched);
1317 urb->hcpriv = NULL;
1318
1319 timer_action (ehci, TIMER_IO_WATCHDOG);
1320 if (unlikely (!ehci->periodic_sched++))
1321 return enable_periodic (ehci);
1322 return 0;
1323}
1324
1325#define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1326
1327static unsigned
1328itd_complete (
1329 struct ehci_hcd *ehci,
1330 struct ehci_itd *itd,
1331 struct pt_regs *regs
1332) {
1333 struct urb *urb = itd->urb;
1334 struct usb_iso_packet_descriptor *desc;
1335 u32 t;
1336 unsigned uframe;
1337 int urb_index = -1;
1338 struct ehci_iso_stream *stream = itd->stream;
1339 struct usb_device *dev;
1340
1341 /* for each uframe with a packet */
1342 for (uframe = 0; uframe < 8; uframe++) {
1343 if (likely (itd->index[uframe] == -1))
1344 continue;
1345 urb_index = itd->index[uframe];
1346 desc = &urb->iso_frame_desc [urb_index];
1347
1348 t = le32_to_cpup (&itd->hw_transaction [uframe]);
1349 itd->hw_transaction [uframe] = 0;
1350 stream->depth -= stream->interval;
1351
1352 /* report transfer status */
1353 if (unlikely (t & ISO_ERRS)) {
1354 urb->error_count++;
1355 if (t & EHCI_ISOC_BUF_ERR)
1356 desc->status = usb_pipein (urb->pipe)
1357 ? -ENOSR /* hc couldn't read */
1358 : -ECOMM; /* hc couldn't write */
1359 else if (t & EHCI_ISOC_BABBLE)
1360 desc->status = -EOVERFLOW;
1361 else /* (t & EHCI_ISOC_XACTERR) */
1362 desc->status = -EPROTO;
1363
1364 /* HC need not update length with this error */
1365 if (!(t & EHCI_ISOC_BABBLE))
1366 desc->actual_length = EHCI_ITD_LENGTH (t);
1367 } else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) {
1368 desc->status = 0;
1369 desc->actual_length = EHCI_ITD_LENGTH (t);
1370 }
1371 }
1372
1373 usb_put_urb (urb);
1374 itd->urb = NULL;
1375 itd->stream = NULL;
1376 list_move (&itd->itd_list, &stream->free_list);
1377 iso_stream_put (ehci, stream);
1378
1379 /* handle completion now? */
1380 if (likely ((urb_index + 1) != urb->number_of_packets))
1381 return 0;
1382
1383 /* ASSERT: it's really the last itd for this urb
1384 list_for_each_entry (itd, &stream->td_list, itd_list)
1385 BUG_ON (itd->urb == urb);
1386 */
1387
1388 /* give urb back to the driver ... can be out-of-order */
1389 dev = usb_get_dev (urb->dev);
1390 ehci_urb_done (ehci, urb, regs);
1391 urb = NULL;
1392
1393 /* defer stopping schedule; completion can submit */
1394 ehci->periodic_sched--;
1395 if (unlikely (!ehci->periodic_sched))
1396 (void) disable_periodic (ehci);
1397 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1398
1399 if (unlikely (list_empty (&stream->td_list))) {
1400 ehci_to_hcd(ehci)->self.bandwidth_allocated
1401 -= stream->bandwidth;
1402 ehci_vdbg (ehci,
1403 "deschedule devp %s ep%d%s-iso\n",
1404 dev->devpath, stream->bEndpointAddress & 0x0f,
1405 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
1406 }
1407 iso_stream_put (ehci, stream);
1408 usb_put_dev (dev);
1409
1410 return 1;
1411}
1412
1413/*-------------------------------------------------------------------------*/
1414
5db539e4
OK		 1415static int itd_submit (struct ehci_hcd *ehci, struct urb *urb,
1416 unsigned mem_flags)
1da177e4
LT		 1417{
1418 int status = -EINVAL;
1419 unsigned long flags;
1420 struct ehci_iso_stream *stream;
1421
1422 /* Get iso_stream head */
1423 stream = iso_stream_find (ehci, urb);
1424 if (unlikely (stream == NULL)) {
1425 ehci_dbg (ehci, "can't get iso stream\n");
1426 return -ENOMEM;
1427 }
1428 if (unlikely (urb->interval != stream->interval)) {
1429 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
1430 stream->interval, urb->interval);
1431 goto done;
1432 }
1433
1434#ifdef EHCI_URB_TRACE
1435 ehci_dbg (ehci,
1436 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1437 __FUNCTION__, urb->dev->devpath, urb,
1438 usb_pipeendpoint (urb->pipe),
1439 usb_pipein (urb->pipe) ? "in" : "out",
1440 urb->transfer_buffer_length,
1441 urb->number_of_packets, urb->interval,
1442 stream);
1443#endif
1444
1445 /* allocate ITDs w/o locking anything */
1446 status = itd_urb_transaction (stream, ehci, urb, mem_flags);
1447 if (unlikely (status < 0)) {
1448 ehci_dbg (ehci, "can't init itds\n");
1449 goto done;
1450 }
1451
1452 /* schedule ... need to lock */
1453 spin_lock_irqsave (&ehci->lock, flags);
1454 status = iso_stream_schedule (ehci, urb, stream);
1455 if (likely (status == 0))
1456 itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
1457 spin_unlock_irqrestore (&ehci->lock, flags);
1458
1459done:
1460 if (unlikely (status < 0))
1461 iso_stream_put (ehci, stream);
1462 return status;
1463}
1464
1465#ifdef CONFIG_USB_EHCI_SPLIT_ISO
1466
1467/*-------------------------------------------------------------------------*/
1468
1469/*
1470 * "Split ISO TDs" ... used for USB 1.1 devices going through the
1471 * TTs in USB 2.0 hubs. These need microframe scheduling.
1472 */
1473
1474static inline void
1475sitd_sched_init (
1476 struct ehci_iso_sched *iso_sched,
1477 struct ehci_iso_stream *stream,
1478 struct urb *urb
1479)
1480{
1481 unsigned i;
1482 dma_addr_t dma = urb->transfer_dma;
1483
1484 /* how many frames are needed for these transfers */
1485 iso_sched->span = urb->number_of_packets * stream->interval;
1486
1487 /* figure out per-frame sitd fields that we'll need later
1488 * when we fit new sitds into the schedule.
1489 */
1490 for (i = 0; i < urb->number_of_packets; i++) {
1491 struct ehci_iso_packet *packet = &iso_sched->packet [i];
1492 unsigned length;
1493 dma_addr_t buf;
1494 u32 trans;
1495
1496 length = urb->iso_frame_desc [i].length & 0x03ff;
1497 buf = dma + urb->iso_frame_desc [i].offset;
1498
1499 trans = SITD_STS_ACTIVE;
1500 if (((i + 1) == urb->number_of_packets)
1501 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1502 trans |= SITD_IOC;
1503 trans |= length << 16;
1504 packet->transaction = cpu_to_le32 (trans);
1505
1506 /* might need to cross a buffer page within a td */
1507 packet->bufp = buf;
1508 packet->buf1 = (buf + length) & ~0x0fff;
1509 if (packet->buf1 != (buf & ~(u64)0x0fff))
1510 packet->cross = 1;
1511
1512 /* OUT uses multiple start-splits */
1513 if (stream->bEndpointAddress & USB_DIR_IN)
1514 continue;
1515 length = (length + 187) / 188;
1516 if (length > 1) /* BEGIN vs ALL */
1517 length |= 1 << 3;
1518 packet->buf1 |= length;
1519 }
1520}
1521
1522static int
1523sitd_urb_transaction (
1524 struct ehci_iso_stream *stream,
1525 struct ehci_hcd *ehci,
1526 struct urb *urb,
5db539e4 1527 unsigned mem_flags
1da177e4
LT		 1528)
1529{
1530 struct ehci_sitd *sitd;
1531 dma_addr_t sitd_dma;
1532 int i;
1533 struct ehci_iso_sched *iso_sched;
1534 unsigned long flags;
1535
1536 iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1537 if (iso_sched == NULL)
1538 return -ENOMEM;
1539
1540 sitd_sched_init (iso_sched, stream, urb);
1541
1542 /* allocate/init sITDs */
1543 spin_lock_irqsave (&ehci->lock, flags);
1544 for (i = 0; i < urb->number_of_packets; i++) {
1545
1546 /* NOTE: for now, we don't try to handle wraparound cases
1547 * for IN (using sitd->hw_backpointer, like a FSTN), which
1548 * means we never need two sitds for full speed packets.
1549 */
1550
1551 /* free_list.next might be cache-hot ... but maybe
1552 * the HC caches it too. avoid that issue for now.
1553 */
1554
1555 /* prefer previously-allocated sitds */
1556 if (!list_empty(&stream->free_list)) {
1557 sitd = list_entry (stream->free_list.prev,
1558 struct ehci_sitd, sitd_list);
1559 list_del (&sitd->sitd_list);
1560 sitd_dma = sitd->sitd_dma;
1561 } else
1562 sitd = NULL;
1563
1564 if (!sitd) {
1565 spin_unlock_irqrestore (&ehci->lock, flags);
1566 sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags,
1567 &sitd_dma);
1568 spin_lock_irqsave (&ehci->lock, flags);
1569 }
1570
1571 if (!sitd) {
1572 iso_sched_free (stream, iso_sched);
1573 spin_unlock_irqrestore (&ehci->lock, flags);
1574 return -ENOMEM;
1575 }
1576 memset (sitd, 0, sizeof *sitd);
1577 sitd->sitd_dma = sitd_dma;
1578 list_add (&sitd->sitd_list, &iso_sched->td_list);
1579 }
1580
1581 /* temporarily store schedule info in hcpriv */
1582 urb->hcpriv = iso_sched;
1583 urb->error_count = 0;
1584
1585 spin_unlock_irqrestore (&ehci->lock, flags);
1586 return 0;
1587}
1588
1589/*-------------------------------------------------------------------------*/
1590
1591static inline void
1592sitd_patch (
1593 struct ehci_iso_stream *stream,
1594 struct ehci_sitd *sitd,
1595 struct ehci_iso_sched *iso_sched,
1596 unsigned index
1597)
1598{
1599 struct ehci_iso_packet *uf = &iso_sched->packet [index];
1600 u64 bufp = uf->bufp;
1601
1602 sitd->hw_next = EHCI_LIST_END;
1603 sitd->hw_fullspeed_ep = stream->address;
1604 sitd->hw_uframe = stream->splits;
1605 sitd->hw_results = uf->transaction;
1606 sitd->hw_backpointer = EHCI_LIST_END;
1607
1608 bufp = uf->bufp;
1609 sitd->hw_buf [0] = cpu_to_le32 (bufp);
1610 sitd->hw_buf_hi [0] = cpu_to_le32 (bufp >> 32);
1611
1612 sitd->hw_buf [1] = cpu_to_le32 (uf->buf1);
1613 if (uf->cross)
1614 bufp += 4096;
1615 sitd->hw_buf_hi [1] = cpu_to_le32 (bufp >> 32);
1616 sitd->index = index;
1617}
1618
1619static inline void
1620sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
1621{
1622 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */
1623 sitd->sitd_next = ehci->pshadow [frame];
1624 sitd->hw_next = ehci->periodic [frame];
1625 ehci->pshadow [frame].sitd = sitd;
1626 sitd->frame = frame;
1627 wmb ();
1628 ehci->periodic [frame] = cpu_to_le32 (sitd->sitd_dma) | Q_TYPE_SITD;
1629}
1630
1631/* fit urb's sitds into the selected schedule slot; activate as needed */
1632static int
1633sitd_link_urb (
1634 struct ehci_hcd *ehci,
1635 struct urb *urb,
1636 unsigned mod,
1637 struct ehci_iso_stream *stream
1638)
1639{
1640 int packet;
1641 unsigned next_uframe;
1642 struct ehci_iso_sched *sched = urb->hcpriv;
1643 struct ehci_sitd *sitd;
1644
1645 next_uframe = stream->next_uframe;
1646
1647 if (list_empty(&stream->td_list)) {
1648 /* usbfs ignores TT bandwidth */
1649 ehci_to_hcd(ehci)->self.bandwidth_allocated
1650 += stream->bandwidth;
1651 ehci_vdbg (ehci,
1652 "sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
1653 urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1654 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1655 (next_uframe >> 3) % ehci->periodic_size,
1656 stream->interval, le32_to_cpu (stream->splits));
1657 stream->start = jiffies;
1658 }
1659 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1660
1661 /* fill sITDs frame by frame */
1662 for (packet = 0, sitd = NULL;
1663 packet < urb->number_of_packets;
1664 packet++) {
1665
1666 /* ASSERT: we have all necessary sitds */
1667 BUG_ON (list_empty (&sched->td_list));
1668
1669 /* ASSERT: no itds for this endpoint in this frame */
1670
1671 sitd = list_entry (sched->td_list.next,
1672 struct ehci_sitd, sitd_list);
1673 list_move_tail (&sitd->sitd_list, &stream->td_list);
1674 sitd->stream = iso_stream_get (stream);
1675 sitd->urb = usb_get_urb (urb);
1676
1677 sitd_patch (stream, sitd, sched, packet);
1678 sitd_link (ehci, (next_uframe >> 3) % ehci->periodic_size,
1679 sitd);
1680
1681 next_uframe += stream->interval << 3;
1682 stream->depth += stream->interval << 3;
1683 }
1684 stream->next_uframe = next_uframe % mod;
1685
1686 /* don't need that schedule data any more */
1687 iso_sched_free (stream, sched);
1688 urb->hcpriv = NULL;
1689
1690 timer_action (ehci, TIMER_IO_WATCHDOG);
1691 if (!ehci->periodic_sched++)
1692 return enable_periodic (ehci);
1693 return 0;
1694}
1695
1696/*-------------------------------------------------------------------------*/
1697
1698#define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
1699 | SITD_STS_XACT | SITD_STS_MMF)
1700
1701static unsigned
1702sitd_complete (
1703 struct ehci_hcd *ehci,
1704 struct ehci_sitd *sitd,
1705 struct pt_regs *regs
1706) {
1707 struct urb *urb = sitd->urb;
1708 struct usb_iso_packet_descriptor *desc;
1709 u32 t;
1710 int urb_index = -1;
1711 struct ehci_iso_stream *stream = sitd->stream;
1712 struct usb_device *dev;
1713
1714 urb_index = sitd->index;
1715 desc = &urb->iso_frame_desc [urb_index];
1716 t = le32_to_cpup (&sitd->hw_results);
1717
1718 /* report transfer status */
1719 if (t & SITD_ERRS) {
1720 urb->error_count++;
1721 if (t & SITD_STS_DBE)
1722 desc->status = usb_pipein (urb->pipe)
1723 ? -ENOSR /* hc couldn't read */
1724 : -ECOMM; /* hc couldn't write */
1725 else if (t & SITD_STS_BABBLE)
1726 desc->status = -EOVERFLOW;
1727 else /* XACT, MMF, etc */
1728 desc->status = -EPROTO;
1729 } else {
1730 desc->status = 0;
1731 desc->actual_length = desc->length - SITD_LENGTH (t);
1732 }
1733
1734 usb_put_urb (urb);
1735 sitd->urb = NULL;
1736 sitd->stream = NULL;
1737 list_move (&sitd->sitd_list, &stream->free_list);
1738 stream->depth -= stream->interval << 3;
1739 iso_stream_put (ehci, stream);
1740
1741 /* handle completion now? */
1742 if ((urb_index + 1) != urb->number_of_packets)
1743 return 0;
1744
1745 /* ASSERT: it's really the last sitd for this urb
1746 list_for_each_entry (sitd, &stream->td_list, sitd_list)
1747 BUG_ON (sitd->urb == urb);
1748 */
1749
1750 /* give urb back to the driver */
1751 dev = usb_get_dev (urb->dev);
1752 ehci_urb_done (ehci, urb, regs);
1753 urb = NULL;
1754
1755 /* defer stopping schedule; completion can submit */
1756 ehci->periodic_sched--;
1757 if (!ehci->periodic_sched)
1758 (void) disable_periodic (ehci);
1759 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1760
1761 if (list_empty (&stream->td_list)) {
1762 ehci_to_hcd(ehci)->self.bandwidth_allocated
1763 -= stream->bandwidth;
1764 ehci_vdbg (ehci,
1765 "deschedule devp %s ep%d%s-iso\n",
1766 dev->devpath, stream->bEndpointAddress & 0x0f,
1767 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
1768 }
1769 iso_stream_put (ehci, stream);
1770 usb_put_dev (dev);
1771
1772 return 1;
1773}
1774
1775
5db539e4
OK		 1776static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
1777 unsigned mem_flags)
1da177e4
LT		 1778{
1779 int status = -EINVAL;
1780 unsigned long flags;
1781 struct ehci_iso_stream *stream;
1782
1783 /* Get iso_stream head */
1784 stream = iso_stream_find (ehci, urb);
1785 if (stream == NULL) {
1786 ehci_dbg (ehci, "can't get iso stream\n");
1787 return -ENOMEM;
1788 }
1789 if (urb->interval != stream->interval) {
1790 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
1791 stream->interval, urb->interval);
1792 goto done;
1793 }
1794
1795#ifdef EHCI_URB_TRACE
1796 ehci_dbg (ehci,
1797 "submit %p dev%s ep%d%s-iso len %d\n",
1798 urb, urb->dev->devpath,
1799 usb_pipeendpoint (urb->pipe),
1800 usb_pipein (urb->pipe) ? "in" : "out",
1801 urb->transfer_buffer_length);
1802#endif
1803
1804 /* allocate SITDs */
1805 status = sitd_urb_transaction (stream, ehci, urb, mem_flags);
1806 if (status < 0) {
1807 ehci_dbg (ehci, "can't init sitds\n");
1808 goto done;
1809 }
1810
1811 /* schedule ... need to lock */
1812 spin_lock_irqsave (&ehci->lock, flags);
1813 status = iso_stream_schedule (ehci, urb, stream);
1814 if (status == 0)
1815 sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
1816 spin_unlock_irqrestore (&ehci->lock, flags);
1817
1818done:
1819 if (status < 0)
1820 iso_stream_put (ehci, stream);
1821 return status;
1822}
1823
1824#else
1825
1826static inline int
5db539e4
OK		 1827sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
1828 unsigned mem_flags)
1da177e4
LT		 1829{
1830 ehci_dbg (ehci, "split iso support is disabled\n");
1831 return -ENOSYS;
1832}
1833
1834static inline unsigned
1835sitd_complete (
1836 struct ehci_hcd *ehci,
1837 struct ehci_sitd *sitd,
1838 struct pt_regs *regs
1839) {
1840 ehci_err (ehci, "sitd_complete %p?\n", sitd);
1841 return 0;
1842}
1843
1844#endif /* USB_EHCI_SPLIT_ISO */
1845
1846/*-------------------------------------------------------------------------*/
1847
1848static void
1849scan_periodic (struct ehci_hcd *ehci, struct pt_regs *regs)
1850{
1851 unsigned frame, clock, now_uframe, mod;
1852 unsigned modified;
1853
1854 mod = ehci->periodic_size << 3;
1855
1856 /*
1857 * When running, scan from last scan point up to "now"
1858 * else clean up by scanning everything that's left.
1859 * Touches as few pages as possible: cache-friendly.
1860 */
1861 now_uframe = ehci->next_uframe;
1862 if (HC_IS_RUNNING (ehci_to_hcd(ehci)->state))
1863 clock = readl (&ehci->regs->frame_index);
1864 else
1865 clock = now_uframe + mod - 1;
1866 clock %= mod;
1867
1868 for (;;) {
1869 union ehci_shadow q, *q_p;
1870 __le32 type, *hw_p;
1871 unsigned uframes;
1872
1873 /* don't scan past the live uframe */
1874 frame = now_uframe >> 3;
1875 if (frame == (clock >> 3))
1876 uframes = now_uframe & 0x07;
1877 else {
1878 /* safe to scan the whole frame at once */
1879 now_uframe |= 0x07;
1880 uframes = 8;
1881 }
1882
1883restart:
1884 /* scan each element in frame's queue for completions */
1885 q_p = &ehci->pshadow [frame];
1886 hw_p = &ehci->periodic [frame];
1887 q.ptr = q_p->ptr;
1888 type = Q_NEXT_TYPE (*hw_p);
1889 modified = 0;
1890
1891 while (q.ptr != NULL) {
1892 unsigned uf;
1893 union ehci_shadow temp;
1894 int live;
1895
1896 live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state);
1897 switch (type) {
1898 case Q_TYPE_QH:
1899 /* handle any completions */
1900 temp.qh = qh_get (q.qh);
1901 type = Q_NEXT_TYPE (q.qh->hw_next);
1902 q = q.qh->qh_next;
1903 modified = qh_completions (ehci, temp.qh, regs);
1904 if (unlikely (list_empty (&temp.qh->qtd_list)))
1905 intr_deschedule (ehci, temp.qh);
1906 qh_put (temp.qh);
1907 break;
1908 case Q_TYPE_FSTN:
1909 /* for "save place" FSTNs, look at QH entries
1910 * in the previous frame for completions.
1911 */
1912 if (q.fstn->hw_prev != EHCI_LIST_END) {
1913 dbg ("ignoring completions from FSTNs");
1914 }
1915 type = Q_NEXT_TYPE (q.fstn->hw_next);
1916 q = q.fstn->fstn_next;
1917 break;
1918 case Q_TYPE_ITD:
1919 /* skip itds for later in the frame */
1920 rmb ();
1921 for (uf = live ? uframes : 8; uf < 8; uf++) {
1922 if (0 == (q.itd->hw_transaction [uf]
1923 & ITD_ACTIVE))
1924 continue;
1925 q_p = &q.itd->itd_next;
1926 hw_p = &q.itd->hw_next;
1927 type = Q_NEXT_TYPE (q.itd->hw_next);
1928 q = *q_p;
1929 break;
1930 }
1931 if (uf != 8)
1932 break;
1933
1934 /* this one's ready ... HC won't cache the
1935 * pointer for much longer, if at all.
1936 */
1937 *q_p = q.itd->itd_next;
1938 *hw_p = q.itd->hw_next;
1939 type = Q_NEXT_TYPE (q.itd->hw_next);
1940 wmb();
1941 modified = itd_complete (ehci, q.itd, regs);
1942 q = *q_p;
1943 break;
1944 case Q_TYPE_SITD:
1945 if ((q.sitd->hw_results & SITD_ACTIVE)
1946 && live) {
1947 q_p = &q.sitd->sitd_next;
1948 hw_p = &q.sitd->hw_next;
1949 type = Q_NEXT_TYPE (q.sitd->hw_next);
1950 q = *q_p;
1951 break;
1952 }
1953 *q_p = q.sitd->sitd_next;
1954 *hw_p = q.sitd->hw_next;
1955 type = Q_NEXT_TYPE (q.sitd->hw_next);
1956 wmb();
1957 modified = sitd_complete (ehci, q.sitd, regs);
1958 q = *q_p;
1959 break;
1960 default:
1961 dbg ("corrupt type %d frame %d shadow %p",
1962 type, frame, q.ptr);
1963 // BUG ();
1964 q.ptr = NULL;
1965 }
1966
1967 /* assume completion callbacks modify the queue */
1968 if (unlikely (modified))
1969 goto restart;
1970 }
1971
1972 /* stop when we catch up to the HC */
1973
1974 // FIXME: this assumes we won't get lapped when
1975 // latencies climb; that should be rare, but...
1976 // detect it, and just go all the way around.
1977 // FLR might help detect this case, so long as latencies
1978 // don't exceed periodic_size msec (default 1.024 sec).
1979
1980 // FIXME: likewise assumes HC doesn't halt mid-scan
1981
1982 if (now_uframe == clock) {
1983 unsigned now;
1984
1985 if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state))
1986 break;
1987 ehci->next_uframe = now_uframe;
1988 now = readl (&ehci->regs->frame_index) % mod;
1989 if (now_uframe == now)
1990 break;
1991
1992 /* rescan the rest of this frame, then ... */
1993 clock = now;
1994 } else {
1995 now_uframe++;
1996 now_uframe %= mod;
1997 }
1998 }
1999}
Linux kernel - Deliverables
This page took 0.16153 seconds and 5 git commands to generate.