2 * Copyright (C) 2001-2004 by David Brownell
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the
6 * Free Software Foundation; either version 2 of the License, or (at your
7 * option) any later version.
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software Foundation,
16 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 /* this file is part of ehci-hcd.c */
21 /*-------------------------------------------------------------------------*/
24 * EHCI hardware queue manipulation ... the core. QH/QTD manipulation.
26 * Control, bulk, and interrupt traffic all use "qh" lists. They list "qtd"
27 * entries describing USB transactions, max 16-20kB/entry (with 4kB-aligned
28 * buffers needed for the larger number). We use one QH per endpoint, queue
29 * multiple urbs (all three types) per endpoint. URBs may need several qtds.
31 * ISO traffic uses "ISO TD" (itd, and sitd) records, and (along with
32 * interrupts) needs careful scheduling. Performance improvements can be
33 * an ongoing challenge. That's in "ehci-sched.c".
35 * USB 1.1 devices are handled (a) by "companion" OHCI or UHCI root hubs,
36 * or otherwise through transaction translators (TTs) in USB 2.0 hubs using
37 * (b) special fields in qh entries or (c) split iso entries. TTs will
38 * buffer low/full speed data so the host collects it at high speed.
41 /*-------------------------------------------------------------------------*/
43 /* fill a qtd, returning how much of the buffer we were able to queue up */
46 qtd_fill(struct ehci_hcd
*ehci
, struct ehci_qtd
*qtd
, dma_addr_t buf
,
47 size_t len
, int token
, int maxpacket
)
52 /* one buffer entry per 4K ... first might be short or unaligned */
53 qtd
->hw_buf
[0] = cpu_to_hc32(ehci
, (u32
)addr
);
54 qtd
->hw_buf_hi
[0] = cpu_to_hc32(ehci
, (u32
)(addr
>> 32));
55 count
= 0x1000 - (buf
& 0x0fff); /* rest of that page */
56 if (likely (len
< count
)) /* ... iff needed */
62 /* per-qtd limit: from 16K to 20K (best alignment) */
63 for (i
= 1; count
< len
&& i
< 5; i
++) {
65 qtd
->hw_buf
[i
] = cpu_to_hc32(ehci
, (u32
)addr
);
66 qtd
->hw_buf_hi
[i
] = cpu_to_hc32(ehci
,
69 if ((count
+ 0x1000) < len
)
75 /* short packets may only terminate transfers */
77 count
-= (count
% maxpacket
);
79 qtd
->hw_token
= cpu_to_hc32(ehci
, (count
<< 16) | token
);
85 /*-------------------------------------------------------------------------*/
88 qh_update (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
, struct ehci_qtd
*qtd
)
90 struct ehci_qh_hw
*hw
= qh
->hw
;
92 /* writes to an active overlay are unsafe */
93 BUG_ON(qh
->qh_state
!= QH_STATE_IDLE
);
95 hw
->hw_qtd_next
= QTD_NEXT(ehci
, qtd
->qtd_dma
);
96 hw
->hw_alt_next
= EHCI_LIST_END(ehci
);
98 /* Except for control endpoints, we make hardware maintain data
99 * toggle (like OHCI) ... here (re)initialize the toggle in the QH,
100 * and set the pseudo-toggle in udev. Only usb_clear_halt() will
103 if (!(hw
->hw_info1
& cpu_to_hc32(ehci
, QH_TOGGLE_CTL
))) {
104 unsigned is_out
, epnum
;
107 epnum
= (hc32_to_cpup(ehci
, &hw
->hw_info1
) >> 8) & 0x0f;
108 if (unlikely (!usb_gettoggle (qh
->dev
, epnum
, is_out
))) {
109 hw
->hw_token
&= ~cpu_to_hc32(ehci
, QTD_TOGGLE
);
110 usb_settoggle (qh
->dev
, epnum
, is_out
, 1);
114 hw
->hw_token
&= cpu_to_hc32(ehci
, QTD_TOGGLE
| QTD_STS_PING
);
117 /* if it weren't for a common silicon quirk (writing the dummy into the qh
118 * overlay, so qh->hw_token wrongly becomes inactive/halted), only fault
119 * recovery (including urb dequeue) would need software changes to a QH...
122 qh_refresh (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
124 struct ehci_qtd
*qtd
;
126 if (list_empty (&qh
->qtd_list
))
129 qtd
= list_entry (qh
->qtd_list
.next
,
130 struct ehci_qtd
, qtd_list
);
132 * first qtd may already be partially processed.
133 * If we come here during unlink, the QH overlay region
134 * might have reference to the just unlinked qtd. The
135 * qtd is updated in qh_completions(). Update the QH
138 if (qh
->hw
->hw_token
& ACTIVE_BIT(ehci
)) {
139 qh
->hw
->hw_qtd_next
= qtd
->hw_next
;
145 qh_update (ehci
, qh
, qtd
);
148 /*-------------------------------------------------------------------------*/
150 static void qh_link_async(struct ehci_hcd
*ehci
, struct ehci_qh
*qh
);
152 static void ehci_clear_tt_buffer_complete(struct usb_hcd
*hcd
,
153 struct usb_host_endpoint
*ep
)
155 struct ehci_hcd
*ehci
= hcd_to_ehci(hcd
);
156 struct ehci_qh
*qh
= ep
->hcpriv
;
159 spin_lock_irqsave(&ehci
->lock
, flags
);
161 if (qh
->qh_state
== QH_STATE_IDLE
&& !list_empty(&qh
->qtd_list
)
162 && ehci
->rh_state
== EHCI_RH_RUNNING
)
163 qh_link_async(ehci
, qh
);
164 spin_unlock_irqrestore(&ehci
->lock
, flags
);
167 static void ehci_clear_tt_buffer(struct ehci_hcd
*ehci
, struct ehci_qh
*qh
,
168 struct urb
*urb
, u32 token
)
171 /* If an async split transaction gets an error or is unlinked,
172 * the TT buffer may be left in an indeterminate state. We
173 * have to clear the TT buffer.
175 * Note: this routine is never called for Isochronous transfers.
177 if (urb
->dev
->tt
&& !usb_pipeint(urb
->pipe
) && !qh
->clearing_tt
) {
179 struct usb_device
*tt
= urb
->dev
->tt
->hub
;
181 "clear tt buffer port %d, a%d ep%d t%08x\n",
182 urb
->dev
->ttport
, urb
->dev
->devnum
,
183 usb_pipeendpoint(urb
->pipe
), token
);
185 if (!ehci_is_TDI(ehci
)
186 || urb
->dev
->tt
->hub
!=
187 ehci_to_hcd(ehci
)->self
.root_hub
) {
188 if (usb_hub_clear_tt_buffer(urb
) == 0)
192 /* REVISIT ARC-derived cores don't clear the root
193 * hub TT buffer in this way...
199 static int qtd_copy_status (
200 struct ehci_hcd
*ehci
,
206 int status
= -EINPROGRESS
;
208 /* count IN/OUT bytes, not SETUP (even short packets) */
209 if (likely (QTD_PID (token
) != 2))
210 urb
->actual_length
+= length
- QTD_LENGTH (token
);
212 /* don't modify error codes */
213 if (unlikely(urb
->unlinked
))
216 /* force cleanup after short read; not always an error */
217 if (unlikely (IS_SHORT_READ (token
)))
220 /* serious "can't proceed" faults reported by the hardware */
221 if (token
& QTD_STS_HALT
) {
222 if (token
& QTD_STS_BABBLE
) {
223 /* FIXME "must" disable babbling device's port too */
225 /* CERR nonzero + halt --> stall */
226 } else if (QTD_CERR(token
)) {
229 /* In theory, more than one of the following bits can be set
230 * since they are sticky and the transaction is retried.
231 * Which to test first is rather arbitrary.
233 } else if (token
& QTD_STS_MMF
) {
234 /* fs/ls interrupt xfer missed the complete-split */
236 } else if (token
& QTD_STS_DBE
) {
237 status
= (QTD_PID (token
) == 1) /* IN ? */
238 ? -ENOSR
/* hc couldn't read data */
239 : -ECOMM
; /* hc couldn't write data */
240 } else if (token
& QTD_STS_XACT
) {
241 /* timeout, bad CRC, wrong PID, etc */
242 ehci_dbg(ehci
, "devpath %s ep%d%s 3strikes\n",
244 usb_pipeendpoint(urb
->pipe
),
245 usb_pipein(urb
->pipe
) ? "in" : "out");
247 } else { /* unknown */
252 "dev%d ep%d%s qtd token %08x --> status %d\n",
253 usb_pipedevice (urb
->pipe
),
254 usb_pipeendpoint (urb
->pipe
),
255 usb_pipein (urb
->pipe
) ? "in" : "out",
263 ehci_urb_done(struct ehci_hcd
*ehci
, struct urb
*urb
, int status
)
264 __releases(ehci
->lock
)
265 __acquires(ehci
->lock
)
267 if (usb_pipetype(urb
->pipe
) == PIPE_INTERRUPT
) {
268 /* ... update hc-wide periodic stats */
269 ehci_to_hcd(ehci
)->self
.bandwidth_int_reqs
--;
272 if (unlikely(urb
->unlinked
)) {
273 COUNT(ehci
->stats
.unlink
);
275 /* report non-error and short read status as zero */
276 if (status
== -EINPROGRESS
|| status
== -EREMOTEIO
)
278 COUNT(ehci
->stats
.complete
);
281 #ifdef EHCI_URB_TRACE
283 "%s %s urb %p ep%d%s status %d len %d/%d\n",
284 __func__
, urb
->dev
->devpath
, urb
,
285 usb_pipeendpoint (urb
->pipe
),
286 usb_pipein (urb
->pipe
) ? "in" : "out",
288 urb
->actual_length
, urb
->transfer_buffer_length
);
291 /* complete() can reenter this HCD */
292 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci
), urb
);
293 spin_unlock (&ehci
->lock
);
294 usb_hcd_giveback_urb(ehci_to_hcd(ehci
), urb
, status
);
295 spin_lock (&ehci
->lock
);
298 static int qh_schedule (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
);
301 * Process and free completed qtds for a qh, returning URBs to drivers.
302 * Chases up to qh->hw_current. Returns number of completions called,
303 * indicating how much "real" work we did.
306 qh_completions (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
308 struct ehci_qtd
*last
, *end
= qh
->dummy
;
309 struct list_head
*entry
, *tmp
;
314 struct ehci_qh_hw
*hw
= qh
->hw
;
316 if (unlikely (list_empty (&qh
->qtd_list
)))
319 /* completions (or tasks on other cpus) must never clobber HALT
320 * till we've gone through and cleaned everything up, even when
321 * they add urbs to this qh's queue or mark them for unlinking.
323 * NOTE: unlinking expects to be done in queue order.
325 * It's a bug for qh->qh_state to be anything other than
326 * QH_STATE_IDLE, unless our caller is scan_async() or
329 state
= qh
->qh_state
;
330 qh
->qh_state
= QH_STATE_COMPLETING
;
331 stopped
= (state
== QH_STATE_IDLE
);
335 last_status
= -EINPROGRESS
;
336 qh
->needs_rescan
= 0;
338 /* remove de-activated QTDs from front of queue.
339 * after faults (including short reads), cleanup this urb
340 * then let the queue advance.
341 * if queue is stopped, handles unlinks.
343 list_for_each_safe (entry
, tmp
, &qh
->qtd_list
) {
344 struct ehci_qtd
*qtd
;
348 qtd
= list_entry (entry
, struct ehci_qtd
, qtd_list
);
351 /* clean up any state from previous QTD ...*/
353 if (likely (last
->urb
!= urb
)) {
354 ehci_urb_done(ehci
, last
->urb
, last_status
);
356 last_status
= -EINPROGRESS
;
358 ehci_qtd_free (ehci
, last
);
362 /* ignore urbs submitted during completions we reported */
366 /* hardware copies qtd out of qh overlay */
368 token
= hc32_to_cpu(ehci
, qtd
->hw_token
);
370 /* always clean up qtds the hc de-activated */
372 if ((token
& QTD_STS_ACTIVE
) == 0) {
374 /* Report Data Buffer Error: non-fatal but useful */
375 if (token
& QTD_STS_DBE
)
377 "detected DataBufferErr for urb %p ep%d%s len %d, qtd %p [qh %p]\n",
379 usb_endpoint_num(&urb
->ep
->desc
),
380 usb_endpoint_dir_in(&urb
->ep
->desc
) ? "in" : "out",
381 urb
->transfer_buffer_length
,
385 /* on STALL, error, and short reads this urb must
386 * complete and all its qtds must be recycled.
388 if ((token
& QTD_STS_HALT
) != 0) {
390 /* retry transaction errors until we
391 * reach the software xacterr limit
393 if ((token
& QTD_STS_XACT
) &&
394 QTD_CERR(token
) == 0 &&
395 ++qh
->xacterrs
< QH_XACTERR_MAX
&&
398 "detected XactErr len %zu/%zu retry %d\n",
399 qtd
->length
- QTD_LENGTH(token
), qtd
->length
, qh
->xacterrs
);
401 /* reset the token in the qtd and the
402 * qh overlay (which still contains
403 * the qtd) so that we pick up from
406 token
&= ~QTD_STS_HALT
;
407 token
|= QTD_STS_ACTIVE
|
408 (EHCI_TUNE_CERR
<< 10);
409 qtd
->hw_token
= cpu_to_hc32(ehci
,
412 hw
->hw_token
= cpu_to_hc32(ehci
,
418 /* magic dummy for some short reads; qh won't advance.
419 * that silicon quirk can kick in with this dummy too.
421 * other short reads won't stop the queue, including
422 * control transfers (status stage handles that) or
423 * most other single-qtd reads ... the queue stops if
424 * URB_SHORT_NOT_OK was set so the driver submitting
425 * the urbs could clean it up.
427 } else if (IS_SHORT_READ (token
)
428 && !(qtd
->hw_alt_next
429 & EHCI_LIST_END(ehci
))) {
433 /* stop scanning when we reach qtds the hc is using */
434 } else if (likely (!stopped
435 && ehci
->rh_state
>= EHCI_RH_RUNNING
)) {
438 /* scan the whole queue for unlinks whenever it stops */
442 /* cancel everything if we halt, suspend, etc */
443 if (ehci
->rh_state
< EHCI_RH_RUNNING
)
444 last_status
= -ESHUTDOWN
;
446 /* this qtd is active; skip it unless a previous qtd
447 * for its urb faulted, or its urb was canceled.
449 else if (last_status
== -EINPROGRESS
&& !urb
->unlinked
)
453 * If this was the active qtd when the qh was unlinked
454 * and the overlay's token is active, then the overlay
455 * hasn't been written back to the qtd yet so use its
456 * token instead of the qtd's. After the qtd is
457 * processed and removed, the overlay won't be valid
460 if (state
== QH_STATE_IDLE
&&
461 qh
->qtd_list
.next
== &qtd
->qtd_list
&&
462 (hw
->hw_token
& ACTIVE_BIT(ehci
))) {
463 token
= hc32_to_cpu(ehci
, hw
->hw_token
);
464 hw
->hw_token
&= ~ACTIVE_BIT(ehci
);
466 /* An unlink may leave an incomplete
467 * async transaction in the TT buffer.
468 * We have to clear it.
470 ehci_clear_tt_buffer(ehci
, qh
, urb
, token
);
474 /* unless we already know the urb's status, collect qtd status
475 * and update count of bytes transferred. in common short read
476 * cases with only one data qtd (including control transfers),
477 * queue processing won't halt. but with two or more qtds (for
478 * example, with a 32 KB transfer), when the first qtd gets a
479 * short read the second must be removed by hand.
481 if (last_status
== -EINPROGRESS
) {
482 last_status
= qtd_copy_status(ehci
, urb
,
484 if (last_status
== -EREMOTEIO
486 & EHCI_LIST_END(ehci
)))
487 last_status
= -EINPROGRESS
;
489 /* As part of low/full-speed endpoint-halt processing
490 * we must clear the TT buffer (11.17.5).
492 if (unlikely(last_status
!= -EINPROGRESS
&&
493 last_status
!= -EREMOTEIO
)) {
494 /* The TT's in some hubs malfunction when they
495 * receive this request following a STALL (they
496 * stop sending isochronous packets). Since a
497 * STALL can't leave the TT buffer in a busy
498 * state (if you believe Figures 11-48 - 11-51
499 * in the USB 2.0 spec), we won't clear the TT
500 * buffer in this case. Strictly speaking this
501 * is a violation of the spec.
503 if (last_status
!= -EPIPE
)
504 ehci_clear_tt_buffer(ehci
, qh
, urb
,
509 /* if we're removing something not at the queue head,
510 * patch the hardware queue pointer.
512 if (stopped
&& qtd
->qtd_list
.prev
!= &qh
->qtd_list
) {
513 last
= list_entry (qtd
->qtd_list
.prev
,
514 struct ehci_qtd
, qtd_list
);
515 last
->hw_next
= qtd
->hw_next
;
518 /* remove qtd; it's recycled after possible urb completion */
519 list_del (&qtd
->qtd_list
);
522 /* reinit the xacterr counter for the next qtd */
526 /* last urb's completion might still need calling */
527 if (likely (last
!= NULL
)) {
528 ehci_urb_done(ehci
, last
->urb
, last_status
);
530 ehci_qtd_free (ehci
, last
);
533 /* Do we need to rescan for URBs dequeued during a giveback? */
534 if (unlikely(qh
->needs_rescan
)) {
535 /* If the QH is already unlinked, do the rescan now. */
536 if (state
== QH_STATE_IDLE
)
539 /* Otherwise we have to wait until the QH is fully unlinked.
540 * Our caller will start an unlink if qh->needs_rescan is
541 * set. But if an unlink has already started, nothing needs
544 if (state
!= QH_STATE_LINKED
)
545 qh
->needs_rescan
= 0;
548 /* restore original state; caller must unlink or relink */
549 qh
->qh_state
= state
;
551 /* be sure the hardware's done with the qh before refreshing
552 * it after fault cleanup, or recovering from silicon wrongly
553 * overlaying the dummy qtd (which reduces DMA chatter).
555 if (stopped
!= 0 || hw
->hw_qtd_next
== EHCI_LIST_END(ehci
)) {
558 qh_refresh(ehci
, qh
);
560 case QH_STATE_LINKED
:
561 /* We won't refresh a QH that's linked (after the HC
562 * stopped the queue). That avoids a race:
563 * - HC reads first part of QH;
564 * - CPU updates that first part and the token;
565 * - HC reads rest of that QH, including token
566 * Result: HC gets an inconsistent image, and then
567 * DMAs to/from the wrong memory (corrupting it).
569 * That should be rare for interrupt transfers,
570 * except maybe high bandwidth ...
573 /* Tell the caller to start an unlink */
574 qh
->needs_rescan
= 1;
576 /* otherwise, unlink already started */
583 /*-------------------------------------------------------------------------*/
585 // high bandwidth multiplier, as encoded in highspeed endpoint descriptors
586 #define hb_mult(wMaxPacketSize) (1 + (((wMaxPacketSize) >> 11) & 0x03))
587 // ... and packet size, for any kind of endpoint descriptor
588 #define max_packet(wMaxPacketSize) ((wMaxPacketSize) & 0x07ff)
591 * reverse of qh_urb_transaction: free a list of TDs.
592 * used for cleanup after errors, before HC sees an URB's TDs.
594 static void qtd_list_free (
595 struct ehci_hcd
*ehci
,
597 struct list_head
*qtd_list
599 struct list_head
*entry
, *temp
;
601 list_for_each_safe (entry
, temp
, qtd_list
) {
602 struct ehci_qtd
*qtd
;
604 qtd
= list_entry (entry
, struct ehci_qtd
, qtd_list
);
605 list_del (&qtd
->qtd_list
);
606 ehci_qtd_free (ehci
, qtd
);
611 * create a list of filled qtds for this URB; won't link into qh.
613 static struct list_head
*
615 struct ehci_hcd
*ehci
,
617 struct list_head
*head
,
620 struct ehci_qtd
*qtd
, *qtd_prev
;
622 int len
, this_sg_len
, maxpacket
;
626 struct scatterlist
*sg
;
629 * URBs map to sequences of QTDs: one logical transaction
631 qtd
= ehci_qtd_alloc (ehci
, flags
);
634 list_add_tail (&qtd
->qtd_list
, head
);
637 token
= QTD_STS_ACTIVE
;
638 token
|= (EHCI_TUNE_CERR
<< 10);
639 /* for split transactions, SplitXState initialized to zero */
641 len
= urb
->transfer_buffer_length
;
642 is_input
= usb_pipein (urb
->pipe
);
643 if (usb_pipecontrol (urb
->pipe
)) {
645 qtd_fill(ehci
, qtd
, urb
->setup_dma
,
646 sizeof (struct usb_ctrlrequest
),
647 token
| (2 /* "setup" */ << 8), 8);
649 /* ... and always at least one more pid */
652 qtd
= ehci_qtd_alloc (ehci
, flags
);
656 qtd_prev
->hw_next
= QTD_NEXT(ehci
, qtd
->qtd_dma
);
657 list_add_tail (&qtd
->qtd_list
, head
);
659 /* for zero length DATA stages, STATUS is always IN */
661 token
|= (1 /* "in" */ << 8);
665 * data transfer stage: buffer setup
667 i
= urb
->num_mapped_sgs
;
668 if (len
> 0 && i
> 0) {
670 buf
= sg_dma_address(sg
);
672 /* urb->transfer_buffer_length may be smaller than the
673 * size of the scatterlist (or vice versa)
675 this_sg_len
= min_t(int, sg_dma_len(sg
), len
);
678 buf
= urb
->transfer_dma
;
683 token
|= (1 /* "in" */ << 8);
684 /* else it's already initted to "out" pid (0 << 8) */
686 maxpacket
= max_packet(usb_maxpacket(urb
->dev
, urb
->pipe
, !is_input
));
689 * buffer gets wrapped in one or more qtds;
690 * last one may be "short" (including zero len)
691 * and may serve as a control status ack
696 this_qtd_len
= qtd_fill(ehci
, qtd
, buf
, this_sg_len
, token
,
698 this_sg_len
-= this_qtd_len
;
703 * short reads advance to a "magic" dummy instead of the next
704 * qtd ... that forces the queue to stop, for manual cleanup.
705 * (this will usually be overridden later.)
708 qtd
->hw_alt_next
= ehci
->async
->hw
->hw_alt_next
;
710 /* qh makes control packets use qtd toggle; maybe switch it */
711 if ((maxpacket
& (this_qtd_len
+ (maxpacket
- 1))) == 0)
714 if (likely(this_sg_len
<= 0)) {
715 if (--i
<= 0 || len
<= 0)
718 buf
= sg_dma_address(sg
);
719 this_sg_len
= min_t(int, sg_dma_len(sg
), len
);
723 qtd
= ehci_qtd_alloc (ehci
, flags
);
727 qtd_prev
->hw_next
= QTD_NEXT(ehci
, qtd
->qtd_dma
);
728 list_add_tail (&qtd
->qtd_list
, head
);
732 * unless the caller requires manual cleanup after short reads,
733 * have the alt_next mechanism keep the queue running after the
734 * last data qtd (the only one, for control and most other cases).
736 if (likely ((urb
->transfer_flags
& URB_SHORT_NOT_OK
) == 0
737 || usb_pipecontrol (urb
->pipe
)))
738 qtd
->hw_alt_next
= EHCI_LIST_END(ehci
);
741 * control requests may need a terminating data "status" ack;
742 * other OUT ones may need a terminating short packet
745 if (likely (urb
->transfer_buffer_length
!= 0)) {
748 if (usb_pipecontrol (urb
->pipe
)) {
750 token
^= 0x0100; /* "in" <--> "out" */
751 token
|= QTD_TOGGLE
; /* force DATA1 */
752 } else if (usb_pipeout(urb
->pipe
)
753 && (urb
->transfer_flags
& URB_ZERO_PACKET
)
754 && !(urb
->transfer_buffer_length
% maxpacket
)) {
759 qtd
= ehci_qtd_alloc (ehci
, flags
);
763 qtd_prev
->hw_next
= QTD_NEXT(ehci
, qtd
->qtd_dma
);
764 list_add_tail (&qtd
->qtd_list
, head
);
766 /* never any data in such packets */
767 qtd_fill(ehci
, qtd
, 0, 0, token
, 0);
771 /* by default, enable interrupt on urb completion */
772 if (likely (!(urb
->transfer_flags
& URB_NO_INTERRUPT
)))
773 qtd
->hw_token
|= cpu_to_hc32(ehci
, QTD_IOC
);
777 qtd_list_free (ehci
, urb
, head
);
781 /*-------------------------------------------------------------------------*/
783 // Would be best to create all qh's from config descriptors,
784 // when each interface/altsetting is established. Unlink
785 // any previous qh and cancel its urbs first; endpoints are
786 // implicitly reset then (data toggle too).
787 // That'd mean updating how usbcore talks to HCDs. (2.7?)
791 * Each QH holds a qtd list; a QH is used for everything except iso.
793 * For interrupt urbs, the scheduler must set the microframe scheduling
794 * mask(s) each time the QH gets scheduled. For highspeed, that's
795 * just one microframe in the s-mask. For split interrupt transactions
796 * there are additional complications: c-mask, maybe FSTNs.
798 static struct ehci_qh
*
800 struct ehci_hcd
*ehci
,
804 struct ehci_qh
*qh
= ehci_qh_alloc (ehci
, flags
);
805 u32 info1
= 0, info2
= 0;
808 struct usb_tt
*tt
= urb
->dev
->tt
;
809 struct ehci_qh_hw
*hw
;
815 * init endpoint/device data for this QH
817 info1
|= usb_pipeendpoint (urb
->pipe
) << 8;
818 info1
|= usb_pipedevice (urb
->pipe
) << 0;
820 is_input
= usb_pipein (urb
->pipe
);
821 type
= usb_pipetype (urb
->pipe
);
822 maxp
= usb_maxpacket (urb
->dev
, urb
->pipe
, !is_input
);
824 /* 1024 byte maxpacket is a hardware ceiling. High bandwidth
825 * acts like up to 3KB, but is built from smaller packets.
827 if (max_packet(maxp
) > 1024) {
828 ehci_dbg(ehci
, "bogus qh maxpacket %d\n", max_packet(maxp
));
832 /* Compute interrupt scheduling parameters just once, and save.
833 * - allowing for high bandwidth, how many nsec/uframe are used?
834 * - split transactions need a second CSPLIT uframe; same question
835 * - splits also need a schedule gap (for full/low speed I/O)
836 * - qh has a polling interval
838 * For control/bulk requests, the HC or TT handles these.
840 if (type
== PIPE_INTERRUPT
) {
841 qh
->usecs
= NS_TO_US(usb_calc_bus_time(USB_SPEED_HIGH
,
843 hb_mult(maxp
) * max_packet(maxp
)));
844 qh
->start
= NO_FRAME
;
846 if (urb
->dev
->speed
== USB_SPEED_HIGH
) {
850 qh
->period
= urb
->interval
>> 3;
851 if (qh
->period
== 0 && urb
->interval
!= 1) {
852 /* NOTE interval 2 or 4 uframes could work.
853 * But interval 1 scheduling is simpler, and
854 * includes high bandwidth.
857 } else if (qh
->period
> ehci
->periodic_size
) {
858 qh
->period
= ehci
->periodic_size
;
859 urb
->interval
= qh
->period
<< 3;
864 /* gap is f(FS/LS transfer times) */
865 qh
->gap_uf
= 1 + usb_calc_bus_time (urb
->dev
->speed
,
866 is_input
, 0, maxp
) / (125 * 1000);
868 /* FIXME this just approximates SPLIT/CSPLIT times */
869 if (is_input
) { // SPLIT, gap, CSPLIT+DATA
870 qh
->c_usecs
= qh
->usecs
+ HS_USECS (0);
871 qh
->usecs
= HS_USECS (1);
872 } else { // SPLIT+DATA, gap, CSPLIT
873 qh
->usecs
+= HS_USECS (1);
874 qh
->c_usecs
= HS_USECS (0);
877 think_time
= tt
? tt
->think_time
: 0;
878 qh
->tt_usecs
= NS_TO_US (think_time
+
879 usb_calc_bus_time (urb
->dev
->speed
,
880 is_input
, 0, max_packet (maxp
)));
881 qh
->period
= urb
->interval
;
882 if (qh
->period
> ehci
->periodic_size
) {
883 qh
->period
= ehci
->periodic_size
;
884 urb
->interval
= qh
->period
;
889 /* support for tt scheduling, and access to toggles */
893 switch (urb
->dev
->speed
) {
895 info1
|= QH_LOW_SPEED
;
899 /* EPS 0 means "full" */
900 if (type
!= PIPE_INTERRUPT
)
901 info1
|= (EHCI_TUNE_RL_TT
<< 28);
902 if (type
== PIPE_CONTROL
) {
903 info1
|= QH_CONTROL_EP
; /* for TT */
904 info1
|= QH_TOGGLE_CTL
; /* toggle from qtd */
908 info2
|= (EHCI_TUNE_MULT_TT
<< 30);
910 /* Some Freescale processors have an erratum in which the
911 * port number in the queue head was 0..N-1 instead of 1..N.
913 if (ehci_has_fsl_portno_bug(ehci
))
914 info2
|= (urb
->dev
->ttport
-1) << 23;
916 info2
|= urb
->dev
->ttport
<< 23;
918 /* set the address of the TT; for TDI's integrated
919 * root hub tt, leave it zeroed.
921 if (tt
&& tt
->hub
!= ehci_to_hcd(ehci
)->self
.root_hub
)
922 info2
|= tt
->hub
->devnum
<< 16;
924 /* NOTE: if (PIPE_INTERRUPT) { scheduler sets c-mask } */
928 case USB_SPEED_HIGH
: /* no TT involved */
929 info1
|= QH_HIGH_SPEED
;
930 if (type
== PIPE_CONTROL
) {
931 info1
|= (EHCI_TUNE_RL_HS
<< 28);
932 info1
|= 64 << 16; /* usb2 fixed maxpacket */
933 info1
|= QH_TOGGLE_CTL
; /* toggle from qtd */
934 info2
|= (EHCI_TUNE_MULT_HS
<< 30);
935 } else if (type
== PIPE_BULK
) {
936 info1
|= (EHCI_TUNE_RL_HS
<< 28);
937 /* The USB spec says that high speed bulk endpoints
938 * always use 512 byte maxpacket. But some device
939 * vendors decided to ignore that, and MSFT is happy
940 * to help them do so. So now people expect to use
941 * such nonconformant devices with Linux too; sigh.
943 info1
|= max_packet(maxp
) << 16;
944 info2
|= (EHCI_TUNE_MULT_HS
<< 30);
945 } else { /* PIPE_INTERRUPT */
946 info1
|= max_packet (maxp
) << 16;
947 info2
|= hb_mult (maxp
) << 30;
951 ehci_dbg(ehci
, "bogus dev %p speed %d\n", urb
->dev
,
954 qh_destroy(ehci
, qh
);
958 /* NOTE: if (PIPE_INTERRUPT) { scheduler sets s-mask } */
960 /* init as live, toggle clear, advance to dummy */
961 qh
->qh_state
= QH_STATE_IDLE
;
963 hw
->hw_info1
= cpu_to_hc32(ehci
, info1
);
964 hw
->hw_info2
= cpu_to_hc32(ehci
, info2
);
965 qh
->is_out
= !is_input
;
966 usb_settoggle (urb
->dev
, usb_pipeendpoint (urb
->pipe
), !is_input
, 1);
967 qh_refresh (ehci
, qh
);
971 /*-------------------------------------------------------------------------*/
973 static void enable_async(struct ehci_hcd
*ehci
)
975 if (ehci
->async_count
++)
978 /* Stop waiting to turn off the async schedule */
979 ehci
->enabled_hrtimer_events
&= ~BIT(EHCI_HRTIMER_DISABLE_ASYNC
);
981 /* Don't start the schedule until ASS is 0 */
983 turn_on_io_watchdog(ehci
);
986 static void disable_async(struct ehci_hcd
*ehci
)
988 if (--ehci
->async_count
)
991 /* The async schedule and async_unlink list are supposed to be empty */
992 WARN_ON(ehci
->async
->qh_next
.qh
|| ehci
->async_unlink
);
994 /* Don't turn off the schedule until ASS is 1 */
998 /* move qh (and its qtds) onto async queue; maybe enable queue. */
1000 static void qh_link_async (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
1002 __hc32 dma
= QH_NEXT(ehci
, qh
->qh_dma
);
1003 struct ehci_qh
*head
;
1005 /* Don't link a QH if there's a Clear-TT-Buffer pending */
1006 if (unlikely(qh
->clearing_tt
))
1009 WARN_ON(qh
->qh_state
!= QH_STATE_IDLE
);
1011 /* clear halt and/or toggle; and maybe recover from silicon quirk */
1012 qh_refresh(ehci
, qh
);
1014 /* splice right after start */
1016 qh
->qh_next
= head
->qh_next
;
1017 qh
->hw
->hw_next
= head
->hw
->hw_next
;
1020 head
->qh_next
.qh
= qh
;
1021 head
->hw
->hw_next
= dma
;
1024 qh
->qh_state
= QH_STATE_LINKED
;
1025 /* qtd completions reported later by interrupt */
1030 /*-------------------------------------------------------------------------*/
1033 * For control/bulk/interrupt, return QH with these TDs appended.
1034 * Allocates and initializes the QH if necessary.
1035 * Returns null if it can't allocate a QH it needs to.
1036 * If the QH has TDs (urbs) already, that's great.
1038 static struct ehci_qh
*qh_append_tds (
1039 struct ehci_hcd
*ehci
,
1041 struct list_head
*qtd_list
,
1046 struct ehci_qh
*qh
= NULL
;
1047 __hc32 qh_addr_mask
= cpu_to_hc32(ehci
, 0x7f);
1049 qh
= (struct ehci_qh
*) *ptr
;
1050 if (unlikely (qh
== NULL
)) {
1051 /* can't sleep here, we have ehci->lock... */
1052 qh
= qh_make (ehci
, urb
, GFP_ATOMIC
);
1055 if (likely (qh
!= NULL
)) {
1056 struct ehci_qtd
*qtd
;
1058 if (unlikely (list_empty (qtd_list
)))
1061 qtd
= list_entry (qtd_list
->next
, struct ehci_qtd
,
1064 /* control qh may need patching ... */
1065 if (unlikely (epnum
== 0)) {
1067 /* usb_reset_device() briefly reverts to address 0 */
1068 if (usb_pipedevice (urb
->pipe
) == 0)
1069 qh
->hw
->hw_info1
&= ~qh_addr_mask
;
1072 /* just one way to queue requests: swap with the dummy qtd.
1073 * only hc or qh_refresh() ever modify the overlay.
1075 if (likely (qtd
!= NULL
)) {
1076 struct ehci_qtd
*dummy
;
1080 /* to avoid racing the HC, use the dummy td instead of
1081 * the first td of our list (becomes new dummy). both
1082 * tds stay deactivated until we're done, when the
1083 * HC is allowed to fetch the old dummy (4.10.2).
1085 token
= qtd
->hw_token
;
1086 qtd
->hw_token
= HALT_BIT(ehci
);
1090 dma
= dummy
->qtd_dma
;
1092 dummy
->qtd_dma
= dma
;
1094 list_del (&qtd
->qtd_list
);
1095 list_add (&dummy
->qtd_list
, qtd_list
);
1096 list_splice_tail(qtd_list
, &qh
->qtd_list
);
1098 ehci_qtd_init(ehci
, qtd
, qtd
->qtd_dma
);
1101 /* hc must see the new dummy at list end */
1103 qtd
= list_entry (qh
->qtd_list
.prev
,
1104 struct ehci_qtd
, qtd_list
);
1105 qtd
->hw_next
= QTD_NEXT(ehci
, dma
);
1107 /* let the hc process these next qtds */
1109 dummy
->hw_token
= token
;
1117 /*-------------------------------------------------------------------------*/
1121 struct ehci_hcd
*ehci
,
1123 struct list_head
*qtd_list
,
1127 unsigned long flags
;
1128 struct ehci_qh
*qh
= NULL
;
1131 epnum
= urb
->ep
->desc
.bEndpointAddress
;
1133 #ifdef EHCI_URB_TRACE
1135 struct ehci_qtd
*qtd
;
1136 qtd
= list_entry(qtd_list
->next
, struct ehci_qtd
, qtd_list
);
1138 "%s %s urb %p ep%d%s len %d, qtd %p [qh %p]\n",
1139 __func__
, urb
->dev
->devpath
, urb
,
1140 epnum
& 0x0f, (epnum
& USB_DIR_IN
) ? "in" : "out",
1141 urb
->transfer_buffer_length
,
1142 qtd
, urb
->ep
->hcpriv
);
1146 spin_lock_irqsave (&ehci
->lock
, flags
);
1147 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci
)))) {
1151 rc
= usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci
), urb
);
1155 qh
= qh_append_tds(ehci
, urb
, qtd_list
, epnum
, &urb
->ep
->hcpriv
);
1156 if (unlikely(qh
== NULL
)) {
1157 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci
), urb
);
1162 /* Control/bulk operations through TTs don't need scheduling,
1163 * the HC and TT handle it when the TT has a buffer ready.
1165 if (likely (qh
->qh_state
== QH_STATE_IDLE
))
1166 qh_link_async(ehci
, qh
);
1168 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1169 if (unlikely (qh
== NULL
))
1170 qtd_list_free (ehci
, urb
, qtd_list
);
1174 /*-------------------------------------------------------------------------*/
1176 static void single_unlink_async(struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
1178 struct ehci_qh
*prev
;
1180 /* Add to the end of the list of QHs waiting for the next IAAD */
1181 qh
->qh_state
= QH_STATE_UNLINK_WAIT
;
1182 if (ehci
->async_unlink
)
1183 ehci
->async_unlink_last
->unlink_next
= qh
;
1185 ehci
->async_unlink
= qh
;
1186 ehci
->async_unlink_last
= qh
;
1188 /* Unlink it from the schedule */
1190 while (prev
->qh_next
.qh
!= qh
)
1191 prev
= prev
->qh_next
.qh
;
1193 prev
->hw
->hw_next
= qh
->hw
->hw_next
;
1194 prev
->qh_next
= qh
->qh_next
;
1195 if (ehci
->qh_scan_next
== qh
)
1196 ehci
->qh_scan_next
= qh
->qh_next
.qh
;
1199 static void start_iaa_cycle(struct ehci_hcd
*ehci
, bool nested
)
1202 * Do nothing if an IAA cycle is already running or
1203 * if one will be started shortly.
1205 if (ehci
->async_iaa
|| ehci
->async_unlinking
)
1208 /* If the controller isn't running, we don't have to wait for it */
1209 if (unlikely(ehci
->rh_state
< EHCI_RH_RUNNING
)) {
1211 /* Do all the waiting QHs */
1212 ehci
->async_iaa
= ehci
->async_unlink
;
1213 ehci
->async_unlink
= NULL
;
1215 if (!nested
) /* Avoid recursion */
1216 end_unlink_async(ehci
);
1218 /* Otherwise start a new IAA cycle */
1219 } else if (likely(ehci
->rh_state
== EHCI_RH_RUNNING
)) {
1222 /* Do only the first waiting QH (nVidia bug?) */
1223 qh
= ehci
->async_unlink
;
1226 * Intel (?) bug: The HC can write back the overlay region
1227 * even after the IAA interrupt occurs. In self-defense,
1228 * always go through two IAA cycles for each QH.
1230 if (qh
->qh_state
== QH_STATE_UNLINK_WAIT
) {
1231 qh
->qh_state
= QH_STATE_UNLINK
;
1233 ehci
->async_iaa
= qh
;
1234 ehci
->async_unlink
= qh
->unlink_next
;
1235 qh
->unlink_next
= NULL
;
1238 /* Make sure the unlinks are all visible to the hardware */
1241 ehci_writel(ehci
, ehci
->command
| CMD_IAAD
,
1242 &ehci
->regs
->command
);
1243 ehci_readl(ehci
, &ehci
->regs
->command
);
1244 ehci_enable_event(ehci
, EHCI_HRTIMER_IAA_WATCHDOG
, true);
1248 /* the async qh for the qtds being unlinked are now gone from the HC */
1250 static void end_unlink_async(struct ehci_hcd
*ehci
)
1254 if (ehci
->has_synopsys_hc_bug
)
1255 ehci_writel(ehci
, (u32
) ehci
->async
->qh_dma
,
1256 &ehci
->regs
->async_next
);
1258 /* Process the idle QHs */
1260 ehci
->async_unlinking
= true;
1261 while (ehci
->async_iaa
) {
1262 qh
= ehci
->async_iaa
;
1263 ehci
->async_iaa
= qh
->unlink_next
;
1264 qh
->unlink_next
= NULL
;
1266 qh
->qh_state
= QH_STATE_IDLE
;
1267 qh
->qh_next
.qh
= NULL
;
1269 qh_completions(ehci
, qh
);
1270 if (!list_empty(&qh
->qtd_list
) &&
1271 ehci
->rh_state
== EHCI_RH_RUNNING
)
1272 qh_link_async(ehci
, qh
);
1273 disable_async(ehci
);
1275 ehci
->async_unlinking
= false;
1277 /* Start a new IAA cycle if any QHs are waiting for it */
1278 if (ehci
->async_unlink
) {
1279 start_iaa_cycle(ehci
, true);
1280 if (unlikely(ehci
->rh_state
< EHCI_RH_RUNNING
))
1285 static void start_unlink_async(struct ehci_hcd
*ehci
, struct ehci_qh
*qh
);
1287 static void unlink_empty_async(struct ehci_hcd
*ehci
)
1290 struct ehci_qh
*qh_to_unlink
= NULL
;
1291 bool check_unlinks_later
= false;
1294 /* Find the last async QH which has been empty for a timer cycle */
1295 for (qh
= ehci
->async
->qh_next
.qh
; qh
; qh
= qh
->qh_next
.qh
) {
1296 if (list_empty(&qh
->qtd_list
) &&
1297 qh
->qh_state
== QH_STATE_LINKED
) {
1299 if (qh
->unlink_cycle
== ehci
->async_unlink_cycle
)
1300 check_unlinks_later
= true;
1306 /* If nothing else is being unlinked, unlink the last empty QH */
1307 if (!ehci
->async_iaa
&& !ehci
->async_unlink
&& qh_to_unlink
) {
1308 start_unlink_async(ehci
, qh_to_unlink
);
1312 /* Other QHs will be handled later */
1314 ehci_enable_event(ehci
, EHCI_HRTIMER_ASYNC_UNLINKS
, true);
1315 ++ehci
->async_unlink_cycle
;
1319 /* The root hub is suspended; unlink all the async QHs */
1320 static void unlink_empty_async_suspended(struct ehci_hcd
*ehci
)
1324 while (ehci
->async
->qh_next
.qh
) {
1325 qh
= ehci
->async
->qh_next
.qh
;
1326 WARN_ON(!list_empty(&qh
->qtd_list
));
1327 single_unlink_async(ehci
, qh
);
1329 start_iaa_cycle(ehci
, false);
1332 /* makes sure the async qh will become idle */
1333 /* caller must own ehci->lock */
1335 static void start_unlink_async(struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
1338 * If the QH isn't linked then there's nothing we can do
1339 * unless we were called during a giveback, in which case
1340 * qh_completions() has to deal with it.
1342 if (qh
->qh_state
!= QH_STATE_LINKED
) {
1343 if (qh
->qh_state
== QH_STATE_COMPLETING
)
1344 qh
->needs_rescan
= 1;
1348 single_unlink_async(ehci
, qh
);
1349 start_iaa_cycle(ehci
, false);
1352 /*-------------------------------------------------------------------------*/
1354 static void scan_async (struct ehci_hcd
*ehci
)
1357 bool check_unlinks_later
= false;
1359 ehci
->qh_scan_next
= ehci
->async
->qh_next
.qh
;
1360 while (ehci
->qh_scan_next
) {
1361 qh
= ehci
->qh_scan_next
;
1362 ehci
->qh_scan_next
= qh
->qh_next
.qh
;
1364 /* clean any finished work for this qh */
1365 if (!list_empty(&qh
->qtd_list
)) {
1369 * Unlinks could happen here; completion reporting
1370 * drops the lock. That's why ehci->qh_scan_next
1371 * always holds the next qh to scan; if the next qh
1372 * gets unlinked then ehci->qh_scan_next is adjusted
1373 * in single_unlink_async().
1375 temp
= qh_completions(ehci
, qh
);
1376 if (qh
->needs_rescan
) {
1377 start_unlink_async(ehci
, qh
);
1378 } else if (list_empty(&qh
->qtd_list
)
1379 && qh
->qh_state
== QH_STATE_LINKED
) {
1380 qh
->unlink_cycle
= ehci
->async_unlink_cycle
;
1381 check_unlinks_later
= true;
1382 } else if (temp
!= 0)
1388 * Unlink empty entries, reducing DMA usage as well
1389 * as HCD schedule-scanning costs. Delay for any qh
1390 * we just scanned, there's a not-unusual case that it
1391 * doesn't stay idle for long.
1393 if (check_unlinks_later
&& ehci
->rh_state
== EHCI_RH_RUNNING
&&
1394 !(ehci
->enabled_hrtimer_events
&
1395 BIT(EHCI_HRTIMER_ASYNC_UNLINKS
))) {
1396 ehci_enable_event(ehci
, EHCI_HRTIMER_ASYNC_UNLINKS
, true);
1397 ++ehci
->async_unlink_cycle
;