2 * Back-end of the driver for virtual network devices. This portion of the
3 * driver exports a 'unified' network-device interface that can be accessed
4 * by any operating system that implements a compatible front end. A
5 * reference front-end implementation can be found in:
6 * drivers/net/xen-netfront.c
8 * Copyright (c) 2002-2005, K A Fraser
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
37 #include <linux/kthread.h>
38 #include <linux/if_vlan.h>
39 #include <linux/udp.h>
40 #include <linux/highmem.h>
45 #include <xen/events.h>
46 #include <xen/interface/memory.h>
48 #include <asm/xen/hypercall.h>
49 #include <asm/xen/page.h>
51 /* Provide an option to disable split event channels at load time as
52 * event channels are limited resource. Split event channels are
55 bool separate_tx_rx_irq
= 1;
56 module_param(separate_tx_rx_irq
, bool, 0644);
58 /* The time that packets can stay on the guest Rx internal queue
59 * before they are dropped.
61 unsigned int rx_drain_timeout_msecs
= 10000;
62 module_param(rx_drain_timeout_msecs
, uint
, 0444);
64 /* The length of time before the frontend is considered unresponsive
65 * because it isn't providing Rx slots.
67 unsigned int rx_stall_timeout_msecs
= 60000;
68 module_param(rx_stall_timeout_msecs
, uint
, 0444);
70 unsigned int xenvif_max_queues
;
71 module_param_named(max_queues
, xenvif_max_queues
, uint
, 0644);
72 MODULE_PARM_DESC(max_queues
,
73 "Maximum number of queues per virtual interface");
76 * This is the maximum slots a skb can have. If a guest sends a skb
77 * which exceeds this limit it is considered malicious.
79 #define FATAL_SKB_SLOTS_DEFAULT 20
80 static unsigned int fatal_skb_slots
= FATAL_SKB_SLOTS_DEFAULT
;
81 module_param(fatal_skb_slots
, uint
, 0444);
83 /* The amount to copy out of the first guest Tx slot into the skb's
84 * linear area. If the first slot has more data, it will be mapped
85 * and put into the first frag.
87 * This is sized to avoid pulling headers from the frags for most
90 #define XEN_NETBACK_TX_COPY_LEN 128
93 static void xenvif_idx_release(struct xenvif_queue
*queue
, u16 pending_idx
,
96 static void make_tx_response(struct xenvif_queue
*queue
,
97 struct xen_netif_tx_request
*txp
,
100 static inline int tx_work_todo(struct xenvif_queue
*queue
);
102 static struct xen_netif_rx_response
*make_rx_response(struct xenvif_queue
*queue
,
109 static inline unsigned long idx_to_pfn(struct xenvif_queue
*queue
,
112 return page_to_pfn(queue
->mmap_pages
[idx
]);
115 static inline unsigned long idx_to_kaddr(struct xenvif_queue
*queue
,
118 return (unsigned long)pfn_to_kaddr(idx_to_pfn(queue
, idx
));
121 #define callback_param(vif, pending_idx) \
122 (vif->pending_tx_info[pending_idx].callback_struct)
124 /* Find the containing VIF's structure from a pointer in pending_tx_info array
126 static inline struct xenvif_queue
*ubuf_to_queue(const struct ubuf_info
*ubuf
)
128 u16 pending_idx
= ubuf
->desc
;
129 struct pending_tx_info
*temp
=
130 container_of(ubuf
, struct pending_tx_info
, callback_struct
);
131 return container_of(temp
- pending_idx
,
136 static u16
frag_get_pending_idx(skb_frag_t
*frag
)
138 return (u16
)frag
->page_offset
;
141 static void frag_set_pending_idx(skb_frag_t
*frag
, u16 pending_idx
)
143 frag
->page_offset
= pending_idx
;
146 static inline pending_ring_idx_t
pending_index(unsigned i
)
148 return i
& (MAX_PENDING_REQS
-1);
151 bool xenvif_rx_ring_slots_available(struct xenvif_queue
*queue
, int needed
)
156 prod
= queue
->rx
.sring
->req_prod
;
157 cons
= queue
->rx
.req_cons
;
159 if (prod
- cons
>= needed
)
162 queue
->rx
.sring
->req_event
= prod
+ 1;
164 /* Make sure event is visible before we check prod
168 } while (queue
->rx
.sring
->req_prod
!= prod
);
173 void xenvif_rx_queue_tail(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
177 spin_lock_irqsave(&queue
->rx_queue
.lock
, flags
);
179 __skb_queue_tail(&queue
->rx_queue
, skb
);
181 queue
->rx_queue_len
+= skb
->len
;
182 if (queue
->rx_queue_len
> queue
->rx_queue_max
)
183 netif_tx_stop_queue(netdev_get_tx_queue(queue
->vif
->dev
, queue
->id
));
185 spin_unlock_irqrestore(&queue
->rx_queue
.lock
, flags
);
188 static struct sk_buff
*xenvif_rx_dequeue(struct xenvif_queue
*queue
)
192 spin_lock_irq(&queue
->rx_queue
.lock
);
194 skb
= __skb_dequeue(&queue
->rx_queue
);
196 queue
->rx_queue_len
-= skb
->len
;
198 spin_unlock_irq(&queue
->rx_queue
.lock
);
203 static void xenvif_rx_queue_maybe_wake(struct xenvif_queue
*queue
)
205 spin_lock_irq(&queue
->rx_queue
.lock
);
207 if (queue
->rx_queue_len
< queue
->rx_queue_max
)
208 netif_tx_wake_queue(netdev_get_tx_queue(queue
->vif
->dev
, queue
->id
));
210 spin_unlock_irq(&queue
->rx_queue
.lock
);
214 static void xenvif_rx_queue_purge(struct xenvif_queue
*queue
)
217 while ((skb
= xenvif_rx_dequeue(queue
)) != NULL
)
221 static void xenvif_rx_queue_drop_expired(struct xenvif_queue
*queue
)
226 skb
= skb_peek(&queue
->rx_queue
);
229 if (time_before(jiffies
, XENVIF_RX_CB(skb
)->expires
))
231 xenvif_rx_dequeue(queue
);
236 struct netrx_pending_operations
{
237 unsigned copy_prod
, copy_cons
;
238 unsigned meta_prod
, meta_cons
;
239 struct gnttab_copy
*copy
;
240 struct xenvif_rx_meta
*meta
;
242 grant_ref_t copy_gref
;
245 static struct xenvif_rx_meta
*get_next_rx_buffer(struct xenvif_queue
*queue
,
246 struct netrx_pending_operations
*npo
)
248 struct xenvif_rx_meta
*meta
;
249 struct xen_netif_rx_request
*req
;
251 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
253 meta
= npo
->meta
+ npo
->meta_prod
++;
254 meta
->gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
260 npo
->copy_gref
= req
->gref
;
266 * Set up the grant operations for this fragment. If it's a flipping
267 * interface, we also set up the unmap request from here.
269 static void xenvif_gop_frag_copy(struct xenvif_queue
*queue
, struct sk_buff
*skb
,
270 struct netrx_pending_operations
*npo
,
271 struct page
*page
, unsigned long size
,
272 unsigned long offset
, int *head
)
274 struct gnttab_copy
*copy_gop
;
275 struct xenvif_rx_meta
*meta
;
277 int gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
279 /* Data must not cross a page boundary. */
280 BUG_ON(size
+ offset
> PAGE_SIZE
<<compound_order(page
));
282 meta
= npo
->meta
+ npo
->meta_prod
- 1;
284 /* Skip unused frames from start of page */
285 page
+= offset
>> PAGE_SHIFT
;
286 offset
&= ~PAGE_MASK
;
289 struct xen_page_foreign
*foreign
;
291 BUG_ON(offset
>= PAGE_SIZE
);
292 BUG_ON(npo
->copy_off
> MAX_BUFFER_OFFSET
);
294 if (npo
->copy_off
== MAX_BUFFER_OFFSET
)
295 meta
= get_next_rx_buffer(queue
, npo
);
297 bytes
= PAGE_SIZE
- offset
;
301 if (npo
->copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
302 bytes
= MAX_BUFFER_OFFSET
- npo
->copy_off
;
304 copy_gop
= npo
->copy
+ npo
->copy_prod
++;
305 copy_gop
->flags
= GNTCOPY_dest_gref
;
306 copy_gop
->len
= bytes
;
308 foreign
= xen_page_foreign(page
);
310 copy_gop
->source
.domid
= foreign
->domid
;
311 copy_gop
->source
.u
.ref
= foreign
->gref
;
312 copy_gop
->flags
|= GNTCOPY_source_gref
;
314 copy_gop
->source
.domid
= DOMID_SELF
;
315 copy_gop
->source
.u
.gmfn
=
316 virt_to_mfn(page_address(page
));
318 copy_gop
->source
.offset
= offset
;
320 copy_gop
->dest
.domid
= queue
->vif
->domid
;
321 copy_gop
->dest
.offset
= npo
->copy_off
;
322 copy_gop
->dest
.u
.ref
= npo
->copy_gref
;
324 npo
->copy_off
+= bytes
;
331 if (offset
== PAGE_SIZE
&& size
) {
332 BUG_ON(!PageCompound(page
));
337 /* Leave a gap for the GSO descriptor. */
338 if (skb_is_gso(skb
)) {
339 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
)
340 gso_type
= XEN_NETIF_GSO_TYPE_TCPV4
;
341 else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
)
342 gso_type
= XEN_NETIF_GSO_TYPE_TCPV6
;
345 if (*head
&& ((1 << gso_type
) & queue
->vif
->gso_mask
))
346 queue
->rx
.req_cons
++;
348 *head
= 0; /* There must be something in this buffer now. */
354 * Prepare an SKB to be transmitted to the frontend.
356 * This function is responsible for allocating grant operations, meta
359 * It returns the number of meta structures consumed. The number of
360 * ring slots used is always equal to the number of meta slots used
361 * plus the number of GSO descriptors used. Currently, we use either
362 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
363 * frontend-side LRO).
365 static int xenvif_gop_skb(struct sk_buff
*skb
,
366 struct netrx_pending_operations
*npo
,
367 struct xenvif_queue
*queue
)
369 struct xenvif
*vif
= netdev_priv(skb
->dev
);
370 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
372 struct xen_netif_rx_request
*req
;
373 struct xenvif_rx_meta
*meta
;
379 old_meta_prod
= npo
->meta_prod
;
381 gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
382 if (skb_is_gso(skb
)) {
383 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
)
384 gso_type
= XEN_NETIF_GSO_TYPE_TCPV4
;
385 else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
)
386 gso_type
= XEN_NETIF_GSO_TYPE_TCPV6
;
389 /* Set up a GSO prefix descriptor, if necessary */
390 if ((1 << gso_type
) & vif
->gso_prefix_mask
) {
391 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
392 meta
= npo
->meta
+ npo
->meta_prod
++;
393 meta
->gso_type
= gso_type
;
394 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
399 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
400 meta
= npo
->meta
+ npo
->meta_prod
++;
402 if ((1 << gso_type
) & vif
->gso_mask
) {
403 meta
->gso_type
= gso_type
;
404 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
406 meta
->gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
413 npo
->copy_gref
= req
->gref
;
416 while (data
< skb_tail_pointer(skb
)) {
417 unsigned int offset
= offset_in_page(data
);
418 unsigned int len
= PAGE_SIZE
- offset
;
420 if (data
+ len
> skb_tail_pointer(skb
))
421 len
= skb_tail_pointer(skb
) - data
;
423 xenvif_gop_frag_copy(queue
, skb
, npo
,
424 virt_to_page(data
), len
, offset
, &head
);
428 for (i
= 0; i
< nr_frags
; i
++) {
429 xenvif_gop_frag_copy(queue
, skb
, npo
,
430 skb_frag_page(&skb_shinfo(skb
)->frags
[i
]),
431 skb_frag_size(&skb_shinfo(skb
)->frags
[i
]),
432 skb_shinfo(skb
)->frags
[i
].page_offset
,
436 return npo
->meta_prod
- old_meta_prod
;
440 * This is a twin to xenvif_gop_skb. Assume that xenvif_gop_skb was
441 * used to set up the operations on the top of
442 * netrx_pending_operations, which have since been done. Check that
443 * they didn't give any errors and advance over them.
445 static int xenvif_check_gop(struct xenvif
*vif
, int nr_meta_slots
,
446 struct netrx_pending_operations
*npo
)
448 struct gnttab_copy
*copy_op
;
449 int status
= XEN_NETIF_RSP_OKAY
;
452 for (i
= 0; i
< nr_meta_slots
; i
++) {
453 copy_op
= npo
->copy
+ npo
->copy_cons
++;
454 if (copy_op
->status
!= GNTST_okay
) {
456 "Bad status %d from copy to DOM%d.\n",
457 copy_op
->status
, vif
->domid
);
458 status
= XEN_NETIF_RSP_ERROR
;
465 static void xenvif_add_frag_responses(struct xenvif_queue
*queue
, int status
,
466 struct xenvif_rx_meta
*meta
,
470 unsigned long offset
;
472 /* No fragments used */
473 if (nr_meta_slots
<= 1)
478 for (i
= 0; i
< nr_meta_slots
; i
++) {
480 if (i
== nr_meta_slots
- 1)
483 flags
= XEN_NETRXF_more_data
;
486 make_rx_response(queue
, meta
[i
].id
, status
, offset
,
487 meta
[i
].size
, flags
);
491 void xenvif_kick_thread(struct xenvif_queue
*queue
)
496 static void xenvif_rx_action(struct xenvif_queue
*queue
)
500 struct xen_netif_rx_response
*resp
;
501 struct sk_buff_head rxq
;
505 unsigned long offset
;
506 bool need_to_notify
= false;
508 struct netrx_pending_operations npo
= {
509 .copy
= queue
->grant_copy_op
,
513 skb_queue_head_init(&rxq
);
515 while (xenvif_rx_ring_slots_available(queue
, XEN_NETBK_RX_SLOTS_MAX
)
516 && (skb
= xenvif_rx_dequeue(queue
)) != NULL
) {
517 RING_IDX old_req_cons
;
518 RING_IDX ring_slots_used
;
520 queue
->last_rx_time
= jiffies
;
522 old_req_cons
= queue
->rx
.req_cons
;
523 XENVIF_RX_CB(skb
)->meta_slots_used
= xenvif_gop_skb(skb
, &npo
, queue
);
524 ring_slots_used
= queue
->rx
.req_cons
- old_req_cons
;
526 __skb_queue_tail(&rxq
, skb
);
529 BUG_ON(npo
.meta_prod
> ARRAY_SIZE(queue
->meta
));
534 BUG_ON(npo
.copy_prod
> MAX_GRANT_COPY_OPS
);
535 gnttab_batch_copy(queue
->grant_copy_op
, npo
.copy_prod
);
537 while ((skb
= __skb_dequeue(&rxq
)) != NULL
) {
539 if ((1 << queue
->meta
[npo
.meta_cons
].gso_type
) &
540 queue
->vif
->gso_prefix_mask
) {
541 resp
= RING_GET_RESPONSE(&queue
->rx
,
542 queue
->rx
.rsp_prod_pvt
++);
544 resp
->flags
= XEN_NETRXF_gso_prefix
| XEN_NETRXF_more_data
;
546 resp
->offset
= queue
->meta
[npo
.meta_cons
].gso_size
;
547 resp
->id
= queue
->meta
[npo
.meta_cons
].id
;
548 resp
->status
= XENVIF_RX_CB(skb
)->meta_slots_used
;
551 XENVIF_RX_CB(skb
)->meta_slots_used
--;
555 queue
->stats
.tx_bytes
+= skb
->len
;
556 queue
->stats
.tx_packets
++;
558 status
= xenvif_check_gop(queue
->vif
,
559 XENVIF_RX_CB(skb
)->meta_slots_used
,
562 if (XENVIF_RX_CB(skb
)->meta_slots_used
== 1)
565 flags
= XEN_NETRXF_more_data
;
567 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) /* local packet? */
568 flags
|= XEN_NETRXF_csum_blank
| XEN_NETRXF_data_validated
;
569 else if (skb
->ip_summed
== CHECKSUM_UNNECESSARY
)
570 /* remote but checksummed. */
571 flags
|= XEN_NETRXF_data_validated
;
574 resp
= make_rx_response(queue
, queue
->meta
[npo
.meta_cons
].id
,
576 queue
->meta
[npo
.meta_cons
].size
,
579 if ((1 << queue
->meta
[npo
.meta_cons
].gso_type
) &
580 queue
->vif
->gso_mask
) {
581 struct xen_netif_extra_info
*gso
=
582 (struct xen_netif_extra_info
*)
583 RING_GET_RESPONSE(&queue
->rx
,
584 queue
->rx
.rsp_prod_pvt
++);
586 resp
->flags
|= XEN_NETRXF_extra_info
;
588 gso
->u
.gso
.type
= queue
->meta
[npo
.meta_cons
].gso_type
;
589 gso
->u
.gso
.size
= queue
->meta
[npo
.meta_cons
].gso_size
;
591 gso
->u
.gso
.features
= 0;
593 gso
->type
= XEN_NETIF_EXTRA_TYPE_GSO
;
597 xenvif_add_frag_responses(queue
, status
,
598 queue
->meta
+ npo
.meta_cons
+ 1,
599 XENVIF_RX_CB(skb
)->meta_slots_used
);
601 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue
->rx
, ret
);
603 need_to_notify
|= !!ret
;
605 npo
.meta_cons
+= XENVIF_RX_CB(skb
)->meta_slots_used
;
611 notify_remote_via_irq(queue
->rx_irq
);
614 void xenvif_napi_schedule_or_enable_events(struct xenvif_queue
*queue
)
618 RING_FINAL_CHECK_FOR_REQUESTS(&queue
->tx
, more_to_do
);
621 napi_schedule(&queue
->napi
);
624 static void tx_add_credit(struct xenvif_queue
*queue
)
626 unsigned long max_burst
, max_credit
;
629 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
630 * Otherwise the interface can seize up due to insufficient credit.
632 max_burst
= RING_GET_REQUEST(&queue
->tx
, queue
->tx
.req_cons
)->size
;
633 max_burst
= min(max_burst
, 131072UL);
634 max_burst
= max(max_burst
, queue
->credit_bytes
);
636 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
637 max_credit
= queue
->remaining_credit
+ queue
->credit_bytes
;
638 if (max_credit
< queue
->remaining_credit
)
639 max_credit
= ULONG_MAX
; /* wrapped: clamp to ULONG_MAX */
641 queue
->remaining_credit
= min(max_credit
, max_burst
);
644 static void tx_credit_callback(unsigned long data
)
646 struct xenvif_queue
*queue
= (struct xenvif_queue
*)data
;
647 tx_add_credit(queue
);
648 xenvif_napi_schedule_or_enable_events(queue
);
651 static void xenvif_tx_err(struct xenvif_queue
*queue
,
652 struct xen_netif_tx_request
*txp
, RING_IDX end
)
654 RING_IDX cons
= queue
->tx
.req_cons
;
660 spin_lock_irqsave(&queue
->response_lock
, flags
);
661 make_tx_response(queue
, txp
, XEN_NETIF_RSP_ERROR
);
662 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue
->tx
, notify
);
663 spin_unlock_irqrestore(&queue
->response_lock
, flags
);
665 notify_remote_via_irq(queue
->tx_irq
);
669 txp
= RING_GET_REQUEST(&queue
->tx
, cons
++);
671 queue
->tx
.req_cons
= cons
;
674 static void xenvif_fatal_tx_err(struct xenvif
*vif
)
676 netdev_err(vif
->dev
, "fatal error; disabling device\n");
677 vif
->disabled
= true;
678 /* Disable the vif from queue 0's kthread */
680 xenvif_kick_thread(&vif
->queues
[0]);
683 static int xenvif_count_requests(struct xenvif_queue
*queue
,
684 struct xen_netif_tx_request
*first
,
685 struct xen_netif_tx_request
*txp
,
688 RING_IDX cons
= queue
->tx
.req_cons
;
693 if (!(first
->flags
& XEN_NETTXF_more_data
))
697 struct xen_netif_tx_request dropped_tx
= { 0 };
699 if (slots
>= work_to_do
) {
700 netdev_err(queue
->vif
->dev
,
701 "Asked for %d slots but exceeds this limit\n",
703 xenvif_fatal_tx_err(queue
->vif
);
707 /* This guest is really using too many slots and
708 * considered malicious.
710 if (unlikely(slots
>= fatal_skb_slots
)) {
711 netdev_err(queue
->vif
->dev
,
712 "Malicious frontend using %d slots, threshold %u\n",
713 slots
, fatal_skb_slots
);
714 xenvif_fatal_tx_err(queue
->vif
);
718 /* Xen network protocol had implicit dependency on
719 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to
720 * the historical MAX_SKB_FRAGS value 18 to honor the
721 * same behavior as before. Any packet using more than
722 * 18 slots but less than fatal_skb_slots slots is
725 if (!drop_err
&& slots
>= XEN_NETBK_LEGACY_SLOTS_MAX
) {
727 netdev_dbg(queue
->vif
->dev
,
728 "Too many slots (%d) exceeding limit (%d), dropping packet\n",
729 slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
736 memcpy(txp
, RING_GET_REQUEST(&queue
->tx
, cons
+ slots
),
739 /* If the guest submitted a frame >= 64 KiB then
740 * first->size overflowed and following slots will
741 * appear to be larger than the frame.
743 * This cannot be fatal error as there are buggy
744 * frontends that do this.
746 * Consume all slots and drop the packet.
748 if (!drop_err
&& txp
->size
> first
->size
) {
750 netdev_dbg(queue
->vif
->dev
,
751 "Invalid tx request, slot size %u > remaining size %u\n",
752 txp
->size
, first
->size
);
756 first
->size
-= txp
->size
;
759 if (unlikely((txp
->offset
+ txp
->size
) > PAGE_SIZE
)) {
760 netdev_err(queue
->vif
->dev
, "Cross page boundary, txp->offset: %x, size: %u\n",
761 txp
->offset
, txp
->size
);
762 xenvif_fatal_tx_err(queue
->vif
);
766 more_data
= txp
->flags
& XEN_NETTXF_more_data
;
774 xenvif_tx_err(queue
, first
, cons
+ slots
);
782 struct xenvif_tx_cb
{
786 #define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb)
788 static inline void xenvif_tx_create_map_op(struct xenvif_queue
*queue
,
790 struct xen_netif_tx_request
*txp
,
791 struct gnttab_map_grant_ref
*mop
)
793 queue
->pages_to_map
[mop
-queue
->tx_map_ops
] = queue
->mmap_pages
[pending_idx
];
794 gnttab_set_map_op(mop
, idx_to_kaddr(queue
, pending_idx
),
795 GNTMAP_host_map
| GNTMAP_readonly
,
796 txp
->gref
, queue
->vif
->domid
);
798 memcpy(&queue
->pending_tx_info
[pending_idx
].req
, txp
,
802 static inline struct sk_buff
*xenvif_alloc_skb(unsigned int size
)
804 struct sk_buff
*skb
=
805 alloc_skb(size
+ NET_SKB_PAD
+ NET_IP_ALIGN
,
806 GFP_ATOMIC
| __GFP_NOWARN
);
807 if (unlikely(skb
== NULL
))
810 /* Packets passed to netif_rx() must have some headroom. */
811 skb_reserve(skb
, NET_SKB_PAD
+ NET_IP_ALIGN
);
813 /* Initialize it here to avoid later surprises */
814 skb_shinfo(skb
)->destructor_arg
= NULL
;
819 static struct gnttab_map_grant_ref
*xenvif_get_requests(struct xenvif_queue
*queue
,
821 struct xen_netif_tx_request
*txp
,
822 struct gnttab_map_grant_ref
*gop
)
824 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
825 skb_frag_t
*frags
= shinfo
->frags
;
826 u16 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
828 pending_ring_idx_t index
;
829 unsigned int nr_slots
, frag_overflow
= 0;
831 /* At this point shinfo->nr_frags is in fact the number of
832 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
834 if (shinfo
->nr_frags
> MAX_SKB_FRAGS
) {
835 frag_overflow
= shinfo
->nr_frags
- MAX_SKB_FRAGS
;
836 BUG_ON(frag_overflow
> MAX_SKB_FRAGS
);
837 shinfo
->nr_frags
= MAX_SKB_FRAGS
;
839 nr_slots
= shinfo
->nr_frags
;
841 /* Skip first skb fragment if it is on same page as header fragment. */
842 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
844 for (shinfo
->nr_frags
= start
; shinfo
->nr_frags
< nr_slots
;
845 shinfo
->nr_frags
++, txp
++, gop
++) {
846 index
= pending_index(queue
->pending_cons
++);
847 pending_idx
= queue
->pending_ring
[index
];
848 xenvif_tx_create_map_op(queue
, pending_idx
, txp
, gop
);
849 frag_set_pending_idx(&frags
[shinfo
->nr_frags
], pending_idx
);
853 struct sk_buff
*nskb
= xenvif_alloc_skb(0);
854 if (unlikely(nskb
== NULL
)) {
856 netdev_err(queue
->vif
->dev
,
857 "Can't allocate the frag_list skb.\n");
861 shinfo
= skb_shinfo(nskb
);
862 frags
= shinfo
->frags
;
864 for (shinfo
->nr_frags
= 0; shinfo
->nr_frags
< frag_overflow
;
865 shinfo
->nr_frags
++, txp
++, gop
++) {
866 index
= pending_index(queue
->pending_cons
++);
867 pending_idx
= queue
->pending_ring
[index
];
868 xenvif_tx_create_map_op(queue
, pending_idx
, txp
, gop
);
869 frag_set_pending_idx(&frags
[shinfo
->nr_frags
],
873 skb_shinfo(skb
)->frag_list
= nskb
;
879 static inline void xenvif_grant_handle_set(struct xenvif_queue
*queue
,
881 grant_handle_t handle
)
883 if (unlikely(queue
->grant_tx_handle
[pending_idx
] !=
884 NETBACK_INVALID_HANDLE
)) {
885 netdev_err(queue
->vif
->dev
,
886 "Trying to overwrite active handle! pending_idx: %x\n",
890 queue
->grant_tx_handle
[pending_idx
] = handle
;
893 static inline void xenvif_grant_handle_reset(struct xenvif_queue
*queue
,
896 if (unlikely(queue
->grant_tx_handle
[pending_idx
] ==
897 NETBACK_INVALID_HANDLE
)) {
898 netdev_err(queue
->vif
->dev
,
899 "Trying to unmap invalid handle! pending_idx: %x\n",
903 queue
->grant_tx_handle
[pending_idx
] = NETBACK_INVALID_HANDLE
;
906 static int xenvif_tx_check_gop(struct xenvif_queue
*queue
,
908 struct gnttab_map_grant_ref
**gopp_map
,
909 struct gnttab_copy
**gopp_copy
)
911 struct gnttab_map_grant_ref
*gop_map
= *gopp_map
;
912 u16 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
913 /* This always points to the shinfo of the skb being checked, which
914 * could be either the first or the one on the frag_list
916 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
917 /* If this is non-NULL, we are currently checking the frag_list skb, and
918 * this points to the shinfo of the first one
920 struct skb_shared_info
*first_shinfo
= NULL
;
921 int nr_frags
= shinfo
->nr_frags
;
922 const bool sharedslot
= nr_frags
&&
923 frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
;
926 /* Check status of header. */
927 err
= (*gopp_copy
)->status
;
930 netdev_dbg(queue
->vif
->dev
,
931 "Grant copy of header failed! status: %d pending_idx: %u ref: %u\n",
932 (*gopp_copy
)->status
,
934 (*gopp_copy
)->source
.u
.ref
);
935 /* The first frag might still have this slot mapped */
937 xenvif_idx_release(queue
, pending_idx
,
938 XEN_NETIF_RSP_ERROR
);
943 for (i
= 0; i
< nr_frags
; i
++, gop_map
++) {
946 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[i
]);
948 /* Check error status: if okay then remember grant handle. */
949 newerr
= gop_map
->status
;
951 if (likely(!newerr
)) {
952 xenvif_grant_handle_set(queue
,
955 /* Had a previous error? Invalidate this fragment. */
957 xenvif_idx_unmap(queue
, pending_idx
);
958 /* If the mapping of the first frag was OK, but
959 * the header's copy failed, and they are
960 * sharing a slot, send an error
962 if (i
== 0 && sharedslot
)
963 xenvif_idx_release(queue
, pending_idx
,
964 XEN_NETIF_RSP_ERROR
);
966 xenvif_idx_release(queue
, pending_idx
,
972 /* Error on this fragment: respond to client with an error. */
974 netdev_dbg(queue
->vif
->dev
,
975 "Grant map of %d. frag failed! status: %d pending_idx: %u ref: %u\n",
981 xenvif_idx_release(queue
, pending_idx
, XEN_NETIF_RSP_ERROR
);
983 /* Not the first error? Preceding frags already invalidated. */
987 /* First error: if the header haven't shared a slot with the
988 * first frag, release it as well.
991 xenvif_idx_release(queue
,
992 XENVIF_TX_CB(skb
)->pending_idx
,
995 /* Invalidate preceding fragments of this skb. */
996 for (j
= 0; j
< i
; j
++) {
997 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[j
]);
998 xenvif_idx_unmap(queue
, pending_idx
);
999 xenvif_idx_release(queue
, pending_idx
,
1000 XEN_NETIF_RSP_OKAY
);
1003 /* And if we found the error while checking the frag_list, unmap
1004 * the first skb's frags
1007 for (j
= 0; j
< first_shinfo
->nr_frags
; j
++) {
1008 pending_idx
= frag_get_pending_idx(&first_shinfo
->frags
[j
]);
1009 xenvif_idx_unmap(queue
, pending_idx
);
1010 xenvif_idx_release(queue
, pending_idx
,
1011 XEN_NETIF_RSP_OKAY
);
1015 /* Remember the error: invalidate all subsequent fragments. */
1019 if (skb_has_frag_list(skb
) && !first_shinfo
) {
1020 first_shinfo
= skb_shinfo(skb
);
1021 shinfo
= skb_shinfo(skb_shinfo(skb
)->frag_list
);
1022 nr_frags
= shinfo
->nr_frags
;
1027 *gopp_map
= gop_map
;
1031 static void xenvif_fill_frags(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1033 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1034 int nr_frags
= shinfo
->nr_frags
;
1036 u16 prev_pending_idx
= INVALID_PENDING_IDX
;
1038 for (i
= 0; i
< nr_frags
; i
++) {
1039 skb_frag_t
*frag
= shinfo
->frags
+ i
;
1040 struct xen_netif_tx_request
*txp
;
1044 pending_idx
= frag_get_pending_idx(frag
);
1046 /* If this is not the first frag, chain it to the previous*/
1047 if (prev_pending_idx
== INVALID_PENDING_IDX
)
1048 skb_shinfo(skb
)->destructor_arg
=
1049 &callback_param(queue
, pending_idx
);
1051 callback_param(queue
, prev_pending_idx
).ctx
=
1052 &callback_param(queue
, pending_idx
);
1054 callback_param(queue
, pending_idx
).ctx
= NULL
;
1055 prev_pending_idx
= pending_idx
;
1057 txp
= &queue
->pending_tx_info
[pending_idx
].req
;
1058 page
= virt_to_page(idx_to_kaddr(queue
, pending_idx
));
1059 __skb_fill_page_desc(skb
, i
, page
, txp
->offset
, txp
->size
);
1060 skb
->len
+= txp
->size
;
1061 skb
->data_len
+= txp
->size
;
1062 skb
->truesize
+= txp
->size
;
1064 /* Take an extra reference to offset network stack's put_page */
1065 get_page(queue
->mmap_pages
[pending_idx
]);
1069 static int xenvif_get_extras(struct xenvif_queue
*queue
,
1070 struct xen_netif_extra_info
*extras
,
1073 struct xen_netif_extra_info extra
;
1074 RING_IDX cons
= queue
->tx
.req_cons
;
1077 if (unlikely(work_to_do
-- <= 0)) {
1078 netdev_err(queue
->vif
->dev
, "Missing extra info\n");
1079 xenvif_fatal_tx_err(queue
->vif
);
1083 memcpy(&extra
, RING_GET_REQUEST(&queue
->tx
, cons
),
1085 if (unlikely(!extra
.type
||
1086 extra
.type
>= XEN_NETIF_EXTRA_TYPE_MAX
)) {
1087 queue
->tx
.req_cons
= ++cons
;
1088 netdev_err(queue
->vif
->dev
,
1089 "Invalid extra type: %d\n", extra
.type
);
1090 xenvif_fatal_tx_err(queue
->vif
);
1094 memcpy(&extras
[extra
.type
- 1], &extra
, sizeof(extra
));
1095 queue
->tx
.req_cons
= ++cons
;
1096 } while (extra
.flags
& XEN_NETIF_EXTRA_FLAG_MORE
);
1101 static int xenvif_set_skb_gso(struct xenvif
*vif
,
1102 struct sk_buff
*skb
,
1103 struct xen_netif_extra_info
*gso
)
1105 if (!gso
->u
.gso
.size
) {
1106 netdev_err(vif
->dev
, "GSO size must not be zero.\n");
1107 xenvif_fatal_tx_err(vif
);
1111 switch (gso
->u
.gso
.type
) {
1112 case XEN_NETIF_GSO_TYPE_TCPV4
:
1113 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
1115 case XEN_NETIF_GSO_TYPE_TCPV6
:
1116 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV6
;
1119 netdev_err(vif
->dev
, "Bad GSO type %d.\n", gso
->u
.gso
.type
);
1120 xenvif_fatal_tx_err(vif
);
1124 skb_shinfo(skb
)->gso_size
= gso
->u
.gso
.size
;
1125 /* gso_segs will be calculated later */
1130 static int checksum_setup(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1132 bool recalculate_partial_csum
= false;
1134 /* A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1135 * peers can fail to set NETRXF_csum_blank when sending a GSO
1136 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1137 * recalculate the partial checksum.
1139 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
&& skb_is_gso(skb
)) {
1140 queue
->stats
.rx_gso_checksum_fixup
++;
1141 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1142 recalculate_partial_csum
= true;
1145 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1146 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1149 return skb_checksum_setup(skb
, recalculate_partial_csum
);
1152 static bool tx_credit_exceeded(struct xenvif_queue
*queue
, unsigned size
)
1154 u64 now
= get_jiffies_64();
1155 u64 next_credit
= queue
->credit_window_start
+
1156 msecs_to_jiffies(queue
->credit_usec
/ 1000);
1158 /* Timer could already be pending in rare cases. */
1159 if (timer_pending(&queue
->credit_timeout
))
1162 /* Passed the point where we can replenish credit? */
1163 if (time_after_eq64(now
, next_credit
)) {
1164 queue
->credit_window_start
= now
;
1165 tx_add_credit(queue
);
1168 /* Still too big to send right now? Set a callback. */
1169 if (size
> queue
->remaining_credit
) {
1170 queue
->credit_timeout
.data
=
1171 (unsigned long)queue
;
1172 queue
->credit_timeout
.function
=
1174 mod_timer(&queue
->credit_timeout
,
1176 queue
->credit_window_start
= next_credit
;
1184 static void xenvif_tx_build_gops(struct xenvif_queue
*queue
,
1189 struct gnttab_map_grant_ref
*gop
= queue
->tx_map_ops
, *request_gop
;
1190 struct sk_buff
*skb
;
1193 while (skb_queue_len(&queue
->tx_queue
) < budget
) {
1194 struct xen_netif_tx_request txreq
;
1195 struct xen_netif_tx_request txfrags
[XEN_NETBK_LEGACY_SLOTS_MAX
];
1196 struct xen_netif_extra_info extras
[XEN_NETIF_EXTRA_TYPE_MAX
-1];
1200 unsigned int data_len
;
1201 pending_ring_idx_t index
;
1203 if (queue
->tx
.sring
->req_prod
- queue
->tx
.req_cons
>
1204 XEN_NETIF_TX_RING_SIZE
) {
1205 netdev_err(queue
->vif
->dev
,
1206 "Impossible number of requests. "
1207 "req_prod %d, req_cons %d, size %ld\n",
1208 queue
->tx
.sring
->req_prod
, queue
->tx
.req_cons
,
1209 XEN_NETIF_TX_RING_SIZE
);
1210 xenvif_fatal_tx_err(queue
->vif
);
1214 work_to_do
= RING_HAS_UNCONSUMED_REQUESTS(&queue
->tx
);
1218 idx
= queue
->tx
.req_cons
;
1219 rmb(); /* Ensure that we see the request before we copy it. */
1220 memcpy(&txreq
, RING_GET_REQUEST(&queue
->tx
, idx
), sizeof(txreq
));
1222 /* Credit-based scheduling. */
1223 if (txreq
.size
> queue
->remaining_credit
&&
1224 tx_credit_exceeded(queue
, txreq
.size
))
1227 queue
->remaining_credit
-= txreq
.size
;
1230 queue
->tx
.req_cons
= ++idx
;
1232 memset(extras
, 0, sizeof(extras
));
1233 if (txreq
.flags
& XEN_NETTXF_extra_info
) {
1234 work_to_do
= xenvif_get_extras(queue
, extras
,
1236 idx
= queue
->tx
.req_cons
;
1237 if (unlikely(work_to_do
< 0))
1241 ret
= xenvif_count_requests(queue
, &txreq
, txfrags
, work_to_do
);
1242 if (unlikely(ret
< 0))
1247 if (unlikely(txreq
.size
< ETH_HLEN
)) {
1248 netdev_dbg(queue
->vif
->dev
,
1249 "Bad packet size: %d\n", txreq
.size
);
1250 xenvif_tx_err(queue
, &txreq
, idx
);
1254 /* No crossing a page as the payload mustn't fragment. */
1255 if (unlikely((txreq
.offset
+ txreq
.size
) > PAGE_SIZE
)) {
1256 netdev_err(queue
->vif
->dev
,
1257 "txreq.offset: %x, size: %u, end: %lu\n",
1258 txreq
.offset
, txreq
.size
,
1259 (txreq
.offset
&~PAGE_MASK
) + txreq
.size
);
1260 xenvif_fatal_tx_err(queue
->vif
);
1264 index
= pending_index(queue
->pending_cons
);
1265 pending_idx
= queue
->pending_ring
[index
];
1267 data_len
= (txreq
.size
> XEN_NETBACK_TX_COPY_LEN
&&
1268 ret
< XEN_NETBK_LEGACY_SLOTS_MAX
) ?
1269 XEN_NETBACK_TX_COPY_LEN
: txreq
.size
;
1271 skb
= xenvif_alloc_skb(data_len
);
1272 if (unlikely(skb
== NULL
)) {
1273 netdev_dbg(queue
->vif
->dev
,
1274 "Can't allocate a skb in start_xmit.\n");
1275 xenvif_tx_err(queue
, &txreq
, idx
);
1279 if (extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1].type
) {
1280 struct xen_netif_extra_info
*gso
;
1281 gso
= &extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1];
1283 if (xenvif_set_skb_gso(queue
->vif
, skb
, gso
)) {
1284 /* Failure in xenvif_set_skb_gso is fatal. */
1290 XENVIF_TX_CB(skb
)->pending_idx
= pending_idx
;
1292 __skb_put(skb
, data_len
);
1293 queue
->tx_copy_ops
[*copy_ops
].source
.u
.ref
= txreq
.gref
;
1294 queue
->tx_copy_ops
[*copy_ops
].source
.domid
= queue
->vif
->domid
;
1295 queue
->tx_copy_ops
[*copy_ops
].source
.offset
= txreq
.offset
;
1297 queue
->tx_copy_ops
[*copy_ops
].dest
.u
.gmfn
=
1298 virt_to_mfn(skb
->data
);
1299 queue
->tx_copy_ops
[*copy_ops
].dest
.domid
= DOMID_SELF
;
1300 queue
->tx_copy_ops
[*copy_ops
].dest
.offset
=
1301 offset_in_page(skb
->data
);
1303 queue
->tx_copy_ops
[*copy_ops
].len
= data_len
;
1304 queue
->tx_copy_ops
[*copy_ops
].flags
= GNTCOPY_source_gref
;
1308 skb_shinfo(skb
)->nr_frags
= ret
;
1309 if (data_len
< txreq
.size
) {
1310 skb_shinfo(skb
)->nr_frags
++;
1311 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1313 xenvif_tx_create_map_op(queue
, pending_idx
, &txreq
, gop
);
1316 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1317 INVALID_PENDING_IDX
);
1318 memcpy(&queue
->pending_tx_info
[pending_idx
].req
, &txreq
,
1322 queue
->pending_cons
++;
1324 request_gop
= xenvif_get_requests(queue
, skb
, txfrags
, gop
);
1325 if (request_gop
== NULL
) {
1327 xenvif_tx_err(queue
, &txreq
, idx
);
1332 __skb_queue_tail(&queue
->tx_queue
, skb
);
1334 queue
->tx
.req_cons
= idx
;
1336 if (((gop
-queue
->tx_map_ops
) >= ARRAY_SIZE(queue
->tx_map_ops
)) ||
1337 (*copy_ops
>= ARRAY_SIZE(queue
->tx_copy_ops
)))
1341 (*map_ops
) = gop
- queue
->tx_map_ops
;
1345 /* Consolidate skb with a frag_list into a brand new one with local pages on
1346 * frags. Returns 0 or -ENOMEM if can't allocate new pages.
1348 static int xenvif_handle_frag_list(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1350 unsigned int offset
= skb_headlen(skb
);
1351 skb_frag_t frags
[MAX_SKB_FRAGS
];
1353 struct ubuf_info
*uarg
;
1354 struct sk_buff
*nskb
= skb_shinfo(skb
)->frag_list
;
1356 queue
->stats
.tx_zerocopy_sent
+= 2;
1357 queue
->stats
.tx_frag_overflow
++;
1359 xenvif_fill_frags(queue
, nskb
);
1360 /* Subtract frags size, we will correct it later */
1361 skb
->truesize
-= skb
->data_len
;
1362 skb
->len
+= nskb
->len
;
1363 skb
->data_len
+= nskb
->len
;
1365 /* create a brand new frags array and coalesce there */
1366 for (i
= 0; offset
< skb
->len
; i
++) {
1370 BUG_ON(i
>= MAX_SKB_FRAGS
);
1371 page
= alloc_page(GFP_ATOMIC
);
1374 skb
->truesize
+= skb
->data_len
;
1375 for (j
= 0; j
< i
; j
++)
1376 put_page(frags
[j
].page
.p
);
1380 if (offset
+ PAGE_SIZE
< skb
->len
)
1383 len
= skb
->len
- offset
;
1384 if (skb_copy_bits(skb
, offset
, page_address(page
), len
))
1388 frags
[i
].page
.p
= page
;
1389 frags
[i
].page_offset
= 0;
1390 skb_frag_size_set(&frags
[i
], len
);
1393 /* Copied all the bits from the frag list -- free it. */
1394 skb_frag_list_init(skb
);
1395 xenvif_skb_zerocopy_prepare(queue
, nskb
);
1398 /* Release all the original (foreign) frags. */
1399 for (f
= 0; f
< skb_shinfo(skb
)->nr_frags
; f
++)
1400 skb_frag_unref(skb
, f
);
1401 uarg
= skb_shinfo(skb
)->destructor_arg
;
1402 /* increase inflight counter to offset decrement in callback */
1403 atomic_inc(&queue
->inflight_packets
);
1404 uarg
->callback(uarg
, true);
1405 skb_shinfo(skb
)->destructor_arg
= NULL
;
1407 /* Fill the skb with the new (local) frags. */
1408 memcpy(skb_shinfo(skb
)->frags
, frags
, i
* sizeof(skb_frag_t
));
1409 skb_shinfo(skb
)->nr_frags
= i
;
1410 skb
->truesize
+= i
* PAGE_SIZE
;
1415 static int xenvif_tx_submit(struct xenvif_queue
*queue
)
1417 struct gnttab_map_grant_ref
*gop_map
= queue
->tx_map_ops
;
1418 struct gnttab_copy
*gop_copy
= queue
->tx_copy_ops
;
1419 struct sk_buff
*skb
;
1422 while ((skb
= __skb_dequeue(&queue
->tx_queue
)) != NULL
) {
1423 struct xen_netif_tx_request
*txp
;
1427 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
1428 txp
= &queue
->pending_tx_info
[pending_idx
].req
;
1430 /* Check the remap error code. */
1431 if (unlikely(xenvif_tx_check_gop(queue
, skb
, &gop_map
, &gop_copy
))) {
1432 /* If there was an error, xenvif_tx_check_gop is
1433 * expected to release all the frags which were mapped,
1434 * so kfree_skb shouldn't do it again
1436 skb_shinfo(skb
)->nr_frags
= 0;
1437 if (skb_has_frag_list(skb
)) {
1438 struct sk_buff
*nskb
=
1439 skb_shinfo(skb
)->frag_list
;
1440 skb_shinfo(nskb
)->nr_frags
= 0;
1446 data_len
= skb
->len
;
1447 callback_param(queue
, pending_idx
).ctx
= NULL
;
1448 if (data_len
< txp
->size
) {
1449 /* Append the packet payload as a fragment. */
1450 txp
->offset
+= data_len
;
1451 txp
->size
-= data_len
;
1453 /* Schedule a response immediately. */
1454 xenvif_idx_release(queue
, pending_idx
,
1455 XEN_NETIF_RSP_OKAY
);
1458 if (txp
->flags
& XEN_NETTXF_csum_blank
)
1459 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1460 else if (txp
->flags
& XEN_NETTXF_data_validated
)
1461 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1463 xenvif_fill_frags(queue
, skb
);
1465 if (unlikely(skb_has_frag_list(skb
))) {
1466 if (xenvif_handle_frag_list(queue
, skb
)) {
1467 if (net_ratelimit())
1468 netdev_err(queue
->vif
->dev
,
1469 "Not enough memory to consolidate frag_list!\n");
1470 xenvif_skb_zerocopy_prepare(queue
, skb
);
1476 skb
->dev
= queue
->vif
->dev
;
1477 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1478 skb_reset_network_header(skb
);
1480 if (checksum_setup(queue
, skb
)) {
1481 netdev_dbg(queue
->vif
->dev
,
1482 "Can't setup checksum in net_tx_action\n");
1483 /* We have to set this flag to trigger the callback */
1484 if (skb_shinfo(skb
)->destructor_arg
)
1485 xenvif_skb_zerocopy_prepare(queue
, skb
);
1490 skb_probe_transport_header(skb
, 0);
1492 /* If the packet is GSO then we will have just set up the
1493 * transport header offset in checksum_setup so it's now
1494 * straightforward to calculate gso_segs.
1496 if (skb_is_gso(skb
)) {
1497 int mss
= skb_shinfo(skb
)->gso_size
;
1498 int hdrlen
= skb_transport_header(skb
) -
1499 skb_mac_header(skb
) +
1502 skb_shinfo(skb
)->gso_segs
=
1503 DIV_ROUND_UP(skb
->len
- hdrlen
, mss
);
1506 queue
->stats
.rx_bytes
+= skb
->len
;
1507 queue
->stats
.rx_packets
++;
1511 /* Set this flag right before netif_receive_skb, otherwise
1512 * someone might think this packet already left netback, and
1513 * do a skb_copy_ubufs while we are still in control of the
1514 * skb. E.g. the __pskb_pull_tail earlier can do such thing.
1516 if (skb_shinfo(skb
)->destructor_arg
) {
1517 xenvif_skb_zerocopy_prepare(queue
, skb
);
1518 queue
->stats
.tx_zerocopy_sent
++;
1521 netif_receive_skb(skb
);
1527 void xenvif_zerocopy_callback(struct ubuf_info
*ubuf
, bool zerocopy_success
)
1529 unsigned long flags
;
1530 pending_ring_idx_t index
;
1531 struct xenvif_queue
*queue
= ubuf_to_queue(ubuf
);
1533 /* This is the only place where we grab this lock, to protect callbacks
1536 spin_lock_irqsave(&queue
->callback_lock
, flags
);
1538 u16 pending_idx
= ubuf
->desc
;
1539 ubuf
= (struct ubuf_info
*) ubuf
->ctx
;
1540 BUG_ON(queue
->dealloc_prod
- queue
->dealloc_cons
>=
1542 index
= pending_index(queue
->dealloc_prod
);
1543 queue
->dealloc_ring
[index
] = pending_idx
;
1544 /* Sync with xenvif_tx_dealloc_action:
1545 * insert idx then incr producer.
1548 queue
->dealloc_prod
++;
1550 wake_up(&queue
->dealloc_wq
);
1551 spin_unlock_irqrestore(&queue
->callback_lock
, flags
);
1553 if (likely(zerocopy_success
))
1554 queue
->stats
.tx_zerocopy_success
++;
1556 queue
->stats
.tx_zerocopy_fail
++;
1557 xenvif_skb_zerocopy_complete(queue
);
1560 static inline void xenvif_tx_dealloc_action(struct xenvif_queue
*queue
)
1562 struct gnttab_unmap_grant_ref
*gop
;
1563 pending_ring_idx_t dc
, dp
;
1564 u16 pending_idx
, pending_idx_release
[MAX_PENDING_REQS
];
1567 dc
= queue
->dealloc_cons
;
1568 gop
= queue
->tx_unmap_ops
;
1570 /* Free up any grants we have finished using */
1572 dp
= queue
->dealloc_prod
;
1574 /* Ensure we see all indices enqueued by all
1575 * xenvif_zerocopy_callback().
1580 BUG_ON(gop
- queue
->tx_unmap_ops
> MAX_PENDING_REQS
);
1582 queue
->dealloc_ring
[pending_index(dc
++)];
1584 pending_idx_release
[gop
-queue
->tx_unmap_ops
] =
1586 queue
->pages_to_unmap
[gop
-queue
->tx_unmap_ops
] =
1587 queue
->mmap_pages
[pending_idx
];
1588 gnttab_set_unmap_op(gop
,
1589 idx_to_kaddr(queue
, pending_idx
),
1591 queue
->grant_tx_handle
[pending_idx
]);
1592 xenvif_grant_handle_reset(queue
, pending_idx
);
1596 } while (dp
!= queue
->dealloc_prod
);
1598 queue
->dealloc_cons
= dc
;
1600 if (gop
- queue
->tx_unmap_ops
> 0) {
1602 ret
= gnttab_unmap_refs(queue
->tx_unmap_ops
,
1604 queue
->pages_to_unmap
,
1605 gop
- queue
->tx_unmap_ops
);
1607 netdev_err(queue
->vif
->dev
, "Unmap fail: nr_ops %tx ret %d\n",
1608 gop
- queue
->tx_unmap_ops
, ret
);
1609 for (i
= 0; i
< gop
- queue
->tx_unmap_ops
; ++i
) {
1610 if (gop
[i
].status
!= GNTST_okay
)
1611 netdev_err(queue
->vif
->dev
,
1612 " host_addr: %llx handle: %x status: %d\n",
1621 for (i
= 0; i
< gop
- queue
->tx_unmap_ops
; ++i
)
1622 xenvif_idx_release(queue
, pending_idx_release
[i
],
1623 XEN_NETIF_RSP_OKAY
);
1627 /* Called after netfront has transmitted */
1628 int xenvif_tx_action(struct xenvif_queue
*queue
, int budget
)
1630 unsigned nr_mops
, nr_cops
= 0;
1633 if (unlikely(!tx_work_todo(queue
)))
1636 xenvif_tx_build_gops(queue
, budget
, &nr_cops
, &nr_mops
);
1641 gnttab_batch_copy(queue
->tx_copy_ops
, nr_cops
);
1643 ret
= gnttab_map_refs(queue
->tx_map_ops
,
1645 queue
->pages_to_map
,
1650 work_done
= xenvif_tx_submit(queue
);
1655 static void xenvif_idx_release(struct xenvif_queue
*queue
, u16 pending_idx
,
1658 struct pending_tx_info
*pending_tx_info
;
1659 pending_ring_idx_t index
;
1661 unsigned long flags
;
1663 pending_tx_info
= &queue
->pending_tx_info
[pending_idx
];
1665 spin_lock_irqsave(&queue
->response_lock
, flags
);
1667 make_tx_response(queue
, &pending_tx_info
->req
, status
);
1669 /* Release the pending index before pusing the Tx response so
1670 * its available before a new Tx request is pushed by the
1673 index
= pending_index(queue
->pending_prod
++);
1674 queue
->pending_ring
[index
] = pending_idx
;
1676 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue
->tx
, notify
);
1678 spin_unlock_irqrestore(&queue
->response_lock
, flags
);
1681 notify_remote_via_irq(queue
->tx_irq
);
1685 static void make_tx_response(struct xenvif_queue
*queue
,
1686 struct xen_netif_tx_request
*txp
,
1689 RING_IDX i
= queue
->tx
.rsp_prod_pvt
;
1690 struct xen_netif_tx_response
*resp
;
1692 resp
= RING_GET_RESPONSE(&queue
->tx
, i
);
1696 if (txp
->flags
& XEN_NETTXF_extra_info
)
1697 RING_GET_RESPONSE(&queue
->tx
, ++i
)->status
= XEN_NETIF_RSP_NULL
;
1699 queue
->tx
.rsp_prod_pvt
= ++i
;
1702 static struct xen_netif_rx_response
*make_rx_response(struct xenvif_queue
*queue
,
1709 RING_IDX i
= queue
->rx
.rsp_prod_pvt
;
1710 struct xen_netif_rx_response
*resp
;
1712 resp
= RING_GET_RESPONSE(&queue
->rx
, i
);
1713 resp
->offset
= offset
;
1714 resp
->flags
= flags
;
1716 resp
->status
= (s16
)size
;
1718 resp
->status
= (s16
)st
;
1720 queue
->rx
.rsp_prod_pvt
= ++i
;
1725 void xenvif_idx_unmap(struct xenvif_queue
*queue
, u16 pending_idx
)
1728 struct gnttab_unmap_grant_ref tx_unmap_op
;
1730 gnttab_set_unmap_op(&tx_unmap_op
,
1731 idx_to_kaddr(queue
, pending_idx
),
1733 queue
->grant_tx_handle
[pending_idx
]);
1734 xenvif_grant_handle_reset(queue
, pending_idx
);
1736 ret
= gnttab_unmap_refs(&tx_unmap_op
, NULL
,
1737 &queue
->mmap_pages
[pending_idx
], 1);
1739 netdev_err(queue
->vif
->dev
,
1740 "Unmap fail: ret: %d pending_idx: %d host_addr: %llx handle: %x status: %d\n",
1743 tx_unmap_op
.host_addr
,
1745 tx_unmap_op
.status
);
1750 static inline int tx_work_todo(struct xenvif_queue
*queue
)
1752 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue
->tx
)))
1758 static inline bool tx_dealloc_work_todo(struct xenvif_queue
*queue
)
1760 return queue
->dealloc_cons
!= queue
->dealloc_prod
;
1763 void xenvif_unmap_frontend_rings(struct xenvif_queue
*queue
)
1765 if (queue
->tx
.sring
)
1766 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue
->vif
),
1768 if (queue
->rx
.sring
)
1769 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue
->vif
),
1773 int xenvif_map_frontend_rings(struct xenvif_queue
*queue
,
1774 grant_ref_t tx_ring_ref
,
1775 grant_ref_t rx_ring_ref
)
1778 struct xen_netif_tx_sring
*txs
;
1779 struct xen_netif_rx_sring
*rxs
;
1783 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue
->vif
),
1784 tx_ring_ref
, &addr
);
1788 txs
= (struct xen_netif_tx_sring
*)addr
;
1789 BACK_RING_INIT(&queue
->tx
, txs
, PAGE_SIZE
);
1791 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue
->vif
),
1792 rx_ring_ref
, &addr
);
1796 rxs
= (struct xen_netif_rx_sring
*)addr
;
1797 BACK_RING_INIT(&queue
->rx
, rxs
, PAGE_SIZE
);
1802 xenvif_unmap_frontend_rings(queue
);
1806 static void xenvif_queue_carrier_off(struct xenvif_queue
*queue
)
1808 struct xenvif
*vif
= queue
->vif
;
1810 queue
->stalled
= true;
1812 /* At least one queue has stalled? Disable the carrier. */
1813 spin_lock(&vif
->lock
);
1814 if (vif
->stalled_queues
++ == 0) {
1815 netdev_info(vif
->dev
, "Guest Rx stalled");
1816 netif_carrier_off(vif
->dev
);
1818 spin_unlock(&vif
->lock
);
1821 static void xenvif_queue_carrier_on(struct xenvif_queue
*queue
)
1823 struct xenvif
*vif
= queue
->vif
;
1825 queue
->last_rx_time
= jiffies
; /* Reset Rx stall detection. */
1826 queue
->stalled
= false;
1828 /* All queues are ready? Enable the carrier. */
1829 spin_lock(&vif
->lock
);
1830 if (--vif
->stalled_queues
== 0) {
1831 netdev_info(vif
->dev
, "Guest Rx ready");
1832 netif_carrier_on(vif
->dev
);
1834 spin_unlock(&vif
->lock
);
1837 static bool xenvif_rx_queue_stalled(struct xenvif_queue
*queue
)
1839 RING_IDX prod
, cons
;
1841 prod
= queue
->rx
.sring
->req_prod
;
1842 cons
= queue
->rx
.req_cons
;
1844 return !queue
->stalled
1845 && prod
- cons
< XEN_NETBK_RX_SLOTS_MAX
1846 && time_after(jiffies
,
1847 queue
->last_rx_time
+ queue
->vif
->stall_timeout
);
1850 static bool xenvif_rx_queue_ready(struct xenvif_queue
*queue
)
1852 RING_IDX prod
, cons
;
1854 prod
= queue
->rx
.sring
->req_prod
;
1855 cons
= queue
->rx
.req_cons
;
1857 return queue
->stalled
1858 && prod
- cons
>= XEN_NETBK_RX_SLOTS_MAX
;
1861 static bool xenvif_have_rx_work(struct xenvif_queue
*queue
)
1863 return (!skb_queue_empty(&queue
->rx_queue
)
1864 && xenvif_rx_ring_slots_available(queue
, XEN_NETBK_RX_SLOTS_MAX
))
1865 || (queue
->vif
->stall_timeout
&&
1866 (xenvif_rx_queue_stalled(queue
)
1867 || xenvif_rx_queue_ready(queue
)))
1868 || kthread_should_stop()
1869 || queue
->vif
->disabled
;
1872 static long xenvif_rx_queue_timeout(struct xenvif_queue
*queue
)
1874 struct sk_buff
*skb
;
1877 skb
= skb_peek(&queue
->rx_queue
);
1879 return MAX_SCHEDULE_TIMEOUT
;
1881 timeout
= XENVIF_RX_CB(skb
)->expires
- jiffies
;
1882 return timeout
< 0 ? 0 : timeout
;
1885 /* Wait until the guest Rx thread has work.
1887 * The timeout needs to be adjusted based on the current head of the
1888 * queue (and not just the head at the beginning). In particular, if
1889 * the queue is initially empty an infinite timeout is used and this
1890 * needs to be reduced when a skb is queued.
1892 * This cannot be done with wait_event_timeout() because it only
1893 * calculates the timeout once.
1895 static void xenvif_wait_for_rx_work(struct xenvif_queue
*queue
)
1899 if (xenvif_have_rx_work(queue
))
1905 prepare_to_wait(&queue
->wq
, &wait
, TASK_INTERRUPTIBLE
);
1906 if (xenvif_have_rx_work(queue
))
1908 ret
= schedule_timeout(xenvif_rx_queue_timeout(queue
));
1912 finish_wait(&queue
->wq
, &wait
);
1915 int xenvif_kthread_guest_rx(void *data
)
1917 struct xenvif_queue
*queue
= data
;
1918 struct xenvif
*vif
= queue
->vif
;
1920 if (!vif
->stall_timeout
)
1921 xenvif_queue_carrier_on(queue
);
1924 xenvif_wait_for_rx_work(queue
);
1926 if (kthread_should_stop())
1929 /* This frontend is found to be rogue, disable it in
1930 * kthread context. Currently this is only set when
1931 * netback finds out frontend sends malformed packet,
1932 * but we cannot disable the interface in softirq
1933 * context so we defer it here, if this thread is
1934 * associated with queue 0.
1936 if (unlikely(vif
->disabled
&& queue
->id
== 0)) {
1937 xenvif_carrier_off(vif
);
1941 if (!skb_queue_empty(&queue
->rx_queue
))
1942 xenvif_rx_action(queue
);
1944 /* If the guest hasn't provided any Rx slots for a
1945 * while it's probably not responsive, drop the
1946 * carrier so packets are dropped earlier.
1948 if (vif
->stall_timeout
) {
1949 if (xenvif_rx_queue_stalled(queue
))
1950 xenvif_queue_carrier_off(queue
);
1951 else if (xenvif_rx_queue_ready(queue
))
1952 xenvif_queue_carrier_on(queue
);
1955 /* Queued packets may have foreign pages from other
1956 * domains. These cannot be queued indefinitely as
1957 * this would starve guests of grant refs and transmit
1960 xenvif_rx_queue_drop_expired(queue
);
1962 xenvif_rx_queue_maybe_wake(queue
);
1967 /* Bin any remaining skbs */
1968 xenvif_rx_queue_purge(queue
);
1973 static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue
*queue
)
1975 /* Dealloc thread must remain running until all inflight
1978 return kthread_should_stop() &&
1979 !atomic_read(&queue
->inflight_packets
);
1982 int xenvif_dealloc_kthread(void *data
)
1984 struct xenvif_queue
*queue
= data
;
1987 wait_event_interruptible(queue
->dealloc_wq
,
1988 tx_dealloc_work_todo(queue
) ||
1989 xenvif_dealloc_kthread_should_stop(queue
));
1990 if (xenvif_dealloc_kthread_should_stop(queue
))
1993 xenvif_tx_dealloc_action(queue
);
1997 /* Unmap anything remaining*/
1998 if (tx_dealloc_work_todo(queue
))
1999 xenvif_tx_dealloc_action(queue
);
2004 static int __init
netback_init(void)
2011 /* Allow as many queues as there are CPUs, by default */
2012 xenvif_max_queues
= num_online_cpus();
2014 if (fatal_skb_slots
< XEN_NETBK_LEGACY_SLOTS_MAX
) {
2015 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n",
2016 fatal_skb_slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
2017 fatal_skb_slots
= XEN_NETBK_LEGACY_SLOTS_MAX
;
2020 rc
= xenvif_xenbus_init();
2024 #ifdef CONFIG_DEBUG_FS
2025 xen_netback_dbg_root
= debugfs_create_dir("xen-netback", NULL
);
2026 if (IS_ERR_OR_NULL(xen_netback_dbg_root
))
2027 pr_warn("Init of debugfs returned %ld!\n",
2028 PTR_ERR(xen_netback_dbg_root
));
2029 #endif /* CONFIG_DEBUG_FS */
2037 module_init(netback_init
);
2039 static void __exit
netback_fini(void)
2041 #ifdef CONFIG_DEBUG_FS
2042 if (!IS_ERR_OR_NULL(xen_netback_dbg_root
))
2043 debugfs_remove_recursive(xen_netback_dbg_root
);
2044 #endif /* CONFIG_DEBUG_FS */
2045 xenvif_xenbus_fini();
2047 module_exit(netback_fini
);
2049 MODULE_LICENSE("Dual BSD/GPL");
2050 MODULE_ALIAS("xen-backend:vif");