2 * Back-end of the driver for virtual network devices. This portion of the
3 * driver exports a 'unified' network-device interface that can be accessed
4 * by any operating system that implements a compatible front end. A
5 * reference front-end implementation can be found in:
6 * drivers/net/xen-netfront.c
8 * Copyright (c) 2002-2005, K A Fraser
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
37 #include <linux/kthread.h>
38 #include <linux/if_vlan.h>
39 #include <linux/udp.h>
40 #include <linux/highmem.h>
45 #include <xen/events.h>
46 #include <xen/interface/memory.h>
49 #include <asm/xen/hypercall.h>
51 /* Provide an option to disable split event channels at load time as
52 * event channels are limited resource. Split event channels are
55 bool separate_tx_rx_irq
= true;
56 module_param(separate_tx_rx_irq
, bool, 0644);
58 /* The time that packets can stay on the guest Rx internal queue
59 * before they are dropped.
61 unsigned int rx_drain_timeout_msecs
= 10000;
62 module_param(rx_drain_timeout_msecs
, uint
, 0444);
64 /* The length of time before the frontend is considered unresponsive
65 * because it isn't providing Rx slots.
67 unsigned int rx_stall_timeout_msecs
= 60000;
68 module_param(rx_stall_timeout_msecs
, uint
, 0444);
70 unsigned int xenvif_max_queues
;
71 module_param_named(max_queues
, xenvif_max_queues
, uint
, 0644);
72 MODULE_PARM_DESC(max_queues
,
73 "Maximum number of queues per virtual interface");
76 * This is the maximum slots a skb can have. If a guest sends a skb
77 * which exceeds this limit it is considered malicious.
79 #define FATAL_SKB_SLOTS_DEFAULT 20
80 static unsigned int fatal_skb_slots
= FATAL_SKB_SLOTS_DEFAULT
;
81 module_param(fatal_skb_slots
, uint
, 0444);
83 /* The amount to copy out of the first guest Tx slot into the skb's
84 * linear area. If the first slot has more data, it will be mapped
85 * and put into the first frag.
87 * This is sized to avoid pulling headers from the frags for most
90 #define XEN_NETBACK_TX_COPY_LEN 128
93 static void xenvif_idx_release(struct xenvif_queue
*queue
, u16 pending_idx
,
96 static void make_tx_response(struct xenvif_queue
*queue
,
97 struct xen_netif_tx_request
*txp
,
99 static void push_tx_responses(struct xenvif_queue
*queue
);
101 static inline int tx_work_todo(struct xenvif_queue
*queue
);
103 static struct xen_netif_rx_response
*make_rx_response(struct xenvif_queue
*queue
,
110 static inline unsigned long idx_to_pfn(struct xenvif_queue
*queue
,
113 return page_to_pfn(queue
->mmap_pages
[idx
]);
116 static inline unsigned long idx_to_kaddr(struct xenvif_queue
*queue
,
119 return (unsigned long)pfn_to_kaddr(idx_to_pfn(queue
, idx
));
122 #define callback_param(vif, pending_idx) \
123 (vif->pending_tx_info[pending_idx].callback_struct)
125 /* Find the containing VIF's structure from a pointer in pending_tx_info array
127 static inline struct xenvif_queue
*ubuf_to_queue(const struct ubuf_info
*ubuf
)
129 u16 pending_idx
= ubuf
->desc
;
130 struct pending_tx_info
*temp
=
131 container_of(ubuf
, struct pending_tx_info
, callback_struct
);
132 return container_of(temp
- pending_idx
,
137 static u16
frag_get_pending_idx(skb_frag_t
*frag
)
139 return (u16
)frag
->page_offset
;
142 static void frag_set_pending_idx(skb_frag_t
*frag
, u16 pending_idx
)
144 frag
->page_offset
= pending_idx
;
147 static inline pending_ring_idx_t
pending_index(unsigned i
)
149 return i
& (MAX_PENDING_REQS
-1);
152 static int xenvif_rx_ring_slots_needed(struct xenvif
*vif
)
155 return DIV_ROUND_UP(vif
->dev
->gso_max_size
, PAGE_SIZE
) + 1;
157 return DIV_ROUND_UP(vif
->dev
->mtu
, PAGE_SIZE
);
160 static bool xenvif_rx_ring_slots_available(struct xenvif_queue
*queue
)
165 needed
= xenvif_rx_ring_slots_needed(queue
->vif
);
168 prod
= queue
->rx
.sring
->req_prod
;
169 cons
= queue
->rx
.req_cons
;
171 if (prod
- cons
>= needed
)
174 queue
->rx
.sring
->req_event
= prod
+ 1;
176 /* Make sure event is visible before we check prod
180 } while (queue
->rx
.sring
->req_prod
!= prod
);
185 void xenvif_rx_queue_tail(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
189 spin_lock_irqsave(&queue
->rx_queue
.lock
, flags
);
191 __skb_queue_tail(&queue
->rx_queue
, skb
);
193 queue
->rx_queue_len
+= skb
->len
;
194 if (queue
->rx_queue_len
> queue
->rx_queue_max
)
195 netif_tx_stop_queue(netdev_get_tx_queue(queue
->vif
->dev
, queue
->id
));
197 spin_unlock_irqrestore(&queue
->rx_queue
.lock
, flags
);
200 static struct sk_buff
*xenvif_rx_dequeue(struct xenvif_queue
*queue
)
204 spin_lock_irq(&queue
->rx_queue
.lock
);
206 skb
= __skb_dequeue(&queue
->rx_queue
);
208 queue
->rx_queue_len
-= skb
->len
;
210 spin_unlock_irq(&queue
->rx_queue
.lock
);
215 static void xenvif_rx_queue_maybe_wake(struct xenvif_queue
*queue
)
217 spin_lock_irq(&queue
->rx_queue
.lock
);
219 if (queue
->rx_queue_len
< queue
->rx_queue_max
)
220 netif_tx_wake_queue(netdev_get_tx_queue(queue
->vif
->dev
, queue
->id
));
222 spin_unlock_irq(&queue
->rx_queue
.lock
);
226 static void xenvif_rx_queue_purge(struct xenvif_queue
*queue
)
229 while ((skb
= xenvif_rx_dequeue(queue
)) != NULL
)
233 static void xenvif_rx_queue_drop_expired(struct xenvif_queue
*queue
)
238 skb
= skb_peek(&queue
->rx_queue
);
241 if (time_before(jiffies
, XENVIF_RX_CB(skb
)->expires
))
243 xenvif_rx_dequeue(queue
);
248 struct netrx_pending_operations
{
249 unsigned copy_prod
, copy_cons
;
250 unsigned meta_prod
, meta_cons
;
251 struct gnttab_copy
*copy
;
252 struct xenvif_rx_meta
*meta
;
254 grant_ref_t copy_gref
;
257 static struct xenvif_rx_meta
*get_next_rx_buffer(struct xenvif_queue
*queue
,
258 struct netrx_pending_operations
*npo
)
260 struct xenvif_rx_meta
*meta
;
261 struct xen_netif_rx_request
*req
;
263 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
265 meta
= npo
->meta
+ npo
->meta_prod
++;
266 meta
->gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
272 npo
->copy_gref
= req
->gref
;
278 * Set up the grant operations for this fragment. If it's a flipping
279 * interface, we also set up the unmap request from here.
281 static void xenvif_gop_frag_copy(struct xenvif_queue
*queue
, struct sk_buff
*skb
,
282 struct netrx_pending_operations
*npo
,
283 struct page
*page
, unsigned long size
,
284 unsigned long offset
, int *head
)
286 struct gnttab_copy
*copy_gop
;
287 struct xenvif_rx_meta
*meta
;
289 int gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
291 /* Data must not cross a page boundary. */
292 BUG_ON(size
+ offset
> PAGE_SIZE
<<compound_order(page
));
294 meta
= npo
->meta
+ npo
->meta_prod
- 1;
296 /* Skip unused frames from start of page */
297 page
+= offset
>> PAGE_SHIFT
;
298 offset
&= ~PAGE_MASK
;
301 struct xen_page_foreign
*foreign
;
303 BUG_ON(offset
>= PAGE_SIZE
);
304 BUG_ON(npo
->copy_off
> MAX_BUFFER_OFFSET
);
306 if (npo
->copy_off
== MAX_BUFFER_OFFSET
)
307 meta
= get_next_rx_buffer(queue
, npo
);
309 bytes
= PAGE_SIZE
- offset
;
313 if (npo
->copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
314 bytes
= MAX_BUFFER_OFFSET
- npo
->copy_off
;
316 copy_gop
= npo
->copy
+ npo
->copy_prod
++;
317 copy_gop
->flags
= GNTCOPY_dest_gref
;
318 copy_gop
->len
= bytes
;
320 foreign
= xen_page_foreign(page
);
322 copy_gop
->source
.domid
= foreign
->domid
;
323 copy_gop
->source
.u
.ref
= foreign
->gref
;
324 copy_gop
->flags
|= GNTCOPY_source_gref
;
326 copy_gop
->source
.domid
= DOMID_SELF
;
327 copy_gop
->source
.u
.gmfn
=
328 virt_to_mfn(page_address(page
));
330 copy_gop
->source
.offset
= offset
;
332 copy_gop
->dest
.domid
= queue
->vif
->domid
;
333 copy_gop
->dest
.offset
= npo
->copy_off
;
334 copy_gop
->dest
.u
.ref
= npo
->copy_gref
;
336 npo
->copy_off
+= bytes
;
343 if (offset
== PAGE_SIZE
&& size
) {
344 BUG_ON(!PageCompound(page
));
349 /* Leave a gap for the GSO descriptor. */
350 if (skb_is_gso(skb
)) {
351 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
)
352 gso_type
= XEN_NETIF_GSO_TYPE_TCPV4
;
353 else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
)
354 gso_type
= XEN_NETIF_GSO_TYPE_TCPV6
;
357 if (*head
&& ((1 << gso_type
) & queue
->vif
->gso_mask
))
358 queue
->rx
.req_cons
++;
360 *head
= 0; /* There must be something in this buffer now. */
366 * Prepare an SKB to be transmitted to the frontend.
368 * This function is responsible for allocating grant operations, meta
371 * It returns the number of meta structures consumed. The number of
372 * ring slots used is always equal to the number of meta slots used
373 * plus the number of GSO descriptors used. Currently, we use either
374 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
375 * frontend-side LRO).
377 static int xenvif_gop_skb(struct sk_buff
*skb
,
378 struct netrx_pending_operations
*npo
,
379 struct xenvif_queue
*queue
)
381 struct xenvif
*vif
= netdev_priv(skb
->dev
);
382 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
384 struct xen_netif_rx_request
*req
;
385 struct xenvif_rx_meta
*meta
;
391 old_meta_prod
= npo
->meta_prod
;
393 gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
394 if (skb_is_gso(skb
)) {
395 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
)
396 gso_type
= XEN_NETIF_GSO_TYPE_TCPV4
;
397 else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
)
398 gso_type
= XEN_NETIF_GSO_TYPE_TCPV6
;
401 /* Set up a GSO prefix descriptor, if necessary */
402 if ((1 << gso_type
) & vif
->gso_prefix_mask
) {
403 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
404 meta
= npo
->meta
+ npo
->meta_prod
++;
405 meta
->gso_type
= gso_type
;
406 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
411 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
412 meta
= npo
->meta
+ npo
->meta_prod
++;
414 if ((1 << gso_type
) & vif
->gso_mask
) {
415 meta
->gso_type
= gso_type
;
416 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
418 meta
->gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
425 npo
->copy_gref
= req
->gref
;
428 while (data
< skb_tail_pointer(skb
)) {
429 unsigned int offset
= offset_in_page(data
);
430 unsigned int len
= PAGE_SIZE
- offset
;
432 if (data
+ len
> skb_tail_pointer(skb
))
433 len
= skb_tail_pointer(skb
) - data
;
435 xenvif_gop_frag_copy(queue
, skb
, npo
,
436 virt_to_page(data
), len
, offset
, &head
);
440 for (i
= 0; i
< nr_frags
; i
++) {
441 xenvif_gop_frag_copy(queue
, skb
, npo
,
442 skb_frag_page(&skb_shinfo(skb
)->frags
[i
]),
443 skb_frag_size(&skb_shinfo(skb
)->frags
[i
]),
444 skb_shinfo(skb
)->frags
[i
].page_offset
,
448 return npo
->meta_prod
- old_meta_prod
;
452 * This is a twin to xenvif_gop_skb. Assume that xenvif_gop_skb was
453 * used to set up the operations on the top of
454 * netrx_pending_operations, which have since been done. Check that
455 * they didn't give any errors and advance over them.
457 static int xenvif_check_gop(struct xenvif
*vif
, int nr_meta_slots
,
458 struct netrx_pending_operations
*npo
)
460 struct gnttab_copy
*copy_op
;
461 int status
= XEN_NETIF_RSP_OKAY
;
464 for (i
= 0; i
< nr_meta_slots
; i
++) {
465 copy_op
= npo
->copy
+ npo
->copy_cons
++;
466 if (copy_op
->status
!= GNTST_okay
) {
468 "Bad status %d from copy to DOM%d.\n",
469 copy_op
->status
, vif
->domid
);
470 status
= XEN_NETIF_RSP_ERROR
;
477 static void xenvif_add_frag_responses(struct xenvif_queue
*queue
, int status
,
478 struct xenvif_rx_meta
*meta
,
482 unsigned long offset
;
484 /* No fragments used */
485 if (nr_meta_slots
<= 1)
490 for (i
= 0; i
< nr_meta_slots
; i
++) {
492 if (i
== nr_meta_slots
- 1)
495 flags
= XEN_NETRXF_more_data
;
498 make_rx_response(queue
, meta
[i
].id
, status
, offset
,
499 meta
[i
].size
, flags
);
503 void xenvif_kick_thread(struct xenvif_queue
*queue
)
508 static void xenvif_rx_action(struct xenvif_queue
*queue
)
512 struct xen_netif_rx_response
*resp
;
513 struct sk_buff_head rxq
;
517 unsigned long offset
;
518 bool need_to_notify
= false;
520 struct netrx_pending_operations npo
= {
521 .copy
= queue
->grant_copy_op
,
525 skb_queue_head_init(&rxq
);
527 while (xenvif_rx_ring_slots_available(queue
)
528 && (skb
= xenvif_rx_dequeue(queue
)) != NULL
) {
529 queue
->last_rx_time
= jiffies
;
531 XENVIF_RX_CB(skb
)->meta_slots_used
= xenvif_gop_skb(skb
, &npo
, queue
);
533 __skb_queue_tail(&rxq
, skb
);
536 BUG_ON(npo
.meta_prod
> ARRAY_SIZE(queue
->meta
));
541 BUG_ON(npo
.copy_prod
> MAX_GRANT_COPY_OPS
);
542 gnttab_batch_copy(queue
->grant_copy_op
, npo
.copy_prod
);
544 while ((skb
= __skb_dequeue(&rxq
)) != NULL
) {
546 if ((1 << queue
->meta
[npo
.meta_cons
].gso_type
) &
547 queue
->vif
->gso_prefix_mask
) {
548 resp
= RING_GET_RESPONSE(&queue
->rx
,
549 queue
->rx
.rsp_prod_pvt
++);
551 resp
->flags
= XEN_NETRXF_gso_prefix
| XEN_NETRXF_more_data
;
553 resp
->offset
= queue
->meta
[npo
.meta_cons
].gso_size
;
554 resp
->id
= queue
->meta
[npo
.meta_cons
].id
;
555 resp
->status
= XENVIF_RX_CB(skb
)->meta_slots_used
;
558 XENVIF_RX_CB(skb
)->meta_slots_used
--;
562 queue
->stats
.tx_bytes
+= skb
->len
;
563 queue
->stats
.tx_packets
++;
565 status
= xenvif_check_gop(queue
->vif
,
566 XENVIF_RX_CB(skb
)->meta_slots_used
,
569 if (XENVIF_RX_CB(skb
)->meta_slots_used
== 1)
572 flags
= XEN_NETRXF_more_data
;
574 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) /* local packet? */
575 flags
|= XEN_NETRXF_csum_blank
| XEN_NETRXF_data_validated
;
576 else if (skb
->ip_summed
== CHECKSUM_UNNECESSARY
)
577 /* remote but checksummed. */
578 flags
|= XEN_NETRXF_data_validated
;
581 resp
= make_rx_response(queue
, queue
->meta
[npo
.meta_cons
].id
,
583 queue
->meta
[npo
.meta_cons
].size
,
586 if ((1 << queue
->meta
[npo
.meta_cons
].gso_type
) &
587 queue
->vif
->gso_mask
) {
588 struct xen_netif_extra_info
*gso
=
589 (struct xen_netif_extra_info
*)
590 RING_GET_RESPONSE(&queue
->rx
,
591 queue
->rx
.rsp_prod_pvt
++);
593 resp
->flags
|= XEN_NETRXF_extra_info
;
595 gso
->u
.gso
.type
= queue
->meta
[npo
.meta_cons
].gso_type
;
596 gso
->u
.gso
.size
= queue
->meta
[npo
.meta_cons
].gso_size
;
598 gso
->u
.gso
.features
= 0;
600 gso
->type
= XEN_NETIF_EXTRA_TYPE_GSO
;
604 xenvif_add_frag_responses(queue
, status
,
605 queue
->meta
+ npo
.meta_cons
+ 1,
606 XENVIF_RX_CB(skb
)->meta_slots_used
);
608 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue
->rx
, ret
);
610 need_to_notify
|= !!ret
;
612 npo
.meta_cons
+= XENVIF_RX_CB(skb
)->meta_slots_used
;
618 notify_remote_via_irq(queue
->rx_irq
);
621 void xenvif_napi_schedule_or_enable_events(struct xenvif_queue
*queue
)
625 RING_FINAL_CHECK_FOR_REQUESTS(&queue
->tx
, more_to_do
);
628 napi_schedule(&queue
->napi
);
631 static void tx_add_credit(struct xenvif_queue
*queue
)
633 unsigned long max_burst
, max_credit
;
636 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
637 * Otherwise the interface can seize up due to insufficient credit.
639 max_burst
= RING_GET_REQUEST(&queue
->tx
, queue
->tx
.req_cons
)->size
;
640 max_burst
= min(max_burst
, 131072UL);
641 max_burst
= max(max_burst
, queue
->credit_bytes
);
643 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
644 max_credit
= queue
->remaining_credit
+ queue
->credit_bytes
;
645 if (max_credit
< queue
->remaining_credit
)
646 max_credit
= ULONG_MAX
; /* wrapped: clamp to ULONG_MAX */
648 queue
->remaining_credit
= min(max_credit
, max_burst
);
651 void xenvif_tx_credit_callback(unsigned long data
)
653 struct xenvif_queue
*queue
= (struct xenvif_queue
*)data
;
654 tx_add_credit(queue
);
655 xenvif_napi_schedule_or_enable_events(queue
);
658 static void xenvif_tx_err(struct xenvif_queue
*queue
,
659 struct xen_netif_tx_request
*txp
, RING_IDX end
)
661 RING_IDX cons
= queue
->tx
.req_cons
;
665 spin_lock_irqsave(&queue
->response_lock
, flags
);
666 make_tx_response(queue
, txp
, XEN_NETIF_RSP_ERROR
);
667 push_tx_responses(queue
);
668 spin_unlock_irqrestore(&queue
->response_lock
, flags
);
671 txp
= RING_GET_REQUEST(&queue
->tx
, cons
++);
673 queue
->tx
.req_cons
= cons
;
676 static void xenvif_fatal_tx_err(struct xenvif
*vif
)
678 netdev_err(vif
->dev
, "fatal error; disabling device\n");
679 vif
->disabled
= true;
680 /* Disable the vif from queue 0's kthread */
682 xenvif_kick_thread(&vif
->queues
[0]);
685 static int xenvif_count_requests(struct xenvif_queue
*queue
,
686 struct xen_netif_tx_request
*first
,
687 struct xen_netif_tx_request
*txp
,
690 RING_IDX cons
= queue
->tx
.req_cons
;
695 if (!(first
->flags
& XEN_NETTXF_more_data
))
699 struct xen_netif_tx_request dropped_tx
= { 0 };
701 if (slots
>= work_to_do
) {
702 netdev_err(queue
->vif
->dev
,
703 "Asked for %d slots but exceeds this limit\n",
705 xenvif_fatal_tx_err(queue
->vif
);
709 /* This guest is really using too many slots and
710 * considered malicious.
712 if (unlikely(slots
>= fatal_skb_slots
)) {
713 netdev_err(queue
->vif
->dev
,
714 "Malicious frontend using %d slots, threshold %u\n",
715 slots
, fatal_skb_slots
);
716 xenvif_fatal_tx_err(queue
->vif
);
720 /* Xen network protocol had implicit dependency on
721 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to
722 * the historical MAX_SKB_FRAGS value 18 to honor the
723 * same behavior as before. Any packet using more than
724 * 18 slots but less than fatal_skb_slots slots is
727 if (!drop_err
&& slots
>= XEN_NETBK_LEGACY_SLOTS_MAX
) {
729 netdev_dbg(queue
->vif
->dev
,
730 "Too many slots (%d) exceeding limit (%d), dropping packet\n",
731 slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
738 memcpy(txp
, RING_GET_REQUEST(&queue
->tx
, cons
+ slots
),
741 /* If the guest submitted a frame >= 64 KiB then
742 * first->size overflowed and following slots will
743 * appear to be larger than the frame.
745 * This cannot be fatal error as there are buggy
746 * frontends that do this.
748 * Consume all slots and drop the packet.
750 if (!drop_err
&& txp
->size
> first
->size
) {
752 netdev_dbg(queue
->vif
->dev
,
753 "Invalid tx request, slot size %u > remaining size %u\n",
754 txp
->size
, first
->size
);
758 first
->size
-= txp
->size
;
761 if (unlikely((txp
->offset
+ txp
->size
) > PAGE_SIZE
)) {
762 netdev_err(queue
->vif
->dev
, "Cross page boundary, txp->offset: %u, size: %u\n",
763 txp
->offset
, txp
->size
);
764 xenvif_fatal_tx_err(queue
->vif
);
768 more_data
= txp
->flags
& XEN_NETTXF_more_data
;
776 xenvif_tx_err(queue
, first
, cons
+ slots
);
784 struct xenvif_tx_cb
{
788 #define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb)
790 static inline void xenvif_tx_create_map_op(struct xenvif_queue
*queue
,
792 struct xen_netif_tx_request
*txp
,
793 struct gnttab_map_grant_ref
*mop
)
795 queue
->pages_to_map
[mop
-queue
->tx_map_ops
] = queue
->mmap_pages
[pending_idx
];
796 gnttab_set_map_op(mop
, idx_to_kaddr(queue
, pending_idx
),
797 GNTMAP_host_map
| GNTMAP_readonly
,
798 txp
->gref
, queue
->vif
->domid
);
800 memcpy(&queue
->pending_tx_info
[pending_idx
].req
, txp
,
804 static inline struct sk_buff
*xenvif_alloc_skb(unsigned int size
)
806 struct sk_buff
*skb
=
807 alloc_skb(size
+ NET_SKB_PAD
+ NET_IP_ALIGN
,
808 GFP_ATOMIC
| __GFP_NOWARN
);
809 if (unlikely(skb
== NULL
))
812 /* Packets passed to netif_rx() must have some headroom. */
813 skb_reserve(skb
, NET_SKB_PAD
+ NET_IP_ALIGN
);
815 /* Initialize it here to avoid later surprises */
816 skb_shinfo(skb
)->destructor_arg
= NULL
;
821 static struct gnttab_map_grant_ref
*xenvif_get_requests(struct xenvif_queue
*queue
,
823 struct xen_netif_tx_request
*txp
,
824 struct gnttab_map_grant_ref
*gop
,
825 unsigned int frag_overflow
,
826 struct sk_buff
*nskb
)
828 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
829 skb_frag_t
*frags
= shinfo
->frags
;
830 u16 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
832 pending_ring_idx_t index
;
833 unsigned int nr_slots
;
835 nr_slots
= shinfo
->nr_frags
;
837 /* Skip first skb fragment if it is on same page as header fragment. */
838 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
840 for (shinfo
->nr_frags
= start
; shinfo
->nr_frags
< nr_slots
;
841 shinfo
->nr_frags
++, txp
++, gop
++) {
842 index
= pending_index(queue
->pending_cons
++);
843 pending_idx
= queue
->pending_ring
[index
];
844 xenvif_tx_create_map_op(queue
, pending_idx
, txp
, gop
);
845 frag_set_pending_idx(&frags
[shinfo
->nr_frags
], pending_idx
);
850 shinfo
= skb_shinfo(nskb
);
851 frags
= shinfo
->frags
;
853 for (shinfo
->nr_frags
= 0; shinfo
->nr_frags
< frag_overflow
;
854 shinfo
->nr_frags
++, txp
++, gop
++) {
855 index
= pending_index(queue
->pending_cons
++);
856 pending_idx
= queue
->pending_ring
[index
];
857 xenvif_tx_create_map_op(queue
, pending_idx
, txp
, gop
);
858 frag_set_pending_idx(&frags
[shinfo
->nr_frags
],
862 skb_shinfo(skb
)->frag_list
= nskb
;
868 static inline void xenvif_grant_handle_set(struct xenvif_queue
*queue
,
870 grant_handle_t handle
)
872 if (unlikely(queue
->grant_tx_handle
[pending_idx
] !=
873 NETBACK_INVALID_HANDLE
)) {
874 netdev_err(queue
->vif
->dev
,
875 "Trying to overwrite active handle! pending_idx: 0x%x\n",
879 queue
->grant_tx_handle
[pending_idx
] = handle
;
882 static inline void xenvif_grant_handle_reset(struct xenvif_queue
*queue
,
885 if (unlikely(queue
->grant_tx_handle
[pending_idx
] ==
886 NETBACK_INVALID_HANDLE
)) {
887 netdev_err(queue
->vif
->dev
,
888 "Trying to unmap invalid handle! pending_idx: 0x%x\n",
892 queue
->grant_tx_handle
[pending_idx
] = NETBACK_INVALID_HANDLE
;
895 static int xenvif_tx_check_gop(struct xenvif_queue
*queue
,
897 struct gnttab_map_grant_ref
**gopp_map
,
898 struct gnttab_copy
**gopp_copy
)
900 struct gnttab_map_grant_ref
*gop_map
= *gopp_map
;
901 u16 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
902 /* This always points to the shinfo of the skb being checked, which
903 * could be either the first or the one on the frag_list
905 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
906 /* If this is non-NULL, we are currently checking the frag_list skb, and
907 * this points to the shinfo of the first one
909 struct skb_shared_info
*first_shinfo
= NULL
;
910 int nr_frags
= shinfo
->nr_frags
;
911 const bool sharedslot
= nr_frags
&&
912 frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
;
915 /* Check status of header. */
916 err
= (*gopp_copy
)->status
;
919 netdev_dbg(queue
->vif
->dev
,
920 "Grant copy of header failed! status: %d pending_idx: %u ref: %u\n",
921 (*gopp_copy
)->status
,
923 (*gopp_copy
)->source
.u
.ref
);
924 /* The first frag might still have this slot mapped */
926 xenvif_idx_release(queue
, pending_idx
,
927 XEN_NETIF_RSP_ERROR
);
932 for (i
= 0; i
< nr_frags
; i
++, gop_map
++) {
935 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[i
]);
937 /* Check error status: if okay then remember grant handle. */
938 newerr
= gop_map
->status
;
940 if (likely(!newerr
)) {
941 xenvif_grant_handle_set(queue
,
944 /* Had a previous error? Invalidate this fragment. */
946 xenvif_idx_unmap(queue
, pending_idx
);
947 /* If the mapping of the first frag was OK, but
948 * the header's copy failed, and they are
949 * sharing a slot, send an error
951 if (i
== 0 && sharedslot
)
952 xenvif_idx_release(queue
, pending_idx
,
953 XEN_NETIF_RSP_ERROR
);
955 xenvif_idx_release(queue
, pending_idx
,
961 /* Error on this fragment: respond to client with an error. */
963 netdev_dbg(queue
->vif
->dev
,
964 "Grant map of %d. frag failed! status: %d pending_idx: %u ref: %u\n",
970 xenvif_idx_release(queue
, pending_idx
, XEN_NETIF_RSP_ERROR
);
972 /* Not the first error? Preceding frags already invalidated. */
976 /* First error: if the header haven't shared a slot with the
977 * first frag, release it as well.
980 xenvif_idx_release(queue
,
981 XENVIF_TX_CB(skb
)->pending_idx
,
984 /* Invalidate preceding fragments of this skb. */
985 for (j
= 0; j
< i
; j
++) {
986 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[j
]);
987 xenvif_idx_unmap(queue
, pending_idx
);
988 xenvif_idx_release(queue
, pending_idx
,
992 /* And if we found the error while checking the frag_list, unmap
993 * the first skb's frags
996 for (j
= 0; j
< first_shinfo
->nr_frags
; j
++) {
997 pending_idx
= frag_get_pending_idx(&first_shinfo
->frags
[j
]);
998 xenvif_idx_unmap(queue
, pending_idx
);
999 xenvif_idx_release(queue
, pending_idx
,
1000 XEN_NETIF_RSP_OKAY
);
1004 /* Remember the error: invalidate all subsequent fragments. */
1008 if (skb_has_frag_list(skb
) && !first_shinfo
) {
1009 first_shinfo
= skb_shinfo(skb
);
1010 shinfo
= skb_shinfo(skb_shinfo(skb
)->frag_list
);
1011 nr_frags
= shinfo
->nr_frags
;
1016 *gopp_map
= gop_map
;
1020 static void xenvif_fill_frags(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1022 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1023 int nr_frags
= shinfo
->nr_frags
;
1025 u16 prev_pending_idx
= INVALID_PENDING_IDX
;
1027 for (i
= 0; i
< nr_frags
; i
++) {
1028 skb_frag_t
*frag
= shinfo
->frags
+ i
;
1029 struct xen_netif_tx_request
*txp
;
1033 pending_idx
= frag_get_pending_idx(frag
);
1035 /* If this is not the first frag, chain it to the previous*/
1036 if (prev_pending_idx
== INVALID_PENDING_IDX
)
1037 skb_shinfo(skb
)->destructor_arg
=
1038 &callback_param(queue
, pending_idx
);
1040 callback_param(queue
, prev_pending_idx
).ctx
=
1041 &callback_param(queue
, pending_idx
);
1043 callback_param(queue
, pending_idx
).ctx
= NULL
;
1044 prev_pending_idx
= pending_idx
;
1046 txp
= &queue
->pending_tx_info
[pending_idx
].req
;
1047 page
= virt_to_page(idx_to_kaddr(queue
, pending_idx
));
1048 __skb_fill_page_desc(skb
, i
, page
, txp
->offset
, txp
->size
);
1049 skb
->len
+= txp
->size
;
1050 skb
->data_len
+= txp
->size
;
1051 skb
->truesize
+= txp
->size
;
1053 /* Take an extra reference to offset network stack's put_page */
1054 get_page(queue
->mmap_pages
[pending_idx
]);
1058 static int xenvif_get_extras(struct xenvif_queue
*queue
,
1059 struct xen_netif_extra_info
*extras
,
1062 struct xen_netif_extra_info extra
;
1063 RING_IDX cons
= queue
->tx
.req_cons
;
1066 if (unlikely(work_to_do
-- <= 0)) {
1067 netdev_err(queue
->vif
->dev
, "Missing extra info\n");
1068 xenvif_fatal_tx_err(queue
->vif
);
1072 memcpy(&extra
, RING_GET_REQUEST(&queue
->tx
, cons
),
1074 if (unlikely(!extra
.type
||
1075 extra
.type
>= XEN_NETIF_EXTRA_TYPE_MAX
)) {
1076 queue
->tx
.req_cons
= ++cons
;
1077 netdev_err(queue
->vif
->dev
,
1078 "Invalid extra type: %d\n", extra
.type
);
1079 xenvif_fatal_tx_err(queue
->vif
);
1083 memcpy(&extras
[extra
.type
- 1], &extra
, sizeof(extra
));
1084 queue
->tx
.req_cons
= ++cons
;
1085 } while (extra
.flags
& XEN_NETIF_EXTRA_FLAG_MORE
);
1090 static int xenvif_set_skb_gso(struct xenvif
*vif
,
1091 struct sk_buff
*skb
,
1092 struct xen_netif_extra_info
*gso
)
1094 if (!gso
->u
.gso
.size
) {
1095 netdev_err(vif
->dev
, "GSO size must not be zero.\n");
1096 xenvif_fatal_tx_err(vif
);
1100 switch (gso
->u
.gso
.type
) {
1101 case XEN_NETIF_GSO_TYPE_TCPV4
:
1102 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
1104 case XEN_NETIF_GSO_TYPE_TCPV6
:
1105 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV6
;
1108 netdev_err(vif
->dev
, "Bad GSO type %d.\n", gso
->u
.gso
.type
);
1109 xenvif_fatal_tx_err(vif
);
1113 skb_shinfo(skb
)->gso_size
= gso
->u
.gso
.size
;
1114 /* gso_segs will be calculated later */
1119 static int checksum_setup(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1121 bool recalculate_partial_csum
= false;
1123 /* A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1124 * peers can fail to set NETRXF_csum_blank when sending a GSO
1125 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1126 * recalculate the partial checksum.
1128 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
&& skb_is_gso(skb
)) {
1129 queue
->stats
.rx_gso_checksum_fixup
++;
1130 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1131 recalculate_partial_csum
= true;
1134 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1135 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1138 return skb_checksum_setup(skb
, recalculate_partial_csum
);
1141 static bool tx_credit_exceeded(struct xenvif_queue
*queue
, unsigned size
)
1143 u64 now
= get_jiffies_64();
1144 u64 next_credit
= queue
->credit_window_start
+
1145 msecs_to_jiffies(queue
->credit_usec
/ 1000);
1147 /* Timer could already be pending in rare cases. */
1148 if (timer_pending(&queue
->credit_timeout
))
1151 /* Passed the point where we can replenish credit? */
1152 if (time_after_eq64(now
, next_credit
)) {
1153 queue
->credit_window_start
= now
;
1154 tx_add_credit(queue
);
1157 /* Still too big to send right now? Set a callback. */
1158 if (size
> queue
->remaining_credit
) {
1159 queue
->credit_timeout
.data
=
1160 (unsigned long)queue
;
1161 mod_timer(&queue
->credit_timeout
,
1163 queue
->credit_window_start
= next_credit
;
1171 /* No locking is required in xenvif_mcast_add/del() as they are
1172 * only ever invoked from NAPI poll. An RCU list is used because
1173 * xenvif_mcast_match() is called asynchronously, during start_xmit.
1176 static int xenvif_mcast_add(struct xenvif
*vif
, const u8
*addr
)
1178 struct xenvif_mcast_addr
*mcast
;
1180 if (vif
->fe_mcast_count
== XEN_NETBK_MCAST_MAX
) {
1181 if (net_ratelimit())
1182 netdev_err(vif
->dev
,
1183 "Too many multicast addresses\n");
1187 mcast
= kzalloc(sizeof(*mcast
), GFP_ATOMIC
);
1191 ether_addr_copy(mcast
->addr
, addr
);
1192 list_add_tail_rcu(&mcast
->entry
, &vif
->fe_mcast_addr
);
1193 vif
->fe_mcast_count
++;
1198 static void xenvif_mcast_del(struct xenvif
*vif
, const u8
*addr
)
1200 struct xenvif_mcast_addr
*mcast
;
1202 list_for_each_entry_rcu(mcast
, &vif
->fe_mcast_addr
, entry
) {
1203 if (ether_addr_equal(addr
, mcast
->addr
)) {
1204 --vif
->fe_mcast_count
;
1205 list_del_rcu(&mcast
->entry
);
1206 kfree_rcu(mcast
, rcu
);
1212 bool xenvif_mcast_match(struct xenvif
*vif
, const u8
*addr
)
1214 struct xenvif_mcast_addr
*mcast
;
1217 list_for_each_entry_rcu(mcast
, &vif
->fe_mcast_addr
, entry
) {
1218 if (ether_addr_equal(addr
, mcast
->addr
)) {
1228 void xenvif_mcast_addr_list_free(struct xenvif
*vif
)
1230 /* No need for locking or RCU here. NAPI poll and TX queue
1233 while (!list_empty(&vif
->fe_mcast_addr
)) {
1234 struct xenvif_mcast_addr
*mcast
;
1236 mcast
= list_first_entry(&vif
->fe_mcast_addr
,
1237 struct xenvif_mcast_addr
,
1239 --vif
->fe_mcast_count
;
1240 list_del(&mcast
->entry
);
1245 static void xenvif_tx_build_gops(struct xenvif_queue
*queue
,
1250 struct gnttab_map_grant_ref
*gop
= queue
->tx_map_ops
;
1251 struct sk_buff
*skb
, *nskb
;
1253 unsigned int frag_overflow
;
1255 while (skb_queue_len(&queue
->tx_queue
) < budget
) {
1256 struct xen_netif_tx_request txreq
;
1257 struct xen_netif_tx_request txfrags
[XEN_NETBK_LEGACY_SLOTS_MAX
];
1258 struct xen_netif_extra_info extras
[XEN_NETIF_EXTRA_TYPE_MAX
-1];
1262 unsigned int data_len
;
1263 pending_ring_idx_t index
;
1265 if (queue
->tx
.sring
->req_prod
- queue
->tx
.req_cons
>
1266 XEN_NETIF_TX_RING_SIZE
) {
1267 netdev_err(queue
->vif
->dev
,
1268 "Impossible number of requests. "
1269 "req_prod %d, req_cons %d, size %ld\n",
1270 queue
->tx
.sring
->req_prod
, queue
->tx
.req_cons
,
1271 XEN_NETIF_TX_RING_SIZE
);
1272 xenvif_fatal_tx_err(queue
->vif
);
1276 work_to_do
= RING_HAS_UNCONSUMED_REQUESTS(&queue
->tx
);
1280 idx
= queue
->tx
.req_cons
;
1281 rmb(); /* Ensure that we see the request before we copy it. */
1282 memcpy(&txreq
, RING_GET_REQUEST(&queue
->tx
, idx
), sizeof(txreq
));
1284 /* Credit-based scheduling. */
1285 if (txreq
.size
> queue
->remaining_credit
&&
1286 tx_credit_exceeded(queue
, txreq
.size
))
1289 queue
->remaining_credit
-= txreq
.size
;
1292 queue
->tx
.req_cons
= ++idx
;
1294 memset(extras
, 0, sizeof(extras
));
1295 if (txreq
.flags
& XEN_NETTXF_extra_info
) {
1296 work_to_do
= xenvif_get_extras(queue
, extras
,
1298 idx
= queue
->tx
.req_cons
;
1299 if (unlikely(work_to_do
< 0))
1303 if (extras
[XEN_NETIF_EXTRA_TYPE_MCAST_ADD
- 1].type
) {
1304 struct xen_netif_extra_info
*extra
;
1306 extra
= &extras
[XEN_NETIF_EXTRA_TYPE_MCAST_ADD
- 1];
1307 ret
= xenvif_mcast_add(queue
->vif
, extra
->u
.mcast
.addr
);
1309 make_tx_response(queue
, &txreq
,
1311 XEN_NETIF_RSP_OKAY
:
1312 XEN_NETIF_RSP_ERROR
);
1313 push_tx_responses(queue
);
1317 if (extras
[XEN_NETIF_EXTRA_TYPE_MCAST_DEL
- 1].type
) {
1318 struct xen_netif_extra_info
*extra
;
1320 extra
= &extras
[XEN_NETIF_EXTRA_TYPE_MCAST_DEL
- 1];
1321 xenvif_mcast_del(queue
->vif
, extra
->u
.mcast
.addr
);
1323 make_tx_response(queue
, &txreq
, XEN_NETIF_RSP_OKAY
);
1324 push_tx_responses(queue
);
1328 ret
= xenvif_count_requests(queue
, &txreq
, txfrags
, work_to_do
);
1329 if (unlikely(ret
< 0))
1334 if (unlikely(txreq
.size
< ETH_HLEN
)) {
1335 netdev_dbg(queue
->vif
->dev
,
1336 "Bad packet size: %d\n", txreq
.size
);
1337 xenvif_tx_err(queue
, &txreq
, idx
);
1341 /* No crossing a page as the payload mustn't fragment. */
1342 if (unlikely((txreq
.offset
+ txreq
.size
) > PAGE_SIZE
)) {
1343 netdev_err(queue
->vif
->dev
,
1344 "txreq.offset: %u, size: %u, end: %lu\n",
1345 txreq
.offset
, txreq
.size
,
1346 (unsigned long)(txreq
.offset
&~PAGE_MASK
) + txreq
.size
);
1347 xenvif_fatal_tx_err(queue
->vif
);
1351 index
= pending_index(queue
->pending_cons
);
1352 pending_idx
= queue
->pending_ring
[index
];
1354 data_len
= (txreq
.size
> XEN_NETBACK_TX_COPY_LEN
&&
1355 ret
< XEN_NETBK_LEGACY_SLOTS_MAX
) ?
1356 XEN_NETBACK_TX_COPY_LEN
: txreq
.size
;
1358 skb
= xenvif_alloc_skb(data_len
);
1359 if (unlikely(skb
== NULL
)) {
1360 netdev_dbg(queue
->vif
->dev
,
1361 "Can't allocate a skb in start_xmit.\n");
1362 xenvif_tx_err(queue
, &txreq
, idx
);
1366 skb_shinfo(skb
)->nr_frags
= ret
;
1367 if (data_len
< txreq
.size
)
1368 skb_shinfo(skb
)->nr_frags
++;
1369 /* At this point shinfo->nr_frags is in fact the number of
1370 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
1374 if (skb_shinfo(skb
)->nr_frags
> MAX_SKB_FRAGS
) {
1375 frag_overflow
= skb_shinfo(skb
)->nr_frags
- MAX_SKB_FRAGS
;
1376 BUG_ON(frag_overflow
> MAX_SKB_FRAGS
);
1377 skb_shinfo(skb
)->nr_frags
= MAX_SKB_FRAGS
;
1378 nskb
= xenvif_alloc_skb(0);
1379 if (unlikely(nskb
== NULL
)) {
1381 xenvif_tx_err(queue
, &txreq
, idx
);
1382 if (net_ratelimit())
1383 netdev_err(queue
->vif
->dev
,
1384 "Can't allocate the frag_list skb.\n");
1389 if (extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1].type
) {
1390 struct xen_netif_extra_info
*gso
;
1391 gso
= &extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1];
1393 if (xenvif_set_skb_gso(queue
->vif
, skb
, gso
)) {
1394 /* Failure in xenvif_set_skb_gso is fatal. */
1401 XENVIF_TX_CB(skb
)->pending_idx
= pending_idx
;
1403 __skb_put(skb
, data_len
);
1404 queue
->tx_copy_ops
[*copy_ops
].source
.u
.ref
= txreq
.gref
;
1405 queue
->tx_copy_ops
[*copy_ops
].source
.domid
= queue
->vif
->domid
;
1406 queue
->tx_copy_ops
[*copy_ops
].source
.offset
= txreq
.offset
;
1408 queue
->tx_copy_ops
[*copy_ops
].dest
.u
.gmfn
=
1409 virt_to_mfn(skb
->data
);
1410 queue
->tx_copy_ops
[*copy_ops
].dest
.domid
= DOMID_SELF
;
1411 queue
->tx_copy_ops
[*copy_ops
].dest
.offset
=
1412 offset_in_page(skb
->data
);
1414 queue
->tx_copy_ops
[*copy_ops
].len
= data_len
;
1415 queue
->tx_copy_ops
[*copy_ops
].flags
= GNTCOPY_source_gref
;
1419 if (data_len
< txreq
.size
) {
1420 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1422 xenvif_tx_create_map_op(queue
, pending_idx
, &txreq
, gop
);
1425 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1426 INVALID_PENDING_IDX
);
1427 memcpy(&queue
->pending_tx_info
[pending_idx
].req
, &txreq
,
1431 queue
->pending_cons
++;
1433 gop
= xenvif_get_requests(queue
, skb
, txfrags
, gop
,
1434 frag_overflow
, nskb
);
1436 __skb_queue_tail(&queue
->tx_queue
, skb
);
1438 queue
->tx
.req_cons
= idx
;
1440 if (((gop
-queue
->tx_map_ops
) >= ARRAY_SIZE(queue
->tx_map_ops
)) ||
1441 (*copy_ops
>= ARRAY_SIZE(queue
->tx_copy_ops
)))
1445 (*map_ops
) = gop
- queue
->tx_map_ops
;
1449 /* Consolidate skb with a frag_list into a brand new one with local pages on
1450 * frags. Returns 0 or -ENOMEM if can't allocate new pages.
1452 static int xenvif_handle_frag_list(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1454 unsigned int offset
= skb_headlen(skb
);
1455 skb_frag_t frags
[MAX_SKB_FRAGS
];
1457 struct ubuf_info
*uarg
;
1458 struct sk_buff
*nskb
= skb_shinfo(skb
)->frag_list
;
1460 queue
->stats
.tx_zerocopy_sent
+= 2;
1461 queue
->stats
.tx_frag_overflow
++;
1463 xenvif_fill_frags(queue
, nskb
);
1464 /* Subtract frags size, we will correct it later */
1465 skb
->truesize
-= skb
->data_len
;
1466 skb
->len
+= nskb
->len
;
1467 skb
->data_len
+= nskb
->len
;
1469 /* create a brand new frags array and coalesce there */
1470 for (i
= 0; offset
< skb
->len
; i
++) {
1474 BUG_ON(i
>= MAX_SKB_FRAGS
);
1475 page
= alloc_page(GFP_ATOMIC
);
1478 skb
->truesize
+= skb
->data_len
;
1479 for (j
= 0; j
< i
; j
++)
1480 put_page(frags
[j
].page
.p
);
1484 if (offset
+ PAGE_SIZE
< skb
->len
)
1487 len
= skb
->len
- offset
;
1488 if (skb_copy_bits(skb
, offset
, page_address(page
), len
))
1492 frags
[i
].page
.p
= page
;
1493 frags
[i
].page_offset
= 0;
1494 skb_frag_size_set(&frags
[i
], len
);
1497 /* Copied all the bits from the frag list -- free it. */
1498 skb_frag_list_init(skb
);
1499 xenvif_skb_zerocopy_prepare(queue
, nskb
);
1502 /* Release all the original (foreign) frags. */
1503 for (f
= 0; f
< skb_shinfo(skb
)->nr_frags
; f
++)
1504 skb_frag_unref(skb
, f
);
1505 uarg
= skb_shinfo(skb
)->destructor_arg
;
1506 /* increase inflight counter to offset decrement in callback */
1507 atomic_inc(&queue
->inflight_packets
);
1508 uarg
->callback(uarg
, true);
1509 skb_shinfo(skb
)->destructor_arg
= NULL
;
1511 /* Fill the skb with the new (local) frags. */
1512 memcpy(skb_shinfo(skb
)->frags
, frags
, i
* sizeof(skb_frag_t
));
1513 skb_shinfo(skb
)->nr_frags
= i
;
1514 skb
->truesize
+= i
* PAGE_SIZE
;
1519 static int xenvif_tx_submit(struct xenvif_queue
*queue
)
1521 struct gnttab_map_grant_ref
*gop_map
= queue
->tx_map_ops
;
1522 struct gnttab_copy
*gop_copy
= queue
->tx_copy_ops
;
1523 struct sk_buff
*skb
;
1526 while ((skb
= __skb_dequeue(&queue
->tx_queue
)) != NULL
) {
1527 struct xen_netif_tx_request
*txp
;
1531 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
1532 txp
= &queue
->pending_tx_info
[pending_idx
].req
;
1534 /* Check the remap error code. */
1535 if (unlikely(xenvif_tx_check_gop(queue
, skb
, &gop_map
, &gop_copy
))) {
1536 /* If there was an error, xenvif_tx_check_gop is
1537 * expected to release all the frags which were mapped,
1538 * so kfree_skb shouldn't do it again
1540 skb_shinfo(skb
)->nr_frags
= 0;
1541 if (skb_has_frag_list(skb
)) {
1542 struct sk_buff
*nskb
=
1543 skb_shinfo(skb
)->frag_list
;
1544 skb_shinfo(nskb
)->nr_frags
= 0;
1550 data_len
= skb
->len
;
1551 callback_param(queue
, pending_idx
).ctx
= NULL
;
1552 if (data_len
< txp
->size
) {
1553 /* Append the packet payload as a fragment. */
1554 txp
->offset
+= data_len
;
1555 txp
->size
-= data_len
;
1557 /* Schedule a response immediately. */
1558 xenvif_idx_release(queue
, pending_idx
,
1559 XEN_NETIF_RSP_OKAY
);
1562 if (txp
->flags
& XEN_NETTXF_csum_blank
)
1563 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1564 else if (txp
->flags
& XEN_NETTXF_data_validated
)
1565 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1567 xenvif_fill_frags(queue
, skb
);
1569 if (unlikely(skb_has_frag_list(skb
))) {
1570 if (xenvif_handle_frag_list(queue
, skb
)) {
1571 if (net_ratelimit())
1572 netdev_err(queue
->vif
->dev
,
1573 "Not enough memory to consolidate frag_list!\n");
1574 xenvif_skb_zerocopy_prepare(queue
, skb
);
1580 skb
->dev
= queue
->vif
->dev
;
1581 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1582 skb_reset_network_header(skb
);
1584 if (checksum_setup(queue
, skb
)) {
1585 netdev_dbg(queue
->vif
->dev
,
1586 "Can't setup checksum in net_tx_action\n");
1587 /* We have to set this flag to trigger the callback */
1588 if (skb_shinfo(skb
)->destructor_arg
)
1589 xenvif_skb_zerocopy_prepare(queue
, skb
);
1594 skb_probe_transport_header(skb
, 0);
1596 /* If the packet is GSO then we will have just set up the
1597 * transport header offset in checksum_setup so it's now
1598 * straightforward to calculate gso_segs.
1600 if (skb_is_gso(skb
)) {
1601 int mss
= skb_shinfo(skb
)->gso_size
;
1602 int hdrlen
= skb_transport_header(skb
) -
1603 skb_mac_header(skb
) +
1606 skb_shinfo(skb
)->gso_segs
=
1607 DIV_ROUND_UP(skb
->len
- hdrlen
, mss
);
1610 queue
->stats
.rx_bytes
+= skb
->len
;
1611 queue
->stats
.rx_packets
++;
1615 /* Set this flag right before netif_receive_skb, otherwise
1616 * someone might think this packet already left netback, and
1617 * do a skb_copy_ubufs while we are still in control of the
1618 * skb. E.g. the __pskb_pull_tail earlier can do such thing.
1620 if (skb_shinfo(skb
)->destructor_arg
) {
1621 xenvif_skb_zerocopy_prepare(queue
, skb
);
1622 queue
->stats
.tx_zerocopy_sent
++;
1625 netif_receive_skb(skb
);
1631 void xenvif_zerocopy_callback(struct ubuf_info
*ubuf
, bool zerocopy_success
)
1633 unsigned long flags
;
1634 pending_ring_idx_t index
;
1635 struct xenvif_queue
*queue
= ubuf_to_queue(ubuf
);
1637 /* This is the only place where we grab this lock, to protect callbacks
1640 spin_lock_irqsave(&queue
->callback_lock
, flags
);
1642 u16 pending_idx
= ubuf
->desc
;
1643 ubuf
= (struct ubuf_info
*) ubuf
->ctx
;
1644 BUG_ON(queue
->dealloc_prod
- queue
->dealloc_cons
>=
1646 index
= pending_index(queue
->dealloc_prod
);
1647 queue
->dealloc_ring
[index
] = pending_idx
;
1648 /* Sync with xenvif_tx_dealloc_action:
1649 * insert idx then incr producer.
1652 queue
->dealloc_prod
++;
1654 spin_unlock_irqrestore(&queue
->callback_lock
, flags
);
1656 if (likely(zerocopy_success
))
1657 queue
->stats
.tx_zerocopy_success
++;
1659 queue
->stats
.tx_zerocopy_fail
++;
1660 xenvif_skb_zerocopy_complete(queue
);
1663 static inline void xenvif_tx_dealloc_action(struct xenvif_queue
*queue
)
1665 struct gnttab_unmap_grant_ref
*gop
;
1666 pending_ring_idx_t dc
, dp
;
1667 u16 pending_idx
, pending_idx_release
[MAX_PENDING_REQS
];
1670 dc
= queue
->dealloc_cons
;
1671 gop
= queue
->tx_unmap_ops
;
1673 /* Free up any grants we have finished using */
1675 dp
= queue
->dealloc_prod
;
1677 /* Ensure we see all indices enqueued by all
1678 * xenvif_zerocopy_callback().
1683 BUG_ON(gop
- queue
->tx_unmap_ops
>= MAX_PENDING_REQS
);
1685 queue
->dealloc_ring
[pending_index(dc
++)];
1687 pending_idx_release
[gop
- queue
->tx_unmap_ops
] =
1689 queue
->pages_to_unmap
[gop
- queue
->tx_unmap_ops
] =
1690 queue
->mmap_pages
[pending_idx
];
1691 gnttab_set_unmap_op(gop
,
1692 idx_to_kaddr(queue
, pending_idx
),
1694 queue
->grant_tx_handle
[pending_idx
]);
1695 xenvif_grant_handle_reset(queue
, pending_idx
);
1699 } while (dp
!= queue
->dealloc_prod
);
1701 queue
->dealloc_cons
= dc
;
1703 if (gop
- queue
->tx_unmap_ops
> 0) {
1705 ret
= gnttab_unmap_refs(queue
->tx_unmap_ops
,
1707 queue
->pages_to_unmap
,
1708 gop
- queue
->tx_unmap_ops
);
1710 netdev_err(queue
->vif
->dev
, "Unmap fail: nr_ops %tu ret %d\n",
1711 gop
- queue
->tx_unmap_ops
, ret
);
1712 for (i
= 0; i
< gop
- queue
->tx_unmap_ops
; ++i
) {
1713 if (gop
[i
].status
!= GNTST_okay
)
1714 netdev_err(queue
->vif
->dev
,
1715 " host_addr: 0x%llx handle: 0x%x status: %d\n",
1724 for (i
= 0; i
< gop
- queue
->tx_unmap_ops
; ++i
)
1725 xenvif_idx_release(queue
, pending_idx_release
[i
],
1726 XEN_NETIF_RSP_OKAY
);
1730 /* Called after netfront has transmitted */
1731 int xenvif_tx_action(struct xenvif_queue
*queue
, int budget
)
1733 unsigned nr_mops
, nr_cops
= 0;
1736 if (unlikely(!tx_work_todo(queue
)))
1739 xenvif_tx_build_gops(queue
, budget
, &nr_cops
, &nr_mops
);
1744 gnttab_batch_copy(queue
->tx_copy_ops
, nr_cops
);
1746 ret
= gnttab_map_refs(queue
->tx_map_ops
,
1748 queue
->pages_to_map
,
1753 work_done
= xenvif_tx_submit(queue
);
1758 static void xenvif_idx_release(struct xenvif_queue
*queue
, u16 pending_idx
,
1761 struct pending_tx_info
*pending_tx_info
;
1762 pending_ring_idx_t index
;
1763 unsigned long flags
;
1765 pending_tx_info
= &queue
->pending_tx_info
[pending_idx
];
1767 spin_lock_irqsave(&queue
->response_lock
, flags
);
1769 make_tx_response(queue
, &pending_tx_info
->req
, status
);
1771 /* Release the pending index before pusing the Tx response so
1772 * its available before a new Tx request is pushed by the
1775 index
= pending_index(queue
->pending_prod
++);
1776 queue
->pending_ring
[index
] = pending_idx
;
1778 push_tx_responses(queue
);
1780 spin_unlock_irqrestore(&queue
->response_lock
, flags
);
1784 static void make_tx_response(struct xenvif_queue
*queue
,
1785 struct xen_netif_tx_request
*txp
,
1788 RING_IDX i
= queue
->tx
.rsp_prod_pvt
;
1789 struct xen_netif_tx_response
*resp
;
1791 resp
= RING_GET_RESPONSE(&queue
->tx
, i
);
1795 if (txp
->flags
& XEN_NETTXF_extra_info
)
1796 RING_GET_RESPONSE(&queue
->tx
, ++i
)->status
= XEN_NETIF_RSP_NULL
;
1798 queue
->tx
.rsp_prod_pvt
= ++i
;
1801 static void push_tx_responses(struct xenvif_queue
*queue
)
1805 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue
->tx
, notify
);
1807 notify_remote_via_irq(queue
->tx_irq
);
1810 static struct xen_netif_rx_response
*make_rx_response(struct xenvif_queue
*queue
,
1817 RING_IDX i
= queue
->rx
.rsp_prod_pvt
;
1818 struct xen_netif_rx_response
*resp
;
1820 resp
= RING_GET_RESPONSE(&queue
->rx
, i
);
1821 resp
->offset
= offset
;
1822 resp
->flags
= flags
;
1824 resp
->status
= (s16
)size
;
1826 resp
->status
= (s16
)st
;
1828 queue
->rx
.rsp_prod_pvt
= ++i
;
1833 void xenvif_idx_unmap(struct xenvif_queue
*queue
, u16 pending_idx
)
1836 struct gnttab_unmap_grant_ref tx_unmap_op
;
1838 gnttab_set_unmap_op(&tx_unmap_op
,
1839 idx_to_kaddr(queue
, pending_idx
),
1841 queue
->grant_tx_handle
[pending_idx
]);
1842 xenvif_grant_handle_reset(queue
, pending_idx
);
1844 ret
= gnttab_unmap_refs(&tx_unmap_op
, NULL
,
1845 &queue
->mmap_pages
[pending_idx
], 1);
1847 netdev_err(queue
->vif
->dev
,
1848 "Unmap fail: ret: %d pending_idx: %d host_addr: %llx handle: 0x%x status: %d\n",
1851 tx_unmap_op
.host_addr
,
1853 tx_unmap_op
.status
);
1858 static inline int tx_work_todo(struct xenvif_queue
*queue
)
1860 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue
->tx
)))
1866 static inline bool tx_dealloc_work_todo(struct xenvif_queue
*queue
)
1868 return queue
->dealloc_cons
!= queue
->dealloc_prod
;
1871 void xenvif_unmap_frontend_rings(struct xenvif_queue
*queue
)
1873 if (queue
->tx
.sring
)
1874 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue
->vif
),
1876 if (queue
->rx
.sring
)
1877 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue
->vif
),
1881 int xenvif_map_frontend_rings(struct xenvif_queue
*queue
,
1882 grant_ref_t tx_ring_ref
,
1883 grant_ref_t rx_ring_ref
)
1886 struct xen_netif_tx_sring
*txs
;
1887 struct xen_netif_rx_sring
*rxs
;
1891 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue
->vif
),
1892 &tx_ring_ref
, 1, &addr
);
1896 txs
= (struct xen_netif_tx_sring
*)addr
;
1897 BACK_RING_INIT(&queue
->tx
, txs
, PAGE_SIZE
);
1899 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue
->vif
),
1900 &rx_ring_ref
, 1, &addr
);
1904 rxs
= (struct xen_netif_rx_sring
*)addr
;
1905 BACK_RING_INIT(&queue
->rx
, rxs
, PAGE_SIZE
);
1910 xenvif_unmap_frontend_rings(queue
);
1914 static void xenvif_queue_carrier_off(struct xenvif_queue
*queue
)
1916 struct xenvif
*vif
= queue
->vif
;
1918 queue
->stalled
= true;
1920 /* At least one queue has stalled? Disable the carrier. */
1921 spin_lock(&vif
->lock
);
1922 if (vif
->stalled_queues
++ == 0) {
1923 netdev_info(vif
->dev
, "Guest Rx stalled");
1924 netif_carrier_off(vif
->dev
);
1926 spin_unlock(&vif
->lock
);
1929 static void xenvif_queue_carrier_on(struct xenvif_queue
*queue
)
1931 struct xenvif
*vif
= queue
->vif
;
1933 queue
->last_rx_time
= jiffies
; /* Reset Rx stall detection. */
1934 queue
->stalled
= false;
1936 /* All queues are ready? Enable the carrier. */
1937 spin_lock(&vif
->lock
);
1938 if (--vif
->stalled_queues
== 0) {
1939 netdev_info(vif
->dev
, "Guest Rx ready");
1940 netif_carrier_on(vif
->dev
);
1942 spin_unlock(&vif
->lock
);
1945 static bool xenvif_rx_queue_stalled(struct xenvif_queue
*queue
)
1947 RING_IDX prod
, cons
;
1949 prod
= queue
->rx
.sring
->req_prod
;
1950 cons
= queue
->rx
.req_cons
;
1952 return !queue
->stalled
&& prod
- cons
< 1
1953 && time_after(jiffies
,
1954 queue
->last_rx_time
+ queue
->vif
->stall_timeout
);
1957 static bool xenvif_rx_queue_ready(struct xenvif_queue
*queue
)
1959 RING_IDX prod
, cons
;
1961 prod
= queue
->rx
.sring
->req_prod
;
1962 cons
= queue
->rx
.req_cons
;
1964 return queue
->stalled
&& prod
- cons
>= 1;
1967 static bool xenvif_have_rx_work(struct xenvif_queue
*queue
)
1969 return (!skb_queue_empty(&queue
->rx_queue
)
1970 && xenvif_rx_ring_slots_available(queue
))
1971 || (queue
->vif
->stall_timeout
&&
1972 (xenvif_rx_queue_stalled(queue
)
1973 || xenvif_rx_queue_ready(queue
)))
1974 || kthread_should_stop()
1975 || queue
->vif
->disabled
;
1978 static long xenvif_rx_queue_timeout(struct xenvif_queue
*queue
)
1980 struct sk_buff
*skb
;
1983 skb
= skb_peek(&queue
->rx_queue
);
1985 return MAX_SCHEDULE_TIMEOUT
;
1987 timeout
= XENVIF_RX_CB(skb
)->expires
- jiffies
;
1988 return timeout
< 0 ? 0 : timeout
;
1991 /* Wait until the guest Rx thread has work.
1993 * The timeout needs to be adjusted based on the current head of the
1994 * queue (and not just the head at the beginning). In particular, if
1995 * the queue is initially empty an infinite timeout is used and this
1996 * needs to be reduced when a skb is queued.
1998 * This cannot be done with wait_event_timeout() because it only
1999 * calculates the timeout once.
2001 static void xenvif_wait_for_rx_work(struct xenvif_queue
*queue
)
2005 if (xenvif_have_rx_work(queue
))
2011 prepare_to_wait(&queue
->wq
, &wait
, TASK_INTERRUPTIBLE
);
2012 if (xenvif_have_rx_work(queue
))
2014 ret
= schedule_timeout(xenvif_rx_queue_timeout(queue
));
2018 finish_wait(&queue
->wq
, &wait
);
2021 int xenvif_kthread_guest_rx(void *data
)
2023 struct xenvif_queue
*queue
= data
;
2024 struct xenvif
*vif
= queue
->vif
;
2026 if (!vif
->stall_timeout
)
2027 xenvif_queue_carrier_on(queue
);
2030 xenvif_wait_for_rx_work(queue
);
2032 if (kthread_should_stop())
2035 /* This frontend is found to be rogue, disable it in
2036 * kthread context. Currently this is only set when
2037 * netback finds out frontend sends malformed packet,
2038 * but we cannot disable the interface in softirq
2039 * context so we defer it here, if this thread is
2040 * associated with queue 0.
2042 if (unlikely(vif
->disabled
&& queue
->id
== 0)) {
2043 xenvif_carrier_off(vif
);
2047 if (!skb_queue_empty(&queue
->rx_queue
))
2048 xenvif_rx_action(queue
);
2050 /* If the guest hasn't provided any Rx slots for a
2051 * while it's probably not responsive, drop the
2052 * carrier so packets are dropped earlier.
2054 if (vif
->stall_timeout
) {
2055 if (xenvif_rx_queue_stalled(queue
))
2056 xenvif_queue_carrier_off(queue
);
2057 else if (xenvif_rx_queue_ready(queue
))
2058 xenvif_queue_carrier_on(queue
);
2061 /* Queued packets may have foreign pages from other
2062 * domains. These cannot be queued indefinitely as
2063 * this would starve guests of grant refs and transmit
2066 xenvif_rx_queue_drop_expired(queue
);
2068 xenvif_rx_queue_maybe_wake(queue
);
2073 /* Bin any remaining skbs */
2074 xenvif_rx_queue_purge(queue
);
2079 static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue
*queue
)
2081 /* Dealloc thread must remain running until all inflight
2084 return kthread_should_stop() &&
2085 !atomic_read(&queue
->inflight_packets
);
2088 int xenvif_dealloc_kthread(void *data
)
2090 struct xenvif_queue
*queue
= data
;
2093 wait_event_interruptible(queue
->dealloc_wq
,
2094 tx_dealloc_work_todo(queue
) ||
2095 xenvif_dealloc_kthread_should_stop(queue
));
2096 if (xenvif_dealloc_kthread_should_stop(queue
))
2099 xenvif_tx_dealloc_action(queue
);
2103 /* Unmap anything remaining*/
2104 if (tx_dealloc_work_todo(queue
))
2105 xenvif_tx_dealloc_action(queue
);
2110 static int __init
netback_init(void)
2117 /* Allow as many queues as there are CPUs, by default */
2118 xenvif_max_queues
= num_online_cpus();
2120 if (fatal_skb_slots
< XEN_NETBK_LEGACY_SLOTS_MAX
) {
2121 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n",
2122 fatal_skb_slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
2123 fatal_skb_slots
= XEN_NETBK_LEGACY_SLOTS_MAX
;
2126 rc
= xenvif_xenbus_init();
2130 #ifdef CONFIG_DEBUG_FS
2131 xen_netback_dbg_root
= debugfs_create_dir("xen-netback", NULL
);
2132 if (IS_ERR_OR_NULL(xen_netback_dbg_root
))
2133 pr_warn("Init of debugfs returned %ld!\n",
2134 PTR_ERR(xen_netback_dbg_root
));
2135 #endif /* CONFIG_DEBUG_FS */
2143 module_init(netback_init
);
2145 static void __exit
netback_fini(void)
2147 #ifdef CONFIG_DEBUG_FS
2148 if (!IS_ERR_OR_NULL(xen_netback_dbg_root
))
2149 debugfs_remove_recursive(xen_netback_dbg_root
);
2150 #endif /* CONFIG_DEBUG_FS */
2151 xenvif_xenbus_fini();
2153 module_exit(netback_fini
);
2155 MODULE_LICENSE("Dual BSD/GPL");
2156 MODULE_ALIAS("xen-backend:vif");