2 * Back-end of the driver for virtual network devices. This portion of the
3 * driver exports a 'unified' network-device interface that can be accessed
4 * by any operating system that implements a compatible front end. A
5 * reference front-end implementation can be found in:
6 * drivers/net/xen-netfront.c
8 * Copyright (c) 2002-2005, K A Fraser
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
37 #include <linux/kthread.h>
38 #include <linux/if_vlan.h>
39 #include <linux/udp.h>
40 #include <linux/highmem.h>
45 #include <xen/events.h>
46 #include <xen/interface/memory.h>
48 #include <asm/xen/hypercall.h>
49 #include <asm/xen/page.h>
51 /* Provide an option to disable split event channels at load time as
52 * event channels are limited resource. Split event channels are
55 bool separate_tx_rx_irq
= 1;
56 module_param(separate_tx_rx_irq
, bool, 0644);
58 /* The time that packets can stay on the guest Rx internal queue
59 * before they are dropped.
61 unsigned int rx_drain_timeout_msecs
= 10000;
62 module_param(rx_drain_timeout_msecs
, uint
, 0444);
64 /* The length of time before the frontend is considered unresponsive
65 * because it isn't providing Rx slots.
67 unsigned int rx_stall_timeout_msecs
= 60000;
68 module_param(rx_stall_timeout_msecs
, uint
, 0444);
70 unsigned int xenvif_max_queues
;
71 module_param_named(max_queues
, xenvif_max_queues
, uint
, 0644);
72 MODULE_PARM_DESC(max_queues
,
73 "Maximum number of queues per virtual interface");
76 * This is the maximum slots a skb can have. If a guest sends a skb
77 * which exceeds this limit it is considered malicious.
79 #define FATAL_SKB_SLOTS_DEFAULT 20
80 static unsigned int fatal_skb_slots
= FATAL_SKB_SLOTS_DEFAULT
;
81 module_param(fatal_skb_slots
, uint
, 0444);
83 /* The amount to copy out of the first guest Tx slot into the skb's
84 * linear area. If the first slot has more data, it will be mapped
85 * and put into the first frag.
87 * This is sized to avoid pulling headers from the frags for most
90 #define XEN_NETBACK_TX_COPY_LEN 128
93 static void xenvif_idx_release(struct xenvif_queue
*queue
, u16 pending_idx
,
96 static void make_tx_response(struct xenvif_queue
*queue
,
97 struct xen_netif_tx_request
*txp
,
99 static void push_tx_responses(struct xenvif_queue
*queue
);
101 static inline int tx_work_todo(struct xenvif_queue
*queue
);
103 static struct xen_netif_rx_response
*make_rx_response(struct xenvif_queue
*queue
,
110 static inline unsigned long idx_to_pfn(struct xenvif_queue
*queue
,
113 return page_to_pfn(queue
->mmap_pages
[idx
]);
116 static inline unsigned long idx_to_kaddr(struct xenvif_queue
*queue
,
119 return (unsigned long)pfn_to_kaddr(idx_to_pfn(queue
, idx
));
122 #define callback_param(vif, pending_idx) \
123 (vif->pending_tx_info[pending_idx].callback_struct)
125 /* Find the containing VIF's structure from a pointer in pending_tx_info array
127 static inline struct xenvif_queue
*ubuf_to_queue(const struct ubuf_info
*ubuf
)
129 u16 pending_idx
= ubuf
->desc
;
130 struct pending_tx_info
*temp
=
131 container_of(ubuf
, struct pending_tx_info
, callback_struct
);
132 return container_of(temp
- pending_idx
,
137 static u16
frag_get_pending_idx(skb_frag_t
*frag
)
139 return (u16
)frag
->page_offset
;
142 static void frag_set_pending_idx(skb_frag_t
*frag
, u16 pending_idx
)
144 frag
->page_offset
= pending_idx
;
147 static inline pending_ring_idx_t
pending_index(unsigned i
)
149 return i
& (MAX_PENDING_REQS
-1);
152 bool xenvif_rx_ring_slots_available(struct xenvif_queue
*queue
, int needed
)
157 prod
= queue
->rx
.sring
->req_prod
;
158 cons
= queue
->rx
.req_cons
;
160 if (prod
- cons
>= needed
)
163 queue
->rx
.sring
->req_event
= prod
+ 1;
165 /* Make sure event is visible before we check prod
169 } while (queue
->rx
.sring
->req_prod
!= prod
);
174 void xenvif_rx_queue_tail(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
178 spin_lock_irqsave(&queue
->rx_queue
.lock
, flags
);
180 __skb_queue_tail(&queue
->rx_queue
, skb
);
182 queue
->rx_queue_len
+= skb
->len
;
183 if (queue
->rx_queue_len
> queue
->rx_queue_max
)
184 netif_tx_stop_queue(netdev_get_tx_queue(queue
->vif
->dev
, queue
->id
));
186 spin_unlock_irqrestore(&queue
->rx_queue
.lock
, flags
);
189 static struct sk_buff
*xenvif_rx_dequeue(struct xenvif_queue
*queue
)
193 spin_lock_irq(&queue
->rx_queue
.lock
);
195 skb
= __skb_dequeue(&queue
->rx_queue
);
197 queue
->rx_queue_len
-= skb
->len
;
199 spin_unlock_irq(&queue
->rx_queue
.lock
);
204 static void xenvif_rx_queue_maybe_wake(struct xenvif_queue
*queue
)
206 spin_lock_irq(&queue
->rx_queue
.lock
);
208 if (queue
->rx_queue_len
< queue
->rx_queue_max
)
209 netif_tx_wake_queue(netdev_get_tx_queue(queue
->vif
->dev
, queue
->id
));
211 spin_unlock_irq(&queue
->rx_queue
.lock
);
215 static void xenvif_rx_queue_purge(struct xenvif_queue
*queue
)
218 while ((skb
= xenvif_rx_dequeue(queue
)) != NULL
)
222 static void xenvif_rx_queue_drop_expired(struct xenvif_queue
*queue
)
227 skb
= skb_peek(&queue
->rx_queue
);
230 if (time_before(jiffies
, XENVIF_RX_CB(skb
)->expires
))
232 xenvif_rx_dequeue(queue
);
237 struct netrx_pending_operations
{
238 unsigned copy_prod
, copy_cons
;
239 unsigned meta_prod
, meta_cons
;
240 struct gnttab_copy
*copy
;
241 struct xenvif_rx_meta
*meta
;
243 grant_ref_t copy_gref
;
246 static struct xenvif_rx_meta
*get_next_rx_buffer(struct xenvif_queue
*queue
,
247 struct netrx_pending_operations
*npo
)
249 struct xenvif_rx_meta
*meta
;
250 struct xen_netif_rx_request
*req
;
252 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
254 meta
= npo
->meta
+ npo
->meta_prod
++;
255 meta
->gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
261 npo
->copy_gref
= req
->gref
;
267 * Set up the grant operations for this fragment. If it's a flipping
268 * interface, we also set up the unmap request from here.
270 static void xenvif_gop_frag_copy(struct xenvif_queue
*queue
, struct sk_buff
*skb
,
271 struct netrx_pending_operations
*npo
,
272 struct page
*page
, unsigned long size
,
273 unsigned long offset
, int *head
)
275 struct gnttab_copy
*copy_gop
;
276 struct xenvif_rx_meta
*meta
;
278 int gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
280 /* Data must not cross a page boundary. */
281 BUG_ON(size
+ offset
> PAGE_SIZE
<<compound_order(page
));
283 meta
= npo
->meta
+ npo
->meta_prod
- 1;
285 /* Skip unused frames from start of page */
286 page
+= offset
>> PAGE_SHIFT
;
287 offset
&= ~PAGE_MASK
;
290 struct xen_page_foreign
*foreign
;
292 BUG_ON(offset
>= PAGE_SIZE
);
293 BUG_ON(npo
->copy_off
> MAX_BUFFER_OFFSET
);
295 if (npo
->copy_off
== MAX_BUFFER_OFFSET
)
296 meta
= get_next_rx_buffer(queue
, npo
);
298 bytes
= PAGE_SIZE
- offset
;
302 if (npo
->copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
303 bytes
= MAX_BUFFER_OFFSET
- npo
->copy_off
;
305 copy_gop
= npo
->copy
+ npo
->copy_prod
++;
306 copy_gop
->flags
= GNTCOPY_dest_gref
;
307 copy_gop
->len
= bytes
;
309 foreign
= xen_page_foreign(page
);
311 copy_gop
->source
.domid
= foreign
->domid
;
312 copy_gop
->source
.u
.ref
= foreign
->gref
;
313 copy_gop
->flags
|= GNTCOPY_source_gref
;
315 copy_gop
->source
.domid
= DOMID_SELF
;
316 copy_gop
->source
.u
.gmfn
=
317 virt_to_mfn(page_address(page
));
319 copy_gop
->source
.offset
= offset
;
321 copy_gop
->dest
.domid
= queue
->vif
->domid
;
322 copy_gop
->dest
.offset
= npo
->copy_off
;
323 copy_gop
->dest
.u
.ref
= npo
->copy_gref
;
325 npo
->copy_off
+= bytes
;
332 if (offset
== PAGE_SIZE
&& size
) {
333 BUG_ON(!PageCompound(page
));
338 /* Leave a gap for the GSO descriptor. */
339 if (skb_is_gso(skb
)) {
340 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
)
341 gso_type
= XEN_NETIF_GSO_TYPE_TCPV4
;
342 else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
)
343 gso_type
= XEN_NETIF_GSO_TYPE_TCPV6
;
346 if (*head
&& ((1 << gso_type
) & queue
->vif
->gso_mask
))
347 queue
->rx
.req_cons
++;
349 *head
= 0; /* There must be something in this buffer now. */
355 * Prepare an SKB to be transmitted to the frontend.
357 * This function is responsible for allocating grant operations, meta
360 * It returns the number of meta structures consumed. The number of
361 * ring slots used is always equal to the number of meta slots used
362 * plus the number of GSO descriptors used. Currently, we use either
363 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
364 * frontend-side LRO).
366 static int xenvif_gop_skb(struct sk_buff
*skb
,
367 struct netrx_pending_operations
*npo
,
368 struct xenvif_queue
*queue
)
370 struct xenvif
*vif
= netdev_priv(skb
->dev
);
371 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
373 struct xen_netif_rx_request
*req
;
374 struct xenvif_rx_meta
*meta
;
380 old_meta_prod
= npo
->meta_prod
;
382 gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
383 if (skb_is_gso(skb
)) {
384 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
)
385 gso_type
= XEN_NETIF_GSO_TYPE_TCPV4
;
386 else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
)
387 gso_type
= XEN_NETIF_GSO_TYPE_TCPV6
;
390 /* Set up a GSO prefix descriptor, if necessary */
391 if ((1 << gso_type
) & vif
->gso_prefix_mask
) {
392 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
393 meta
= npo
->meta
+ npo
->meta_prod
++;
394 meta
->gso_type
= gso_type
;
395 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
400 req
= RING_GET_REQUEST(&queue
->rx
, queue
->rx
.req_cons
++);
401 meta
= npo
->meta
+ npo
->meta_prod
++;
403 if ((1 << gso_type
) & vif
->gso_mask
) {
404 meta
->gso_type
= gso_type
;
405 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
407 meta
->gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
414 npo
->copy_gref
= req
->gref
;
417 while (data
< skb_tail_pointer(skb
)) {
418 unsigned int offset
= offset_in_page(data
);
419 unsigned int len
= PAGE_SIZE
- offset
;
421 if (data
+ len
> skb_tail_pointer(skb
))
422 len
= skb_tail_pointer(skb
) - data
;
424 xenvif_gop_frag_copy(queue
, skb
, npo
,
425 virt_to_page(data
), len
, offset
, &head
);
429 for (i
= 0; i
< nr_frags
; i
++) {
430 xenvif_gop_frag_copy(queue
, skb
, npo
,
431 skb_frag_page(&skb_shinfo(skb
)->frags
[i
]),
432 skb_frag_size(&skb_shinfo(skb
)->frags
[i
]),
433 skb_shinfo(skb
)->frags
[i
].page_offset
,
437 return npo
->meta_prod
- old_meta_prod
;
441 * This is a twin to xenvif_gop_skb. Assume that xenvif_gop_skb was
442 * used to set up the operations on the top of
443 * netrx_pending_operations, which have since been done. Check that
444 * they didn't give any errors and advance over them.
446 static int xenvif_check_gop(struct xenvif
*vif
, int nr_meta_slots
,
447 struct netrx_pending_operations
*npo
)
449 struct gnttab_copy
*copy_op
;
450 int status
= XEN_NETIF_RSP_OKAY
;
453 for (i
= 0; i
< nr_meta_slots
; i
++) {
454 copy_op
= npo
->copy
+ npo
->copy_cons
++;
455 if (copy_op
->status
!= GNTST_okay
) {
457 "Bad status %d from copy to DOM%d.\n",
458 copy_op
->status
, vif
->domid
);
459 status
= XEN_NETIF_RSP_ERROR
;
466 static void xenvif_add_frag_responses(struct xenvif_queue
*queue
, int status
,
467 struct xenvif_rx_meta
*meta
,
471 unsigned long offset
;
473 /* No fragments used */
474 if (nr_meta_slots
<= 1)
479 for (i
= 0; i
< nr_meta_slots
; i
++) {
481 if (i
== nr_meta_slots
- 1)
484 flags
= XEN_NETRXF_more_data
;
487 make_rx_response(queue
, meta
[i
].id
, status
, offset
,
488 meta
[i
].size
, flags
);
492 void xenvif_kick_thread(struct xenvif_queue
*queue
)
497 static void xenvif_rx_action(struct xenvif_queue
*queue
)
501 struct xen_netif_rx_response
*resp
;
502 struct sk_buff_head rxq
;
506 unsigned long offset
;
507 bool need_to_notify
= false;
509 struct netrx_pending_operations npo
= {
510 .copy
= queue
->grant_copy_op
,
514 skb_queue_head_init(&rxq
);
516 while (xenvif_rx_ring_slots_available(queue
, XEN_NETBK_RX_SLOTS_MAX
)
517 && (skb
= xenvif_rx_dequeue(queue
)) != NULL
) {
518 RING_IDX old_req_cons
;
519 RING_IDX ring_slots_used
;
521 queue
->last_rx_time
= jiffies
;
523 old_req_cons
= queue
->rx
.req_cons
;
524 XENVIF_RX_CB(skb
)->meta_slots_used
= xenvif_gop_skb(skb
, &npo
, queue
);
525 ring_slots_used
= queue
->rx
.req_cons
- old_req_cons
;
527 __skb_queue_tail(&rxq
, skb
);
530 BUG_ON(npo
.meta_prod
> ARRAY_SIZE(queue
->meta
));
535 BUG_ON(npo
.copy_prod
> MAX_GRANT_COPY_OPS
);
536 gnttab_batch_copy(queue
->grant_copy_op
, npo
.copy_prod
);
538 while ((skb
= __skb_dequeue(&rxq
)) != NULL
) {
540 if ((1 << queue
->meta
[npo
.meta_cons
].gso_type
) &
541 queue
->vif
->gso_prefix_mask
) {
542 resp
= RING_GET_RESPONSE(&queue
->rx
,
543 queue
->rx
.rsp_prod_pvt
++);
545 resp
->flags
= XEN_NETRXF_gso_prefix
| XEN_NETRXF_more_data
;
547 resp
->offset
= queue
->meta
[npo
.meta_cons
].gso_size
;
548 resp
->id
= queue
->meta
[npo
.meta_cons
].id
;
549 resp
->status
= XENVIF_RX_CB(skb
)->meta_slots_used
;
552 XENVIF_RX_CB(skb
)->meta_slots_used
--;
556 queue
->stats
.tx_bytes
+= skb
->len
;
557 queue
->stats
.tx_packets
++;
559 status
= xenvif_check_gop(queue
->vif
,
560 XENVIF_RX_CB(skb
)->meta_slots_used
,
563 if (XENVIF_RX_CB(skb
)->meta_slots_used
== 1)
566 flags
= XEN_NETRXF_more_data
;
568 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) /* local packet? */
569 flags
|= XEN_NETRXF_csum_blank
| XEN_NETRXF_data_validated
;
570 else if (skb
->ip_summed
== CHECKSUM_UNNECESSARY
)
571 /* remote but checksummed. */
572 flags
|= XEN_NETRXF_data_validated
;
575 resp
= make_rx_response(queue
, queue
->meta
[npo
.meta_cons
].id
,
577 queue
->meta
[npo
.meta_cons
].size
,
580 if ((1 << queue
->meta
[npo
.meta_cons
].gso_type
) &
581 queue
->vif
->gso_mask
) {
582 struct xen_netif_extra_info
*gso
=
583 (struct xen_netif_extra_info
*)
584 RING_GET_RESPONSE(&queue
->rx
,
585 queue
->rx
.rsp_prod_pvt
++);
587 resp
->flags
|= XEN_NETRXF_extra_info
;
589 gso
->u
.gso
.type
= queue
->meta
[npo
.meta_cons
].gso_type
;
590 gso
->u
.gso
.size
= queue
->meta
[npo
.meta_cons
].gso_size
;
592 gso
->u
.gso
.features
= 0;
594 gso
->type
= XEN_NETIF_EXTRA_TYPE_GSO
;
598 xenvif_add_frag_responses(queue
, status
,
599 queue
->meta
+ npo
.meta_cons
+ 1,
600 XENVIF_RX_CB(skb
)->meta_slots_used
);
602 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue
->rx
, ret
);
604 need_to_notify
|= !!ret
;
606 npo
.meta_cons
+= XENVIF_RX_CB(skb
)->meta_slots_used
;
612 notify_remote_via_irq(queue
->rx_irq
);
615 void xenvif_napi_schedule_or_enable_events(struct xenvif_queue
*queue
)
619 RING_FINAL_CHECK_FOR_REQUESTS(&queue
->tx
, more_to_do
);
622 napi_schedule(&queue
->napi
);
625 static void tx_add_credit(struct xenvif_queue
*queue
)
627 unsigned long max_burst
, max_credit
;
630 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
631 * Otherwise the interface can seize up due to insufficient credit.
633 max_burst
= RING_GET_REQUEST(&queue
->tx
, queue
->tx
.req_cons
)->size
;
634 max_burst
= min(max_burst
, 131072UL);
635 max_burst
= max(max_burst
, queue
->credit_bytes
);
637 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
638 max_credit
= queue
->remaining_credit
+ queue
->credit_bytes
;
639 if (max_credit
< queue
->remaining_credit
)
640 max_credit
= ULONG_MAX
; /* wrapped: clamp to ULONG_MAX */
642 queue
->remaining_credit
= min(max_credit
, max_burst
);
645 static void tx_credit_callback(unsigned long data
)
647 struct xenvif_queue
*queue
= (struct xenvif_queue
*)data
;
648 tx_add_credit(queue
);
649 xenvif_napi_schedule_or_enable_events(queue
);
652 static void xenvif_tx_err(struct xenvif_queue
*queue
,
653 struct xen_netif_tx_request
*txp
, RING_IDX end
)
655 RING_IDX cons
= queue
->tx
.req_cons
;
659 spin_lock_irqsave(&queue
->response_lock
, flags
);
660 make_tx_response(queue
, txp
, XEN_NETIF_RSP_ERROR
);
661 push_tx_responses(queue
);
662 spin_unlock_irqrestore(&queue
->response_lock
, flags
);
665 txp
= RING_GET_REQUEST(&queue
->tx
, cons
++);
667 queue
->tx
.req_cons
= cons
;
670 static void xenvif_fatal_tx_err(struct xenvif
*vif
)
672 netdev_err(vif
->dev
, "fatal error; disabling device\n");
673 vif
->disabled
= true;
674 /* Disable the vif from queue 0's kthread */
676 xenvif_kick_thread(&vif
->queues
[0]);
679 static int xenvif_count_requests(struct xenvif_queue
*queue
,
680 struct xen_netif_tx_request
*first
,
681 struct xen_netif_tx_request
*txp
,
684 RING_IDX cons
= queue
->tx
.req_cons
;
689 if (!(first
->flags
& XEN_NETTXF_more_data
))
693 struct xen_netif_tx_request dropped_tx
= { 0 };
695 if (slots
>= work_to_do
) {
696 netdev_err(queue
->vif
->dev
,
697 "Asked for %d slots but exceeds this limit\n",
699 xenvif_fatal_tx_err(queue
->vif
);
703 /* This guest is really using too many slots and
704 * considered malicious.
706 if (unlikely(slots
>= fatal_skb_slots
)) {
707 netdev_err(queue
->vif
->dev
,
708 "Malicious frontend using %d slots, threshold %u\n",
709 slots
, fatal_skb_slots
);
710 xenvif_fatal_tx_err(queue
->vif
);
714 /* Xen network protocol had implicit dependency on
715 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to
716 * the historical MAX_SKB_FRAGS value 18 to honor the
717 * same behavior as before. Any packet using more than
718 * 18 slots but less than fatal_skb_slots slots is
721 if (!drop_err
&& slots
>= XEN_NETBK_LEGACY_SLOTS_MAX
) {
723 netdev_dbg(queue
->vif
->dev
,
724 "Too many slots (%d) exceeding limit (%d), dropping packet\n",
725 slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
732 memcpy(txp
, RING_GET_REQUEST(&queue
->tx
, cons
+ slots
),
735 /* If the guest submitted a frame >= 64 KiB then
736 * first->size overflowed and following slots will
737 * appear to be larger than the frame.
739 * This cannot be fatal error as there are buggy
740 * frontends that do this.
742 * Consume all slots and drop the packet.
744 if (!drop_err
&& txp
->size
> first
->size
) {
746 netdev_dbg(queue
->vif
->dev
,
747 "Invalid tx request, slot size %u > remaining size %u\n",
748 txp
->size
, first
->size
);
752 first
->size
-= txp
->size
;
755 if (unlikely((txp
->offset
+ txp
->size
) > PAGE_SIZE
)) {
756 netdev_err(queue
->vif
->dev
, "Cross page boundary, txp->offset: %x, size: %u\n",
757 txp
->offset
, txp
->size
);
758 xenvif_fatal_tx_err(queue
->vif
);
762 more_data
= txp
->flags
& XEN_NETTXF_more_data
;
770 xenvif_tx_err(queue
, first
, cons
+ slots
);
778 struct xenvif_tx_cb
{
782 #define XENVIF_TX_CB(skb) ((struct xenvif_tx_cb *)(skb)->cb)
784 static inline void xenvif_tx_create_map_op(struct xenvif_queue
*queue
,
786 struct xen_netif_tx_request
*txp
,
787 struct gnttab_map_grant_ref
*mop
)
789 queue
->pages_to_map
[mop
-queue
->tx_map_ops
] = queue
->mmap_pages
[pending_idx
];
790 gnttab_set_map_op(mop
, idx_to_kaddr(queue
, pending_idx
),
791 GNTMAP_host_map
| GNTMAP_readonly
,
792 txp
->gref
, queue
->vif
->domid
);
794 memcpy(&queue
->pending_tx_info
[pending_idx
].req
, txp
,
798 static inline struct sk_buff
*xenvif_alloc_skb(unsigned int size
)
800 struct sk_buff
*skb
=
801 alloc_skb(size
+ NET_SKB_PAD
+ NET_IP_ALIGN
,
802 GFP_ATOMIC
| __GFP_NOWARN
);
803 if (unlikely(skb
== NULL
))
806 /* Packets passed to netif_rx() must have some headroom. */
807 skb_reserve(skb
, NET_SKB_PAD
+ NET_IP_ALIGN
);
809 /* Initialize it here to avoid later surprises */
810 skb_shinfo(skb
)->destructor_arg
= NULL
;
815 static struct gnttab_map_grant_ref
*xenvif_get_requests(struct xenvif_queue
*queue
,
817 struct xen_netif_tx_request
*txp
,
818 struct gnttab_map_grant_ref
*gop
)
820 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
821 skb_frag_t
*frags
= shinfo
->frags
;
822 u16 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
824 pending_ring_idx_t index
;
825 unsigned int nr_slots
, frag_overflow
= 0;
827 /* At this point shinfo->nr_frags is in fact the number of
828 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
830 if (shinfo
->nr_frags
> MAX_SKB_FRAGS
) {
831 frag_overflow
= shinfo
->nr_frags
- MAX_SKB_FRAGS
;
832 BUG_ON(frag_overflow
> MAX_SKB_FRAGS
);
833 shinfo
->nr_frags
= MAX_SKB_FRAGS
;
835 nr_slots
= shinfo
->nr_frags
;
837 /* Skip first skb fragment if it is on same page as header fragment. */
838 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
840 for (shinfo
->nr_frags
= start
; shinfo
->nr_frags
< nr_slots
;
841 shinfo
->nr_frags
++, txp
++, gop
++) {
842 index
= pending_index(queue
->pending_cons
++);
843 pending_idx
= queue
->pending_ring
[index
];
844 xenvif_tx_create_map_op(queue
, pending_idx
, txp
, gop
);
845 frag_set_pending_idx(&frags
[shinfo
->nr_frags
], pending_idx
);
849 struct sk_buff
*nskb
= xenvif_alloc_skb(0);
850 if (unlikely(nskb
== NULL
)) {
852 netdev_err(queue
->vif
->dev
,
853 "Can't allocate the frag_list skb.\n");
857 shinfo
= skb_shinfo(nskb
);
858 frags
= shinfo
->frags
;
860 for (shinfo
->nr_frags
= 0; shinfo
->nr_frags
< frag_overflow
;
861 shinfo
->nr_frags
++, txp
++, gop
++) {
862 index
= pending_index(queue
->pending_cons
++);
863 pending_idx
= queue
->pending_ring
[index
];
864 xenvif_tx_create_map_op(queue
, pending_idx
, txp
, gop
);
865 frag_set_pending_idx(&frags
[shinfo
->nr_frags
],
869 skb_shinfo(skb
)->frag_list
= nskb
;
875 static inline void xenvif_grant_handle_set(struct xenvif_queue
*queue
,
877 grant_handle_t handle
)
879 if (unlikely(queue
->grant_tx_handle
[pending_idx
] !=
880 NETBACK_INVALID_HANDLE
)) {
881 netdev_err(queue
->vif
->dev
,
882 "Trying to overwrite active handle! pending_idx: %x\n",
886 queue
->grant_tx_handle
[pending_idx
] = handle
;
889 static inline void xenvif_grant_handle_reset(struct xenvif_queue
*queue
,
892 if (unlikely(queue
->grant_tx_handle
[pending_idx
] ==
893 NETBACK_INVALID_HANDLE
)) {
894 netdev_err(queue
->vif
->dev
,
895 "Trying to unmap invalid handle! pending_idx: %x\n",
899 queue
->grant_tx_handle
[pending_idx
] = NETBACK_INVALID_HANDLE
;
902 static int xenvif_tx_check_gop(struct xenvif_queue
*queue
,
904 struct gnttab_map_grant_ref
**gopp_map
,
905 struct gnttab_copy
**gopp_copy
)
907 struct gnttab_map_grant_ref
*gop_map
= *gopp_map
;
908 u16 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
909 /* This always points to the shinfo of the skb being checked, which
910 * could be either the first or the one on the frag_list
912 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
913 /* If this is non-NULL, we are currently checking the frag_list skb, and
914 * this points to the shinfo of the first one
916 struct skb_shared_info
*first_shinfo
= NULL
;
917 int nr_frags
= shinfo
->nr_frags
;
918 const bool sharedslot
= nr_frags
&&
919 frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
;
922 /* Check status of header. */
923 err
= (*gopp_copy
)->status
;
926 netdev_dbg(queue
->vif
->dev
,
927 "Grant copy of header failed! status: %d pending_idx: %u ref: %u\n",
928 (*gopp_copy
)->status
,
930 (*gopp_copy
)->source
.u
.ref
);
931 /* The first frag might still have this slot mapped */
933 xenvif_idx_release(queue
, pending_idx
,
934 XEN_NETIF_RSP_ERROR
);
939 for (i
= 0; i
< nr_frags
; i
++, gop_map
++) {
942 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[i
]);
944 /* Check error status: if okay then remember grant handle. */
945 newerr
= gop_map
->status
;
947 if (likely(!newerr
)) {
948 xenvif_grant_handle_set(queue
,
951 /* Had a previous error? Invalidate this fragment. */
953 xenvif_idx_unmap(queue
, pending_idx
);
954 /* If the mapping of the first frag was OK, but
955 * the header's copy failed, and they are
956 * sharing a slot, send an error
958 if (i
== 0 && sharedslot
)
959 xenvif_idx_release(queue
, pending_idx
,
960 XEN_NETIF_RSP_ERROR
);
962 xenvif_idx_release(queue
, pending_idx
,
968 /* Error on this fragment: respond to client with an error. */
970 netdev_dbg(queue
->vif
->dev
,
971 "Grant map of %d. frag failed! status: %d pending_idx: %u ref: %u\n",
977 xenvif_idx_release(queue
, pending_idx
, XEN_NETIF_RSP_ERROR
);
979 /* Not the first error? Preceding frags already invalidated. */
983 /* First error: if the header haven't shared a slot with the
984 * first frag, release it as well.
987 xenvif_idx_release(queue
,
988 XENVIF_TX_CB(skb
)->pending_idx
,
991 /* Invalidate preceding fragments of this skb. */
992 for (j
= 0; j
< i
; j
++) {
993 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[j
]);
994 xenvif_idx_unmap(queue
, pending_idx
);
995 xenvif_idx_release(queue
, pending_idx
,
999 /* And if we found the error while checking the frag_list, unmap
1000 * the first skb's frags
1003 for (j
= 0; j
< first_shinfo
->nr_frags
; j
++) {
1004 pending_idx
= frag_get_pending_idx(&first_shinfo
->frags
[j
]);
1005 xenvif_idx_unmap(queue
, pending_idx
);
1006 xenvif_idx_release(queue
, pending_idx
,
1007 XEN_NETIF_RSP_OKAY
);
1011 /* Remember the error: invalidate all subsequent fragments. */
1015 if (skb_has_frag_list(skb
) && !first_shinfo
) {
1016 first_shinfo
= skb_shinfo(skb
);
1017 shinfo
= skb_shinfo(skb_shinfo(skb
)->frag_list
);
1018 nr_frags
= shinfo
->nr_frags
;
1023 *gopp_map
= gop_map
;
1027 static void xenvif_fill_frags(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1029 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1030 int nr_frags
= shinfo
->nr_frags
;
1032 u16 prev_pending_idx
= INVALID_PENDING_IDX
;
1034 for (i
= 0; i
< nr_frags
; i
++) {
1035 skb_frag_t
*frag
= shinfo
->frags
+ i
;
1036 struct xen_netif_tx_request
*txp
;
1040 pending_idx
= frag_get_pending_idx(frag
);
1042 /* If this is not the first frag, chain it to the previous*/
1043 if (prev_pending_idx
== INVALID_PENDING_IDX
)
1044 skb_shinfo(skb
)->destructor_arg
=
1045 &callback_param(queue
, pending_idx
);
1047 callback_param(queue
, prev_pending_idx
).ctx
=
1048 &callback_param(queue
, pending_idx
);
1050 callback_param(queue
, pending_idx
).ctx
= NULL
;
1051 prev_pending_idx
= pending_idx
;
1053 txp
= &queue
->pending_tx_info
[pending_idx
].req
;
1054 page
= virt_to_page(idx_to_kaddr(queue
, pending_idx
));
1055 __skb_fill_page_desc(skb
, i
, page
, txp
->offset
, txp
->size
);
1056 skb
->len
+= txp
->size
;
1057 skb
->data_len
+= txp
->size
;
1058 skb
->truesize
+= txp
->size
;
1060 /* Take an extra reference to offset network stack's put_page */
1061 get_page(queue
->mmap_pages
[pending_idx
]);
1065 static int xenvif_get_extras(struct xenvif_queue
*queue
,
1066 struct xen_netif_extra_info
*extras
,
1069 struct xen_netif_extra_info extra
;
1070 RING_IDX cons
= queue
->tx
.req_cons
;
1073 if (unlikely(work_to_do
-- <= 0)) {
1074 netdev_err(queue
->vif
->dev
, "Missing extra info\n");
1075 xenvif_fatal_tx_err(queue
->vif
);
1079 memcpy(&extra
, RING_GET_REQUEST(&queue
->tx
, cons
),
1081 if (unlikely(!extra
.type
||
1082 extra
.type
>= XEN_NETIF_EXTRA_TYPE_MAX
)) {
1083 queue
->tx
.req_cons
= ++cons
;
1084 netdev_err(queue
->vif
->dev
,
1085 "Invalid extra type: %d\n", extra
.type
);
1086 xenvif_fatal_tx_err(queue
->vif
);
1090 memcpy(&extras
[extra
.type
- 1], &extra
, sizeof(extra
));
1091 queue
->tx
.req_cons
= ++cons
;
1092 } while (extra
.flags
& XEN_NETIF_EXTRA_FLAG_MORE
);
1097 static int xenvif_set_skb_gso(struct xenvif
*vif
,
1098 struct sk_buff
*skb
,
1099 struct xen_netif_extra_info
*gso
)
1101 if (!gso
->u
.gso
.size
) {
1102 netdev_err(vif
->dev
, "GSO size must not be zero.\n");
1103 xenvif_fatal_tx_err(vif
);
1107 switch (gso
->u
.gso
.type
) {
1108 case XEN_NETIF_GSO_TYPE_TCPV4
:
1109 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
1111 case XEN_NETIF_GSO_TYPE_TCPV6
:
1112 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV6
;
1115 netdev_err(vif
->dev
, "Bad GSO type %d.\n", gso
->u
.gso
.type
);
1116 xenvif_fatal_tx_err(vif
);
1120 skb_shinfo(skb
)->gso_size
= gso
->u
.gso
.size
;
1121 /* gso_segs will be calculated later */
1126 static int checksum_setup(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1128 bool recalculate_partial_csum
= false;
1130 /* A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1131 * peers can fail to set NETRXF_csum_blank when sending a GSO
1132 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1133 * recalculate the partial checksum.
1135 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
&& skb_is_gso(skb
)) {
1136 queue
->stats
.rx_gso_checksum_fixup
++;
1137 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1138 recalculate_partial_csum
= true;
1141 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1142 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1145 return skb_checksum_setup(skb
, recalculate_partial_csum
);
1148 static bool tx_credit_exceeded(struct xenvif_queue
*queue
, unsigned size
)
1150 u64 now
= get_jiffies_64();
1151 u64 next_credit
= queue
->credit_window_start
+
1152 msecs_to_jiffies(queue
->credit_usec
/ 1000);
1154 /* Timer could already be pending in rare cases. */
1155 if (timer_pending(&queue
->credit_timeout
))
1158 /* Passed the point where we can replenish credit? */
1159 if (time_after_eq64(now
, next_credit
)) {
1160 queue
->credit_window_start
= now
;
1161 tx_add_credit(queue
);
1164 /* Still too big to send right now? Set a callback. */
1165 if (size
> queue
->remaining_credit
) {
1166 queue
->credit_timeout
.data
=
1167 (unsigned long)queue
;
1168 queue
->credit_timeout
.function
=
1170 mod_timer(&queue
->credit_timeout
,
1172 queue
->credit_window_start
= next_credit
;
1180 static void xenvif_tx_build_gops(struct xenvif_queue
*queue
,
1185 struct gnttab_map_grant_ref
*gop
= queue
->tx_map_ops
, *request_gop
;
1186 struct sk_buff
*skb
;
1189 while (skb_queue_len(&queue
->tx_queue
) < budget
) {
1190 struct xen_netif_tx_request txreq
;
1191 struct xen_netif_tx_request txfrags
[XEN_NETBK_LEGACY_SLOTS_MAX
];
1192 struct xen_netif_extra_info extras
[XEN_NETIF_EXTRA_TYPE_MAX
-1];
1196 unsigned int data_len
;
1197 pending_ring_idx_t index
;
1199 if (queue
->tx
.sring
->req_prod
- queue
->tx
.req_cons
>
1200 XEN_NETIF_TX_RING_SIZE
) {
1201 netdev_err(queue
->vif
->dev
,
1202 "Impossible number of requests. "
1203 "req_prod %d, req_cons %d, size %ld\n",
1204 queue
->tx
.sring
->req_prod
, queue
->tx
.req_cons
,
1205 XEN_NETIF_TX_RING_SIZE
);
1206 xenvif_fatal_tx_err(queue
->vif
);
1210 work_to_do
= RING_HAS_UNCONSUMED_REQUESTS(&queue
->tx
);
1214 idx
= queue
->tx
.req_cons
;
1215 rmb(); /* Ensure that we see the request before we copy it. */
1216 memcpy(&txreq
, RING_GET_REQUEST(&queue
->tx
, idx
), sizeof(txreq
));
1218 /* Credit-based scheduling. */
1219 if (txreq
.size
> queue
->remaining_credit
&&
1220 tx_credit_exceeded(queue
, txreq
.size
))
1223 queue
->remaining_credit
-= txreq
.size
;
1226 queue
->tx
.req_cons
= ++idx
;
1228 memset(extras
, 0, sizeof(extras
));
1229 if (txreq
.flags
& XEN_NETTXF_extra_info
) {
1230 work_to_do
= xenvif_get_extras(queue
, extras
,
1232 idx
= queue
->tx
.req_cons
;
1233 if (unlikely(work_to_do
< 0))
1237 ret
= xenvif_count_requests(queue
, &txreq
, txfrags
, work_to_do
);
1238 if (unlikely(ret
< 0))
1243 if (unlikely(txreq
.size
< ETH_HLEN
)) {
1244 netdev_dbg(queue
->vif
->dev
,
1245 "Bad packet size: %d\n", txreq
.size
);
1246 xenvif_tx_err(queue
, &txreq
, idx
);
1250 /* No crossing a page as the payload mustn't fragment. */
1251 if (unlikely((txreq
.offset
+ txreq
.size
) > PAGE_SIZE
)) {
1252 netdev_err(queue
->vif
->dev
,
1253 "txreq.offset: %x, size: %u, end: %lu\n",
1254 txreq
.offset
, txreq
.size
,
1255 (txreq
.offset
&~PAGE_MASK
) + txreq
.size
);
1256 xenvif_fatal_tx_err(queue
->vif
);
1260 index
= pending_index(queue
->pending_cons
);
1261 pending_idx
= queue
->pending_ring
[index
];
1263 data_len
= (txreq
.size
> XEN_NETBACK_TX_COPY_LEN
&&
1264 ret
< XEN_NETBK_LEGACY_SLOTS_MAX
) ?
1265 XEN_NETBACK_TX_COPY_LEN
: txreq
.size
;
1267 skb
= xenvif_alloc_skb(data_len
);
1268 if (unlikely(skb
== NULL
)) {
1269 netdev_dbg(queue
->vif
->dev
,
1270 "Can't allocate a skb in start_xmit.\n");
1271 xenvif_tx_err(queue
, &txreq
, idx
);
1275 if (extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1].type
) {
1276 struct xen_netif_extra_info
*gso
;
1277 gso
= &extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1];
1279 if (xenvif_set_skb_gso(queue
->vif
, skb
, gso
)) {
1280 /* Failure in xenvif_set_skb_gso is fatal. */
1286 XENVIF_TX_CB(skb
)->pending_idx
= pending_idx
;
1288 __skb_put(skb
, data_len
);
1289 queue
->tx_copy_ops
[*copy_ops
].source
.u
.ref
= txreq
.gref
;
1290 queue
->tx_copy_ops
[*copy_ops
].source
.domid
= queue
->vif
->domid
;
1291 queue
->tx_copy_ops
[*copy_ops
].source
.offset
= txreq
.offset
;
1293 queue
->tx_copy_ops
[*copy_ops
].dest
.u
.gmfn
=
1294 virt_to_mfn(skb
->data
);
1295 queue
->tx_copy_ops
[*copy_ops
].dest
.domid
= DOMID_SELF
;
1296 queue
->tx_copy_ops
[*copy_ops
].dest
.offset
=
1297 offset_in_page(skb
->data
);
1299 queue
->tx_copy_ops
[*copy_ops
].len
= data_len
;
1300 queue
->tx_copy_ops
[*copy_ops
].flags
= GNTCOPY_source_gref
;
1304 skb_shinfo(skb
)->nr_frags
= ret
;
1305 if (data_len
< txreq
.size
) {
1306 skb_shinfo(skb
)->nr_frags
++;
1307 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1309 xenvif_tx_create_map_op(queue
, pending_idx
, &txreq
, gop
);
1312 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1313 INVALID_PENDING_IDX
);
1314 memcpy(&queue
->pending_tx_info
[pending_idx
].req
, &txreq
,
1318 queue
->pending_cons
++;
1320 request_gop
= xenvif_get_requests(queue
, skb
, txfrags
, gop
);
1321 if (request_gop
== NULL
) {
1323 xenvif_tx_err(queue
, &txreq
, idx
);
1328 __skb_queue_tail(&queue
->tx_queue
, skb
);
1330 queue
->tx
.req_cons
= idx
;
1332 if (((gop
-queue
->tx_map_ops
) >= ARRAY_SIZE(queue
->tx_map_ops
)) ||
1333 (*copy_ops
>= ARRAY_SIZE(queue
->tx_copy_ops
)))
1337 (*map_ops
) = gop
- queue
->tx_map_ops
;
1341 /* Consolidate skb with a frag_list into a brand new one with local pages on
1342 * frags. Returns 0 or -ENOMEM if can't allocate new pages.
1344 static int xenvif_handle_frag_list(struct xenvif_queue
*queue
, struct sk_buff
*skb
)
1346 unsigned int offset
= skb_headlen(skb
);
1347 skb_frag_t frags
[MAX_SKB_FRAGS
];
1349 struct ubuf_info
*uarg
;
1350 struct sk_buff
*nskb
= skb_shinfo(skb
)->frag_list
;
1352 queue
->stats
.tx_zerocopy_sent
+= 2;
1353 queue
->stats
.tx_frag_overflow
++;
1355 xenvif_fill_frags(queue
, nskb
);
1356 /* Subtract frags size, we will correct it later */
1357 skb
->truesize
-= skb
->data_len
;
1358 skb
->len
+= nskb
->len
;
1359 skb
->data_len
+= nskb
->len
;
1361 /* create a brand new frags array and coalesce there */
1362 for (i
= 0; offset
< skb
->len
; i
++) {
1366 BUG_ON(i
>= MAX_SKB_FRAGS
);
1367 page
= alloc_page(GFP_ATOMIC
);
1370 skb
->truesize
+= skb
->data_len
;
1371 for (j
= 0; j
< i
; j
++)
1372 put_page(frags
[j
].page
.p
);
1376 if (offset
+ PAGE_SIZE
< skb
->len
)
1379 len
= skb
->len
- offset
;
1380 if (skb_copy_bits(skb
, offset
, page_address(page
), len
))
1384 frags
[i
].page
.p
= page
;
1385 frags
[i
].page_offset
= 0;
1386 skb_frag_size_set(&frags
[i
], len
);
1389 /* Copied all the bits from the frag list -- free it. */
1390 skb_frag_list_init(skb
);
1391 xenvif_skb_zerocopy_prepare(queue
, nskb
);
1394 /* Release all the original (foreign) frags. */
1395 for (f
= 0; f
< skb_shinfo(skb
)->nr_frags
; f
++)
1396 skb_frag_unref(skb
, f
);
1397 uarg
= skb_shinfo(skb
)->destructor_arg
;
1398 /* increase inflight counter to offset decrement in callback */
1399 atomic_inc(&queue
->inflight_packets
);
1400 uarg
->callback(uarg
, true);
1401 skb_shinfo(skb
)->destructor_arg
= NULL
;
1403 /* Fill the skb with the new (local) frags. */
1404 memcpy(skb_shinfo(skb
)->frags
, frags
, i
* sizeof(skb_frag_t
));
1405 skb_shinfo(skb
)->nr_frags
= i
;
1406 skb
->truesize
+= i
* PAGE_SIZE
;
1411 static int xenvif_tx_submit(struct xenvif_queue
*queue
)
1413 struct gnttab_map_grant_ref
*gop_map
= queue
->tx_map_ops
;
1414 struct gnttab_copy
*gop_copy
= queue
->tx_copy_ops
;
1415 struct sk_buff
*skb
;
1418 while ((skb
= __skb_dequeue(&queue
->tx_queue
)) != NULL
) {
1419 struct xen_netif_tx_request
*txp
;
1423 pending_idx
= XENVIF_TX_CB(skb
)->pending_idx
;
1424 txp
= &queue
->pending_tx_info
[pending_idx
].req
;
1426 /* Check the remap error code. */
1427 if (unlikely(xenvif_tx_check_gop(queue
, skb
, &gop_map
, &gop_copy
))) {
1428 /* If there was an error, xenvif_tx_check_gop is
1429 * expected to release all the frags which were mapped,
1430 * so kfree_skb shouldn't do it again
1432 skb_shinfo(skb
)->nr_frags
= 0;
1433 if (skb_has_frag_list(skb
)) {
1434 struct sk_buff
*nskb
=
1435 skb_shinfo(skb
)->frag_list
;
1436 skb_shinfo(nskb
)->nr_frags
= 0;
1442 data_len
= skb
->len
;
1443 callback_param(queue
, pending_idx
).ctx
= NULL
;
1444 if (data_len
< txp
->size
) {
1445 /* Append the packet payload as a fragment. */
1446 txp
->offset
+= data_len
;
1447 txp
->size
-= data_len
;
1449 /* Schedule a response immediately. */
1450 xenvif_idx_release(queue
, pending_idx
,
1451 XEN_NETIF_RSP_OKAY
);
1454 if (txp
->flags
& XEN_NETTXF_csum_blank
)
1455 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1456 else if (txp
->flags
& XEN_NETTXF_data_validated
)
1457 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1459 xenvif_fill_frags(queue
, skb
);
1461 if (unlikely(skb_has_frag_list(skb
))) {
1462 if (xenvif_handle_frag_list(queue
, skb
)) {
1463 if (net_ratelimit())
1464 netdev_err(queue
->vif
->dev
,
1465 "Not enough memory to consolidate frag_list!\n");
1466 xenvif_skb_zerocopy_prepare(queue
, skb
);
1472 skb
->dev
= queue
->vif
->dev
;
1473 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1474 skb_reset_network_header(skb
);
1476 if (checksum_setup(queue
, skb
)) {
1477 netdev_dbg(queue
->vif
->dev
,
1478 "Can't setup checksum in net_tx_action\n");
1479 /* We have to set this flag to trigger the callback */
1480 if (skb_shinfo(skb
)->destructor_arg
)
1481 xenvif_skb_zerocopy_prepare(queue
, skb
);
1486 skb_probe_transport_header(skb
, 0);
1488 /* If the packet is GSO then we will have just set up the
1489 * transport header offset in checksum_setup so it's now
1490 * straightforward to calculate gso_segs.
1492 if (skb_is_gso(skb
)) {
1493 int mss
= skb_shinfo(skb
)->gso_size
;
1494 int hdrlen
= skb_transport_header(skb
) -
1495 skb_mac_header(skb
) +
1498 skb_shinfo(skb
)->gso_segs
=
1499 DIV_ROUND_UP(skb
->len
- hdrlen
, mss
);
1502 queue
->stats
.rx_bytes
+= skb
->len
;
1503 queue
->stats
.rx_packets
++;
1507 /* Set this flag right before netif_receive_skb, otherwise
1508 * someone might think this packet already left netback, and
1509 * do a skb_copy_ubufs while we are still in control of the
1510 * skb. E.g. the __pskb_pull_tail earlier can do such thing.
1512 if (skb_shinfo(skb
)->destructor_arg
) {
1513 xenvif_skb_zerocopy_prepare(queue
, skb
);
1514 queue
->stats
.tx_zerocopy_sent
++;
1517 netif_receive_skb(skb
);
1523 void xenvif_zerocopy_callback(struct ubuf_info
*ubuf
, bool zerocopy_success
)
1525 unsigned long flags
;
1526 pending_ring_idx_t index
;
1527 struct xenvif_queue
*queue
= ubuf_to_queue(ubuf
);
1529 /* This is the only place where we grab this lock, to protect callbacks
1532 spin_lock_irqsave(&queue
->callback_lock
, flags
);
1534 u16 pending_idx
= ubuf
->desc
;
1535 ubuf
= (struct ubuf_info
*) ubuf
->ctx
;
1536 BUG_ON(queue
->dealloc_prod
- queue
->dealloc_cons
>=
1538 index
= pending_index(queue
->dealloc_prod
);
1539 queue
->dealloc_ring
[index
] = pending_idx
;
1540 /* Sync with xenvif_tx_dealloc_action:
1541 * insert idx then incr producer.
1544 queue
->dealloc_prod
++;
1546 wake_up(&queue
->dealloc_wq
);
1547 spin_unlock_irqrestore(&queue
->callback_lock
, flags
);
1549 if (likely(zerocopy_success
))
1550 queue
->stats
.tx_zerocopy_success
++;
1552 queue
->stats
.tx_zerocopy_fail
++;
1553 xenvif_skb_zerocopy_complete(queue
);
1556 static inline void xenvif_tx_dealloc_action(struct xenvif_queue
*queue
)
1558 struct gnttab_unmap_grant_ref
*gop
;
1559 pending_ring_idx_t dc
, dp
;
1560 u16 pending_idx
, pending_idx_release
[MAX_PENDING_REQS
];
1563 dc
= queue
->dealloc_cons
;
1564 gop
= queue
->tx_unmap_ops
;
1566 /* Free up any grants we have finished using */
1568 dp
= queue
->dealloc_prod
;
1570 /* Ensure we see all indices enqueued by all
1571 * xenvif_zerocopy_callback().
1576 BUG_ON(gop
- queue
->tx_unmap_ops
> MAX_PENDING_REQS
);
1578 queue
->dealloc_ring
[pending_index(dc
++)];
1580 pending_idx_release
[gop
-queue
->tx_unmap_ops
] =
1582 queue
->pages_to_unmap
[gop
-queue
->tx_unmap_ops
] =
1583 queue
->mmap_pages
[pending_idx
];
1584 gnttab_set_unmap_op(gop
,
1585 idx_to_kaddr(queue
, pending_idx
),
1587 queue
->grant_tx_handle
[pending_idx
]);
1588 xenvif_grant_handle_reset(queue
, pending_idx
);
1592 } while (dp
!= queue
->dealloc_prod
);
1594 queue
->dealloc_cons
= dc
;
1596 if (gop
- queue
->tx_unmap_ops
> 0) {
1598 ret
= gnttab_unmap_refs(queue
->tx_unmap_ops
,
1600 queue
->pages_to_unmap
,
1601 gop
- queue
->tx_unmap_ops
);
1603 netdev_err(queue
->vif
->dev
, "Unmap fail: nr_ops %tx ret %d\n",
1604 gop
- queue
->tx_unmap_ops
, ret
);
1605 for (i
= 0; i
< gop
- queue
->tx_unmap_ops
; ++i
) {
1606 if (gop
[i
].status
!= GNTST_okay
)
1607 netdev_err(queue
->vif
->dev
,
1608 " host_addr: %llx handle: %x status: %d\n",
1617 for (i
= 0; i
< gop
- queue
->tx_unmap_ops
; ++i
)
1618 xenvif_idx_release(queue
, pending_idx_release
[i
],
1619 XEN_NETIF_RSP_OKAY
);
1623 /* Called after netfront has transmitted */
1624 int xenvif_tx_action(struct xenvif_queue
*queue
, int budget
)
1626 unsigned nr_mops
, nr_cops
= 0;
1629 if (unlikely(!tx_work_todo(queue
)))
1632 xenvif_tx_build_gops(queue
, budget
, &nr_cops
, &nr_mops
);
1637 gnttab_batch_copy(queue
->tx_copy_ops
, nr_cops
);
1639 ret
= gnttab_map_refs(queue
->tx_map_ops
,
1641 queue
->pages_to_map
,
1646 work_done
= xenvif_tx_submit(queue
);
1651 static void xenvif_idx_release(struct xenvif_queue
*queue
, u16 pending_idx
,
1654 struct pending_tx_info
*pending_tx_info
;
1655 pending_ring_idx_t index
;
1656 unsigned long flags
;
1658 pending_tx_info
= &queue
->pending_tx_info
[pending_idx
];
1660 spin_lock_irqsave(&queue
->response_lock
, flags
);
1662 make_tx_response(queue
, &pending_tx_info
->req
, status
);
1664 /* Release the pending index before pusing the Tx response so
1665 * its available before a new Tx request is pushed by the
1668 index
= pending_index(queue
->pending_prod
++);
1669 queue
->pending_ring
[index
] = pending_idx
;
1671 push_tx_responses(queue
);
1673 spin_unlock_irqrestore(&queue
->response_lock
, flags
);
1677 static void make_tx_response(struct xenvif_queue
*queue
,
1678 struct xen_netif_tx_request
*txp
,
1681 RING_IDX i
= queue
->tx
.rsp_prod_pvt
;
1682 struct xen_netif_tx_response
*resp
;
1684 resp
= RING_GET_RESPONSE(&queue
->tx
, i
);
1688 if (txp
->flags
& XEN_NETTXF_extra_info
)
1689 RING_GET_RESPONSE(&queue
->tx
, ++i
)->status
= XEN_NETIF_RSP_NULL
;
1691 queue
->tx
.rsp_prod_pvt
= ++i
;
1694 static void push_tx_responses(struct xenvif_queue
*queue
)
1698 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue
->tx
, notify
);
1700 notify_remote_via_irq(queue
->tx_irq
);
1703 static struct xen_netif_rx_response
*make_rx_response(struct xenvif_queue
*queue
,
1710 RING_IDX i
= queue
->rx
.rsp_prod_pvt
;
1711 struct xen_netif_rx_response
*resp
;
1713 resp
= RING_GET_RESPONSE(&queue
->rx
, i
);
1714 resp
->offset
= offset
;
1715 resp
->flags
= flags
;
1717 resp
->status
= (s16
)size
;
1719 resp
->status
= (s16
)st
;
1721 queue
->rx
.rsp_prod_pvt
= ++i
;
1726 void xenvif_idx_unmap(struct xenvif_queue
*queue
, u16 pending_idx
)
1729 struct gnttab_unmap_grant_ref tx_unmap_op
;
1731 gnttab_set_unmap_op(&tx_unmap_op
,
1732 idx_to_kaddr(queue
, pending_idx
),
1734 queue
->grant_tx_handle
[pending_idx
]);
1735 xenvif_grant_handle_reset(queue
, pending_idx
);
1737 ret
= gnttab_unmap_refs(&tx_unmap_op
, NULL
,
1738 &queue
->mmap_pages
[pending_idx
], 1);
1740 netdev_err(queue
->vif
->dev
,
1741 "Unmap fail: ret: %d pending_idx: %d host_addr: %llx handle: %x status: %d\n",
1744 tx_unmap_op
.host_addr
,
1746 tx_unmap_op
.status
);
1751 static inline int tx_work_todo(struct xenvif_queue
*queue
)
1753 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&queue
->tx
)))
1759 static inline bool tx_dealloc_work_todo(struct xenvif_queue
*queue
)
1761 return queue
->dealloc_cons
!= queue
->dealloc_prod
;
1764 void xenvif_unmap_frontend_rings(struct xenvif_queue
*queue
)
1766 if (queue
->tx
.sring
)
1767 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue
->vif
),
1769 if (queue
->rx
.sring
)
1770 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(queue
->vif
),
1774 int xenvif_map_frontend_rings(struct xenvif_queue
*queue
,
1775 grant_ref_t tx_ring_ref
,
1776 grant_ref_t rx_ring_ref
)
1779 struct xen_netif_tx_sring
*txs
;
1780 struct xen_netif_rx_sring
*rxs
;
1784 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue
->vif
),
1785 tx_ring_ref
, &addr
);
1789 txs
= (struct xen_netif_tx_sring
*)addr
;
1790 BACK_RING_INIT(&queue
->tx
, txs
, PAGE_SIZE
);
1792 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(queue
->vif
),
1793 rx_ring_ref
, &addr
);
1797 rxs
= (struct xen_netif_rx_sring
*)addr
;
1798 BACK_RING_INIT(&queue
->rx
, rxs
, PAGE_SIZE
);
1803 xenvif_unmap_frontend_rings(queue
);
1807 static void xenvif_queue_carrier_off(struct xenvif_queue
*queue
)
1809 struct xenvif
*vif
= queue
->vif
;
1811 queue
->stalled
= true;
1813 /* At least one queue has stalled? Disable the carrier. */
1814 spin_lock(&vif
->lock
);
1815 if (vif
->stalled_queues
++ == 0) {
1816 netdev_info(vif
->dev
, "Guest Rx stalled");
1817 netif_carrier_off(vif
->dev
);
1819 spin_unlock(&vif
->lock
);
1822 static void xenvif_queue_carrier_on(struct xenvif_queue
*queue
)
1824 struct xenvif
*vif
= queue
->vif
;
1826 queue
->last_rx_time
= jiffies
; /* Reset Rx stall detection. */
1827 queue
->stalled
= false;
1829 /* All queues are ready? Enable the carrier. */
1830 spin_lock(&vif
->lock
);
1831 if (--vif
->stalled_queues
== 0) {
1832 netdev_info(vif
->dev
, "Guest Rx ready");
1833 netif_carrier_on(vif
->dev
);
1835 spin_unlock(&vif
->lock
);
1838 static bool xenvif_rx_queue_stalled(struct xenvif_queue
*queue
)
1840 RING_IDX prod
, cons
;
1842 prod
= queue
->rx
.sring
->req_prod
;
1843 cons
= queue
->rx
.req_cons
;
1845 return !queue
->stalled
1846 && prod
- cons
< XEN_NETBK_RX_SLOTS_MAX
1847 && time_after(jiffies
,
1848 queue
->last_rx_time
+ queue
->vif
->stall_timeout
);
1851 static bool xenvif_rx_queue_ready(struct xenvif_queue
*queue
)
1853 RING_IDX prod
, cons
;
1855 prod
= queue
->rx
.sring
->req_prod
;
1856 cons
= queue
->rx
.req_cons
;
1858 return queue
->stalled
1859 && prod
- cons
>= XEN_NETBK_RX_SLOTS_MAX
;
1862 static bool xenvif_have_rx_work(struct xenvif_queue
*queue
)
1864 return (!skb_queue_empty(&queue
->rx_queue
)
1865 && xenvif_rx_ring_slots_available(queue
, XEN_NETBK_RX_SLOTS_MAX
))
1866 || (queue
->vif
->stall_timeout
&&
1867 (xenvif_rx_queue_stalled(queue
)
1868 || xenvif_rx_queue_ready(queue
)))
1869 || kthread_should_stop()
1870 || queue
->vif
->disabled
;
1873 static long xenvif_rx_queue_timeout(struct xenvif_queue
*queue
)
1875 struct sk_buff
*skb
;
1878 skb
= skb_peek(&queue
->rx_queue
);
1880 return MAX_SCHEDULE_TIMEOUT
;
1882 timeout
= XENVIF_RX_CB(skb
)->expires
- jiffies
;
1883 return timeout
< 0 ? 0 : timeout
;
1886 /* Wait until the guest Rx thread has work.
1888 * The timeout needs to be adjusted based on the current head of the
1889 * queue (and not just the head at the beginning). In particular, if
1890 * the queue is initially empty an infinite timeout is used and this
1891 * needs to be reduced when a skb is queued.
1893 * This cannot be done with wait_event_timeout() because it only
1894 * calculates the timeout once.
1896 static void xenvif_wait_for_rx_work(struct xenvif_queue
*queue
)
1900 if (xenvif_have_rx_work(queue
))
1906 prepare_to_wait(&queue
->wq
, &wait
, TASK_INTERRUPTIBLE
);
1907 if (xenvif_have_rx_work(queue
))
1909 ret
= schedule_timeout(xenvif_rx_queue_timeout(queue
));
1913 finish_wait(&queue
->wq
, &wait
);
1916 int xenvif_kthread_guest_rx(void *data
)
1918 struct xenvif_queue
*queue
= data
;
1919 struct xenvif
*vif
= queue
->vif
;
1921 if (!vif
->stall_timeout
)
1922 xenvif_queue_carrier_on(queue
);
1925 xenvif_wait_for_rx_work(queue
);
1927 if (kthread_should_stop())
1930 /* This frontend is found to be rogue, disable it in
1931 * kthread context. Currently this is only set when
1932 * netback finds out frontend sends malformed packet,
1933 * but we cannot disable the interface in softirq
1934 * context so we defer it here, if this thread is
1935 * associated with queue 0.
1937 if (unlikely(vif
->disabled
&& queue
->id
== 0)) {
1938 xenvif_carrier_off(vif
);
1942 if (!skb_queue_empty(&queue
->rx_queue
))
1943 xenvif_rx_action(queue
);
1945 /* If the guest hasn't provided any Rx slots for a
1946 * while it's probably not responsive, drop the
1947 * carrier so packets are dropped earlier.
1949 if (vif
->stall_timeout
) {
1950 if (xenvif_rx_queue_stalled(queue
))
1951 xenvif_queue_carrier_off(queue
);
1952 else if (xenvif_rx_queue_ready(queue
))
1953 xenvif_queue_carrier_on(queue
);
1956 /* Queued packets may have foreign pages from other
1957 * domains. These cannot be queued indefinitely as
1958 * this would starve guests of grant refs and transmit
1961 xenvif_rx_queue_drop_expired(queue
);
1963 xenvif_rx_queue_maybe_wake(queue
);
1968 /* Bin any remaining skbs */
1969 xenvif_rx_queue_purge(queue
);
1974 static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue
*queue
)
1976 /* Dealloc thread must remain running until all inflight
1979 return kthread_should_stop() &&
1980 !atomic_read(&queue
->inflight_packets
);
1983 int xenvif_dealloc_kthread(void *data
)
1985 struct xenvif_queue
*queue
= data
;
1988 wait_event_interruptible(queue
->dealloc_wq
,
1989 tx_dealloc_work_todo(queue
) ||
1990 xenvif_dealloc_kthread_should_stop(queue
));
1991 if (xenvif_dealloc_kthread_should_stop(queue
))
1994 xenvif_tx_dealloc_action(queue
);
1998 /* Unmap anything remaining*/
1999 if (tx_dealloc_work_todo(queue
))
2000 xenvif_tx_dealloc_action(queue
);
2005 static int __init
netback_init(void)
2012 /* Allow as many queues as there are CPUs, by default */
2013 xenvif_max_queues
= num_online_cpus();
2015 if (fatal_skb_slots
< XEN_NETBK_LEGACY_SLOTS_MAX
) {
2016 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n",
2017 fatal_skb_slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
2018 fatal_skb_slots
= XEN_NETBK_LEGACY_SLOTS_MAX
;
2021 rc
= xenvif_xenbus_init();
2025 #ifdef CONFIG_DEBUG_FS
2026 xen_netback_dbg_root
= debugfs_create_dir("xen-netback", NULL
);
2027 if (IS_ERR_OR_NULL(xen_netback_dbg_root
))
2028 pr_warn("Init of debugfs returned %ld!\n",
2029 PTR_ERR(xen_netback_dbg_root
));
2030 #endif /* CONFIG_DEBUG_FS */
2038 module_init(netback_init
);
2040 static void __exit
netback_fini(void)
2042 #ifdef CONFIG_DEBUG_FS
2043 if (!IS_ERR_OR_NULL(xen_netback_dbg_root
))
2044 debugfs_remove_recursive(xen_netback_dbg_root
);
2045 #endif /* CONFIG_DEBUG_FS */
2046 xenvif_xenbus_fini();
2048 module_exit(netback_fini
);
2050 MODULE_LICENSE("Dual BSD/GPL");
2051 MODULE_ALIAS("xen-backend:vif");