2 * Back-end of the driver for virtual network devices. This portion of the
3 * driver exports a 'unified' network-device interface that can be accessed
4 * by any operating system that implements a compatible front end. A
5 * reference front-end implementation can be found in:
6 * drivers/net/xen-netfront.c
8 * Copyright (c) 2002-2005, K A Fraser
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
37 #include <linux/kthread.h>
38 #include <linux/if_vlan.h>
39 #include <linux/udp.h>
44 #include <xen/events.h>
45 #include <xen/interface/memory.h>
47 #include <asm/xen/hypercall.h>
48 #include <asm/xen/page.h>
51 * This is the maximum slots a skb can have. If a guest sends a skb
52 * which exceeds this limit it is considered malicious.
54 #define FATAL_SKB_SLOTS_DEFAULT 20
55 static unsigned int fatal_skb_slots
= FATAL_SKB_SLOTS_DEFAULT
;
56 module_param(fatal_skb_slots
, uint
, 0444);
59 * To avoid confusion, we define XEN_NETBK_LEGACY_SLOTS_MAX indicating
60 * the maximum slots a valid packet can use. Now this value is defined
61 * to be XEN_NETIF_NR_SLOTS_MIN, which is supposed to be supported by
64 #define XEN_NETBK_LEGACY_SLOTS_MAX XEN_NETIF_NR_SLOTS_MIN
66 typedef unsigned int pending_ring_idx_t
;
67 #define INVALID_PENDING_RING_IDX (~0U)
69 struct pending_tx_info
{
70 struct xen_netif_tx_request req
; /* coalesced tx request */
72 pending_ring_idx_t head
; /* head != INVALID_PENDING_RING_IDX
73 * if it is head of one or more tx
78 struct netbk_rx_meta
{
84 #define MAX_PENDING_REQS 256
86 /* Discriminate from any valid pending_idx value. */
87 #define INVALID_PENDING_IDX 0xFFFF
89 #define MAX_BUFFER_OFFSET PAGE_SIZE
91 /* extra field used in struct page */
94 #if BITS_PER_LONG < 64
96 #define GROUP_WIDTH (BITS_PER_LONG - IDX_WIDTH)
97 unsigned int group
:GROUP_WIDTH
;
98 unsigned int idx
:IDX_WIDTH
;
100 unsigned int group
, idx
;
107 wait_queue_head_t wq
;
108 struct task_struct
*task
;
110 struct sk_buff_head rx_queue
;
111 struct sk_buff_head tx_queue
;
113 struct timer_list net_timer
;
115 struct page
*mmap_pages
[MAX_PENDING_REQS
];
117 pending_ring_idx_t pending_prod
;
118 pending_ring_idx_t pending_cons
;
119 struct list_head net_schedule_list
;
121 /* Protect the net_schedule_list in netif. */
122 spinlock_t net_schedule_list_lock
;
124 atomic_t netfront_count
;
126 struct pending_tx_info pending_tx_info
[MAX_PENDING_REQS
];
127 /* Coalescing tx requests before copying makes number of grant
128 * copy ops greater or equal to number of slots required. In
129 * worst case a tx request consumes 2 gnttab_copy.
131 struct gnttab_copy tx_copy_ops
[2*MAX_PENDING_REQS
];
133 u16 pending_ring
[MAX_PENDING_REQS
];
136 * Given MAX_BUFFER_OFFSET of 4096 the worst case is that each
137 * head/fragment page uses 2 copy operations because it
138 * straddles two buffers in the frontend.
140 struct gnttab_copy grant_copy_op
[2*XEN_NETIF_RX_RING_SIZE
];
141 struct netbk_rx_meta meta
[2*XEN_NETIF_RX_RING_SIZE
];
144 static struct xen_netbk
*xen_netbk
;
145 static int xen_netbk_group_nr
;
148 * If head != INVALID_PENDING_RING_IDX, it means this tx request is head of
149 * one or more merged tx requests, otherwise it is the continuation of
150 * previous tx request.
152 static inline int pending_tx_is_head(struct xen_netbk
*netbk
, RING_IDX idx
)
154 return netbk
->pending_tx_info
[idx
].head
!= INVALID_PENDING_RING_IDX
;
157 void xen_netbk_add_xenvif(struct xenvif
*vif
)
160 int min_netfront_count
;
162 struct xen_netbk
*netbk
;
164 min_netfront_count
= atomic_read(&xen_netbk
[0].netfront_count
);
165 for (i
= 0; i
< xen_netbk_group_nr
; i
++) {
166 int netfront_count
= atomic_read(&xen_netbk
[i
].netfront_count
);
167 if (netfront_count
< min_netfront_count
) {
169 min_netfront_count
= netfront_count
;
173 netbk
= &xen_netbk
[min_group
];
176 atomic_inc(&netbk
->netfront_count
);
179 void xen_netbk_remove_xenvif(struct xenvif
*vif
)
181 struct xen_netbk
*netbk
= vif
->netbk
;
183 atomic_dec(&netbk
->netfront_count
);
186 static void xen_netbk_idx_release(struct xen_netbk
*netbk
, u16 pending_idx
,
188 static void make_tx_response(struct xenvif
*vif
,
189 struct xen_netif_tx_request
*txp
,
191 static struct xen_netif_rx_response
*make_rx_response(struct xenvif
*vif
,
198 static inline unsigned long idx_to_pfn(struct xen_netbk
*netbk
,
201 return page_to_pfn(netbk
->mmap_pages
[idx
]);
204 static inline unsigned long idx_to_kaddr(struct xen_netbk
*netbk
,
207 return (unsigned long)pfn_to_kaddr(idx_to_pfn(netbk
, idx
));
210 /* extra field used in struct page */
211 static inline void set_page_ext(struct page
*pg
, struct xen_netbk
*netbk
,
214 unsigned int group
= netbk
- xen_netbk
;
215 union page_ext ext
= { .e
= { .group
= group
+ 1, .idx
= idx
} };
217 BUILD_BUG_ON(sizeof(ext
) > sizeof(ext
.mapping
));
218 pg
->mapping
= ext
.mapping
;
221 static int get_page_ext(struct page
*pg
,
222 unsigned int *pgroup
, unsigned int *pidx
)
224 union page_ext ext
= { .mapping
= pg
->mapping
};
225 struct xen_netbk
*netbk
;
226 unsigned int group
, idx
;
228 group
= ext
.e
.group
- 1;
230 if (group
< 0 || group
>= xen_netbk_group_nr
)
233 netbk
= &xen_netbk
[group
];
237 if ((idx
< 0) || (idx
>= MAX_PENDING_REQS
))
240 if (netbk
->mmap_pages
[idx
] != pg
)
250 * This is the amount of packet we copy rather than map, so that the
251 * guest can't fiddle with the contents of the headers while we do
252 * packet processing on them (netfilter, routing, etc).
254 #define PKT_PROT_LEN (ETH_HLEN + \
256 sizeof(struct iphdr) + MAX_IPOPTLEN + \
257 sizeof(struct tcphdr) + MAX_TCP_OPTION_SPACE)
259 static u16
frag_get_pending_idx(skb_frag_t
*frag
)
261 return (u16
)frag
->page_offset
;
264 static void frag_set_pending_idx(skb_frag_t
*frag
, u16 pending_idx
)
266 frag
->page_offset
= pending_idx
;
269 static inline pending_ring_idx_t
pending_index(unsigned i
)
271 return i
& (MAX_PENDING_REQS
-1);
274 static inline pending_ring_idx_t
nr_pending_reqs(struct xen_netbk
*netbk
)
276 return MAX_PENDING_REQS
-
277 netbk
->pending_prod
+ netbk
->pending_cons
;
280 static void xen_netbk_kick_thread(struct xen_netbk
*netbk
)
285 static int max_required_rx_slots(struct xenvif
*vif
)
287 int max
= DIV_ROUND_UP(vif
->dev
->mtu
, PAGE_SIZE
);
289 /* XXX FIXME: RX path dependent on MAX_SKB_FRAGS */
290 if (vif
->can_sg
|| vif
->gso
|| vif
->gso_prefix
)
291 max
+= MAX_SKB_FRAGS
+ 1; /* extra_info + frags */
296 int xen_netbk_rx_ring_full(struct xenvif
*vif
)
298 RING_IDX peek
= vif
->rx_req_cons_peek
;
299 RING_IDX needed
= max_required_rx_slots(vif
);
301 return ((vif
->rx
.sring
->req_prod
- peek
) < needed
) ||
302 ((vif
->rx
.rsp_prod_pvt
+ XEN_NETIF_RX_RING_SIZE
- peek
) < needed
);
305 int xen_netbk_must_stop_queue(struct xenvif
*vif
)
307 if (!xen_netbk_rx_ring_full(vif
))
310 vif
->rx
.sring
->req_event
= vif
->rx_req_cons_peek
+
311 max_required_rx_slots(vif
);
312 mb(); /* request notification /then/ check the queue */
314 return xen_netbk_rx_ring_full(vif
);
318 * Returns true if we should start a new receive buffer instead of
319 * adding 'size' bytes to a buffer which currently contains 'offset'
322 static bool start_new_rx_buffer(int offset
, unsigned long size
, int head
)
324 /* simple case: we have completely filled the current buffer. */
325 if (offset
== MAX_BUFFER_OFFSET
)
329 * complex case: start a fresh buffer if the current frag
330 * would overflow the current buffer but only if:
331 * (i) this frag would fit completely in the next buffer
332 * and (ii) there is already some data in the current buffer
333 * and (iii) this is not the head buffer.
336 * - (i) stops us splitting a frag into two copies
337 * unless the frag is too large for a single buffer.
338 * - (ii) stops us from leaving a buffer pointlessly empty.
339 * - (iii) stops us leaving the first buffer
340 * empty. Strictly speaking this is already covered
341 * by (ii) but is explicitly checked because
342 * netfront relies on the first buffer being
343 * non-empty and can crash otherwise.
345 * This means we will effectively linearise small
346 * frags but do not needlessly split large buffers
347 * into multiple copies tend to give large frags their
348 * own buffers as before.
350 if ((offset
+ size
> MAX_BUFFER_OFFSET
) &&
351 (size
<= MAX_BUFFER_OFFSET
) && offset
&& !head
)
358 * Figure out how many ring slots we're going to need to send @skb to
359 * the guest. This function is essentially a dry run of
360 * netbk_gop_frag_copy.
362 unsigned int xen_netbk_count_skb_slots(struct xenvif
*vif
, struct sk_buff
*skb
)
367 count
= DIV_ROUND_UP(skb_headlen(skb
), PAGE_SIZE
);
369 copy_off
= skb_headlen(skb
) % PAGE_SIZE
;
371 if (skb_shinfo(skb
)->gso_size
)
374 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
375 unsigned long size
= skb_frag_size(&skb_shinfo(skb
)->frags
[i
]);
376 unsigned long offset
= skb_shinfo(skb
)->frags
[i
].page_offset
;
379 offset
&= ~PAGE_MASK
;
382 BUG_ON(offset
>= PAGE_SIZE
);
383 BUG_ON(copy_off
> MAX_BUFFER_OFFSET
);
385 bytes
= PAGE_SIZE
- offset
;
390 if (start_new_rx_buffer(copy_off
, bytes
, 0)) {
395 if (copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
396 bytes
= MAX_BUFFER_OFFSET
- copy_off
;
403 if (offset
== PAGE_SIZE
)
410 struct netrx_pending_operations
{
411 unsigned copy_prod
, copy_cons
;
412 unsigned meta_prod
, meta_cons
;
413 struct gnttab_copy
*copy
;
414 struct netbk_rx_meta
*meta
;
416 grant_ref_t copy_gref
;
419 static struct netbk_rx_meta
*get_next_rx_buffer(struct xenvif
*vif
,
420 struct netrx_pending_operations
*npo
)
422 struct netbk_rx_meta
*meta
;
423 struct xen_netif_rx_request
*req
;
425 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
427 meta
= npo
->meta
+ npo
->meta_prod
++;
433 npo
->copy_gref
= req
->gref
;
439 * Set up the grant operations for this fragment. If it's a flipping
440 * interface, we also set up the unmap request from here.
442 static void netbk_gop_frag_copy(struct xenvif
*vif
, struct sk_buff
*skb
,
443 struct netrx_pending_operations
*npo
,
444 struct page
*page
, unsigned long size
,
445 unsigned long offset
, int *head
)
447 struct gnttab_copy
*copy_gop
;
448 struct netbk_rx_meta
*meta
;
450 * These variables are used iff get_page_ext returns true,
451 * in which case they are guaranteed to be initialized.
453 unsigned int uninitialized_var(group
), uninitialized_var(idx
);
454 int foreign
= get_page_ext(page
, &group
, &idx
);
457 /* Data must not cross a page boundary. */
458 BUG_ON(size
+ offset
> PAGE_SIZE
<<compound_order(page
));
460 meta
= npo
->meta
+ npo
->meta_prod
- 1;
462 /* Skip unused frames from start of page */
463 page
+= offset
>> PAGE_SHIFT
;
464 offset
&= ~PAGE_MASK
;
467 BUG_ON(offset
>= PAGE_SIZE
);
468 BUG_ON(npo
->copy_off
> MAX_BUFFER_OFFSET
);
470 bytes
= PAGE_SIZE
- offset
;
475 if (start_new_rx_buffer(npo
->copy_off
, bytes
, *head
)) {
477 * Netfront requires there to be some data in the head
482 meta
= get_next_rx_buffer(vif
, npo
);
485 if (npo
->copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
486 bytes
= MAX_BUFFER_OFFSET
- npo
->copy_off
;
488 copy_gop
= npo
->copy
+ npo
->copy_prod
++;
489 copy_gop
->flags
= GNTCOPY_dest_gref
;
491 struct xen_netbk
*netbk
= &xen_netbk
[group
];
492 struct pending_tx_info
*src_pend
;
494 src_pend
= &netbk
->pending_tx_info
[idx
];
496 copy_gop
->source
.domid
= src_pend
->vif
->domid
;
497 copy_gop
->source
.u
.ref
= src_pend
->req
.gref
;
498 copy_gop
->flags
|= GNTCOPY_source_gref
;
500 void *vaddr
= page_address(page
);
501 copy_gop
->source
.domid
= DOMID_SELF
;
502 copy_gop
->source
.u
.gmfn
= virt_to_mfn(vaddr
);
504 copy_gop
->source
.offset
= offset
;
505 copy_gop
->dest
.domid
= vif
->domid
;
507 copy_gop
->dest
.offset
= npo
->copy_off
;
508 copy_gop
->dest
.u
.ref
= npo
->copy_gref
;
509 copy_gop
->len
= bytes
;
511 npo
->copy_off
+= bytes
;
518 if (offset
== PAGE_SIZE
&& size
) {
519 BUG_ON(!PageCompound(page
));
524 /* Leave a gap for the GSO descriptor. */
525 if (*head
&& skb_shinfo(skb
)->gso_size
&& !vif
->gso_prefix
)
528 *head
= 0; /* There must be something in this buffer now. */
534 * Prepare an SKB to be transmitted to the frontend.
536 * This function is responsible for allocating grant operations, meta
539 * It returns the number of meta structures consumed. The number of
540 * ring slots used is always equal to the number of meta slots used
541 * plus the number of GSO descriptors used. Currently, we use either
542 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
543 * frontend-side LRO).
545 static int netbk_gop_skb(struct sk_buff
*skb
,
546 struct netrx_pending_operations
*npo
)
548 struct xenvif
*vif
= netdev_priv(skb
->dev
);
549 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
551 struct xen_netif_rx_request
*req
;
552 struct netbk_rx_meta
*meta
;
557 old_meta_prod
= npo
->meta_prod
;
559 /* Set up a GSO prefix descriptor, if necessary */
560 if (skb_shinfo(skb
)->gso_size
&& vif
->gso_prefix
) {
561 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
562 meta
= npo
->meta
+ npo
->meta_prod
++;
563 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
568 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
569 meta
= npo
->meta
+ npo
->meta_prod
++;
571 if (!vif
->gso_prefix
)
572 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
579 npo
->copy_gref
= req
->gref
;
582 while (data
< skb_tail_pointer(skb
)) {
583 unsigned int offset
= offset_in_page(data
);
584 unsigned int len
= PAGE_SIZE
- offset
;
586 if (data
+ len
> skb_tail_pointer(skb
))
587 len
= skb_tail_pointer(skb
) - data
;
589 netbk_gop_frag_copy(vif
, skb
, npo
,
590 virt_to_page(data
), len
, offset
, &head
);
594 for (i
= 0; i
< nr_frags
; i
++) {
595 netbk_gop_frag_copy(vif
, skb
, npo
,
596 skb_frag_page(&skb_shinfo(skb
)->frags
[i
]),
597 skb_frag_size(&skb_shinfo(skb
)->frags
[i
]),
598 skb_shinfo(skb
)->frags
[i
].page_offset
,
602 return npo
->meta_prod
- old_meta_prod
;
606 * This is a twin to netbk_gop_skb. Assume that netbk_gop_skb was
607 * used to set up the operations on the top of
608 * netrx_pending_operations, which have since been done. Check that
609 * they didn't give any errors and advance over them.
611 static int netbk_check_gop(struct xenvif
*vif
, int nr_meta_slots
,
612 struct netrx_pending_operations
*npo
)
614 struct gnttab_copy
*copy_op
;
615 int status
= XEN_NETIF_RSP_OKAY
;
618 for (i
= 0; i
< nr_meta_slots
; i
++) {
619 copy_op
= npo
->copy
+ npo
->copy_cons
++;
620 if (copy_op
->status
!= GNTST_okay
) {
622 "Bad status %d from copy to DOM%d.\n",
623 copy_op
->status
, vif
->domid
);
624 status
= XEN_NETIF_RSP_ERROR
;
631 static void netbk_add_frag_responses(struct xenvif
*vif
, int status
,
632 struct netbk_rx_meta
*meta
,
636 unsigned long offset
;
638 /* No fragments used */
639 if (nr_meta_slots
<= 1)
644 for (i
= 0; i
< nr_meta_slots
; i
++) {
646 if (i
== nr_meta_slots
- 1)
649 flags
= XEN_NETRXF_more_data
;
652 make_rx_response(vif
, meta
[i
].id
, status
, offset
,
653 meta
[i
].size
, flags
);
657 struct skb_cb_overlay
{
661 static void xen_netbk_rx_action(struct xen_netbk
*netbk
)
663 struct xenvif
*vif
= NULL
, *tmp
;
666 struct xen_netif_rx_response
*resp
;
667 struct sk_buff_head rxq
;
673 unsigned long offset
;
674 struct skb_cb_overlay
*sco
;
676 struct netrx_pending_operations npo
= {
677 .copy
= netbk
->grant_copy_op
,
681 skb_queue_head_init(&rxq
);
685 while ((skb
= skb_dequeue(&netbk
->rx_queue
)) != NULL
) {
686 vif
= netdev_priv(skb
->dev
);
687 nr_frags
= skb_shinfo(skb
)->nr_frags
;
689 sco
= (struct skb_cb_overlay
*)skb
->cb
;
690 sco
->meta_slots_used
= netbk_gop_skb(skb
, &npo
);
692 count
+= nr_frags
+ 1;
694 __skb_queue_tail(&rxq
, skb
);
696 /* Filled the batch queue? */
697 /* XXX FIXME: RX path dependent on MAX_SKB_FRAGS */
698 if (count
+ MAX_SKB_FRAGS
>= XEN_NETIF_RX_RING_SIZE
)
702 BUG_ON(npo
.meta_prod
> ARRAY_SIZE(netbk
->meta
));
707 BUG_ON(npo
.copy_prod
> ARRAY_SIZE(netbk
->grant_copy_op
));
708 gnttab_batch_copy(netbk
->grant_copy_op
, npo
.copy_prod
);
710 while ((skb
= __skb_dequeue(&rxq
)) != NULL
) {
711 sco
= (struct skb_cb_overlay
*)skb
->cb
;
713 vif
= netdev_priv(skb
->dev
);
715 if (netbk
->meta
[npo
.meta_cons
].gso_size
&& vif
->gso_prefix
) {
716 resp
= RING_GET_RESPONSE(&vif
->rx
,
717 vif
->rx
.rsp_prod_pvt
++);
719 resp
->flags
= XEN_NETRXF_gso_prefix
| XEN_NETRXF_more_data
;
721 resp
->offset
= netbk
->meta
[npo
.meta_cons
].gso_size
;
722 resp
->id
= netbk
->meta
[npo
.meta_cons
].id
;
723 resp
->status
= sco
->meta_slots_used
;
726 sco
->meta_slots_used
--;
730 vif
->dev
->stats
.tx_bytes
+= skb
->len
;
731 vif
->dev
->stats
.tx_packets
++;
733 status
= netbk_check_gop(vif
, sco
->meta_slots_used
, &npo
);
735 if (sco
->meta_slots_used
== 1)
738 flags
= XEN_NETRXF_more_data
;
740 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) /* local packet? */
741 flags
|= XEN_NETRXF_csum_blank
| XEN_NETRXF_data_validated
;
742 else if (skb
->ip_summed
== CHECKSUM_UNNECESSARY
)
743 /* remote but checksummed. */
744 flags
|= XEN_NETRXF_data_validated
;
747 resp
= make_rx_response(vif
, netbk
->meta
[npo
.meta_cons
].id
,
749 netbk
->meta
[npo
.meta_cons
].size
,
752 if (netbk
->meta
[npo
.meta_cons
].gso_size
&& !vif
->gso_prefix
) {
753 struct xen_netif_extra_info
*gso
=
754 (struct xen_netif_extra_info
*)
755 RING_GET_RESPONSE(&vif
->rx
,
756 vif
->rx
.rsp_prod_pvt
++);
758 resp
->flags
|= XEN_NETRXF_extra_info
;
760 gso
->u
.gso
.size
= netbk
->meta
[npo
.meta_cons
].gso_size
;
761 gso
->u
.gso
.type
= XEN_NETIF_GSO_TYPE_TCPV4
;
763 gso
->u
.gso
.features
= 0;
765 gso
->type
= XEN_NETIF_EXTRA_TYPE_GSO
;
769 netbk_add_frag_responses(vif
, status
,
770 netbk
->meta
+ npo
.meta_cons
+ 1,
771 sco
->meta_slots_used
);
773 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif
->rx
, ret
);
775 xenvif_notify_tx_completion(vif
);
777 if (ret
&& list_empty(&vif
->notify_list
))
778 list_add_tail(&vif
->notify_list
, ¬ify
);
781 npo
.meta_cons
+= sco
->meta_slots_used
;
785 list_for_each_entry_safe(vif
, tmp
, ¬ify
, notify_list
) {
786 notify_remote_via_irq(vif
->irq
);
787 list_del_init(&vif
->notify_list
);
791 /* More work to do? */
792 if (!skb_queue_empty(&netbk
->rx_queue
) &&
793 !timer_pending(&netbk
->net_timer
))
794 xen_netbk_kick_thread(netbk
);
797 void xen_netbk_queue_tx_skb(struct xenvif
*vif
, struct sk_buff
*skb
)
799 struct xen_netbk
*netbk
= vif
->netbk
;
801 skb_queue_tail(&netbk
->rx_queue
, skb
);
803 xen_netbk_kick_thread(netbk
);
806 static void xen_netbk_alarm(unsigned long data
)
808 struct xen_netbk
*netbk
= (struct xen_netbk
*)data
;
809 xen_netbk_kick_thread(netbk
);
812 static int __on_net_schedule_list(struct xenvif
*vif
)
814 return !list_empty(&vif
->schedule_list
);
817 /* Must be called with net_schedule_list_lock held */
818 static void remove_from_net_schedule_list(struct xenvif
*vif
)
820 if (likely(__on_net_schedule_list(vif
))) {
821 list_del_init(&vif
->schedule_list
);
826 static struct xenvif
*poll_net_schedule_list(struct xen_netbk
*netbk
)
828 struct xenvif
*vif
= NULL
;
830 spin_lock_irq(&netbk
->net_schedule_list_lock
);
831 if (list_empty(&netbk
->net_schedule_list
))
834 vif
= list_first_entry(&netbk
->net_schedule_list
,
835 struct xenvif
, schedule_list
);
841 remove_from_net_schedule_list(vif
);
843 spin_unlock_irq(&netbk
->net_schedule_list_lock
);
847 void xen_netbk_schedule_xenvif(struct xenvif
*vif
)
850 struct xen_netbk
*netbk
= vif
->netbk
;
852 if (__on_net_schedule_list(vif
))
855 spin_lock_irqsave(&netbk
->net_schedule_list_lock
, flags
);
856 if (!__on_net_schedule_list(vif
) &&
857 likely(xenvif_schedulable(vif
))) {
858 list_add_tail(&vif
->schedule_list
, &netbk
->net_schedule_list
);
861 spin_unlock_irqrestore(&netbk
->net_schedule_list_lock
, flags
);
865 if ((nr_pending_reqs(netbk
) < (MAX_PENDING_REQS
/2)) &&
866 !list_empty(&netbk
->net_schedule_list
))
867 xen_netbk_kick_thread(netbk
);
870 void xen_netbk_deschedule_xenvif(struct xenvif
*vif
)
872 struct xen_netbk
*netbk
= vif
->netbk
;
873 spin_lock_irq(&netbk
->net_schedule_list_lock
);
874 remove_from_net_schedule_list(vif
);
875 spin_unlock_irq(&netbk
->net_schedule_list_lock
);
878 void xen_netbk_check_rx_xenvif(struct xenvif
*vif
)
882 RING_FINAL_CHECK_FOR_REQUESTS(&vif
->tx
, more_to_do
);
885 xen_netbk_schedule_xenvif(vif
);
888 static void tx_add_credit(struct xenvif
*vif
)
890 unsigned long max_burst
, max_credit
;
893 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
894 * Otherwise the interface can seize up due to insufficient credit.
896 max_burst
= RING_GET_REQUEST(&vif
->tx
, vif
->tx
.req_cons
)->size
;
897 max_burst
= min(max_burst
, 131072UL);
898 max_burst
= max(max_burst
, vif
->credit_bytes
);
900 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
901 max_credit
= vif
->remaining_credit
+ vif
->credit_bytes
;
902 if (max_credit
< vif
->remaining_credit
)
903 max_credit
= ULONG_MAX
; /* wrapped: clamp to ULONG_MAX */
905 vif
->remaining_credit
= min(max_credit
, max_burst
);
908 static void tx_credit_callback(unsigned long data
)
910 struct xenvif
*vif
= (struct xenvif
*)data
;
912 xen_netbk_check_rx_xenvif(vif
);
915 static void netbk_tx_err(struct xenvif
*vif
,
916 struct xen_netif_tx_request
*txp
, RING_IDX end
)
918 RING_IDX cons
= vif
->tx
.req_cons
;
921 make_tx_response(vif
, txp
, XEN_NETIF_RSP_ERROR
);
924 txp
= RING_GET_REQUEST(&vif
->tx
, cons
++);
926 vif
->tx
.req_cons
= cons
;
927 xen_netbk_check_rx_xenvif(vif
);
931 static void netbk_fatal_tx_err(struct xenvif
*vif
)
933 netdev_err(vif
->dev
, "fatal error; disabling device\n");
934 xenvif_carrier_off(vif
);
938 static int netbk_count_requests(struct xenvif
*vif
,
939 struct xen_netif_tx_request
*first
,
940 struct xen_netif_tx_request
*txp
,
943 RING_IDX cons
= vif
->tx
.req_cons
;
948 if (!(first
->flags
& XEN_NETTXF_more_data
))
952 struct xen_netif_tx_request dropped_tx
= { 0 };
954 if (slots
>= work_to_do
) {
956 "Asked for %d slots but exceeds this limit\n",
958 netbk_fatal_tx_err(vif
);
962 /* This guest is really using too many slots and
963 * considered malicious.
965 if (unlikely(slots
>= fatal_skb_slots
)) {
967 "Malicious frontend using %d slots, threshold %u\n",
968 slots
, fatal_skb_slots
);
969 netbk_fatal_tx_err(vif
);
973 /* Xen network protocol had implicit dependency on
974 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to
975 * the historical MAX_SKB_FRAGS value 18 to honor the
976 * same behavior as before. Any packet using more than
977 * 18 slots but less than fatal_skb_slots slots is
980 if (!drop_err
&& slots
>= XEN_NETBK_LEGACY_SLOTS_MAX
) {
983 "Too many slots (%d) exceeding limit (%d), dropping packet\n",
984 slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
991 memcpy(txp
, RING_GET_REQUEST(&vif
->tx
, cons
+ slots
),
994 /* If the guest submitted a frame >= 64 KiB then
995 * first->size overflowed and following slots will
996 * appear to be larger than the frame.
998 * This cannot be fatal error as there are buggy
999 * frontends that do this.
1001 * Consume all slots and drop the packet.
1003 if (!drop_err
&& txp
->size
> first
->size
) {
1004 if (net_ratelimit())
1005 netdev_dbg(vif
->dev
,
1006 "Invalid tx request, slot size %u > remaining size %u\n",
1007 txp
->size
, first
->size
);
1011 first
->size
-= txp
->size
;
1014 if (unlikely((txp
->offset
+ txp
->size
) > PAGE_SIZE
)) {
1015 netdev_err(vif
->dev
, "Cross page boundary, txp->offset: %x, size: %u\n",
1016 txp
->offset
, txp
->size
);
1017 netbk_fatal_tx_err(vif
);
1021 more_data
= txp
->flags
& XEN_NETTXF_more_data
;
1026 } while (more_data
);
1029 netbk_tx_err(vif
, first
, cons
+ slots
);
1036 static struct page
*xen_netbk_alloc_page(struct xen_netbk
*netbk
,
1040 page
= alloc_page(GFP_KERNEL
|__GFP_COLD
);
1043 set_page_ext(page
, netbk
, pending_idx
);
1044 netbk
->mmap_pages
[pending_idx
] = page
;
1048 static struct gnttab_copy
*xen_netbk_get_requests(struct xen_netbk
*netbk
,
1050 struct sk_buff
*skb
,
1051 struct xen_netif_tx_request
*txp
,
1052 struct gnttab_copy
*gop
)
1054 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1055 skb_frag_t
*frags
= shinfo
->frags
;
1056 u16 pending_idx
= *((u16
*)skb
->data
);
1060 pending_ring_idx_t index
, start_idx
= 0;
1061 uint16_t dst_offset
;
1062 unsigned int nr_slots
;
1063 struct pending_tx_info
*first
= NULL
;
1065 /* At this point shinfo->nr_frags is in fact the number of
1066 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
1068 nr_slots
= shinfo
->nr_frags
;
1070 /* Skip first skb fragment if it is on same page as header fragment. */
1071 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
1073 /* Coalesce tx requests, at this point the packet passed in
1074 * should be <= 64K. Any packets larger than 64K have been
1075 * handled in netbk_count_requests().
1077 for (shinfo
->nr_frags
= slot
= start
; slot
< nr_slots
;
1078 shinfo
->nr_frags
++) {
1079 struct pending_tx_info
*pending_tx_info
=
1080 netbk
->pending_tx_info
;
1082 page
= alloc_page(GFP_KERNEL
|__GFP_COLD
);
1088 while (dst_offset
< PAGE_SIZE
&& slot
< nr_slots
) {
1089 gop
->flags
= GNTCOPY_source_gref
;
1091 gop
->source
.u
.ref
= txp
->gref
;
1092 gop
->source
.domid
= vif
->domid
;
1093 gop
->source
.offset
= txp
->offset
;
1095 gop
->dest
.domid
= DOMID_SELF
;
1097 gop
->dest
.offset
= dst_offset
;
1098 gop
->dest
.u
.gmfn
= virt_to_mfn(page_address(page
));
1100 if (dst_offset
+ txp
->size
> PAGE_SIZE
) {
1101 /* This page can only merge a portion
1102 * of tx request. Do not increment any
1103 * pointer / counter here. The txp
1104 * will be dealt with in future
1105 * rounds, eventually hitting the
1108 gop
->len
= PAGE_SIZE
- dst_offset
;
1109 txp
->offset
+= gop
->len
;
1110 txp
->size
-= gop
->len
;
1111 dst_offset
+= gop
->len
; /* quit loop */
1113 /* This tx request can be merged in the page */
1114 gop
->len
= txp
->size
;
1115 dst_offset
+= gop
->len
;
1117 index
= pending_index(netbk
->pending_cons
++);
1119 pending_idx
= netbk
->pending_ring
[index
];
1121 memcpy(&pending_tx_info
[pending_idx
].req
, txp
,
1125 pending_tx_info
[pending_idx
].vif
= vif
;
1127 /* Poison these fields, corresponding
1128 * fields for head tx req will be set
1129 * to correct values after the loop.
1131 netbk
->mmap_pages
[pending_idx
] = (void *)(~0UL);
1132 pending_tx_info
[pending_idx
].head
=
1133 INVALID_PENDING_RING_IDX
;
1136 first
= &pending_tx_info
[pending_idx
];
1138 head_idx
= pending_idx
;
1148 first
->req
.offset
= 0;
1149 first
->req
.size
= dst_offset
;
1150 first
->head
= start_idx
;
1151 set_page_ext(page
, netbk
, head_idx
);
1152 netbk
->mmap_pages
[head_idx
] = page
;
1153 frag_set_pending_idx(&frags
[shinfo
->nr_frags
], head_idx
);
1156 BUG_ON(shinfo
->nr_frags
> MAX_SKB_FRAGS
);
1160 /* Unwind, freeing all pages and sending error responses. */
1161 while (shinfo
->nr_frags
-- > start
) {
1162 xen_netbk_idx_release(netbk
,
1163 frag_get_pending_idx(&frags
[shinfo
->nr_frags
]),
1164 XEN_NETIF_RSP_ERROR
);
1166 /* The head too, if necessary. */
1168 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_ERROR
);
1173 static int xen_netbk_tx_check_gop(struct xen_netbk
*netbk
,
1174 struct sk_buff
*skb
,
1175 struct gnttab_copy
**gopp
)
1177 struct gnttab_copy
*gop
= *gopp
;
1178 u16 pending_idx
= *((u16
*)skb
->data
);
1179 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1180 struct pending_tx_info
*tx_info
;
1181 int nr_frags
= shinfo
->nr_frags
;
1183 u16 peek
; /* peek into next tx request */
1185 /* Check status of header. */
1188 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_ERROR
);
1190 /* Skip first skb fragment if it is on same page as header fragment. */
1191 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
1193 for (i
= start
; i
< nr_frags
; i
++) {
1195 pending_ring_idx_t head
;
1197 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[i
]);
1198 tx_info
= &netbk
->pending_tx_info
[pending_idx
];
1199 head
= tx_info
->head
;
1201 /* Check error status: if okay then remember grant handle. */
1203 newerr
= (++gop
)->status
;
1206 peek
= netbk
->pending_ring
[pending_index(++head
)];
1207 } while (!pending_tx_is_head(netbk
, peek
));
1209 if (likely(!newerr
)) {
1210 /* Had a previous error? Invalidate this fragment. */
1212 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1216 /* Error on this fragment: respond to client with an error. */
1217 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_ERROR
);
1219 /* Not the first error? Preceding frags already invalidated. */
1223 /* First error: invalidate header and preceding fragments. */
1224 pending_idx
= *((u16
*)skb
->data
);
1225 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1226 for (j
= start
; j
< i
; j
++) {
1227 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[j
]);
1228 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1231 /* Remember the error: invalidate all subsequent fragments. */
1239 static void xen_netbk_fill_frags(struct xen_netbk
*netbk
, struct sk_buff
*skb
)
1241 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1242 int nr_frags
= shinfo
->nr_frags
;
1245 for (i
= 0; i
< nr_frags
; i
++) {
1246 skb_frag_t
*frag
= shinfo
->frags
+ i
;
1247 struct xen_netif_tx_request
*txp
;
1251 pending_idx
= frag_get_pending_idx(frag
);
1253 txp
= &netbk
->pending_tx_info
[pending_idx
].req
;
1254 page
= virt_to_page(idx_to_kaddr(netbk
, pending_idx
));
1255 __skb_fill_page_desc(skb
, i
, page
, txp
->offset
, txp
->size
);
1256 skb
->len
+= txp
->size
;
1257 skb
->data_len
+= txp
->size
;
1258 skb
->truesize
+= txp
->size
;
1260 /* Take an extra reference to offset xen_netbk_idx_release */
1261 get_page(netbk
->mmap_pages
[pending_idx
]);
1262 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1266 static int xen_netbk_get_extras(struct xenvif
*vif
,
1267 struct xen_netif_extra_info
*extras
,
1270 struct xen_netif_extra_info extra
;
1271 RING_IDX cons
= vif
->tx
.req_cons
;
1274 if (unlikely(work_to_do
-- <= 0)) {
1275 netdev_err(vif
->dev
, "Missing extra info\n");
1276 netbk_fatal_tx_err(vif
);
1280 memcpy(&extra
, RING_GET_REQUEST(&vif
->tx
, cons
),
1282 if (unlikely(!extra
.type
||
1283 extra
.type
>= XEN_NETIF_EXTRA_TYPE_MAX
)) {
1284 vif
->tx
.req_cons
= ++cons
;
1285 netdev_err(vif
->dev
,
1286 "Invalid extra type: %d\n", extra
.type
);
1287 netbk_fatal_tx_err(vif
);
1291 memcpy(&extras
[extra
.type
- 1], &extra
, sizeof(extra
));
1292 vif
->tx
.req_cons
= ++cons
;
1293 } while (extra
.flags
& XEN_NETIF_EXTRA_FLAG_MORE
);
1298 static int netbk_set_skb_gso(struct xenvif
*vif
,
1299 struct sk_buff
*skb
,
1300 struct xen_netif_extra_info
*gso
)
1302 if (!gso
->u
.gso
.size
) {
1303 netdev_err(vif
->dev
, "GSO size must not be zero.\n");
1304 netbk_fatal_tx_err(vif
);
1308 /* Currently only TCPv4 S.O. is supported. */
1309 if (gso
->u
.gso
.type
!= XEN_NETIF_GSO_TYPE_TCPV4
) {
1310 netdev_err(vif
->dev
, "Bad GSO type %d.\n", gso
->u
.gso
.type
);
1311 netbk_fatal_tx_err(vif
);
1315 skb_shinfo(skb
)->gso_size
= gso
->u
.gso
.size
;
1316 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
1318 /* Header must be checked, and gso_segs computed. */
1319 skb_shinfo(skb
)->gso_type
|= SKB_GSO_DODGY
;
1320 skb_shinfo(skb
)->gso_segs
= 0;
1325 static int checksum_setup(struct xenvif
*vif
, struct sk_buff
*skb
)
1329 int recalculate_partial_csum
= 0;
1332 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1333 * peers can fail to set NETRXF_csum_blank when sending a GSO
1334 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1335 * recalculate the partial checksum.
1337 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
&& skb_is_gso(skb
)) {
1338 vif
->rx_gso_checksum_fixup
++;
1339 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1340 recalculate_partial_csum
= 1;
1343 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1344 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1347 if (skb
->protocol
!= htons(ETH_P_IP
))
1350 iph
= (void *)skb
->data
;
1351 switch (iph
->protocol
) {
1353 if (!skb_partial_csum_set(skb
, 4 * iph
->ihl
,
1354 offsetof(struct tcphdr
, check
)))
1357 if (recalculate_partial_csum
) {
1358 struct tcphdr
*tcph
= tcp_hdr(skb
);
1359 tcph
->check
= ~csum_tcpudp_magic(iph
->saddr
, iph
->daddr
,
1360 skb
->len
- iph
->ihl
*4,
1365 if (!skb_partial_csum_set(skb
, 4 * iph
->ihl
,
1366 offsetof(struct udphdr
, check
)))
1369 if (recalculate_partial_csum
) {
1370 struct udphdr
*udph
= udp_hdr(skb
);
1371 udph
->check
= ~csum_tcpudp_magic(iph
->saddr
, iph
->daddr
,
1372 skb
->len
- iph
->ihl
*4,
1377 if (net_ratelimit())
1378 netdev_err(vif
->dev
,
1379 "Attempting to checksum a non-TCP/UDP packet, dropping a protocol %d packet\n",
1390 static bool tx_credit_exceeded(struct xenvif
*vif
, unsigned size
)
1392 unsigned long now
= jiffies
;
1393 unsigned long next_credit
=
1394 vif
->credit_timeout
.expires
+
1395 msecs_to_jiffies(vif
->credit_usec
/ 1000);
1397 /* Timer could already be pending in rare cases. */
1398 if (timer_pending(&vif
->credit_timeout
))
1401 /* Passed the point where we can replenish credit? */
1402 if (time_after_eq(now
, next_credit
)) {
1403 vif
->credit_timeout
.expires
= now
;
1407 /* Still too big to send right now? Set a callback. */
1408 if (size
> vif
->remaining_credit
) {
1409 vif
->credit_timeout
.data
=
1411 vif
->credit_timeout
.function
=
1413 mod_timer(&vif
->credit_timeout
,
1422 static unsigned xen_netbk_tx_build_gops(struct xen_netbk
*netbk
)
1424 struct gnttab_copy
*gop
= netbk
->tx_copy_ops
, *request_gop
;
1425 struct sk_buff
*skb
;
1428 while ((nr_pending_reqs(netbk
) + XEN_NETBK_LEGACY_SLOTS_MAX
1429 < MAX_PENDING_REQS
) &&
1430 !list_empty(&netbk
->net_schedule_list
)) {
1432 struct xen_netif_tx_request txreq
;
1433 struct xen_netif_tx_request txfrags
[XEN_NETBK_LEGACY_SLOTS_MAX
];
1435 struct xen_netif_extra_info extras
[XEN_NETIF_EXTRA_TYPE_MAX
-1];
1439 unsigned int data_len
;
1440 pending_ring_idx_t index
;
1442 /* Get a netif from the list with work to do. */
1443 vif
= poll_net_schedule_list(netbk
);
1444 /* This can sometimes happen because the test of
1445 * list_empty(net_schedule_list) at the top of the
1446 * loop is unlocked. Just go back and have another
1452 if (vif
->tx
.sring
->req_prod
- vif
->tx
.req_cons
>
1453 XEN_NETIF_TX_RING_SIZE
) {
1454 netdev_err(vif
->dev
,
1455 "Impossible number of requests. "
1456 "req_prod %d, req_cons %d, size %ld\n",
1457 vif
->tx
.sring
->req_prod
, vif
->tx
.req_cons
,
1458 XEN_NETIF_TX_RING_SIZE
);
1459 netbk_fatal_tx_err(vif
);
1463 RING_FINAL_CHECK_FOR_REQUESTS(&vif
->tx
, work_to_do
);
1469 idx
= vif
->tx
.req_cons
;
1470 rmb(); /* Ensure that we see the request before we copy it. */
1471 memcpy(&txreq
, RING_GET_REQUEST(&vif
->tx
, idx
), sizeof(txreq
));
1473 /* Credit-based scheduling. */
1474 if (txreq
.size
> vif
->remaining_credit
&&
1475 tx_credit_exceeded(vif
, txreq
.size
)) {
1480 vif
->remaining_credit
-= txreq
.size
;
1483 vif
->tx
.req_cons
= ++idx
;
1485 memset(extras
, 0, sizeof(extras
));
1486 if (txreq
.flags
& XEN_NETTXF_extra_info
) {
1487 work_to_do
= xen_netbk_get_extras(vif
, extras
,
1489 idx
= vif
->tx
.req_cons
;
1490 if (unlikely(work_to_do
< 0))
1494 ret
= netbk_count_requests(vif
, &txreq
, txfrags
, work_to_do
);
1495 if (unlikely(ret
< 0))
1500 if (unlikely(txreq
.size
< ETH_HLEN
)) {
1501 netdev_dbg(vif
->dev
,
1502 "Bad packet size: %d\n", txreq
.size
);
1503 netbk_tx_err(vif
, &txreq
, idx
);
1507 /* No crossing a page as the payload mustn't fragment. */
1508 if (unlikely((txreq
.offset
+ txreq
.size
) > PAGE_SIZE
)) {
1509 netdev_err(vif
->dev
,
1510 "txreq.offset: %x, size: %u, end: %lu\n",
1511 txreq
.offset
, txreq
.size
,
1512 (txreq
.offset
&~PAGE_MASK
) + txreq
.size
);
1513 netbk_fatal_tx_err(vif
);
1517 index
= pending_index(netbk
->pending_cons
);
1518 pending_idx
= netbk
->pending_ring
[index
];
1520 data_len
= (txreq
.size
> PKT_PROT_LEN
&&
1521 ret
< XEN_NETBK_LEGACY_SLOTS_MAX
) ?
1522 PKT_PROT_LEN
: txreq
.size
;
1524 skb
= alloc_skb(data_len
+ NET_SKB_PAD
+ NET_IP_ALIGN
,
1525 GFP_ATOMIC
| __GFP_NOWARN
);
1526 if (unlikely(skb
== NULL
)) {
1527 netdev_dbg(vif
->dev
,
1528 "Can't allocate a skb in start_xmit.\n");
1529 netbk_tx_err(vif
, &txreq
, idx
);
1533 /* Packets passed to netif_rx() must have some headroom. */
1534 skb_reserve(skb
, NET_SKB_PAD
+ NET_IP_ALIGN
);
1536 if (extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1].type
) {
1537 struct xen_netif_extra_info
*gso
;
1538 gso
= &extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1];
1540 if (netbk_set_skb_gso(vif
, skb
, gso
)) {
1541 /* Failure in netbk_set_skb_gso is fatal. */
1547 /* XXX could copy straight to head */
1548 page
= xen_netbk_alloc_page(netbk
, pending_idx
);
1551 netbk_tx_err(vif
, &txreq
, idx
);
1555 gop
->source
.u
.ref
= txreq
.gref
;
1556 gop
->source
.domid
= vif
->domid
;
1557 gop
->source
.offset
= txreq
.offset
;
1559 gop
->dest
.u
.gmfn
= virt_to_mfn(page_address(page
));
1560 gop
->dest
.domid
= DOMID_SELF
;
1561 gop
->dest
.offset
= txreq
.offset
;
1563 gop
->len
= txreq
.size
;
1564 gop
->flags
= GNTCOPY_source_gref
;
1568 memcpy(&netbk
->pending_tx_info
[pending_idx
].req
,
1569 &txreq
, sizeof(txreq
));
1570 netbk
->pending_tx_info
[pending_idx
].vif
= vif
;
1571 netbk
->pending_tx_info
[pending_idx
].head
= index
;
1572 *((u16
*)skb
->data
) = pending_idx
;
1574 __skb_put(skb
, data_len
);
1576 skb_shinfo(skb
)->nr_frags
= ret
;
1577 if (data_len
< txreq
.size
) {
1578 skb_shinfo(skb
)->nr_frags
++;
1579 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1582 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1583 INVALID_PENDING_IDX
);
1586 netbk
->pending_cons
++;
1588 request_gop
= xen_netbk_get_requests(netbk
, vif
,
1590 if (request_gop
== NULL
) {
1592 netbk_tx_err(vif
, &txreq
, idx
);
1597 __skb_queue_tail(&netbk
->tx_queue
, skb
);
1599 vif
->tx
.req_cons
= idx
;
1600 xen_netbk_check_rx_xenvif(vif
);
1602 if ((gop
-netbk
->tx_copy_ops
) >= ARRAY_SIZE(netbk
->tx_copy_ops
))
1606 return gop
- netbk
->tx_copy_ops
;
1609 static void xen_netbk_tx_submit(struct xen_netbk
*netbk
)
1611 struct gnttab_copy
*gop
= netbk
->tx_copy_ops
;
1612 struct sk_buff
*skb
;
1614 while ((skb
= __skb_dequeue(&netbk
->tx_queue
)) != NULL
) {
1615 struct xen_netif_tx_request
*txp
;
1620 pending_idx
= *((u16
*)skb
->data
);
1621 vif
= netbk
->pending_tx_info
[pending_idx
].vif
;
1622 txp
= &netbk
->pending_tx_info
[pending_idx
].req
;
1624 /* Check the remap error code. */
1625 if (unlikely(xen_netbk_tx_check_gop(netbk
, skb
, &gop
))) {
1626 netdev_dbg(vif
->dev
, "netback grant failed.\n");
1627 skb_shinfo(skb
)->nr_frags
= 0;
1632 data_len
= skb
->len
;
1634 (void *)(idx_to_kaddr(netbk
, pending_idx
)|txp
->offset
),
1636 if (data_len
< txp
->size
) {
1637 /* Append the packet payload as a fragment. */
1638 txp
->offset
+= data_len
;
1639 txp
->size
-= data_len
;
1641 /* Schedule a response immediately. */
1642 xen_netbk_idx_release(netbk
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1645 if (txp
->flags
& XEN_NETTXF_csum_blank
)
1646 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1647 else if (txp
->flags
& XEN_NETTXF_data_validated
)
1648 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1650 xen_netbk_fill_frags(netbk
, skb
);
1653 * If the initial fragment was < PKT_PROT_LEN then
1654 * pull through some bytes from the other fragments to
1655 * increase the linear region to PKT_PROT_LEN bytes.
1657 if (skb_headlen(skb
) < PKT_PROT_LEN
&& skb_is_nonlinear(skb
)) {
1658 int target
= min_t(int, skb
->len
, PKT_PROT_LEN
);
1659 __pskb_pull_tail(skb
, target
- skb_headlen(skb
));
1662 skb
->dev
= vif
->dev
;
1663 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1664 skb_reset_network_header(skb
);
1666 if (checksum_setup(vif
, skb
)) {
1667 netdev_dbg(vif
->dev
,
1668 "Can't setup checksum in net_tx_action\n");
1673 skb_probe_transport_header(skb
, 0);
1675 vif
->dev
->stats
.rx_bytes
+= skb
->len
;
1676 vif
->dev
->stats
.rx_packets
++;
1678 xenvif_receive_skb(vif
, skb
);
1682 /* Called after netfront has transmitted */
1683 static void xen_netbk_tx_action(struct xen_netbk
*netbk
)
1687 nr_gops
= xen_netbk_tx_build_gops(netbk
);
1692 gnttab_batch_copy(netbk
->tx_copy_ops
, nr_gops
);
1694 xen_netbk_tx_submit(netbk
);
1697 static void xen_netbk_idx_release(struct xen_netbk
*netbk
, u16 pending_idx
,
1701 struct pending_tx_info
*pending_tx_info
;
1702 pending_ring_idx_t head
;
1703 u16 peek
; /* peek into next tx request */
1705 BUG_ON(netbk
->mmap_pages
[pending_idx
] == (void *)(~0UL));
1707 /* Already complete? */
1708 if (netbk
->mmap_pages
[pending_idx
] == NULL
)
1711 pending_tx_info
= &netbk
->pending_tx_info
[pending_idx
];
1713 vif
= pending_tx_info
->vif
;
1714 head
= pending_tx_info
->head
;
1716 BUG_ON(!pending_tx_is_head(netbk
, head
));
1717 BUG_ON(netbk
->pending_ring
[pending_index(head
)] != pending_idx
);
1720 pending_ring_idx_t index
;
1721 pending_ring_idx_t idx
= pending_index(head
);
1722 u16 info_idx
= netbk
->pending_ring
[idx
];
1724 pending_tx_info
= &netbk
->pending_tx_info
[info_idx
];
1725 make_tx_response(vif
, &pending_tx_info
->req
, status
);
1727 /* Setting any number other than
1728 * INVALID_PENDING_RING_IDX indicates this slot is
1729 * starting a new packet / ending a previous packet.
1731 pending_tx_info
->head
= 0;
1733 index
= pending_index(netbk
->pending_prod
++);
1734 netbk
->pending_ring
[index
] = netbk
->pending_ring
[info_idx
];
1738 peek
= netbk
->pending_ring
[pending_index(++head
)];
1740 } while (!pending_tx_is_head(netbk
, peek
));
1742 netbk
->mmap_pages
[pending_idx
]->mapping
= 0;
1743 put_page(netbk
->mmap_pages
[pending_idx
]);
1744 netbk
->mmap_pages
[pending_idx
] = NULL
;
1748 static void make_tx_response(struct xenvif
*vif
,
1749 struct xen_netif_tx_request
*txp
,
1752 RING_IDX i
= vif
->tx
.rsp_prod_pvt
;
1753 struct xen_netif_tx_response
*resp
;
1756 resp
= RING_GET_RESPONSE(&vif
->tx
, i
);
1760 if (txp
->flags
& XEN_NETTXF_extra_info
)
1761 RING_GET_RESPONSE(&vif
->tx
, ++i
)->status
= XEN_NETIF_RSP_NULL
;
1763 vif
->tx
.rsp_prod_pvt
= ++i
;
1764 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif
->tx
, notify
);
1766 notify_remote_via_irq(vif
->irq
);
1769 static struct xen_netif_rx_response
*make_rx_response(struct xenvif
*vif
,
1776 RING_IDX i
= vif
->rx
.rsp_prod_pvt
;
1777 struct xen_netif_rx_response
*resp
;
1779 resp
= RING_GET_RESPONSE(&vif
->rx
, i
);
1780 resp
->offset
= offset
;
1781 resp
->flags
= flags
;
1783 resp
->status
= (s16
)size
;
1785 resp
->status
= (s16
)st
;
1787 vif
->rx
.rsp_prod_pvt
= ++i
;
1792 static inline int rx_work_todo(struct xen_netbk
*netbk
)
1794 return !skb_queue_empty(&netbk
->rx_queue
);
1797 static inline int tx_work_todo(struct xen_netbk
*netbk
)
1800 if ((nr_pending_reqs(netbk
) + XEN_NETBK_LEGACY_SLOTS_MAX
1801 < MAX_PENDING_REQS
) &&
1802 !list_empty(&netbk
->net_schedule_list
))
1808 static int xen_netbk_kthread(void *data
)
1810 struct xen_netbk
*netbk
= data
;
1811 while (!kthread_should_stop()) {
1812 wait_event_interruptible(netbk
->wq
,
1813 rx_work_todo(netbk
) ||
1814 tx_work_todo(netbk
) ||
1815 kthread_should_stop());
1818 if (kthread_should_stop())
1821 if (rx_work_todo(netbk
))
1822 xen_netbk_rx_action(netbk
);
1824 if (tx_work_todo(netbk
))
1825 xen_netbk_tx_action(netbk
);
1831 void xen_netbk_unmap_frontend_rings(struct xenvif
*vif
)
1834 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif
),
1837 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif
),
1841 int xen_netbk_map_frontend_rings(struct xenvif
*vif
,
1842 grant_ref_t tx_ring_ref
,
1843 grant_ref_t rx_ring_ref
)
1846 struct xen_netif_tx_sring
*txs
;
1847 struct xen_netif_rx_sring
*rxs
;
1851 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif
),
1852 tx_ring_ref
, &addr
);
1856 txs
= (struct xen_netif_tx_sring
*)addr
;
1857 BACK_RING_INIT(&vif
->tx
, txs
, PAGE_SIZE
);
1859 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif
),
1860 rx_ring_ref
, &addr
);
1864 rxs
= (struct xen_netif_rx_sring
*)addr
;
1865 BACK_RING_INIT(&vif
->rx
, rxs
, PAGE_SIZE
);
1867 vif
->rx_req_cons_peek
= 0;
1872 xen_netbk_unmap_frontend_rings(vif
);
1876 static int __init
netback_init(void)
1885 if (fatal_skb_slots
< XEN_NETBK_LEGACY_SLOTS_MAX
) {
1887 "xen-netback: fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n",
1888 fatal_skb_slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
1889 fatal_skb_slots
= XEN_NETBK_LEGACY_SLOTS_MAX
;
1892 xen_netbk_group_nr
= num_online_cpus();
1893 xen_netbk
= vzalloc(sizeof(struct xen_netbk
) * xen_netbk_group_nr
);
1897 for (group
= 0; group
< xen_netbk_group_nr
; group
++) {
1898 struct xen_netbk
*netbk
= &xen_netbk
[group
];
1899 skb_queue_head_init(&netbk
->rx_queue
);
1900 skb_queue_head_init(&netbk
->tx_queue
);
1902 init_timer(&netbk
->net_timer
);
1903 netbk
->net_timer
.data
= (unsigned long)netbk
;
1904 netbk
->net_timer
.function
= xen_netbk_alarm
;
1906 netbk
->pending_cons
= 0;
1907 netbk
->pending_prod
= MAX_PENDING_REQS
;
1908 for (i
= 0; i
< MAX_PENDING_REQS
; i
++)
1909 netbk
->pending_ring
[i
] = i
;
1911 init_waitqueue_head(&netbk
->wq
);
1912 netbk
->task
= kthread_create(xen_netbk_kthread
,
1914 "netback/%u", group
);
1916 if (IS_ERR(netbk
->task
)) {
1917 printk(KERN_ALERT
"kthread_create() fails at netback\n");
1918 del_timer(&netbk
->net_timer
);
1919 rc
= PTR_ERR(netbk
->task
);
1923 kthread_bind(netbk
->task
, group
);
1925 INIT_LIST_HEAD(&netbk
->net_schedule_list
);
1927 spin_lock_init(&netbk
->net_schedule_list_lock
);
1929 atomic_set(&netbk
->netfront_count
, 0);
1931 wake_up_process(netbk
->task
);
1934 rc
= xenvif_xenbus_init();
1941 while (--group
>= 0) {
1942 struct xen_netbk
*netbk
= &xen_netbk
[group
];
1943 for (i
= 0; i
< MAX_PENDING_REQS
; i
++) {
1944 if (netbk
->mmap_pages
[i
])
1945 __free_page(netbk
->mmap_pages
[i
]);
1947 del_timer(&netbk
->net_timer
);
1948 kthread_stop(netbk
->task
);
1955 module_init(netback_init
);
1957 MODULE_LICENSE("Dual BSD/GPL");
1958 MODULE_ALIAS("xen-backend:vif");