2 * Back-end of the driver for virtual network devices. This portion of the
3 * driver exports a 'unified' network-device interface that can be accessed
4 * by any operating system that implements a compatible front end. A
5 * reference front-end implementation can be found in:
6 * drivers/net/xen-netfront.c
8 * Copyright (c) 2002-2005, K A Fraser
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
37 #include <linux/kthread.h>
38 #include <linux/if_vlan.h>
39 #include <linux/udp.h>
43 #include <xen/events.h>
44 #include <xen/interface/memory.h>
46 #include <asm/xen/hypercall.h>
47 #include <asm/xen/page.h>
49 struct pending_tx_info
{
50 struct xen_netif_tx_request req
;
53 typedef unsigned int pending_ring_idx_t
;
55 struct netbk_rx_meta
{
61 #define MAX_PENDING_REQS 256
63 /* Discriminate from any valid pending_idx value. */
64 #define INVALID_PENDING_IDX 0xFFFF
66 #define MAX_BUFFER_OFFSET PAGE_SIZE
68 /* extra field used in struct page */
71 #if BITS_PER_LONG < 64
73 #define GROUP_WIDTH (BITS_PER_LONG - IDX_WIDTH)
74 unsigned int group
:GROUP_WIDTH
;
75 unsigned int idx
:IDX_WIDTH
;
77 unsigned int group
, idx
;
85 struct task_struct
*task
;
87 struct sk_buff_head rx_queue
;
88 struct sk_buff_head tx_queue
;
90 struct timer_list net_timer
;
92 struct page
*mmap_pages
[MAX_PENDING_REQS
];
94 pending_ring_idx_t pending_prod
;
95 pending_ring_idx_t pending_cons
;
96 struct list_head net_schedule_list
;
98 /* Protect the net_schedule_list in netif. */
99 spinlock_t net_schedule_list_lock
;
101 atomic_t netfront_count
;
103 struct pending_tx_info pending_tx_info
[MAX_PENDING_REQS
];
104 struct gnttab_copy tx_copy_ops
[MAX_PENDING_REQS
];
106 u16 pending_ring
[MAX_PENDING_REQS
];
109 * Given MAX_BUFFER_OFFSET of 4096 the worst case is that each
110 * head/fragment page uses 2 copy operations because it
111 * straddles two buffers in the frontend.
113 struct gnttab_copy grant_copy_op
[2*XEN_NETIF_RX_RING_SIZE
];
114 struct netbk_rx_meta meta
[2*XEN_NETIF_RX_RING_SIZE
];
117 static struct xen_netbk
*xen_netbk
;
118 static int xen_netbk_group_nr
;
120 void xen_netbk_add_xenvif(struct xenvif
*vif
)
123 int min_netfront_count
;
125 struct xen_netbk
*netbk
;
127 min_netfront_count
= atomic_read(&xen_netbk
[0].netfront_count
);
128 for (i
= 0; i
< xen_netbk_group_nr
; i
++) {
129 int netfront_count
= atomic_read(&xen_netbk
[i
].netfront_count
);
130 if (netfront_count
< min_netfront_count
) {
132 min_netfront_count
= netfront_count
;
136 netbk
= &xen_netbk
[min_group
];
139 atomic_inc(&netbk
->netfront_count
);
142 void xen_netbk_remove_xenvif(struct xenvif
*vif
)
144 struct xen_netbk
*netbk
= vif
->netbk
;
146 atomic_dec(&netbk
->netfront_count
);
149 static void xen_netbk_idx_release(struct xen_netbk
*netbk
, u16 pending_idx
);
150 static void make_tx_response(struct xenvif
*vif
,
151 struct xen_netif_tx_request
*txp
,
153 static struct xen_netif_rx_response
*make_rx_response(struct xenvif
*vif
,
160 static inline unsigned long idx_to_pfn(struct xen_netbk
*netbk
,
163 return page_to_pfn(netbk
->mmap_pages
[idx
]);
166 static inline unsigned long idx_to_kaddr(struct xen_netbk
*netbk
,
169 return (unsigned long)pfn_to_kaddr(idx_to_pfn(netbk
, idx
));
172 /* extra field used in struct page */
173 static inline void set_page_ext(struct page
*pg
, struct xen_netbk
*netbk
,
176 unsigned int group
= netbk
- xen_netbk
;
177 union page_ext ext
= { .e
= { .group
= group
+ 1, .idx
= idx
} };
179 BUILD_BUG_ON(sizeof(ext
) > sizeof(ext
.mapping
));
180 pg
->mapping
= ext
.mapping
;
183 static int get_page_ext(struct page
*pg
,
184 unsigned int *pgroup
, unsigned int *pidx
)
186 union page_ext ext
= { .mapping
= pg
->mapping
};
187 struct xen_netbk
*netbk
;
188 unsigned int group
, idx
;
190 group
= ext
.e
.group
- 1;
192 if (group
< 0 || group
>= xen_netbk_group_nr
)
195 netbk
= &xen_netbk
[group
];
199 if ((idx
< 0) || (idx
>= MAX_PENDING_REQS
))
202 if (netbk
->mmap_pages
[idx
] != pg
)
212 * This is the amount of packet we copy rather than map, so that the
213 * guest can't fiddle with the contents of the headers while we do
214 * packet processing on them (netfilter, routing, etc).
216 #define PKT_PROT_LEN (ETH_HLEN + \
218 sizeof(struct iphdr) + MAX_IPOPTLEN + \
219 sizeof(struct tcphdr) + MAX_TCP_OPTION_SPACE)
221 static u16
frag_get_pending_idx(skb_frag_t
*frag
)
223 return (u16
)frag
->page_offset
;
226 static void frag_set_pending_idx(skb_frag_t
*frag
, u16 pending_idx
)
228 frag
->page_offset
= pending_idx
;
231 static inline pending_ring_idx_t
pending_index(unsigned i
)
233 return i
& (MAX_PENDING_REQS
-1);
236 static inline pending_ring_idx_t
nr_pending_reqs(struct xen_netbk
*netbk
)
238 return MAX_PENDING_REQS
-
239 netbk
->pending_prod
+ netbk
->pending_cons
;
242 static void xen_netbk_kick_thread(struct xen_netbk
*netbk
)
247 static int max_required_rx_slots(struct xenvif
*vif
)
249 int max
= DIV_ROUND_UP(vif
->dev
->mtu
, PAGE_SIZE
);
251 if (vif
->can_sg
|| vif
->gso
|| vif
->gso_prefix
)
252 max
+= MAX_SKB_FRAGS
+ 1; /* extra_info + frags */
257 int xen_netbk_rx_ring_full(struct xenvif
*vif
)
259 RING_IDX peek
= vif
->rx_req_cons_peek
;
260 RING_IDX needed
= max_required_rx_slots(vif
);
262 return ((vif
->rx
.sring
->req_prod
- peek
) < needed
) ||
263 ((vif
->rx
.rsp_prod_pvt
+ XEN_NETIF_RX_RING_SIZE
- peek
) < needed
);
266 int xen_netbk_must_stop_queue(struct xenvif
*vif
)
268 if (!xen_netbk_rx_ring_full(vif
))
271 vif
->rx
.sring
->req_event
= vif
->rx_req_cons_peek
+
272 max_required_rx_slots(vif
);
273 mb(); /* request notification /then/ check the queue */
275 return xen_netbk_rx_ring_full(vif
);
279 * Returns true if we should start a new receive buffer instead of
280 * adding 'size' bytes to a buffer which currently contains 'offset'
283 static bool start_new_rx_buffer(int offset
, unsigned long size
, int head
)
285 /* simple case: we have completely filled the current buffer. */
286 if (offset
== MAX_BUFFER_OFFSET
)
290 * complex case: start a fresh buffer if the current frag
291 * would overflow the current buffer but only if:
292 * (i) this frag would fit completely in the next buffer
293 * and (ii) there is already some data in the current buffer
294 * and (iii) this is not the head buffer.
297 * - (i) stops us splitting a frag into two copies
298 * unless the frag is too large for a single buffer.
299 * - (ii) stops us from leaving a buffer pointlessly empty.
300 * - (iii) stops us leaving the first buffer
301 * empty. Strictly speaking this is already covered
302 * by (ii) but is explicitly checked because
303 * netfront relies on the first buffer being
304 * non-empty and can crash otherwise.
306 * This means we will effectively linearise small
307 * frags but do not needlessly split large buffers
308 * into multiple copies tend to give large frags their
309 * own buffers as before.
311 if ((offset
+ size
> MAX_BUFFER_OFFSET
) &&
312 (size
<= MAX_BUFFER_OFFSET
) && offset
&& !head
)
319 * Figure out how many ring slots we're going to need to send @skb to
320 * the guest. This function is essentially a dry run of
321 * netbk_gop_frag_copy.
323 unsigned int xen_netbk_count_skb_slots(struct xenvif
*vif
, struct sk_buff
*skb
)
328 count
= DIV_ROUND_UP(skb_headlen(skb
), PAGE_SIZE
);
330 copy_off
= skb_headlen(skb
) % PAGE_SIZE
;
332 if (skb_shinfo(skb
)->gso_size
)
335 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
336 unsigned long size
= skb_frag_size(&skb_shinfo(skb
)->frags
[i
]);
339 BUG_ON(copy_off
> MAX_BUFFER_OFFSET
);
341 if (start_new_rx_buffer(copy_off
, size
, 0)) {
347 if (copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
348 bytes
= MAX_BUFFER_OFFSET
- copy_off
;
357 struct netrx_pending_operations
{
358 unsigned copy_prod
, copy_cons
;
359 unsigned meta_prod
, meta_cons
;
360 struct gnttab_copy
*copy
;
361 struct netbk_rx_meta
*meta
;
363 grant_ref_t copy_gref
;
366 static struct netbk_rx_meta
*get_next_rx_buffer(struct xenvif
*vif
,
367 struct netrx_pending_operations
*npo
)
369 struct netbk_rx_meta
*meta
;
370 struct xen_netif_rx_request
*req
;
372 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
374 meta
= npo
->meta
+ npo
->meta_prod
++;
380 npo
->copy_gref
= req
->gref
;
386 * Set up the grant operations for this fragment. If it's a flipping
387 * interface, we also set up the unmap request from here.
389 static void netbk_gop_frag_copy(struct xenvif
*vif
, struct sk_buff
*skb
,
390 struct netrx_pending_operations
*npo
,
391 struct page
*page
, unsigned long size
,
392 unsigned long offset
, int *head
)
394 struct gnttab_copy
*copy_gop
;
395 struct netbk_rx_meta
*meta
;
397 * These variables are used iff get_page_ext returns true,
398 * in which case they are guaranteed to be initialized.
400 unsigned int uninitialized_var(group
), uninitialized_var(idx
);
401 int foreign
= get_page_ext(page
, &group
, &idx
);
404 /* Data must not cross a page boundary. */
405 BUG_ON(size
+ offset
> PAGE_SIZE
);
407 meta
= npo
->meta
+ npo
->meta_prod
- 1;
410 BUG_ON(npo
->copy_off
> MAX_BUFFER_OFFSET
);
412 if (start_new_rx_buffer(npo
->copy_off
, size
, *head
)) {
414 * Netfront requires there to be some data in the head
419 meta
= get_next_rx_buffer(vif
, npo
);
423 if (npo
->copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
424 bytes
= MAX_BUFFER_OFFSET
- npo
->copy_off
;
426 copy_gop
= npo
->copy
+ npo
->copy_prod
++;
427 copy_gop
->flags
= GNTCOPY_dest_gref
;
429 struct xen_netbk
*netbk
= &xen_netbk
[group
];
430 struct pending_tx_info
*src_pend
;
432 src_pend
= &netbk
->pending_tx_info
[idx
];
434 copy_gop
->source
.domid
= src_pend
->vif
->domid
;
435 copy_gop
->source
.u
.ref
= src_pend
->req
.gref
;
436 copy_gop
->flags
|= GNTCOPY_source_gref
;
438 void *vaddr
= page_address(page
);
439 copy_gop
->source
.domid
= DOMID_SELF
;
440 copy_gop
->source
.u
.gmfn
= virt_to_mfn(vaddr
);
442 copy_gop
->source
.offset
= offset
;
443 copy_gop
->dest
.domid
= vif
->domid
;
445 copy_gop
->dest
.offset
= npo
->copy_off
;
446 copy_gop
->dest
.u
.ref
= npo
->copy_gref
;
447 copy_gop
->len
= bytes
;
449 npo
->copy_off
+= bytes
;
455 /* Leave a gap for the GSO descriptor. */
456 if (*head
&& skb_shinfo(skb
)->gso_size
&& !vif
->gso_prefix
)
459 *head
= 0; /* There must be something in this buffer now. */
465 * Prepare an SKB to be transmitted to the frontend.
467 * This function is responsible for allocating grant operations, meta
470 * It returns the number of meta structures consumed. The number of
471 * ring slots used is always equal to the number of meta slots used
472 * plus the number of GSO descriptors used. Currently, we use either
473 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
474 * frontend-side LRO).
476 static int netbk_gop_skb(struct sk_buff
*skb
,
477 struct netrx_pending_operations
*npo
)
479 struct xenvif
*vif
= netdev_priv(skb
->dev
);
480 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
482 struct xen_netif_rx_request
*req
;
483 struct netbk_rx_meta
*meta
;
488 old_meta_prod
= npo
->meta_prod
;
490 /* Set up a GSO prefix descriptor, if necessary */
491 if (skb_shinfo(skb
)->gso_size
&& vif
->gso_prefix
) {
492 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
493 meta
= npo
->meta
+ npo
->meta_prod
++;
494 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
499 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
500 meta
= npo
->meta
+ npo
->meta_prod
++;
502 if (!vif
->gso_prefix
)
503 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
510 npo
->copy_gref
= req
->gref
;
513 while (data
< skb_tail_pointer(skb
)) {
514 unsigned int offset
= offset_in_page(data
);
515 unsigned int len
= PAGE_SIZE
- offset
;
517 if (data
+ len
> skb_tail_pointer(skb
))
518 len
= skb_tail_pointer(skb
) - data
;
520 netbk_gop_frag_copy(vif
, skb
, npo
,
521 virt_to_page(data
), len
, offset
, &head
);
525 for (i
= 0; i
< nr_frags
; i
++) {
526 netbk_gop_frag_copy(vif
, skb
, npo
,
527 skb_frag_page(&skb_shinfo(skb
)->frags
[i
]),
528 skb_frag_size(&skb_shinfo(skb
)->frags
[i
]),
529 skb_shinfo(skb
)->frags
[i
].page_offset
,
533 return npo
->meta_prod
- old_meta_prod
;
537 * This is a twin to netbk_gop_skb. Assume that netbk_gop_skb was
538 * used to set up the operations on the top of
539 * netrx_pending_operations, which have since been done. Check that
540 * they didn't give any errors and advance over them.
542 static int netbk_check_gop(struct xenvif
*vif
, int nr_meta_slots
,
543 struct netrx_pending_operations
*npo
)
545 struct gnttab_copy
*copy_op
;
546 int status
= XEN_NETIF_RSP_OKAY
;
549 for (i
= 0; i
< nr_meta_slots
; i
++) {
550 copy_op
= npo
->copy
+ npo
->copy_cons
++;
551 if (copy_op
->status
!= GNTST_okay
) {
553 "Bad status %d from copy to DOM%d.\n",
554 copy_op
->status
, vif
->domid
);
555 status
= XEN_NETIF_RSP_ERROR
;
562 static void netbk_add_frag_responses(struct xenvif
*vif
, int status
,
563 struct netbk_rx_meta
*meta
,
567 unsigned long offset
;
569 /* No fragments used */
570 if (nr_meta_slots
<= 1)
575 for (i
= 0; i
< nr_meta_slots
; i
++) {
577 if (i
== nr_meta_slots
- 1)
580 flags
= XEN_NETRXF_more_data
;
583 make_rx_response(vif
, meta
[i
].id
, status
, offset
,
584 meta
[i
].size
, flags
);
588 struct skb_cb_overlay
{
592 static void xen_netbk_rx_action(struct xen_netbk
*netbk
)
594 struct xenvif
*vif
= NULL
, *tmp
;
597 struct xen_netif_rx_response
*resp
;
598 struct sk_buff_head rxq
;
604 unsigned long offset
;
605 struct skb_cb_overlay
*sco
;
607 struct netrx_pending_operations npo
= {
608 .copy
= netbk
->grant_copy_op
,
612 skb_queue_head_init(&rxq
);
616 while ((skb
= skb_dequeue(&netbk
->rx_queue
)) != NULL
) {
617 vif
= netdev_priv(skb
->dev
);
618 nr_frags
= skb_shinfo(skb
)->nr_frags
;
620 sco
= (struct skb_cb_overlay
*)skb
->cb
;
621 sco
->meta_slots_used
= netbk_gop_skb(skb
, &npo
);
623 count
+= nr_frags
+ 1;
625 __skb_queue_tail(&rxq
, skb
);
627 /* Filled the batch queue? */
628 if (count
+ MAX_SKB_FRAGS
>= XEN_NETIF_RX_RING_SIZE
)
632 BUG_ON(npo
.meta_prod
> ARRAY_SIZE(netbk
->meta
));
637 BUG_ON(npo
.copy_prod
> ARRAY_SIZE(netbk
->grant_copy_op
));
638 ret
= HYPERVISOR_grant_table_op(GNTTABOP_copy
, &netbk
->grant_copy_op
,
642 while ((skb
= __skb_dequeue(&rxq
)) != NULL
) {
643 sco
= (struct skb_cb_overlay
*)skb
->cb
;
645 vif
= netdev_priv(skb
->dev
);
647 if (netbk
->meta
[npo
.meta_cons
].gso_size
&& vif
->gso_prefix
) {
648 resp
= RING_GET_RESPONSE(&vif
->rx
,
649 vif
->rx
.rsp_prod_pvt
++);
651 resp
->flags
= XEN_NETRXF_gso_prefix
| XEN_NETRXF_more_data
;
653 resp
->offset
= netbk
->meta
[npo
.meta_cons
].gso_size
;
654 resp
->id
= netbk
->meta
[npo
.meta_cons
].id
;
655 resp
->status
= sco
->meta_slots_used
;
658 sco
->meta_slots_used
--;
662 vif
->dev
->stats
.tx_bytes
+= skb
->len
;
663 vif
->dev
->stats
.tx_packets
++;
665 status
= netbk_check_gop(vif
, sco
->meta_slots_used
, &npo
);
667 if (sco
->meta_slots_used
== 1)
670 flags
= XEN_NETRXF_more_data
;
672 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) /* local packet? */
673 flags
|= XEN_NETRXF_csum_blank
| XEN_NETRXF_data_validated
;
674 else if (skb
->ip_summed
== CHECKSUM_UNNECESSARY
)
675 /* remote but checksummed. */
676 flags
|= XEN_NETRXF_data_validated
;
679 resp
= make_rx_response(vif
, netbk
->meta
[npo
.meta_cons
].id
,
681 netbk
->meta
[npo
.meta_cons
].size
,
684 if (netbk
->meta
[npo
.meta_cons
].gso_size
&& !vif
->gso_prefix
) {
685 struct xen_netif_extra_info
*gso
=
686 (struct xen_netif_extra_info
*)
687 RING_GET_RESPONSE(&vif
->rx
,
688 vif
->rx
.rsp_prod_pvt
++);
690 resp
->flags
|= XEN_NETRXF_extra_info
;
692 gso
->u
.gso
.size
= netbk
->meta
[npo
.meta_cons
].gso_size
;
693 gso
->u
.gso
.type
= XEN_NETIF_GSO_TYPE_TCPV4
;
695 gso
->u
.gso
.features
= 0;
697 gso
->type
= XEN_NETIF_EXTRA_TYPE_GSO
;
701 netbk_add_frag_responses(vif
, status
,
702 netbk
->meta
+ npo
.meta_cons
+ 1,
703 sco
->meta_slots_used
);
705 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif
->rx
, ret
);
707 if (ret
&& list_empty(&vif
->notify_list
))
708 list_add_tail(&vif
->notify_list
, ¬ify
);
710 xenvif_notify_tx_completion(vif
);
713 npo
.meta_cons
+= sco
->meta_slots_used
;
717 list_for_each_entry_safe(vif
, tmp
, ¬ify
, notify_list
) {
718 notify_remote_via_irq(vif
->irq
);
719 list_del_init(&vif
->notify_list
);
722 /* More work to do? */
723 if (!skb_queue_empty(&netbk
->rx_queue
) &&
724 !timer_pending(&netbk
->net_timer
))
725 xen_netbk_kick_thread(netbk
);
728 void xen_netbk_queue_tx_skb(struct xenvif
*vif
, struct sk_buff
*skb
)
730 struct xen_netbk
*netbk
= vif
->netbk
;
732 skb_queue_tail(&netbk
->rx_queue
, skb
);
734 xen_netbk_kick_thread(netbk
);
737 static void xen_netbk_alarm(unsigned long data
)
739 struct xen_netbk
*netbk
= (struct xen_netbk
*)data
;
740 xen_netbk_kick_thread(netbk
);
743 static int __on_net_schedule_list(struct xenvif
*vif
)
745 return !list_empty(&vif
->schedule_list
);
748 /* Must be called with net_schedule_list_lock held */
749 static void remove_from_net_schedule_list(struct xenvif
*vif
)
751 if (likely(__on_net_schedule_list(vif
))) {
752 list_del_init(&vif
->schedule_list
);
757 static struct xenvif
*poll_net_schedule_list(struct xen_netbk
*netbk
)
759 struct xenvif
*vif
= NULL
;
761 spin_lock_irq(&netbk
->net_schedule_list_lock
);
762 if (list_empty(&netbk
->net_schedule_list
))
765 vif
= list_first_entry(&netbk
->net_schedule_list
,
766 struct xenvif
, schedule_list
);
772 remove_from_net_schedule_list(vif
);
774 spin_unlock_irq(&netbk
->net_schedule_list_lock
);
778 void xen_netbk_schedule_xenvif(struct xenvif
*vif
)
781 struct xen_netbk
*netbk
= vif
->netbk
;
783 if (__on_net_schedule_list(vif
))
786 spin_lock_irqsave(&netbk
->net_schedule_list_lock
, flags
);
787 if (!__on_net_schedule_list(vif
) &&
788 likely(xenvif_schedulable(vif
))) {
789 list_add_tail(&vif
->schedule_list
, &netbk
->net_schedule_list
);
792 spin_unlock_irqrestore(&netbk
->net_schedule_list_lock
, flags
);
796 if ((nr_pending_reqs(netbk
) < (MAX_PENDING_REQS
/2)) &&
797 !list_empty(&netbk
->net_schedule_list
))
798 xen_netbk_kick_thread(netbk
);
801 void xen_netbk_deschedule_xenvif(struct xenvif
*vif
)
803 struct xen_netbk
*netbk
= vif
->netbk
;
804 spin_lock_irq(&netbk
->net_schedule_list_lock
);
805 remove_from_net_schedule_list(vif
);
806 spin_unlock_irq(&netbk
->net_schedule_list_lock
);
809 void xen_netbk_check_rx_xenvif(struct xenvif
*vif
)
813 RING_FINAL_CHECK_FOR_REQUESTS(&vif
->tx
, more_to_do
);
816 xen_netbk_schedule_xenvif(vif
);
819 static void tx_add_credit(struct xenvif
*vif
)
821 unsigned long max_burst
, max_credit
;
824 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
825 * Otherwise the interface can seize up due to insufficient credit.
827 max_burst
= RING_GET_REQUEST(&vif
->tx
, vif
->tx
.req_cons
)->size
;
828 max_burst
= min(max_burst
, 131072UL);
829 max_burst
= max(max_burst
, vif
->credit_bytes
);
831 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
832 max_credit
= vif
->remaining_credit
+ vif
->credit_bytes
;
833 if (max_credit
< vif
->remaining_credit
)
834 max_credit
= ULONG_MAX
; /* wrapped: clamp to ULONG_MAX */
836 vif
->remaining_credit
= min(max_credit
, max_burst
);
839 static void tx_credit_callback(unsigned long data
)
841 struct xenvif
*vif
= (struct xenvif
*)data
;
843 xen_netbk_check_rx_xenvif(vif
);
846 static void netbk_tx_err(struct xenvif
*vif
,
847 struct xen_netif_tx_request
*txp
, RING_IDX end
)
849 RING_IDX cons
= vif
->tx
.req_cons
;
852 make_tx_response(vif
, txp
, XEN_NETIF_RSP_ERROR
);
855 txp
= RING_GET_REQUEST(&vif
->tx
, cons
++);
857 vif
->tx
.req_cons
= cons
;
858 xen_netbk_check_rx_xenvif(vif
);
862 static int netbk_count_requests(struct xenvif
*vif
,
863 struct xen_netif_tx_request
*first
,
864 struct xen_netif_tx_request
*txp
,
867 RING_IDX cons
= vif
->tx
.req_cons
;
870 if (!(first
->flags
& XEN_NETTXF_more_data
))
874 if (frags
>= work_to_do
) {
875 netdev_dbg(vif
->dev
, "Need more frags\n");
879 if (unlikely(frags
>= MAX_SKB_FRAGS
)) {
880 netdev_dbg(vif
->dev
, "Too many frags\n");
884 memcpy(txp
, RING_GET_REQUEST(&vif
->tx
, cons
+ frags
),
886 if (txp
->size
> first
->size
) {
887 netdev_dbg(vif
->dev
, "Frags galore\n");
891 first
->size
-= txp
->size
;
894 if (unlikely((txp
->offset
+ txp
->size
) > PAGE_SIZE
)) {
895 netdev_dbg(vif
->dev
, "txp->offset: %x, size: %u\n",
896 txp
->offset
, txp
->size
);
899 } while ((txp
++)->flags
& XEN_NETTXF_more_data
);
903 static struct page
*xen_netbk_alloc_page(struct xen_netbk
*netbk
,
908 page
= alloc_page(GFP_KERNEL
|__GFP_COLD
);
911 set_page_ext(page
, netbk
, pending_idx
);
912 netbk
->mmap_pages
[pending_idx
] = page
;
916 static struct gnttab_copy
*xen_netbk_get_requests(struct xen_netbk
*netbk
,
919 struct xen_netif_tx_request
*txp
,
920 struct gnttab_copy
*gop
)
922 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
923 skb_frag_t
*frags
= shinfo
->frags
;
924 u16 pending_idx
= *((u16
*)skb
->data
);
927 /* Skip first skb fragment if it is on same page as header fragment. */
928 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
930 for (i
= start
; i
< shinfo
->nr_frags
; i
++, txp
++) {
932 pending_ring_idx_t index
;
933 struct pending_tx_info
*pending_tx_info
=
934 netbk
->pending_tx_info
;
936 index
= pending_index(netbk
->pending_cons
++);
937 pending_idx
= netbk
->pending_ring
[index
];
938 page
= xen_netbk_alloc_page(netbk
, skb
, pending_idx
);
942 gop
->source
.u
.ref
= txp
->gref
;
943 gop
->source
.domid
= vif
->domid
;
944 gop
->source
.offset
= txp
->offset
;
946 gop
->dest
.u
.gmfn
= virt_to_mfn(page_address(page
));
947 gop
->dest
.domid
= DOMID_SELF
;
948 gop
->dest
.offset
= txp
->offset
;
950 gop
->len
= txp
->size
;
951 gop
->flags
= GNTCOPY_source_gref
;
955 memcpy(&pending_tx_info
[pending_idx
].req
, txp
, sizeof(*txp
));
957 pending_tx_info
[pending_idx
].vif
= vif
;
958 frag_set_pending_idx(&frags
[i
], pending_idx
);
964 static int xen_netbk_tx_check_gop(struct xen_netbk
*netbk
,
966 struct gnttab_copy
**gopp
)
968 struct gnttab_copy
*gop
= *gopp
;
969 u16 pending_idx
= *((u16
*)skb
->data
);
970 struct pending_tx_info
*pending_tx_info
= netbk
->pending_tx_info
;
971 struct xenvif
*vif
= pending_tx_info
[pending_idx
].vif
;
972 struct xen_netif_tx_request
*txp
;
973 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
974 int nr_frags
= shinfo
->nr_frags
;
977 /* Check status of header. */
980 pending_ring_idx_t index
;
981 index
= pending_index(netbk
->pending_prod
++);
982 txp
= &pending_tx_info
[pending_idx
].req
;
983 make_tx_response(vif
, txp
, XEN_NETIF_RSP_ERROR
);
984 netbk
->pending_ring
[index
] = pending_idx
;
988 /* Skip first skb fragment if it is on same page as header fragment. */
989 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
991 for (i
= start
; i
< nr_frags
; i
++) {
993 pending_ring_idx_t index
;
995 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[i
]);
997 /* Check error status: if okay then remember grant handle. */
998 newerr
= (++gop
)->status
;
999 if (likely(!newerr
)) {
1000 /* Had a previous error? Invalidate this fragment. */
1002 xen_netbk_idx_release(netbk
, pending_idx
);
1006 /* Error on this fragment: respond to client with an error. */
1007 txp
= &netbk
->pending_tx_info
[pending_idx
].req
;
1008 make_tx_response(vif
, txp
, XEN_NETIF_RSP_ERROR
);
1009 index
= pending_index(netbk
->pending_prod
++);
1010 netbk
->pending_ring
[index
] = pending_idx
;
1013 /* Not the first error? Preceding frags already invalidated. */
1017 /* First error: invalidate header and preceding fragments. */
1018 pending_idx
= *((u16
*)skb
->data
);
1019 xen_netbk_idx_release(netbk
, pending_idx
);
1020 for (j
= start
; j
< i
; j
++) {
1021 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[j
]);
1022 xen_netbk_idx_release(netbk
, pending_idx
);
1025 /* Remember the error: invalidate all subsequent fragments. */
1033 static void xen_netbk_fill_frags(struct xen_netbk
*netbk
, struct sk_buff
*skb
)
1035 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1036 int nr_frags
= shinfo
->nr_frags
;
1039 for (i
= 0; i
< nr_frags
; i
++) {
1040 skb_frag_t
*frag
= shinfo
->frags
+ i
;
1041 struct xen_netif_tx_request
*txp
;
1045 pending_idx
= frag_get_pending_idx(frag
);
1047 txp
= &netbk
->pending_tx_info
[pending_idx
].req
;
1048 page
= virt_to_page(idx_to_kaddr(netbk
, pending_idx
));
1049 __skb_fill_page_desc(skb
, i
, page
, txp
->offset
, txp
->size
);
1050 skb
->len
+= txp
->size
;
1051 skb
->data_len
+= txp
->size
;
1052 skb
->truesize
+= txp
->size
;
1054 /* Take an extra reference to offset xen_netbk_idx_release */
1055 get_page(netbk
->mmap_pages
[pending_idx
]);
1056 xen_netbk_idx_release(netbk
, pending_idx
);
1060 static int xen_netbk_get_extras(struct xenvif
*vif
,
1061 struct xen_netif_extra_info
*extras
,
1064 struct xen_netif_extra_info extra
;
1065 RING_IDX cons
= vif
->tx
.req_cons
;
1068 if (unlikely(work_to_do
-- <= 0)) {
1069 netdev_dbg(vif
->dev
, "Missing extra info\n");
1073 memcpy(&extra
, RING_GET_REQUEST(&vif
->tx
, cons
),
1075 if (unlikely(!extra
.type
||
1076 extra
.type
>= XEN_NETIF_EXTRA_TYPE_MAX
)) {
1077 vif
->tx
.req_cons
= ++cons
;
1078 netdev_dbg(vif
->dev
,
1079 "Invalid extra type: %d\n", extra
.type
);
1083 memcpy(&extras
[extra
.type
- 1], &extra
, sizeof(extra
));
1084 vif
->tx
.req_cons
= ++cons
;
1085 } while (extra
.flags
& XEN_NETIF_EXTRA_FLAG_MORE
);
1090 static int netbk_set_skb_gso(struct xenvif
*vif
,
1091 struct sk_buff
*skb
,
1092 struct xen_netif_extra_info
*gso
)
1094 if (!gso
->u
.gso
.size
) {
1095 netdev_dbg(vif
->dev
, "GSO size must not be zero.\n");
1099 /* Currently only TCPv4 S.O. is supported. */
1100 if (gso
->u
.gso
.type
!= XEN_NETIF_GSO_TYPE_TCPV4
) {
1101 netdev_dbg(vif
->dev
, "Bad GSO type %d.\n", gso
->u
.gso
.type
);
1105 skb_shinfo(skb
)->gso_size
= gso
->u
.gso
.size
;
1106 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
1108 /* Header must be checked, and gso_segs computed. */
1109 skb_shinfo(skb
)->gso_type
|= SKB_GSO_DODGY
;
1110 skb_shinfo(skb
)->gso_segs
= 0;
1115 static int checksum_setup(struct xenvif
*vif
, struct sk_buff
*skb
)
1120 int recalculate_partial_csum
= 0;
1123 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1124 * peers can fail to set NETRXF_csum_blank when sending a GSO
1125 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1126 * recalculate the partial checksum.
1128 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
&& skb_is_gso(skb
)) {
1129 vif
->rx_gso_checksum_fixup
++;
1130 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1131 recalculate_partial_csum
= 1;
1134 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1135 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1138 if (skb
->protocol
!= htons(ETH_P_IP
))
1141 iph
= (void *)skb
->data
;
1142 th
= skb
->data
+ 4 * iph
->ihl
;
1143 if (th
>= skb_tail_pointer(skb
))
1146 skb
->csum_start
= th
- skb
->head
;
1147 switch (iph
->protocol
) {
1149 skb
->csum_offset
= offsetof(struct tcphdr
, check
);
1151 if (recalculate_partial_csum
) {
1152 struct tcphdr
*tcph
= (struct tcphdr
*)th
;
1153 tcph
->check
= ~csum_tcpudp_magic(iph
->saddr
, iph
->daddr
,
1154 skb
->len
- iph
->ihl
*4,
1159 skb
->csum_offset
= offsetof(struct udphdr
, check
);
1161 if (recalculate_partial_csum
) {
1162 struct udphdr
*udph
= (struct udphdr
*)th
;
1163 udph
->check
= ~csum_tcpudp_magic(iph
->saddr
, iph
->daddr
,
1164 skb
->len
- iph
->ihl
*4,
1169 if (net_ratelimit())
1170 netdev_err(vif
->dev
,
1171 "Attempting to checksum a non-TCP/UDP packet, dropping a protocol %d packet\n",
1176 if ((th
+ skb
->csum_offset
+ 2) > skb_tail_pointer(skb
))
1185 static bool tx_credit_exceeded(struct xenvif
*vif
, unsigned size
)
1187 unsigned long now
= jiffies
;
1188 unsigned long next_credit
=
1189 vif
->credit_timeout
.expires
+
1190 msecs_to_jiffies(vif
->credit_usec
/ 1000);
1192 /* Timer could already be pending in rare cases. */
1193 if (timer_pending(&vif
->credit_timeout
))
1196 /* Passed the point where we can replenish credit? */
1197 if (time_after_eq(now
, next_credit
)) {
1198 vif
->credit_timeout
.expires
= now
;
1202 /* Still too big to send right now? Set a callback. */
1203 if (size
> vif
->remaining_credit
) {
1204 vif
->credit_timeout
.data
=
1206 vif
->credit_timeout
.function
=
1208 mod_timer(&vif
->credit_timeout
,
1217 static unsigned xen_netbk_tx_build_gops(struct xen_netbk
*netbk
)
1219 struct gnttab_copy
*gop
= netbk
->tx_copy_ops
, *request_gop
;
1220 struct sk_buff
*skb
;
1223 while (((nr_pending_reqs(netbk
) + MAX_SKB_FRAGS
) < MAX_PENDING_REQS
) &&
1224 !list_empty(&netbk
->net_schedule_list
)) {
1226 struct xen_netif_tx_request txreq
;
1227 struct xen_netif_tx_request txfrags
[MAX_SKB_FRAGS
];
1229 struct xen_netif_extra_info extras
[XEN_NETIF_EXTRA_TYPE_MAX
-1];
1233 unsigned int data_len
;
1234 pending_ring_idx_t index
;
1236 /* Get a netif from the list with work to do. */
1237 vif
= poll_net_schedule_list(netbk
);
1241 RING_FINAL_CHECK_FOR_REQUESTS(&vif
->tx
, work_to_do
);
1247 idx
= vif
->tx
.req_cons
;
1248 rmb(); /* Ensure that we see the request before we copy it. */
1249 memcpy(&txreq
, RING_GET_REQUEST(&vif
->tx
, idx
), sizeof(txreq
));
1251 /* Credit-based scheduling. */
1252 if (txreq
.size
> vif
->remaining_credit
&&
1253 tx_credit_exceeded(vif
, txreq
.size
)) {
1258 vif
->remaining_credit
-= txreq
.size
;
1261 vif
->tx
.req_cons
= ++idx
;
1263 memset(extras
, 0, sizeof(extras
));
1264 if (txreq
.flags
& XEN_NETTXF_extra_info
) {
1265 work_to_do
= xen_netbk_get_extras(vif
, extras
,
1267 idx
= vif
->tx
.req_cons
;
1268 if (unlikely(work_to_do
< 0)) {
1269 netbk_tx_err(vif
, &txreq
, idx
);
1274 ret
= netbk_count_requests(vif
, &txreq
, txfrags
, work_to_do
);
1275 if (unlikely(ret
< 0)) {
1276 netbk_tx_err(vif
, &txreq
, idx
- ret
);
1281 if (unlikely(txreq
.size
< ETH_HLEN
)) {
1282 netdev_dbg(vif
->dev
,
1283 "Bad packet size: %d\n", txreq
.size
);
1284 netbk_tx_err(vif
, &txreq
, idx
);
1288 /* No crossing a page as the payload mustn't fragment. */
1289 if (unlikely((txreq
.offset
+ txreq
.size
) > PAGE_SIZE
)) {
1290 netdev_dbg(vif
->dev
,
1291 "txreq.offset: %x, size: %u, end: %lu\n",
1292 txreq
.offset
, txreq
.size
,
1293 (txreq
.offset
&~PAGE_MASK
) + txreq
.size
);
1294 netbk_tx_err(vif
, &txreq
, idx
);
1298 index
= pending_index(netbk
->pending_cons
);
1299 pending_idx
= netbk
->pending_ring
[index
];
1301 data_len
= (txreq
.size
> PKT_PROT_LEN
&&
1302 ret
< MAX_SKB_FRAGS
) ?
1303 PKT_PROT_LEN
: txreq
.size
;
1305 skb
= alloc_skb(data_len
+ NET_SKB_PAD
+ NET_IP_ALIGN
,
1306 GFP_ATOMIC
| __GFP_NOWARN
);
1307 if (unlikely(skb
== NULL
)) {
1308 netdev_dbg(vif
->dev
,
1309 "Can't allocate a skb in start_xmit.\n");
1310 netbk_tx_err(vif
, &txreq
, idx
);
1314 /* Packets passed to netif_rx() must have some headroom. */
1315 skb_reserve(skb
, NET_SKB_PAD
+ NET_IP_ALIGN
);
1317 if (extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1].type
) {
1318 struct xen_netif_extra_info
*gso
;
1319 gso
= &extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1];
1321 if (netbk_set_skb_gso(vif
, skb
, gso
)) {
1323 netbk_tx_err(vif
, &txreq
, idx
);
1328 /* XXX could copy straight to head */
1329 page
= xen_netbk_alloc_page(netbk
, skb
, pending_idx
);
1332 netbk_tx_err(vif
, &txreq
, idx
);
1336 gop
->source
.u
.ref
= txreq
.gref
;
1337 gop
->source
.domid
= vif
->domid
;
1338 gop
->source
.offset
= txreq
.offset
;
1340 gop
->dest
.u
.gmfn
= virt_to_mfn(page_address(page
));
1341 gop
->dest
.domid
= DOMID_SELF
;
1342 gop
->dest
.offset
= txreq
.offset
;
1344 gop
->len
= txreq
.size
;
1345 gop
->flags
= GNTCOPY_source_gref
;
1349 memcpy(&netbk
->pending_tx_info
[pending_idx
].req
,
1350 &txreq
, sizeof(txreq
));
1351 netbk
->pending_tx_info
[pending_idx
].vif
= vif
;
1352 *((u16
*)skb
->data
) = pending_idx
;
1354 __skb_put(skb
, data_len
);
1356 skb_shinfo(skb
)->nr_frags
= ret
;
1357 if (data_len
< txreq
.size
) {
1358 skb_shinfo(skb
)->nr_frags
++;
1359 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1362 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1363 INVALID_PENDING_IDX
);
1366 __skb_queue_tail(&netbk
->tx_queue
, skb
);
1368 netbk
->pending_cons
++;
1370 request_gop
= xen_netbk_get_requests(netbk
, vif
,
1372 if (request_gop
== NULL
) {
1374 netbk_tx_err(vif
, &txreq
, idx
);
1379 vif
->tx
.req_cons
= idx
;
1380 xen_netbk_check_rx_xenvif(vif
);
1382 if ((gop
-netbk
->tx_copy_ops
) >= ARRAY_SIZE(netbk
->tx_copy_ops
))
1386 return gop
- netbk
->tx_copy_ops
;
1389 static void xen_netbk_tx_submit(struct xen_netbk
*netbk
)
1391 struct gnttab_copy
*gop
= netbk
->tx_copy_ops
;
1392 struct sk_buff
*skb
;
1394 while ((skb
= __skb_dequeue(&netbk
->tx_queue
)) != NULL
) {
1395 struct xen_netif_tx_request
*txp
;
1400 pending_idx
= *((u16
*)skb
->data
);
1401 vif
= netbk
->pending_tx_info
[pending_idx
].vif
;
1402 txp
= &netbk
->pending_tx_info
[pending_idx
].req
;
1404 /* Check the remap error code. */
1405 if (unlikely(xen_netbk_tx_check_gop(netbk
, skb
, &gop
))) {
1406 netdev_dbg(vif
->dev
, "netback grant failed.\n");
1407 skb_shinfo(skb
)->nr_frags
= 0;
1412 data_len
= skb
->len
;
1414 (void *)(idx_to_kaddr(netbk
, pending_idx
)|txp
->offset
),
1416 if (data_len
< txp
->size
) {
1417 /* Append the packet payload as a fragment. */
1418 txp
->offset
+= data_len
;
1419 txp
->size
-= data_len
;
1421 /* Schedule a response immediately. */
1422 xen_netbk_idx_release(netbk
, pending_idx
);
1425 if (txp
->flags
& XEN_NETTXF_csum_blank
)
1426 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1427 else if (txp
->flags
& XEN_NETTXF_data_validated
)
1428 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1430 xen_netbk_fill_frags(netbk
, skb
);
1433 * If the initial fragment was < PKT_PROT_LEN then
1434 * pull through some bytes from the other fragments to
1435 * increase the linear region to PKT_PROT_LEN bytes.
1437 if (skb_headlen(skb
) < PKT_PROT_LEN
&& skb_is_nonlinear(skb
)) {
1438 int target
= min_t(int, skb
->len
, PKT_PROT_LEN
);
1439 __pskb_pull_tail(skb
, target
- skb_headlen(skb
));
1442 skb
->dev
= vif
->dev
;
1443 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1445 if (checksum_setup(vif
, skb
)) {
1446 netdev_dbg(vif
->dev
,
1447 "Can't setup checksum in net_tx_action\n");
1452 vif
->dev
->stats
.rx_bytes
+= skb
->len
;
1453 vif
->dev
->stats
.rx_packets
++;
1455 xenvif_receive_skb(vif
, skb
);
1459 /* Called after netfront has transmitted */
1460 static void xen_netbk_tx_action(struct xen_netbk
*netbk
)
1465 nr_gops
= xen_netbk_tx_build_gops(netbk
);
1469 ret
= HYPERVISOR_grant_table_op(GNTTABOP_copy
,
1470 netbk
->tx_copy_ops
, nr_gops
);
1473 xen_netbk_tx_submit(netbk
);
1477 static void xen_netbk_idx_release(struct xen_netbk
*netbk
, u16 pending_idx
)
1480 struct pending_tx_info
*pending_tx_info
;
1481 pending_ring_idx_t index
;
1483 /* Already complete? */
1484 if (netbk
->mmap_pages
[pending_idx
] == NULL
)
1487 pending_tx_info
= &netbk
->pending_tx_info
[pending_idx
];
1489 vif
= pending_tx_info
->vif
;
1491 make_tx_response(vif
, &pending_tx_info
->req
, XEN_NETIF_RSP_OKAY
);
1493 index
= pending_index(netbk
->pending_prod
++);
1494 netbk
->pending_ring
[index
] = pending_idx
;
1498 netbk
->mmap_pages
[pending_idx
]->mapping
= 0;
1499 put_page(netbk
->mmap_pages
[pending_idx
]);
1500 netbk
->mmap_pages
[pending_idx
] = NULL
;
1503 static void make_tx_response(struct xenvif
*vif
,
1504 struct xen_netif_tx_request
*txp
,
1507 RING_IDX i
= vif
->tx
.rsp_prod_pvt
;
1508 struct xen_netif_tx_response
*resp
;
1511 resp
= RING_GET_RESPONSE(&vif
->tx
, i
);
1515 if (txp
->flags
& XEN_NETTXF_extra_info
)
1516 RING_GET_RESPONSE(&vif
->tx
, ++i
)->status
= XEN_NETIF_RSP_NULL
;
1518 vif
->tx
.rsp_prod_pvt
= ++i
;
1519 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif
->tx
, notify
);
1521 notify_remote_via_irq(vif
->irq
);
1524 static struct xen_netif_rx_response
*make_rx_response(struct xenvif
*vif
,
1531 RING_IDX i
= vif
->rx
.rsp_prod_pvt
;
1532 struct xen_netif_rx_response
*resp
;
1534 resp
= RING_GET_RESPONSE(&vif
->rx
, i
);
1535 resp
->offset
= offset
;
1536 resp
->flags
= flags
;
1538 resp
->status
= (s16
)size
;
1540 resp
->status
= (s16
)st
;
1542 vif
->rx
.rsp_prod_pvt
= ++i
;
1547 static inline int rx_work_todo(struct xen_netbk
*netbk
)
1549 return !skb_queue_empty(&netbk
->rx_queue
);
1552 static inline int tx_work_todo(struct xen_netbk
*netbk
)
1555 if (((nr_pending_reqs(netbk
) + MAX_SKB_FRAGS
) < MAX_PENDING_REQS
) &&
1556 !list_empty(&netbk
->net_schedule_list
))
1562 static int xen_netbk_kthread(void *data
)
1564 struct xen_netbk
*netbk
= data
;
1565 while (!kthread_should_stop()) {
1566 wait_event_interruptible(netbk
->wq
,
1567 rx_work_todo(netbk
) ||
1568 tx_work_todo(netbk
) ||
1569 kthread_should_stop());
1572 if (kthread_should_stop())
1575 if (rx_work_todo(netbk
))
1576 xen_netbk_rx_action(netbk
);
1578 if (tx_work_todo(netbk
))
1579 xen_netbk_tx_action(netbk
);
1585 void xen_netbk_unmap_frontend_rings(struct xenvif
*vif
)
1588 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif
),
1591 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif
),
1595 int xen_netbk_map_frontend_rings(struct xenvif
*vif
,
1596 grant_ref_t tx_ring_ref
,
1597 grant_ref_t rx_ring_ref
)
1600 struct xen_netif_tx_sring
*txs
;
1601 struct xen_netif_rx_sring
*rxs
;
1605 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif
),
1606 tx_ring_ref
, &addr
);
1610 txs
= (struct xen_netif_tx_sring
*)addr
;
1611 BACK_RING_INIT(&vif
->tx
, txs
, PAGE_SIZE
);
1613 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif
),
1614 rx_ring_ref
, &addr
);
1618 rxs
= (struct xen_netif_rx_sring
*)addr
;
1619 BACK_RING_INIT(&vif
->rx
, rxs
, PAGE_SIZE
);
1621 vif
->rx_req_cons_peek
= 0;
1626 xen_netbk_unmap_frontend_rings(vif
);
1630 static int __init
netback_init(void)
1639 xen_netbk_group_nr
= num_online_cpus();
1640 xen_netbk
= vzalloc(sizeof(struct xen_netbk
) * xen_netbk_group_nr
);
1644 for (group
= 0; group
< xen_netbk_group_nr
; group
++) {
1645 struct xen_netbk
*netbk
= &xen_netbk
[group
];
1646 skb_queue_head_init(&netbk
->rx_queue
);
1647 skb_queue_head_init(&netbk
->tx_queue
);
1649 init_timer(&netbk
->net_timer
);
1650 netbk
->net_timer
.data
= (unsigned long)netbk
;
1651 netbk
->net_timer
.function
= xen_netbk_alarm
;
1653 netbk
->pending_cons
= 0;
1654 netbk
->pending_prod
= MAX_PENDING_REQS
;
1655 for (i
= 0; i
< MAX_PENDING_REQS
; i
++)
1656 netbk
->pending_ring
[i
] = i
;
1658 init_waitqueue_head(&netbk
->wq
);
1659 netbk
->task
= kthread_create(xen_netbk_kthread
,
1661 "netback/%u", group
);
1663 if (IS_ERR(netbk
->task
)) {
1664 printk(KERN_ALERT
"kthread_create() fails at netback\n");
1665 del_timer(&netbk
->net_timer
);
1666 rc
= PTR_ERR(netbk
->task
);
1670 kthread_bind(netbk
->task
, group
);
1672 INIT_LIST_HEAD(&netbk
->net_schedule_list
);
1674 spin_lock_init(&netbk
->net_schedule_list_lock
);
1676 atomic_set(&netbk
->netfront_count
, 0);
1678 wake_up_process(netbk
->task
);
1681 rc
= xenvif_xenbus_init();
1688 while (--group
>= 0) {
1689 struct xen_netbk
*netbk
= &xen_netbk
[group
];
1690 for (i
= 0; i
< MAX_PENDING_REQS
; i
++) {
1691 if (netbk
->mmap_pages
[i
])
1692 __free_page(netbk
->mmap_pages
[i
]);
1694 del_timer(&netbk
->net_timer
);
1695 kthread_stop(netbk
->task
);
1702 module_init(netback_init
);
1704 MODULE_LICENSE("Dual BSD/GPL");
1705 MODULE_ALIAS("xen-backend:vif");