2 * Back-end of the driver for virtual network devices. This portion of the
3 * driver exports a 'unified' network-device interface that can be accessed
4 * by any operating system that implements a compatible front end. A
5 * reference front-end implementation can be found in:
6 * drivers/net/xen-netfront.c
8 * Copyright (c) 2002-2005, K A Fraser
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
37 #include <linux/kthread.h>
38 #include <linux/if_vlan.h>
39 #include <linux/udp.h>
44 #include <xen/events.h>
45 #include <xen/interface/memory.h>
47 #include <asm/xen/hypercall.h>
48 #include <asm/xen/page.h>
50 /* Provide an option to disable split event channels at load time as
51 * event channels are limited resource. Split event channels are
54 bool separate_tx_rx_irq
= 1;
55 module_param(separate_tx_rx_irq
, bool, 0644);
58 * This is the maximum slots a skb can have. If a guest sends a skb
59 * which exceeds this limit it is considered malicious.
61 #define FATAL_SKB_SLOTS_DEFAULT 20
62 static unsigned int fatal_skb_slots
= FATAL_SKB_SLOTS_DEFAULT
;
63 module_param(fatal_skb_slots
, uint
, 0444);
66 * To avoid confusion, we define XEN_NETBK_LEGACY_SLOTS_MAX indicating
67 * the maximum slots a valid packet can use. Now this value is defined
68 * to be XEN_NETIF_NR_SLOTS_MIN, which is supposed to be supported by
71 #define XEN_NETBK_LEGACY_SLOTS_MAX XEN_NETIF_NR_SLOTS_MIN
74 * If head != INVALID_PENDING_RING_IDX, it means this tx request is head of
75 * one or more merged tx requests, otherwise it is the continuation of
76 * previous tx request.
78 static inline int pending_tx_is_head(struct xenvif
*vif
, RING_IDX idx
)
80 return vif
->pending_tx_info
[idx
].head
!= INVALID_PENDING_RING_IDX
;
83 static void xenvif_idx_release(struct xenvif
*vif
, u16 pending_idx
,
86 static void make_tx_response(struct xenvif
*vif
,
87 struct xen_netif_tx_request
*txp
,
90 static inline int tx_work_todo(struct xenvif
*vif
);
91 static inline int rx_work_todo(struct xenvif
*vif
);
93 static struct xen_netif_rx_response
*make_rx_response(struct xenvif
*vif
,
100 static inline unsigned long idx_to_pfn(struct xenvif
*vif
,
103 return page_to_pfn(vif
->mmap_pages
[idx
]);
106 static inline unsigned long idx_to_kaddr(struct xenvif
*vif
,
109 return (unsigned long)pfn_to_kaddr(idx_to_pfn(vif
, idx
));
112 /* This is a miniumum size for the linear area to avoid lots of
113 * calls to __pskb_pull_tail() as we set up checksum offsets. The
114 * value 128 was chosen as it covers all IPv4 and most likely
117 #define PKT_PROT_LEN 128
119 static u16
frag_get_pending_idx(skb_frag_t
*frag
)
121 return (u16
)frag
->page_offset
;
124 static void frag_set_pending_idx(skb_frag_t
*frag
, u16 pending_idx
)
126 frag
->page_offset
= pending_idx
;
129 static inline pending_ring_idx_t
pending_index(unsigned i
)
131 return i
& (MAX_PENDING_REQS
-1);
134 static inline pending_ring_idx_t
nr_pending_reqs(struct xenvif
*vif
)
136 return MAX_PENDING_REQS
-
137 vif
->pending_prod
+ vif
->pending_cons
;
140 bool xenvif_rx_ring_slots_available(struct xenvif
*vif
, int needed
)
145 prod
= vif
->rx
.sring
->req_prod
;
146 cons
= vif
->rx
.req_cons
;
148 if (prod
- cons
>= needed
)
151 vif
->rx
.sring
->req_event
= prod
+ 1;
153 /* Make sure event is visible before we check prod
157 } while (vif
->rx
.sring
->req_prod
!= prod
);
163 * Returns true if we should start a new receive buffer instead of
164 * adding 'size' bytes to a buffer which currently contains 'offset'
167 static bool start_new_rx_buffer(int offset
, unsigned long size
, int head
)
169 /* simple case: we have completely filled the current buffer. */
170 if (offset
== MAX_BUFFER_OFFSET
)
174 * complex case: start a fresh buffer if the current frag
175 * would overflow the current buffer but only if:
176 * (i) this frag would fit completely in the next buffer
177 * and (ii) there is already some data in the current buffer
178 * and (iii) this is not the head buffer.
181 * - (i) stops us splitting a frag into two copies
182 * unless the frag is too large for a single buffer.
183 * - (ii) stops us from leaving a buffer pointlessly empty.
184 * - (iii) stops us leaving the first buffer
185 * empty. Strictly speaking this is already covered
186 * by (ii) but is explicitly checked because
187 * netfront relies on the first buffer being
188 * non-empty and can crash otherwise.
190 * This means we will effectively linearise small
191 * frags but do not needlessly split large buffers
192 * into multiple copies tend to give large frags their
193 * own buffers as before.
195 BUG_ON(size
> MAX_BUFFER_OFFSET
);
196 if ((offset
+ size
> MAX_BUFFER_OFFSET
) && offset
&& !head
)
202 struct netrx_pending_operations
{
203 unsigned copy_prod
, copy_cons
;
204 unsigned meta_prod
, meta_cons
;
205 struct gnttab_copy
*copy
;
206 struct xenvif_rx_meta
*meta
;
208 grant_ref_t copy_gref
;
211 static struct xenvif_rx_meta
*get_next_rx_buffer(struct xenvif
*vif
,
212 struct netrx_pending_operations
*npo
)
214 struct xenvif_rx_meta
*meta
;
215 struct xen_netif_rx_request
*req
;
217 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
219 meta
= npo
->meta
+ npo
->meta_prod
++;
220 meta
->gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
226 npo
->copy_gref
= req
->gref
;
232 * Set up the grant operations for this fragment. If it's a flipping
233 * interface, we also set up the unmap request from here.
235 static void xenvif_gop_frag_copy(struct xenvif
*vif
, struct sk_buff
*skb
,
236 struct netrx_pending_operations
*npo
,
237 struct page
*page
, unsigned long size
,
238 unsigned long offset
, int *head
)
240 struct gnttab_copy
*copy_gop
;
241 struct xenvif_rx_meta
*meta
;
243 int gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
245 /* Data must not cross a page boundary. */
246 BUG_ON(size
+ offset
> PAGE_SIZE
<<compound_order(page
));
248 meta
= npo
->meta
+ npo
->meta_prod
- 1;
250 /* Skip unused frames from start of page */
251 page
+= offset
>> PAGE_SHIFT
;
252 offset
&= ~PAGE_MASK
;
255 BUG_ON(offset
>= PAGE_SIZE
);
256 BUG_ON(npo
->copy_off
> MAX_BUFFER_OFFSET
);
258 bytes
= PAGE_SIZE
- offset
;
263 if (start_new_rx_buffer(npo
->copy_off
, bytes
, *head
)) {
265 * Netfront requires there to be some data in the head
270 meta
= get_next_rx_buffer(vif
, npo
);
273 if (npo
->copy_off
+ bytes
> MAX_BUFFER_OFFSET
)
274 bytes
= MAX_BUFFER_OFFSET
- npo
->copy_off
;
276 copy_gop
= npo
->copy
+ npo
->copy_prod
++;
277 copy_gop
->flags
= GNTCOPY_dest_gref
;
278 copy_gop
->len
= bytes
;
280 copy_gop
->source
.domid
= DOMID_SELF
;
281 copy_gop
->source
.u
.gmfn
= virt_to_mfn(page_address(page
));
282 copy_gop
->source
.offset
= offset
;
284 copy_gop
->dest
.domid
= vif
->domid
;
285 copy_gop
->dest
.offset
= npo
->copy_off
;
286 copy_gop
->dest
.u
.ref
= npo
->copy_gref
;
288 npo
->copy_off
+= bytes
;
295 if (offset
== PAGE_SIZE
&& size
) {
296 BUG_ON(!PageCompound(page
));
301 /* Leave a gap for the GSO descriptor. */
302 if (skb_is_gso(skb
)) {
303 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
)
304 gso_type
= XEN_NETIF_GSO_TYPE_TCPV4
;
305 else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
)
306 gso_type
= XEN_NETIF_GSO_TYPE_TCPV6
;
309 if (*head
&& ((1 << gso_type
) & vif
->gso_mask
))
312 *head
= 0; /* There must be something in this buffer now. */
318 * Prepare an SKB to be transmitted to the frontend.
320 * This function is responsible for allocating grant operations, meta
323 * It returns the number of meta structures consumed. The number of
324 * ring slots used is always equal to the number of meta slots used
325 * plus the number of GSO descriptors used. Currently, we use either
326 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
327 * frontend-side LRO).
329 static int xenvif_gop_skb(struct sk_buff
*skb
,
330 struct netrx_pending_operations
*npo
)
332 struct xenvif
*vif
= netdev_priv(skb
->dev
);
333 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
335 struct xen_netif_rx_request
*req
;
336 struct xenvif_rx_meta
*meta
;
342 old_meta_prod
= npo
->meta_prod
;
344 gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
345 if (skb_is_gso(skb
)) {
346 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
)
347 gso_type
= XEN_NETIF_GSO_TYPE_TCPV4
;
348 else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
)
349 gso_type
= XEN_NETIF_GSO_TYPE_TCPV6
;
352 /* Set up a GSO prefix descriptor, if necessary */
353 if ((1 << gso_type
) & vif
->gso_prefix_mask
) {
354 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
355 meta
= npo
->meta
+ npo
->meta_prod
++;
356 meta
->gso_type
= gso_type
;
357 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
362 req
= RING_GET_REQUEST(&vif
->rx
, vif
->rx
.req_cons
++);
363 meta
= npo
->meta
+ npo
->meta_prod
++;
365 if ((1 << gso_type
) & vif
->gso_mask
) {
366 meta
->gso_type
= gso_type
;
367 meta
->gso_size
= skb_shinfo(skb
)->gso_size
;
369 meta
->gso_type
= XEN_NETIF_GSO_TYPE_NONE
;
376 npo
->copy_gref
= req
->gref
;
379 while (data
< skb_tail_pointer(skb
)) {
380 unsigned int offset
= offset_in_page(data
);
381 unsigned int len
= PAGE_SIZE
- offset
;
383 if (data
+ len
> skb_tail_pointer(skb
))
384 len
= skb_tail_pointer(skb
) - data
;
386 xenvif_gop_frag_copy(vif
, skb
, npo
,
387 virt_to_page(data
), len
, offset
, &head
);
391 for (i
= 0; i
< nr_frags
; i
++) {
392 xenvif_gop_frag_copy(vif
, skb
, npo
,
393 skb_frag_page(&skb_shinfo(skb
)->frags
[i
]),
394 skb_frag_size(&skb_shinfo(skb
)->frags
[i
]),
395 skb_shinfo(skb
)->frags
[i
].page_offset
,
399 return npo
->meta_prod
- old_meta_prod
;
403 * This is a twin to xenvif_gop_skb. Assume that xenvif_gop_skb was
404 * used to set up the operations on the top of
405 * netrx_pending_operations, which have since been done. Check that
406 * they didn't give any errors and advance over them.
408 static int xenvif_check_gop(struct xenvif
*vif
, int nr_meta_slots
,
409 struct netrx_pending_operations
*npo
)
411 struct gnttab_copy
*copy_op
;
412 int status
= XEN_NETIF_RSP_OKAY
;
415 for (i
= 0; i
< nr_meta_slots
; i
++) {
416 copy_op
= npo
->copy
+ npo
->copy_cons
++;
417 if (copy_op
->status
!= GNTST_okay
) {
419 "Bad status %d from copy to DOM%d.\n",
420 copy_op
->status
, vif
->domid
);
421 status
= XEN_NETIF_RSP_ERROR
;
428 static void xenvif_add_frag_responses(struct xenvif
*vif
, int status
,
429 struct xenvif_rx_meta
*meta
,
433 unsigned long offset
;
435 /* No fragments used */
436 if (nr_meta_slots
<= 1)
441 for (i
= 0; i
< nr_meta_slots
; i
++) {
443 if (i
== nr_meta_slots
- 1)
446 flags
= XEN_NETRXF_more_data
;
449 make_rx_response(vif
, meta
[i
].id
, status
, offset
,
450 meta
[i
].size
, flags
);
454 struct skb_cb_overlay
{
458 void xenvif_kick_thread(struct xenvif
*vif
)
463 static void xenvif_rx_action(struct xenvif
*vif
)
467 struct xen_netif_rx_response
*resp
;
468 struct sk_buff_head rxq
;
472 unsigned long offset
;
473 struct skb_cb_overlay
*sco
;
474 bool need_to_notify
= false;
476 struct netrx_pending_operations npo
= {
477 .copy
= vif
->grant_copy_op
,
481 skb_queue_head_init(&rxq
);
483 while ((skb
= skb_dequeue(&vif
->rx_queue
)) != NULL
) {
484 RING_IDX max_slots_needed
;
485 RING_IDX old_req_cons
;
486 RING_IDX ring_slots_used
;
489 /* We need a cheap worse case estimate for the number of
493 max_slots_needed
= DIV_ROUND_UP(offset_in_page(skb
->data
) +
496 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
500 size
= skb_frag_size(&skb_shinfo(skb
)->frags
[i
]);
501 offset
= skb_shinfo(skb
)->frags
[i
].page_offset
;
503 /* For a worse-case estimate we need to factor in
504 * the fragment page offset as this will affect the
505 * number of times xenvif_gop_frag_copy() will
506 * call start_new_rx_buffer().
508 max_slots_needed
+= DIV_ROUND_UP(offset
+ size
,
512 /* To avoid the estimate becoming too pessimal for some
513 * frontends that limit posted rx requests, cap the estimate
516 if (max_slots_needed
> MAX_SKB_FRAGS
)
517 max_slots_needed
= MAX_SKB_FRAGS
;
519 /* We may need one more slot for GSO metadata */
520 if (skb_is_gso(skb
) &&
521 (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
||
522 skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
))
525 /* If the skb may not fit then bail out now */
526 if (!xenvif_rx_ring_slots_available(vif
, max_slots_needed
)) {
527 skb_queue_head(&vif
->rx_queue
, skb
);
528 need_to_notify
= true;
529 vif
->rx_last_skb_slots
= max_slots_needed
;
532 vif
->rx_last_skb_slots
= 0;
534 sco
= (struct skb_cb_overlay
*)skb
->cb
;
536 old_req_cons
= vif
->rx
.req_cons
;
537 sco
->meta_slots_used
= xenvif_gop_skb(skb
, &npo
);
538 ring_slots_used
= vif
->rx
.req_cons
- old_req_cons
;
540 BUG_ON(ring_slots_used
> max_slots_needed
);
542 __skb_queue_tail(&rxq
, skb
);
545 BUG_ON(npo
.meta_prod
> ARRAY_SIZE(vif
->meta
));
550 BUG_ON(npo
.copy_prod
> MAX_GRANT_COPY_OPS
);
551 gnttab_batch_copy(vif
->grant_copy_op
, npo
.copy_prod
);
553 while ((skb
= __skb_dequeue(&rxq
)) != NULL
) {
554 sco
= (struct skb_cb_overlay
*)skb
->cb
;
556 if ((1 << vif
->meta
[npo
.meta_cons
].gso_type
) &
557 vif
->gso_prefix_mask
) {
558 resp
= RING_GET_RESPONSE(&vif
->rx
,
559 vif
->rx
.rsp_prod_pvt
++);
561 resp
->flags
= XEN_NETRXF_gso_prefix
| XEN_NETRXF_more_data
;
563 resp
->offset
= vif
->meta
[npo
.meta_cons
].gso_size
;
564 resp
->id
= vif
->meta
[npo
.meta_cons
].id
;
565 resp
->status
= sco
->meta_slots_used
;
568 sco
->meta_slots_used
--;
572 vif
->dev
->stats
.tx_bytes
+= skb
->len
;
573 vif
->dev
->stats
.tx_packets
++;
575 status
= xenvif_check_gop(vif
, sco
->meta_slots_used
, &npo
);
577 if (sco
->meta_slots_used
== 1)
580 flags
= XEN_NETRXF_more_data
;
582 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) /* local packet? */
583 flags
|= XEN_NETRXF_csum_blank
| XEN_NETRXF_data_validated
;
584 else if (skb
->ip_summed
== CHECKSUM_UNNECESSARY
)
585 /* remote but checksummed. */
586 flags
|= XEN_NETRXF_data_validated
;
589 resp
= make_rx_response(vif
, vif
->meta
[npo
.meta_cons
].id
,
591 vif
->meta
[npo
.meta_cons
].size
,
594 if ((1 << vif
->meta
[npo
.meta_cons
].gso_type
) &
596 struct xen_netif_extra_info
*gso
=
597 (struct xen_netif_extra_info
*)
598 RING_GET_RESPONSE(&vif
->rx
,
599 vif
->rx
.rsp_prod_pvt
++);
601 resp
->flags
|= XEN_NETRXF_extra_info
;
603 gso
->u
.gso
.type
= vif
->meta
[npo
.meta_cons
].gso_type
;
604 gso
->u
.gso
.size
= vif
->meta
[npo
.meta_cons
].gso_size
;
606 gso
->u
.gso
.features
= 0;
608 gso
->type
= XEN_NETIF_EXTRA_TYPE_GSO
;
612 xenvif_add_frag_responses(vif
, status
,
613 vif
->meta
+ npo
.meta_cons
+ 1,
614 sco
->meta_slots_used
);
616 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif
->rx
, ret
);
618 need_to_notify
|= !!ret
;
620 npo
.meta_cons
+= sco
->meta_slots_used
;
626 notify_remote_via_irq(vif
->rx_irq
);
629 void xenvif_check_rx_xenvif(struct xenvif
*vif
)
633 RING_FINAL_CHECK_FOR_REQUESTS(&vif
->tx
, more_to_do
);
636 napi_schedule(&vif
->napi
);
639 static void tx_add_credit(struct xenvif
*vif
)
641 unsigned long max_burst
, max_credit
;
644 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
645 * Otherwise the interface can seize up due to insufficient credit.
647 max_burst
= RING_GET_REQUEST(&vif
->tx
, vif
->tx
.req_cons
)->size
;
648 max_burst
= min(max_burst
, 131072UL);
649 max_burst
= max(max_burst
, vif
->credit_bytes
);
651 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
652 max_credit
= vif
->remaining_credit
+ vif
->credit_bytes
;
653 if (max_credit
< vif
->remaining_credit
)
654 max_credit
= ULONG_MAX
; /* wrapped: clamp to ULONG_MAX */
656 vif
->remaining_credit
= min(max_credit
, max_burst
);
659 static void tx_credit_callback(unsigned long data
)
661 struct xenvif
*vif
= (struct xenvif
*)data
;
663 xenvif_check_rx_xenvif(vif
);
666 static void xenvif_tx_err(struct xenvif
*vif
,
667 struct xen_netif_tx_request
*txp
, RING_IDX end
)
669 RING_IDX cons
= vif
->tx
.req_cons
;
672 make_tx_response(vif
, txp
, XEN_NETIF_RSP_ERROR
);
675 txp
= RING_GET_REQUEST(&vif
->tx
, cons
++);
677 vif
->tx
.req_cons
= cons
;
680 static void xenvif_fatal_tx_err(struct xenvif
*vif
)
682 netdev_err(vif
->dev
, "fatal error; disabling device\n");
683 xenvif_carrier_off(vif
);
686 static int xenvif_count_requests(struct xenvif
*vif
,
687 struct xen_netif_tx_request
*first
,
688 struct xen_netif_tx_request
*txp
,
691 RING_IDX cons
= vif
->tx
.req_cons
;
696 if (!(first
->flags
& XEN_NETTXF_more_data
))
700 struct xen_netif_tx_request dropped_tx
= { 0 };
702 if (slots
>= work_to_do
) {
704 "Asked for %d slots but exceeds this limit\n",
706 xenvif_fatal_tx_err(vif
);
710 /* This guest is really using too many slots and
711 * considered malicious.
713 if (unlikely(slots
>= fatal_skb_slots
)) {
715 "Malicious frontend using %d slots, threshold %u\n",
716 slots
, fatal_skb_slots
);
717 xenvif_fatal_tx_err(vif
);
721 /* Xen network protocol had implicit dependency on
722 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to
723 * the historical MAX_SKB_FRAGS value 18 to honor the
724 * same behavior as before. Any packet using more than
725 * 18 slots but less than fatal_skb_slots slots is
728 if (!drop_err
&& slots
>= XEN_NETBK_LEGACY_SLOTS_MAX
) {
731 "Too many slots (%d) exceeding limit (%d), dropping packet\n",
732 slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
739 memcpy(txp
, RING_GET_REQUEST(&vif
->tx
, cons
+ slots
),
742 /* If the guest submitted a frame >= 64 KiB then
743 * first->size overflowed and following slots will
744 * appear to be larger than the frame.
746 * This cannot be fatal error as there are buggy
747 * frontends that do this.
749 * Consume all slots and drop the packet.
751 if (!drop_err
&& txp
->size
> first
->size
) {
754 "Invalid tx request, slot size %u > remaining size %u\n",
755 txp
->size
, first
->size
);
759 first
->size
-= txp
->size
;
762 if (unlikely((txp
->offset
+ txp
->size
) > PAGE_SIZE
)) {
763 netdev_err(vif
->dev
, "Cross page boundary, txp->offset: %x, size: %u\n",
764 txp
->offset
, txp
->size
);
765 xenvif_fatal_tx_err(vif
);
769 more_data
= txp
->flags
& XEN_NETTXF_more_data
;
777 xenvif_tx_err(vif
, first
, cons
+ slots
);
784 static struct page
*xenvif_alloc_page(struct xenvif
*vif
,
789 page
= alloc_page(GFP_ATOMIC
|__GFP_COLD
);
792 vif
->mmap_pages
[pending_idx
] = page
;
797 static struct gnttab_copy
*xenvif_get_requests(struct xenvif
*vif
,
799 struct xen_netif_tx_request
*txp
,
800 struct gnttab_copy
*gop
)
802 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
803 skb_frag_t
*frags
= shinfo
->frags
;
804 u16 pending_idx
= *((u16
*)skb
->data
);
808 pending_ring_idx_t index
, start_idx
= 0;
810 unsigned int nr_slots
;
811 struct pending_tx_info
*first
= NULL
;
813 /* At this point shinfo->nr_frags is in fact the number of
814 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
816 nr_slots
= shinfo
->nr_frags
;
818 /* Skip first skb fragment if it is on same page as header fragment. */
819 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
821 /* Coalesce tx requests, at this point the packet passed in
822 * should be <= 64K. Any packets larger than 64K have been
823 * handled in xenvif_count_requests().
825 for (shinfo
->nr_frags
= slot
= start
; slot
< nr_slots
;
826 shinfo
->nr_frags
++) {
827 struct pending_tx_info
*pending_tx_info
=
828 vif
->pending_tx_info
;
830 page
= alloc_page(GFP_ATOMIC
|__GFP_COLD
);
836 while (dst_offset
< PAGE_SIZE
&& slot
< nr_slots
) {
837 gop
->flags
= GNTCOPY_source_gref
;
839 gop
->source
.u
.ref
= txp
->gref
;
840 gop
->source
.domid
= vif
->domid
;
841 gop
->source
.offset
= txp
->offset
;
843 gop
->dest
.domid
= DOMID_SELF
;
845 gop
->dest
.offset
= dst_offset
;
846 gop
->dest
.u
.gmfn
= virt_to_mfn(page_address(page
));
848 if (dst_offset
+ txp
->size
> PAGE_SIZE
) {
849 /* This page can only merge a portion
850 * of tx request. Do not increment any
851 * pointer / counter here. The txp
852 * will be dealt with in future
853 * rounds, eventually hitting the
856 gop
->len
= PAGE_SIZE
- dst_offset
;
857 txp
->offset
+= gop
->len
;
858 txp
->size
-= gop
->len
;
859 dst_offset
+= gop
->len
; /* quit loop */
861 /* This tx request can be merged in the page */
862 gop
->len
= txp
->size
;
863 dst_offset
+= gop
->len
;
865 index
= pending_index(vif
->pending_cons
++);
867 pending_idx
= vif
->pending_ring
[index
];
869 memcpy(&pending_tx_info
[pending_idx
].req
, txp
,
872 /* Poison these fields, corresponding
873 * fields for head tx req will be set
874 * to correct values after the loop.
876 vif
->mmap_pages
[pending_idx
] = (void *)(~0UL);
877 pending_tx_info
[pending_idx
].head
=
878 INVALID_PENDING_RING_IDX
;
881 first
= &pending_tx_info
[pending_idx
];
883 head_idx
= pending_idx
;
893 first
->req
.offset
= 0;
894 first
->req
.size
= dst_offset
;
895 first
->head
= start_idx
;
896 vif
->mmap_pages
[head_idx
] = page
;
897 frag_set_pending_idx(&frags
[shinfo
->nr_frags
], head_idx
);
900 BUG_ON(shinfo
->nr_frags
> MAX_SKB_FRAGS
);
904 /* Unwind, freeing all pages and sending error responses. */
905 while (shinfo
->nr_frags
-- > start
) {
906 xenvif_idx_release(vif
,
907 frag_get_pending_idx(&frags
[shinfo
->nr_frags
]),
908 XEN_NETIF_RSP_ERROR
);
910 /* The head too, if necessary. */
912 xenvif_idx_release(vif
, pending_idx
, XEN_NETIF_RSP_ERROR
);
917 static int xenvif_tx_check_gop(struct xenvif
*vif
,
919 struct gnttab_copy
**gopp
)
921 struct gnttab_copy
*gop
= *gopp
;
922 u16 pending_idx
= *((u16
*)skb
->data
);
923 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
924 struct pending_tx_info
*tx_info
;
925 int nr_frags
= shinfo
->nr_frags
;
927 u16 peek
; /* peek into next tx request */
929 /* Check status of header. */
932 xenvif_idx_release(vif
, pending_idx
, XEN_NETIF_RSP_ERROR
);
934 /* Skip first skb fragment if it is on same page as header fragment. */
935 start
= (frag_get_pending_idx(&shinfo
->frags
[0]) == pending_idx
);
937 for (i
= start
; i
< nr_frags
; i
++) {
939 pending_ring_idx_t head
;
941 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[i
]);
942 tx_info
= &vif
->pending_tx_info
[pending_idx
];
943 head
= tx_info
->head
;
945 /* Check error status: if okay then remember grant handle. */
947 newerr
= (++gop
)->status
;
950 peek
= vif
->pending_ring
[pending_index(++head
)];
951 } while (!pending_tx_is_head(vif
, peek
));
953 if (likely(!newerr
)) {
954 /* Had a previous error? Invalidate this fragment. */
956 xenvif_idx_release(vif
, pending_idx
,
961 /* Error on this fragment: respond to client with an error. */
962 xenvif_idx_release(vif
, pending_idx
, XEN_NETIF_RSP_ERROR
);
964 /* Not the first error? Preceding frags already invalidated. */
968 /* First error: invalidate header and preceding fragments. */
969 pending_idx
= *((u16
*)skb
->data
);
970 xenvif_idx_release(vif
, pending_idx
, XEN_NETIF_RSP_OKAY
);
971 for (j
= start
; j
< i
; j
++) {
972 pending_idx
= frag_get_pending_idx(&shinfo
->frags
[j
]);
973 xenvif_idx_release(vif
, pending_idx
,
977 /* Remember the error: invalidate all subsequent fragments. */
985 static void xenvif_fill_frags(struct xenvif
*vif
, struct sk_buff
*skb
)
987 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
988 int nr_frags
= shinfo
->nr_frags
;
991 for (i
= 0; i
< nr_frags
; i
++) {
992 skb_frag_t
*frag
= shinfo
->frags
+ i
;
993 struct xen_netif_tx_request
*txp
;
997 pending_idx
= frag_get_pending_idx(frag
);
999 txp
= &vif
->pending_tx_info
[pending_idx
].req
;
1000 page
= virt_to_page(idx_to_kaddr(vif
, pending_idx
));
1001 __skb_fill_page_desc(skb
, i
, page
, txp
->offset
, txp
->size
);
1002 skb
->len
+= txp
->size
;
1003 skb
->data_len
+= txp
->size
;
1004 skb
->truesize
+= txp
->size
;
1006 /* Take an extra reference to offset xenvif_idx_release */
1007 get_page(vif
->mmap_pages
[pending_idx
]);
1008 xenvif_idx_release(vif
, pending_idx
, XEN_NETIF_RSP_OKAY
);
1012 static int xenvif_get_extras(struct xenvif
*vif
,
1013 struct xen_netif_extra_info
*extras
,
1016 struct xen_netif_extra_info extra
;
1017 RING_IDX cons
= vif
->tx
.req_cons
;
1020 if (unlikely(work_to_do
-- <= 0)) {
1021 netdev_err(vif
->dev
, "Missing extra info\n");
1022 xenvif_fatal_tx_err(vif
);
1026 memcpy(&extra
, RING_GET_REQUEST(&vif
->tx
, cons
),
1028 if (unlikely(!extra
.type
||
1029 extra
.type
>= XEN_NETIF_EXTRA_TYPE_MAX
)) {
1030 vif
->tx
.req_cons
= ++cons
;
1031 netdev_err(vif
->dev
,
1032 "Invalid extra type: %d\n", extra
.type
);
1033 xenvif_fatal_tx_err(vif
);
1037 memcpy(&extras
[extra
.type
- 1], &extra
, sizeof(extra
));
1038 vif
->tx
.req_cons
= ++cons
;
1039 } while (extra
.flags
& XEN_NETIF_EXTRA_FLAG_MORE
);
1044 static int xenvif_set_skb_gso(struct xenvif
*vif
,
1045 struct sk_buff
*skb
,
1046 struct xen_netif_extra_info
*gso
)
1048 if (!gso
->u
.gso
.size
) {
1049 netdev_err(vif
->dev
, "GSO size must not be zero.\n");
1050 xenvif_fatal_tx_err(vif
);
1054 switch (gso
->u
.gso
.type
) {
1055 case XEN_NETIF_GSO_TYPE_TCPV4
:
1056 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
1058 case XEN_NETIF_GSO_TYPE_TCPV6
:
1059 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV6
;
1062 netdev_err(vif
->dev
, "Bad GSO type %d.\n", gso
->u
.gso
.type
);
1063 xenvif_fatal_tx_err(vif
);
1067 skb_shinfo(skb
)->gso_size
= gso
->u
.gso
.size
;
1068 /* gso_segs will be calculated later */
1073 static int checksum_setup(struct xenvif
*vif
, struct sk_buff
*skb
)
1075 bool recalculate_partial_csum
= false;
1077 /* A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1078 * peers can fail to set NETRXF_csum_blank when sending a GSO
1079 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1080 * recalculate the partial checksum.
1082 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
&& skb_is_gso(skb
)) {
1083 vif
->rx_gso_checksum_fixup
++;
1084 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1085 recalculate_partial_csum
= true;
1088 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1089 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1092 return skb_checksum_setup(skb
, recalculate_partial_csum
);
1095 static bool tx_credit_exceeded(struct xenvif
*vif
, unsigned size
)
1097 u64 now
= get_jiffies_64();
1098 u64 next_credit
= vif
->credit_window_start
+
1099 msecs_to_jiffies(vif
->credit_usec
/ 1000);
1101 /* Timer could already be pending in rare cases. */
1102 if (timer_pending(&vif
->credit_timeout
))
1105 /* Passed the point where we can replenish credit? */
1106 if (time_after_eq64(now
, next_credit
)) {
1107 vif
->credit_window_start
= now
;
1111 /* Still too big to send right now? Set a callback. */
1112 if (size
> vif
->remaining_credit
) {
1113 vif
->credit_timeout
.data
=
1115 vif
->credit_timeout
.function
=
1117 mod_timer(&vif
->credit_timeout
,
1119 vif
->credit_window_start
= next_credit
;
1127 static unsigned xenvif_tx_build_gops(struct xenvif
*vif
, int budget
)
1129 struct gnttab_copy
*gop
= vif
->tx_copy_ops
, *request_gop
;
1130 struct sk_buff
*skb
;
1133 while ((nr_pending_reqs(vif
) + XEN_NETBK_LEGACY_SLOTS_MAX
1134 < MAX_PENDING_REQS
) &&
1135 (skb_queue_len(&vif
->tx_queue
) < budget
)) {
1136 struct xen_netif_tx_request txreq
;
1137 struct xen_netif_tx_request txfrags
[XEN_NETBK_LEGACY_SLOTS_MAX
];
1139 struct xen_netif_extra_info extras
[XEN_NETIF_EXTRA_TYPE_MAX
-1];
1143 unsigned int data_len
;
1144 pending_ring_idx_t index
;
1146 if (vif
->tx
.sring
->req_prod
- vif
->tx
.req_cons
>
1147 XEN_NETIF_TX_RING_SIZE
) {
1148 netdev_err(vif
->dev
,
1149 "Impossible number of requests. "
1150 "req_prod %d, req_cons %d, size %ld\n",
1151 vif
->tx
.sring
->req_prod
, vif
->tx
.req_cons
,
1152 XEN_NETIF_TX_RING_SIZE
);
1153 xenvif_fatal_tx_err(vif
);
1157 work_to_do
= RING_HAS_UNCONSUMED_REQUESTS(&vif
->tx
);
1161 idx
= vif
->tx
.req_cons
;
1162 rmb(); /* Ensure that we see the request before we copy it. */
1163 memcpy(&txreq
, RING_GET_REQUEST(&vif
->tx
, idx
), sizeof(txreq
));
1165 /* Credit-based scheduling. */
1166 if (txreq
.size
> vif
->remaining_credit
&&
1167 tx_credit_exceeded(vif
, txreq
.size
))
1170 vif
->remaining_credit
-= txreq
.size
;
1173 vif
->tx
.req_cons
= ++idx
;
1175 memset(extras
, 0, sizeof(extras
));
1176 if (txreq
.flags
& XEN_NETTXF_extra_info
) {
1177 work_to_do
= xenvif_get_extras(vif
, extras
,
1179 idx
= vif
->tx
.req_cons
;
1180 if (unlikely(work_to_do
< 0))
1184 ret
= xenvif_count_requests(vif
, &txreq
, txfrags
, work_to_do
);
1185 if (unlikely(ret
< 0))
1190 if (unlikely(txreq
.size
< ETH_HLEN
)) {
1191 netdev_dbg(vif
->dev
,
1192 "Bad packet size: %d\n", txreq
.size
);
1193 xenvif_tx_err(vif
, &txreq
, idx
);
1197 /* No crossing a page as the payload mustn't fragment. */
1198 if (unlikely((txreq
.offset
+ txreq
.size
) > PAGE_SIZE
)) {
1199 netdev_err(vif
->dev
,
1200 "txreq.offset: %x, size: %u, end: %lu\n",
1201 txreq
.offset
, txreq
.size
,
1202 (txreq
.offset
&~PAGE_MASK
) + txreq
.size
);
1203 xenvif_fatal_tx_err(vif
);
1207 index
= pending_index(vif
->pending_cons
);
1208 pending_idx
= vif
->pending_ring
[index
];
1210 data_len
= (txreq
.size
> PKT_PROT_LEN
&&
1211 ret
< XEN_NETBK_LEGACY_SLOTS_MAX
) ?
1212 PKT_PROT_LEN
: txreq
.size
;
1214 skb
= alloc_skb(data_len
+ NET_SKB_PAD
+ NET_IP_ALIGN
,
1215 GFP_ATOMIC
| __GFP_NOWARN
);
1216 if (unlikely(skb
== NULL
)) {
1217 netdev_dbg(vif
->dev
,
1218 "Can't allocate a skb in start_xmit.\n");
1219 xenvif_tx_err(vif
, &txreq
, idx
);
1223 /* Packets passed to netif_rx() must have some headroom. */
1224 skb_reserve(skb
, NET_SKB_PAD
+ NET_IP_ALIGN
);
1226 if (extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1].type
) {
1227 struct xen_netif_extra_info
*gso
;
1228 gso
= &extras
[XEN_NETIF_EXTRA_TYPE_GSO
- 1];
1230 if (xenvif_set_skb_gso(vif
, skb
, gso
)) {
1231 /* Failure in xenvif_set_skb_gso is fatal. */
1237 /* XXX could copy straight to head */
1238 page
= xenvif_alloc_page(vif
, pending_idx
);
1241 xenvif_tx_err(vif
, &txreq
, idx
);
1245 gop
->source
.u
.ref
= txreq
.gref
;
1246 gop
->source
.domid
= vif
->domid
;
1247 gop
->source
.offset
= txreq
.offset
;
1249 gop
->dest
.u
.gmfn
= virt_to_mfn(page_address(page
));
1250 gop
->dest
.domid
= DOMID_SELF
;
1251 gop
->dest
.offset
= txreq
.offset
;
1253 gop
->len
= txreq
.size
;
1254 gop
->flags
= GNTCOPY_source_gref
;
1258 memcpy(&vif
->pending_tx_info
[pending_idx
].req
,
1259 &txreq
, sizeof(txreq
));
1260 vif
->pending_tx_info
[pending_idx
].head
= index
;
1261 *((u16
*)skb
->data
) = pending_idx
;
1263 __skb_put(skb
, data_len
);
1265 skb_shinfo(skb
)->nr_frags
= ret
;
1266 if (data_len
< txreq
.size
) {
1267 skb_shinfo(skb
)->nr_frags
++;
1268 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1271 frag_set_pending_idx(&skb_shinfo(skb
)->frags
[0],
1272 INVALID_PENDING_IDX
);
1275 vif
->pending_cons
++;
1277 request_gop
= xenvif_get_requests(vif
, skb
, txfrags
, gop
);
1278 if (request_gop
== NULL
) {
1280 xenvif_tx_err(vif
, &txreq
, idx
);
1285 __skb_queue_tail(&vif
->tx_queue
, skb
);
1287 vif
->tx
.req_cons
= idx
;
1289 if ((gop
-vif
->tx_copy_ops
) >= ARRAY_SIZE(vif
->tx_copy_ops
))
1293 return gop
- vif
->tx_copy_ops
;
1297 static int xenvif_tx_submit(struct xenvif
*vif
)
1299 struct gnttab_copy
*gop
= vif
->tx_copy_ops
;
1300 struct sk_buff
*skb
;
1303 while ((skb
= __skb_dequeue(&vif
->tx_queue
)) != NULL
) {
1304 struct xen_netif_tx_request
*txp
;
1308 pending_idx
= *((u16
*)skb
->data
);
1309 txp
= &vif
->pending_tx_info
[pending_idx
].req
;
1311 /* Check the remap error code. */
1312 if (unlikely(xenvif_tx_check_gop(vif
, skb
, &gop
))) {
1313 netdev_dbg(vif
->dev
, "netback grant failed.\n");
1314 skb_shinfo(skb
)->nr_frags
= 0;
1319 data_len
= skb
->len
;
1321 (void *)(idx_to_kaddr(vif
, pending_idx
)|txp
->offset
),
1323 if (data_len
< txp
->size
) {
1324 /* Append the packet payload as a fragment. */
1325 txp
->offset
+= data_len
;
1326 txp
->size
-= data_len
;
1328 /* Schedule a response immediately. */
1329 xenvif_idx_release(vif
, pending_idx
,
1330 XEN_NETIF_RSP_OKAY
);
1333 if (txp
->flags
& XEN_NETTXF_csum_blank
)
1334 skb
->ip_summed
= CHECKSUM_PARTIAL
;
1335 else if (txp
->flags
& XEN_NETTXF_data_validated
)
1336 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1338 xenvif_fill_frags(vif
, skb
);
1340 if (skb_is_nonlinear(skb
) && skb_headlen(skb
) < PKT_PROT_LEN
) {
1341 int target
= min_t(int, skb
->len
, PKT_PROT_LEN
);
1342 __pskb_pull_tail(skb
, target
- skb_headlen(skb
));
1345 skb
->dev
= vif
->dev
;
1346 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1347 skb_reset_network_header(skb
);
1349 if (checksum_setup(vif
, skb
)) {
1350 netdev_dbg(vif
->dev
,
1351 "Can't setup checksum in net_tx_action\n");
1356 skb_probe_transport_header(skb
, 0);
1358 /* If the packet is GSO then we will have just set up the
1359 * transport header offset in checksum_setup so it's now
1360 * straightforward to calculate gso_segs.
1362 if (skb_is_gso(skb
)) {
1363 int mss
= skb_shinfo(skb
)->gso_size
;
1364 int hdrlen
= skb_transport_header(skb
) -
1365 skb_mac_header(skb
) +
1368 skb_shinfo(skb
)->gso_segs
=
1369 DIV_ROUND_UP(skb
->len
- hdrlen
, mss
);
1372 vif
->dev
->stats
.rx_bytes
+= skb
->len
;
1373 vif
->dev
->stats
.rx_packets
++;
1377 netif_receive_skb(skb
);
1383 /* Called after netfront has transmitted */
1384 int xenvif_tx_action(struct xenvif
*vif
, int budget
)
1389 if (unlikely(!tx_work_todo(vif
)))
1392 nr_gops
= xenvif_tx_build_gops(vif
, budget
);
1397 gnttab_batch_copy(vif
->tx_copy_ops
, nr_gops
);
1399 work_done
= xenvif_tx_submit(vif
);
1404 static void xenvif_idx_release(struct xenvif
*vif
, u16 pending_idx
,
1407 struct pending_tx_info
*pending_tx_info
;
1408 pending_ring_idx_t head
;
1409 u16 peek
; /* peek into next tx request */
1411 BUG_ON(vif
->mmap_pages
[pending_idx
] == (void *)(~0UL));
1413 /* Already complete? */
1414 if (vif
->mmap_pages
[pending_idx
] == NULL
)
1417 pending_tx_info
= &vif
->pending_tx_info
[pending_idx
];
1419 head
= pending_tx_info
->head
;
1421 BUG_ON(!pending_tx_is_head(vif
, head
));
1422 BUG_ON(vif
->pending_ring
[pending_index(head
)] != pending_idx
);
1425 pending_ring_idx_t index
;
1426 pending_ring_idx_t idx
= pending_index(head
);
1427 u16 info_idx
= vif
->pending_ring
[idx
];
1429 pending_tx_info
= &vif
->pending_tx_info
[info_idx
];
1430 make_tx_response(vif
, &pending_tx_info
->req
, status
);
1432 /* Setting any number other than
1433 * INVALID_PENDING_RING_IDX indicates this slot is
1434 * starting a new packet / ending a previous packet.
1436 pending_tx_info
->head
= 0;
1438 index
= pending_index(vif
->pending_prod
++);
1439 vif
->pending_ring
[index
] = vif
->pending_ring
[info_idx
];
1441 peek
= vif
->pending_ring
[pending_index(++head
)];
1443 } while (!pending_tx_is_head(vif
, peek
));
1445 put_page(vif
->mmap_pages
[pending_idx
]);
1446 vif
->mmap_pages
[pending_idx
] = NULL
;
1450 static void make_tx_response(struct xenvif
*vif
,
1451 struct xen_netif_tx_request
*txp
,
1454 RING_IDX i
= vif
->tx
.rsp_prod_pvt
;
1455 struct xen_netif_tx_response
*resp
;
1458 resp
= RING_GET_RESPONSE(&vif
->tx
, i
);
1462 if (txp
->flags
& XEN_NETTXF_extra_info
)
1463 RING_GET_RESPONSE(&vif
->tx
, ++i
)->status
= XEN_NETIF_RSP_NULL
;
1465 vif
->tx
.rsp_prod_pvt
= ++i
;
1466 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif
->tx
, notify
);
1468 notify_remote_via_irq(vif
->tx_irq
);
1471 static struct xen_netif_rx_response
*make_rx_response(struct xenvif
*vif
,
1478 RING_IDX i
= vif
->rx
.rsp_prod_pvt
;
1479 struct xen_netif_rx_response
*resp
;
1481 resp
= RING_GET_RESPONSE(&vif
->rx
, i
);
1482 resp
->offset
= offset
;
1483 resp
->flags
= flags
;
1485 resp
->status
= (s16
)size
;
1487 resp
->status
= (s16
)st
;
1489 vif
->rx
.rsp_prod_pvt
= ++i
;
1494 static inline int rx_work_todo(struct xenvif
*vif
)
1496 return !skb_queue_empty(&vif
->rx_queue
) &&
1497 xenvif_rx_ring_slots_available(vif
, vif
->rx_last_skb_slots
);
1500 static inline int tx_work_todo(struct xenvif
*vif
)
1503 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&vif
->tx
)) &&
1504 (nr_pending_reqs(vif
) + XEN_NETBK_LEGACY_SLOTS_MAX
1505 < MAX_PENDING_REQS
))
1511 void xenvif_unmap_frontend_rings(struct xenvif
*vif
)
1514 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif
),
1517 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif
),
1521 int xenvif_map_frontend_rings(struct xenvif
*vif
,
1522 grant_ref_t tx_ring_ref
,
1523 grant_ref_t rx_ring_ref
)
1526 struct xen_netif_tx_sring
*txs
;
1527 struct xen_netif_rx_sring
*rxs
;
1531 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif
),
1532 tx_ring_ref
, &addr
);
1536 txs
= (struct xen_netif_tx_sring
*)addr
;
1537 BACK_RING_INIT(&vif
->tx
, txs
, PAGE_SIZE
);
1539 err
= xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif
),
1540 rx_ring_ref
, &addr
);
1544 rxs
= (struct xen_netif_rx_sring
*)addr
;
1545 BACK_RING_INIT(&vif
->rx
, rxs
, PAGE_SIZE
);
1550 xenvif_unmap_frontend_rings(vif
);
1554 void xenvif_stop_queue(struct xenvif
*vif
)
1556 if (!vif
->can_queue
)
1559 netif_stop_queue(vif
->dev
);
1562 static void xenvif_start_queue(struct xenvif
*vif
)
1564 if (xenvif_schedulable(vif
))
1565 netif_wake_queue(vif
->dev
);
1568 int xenvif_kthread(void *data
)
1570 struct xenvif
*vif
= data
;
1571 struct sk_buff
*skb
;
1573 while (!kthread_should_stop()) {
1574 wait_event_interruptible(vif
->wq
,
1575 rx_work_todo(vif
) ||
1576 kthread_should_stop());
1577 if (kthread_should_stop())
1580 if (!skb_queue_empty(&vif
->rx_queue
))
1581 xenvif_rx_action(vif
);
1583 if (skb_queue_empty(&vif
->rx_queue
) &&
1584 netif_queue_stopped(vif
->dev
))
1585 xenvif_start_queue(vif
);
1590 /* Bin any remaining skbs */
1591 while ((skb
= skb_dequeue(&vif
->rx_queue
)) != NULL
)
1597 static int __init
netback_init(void)
1604 if (fatal_skb_slots
< XEN_NETBK_LEGACY_SLOTS_MAX
) {
1605 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n",
1606 fatal_skb_slots
, XEN_NETBK_LEGACY_SLOTS_MAX
);
1607 fatal_skb_slots
= XEN_NETBK_LEGACY_SLOTS_MAX
;
1610 rc
= xenvif_xenbus_init();
1620 module_init(netback_init
);
1622 static void __exit
netback_fini(void)
1624 xenvif_xenbus_fini();
1626 module_exit(netback_fini
);
1628 MODULE_LICENSE("Dual BSD/GPL");
1629 MODULE_ALIAS("xen-backend:vif");