projects / deliverable / linux.git / blob
? search:


919b6509455cfbaf45ba63c7d8b378fa13a8e758
[deliverable/linux.git] / drivers / net / xen-netback / netback.c
1 /*
2 * Back-end of the driver for virtual network devices. This portion of the
3 * driver exports a 'unified' network-device interface that can be accessed
4 * by any operating system that implements a compatible front end. A
5 * reference front-end implementation can be found in:
6 * drivers/net/xen-netfront.c
7 *
8 * Copyright (c) 2002-2005, K A Fraser
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License version 2
12 * as published by the Free Software Foundation; or, when distributed
13 * separately from the Linux kernel or incorporated into other
14 * software packages, subject to the following license:
15 *
16 * Permission is hereby granted, free of charge, to any person obtaining a copy
17 * of this source file (the "Software"), to deal in the Software without
18 * restriction, including without limitation the rights to use, copy, modify,
19 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
20 * and to permit persons to whom the Software is furnished to do so, subject to
21 * the following conditions:
22 *
23 * The above copyright notice and this permission notice shall be included in
24 * all copies or substantial portions of the Software.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
31 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
32 * IN THE SOFTWARE.
33 */
34
35 #include "common.h"
36
37 #include <linux/kthread.h>
38 #include <linux/if_vlan.h>
39 #include <linux/udp.h>
40
41 #include <net/tcp.h>
42
43 #include <xen/xen.h>
44 #include <xen/events.h>
45 #include <xen/interface/memory.h>
46
47 #include <asm/xen/hypercall.h>
48 #include <asm/xen/page.h>
49
50 /* Provide an option to disable split event channels at load time as
51 * event channels are limited resource. Split event channels are
52 * enabled by default.
53 */
54 bool separate_tx_rx_irq = 1;
55 module_param(separate_tx_rx_irq, bool, 0644);
56
57 /*
58 * This is the maximum slots a skb can have. If a guest sends a skb
59 * which exceeds this limit it is considered malicious.
60 */
61 #define FATAL_SKB_SLOTS_DEFAULT 20
62 static unsigned int fatal_skb_slots = FATAL_SKB_SLOTS_DEFAULT;
63 module_param(fatal_skb_slots, uint, 0444);
64
65 /*
66 * To avoid confusion, we define XEN_NETBK_LEGACY_SLOTS_MAX indicating
67 * the maximum slots a valid packet can use. Now this value is defined
68 * to be XEN_NETIF_NR_SLOTS_MIN, which is supposed to be supported by
69 * all backend.
70 */
71 #define XEN_NETBK_LEGACY_SLOTS_MAX XEN_NETIF_NR_SLOTS_MIN
72
73 /*
74 * If head != INVALID_PENDING_RING_IDX, it means this tx request is head of
75 * one or more merged tx requests, otherwise it is the continuation of
76 * previous tx request.
77 */
78 static inline int pending_tx_is_head(struct xenvif *vif, RING_IDX idx)
79 {
80 return vif->pending_tx_info[idx].head != INVALID_PENDING_RING_IDX;
81 }
82
83 static void xenvif_idx_release(struct xenvif *vif, u16 pending_idx,
84 u8 status);
85
86 static void make_tx_response(struct xenvif *vif,
87 struct xen_netif_tx_request *txp,
88 s8 st);
89
90 static inline int tx_work_todo(struct xenvif *vif);
91 static inline int rx_work_todo(struct xenvif *vif);
92
93 static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif,
94 u16 id,
95 s8 st,
96 u16 offset,
97 u16 size,
98 u16 flags);
99
100 static inline unsigned long idx_to_pfn(struct xenvif *vif,
101 u16 idx)
102 {
103 return page_to_pfn(vif->mmap_pages[idx]);
104 }
105
106 static inline unsigned long idx_to_kaddr(struct xenvif *vif,
107 u16 idx)
108 {
109 return (unsigned long)pfn_to_kaddr(idx_to_pfn(vif, idx));
110 }
111
112 /* This is a miniumum size for the linear area to avoid lots of
113 * calls to __pskb_pull_tail() as we set up checksum offsets. The
114 * value 128 was chosen as it covers all IPv4 and most likely
115 * IPv6 headers.
116 */
117 #define PKT_PROT_LEN 128
118
119 static u16 frag_get_pending_idx(skb_frag_t *frag)
120 {
121 return (u16)frag->page_offset;
122 }
123
124 static void frag_set_pending_idx(skb_frag_t *frag, u16 pending_idx)
125 {
126 frag->page_offset = pending_idx;
127 }
128
129 static inline pending_ring_idx_t pending_index(unsigned i)
130 {
131 return i & (MAX_PENDING_REQS-1);
132 }
133
134 static inline pending_ring_idx_t nr_pending_reqs(struct xenvif *vif)
135 {
136 return MAX_PENDING_REQS -
137 vif->pending_prod + vif->pending_cons;
138 }
139
140 static int max_required_rx_slots(struct xenvif *vif)
141 {
142 int max = DIV_ROUND_UP(vif->dev->mtu, PAGE_SIZE);
143
144 /* XXX FIXME: RX path dependent on MAX_SKB_FRAGS */
145 if (vif->can_sg || vif->gso_mask || vif->gso_prefix_mask)
146 max += MAX_SKB_FRAGS + 1; /* extra_info + frags */
147
148 return max;
149 }
150
151 int xenvif_rx_ring_full(struct xenvif *vif)
152 {
153 RING_IDX peek = vif->rx_req_cons_peek;
154 RING_IDX needed = max_required_rx_slots(vif);
155
156 return ((vif->rx.sring->req_prod - peek) < needed) ||
157 ((vif->rx.rsp_prod_pvt + XEN_NETIF_RX_RING_SIZE - peek) < needed);
158 }
159
160 int xenvif_must_stop_queue(struct xenvif *vif)
161 {
162 if (!xenvif_rx_ring_full(vif))
163 return 0;
164
165 vif->rx.sring->req_event = vif->rx_req_cons_peek +
166 max_required_rx_slots(vif);
167 mb(); /* request notification /then/ check the queue */
168
169 return xenvif_rx_ring_full(vif);
170 }
171
172 /*
173 * Returns true if we should start a new receive buffer instead of
174 * adding 'size' bytes to a buffer which currently contains 'offset'
175 * bytes.
176 */
177 static bool start_new_rx_buffer(int offset, unsigned long size, int head)
178 {
179 /* simple case: we have completely filled the current buffer. */
180 if (offset == MAX_BUFFER_OFFSET)
181 return true;
182
183 /*
184 * complex case: start a fresh buffer if the current frag
185 * would overflow the current buffer but only if:
186 * (i) this frag would fit completely in the next buffer
187 * and (ii) there is already some data in the current buffer
188 * and (iii) this is not the head buffer.
189 *
190 * Where:
191 * - (i) stops us splitting a frag into two copies
192 * unless the frag is too large for a single buffer.
193 * - (ii) stops us from leaving a buffer pointlessly empty.
194 * - (iii) stops us leaving the first buffer
195 * empty. Strictly speaking this is already covered
196 * by (ii) but is explicitly checked because
197 * netfront relies on the first buffer being
198 * non-empty and can crash otherwise.
199 *
200 * This means we will effectively linearise small
201 * frags but do not needlessly split large buffers
202 * into multiple copies tend to give large frags their
203 * own buffers as before.
204 */
205 if ((offset + size > MAX_BUFFER_OFFSET) &&
206 (size <= MAX_BUFFER_OFFSET) && offset && !head)
207 return true;
208
209 return false;
210 }
211
212 struct xenvif_count_slot_state {
213 unsigned long copy_off;
214 bool head;
215 };
216
217 unsigned int xenvif_count_frag_slots(struct xenvif *vif,
218 unsigned long offset, unsigned long size,
219 struct xenvif_count_slot_state *state)
220 {
221 unsigned count = 0;
222
223 offset &= ~PAGE_MASK;
224
225 while (size > 0) {
226 unsigned long bytes;
227
228 bytes = PAGE_SIZE - offset;
229
230 if (bytes > size)
231 bytes = size;
232
233 if (start_new_rx_buffer(state->copy_off, bytes, state->head)) {
234 count++;
235 state->copy_off = 0;
236 }
237
238 if (state->copy_off + bytes > MAX_BUFFER_OFFSET)
239 bytes = MAX_BUFFER_OFFSET - state->copy_off;
240
241 state->copy_off += bytes;
242
243 offset += bytes;
244 size -= bytes;
245
246 if (offset == PAGE_SIZE)
247 offset = 0;
248
249 state->head = false;
250 }
251
252 return count;
253 }
254
255 /*
256 * Figure out how many ring slots we're going to need to send @skb to
257 * the guest. This function is essentially a dry run of
258 * xenvif_gop_frag_copy.
259 */
260 unsigned int xenvif_count_skb_slots(struct xenvif *vif, struct sk_buff *skb)
261 {
262 struct xenvif_count_slot_state state;
263 unsigned int count;
264 unsigned char *data;
265 unsigned i;
266
267 state.head = true;
268 state.copy_off = 0;
269
270 /* Slot for the first (partial) page of data. */
271 count = 1;
272
273 /* Need a slot for the GSO prefix for GSO extra data? */
274 if (skb_shinfo(skb)->gso_size)
275 count++;
276
277 data = skb->data;
278 while (data < skb_tail_pointer(skb)) {
279 unsigned long offset = offset_in_page(data);
280 unsigned long size = PAGE_SIZE - offset;
281
282 if (data + size > skb_tail_pointer(skb))
283 size = skb_tail_pointer(skb) - data;
284
285 count += xenvif_count_frag_slots(vif, offset, size, &state);
286
287 data += size;
288 }
289
290 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
291 unsigned long size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
292 unsigned long offset = skb_shinfo(skb)->frags[i].page_offset;
293
294 count += xenvif_count_frag_slots(vif, offset, size, &state);
295 }
296 return count;
297 }
298
299 struct netrx_pending_operations {
300 unsigned copy_prod, copy_cons;
301 unsigned meta_prod, meta_cons;
302 struct gnttab_copy *copy;
303 struct xenvif_rx_meta *meta;
304 int copy_off;
305 grant_ref_t copy_gref;
306 };
307
308 static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif *vif,
309 struct netrx_pending_operations *npo)
310 {
311 struct xenvif_rx_meta *meta;
312 struct xen_netif_rx_request *req;
313
314 req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
315
316 meta = npo->meta + npo->meta_prod++;
317 meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
318 meta->gso_size = 0;
319 meta->size = 0;
320 meta->id = req->id;
321
322 npo->copy_off = 0;
323 npo->copy_gref = req->gref;
324
325 return meta;
326 }
327
328 /*
329 * Set up the grant operations for this fragment. If it's a flipping
330 * interface, we also set up the unmap request from here.
331 */
332 static void xenvif_gop_frag_copy(struct xenvif *vif, struct sk_buff *skb,
333 struct netrx_pending_operations *npo,
334 struct page *page, unsigned long size,
335 unsigned long offset, int *head)
336 {
337 struct gnttab_copy *copy_gop;
338 struct xenvif_rx_meta *meta;
339 unsigned long bytes;
340 int gso_type;
341
342 /* Data must not cross a page boundary. */
343 BUG_ON(size + offset > PAGE_SIZE<<compound_order(page));
344
345 meta = npo->meta + npo->meta_prod - 1;
346
347 /* Skip unused frames from start of page */
348 page += offset >> PAGE_SHIFT;
349 offset &= ~PAGE_MASK;
350
351 while (size > 0) {
352 BUG_ON(offset >= PAGE_SIZE);
353 BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET);
354
355 bytes = PAGE_SIZE - offset;
356
357 if (bytes > size)
358 bytes = size;
359
360 if (start_new_rx_buffer(npo->copy_off, bytes, *head)) {
361 /*
362 * Netfront requires there to be some data in the head
363 * buffer.
364 */
365 BUG_ON(*head);
366
367 meta = get_next_rx_buffer(vif, npo);
368 }
369
370 if (npo->copy_off + bytes > MAX_BUFFER_OFFSET)
371 bytes = MAX_BUFFER_OFFSET - npo->copy_off;
372
373 copy_gop = npo->copy + npo->copy_prod++;
374 copy_gop->flags = GNTCOPY_dest_gref;
375 copy_gop->len = bytes;
376
377 copy_gop->source.domid = DOMID_SELF;
378 copy_gop->source.u.gmfn = virt_to_mfn(page_address(page));
379 copy_gop->source.offset = offset;
380
381 copy_gop->dest.domid = vif->domid;
382 copy_gop->dest.offset = npo->copy_off;
383 copy_gop->dest.u.ref = npo->copy_gref;
384
385 npo->copy_off += bytes;
386 meta->size += bytes;
387
388 offset += bytes;
389 size -= bytes;
390
391 /* Next frame */
392 if (offset == PAGE_SIZE && size) {
393 BUG_ON(!PageCompound(page));
394 page++;
395 offset = 0;
396 }
397
398 /* Leave a gap for the GSO descriptor. */
399 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
400 gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
401 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
402 gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
403 else
404 gso_type = XEN_NETIF_GSO_TYPE_NONE;
405
406 if (*head && ((1 << gso_type) & vif->gso_mask))
407 vif->rx.req_cons++;
408
409 *head = 0; /* There must be something in this buffer now. */
410
411 }
412 }
413
414 /*
415 * Prepare an SKB to be transmitted to the frontend.
416 *
417 * This function is responsible for allocating grant operations, meta
418 * structures, etc.
419 *
420 * It returns the number of meta structures consumed. The number of
421 * ring slots used is always equal to the number of meta slots used
422 * plus the number of GSO descriptors used. Currently, we use either
423 * zero GSO descriptors (for non-GSO packets) or one descriptor (for
424 * frontend-side LRO).
425 */
426 static int xenvif_gop_skb(struct sk_buff *skb,
427 struct netrx_pending_operations *npo)
428 {
429 struct xenvif *vif = netdev_priv(skb->dev);
430 int nr_frags = skb_shinfo(skb)->nr_frags;
431 int i;
432 struct xen_netif_rx_request *req;
433 struct xenvif_rx_meta *meta;
434 unsigned char *data;
435 int head = 1;
436 int old_meta_prod;
437 int gso_type;
438 int gso_size;
439
440 old_meta_prod = npo->meta_prod;
441
442 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
443 gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
444 gso_size = skb_shinfo(skb)->gso_size;
445 } else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
446 gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
447 gso_size = skb_shinfo(skb)->gso_size;
448 } else {
449 gso_type = XEN_NETIF_GSO_TYPE_NONE;
450 gso_size = 0;
451 }
452
453 /* Set up a GSO prefix descriptor, if necessary */
454 if ((1 << skb_shinfo(skb)->gso_type) & vif->gso_prefix_mask) {
455 req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
456 meta = npo->meta + npo->meta_prod++;
457 meta->gso_type = gso_type;
458 meta->gso_size = gso_size;
459 meta->size = 0;
460 meta->id = req->id;
461 }
462
463 req = RING_GET_REQUEST(&vif->rx, vif->rx.req_cons++);
464 meta = npo->meta + npo->meta_prod++;
465
466 if ((1 << gso_type) & vif->gso_mask) {
467 meta->gso_type = gso_type;
468 meta->gso_size = gso_size;
469 } else {
470 meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
471 meta->gso_size = 0;
472 }
473
474 meta->size = 0;
475 meta->id = req->id;
476 npo->copy_off = 0;
477 npo->copy_gref = req->gref;
478
479 data = skb->data;
480 while (data < skb_tail_pointer(skb)) {
481 unsigned int offset = offset_in_page(data);
482 unsigned int len = PAGE_SIZE - offset;
483
484 if (data + len > skb_tail_pointer(skb))
485 len = skb_tail_pointer(skb) - data;
486
487 xenvif_gop_frag_copy(vif, skb, npo,
488 virt_to_page(data), len, offset, &head);
489 data += len;
490 }
491
492 for (i = 0; i < nr_frags; i++) {
493 xenvif_gop_frag_copy(vif, skb, npo,
494 skb_frag_page(&skb_shinfo(skb)->frags[i]),
495 skb_frag_size(&skb_shinfo(skb)->frags[i]),
496 skb_shinfo(skb)->frags[i].page_offset,
497 &head);
498 }
499
500 return npo->meta_prod - old_meta_prod;
501 }
502
503 /*
504 * This is a twin to xenvif_gop_skb. Assume that xenvif_gop_skb was
505 * used to set up the operations on the top of
506 * netrx_pending_operations, which have since been done. Check that
507 * they didn't give any errors and advance over them.
508 */
509 static int xenvif_check_gop(struct xenvif *vif, int nr_meta_slots,
510 struct netrx_pending_operations *npo)
511 {
512 struct gnttab_copy *copy_op;
513 int status = XEN_NETIF_RSP_OKAY;
514 int i;
515
516 for (i = 0; i < nr_meta_slots; i++) {
517 copy_op = npo->copy + npo->copy_cons++;
518 if (copy_op->status != GNTST_okay) {
519 netdev_dbg(vif->dev,
520 "Bad status %d from copy to DOM%d.\n",
521 copy_op->status, vif->domid);
522 status = XEN_NETIF_RSP_ERROR;
523 }
524 }
525
526 return status;
527 }
528
529 static void xenvif_add_frag_responses(struct xenvif *vif, int status,
530 struct xenvif_rx_meta *meta,
531 int nr_meta_slots)
532 {
533 int i;
534 unsigned long offset;
535
536 /* No fragments used */
537 if (nr_meta_slots <= 1)
538 return;
539
540 nr_meta_slots--;
541
542 for (i = 0; i < nr_meta_slots; i++) {
543 int flags;
544 if (i == nr_meta_slots - 1)
545 flags = 0;
546 else
547 flags = XEN_NETRXF_more_data;
548
549 offset = 0;
550 make_rx_response(vif, meta[i].id, status, offset,
551 meta[i].size, flags);
552 }
553 }
554
555 struct skb_cb_overlay {
556 int meta_slots_used;
557 };
558
559 static void xenvif_kick_thread(struct xenvif *vif)
560 {
561 wake_up(&vif->wq);
562 }
563
564 void xenvif_rx_action(struct xenvif *vif)
565 {
566 s8 status;
567 u16 flags;
568 struct xen_netif_rx_response *resp;
569 struct sk_buff_head rxq;
570 struct sk_buff *skb;
571 LIST_HEAD(notify);
572 int ret;
573 int nr_frags;
574 int count;
575 unsigned long offset;
576 struct skb_cb_overlay *sco;
577 int need_to_notify = 0;
578
579 struct netrx_pending_operations npo = {
580 .copy = vif->grant_copy_op,
581 .meta = vif->meta,
582 };
583
584 skb_queue_head_init(&rxq);
585
586 count = 0;
587
588 while ((skb = skb_dequeue(&vif->rx_queue)) != NULL) {
589 vif = netdev_priv(skb->dev);
590 nr_frags = skb_shinfo(skb)->nr_frags;
591
592 sco = (struct skb_cb_overlay *)skb->cb;
593 sco->meta_slots_used = xenvif_gop_skb(skb, &npo);
594
595 count += nr_frags + 1;
596
597 __skb_queue_tail(&rxq, skb);
598
599 /* Filled the batch queue? */
600 /* XXX FIXME: RX path dependent on MAX_SKB_FRAGS */
601 if (count + MAX_SKB_FRAGS >= XEN_NETIF_RX_RING_SIZE)
602 break;
603 }
604
605 BUG_ON(npo.meta_prod > ARRAY_SIZE(vif->meta));
606
607 if (!npo.copy_prod)
608 return;
609
610 BUG_ON(npo.copy_prod > ARRAY_SIZE(vif->grant_copy_op));
611 gnttab_batch_copy(vif->grant_copy_op, npo.copy_prod);
612
613 while ((skb = __skb_dequeue(&rxq)) != NULL) {
614 sco = (struct skb_cb_overlay *)skb->cb;
615
616 vif = netdev_priv(skb->dev);
617
618 if ((1 << vif->meta[npo.meta_cons].gso_type) &
619 vif->gso_prefix_mask) {
620 resp = RING_GET_RESPONSE(&vif->rx,
621 vif->rx.rsp_prod_pvt++);
622
623 resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data;
624
625 resp->offset = vif->meta[npo.meta_cons].gso_size;
626 resp->id = vif->meta[npo.meta_cons].id;
627 resp->status = sco->meta_slots_used;
628
629 npo.meta_cons++;
630 sco->meta_slots_used--;
631 }
632
633
634 vif->dev->stats.tx_bytes += skb->len;
635 vif->dev->stats.tx_packets++;
636
637 status = xenvif_check_gop(vif, sco->meta_slots_used, &npo);
638
639 if (sco->meta_slots_used == 1)
640 flags = 0;
641 else
642 flags = XEN_NETRXF_more_data;
643
644 if (skb->ip_summed == CHECKSUM_PARTIAL) /* local packet? */
645 flags |= XEN_NETRXF_csum_blank | XEN_NETRXF_data_validated;
646 else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
647 /* remote but checksummed. */
648 flags |= XEN_NETRXF_data_validated;
649
650 offset = 0;
651 resp = make_rx_response(vif, vif->meta[npo.meta_cons].id,
652 status, offset,
653 vif->meta[npo.meta_cons].size,
654 flags);
655
656 if ((1 << vif->meta[npo.meta_cons].gso_type) &
657 vif->gso_mask) {
658 struct xen_netif_extra_info *gso =
659 (struct xen_netif_extra_info *)
660 RING_GET_RESPONSE(&vif->rx,
661 vif->rx.rsp_prod_pvt++);
662
663 resp->flags |= XEN_NETRXF_extra_info;
664
665 gso->u.gso.type = vif->meta[npo.meta_cons].gso_type;
666 gso->u.gso.size = vif->meta[npo.meta_cons].gso_size;
667 gso->u.gso.pad = 0;
668 gso->u.gso.features = 0;
669
670 gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
671 gso->flags = 0;
672 }
673
674 xenvif_add_frag_responses(vif, status,
675 vif->meta + npo.meta_cons + 1,
676 sco->meta_slots_used);
677
678 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->rx, ret);
679
680 if (ret)
681 need_to_notify = 1;
682
683 xenvif_notify_tx_completion(vif);
684
685 npo.meta_cons += sco->meta_slots_used;
686 dev_kfree_skb(skb);
687 }
688
689 if (need_to_notify)
690 notify_remote_via_irq(vif->rx_irq);
691
692 /* More work to do? */
693 if (!skb_queue_empty(&vif->rx_queue))
694 xenvif_kick_thread(vif);
695 }
696
697 void xenvif_queue_tx_skb(struct xenvif *vif, struct sk_buff *skb)
698 {
699 skb_queue_tail(&vif->rx_queue, skb);
700
701 xenvif_kick_thread(vif);
702 }
703
704 void xenvif_check_rx_xenvif(struct xenvif *vif)
705 {
706 int more_to_do;
707
708 RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, more_to_do);
709
710 if (more_to_do)
711 napi_schedule(&vif->napi);
712 }
713
714 static void tx_add_credit(struct xenvif *vif)
715 {
716 unsigned long max_burst, max_credit;
717
718 /*
719 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
720 * Otherwise the interface can seize up due to insufficient credit.
721 */
722 max_burst = RING_GET_REQUEST(&vif->tx, vif->tx.req_cons)->size;
723 max_burst = min(max_burst, 131072UL);
724 max_burst = max(max_burst, vif->credit_bytes);
725
726 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
727 max_credit = vif->remaining_credit + vif->credit_bytes;
728 if (max_credit < vif->remaining_credit)
729 max_credit = ULONG_MAX; /* wrapped: clamp to ULONG_MAX */
730
731 vif->remaining_credit = min(max_credit, max_burst);
732 }
733
734 static void tx_credit_callback(unsigned long data)
735 {
736 struct xenvif *vif = (struct xenvif *)data;
737 tx_add_credit(vif);
738 xenvif_check_rx_xenvif(vif);
739 }
740
741 static void xenvif_tx_err(struct xenvif *vif,
742 struct xen_netif_tx_request *txp, RING_IDX end)
743 {
744 RING_IDX cons = vif->tx.req_cons;
745
746 do {
747 make_tx_response(vif, txp, XEN_NETIF_RSP_ERROR);
748 if (cons == end)
749 break;
750 txp = RING_GET_REQUEST(&vif->tx, cons++);
751 } while (1);
752 vif->tx.req_cons = cons;
753 }
754
755 static void xenvif_fatal_tx_err(struct xenvif *vif)
756 {
757 netdev_err(vif->dev, "fatal error; disabling device\n");
758 xenvif_carrier_off(vif);
759 }
760
761 static int xenvif_count_requests(struct xenvif *vif,
762 struct xen_netif_tx_request *first,
763 struct xen_netif_tx_request *txp,
764 int work_to_do)
765 {
766 RING_IDX cons = vif->tx.req_cons;
767 int slots = 0;
768 int drop_err = 0;
769 int more_data;
770
771 if (!(first->flags & XEN_NETTXF_more_data))
772 return 0;
773
774 do {
775 struct xen_netif_tx_request dropped_tx = { 0 };
776
777 if (slots >= work_to_do) {
778 netdev_err(vif->dev,
779 "Asked for %d slots but exceeds this limit\n",
780 work_to_do);
781 xenvif_fatal_tx_err(vif);
782 return -ENODATA;
783 }
784
785 /* This guest is really using too many slots and
786 * considered malicious.
787 */
788 if (unlikely(slots >= fatal_skb_slots)) {
789 netdev_err(vif->dev,
790 "Malicious frontend using %d slots, threshold %u\n",
791 slots, fatal_skb_slots);
792 xenvif_fatal_tx_err(vif);
793 return -E2BIG;
794 }
795
796 /* Xen network protocol had implicit dependency on
797 * MAX_SKB_FRAGS. XEN_NETBK_LEGACY_SLOTS_MAX is set to
798 * the historical MAX_SKB_FRAGS value 18 to honor the
799 * same behavior as before. Any packet using more than
800 * 18 slots but less than fatal_skb_slots slots is
801 * dropped
802 */
803 if (!drop_err && slots >= XEN_NETBK_LEGACY_SLOTS_MAX) {
804 if (net_ratelimit())
805 netdev_dbg(vif->dev,
806 "Too many slots (%d) exceeding limit (%d), dropping packet\n",
807 slots, XEN_NETBK_LEGACY_SLOTS_MAX);
808 drop_err = -E2BIG;
809 }
810
811 if (drop_err)
812 txp = &dropped_tx;
813
814 memcpy(txp, RING_GET_REQUEST(&vif->tx, cons + slots),
815 sizeof(*txp));
816
817 /* If the guest submitted a frame >= 64 KiB then
818 * first->size overflowed and following slots will
819 * appear to be larger than the frame.
820 *
821 * This cannot be fatal error as there are buggy
822 * frontends that do this.
823 *
824 * Consume all slots and drop the packet.
825 */
826 if (!drop_err && txp->size > first->size) {
827 if (net_ratelimit())
828 netdev_dbg(vif->dev,
829 "Invalid tx request, slot size %u > remaining size %u\n",
830 txp->size, first->size);
831 drop_err = -EIO;
832 }
833
834 first->size -= txp->size;
835 slots++;
836
837 if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) {
838 netdev_err(vif->dev, "Cross page boundary, txp->offset: %x, size: %u\n",
839 txp->offset, txp->size);
840 xenvif_fatal_tx_err(vif);
841 return -EINVAL;
842 }
843
844 more_data = txp->flags & XEN_NETTXF_more_data;
845
846 if (!drop_err)
847 txp++;
848
849 } while (more_data);
850
851 if (drop_err) {
852 xenvif_tx_err(vif, first, cons + slots);
853 return drop_err;
854 }
855
856 return slots;
857 }
858
859 static struct page *xenvif_alloc_page(struct xenvif *vif,
860 u16 pending_idx)
861 {
862 struct page *page;
863
864 page = alloc_page(GFP_ATOMIC|__GFP_COLD);
865 if (!page)
866 return NULL;
867 vif->mmap_pages[pending_idx] = page;
868
869 return page;
870 }
871
872 static struct gnttab_copy *xenvif_get_requests(struct xenvif *vif,
873 struct sk_buff *skb,
874 struct xen_netif_tx_request *txp,
875 struct gnttab_copy *gop)
876 {
877 struct skb_shared_info *shinfo = skb_shinfo(skb);
878 skb_frag_t *frags = shinfo->frags;
879 u16 pending_idx = *((u16 *)skb->data);
880 u16 head_idx = 0;
881 int slot, start;
882 struct page *page;
883 pending_ring_idx_t index, start_idx = 0;
884 uint16_t dst_offset;
885 unsigned int nr_slots;
886 struct pending_tx_info *first = NULL;
887
888 /* At this point shinfo->nr_frags is in fact the number of
889 * slots, which can be as large as XEN_NETBK_LEGACY_SLOTS_MAX.
890 */
891 nr_slots = shinfo->nr_frags;
892
893 /* Skip first skb fragment if it is on same page as header fragment. */
894 start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
895
896 /* Coalesce tx requests, at this point the packet passed in
897 * should be <= 64K. Any packets larger than 64K have been
898 * handled in xenvif_count_requests().
899 */
900 for (shinfo->nr_frags = slot = start; slot < nr_slots;
901 shinfo->nr_frags++) {
902 struct pending_tx_info *pending_tx_info =
903 vif->pending_tx_info;
904
905 page = alloc_page(GFP_ATOMIC|__GFP_COLD);
906 if (!page)
907 goto err;
908
909 dst_offset = 0;
910 first = NULL;
911 while (dst_offset < PAGE_SIZE && slot < nr_slots) {
912 gop->flags = GNTCOPY_source_gref;
913
914 gop->source.u.ref = txp->gref;
915 gop->source.domid = vif->domid;
916 gop->source.offset = txp->offset;
917
918 gop->dest.domid = DOMID_SELF;
919
920 gop->dest.offset = dst_offset;
921 gop->dest.u.gmfn = virt_to_mfn(page_address(page));
922
923 if (dst_offset + txp->size > PAGE_SIZE) {
924 /* This page can only merge a portion
925 * of tx request. Do not increment any
926 * pointer / counter here. The txp
927 * will be dealt with in future
928 * rounds, eventually hitting the
929 * `else` branch.
930 */
931 gop->len = PAGE_SIZE - dst_offset;
932 txp->offset += gop->len;
933 txp->size -= gop->len;
934 dst_offset += gop->len; /* quit loop */
935 } else {
936 /* This tx request can be merged in the page */
937 gop->len = txp->size;
938 dst_offset += gop->len;
939
940 index = pending_index(vif->pending_cons++);
941
942 pending_idx = vif->pending_ring[index];
943
944 memcpy(&pending_tx_info[pending_idx].req, txp,
945 sizeof(*txp));
946
947 /* Poison these fields, corresponding
948 * fields for head tx req will be set
949 * to correct values after the loop.
950 */
951 vif->mmap_pages[pending_idx] = (void *)(~0UL);
952 pending_tx_info[pending_idx].head =
953 INVALID_PENDING_RING_IDX;
954
955 if (!first) {
956 first = &pending_tx_info[pending_idx];
957 start_idx = index;
958 head_idx = pending_idx;
959 }
960
961 txp++;
962 slot++;
963 }
964
965 gop++;
966 }
967
968 first->req.offset = 0;
969 first->req.size = dst_offset;
970 first->head = start_idx;
971 vif->mmap_pages[head_idx] = page;
972 frag_set_pending_idx(&frags[shinfo->nr_frags], head_idx);
973 }
974
975 BUG_ON(shinfo->nr_frags > MAX_SKB_FRAGS);
976
977 return gop;
978 err:
979 /* Unwind, freeing all pages and sending error responses. */
980 while (shinfo->nr_frags-- > start) {
981 xenvif_idx_release(vif,
982 frag_get_pending_idx(&frags[shinfo->nr_frags]),
983 XEN_NETIF_RSP_ERROR);
984 }
985 /* The head too, if necessary. */
986 if (start)
987 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
988
989 return NULL;
990 }
991
992 static int xenvif_tx_check_gop(struct xenvif *vif,
993 struct sk_buff *skb,
994 struct gnttab_copy **gopp)
995 {
996 struct gnttab_copy *gop = *gopp;
997 u16 pending_idx = *((u16 *)skb->data);
998 struct skb_shared_info *shinfo = skb_shinfo(skb);
999 struct pending_tx_info *tx_info;
1000 int nr_frags = shinfo->nr_frags;
1001 int i, err, start;
1002 u16 peek; /* peek into next tx request */
1003
1004 /* Check status of header. */
1005 err = gop->status;
1006 if (unlikely(err))
1007 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
1008
1009 /* Skip first skb fragment if it is on same page as header fragment. */
1010 start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
1011
1012 for (i = start; i < nr_frags; i++) {
1013 int j, newerr;
1014 pending_ring_idx_t head;
1015
1016 pending_idx = frag_get_pending_idx(&shinfo->frags[i]);
1017 tx_info = &vif->pending_tx_info[pending_idx];
1018 head = tx_info->head;
1019
1020 /* Check error status: if okay then remember grant handle. */
1021 do {
1022 newerr = (++gop)->status;
1023 if (newerr)
1024 break;
1025 peek = vif->pending_ring[pending_index(++head)];
1026 } while (!pending_tx_is_head(vif, peek));
1027
1028 if (likely(!newerr)) {
1029 /* Had a previous error? Invalidate this fragment. */
1030 if (unlikely(err))
1031 xenvif_idx_release(vif, pending_idx,
1032 XEN_NETIF_RSP_OKAY);
1033 continue;
1034 }
1035
1036 /* Error on this fragment: respond to client with an error. */
1037 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
1038
1039 /* Not the first error? Preceding frags already invalidated. */
1040 if (err)
1041 continue;
1042
1043 /* First error: invalidate header and preceding fragments. */
1044 pending_idx = *((u16 *)skb->data);
1045 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_OKAY);
1046 for (j = start; j < i; j++) {
1047 pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
1048 xenvif_idx_release(vif, pending_idx,
1049 XEN_NETIF_RSP_OKAY);
1050 }
1051
1052 /* Remember the error: invalidate all subsequent fragments. */
1053 err = newerr;
1054 }
1055
1056 *gopp = gop + 1;
1057 return err;
1058 }
1059
1060 static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb)
1061 {
1062 struct skb_shared_info *shinfo = skb_shinfo(skb);
1063 int nr_frags = shinfo->nr_frags;
1064 int i;
1065
1066 for (i = 0; i < nr_frags; i++) {
1067 skb_frag_t *frag = shinfo->frags + i;
1068 struct xen_netif_tx_request *txp;
1069 struct page *page;
1070 u16 pending_idx;
1071
1072 pending_idx = frag_get_pending_idx(frag);
1073
1074 txp = &vif->pending_tx_info[pending_idx].req;
1075 page = virt_to_page(idx_to_kaddr(vif, pending_idx));
1076 __skb_fill_page_desc(skb, i, page, txp->offset, txp->size);
1077 skb->len += txp->size;
1078 skb->data_len += txp->size;
1079 skb->truesize += txp->size;
1080
1081 /* Take an extra reference to offset xenvif_idx_release */
1082 get_page(vif->mmap_pages[pending_idx]);
1083 xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_OKAY);
1084 }
1085 }
1086
1087 static int xenvif_get_extras(struct xenvif *vif,
1088 struct xen_netif_extra_info *extras,
1089 int work_to_do)
1090 {
1091 struct xen_netif_extra_info extra;
1092 RING_IDX cons = vif->tx.req_cons;
1093
1094 do {
1095 if (unlikely(work_to_do-- <= 0)) {
1096 netdev_err(vif->dev, "Missing extra info\n");
1097 xenvif_fatal_tx_err(vif);
1098 return -EBADR;
1099 }
1100
1101 memcpy(&extra, RING_GET_REQUEST(&vif->tx, cons),
1102 sizeof(extra));
1103 if (unlikely(!extra.type ||
1104 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
1105 vif->tx.req_cons = ++cons;
1106 netdev_err(vif->dev,
1107 "Invalid extra type: %d\n", extra.type);
1108 xenvif_fatal_tx_err(vif);
1109 return -EINVAL;
1110 }
1111
1112 memcpy(&extras[extra.type - 1], &extra, sizeof(extra));
1113 vif->tx.req_cons = ++cons;
1114 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE);
1115
1116 return work_to_do;
1117 }
1118
1119 static int xenvif_set_skb_gso(struct xenvif *vif,
1120 struct sk_buff *skb,
1121 struct xen_netif_extra_info *gso)
1122 {
1123 if (!gso->u.gso.size) {
1124 netdev_err(vif->dev, "GSO size must not be zero.\n");
1125 xenvif_fatal_tx_err(vif);
1126 return -EINVAL;
1127 }
1128
1129 switch (gso->u.gso.type) {
1130 case XEN_NETIF_GSO_TYPE_TCPV4:
1131 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1132 break;
1133 case XEN_NETIF_GSO_TYPE_TCPV6:
1134 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
1135 break;
1136 default:
1137 netdev_err(vif->dev, "Bad GSO type %d.\n", gso->u.gso.type);
1138 xenvif_fatal_tx_err(vif);
1139 return -EINVAL;
1140 }
1141
1142 skb_shinfo(skb)->gso_size = gso->u.gso.size;
1143
1144 /* Header must be checked, and gso_segs computed. */
1145 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
1146 skb_shinfo(skb)->gso_segs = 0;
1147
1148 return 0;
1149 }
1150
1151 static inline void maybe_pull_tail(struct sk_buff *skb, unsigned int len)
1152 {
1153 if (skb_is_nonlinear(skb) && skb_headlen(skb) < len) {
1154 /* If we need to pullup then pullup to the max, so we
1155 * won't need to do it again.
1156 */
1157 int target = min_t(int, skb->len, MAX_TCP_HEADER);
1158 __pskb_pull_tail(skb, target - skb_headlen(skb));
1159 }
1160 }
1161
1162 static int checksum_setup_ip(struct xenvif *vif, struct sk_buff *skb,
1163 int recalculate_partial_csum)
1164 {
1165 struct iphdr *iph = (void *)skb->data;
1166 unsigned int header_size;
1167 unsigned int off;
1168 int err = -EPROTO;
1169
1170 off = sizeof(struct iphdr);
1171
1172 header_size = skb->network_header + off + MAX_IPOPTLEN;
1173 maybe_pull_tail(skb, header_size);
1174
1175 off = iph->ihl * 4;
1176
1177 switch (iph->protocol) {
1178 case IPPROTO_TCP:
1179 if (!skb_partial_csum_set(skb, off,
1180 offsetof(struct tcphdr, check)))
1181 goto out;
1182
1183 if (recalculate_partial_csum) {
1184 struct tcphdr *tcph = tcp_hdr(skb);
1185
1186 header_size = skb->network_header +
1187 off +
1188 sizeof(struct tcphdr);
1189 maybe_pull_tail(skb, header_size);
1190
1191 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
1192 skb->len - off,
1193 IPPROTO_TCP, 0);
1194 }
1195 break;
1196 case IPPROTO_UDP:
1197 if (!skb_partial_csum_set(skb, off,
1198 offsetof(struct udphdr, check)))
1199 goto out;
1200
1201 if (recalculate_partial_csum) {
1202 struct udphdr *udph = udp_hdr(skb);
1203
1204 header_size = skb->network_header +
1205 off +
1206 sizeof(struct udphdr);
1207 maybe_pull_tail(skb, header_size);
1208
1209 udph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
1210 skb->len - off,
1211 IPPROTO_UDP, 0);
1212 }
1213 break;
1214 default:
1215 if (net_ratelimit())
1216 netdev_err(vif->dev,
1217 "Attempting to checksum a non-TCP/UDP packet, "
1218 "dropping a protocol %d packet\n",
1219 iph->protocol);
1220 goto out;
1221 }
1222
1223 err = 0;
1224
1225 out:
1226 return err;
1227 }
1228
1229 static int checksum_setup_ipv6(struct xenvif *vif, struct sk_buff *skb,
1230 int recalculate_partial_csum)
1231 {
1232 int err = -EPROTO;
1233 struct ipv6hdr *ipv6h = (void *)skb->data;
1234 u8 nexthdr;
1235 unsigned int header_size;
1236 unsigned int off;
1237 bool fragment;
1238 bool done;
1239
1240 done = false;
1241
1242 off = sizeof(struct ipv6hdr);
1243
1244 header_size = skb->network_header + off;
1245 maybe_pull_tail(skb, header_size);
1246
1247 nexthdr = ipv6h->nexthdr;
1248
1249 while ((off <= sizeof(struct ipv6hdr) + ntohs(ipv6h->payload_len)) &&
1250 !done) {
1251 switch (nexthdr) {
1252 case IPPROTO_DSTOPTS:
1253 case IPPROTO_HOPOPTS:
1254 case IPPROTO_ROUTING: {
1255 struct ipv6_opt_hdr *hp = (void *)(skb->data + off);
1256
1257 header_size = skb->network_header +
1258 off +
1259 sizeof(struct ipv6_opt_hdr);
1260 maybe_pull_tail(skb, header_size);
1261
1262 nexthdr = hp->nexthdr;
1263 off += ipv6_optlen(hp);
1264 break;
1265 }
1266 case IPPROTO_AH: {
1267 struct ip_auth_hdr *hp = (void *)(skb->data + off);
1268
1269 header_size = skb->network_header +
1270 off +
1271 sizeof(struct ip_auth_hdr);
1272 maybe_pull_tail(skb, header_size);
1273
1274 nexthdr = hp->nexthdr;
1275 off += (hp->hdrlen+2)<<2;
1276 break;
1277 }
1278 case IPPROTO_FRAGMENT:
1279 fragment = true;
1280 /* fall through */
1281 default:
1282 done = true;
1283 break;
1284 }
1285 }
1286
1287 if (!done) {
1288 if (net_ratelimit())
1289 netdev_err(vif->dev, "Failed to parse packet header\n");
1290 goto out;
1291 }
1292
1293 if (fragment) {
1294 if (net_ratelimit())
1295 netdev_err(vif->dev, "Packet is a fragment!\n");
1296 goto out;
1297 }
1298
1299 switch (nexthdr) {
1300 case IPPROTO_TCP:
1301 if (!skb_partial_csum_set(skb, off,
1302 offsetof(struct tcphdr, check)))
1303 goto out;
1304
1305 if (recalculate_partial_csum) {
1306 struct tcphdr *tcph = tcp_hdr(skb);
1307
1308 header_size = skb->network_header +
1309 off +
1310 sizeof(struct tcphdr);
1311 maybe_pull_tail(skb, header_size);
1312
1313 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr,
1314 &ipv6h->daddr,
1315 skb->len - off,
1316 IPPROTO_TCP, 0);
1317 }
1318 break;
1319 case IPPROTO_UDP:
1320 if (!skb_partial_csum_set(skb, off,
1321 offsetof(struct udphdr, check)))
1322 goto out;
1323
1324 if (recalculate_partial_csum) {
1325 struct udphdr *udph = udp_hdr(skb);
1326
1327 header_size = skb->network_header +
1328 off +
1329 sizeof(struct udphdr);
1330 maybe_pull_tail(skb, header_size);
1331
1332 udph->check = ~csum_ipv6_magic(&ipv6h->saddr,
1333 &ipv6h->daddr,
1334 skb->len - off,
1335 IPPROTO_UDP, 0);
1336 }
1337 break;
1338 default:
1339 if (net_ratelimit())
1340 netdev_err(vif->dev,
1341 "Attempting to checksum a non-TCP/UDP packet, "
1342 "dropping a protocol %d packet\n",
1343 nexthdr);
1344 goto out;
1345 }
1346
1347 err = 0;
1348
1349 out:
1350 return err;
1351 }
1352
1353 static int checksum_setup(struct xenvif *vif, struct sk_buff *skb)
1354 {
1355 int err = -EPROTO;
1356 int recalculate_partial_csum = 0;
1357
1358 /* A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1359 * peers can fail to set NETRXF_csum_blank when sending a GSO
1360 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1361 * recalculate the partial checksum.
1362 */
1363 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) {
1364 vif->rx_gso_checksum_fixup++;
1365 skb->ip_summed = CHECKSUM_PARTIAL;
1366 recalculate_partial_csum = 1;
1367 }
1368
1369 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1370 if (skb->ip_summed != CHECKSUM_PARTIAL)
1371 return 0;
1372
1373 if (skb->protocol == htons(ETH_P_IP))
1374 err = checksum_setup_ip(vif, skb, recalculate_partial_csum);
1375 else if (skb->protocol == htons(ETH_P_IPV6))
1376 err = checksum_setup_ipv6(vif, skb, recalculate_partial_csum);
1377
1378 return err;
1379 }
1380
1381 static bool tx_credit_exceeded(struct xenvif *vif, unsigned size)
1382 {
1383 u64 now = get_jiffies_64();
1384 u64 next_credit = vif->credit_window_start +
1385 msecs_to_jiffies(vif->credit_usec / 1000);
1386
1387 /* Timer could already be pending in rare cases. */
1388 if (timer_pending(&vif->credit_timeout))
1389 return true;
1390
1391 /* Passed the point where we can replenish credit? */
1392 if (time_after_eq64(now, next_credit)) {
1393 vif->credit_window_start = now;
1394 tx_add_credit(vif);
1395 }
1396
1397 /* Still too big to send right now? Set a callback. */
1398 if (size > vif->remaining_credit) {
1399 vif->credit_timeout.data =
1400 (unsigned long)vif;
1401 vif->credit_timeout.function =
1402 tx_credit_callback;
1403 mod_timer(&vif->credit_timeout,
1404 next_credit);
1405 vif->credit_window_start = next_credit;
1406
1407 return true;
1408 }
1409
1410 return false;
1411 }
1412
1413 static unsigned xenvif_tx_build_gops(struct xenvif *vif)
1414 {
1415 struct gnttab_copy *gop = vif->tx_copy_ops, *request_gop;
1416 struct sk_buff *skb;
1417 int ret;
1418
1419 while ((nr_pending_reqs(vif) + XEN_NETBK_LEGACY_SLOTS_MAX
1420 < MAX_PENDING_REQS)) {
1421 struct xen_netif_tx_request txreq;
1422 struct xen_netif_tx_request txfrags[XEN_NETBK_LEGACY_SLOTS_MAX];
1423 struct page *page;
1424 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX-1];
1425 u16 pending_idx;
1426 RING_IDX idx;
1427 int work_to_do;
1428 unsigned int data_len;
1429 pending_ring_idx_t index;
1430
1431 if (vif->tx.sring->req_prod - vif->tx.req_cons >
1432 XEN_NETIF_TX_RING_SIZE) {
1433 netdev_err(vif->dev,
1434 "Impossible number of requests. "
1435 "req_prod %d, req_cons %d, size %ld\n",
1436 vif->tx.sring->req_prod, vif->tx.req_cons,
1437 XEN_NETIF_TX_RING_SIZE);
1438 xenvif_fatal_tx_err(vif);
1439 continue;
1440 }
1441
1442 RING_FINAL_CHECK_FOR_REQUESTS(&vif->tx, work_to_do);
1443 if (!work_to_do)
1444 break;
1445
1446 idx = vif->tx.req_cons;
1447 rmb(); /* Ensure that we see the request before we copy it. */
1448 memcpy(&txreq, RING_GET_REQUEST(&vif->tx, idx), sizeof(txreq));
1449
1450 /* Credit-based scheduling. */
1451 if (txreq.size > vif->remaining_credit &&
1452 tx_credit_exceeded(vif, txreq.size))
1453 break;
1454
1455 vif->remaining_credit -= txreq.size;
1456
1457 work_to_do--;
1458 vif->tx.req_cons = ++idx;
1459
1460 memset(extras, 0, sizeof(extras));
1461 if (txreq.flags & XEN_NETTXF_extra_info) {
1462 work_to_do = xenvif_get_extras(vif, extras,
1463 work_to_do);
1464 idx = vif->tx.req_cons;
1465 if (unlikely(work_to_do < 0))
1466 break;
1467 }
1468
1469 ret = xenvif_count_requests(vif, &txreq, txfrags, work_to_do);
1470 if (unlikely(ret < 0))
1471 break;
1472
1473 idx += ret;
1474
1475 if (unlikely(txreq.size < ETH_HLEN)) {
1476 netdev_dbg(vif->dev,
1477 "Bad packet size: %d\n", txreq.size);
1478 xenvif_tx_err(vif, &txreq, idx);
1479 break;
1480 }
1481
1482 /* No crossing a page as the payload mustn't fragment. */
1483 if (unlikely((txreq.offset + txreq.size) > PAGE_SIZE)) {
1484 netdev_err(vif->dev,
1485 "txreq.offset: %x, size: %u, end: %lu\n",
1486 txreq.offset, txreq.size,
1487 (txreq.offset&~PAGE_MASK) + txreq.size);
1488 xenvif_fatal_tx_err(vif);
1489 break;
1490 }
1491
1492 index = pending_index(vif->pending_cons);
1493 pending_idx = vif->pending_ring[index];
1494
1495 data_len = (txreq.size > PKT_PROT_LEN &&
1496 ret < XEN_NETBK_LEGACY_SLOTS_MAX) ?
1497 PKT_PROT_LEN : txreq.size;
1498
1499 skb = alloc_skb(data_len + NET_SKB_PAD + NET_IP_ALIGN,
1500 GFP_ATOMIC | __GFP_NOWARN);
1501 if (unlikely(skb == NULL)) {
1502 netdev_dbg(vif->dev,
1503 "Can't allocate a skb in start_xmit.\n");
1504 xenvif_tx_err(vif, &txreq, idx);
1505 break;
1506 }
1507
1508 /* Packets passed to netif_rx() must have some headroom. */
1509 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
1510
1511 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) {
1512 struct xen_netif_extra_info *gso;
1513 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
1514
1515 if (xenvif_set_skb_gso(vif, skb, gso)) {
1516 /* Failure in xenvif_set_skb_gso is fatal. */
1517 kfree_skb(skb);
1518 break;
1519 }
1520 }
1521
1522 /* XXX could copy straight to head */
1523 page = xenvif_alloc_page(vif, pending_idx);
1524 if (!page) {
1525 kfree_skb(skb);
1526 xenvif_tx_err(vif, &txreq, idx);
1527 break;
1528 }
1529
1530 gop->source.u.ref = txreq.gref;
1531 gop->source.domid = vif->domid;
1532 gop->source.offset = txreq.offset;
1533
1534 gop->dest.u.gmfn = virt_to_mfn(page_address(page));
1535 gop->dest.domid = DOMID_SELF;
1536 gop->dest.offset = txreq.offset;
1537
1538 gop->len = txreq.size;
1539 gop->flags = GNTCOPY_source_gref;
1540
1541 gop++;
1542
1543 memcpy(&vif->pending_tx_info[pending_idx].req,
1544 &txreq, sizeof(txreq));
1545 vif->pending_tx_info[pending_idx].head = index;
1546 *((u16 *)skb->data) = pending_idx;
1547
1548 __skb_put(skb, data_len);
1549
1550 skb_shinfo(skb)->nr_frags = ret;
1551 if (data_len < txreq.size) {
1552 skb_shinfo(skb)->nr_frags++;
1553 frag_set_pending_idx(&skb_shinfo(skb)->frags[0],
1554 pending_idx);
1555 } else {
1556 frag_set_pending_idx(&skb_shinfo(skb)->frags[0],
1557 INVALID_PENDING_IDX);
1558 }
1559
1560 vif->pending_cons++;
1561
1562 request_gop = xenvif_get_requests(vif, skb, txfrags, gop);
1563 if (request_gop == NULL) {
1564 kfree_skb(skb);
1565 xenvif_tx_err(vif, &txreq, idx);
1566 break;
1567 }
1568 gop = request_gop;
1569
1570 __skb_queue_tail(&vif->tx_queue, skb);
1571
1572 vif->tx.req_cons = idx;
1573
1574 if ((gop-vif->tx_copy_ops) >= ARRAY_SIZE(vif->tx_copy_ops))
1575 break;
1576 }
1577
1578 return gop - vif->tx_copy_ops;
1579 }
1580
1581
1582 static int xenvif_tx_submit(struct xenvif *vif, int budget)
1583 {
1584 struct gnttab_copy *gop = vif->tx_copy_ops;
1585 struct sk_buff *skb;
1586 int work_done = 0;
1587
1588 while (work_done < budget &&
1589 (skb = __skb_dequeue(&vif->tx_queue)) != NULL) {
1590 struct xen_netif_tx_request *txp;
1591 u16 pending_idx;
1592 unsigned data_len;
1593
1594 pending_idx = *((u16 *)skb->data);
1595 txp = &vif->pending_tx_info[pending_idx].req;
1596
1597 /* Check the remap error code. */
1598 if (unlikely(xenvif_tx_check_gop(vif, skb, &gop))) {
1599 netdev_dbg(vif->dev, "netback grant failed.\n");
1600 skb_shinfo(skb)->nr_frags = 0;
1601 kfree_skb(skb);
1602 continue;
1603 }
1604
1605 data_len = skb->len;
1606 memcpy(skb->data,
1607 (void *)(idx_to_kaddr(vif, pending_idx)|txp->offset),
1608 data_len);
1609 if (data_len < txp->size) {
1610 /* Append the packet payload as a fragment. */
1611 txp->offset += data_len;
1612 txp->size -= data_len;
1613 } else {
1614 /* Schedule a response immediately. */
1615 xenvif_idx_release(vif, pending_idx,
1616 XEN_NETIF_RSP_OKAY);
1617 }
1618
1619 if (txp->flags & XEN_NETTXF_csum_blank)
1620 skb->ip_summed = CHECKSUM_PARTIAL;
1621 else if (txp->flags & XEN_NETTXF_data_validated)
1622 skb->ip_summed = CHECKSUM_UNNECESSARY;
1623
1624 xenvif_fill_frags(vif, skb);
1625
1626 if (skb_is_nonlinear(skb) && skb_headlen(skb) < PKT_PROT_LEN) {
1627 int target = min_t(int, skb->len, PKT_PROT_LEN);
1628 __pskb_pull_tail(skb, target - skb_headlen(skb));
1629 }
1630
1631 skb->dev = vif->dev;
1632 skb->protocol = eth_type_trans(skb, skb->dev);
1633 skb_reset_network_header(skb);
1634
1635 if (checksum_setup(vif, skb)) {
1636 netdev_dbg(vif->dev,
1637 "Can't setup checksum in net_tx_action\n");
1638 kfree_skb(skb);
1639 continue;
1640 }
1641
1642 skb_probe_transport_header(skb, 0);
1643
1644 vif->dev->stats.rx_bytes += skb->len;
1645 vif->dev->stats.rx_packets++;
1646
1647 work_done++;
1648
1649 netif_receive_skb(skb);
1650 }
1651
1652 return work_done;
1653 }
1654
1655 /* Called after netfront has transmitted */
1656 int xenvif_tx_action(struct xenvif *vif, int budget)
1657 {
1658 unsigned nr_gops;
1659 int work_done;
1660
1661 if (unlikely(!tx_work_todo(vif)))
1662 return 0;
1663
1664 nr_gops = xenvif_tx_build_gops(vif);
1665
1666 if (nr_gops == 0)
1667 return 0;
1668
1669 gnttab_batch_copy(vif->tx_copy_ops, nr_gops);
1670
1671 work_done = xenvif_tx_submit(vif, nr_gops);
1672
1673 return work_done;
1674 }
1675
1676 static void xenvif_idx_release(struct xenvif *vif, u16 pending_idx,
1677 u8 status)
1678 {
1679 struct pending_tx_info *pending_tx_info;
1680 pending_ring_idx_t head;
1681 u16 peek; /* peek into next tx request */
1682
1683 BUG_ON(vif->mmap_pages[pending_idx] == (void *)(~0UL));
1684
1685 /* Already complete? */
1686 if (vif->mmap_pages[pending_idx] == NULL)
1687 return;
1688
1689 pending_tx_info = &vif->pending_tx_info[pending_idx];
1690
1691 head = pending_tx_info->head;
1692
1693 BUG_ON(!pending_tx_is_head(vif, head));
1694 BUG_ON(vif->pending_ring[pending_index(head)] != pending_idx);
1695
1696 do {
1697 pending_ring_idx_t index;
1698 pending_ring_idx_t idx = pending_index(head);
1699 u16 info_idx = vif->pending_ring[idx];
1700
1701 pending_tx_info = &vif->pending_tx_info[info_idx];
1702 make_tx_response(vif, &pending_tx_info->req, status);
1703
1704 /* Setting any number other than
1705 * INVALID_PENDING_RING_IDX indicates this slot is
1706 * starting a new packet / ending a previous packet.
1707 */
1708 pending_tx_info->head = 0;
1709
1710 index = pending_index(vif->pending_prod++);
1711 vif->pending_ring[index] = vif->pending_ring[info_idx];
1712
1713 peek = vif->pending_ring[pending_index(++head)];
1714
1715 } while (!pending_tx_is_head(vif, peek));
1716
1717 put_page(vif->mmap_pages[pending_idx]);
1718 vif->mmap_pages[pending_idx] = NULL;
1719 }
1720
1721
1722 static void make_tx_response(struct xenvif *vif,
1723 struct xen_netif_tx_request *txp,
1724 s8 st)
1725 {
1726 RING_IDX i = vif->tx.rsp_prod_pvt;
1727 struct xen_netif_tx_response *resp;
1728 int notify;
1729
1730 resp = RING_GET_RESPONSE(&vif->tx, i);
1731 resp->id = txp->id;
1732 resp->status = st;
1733
1734 if (txp->flags & XEN_NETTXF_extra_info)
1735 RING_GET_RESPONSE(&vif->tx, ++i)->status = XEN_NETIF_RSP_NULL;
1736
1737 vif->tx.rsp_prod_pvt = ++i;
1738 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&vif->tx, notify);
1739 if (notify)
1740 notify_remote_via_irq(vif->tx_irq);
1741 }
1742
1743 static struct xen_netif_rx_response *make_rx_response(struct xenvif *vif,
1744 u16 id,
1745 s8 st,
1746 u16 offset,
1747 u16 size,
1748 u16 flags)
1749 {
1750 RING_IDX i = vif->rx.rsp_prod_pvt;
1751 struct xen_netif_rx_response *resp;
1752
1753 resp = RING_GET_RESPONSE(&vif->rx, i);
1754 resp->offset = offset;
1755 resp->flags = flags;
1756 resp->id = id;
1757 resp->status = (s16)size;
1758 if (st < 0)
1759 resp->status = (s16)st;
1760
1761 vif->rx.rsp_prod_pvt = ++i;
1762
1763 return resp;
1764 }
1765
1766 static inline int rx_work_todo(struct xenvif *vif)
1767 {
1768 return !skb_queue_empty(&vif->rx_queue);
1769 }
1770
1771 static inline int tx_work_todo(struct xenvif *vif)
1772 {
1773
1774 if (likely(RING_HAS_UNCONSUMED_REQUESTS(&vif->tx)) &&
1775 (nr_pending_reqs(vif) + XEN_NETBK_LEGACY_SLOTS_MAX
1776 < MAX_PENDING_REQS))
1777 return 1;
1778
1779 return 0;
1780 }
1781
1782 void xenvif_unmap_frontend_rings(struct xenvif *vif)
1783 {
1784 if (vif->tx.sring)
1785 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif),
1786 vif->tx.sring);
1787 if (vif->rx.sring)
1788 xenbus_unmap_ring_vfree(xenvif_to_xenbus_device(vif),
1789 vif->rx.sring);
1790 }
1791
1792 int xenvif_map_frontend_rings(struct xenvif *vif,
1793 grant_ref_t tx_ring_ref,
1794 grant_ref_t rx_ring_ref)
1795 {
1796 void *addr;
1797 struct xen_netif_tx_sring *txs;
1798 struct xen_netif_rx_sring *rxs;
1799
1800 int err = -ENOMEM;
1801
1802 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif),
1803 tx_ring_ref, &addr);
1804 if (err)
1805 goto err;
1806
1807 txs = (struct xen_netif_tx_sring *)addr;
1808 BACK_RING_INIT(&vif->tx, txs, PAGE_SIZE);
1809
1810 err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif),
1811 rx_ring_ref, &addr);
1812 if (err)
1813 goto err;
1814
1815 rxs = (struct xen_netif_rx_sring *)addr;
1816 BACK_RING_INIT(&vif->rx, rxs, PAGE_SIZE);
1817
1818 vif->rx_req_cons_peek = 0;
1819
1820 return 0;
1821
1822 err:
1823 xenvif_unmap_frontend_rings(vif);
1824 return err;
1825 }
1826
1827 int xenvif_kthread(void *data)
1828 {
1829 struct xenvif *vif = data;
1830
1831 while (!kthread_should_stop()) {
1832 wait_event_interruptible(vif->wq,
1833 rx_work_todo(vif) ||
1834 kthread_should_stop());
1835 if (kthread_should_stop())
1836 break;
1837
1838 if (rx_work_todo(vif))
1839 xenvif_rx_action(vif);
1840
1841 cond_resched();
1842 }
1843
1844 return 0;
1845 }
1846
1847 static int __init netback_init(void)
1848 {
1849 int rc = 0;
1850
1851 if (!xen_domain())
1852 return -ENODEV;
1853
1854 if (fatal_skb_slots < XEN_NETBK_LEGACY_SLOTS_MAX) {
1855 pr_info("fatal_skb_slots too small (%d), bump it to XEN_NETBK_LEGACY_SLOTS_MAX (%d)\n",
1856 fatal_skb_slots, XEN_NETBK_LEGACY_SLOTS_MAX);
1857 fatal_skb_slots = XEN_NETBK_LEGACY_SLOTS_MAX;
1858 }
1859
1860 rc = xenvif_xenbus_init();
1861 if (rc)
1862 goto failed_init;
1863
1864 return 0;
1865
1866 failed_init:
1867 return rc;
1868 }
1869
1870 module_init(netback_init);
1871
1872 static void __exit netback_fini(void)
1873 {
1874 xenvif_xenbus_fini();
1875 }
1876 module_exit(netback_fini);
1877
1878 MODULE_LICENSE("Dual BSD/GPL");
1879 MODULE_ALIAS("xen-backend:vif");
Linux kernel - Deliverables
This page took 0.06685 seconds and 4 git commands to generate.