1 /* sunvnet.c: Sun LDOM Virtual Network Driver.
3 * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
6 #include <linux/module.h>
7 #include <linux/kernel.h>
8 #include <linux/types.h>
9 #include <linux/slab.h>
10 #include <linux/delay.h>
11 #include <linux/init.h>
12 #include <linux/netdevice.h>
13 #include <linux/ethtool.h>
14 #include <linux/etherdevice.h>
15 #include <linux/mutex.h>
16 #include <linux/highmem.h>
17 #include <linux/if_vlan.h>
18 #define CREATE_TRACE_POINTS
19 #include <trace/events/sunvnet.h>
21 #if IS_ENABLED(CONFIG_IPV6)
22 #include <linux/icmpv6.h>
27 #include <net/route.h>
32 #include "sunvnet_common.h"
34 /* Heuristic for the number of times to exponentially backoff and
35 * retry sending an LDC trigger when EAGAIN is encountered
37 #define VNET_MAX_RETRIES 10
39 static int __vnet_tx_trigger(struct vnet_port
*port
, u32 start
);
40 static void vnet_port_reset(struct vnet_port
*port
);
42 static inline u32
vnet_tx_dring_avail(struct vio_dring_state
*dr
)
44 return vio_dring_avail(dr
, VNET_TX_RING_SIZE
);
47 static int vnet_handle_unknown(struct vnet_port
*port
, void *arg
)
49 struct vio_msg_tag
*pkt
= arg
;
51 pr_err("Received unknown msg [%02x:%02x:%04x:%08x]\n",
52 pkt
->type
, pkt
->stype
, pkt
->stype_env
, pkt
->sid
);
53 pr_err("Resetting connection\n");
55 ldc_disconnect(port
->vio
.lp
);
60 static int vnet_port_alloc_tx_ring(struct vnet_port
*port
);
62 int sunvnet_send_attr_common(struct vio_driver_state
*vio
)
64 struct vnet_port
*port
= to_vnet_port(vio
);
65 struct net_device
*dev
= port
->vp
->dev
;
66 struct vio_net_attr_info pkt
;
67 int framelen
= ETH_FRAME_LEN
;
70 err
= vnet_port_alloc_tx_ring(to_vnet_port(vio
));
74 memset(&pkt
, 0, sizeof(pkt
));
75 pkt
.tag
.type
= VIO_TYPE_CTRL
;
76 pkt
.tag
.stype
= VIO_SUBTYPE_INFO
;
77 pkt
.tag
.stype_env
= VIO_ATTR_INFO
;
78 pkt
.tag
.sid
= vio_send_sid(vio
);
79 if (vio_version_before(vio
, 1, 2))
80 pkt
.xfer_mode
= VIO_DRING_MODE
;
82 pkt
.xfer_mode
= VIO_NEW_DRING_MODE
;
83 pkt
.addr_type
= VNET_ADDR_ETHERMAC
;
85 for (i
= 0; i
< 6; i
++)
86 pkt
.addr
|= (u64
)dev
->dev_addr
[i
] << ((5 - i
) * 8);
87 if (vio_version_after(vio
, 1, 3)) {
89 port
->rmtu
= min(VNET_MAXPACKET
, port
->rmtu
);
92 port
->rmtu
= VNET_MAXPACKET
;
95 if (vio_version_after_eq(vio
, 1, 6))
96 pkt
.options
= VIO_TX_DRING
;
97 } else if (vio_version_before(vio
, 1, 3)) {
100 pkt
.mtu
= framelen
+ VLAN_HLEN
;
104 if (vio_version_after_eq(vio
, 1, 7) && port
->tso
) {
105 pkt
.cflags
|= VNET_LSO_IPV4_CAPAB
;
107 port
->tsolen
= VNET_MAXTSO
;
108 pkt
.ipv4_lso_maxlen
= port
->tsolen
;
111 pkt
.plnk_updt
= PHYSLINK_UPDATE_NONE
;
113 viodbg(HS
, "SEND NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] "
114 "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] "
115 "cflags[0x%04x] lso_max[%u]\n",
116 pkt
.xfer_mode
, pkt
.addr_type
,
117 (unsigned long long)pkt
.addr
,
118 pkt
.ack_freq
, pkt
.plnk_updt
, pkt
.options
,
119 (unsigned long long)pkt
.mtu
, pkt
.cflags
, pkt
.ipv4_lso_maxlen
);
122 return vio_ldc_send(vio
, &pkt
, sizeof(pkt
));
124 EXPORT_SYMBOL_GPL(sunvnet_send_attr_common
);
126 static int handle_attr_info(struct vio_driver_state
*vio
,
127 struct vio_net_attr_info
*pkt
)
129 struct vnet_port
*port
= to_vnet_port(vio
);
133 viodbg(HS
, "GOT NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] "
134 "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] "
135 " (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n",
136 pkt
->xfer_mode
, pkt
->addr_type
,
137 (unsigned long long)pkt
->addr
,
138 pkt
->ack_freq
, pkt
->plnk_updt
, pkt
->options
,
139 (unsigned long long)pkt
->mtu
, port
->rmtu
, pkt
->cflags
,
140 pkt
->ipv4_lso_maxlen
);
142 pkt
->tag
.sid
= vio_send_sid(vio
);
144 xfer_mode
= pkt
->xfer_mode
;
145 /* for version < 1.2, VIO_DRING_MODE = 0x3 and no bitmask */
146 if (vio_version_before(vio
, 1, 2) && xfer_mode
== VIO_DRING_MODE
)
147 xfer_mode
= VIO_NEW_DRING_MODE
;
150 * < v1.3 - ETH_FRAME_LEN exactly
151 * > v1.3 - MIN(pkt.mtu, VNET_MAXPACKET, port->rmtu) and change
153 * = v1.3 - ETH_FRAME_LEN + VLAN_HLEN exactly
155 if (vio_version_before(vio
, 1, 3)) {
156 localmtu
= ETH_FRAME_LEN
;
157 } else if (vio_version_after(vio
, 1, 3)) {
158 localmtu
= port
->rmtu
? port
->rmtu
: VNET_MAXPACKET
;
159 localmtu
= min(pkt
->mtu
, localmtu
);
162 localmtu
= ETH_FRAME_LEN
+ VLAN_HLEN
;
164 port
->rmtu
= localmtu
;
166 /* LSO negotiation */
167 if (vio_version_after_eq(vio
, 1, 7))
168 port
->tso
&= !!(pkt
->cflags
& VNET_LSO_IPV4_CAPAB
);
173 port
->tsolen
= VNET_MAXTSO
;
174 port
->tsolen
= min(port
->tsolen
, pkt
->ipv4_lso_maxlen
);
175 if (port
->tsolen
< VNET_MINTSO
) {
178 pkt
->cflags
&= ~VNET_LSO_IPV4_CAPAB
;
180 pkt
->ipv4_lso_maxlen
= port
->tsolen
;
182 pkt
->cflags
&= ~VNET_LSO_IPV4_CAPAB
;
183 pkt
->ipv4_lso_maxlen
= 0;
186 /* for version >= 1.6, ACK packet mode we support */
187 if (vio_version_after_eq(vio
, 1, 6)) {
188 pkt
->xfer_mode
= VIO_NEW_DRING_MODE
;
189 pkt
->options
= VIO_TX_DRING
;
192 if (!(xfer_mode
| VIO_NEW_DRING_MODE
) ||
193 pkt
->addr_type
!= VNET_ADDR_ETHERMAC
||
194 pkt
->mtu
!= localmtu
) {
195 viodbg(HS
, "SEND NET ATTR NACK\n");
197 pkt
->tag
.stype
= VIO_SUBTYPE_NACK
;
199 (void) vio_ldc_send(vio
, pkt
, sizeof(*pkt
));
203 viodbg(HS
, "SEND NET ATTR ACK xmode[0x%x] atype[0x%x] "
204 "addr[%llx] ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] "
205 "mtu[%llu] (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n",
206 pkt
->xfer_mode
, pkt
->addr_type
,
207 (unsigned long long)pkt
->addr
,
208 pkt
->ack_freq
, pkt
->plnk_updt
, pkt
->options
,
209 (unsigned long long)pkt
->mtu
, port
->rmtu
, pkt
->cflags
,
210 pkt
->ipv4_lso_maxlen
);
212 pkt
->tag
.stype
= VIO_SUBTYPE_ACK
;
214 return vio_ldc_send(vio
, pkt
, sizeof(*pkt
));
219 static int handle_attr_ack(struct vio_driver_state
*vio
,
220 struct vio_net_attr_info
*pkt
)
222 viodbg(HS
, "GOT NET ATTR ACK\n");
227 static int handle_attr_nack(struct vio_driver_state
*vio
,
228 struct vio_net_attr_info
*pkt
)
230 viodbg(HS
, "GOT NET ATTR NACK\n");
235 int sunvnet_handle_attr_common(struct vio_driver_state
*vio
, void *arg
)
237 struct vio_net_attr_info
*pkt
= arg
;
239 switch (pkt
->tag
.stype
) {
240 case VIO_SUBTYPE_INFO
:
241 return handle_attr_info(vio
, pkt
);
243 case VIO_SUBTYPE_ACK
:
244 return handle_attr_ack(vio
, pkt
);
246 case VIO_SUBTYPE_NACK
:
247 return handle_attr_nack(vio
, pkt
);
253 EXPORT_SYMBOL_GPL(sunvnet_handle_attr_common
);
255 void sunvnet_handshake_complete_common(struct vio_driver_state
*vio
)
257 struct vio_dring_state
*dr
;
259 dr
= &vio
->drings
[VIO_DRIVER_RX_RING
];
260 dr
->snd_nxt
= dr
->rcv_nxt
= 1;
262 dr
= &vio
->drings
[VIO_DRIVER_TX_RING
];
263 dr
->snd_nxt
= dr
->rcv_nxt
= 1;
265 EXPORT_SYMBOL_GPL(sunvnet_handshake_complete_common
);
267 /* The hypervisor interface that implements copying to/from imported
268 * memory from another domain requires that copies are done to 8-byte
269 * aligned buffers, and that the lengths of such copies are also 8-byte
272 * So we align skb->data to an 8-byte multiple and pad-out the data
273 * area so we can round the copy length up to the next multiple of
276 * The transmitter puts the actual start of the packet 6 bytes into
277 * the buffer it sends over, so that the IP headers after the ethernet
278 * header are aligned properly. These 6 bytes are not in the descriptor
279 * length, they are simply implied. This offset is represented using
280 * the VNET_PACKET_SKIP macro.
282 static struct sk_buff
*alloc_and_align_skb(struct net_device
*dev
,
285 struct sk_buff
*skb
= netdev_alloc_skb(dev
, len
+VNET_PACKET_SKIP
+8+8);
286 unsigned long addr
, off
;
291 addr
= (unsigned long) skb
->data
;
292 off
= ((addr
+ 7UL) & ~7UL) - addr
;
294 skb_reserve(skb
, off
);
299 static inline void vnet_fullcsum(struct sk_buff
*skb
)
301 struct iphdr
*iph
= ip_hdr(skb
);
302 int offset
= skb_transport_offset(skb
);
304 if (skb
->protocol
!= htons(ETH_P_IP
))
306 if (iph
->protocol
!= IPPROTO_TCP
&&
307 iph
->protocol
!= IPPROTO_UDP
)
309 skb
->ip_summed
= CHECKSUM_NONE
;
312 if (iph
->protocol
== IPPROTO_TCP
) {
313 struct tcphdr
*ptcp
= tcp_hdr(skb
);
316 skb
->csum
= skb_checksum(skb
, offset
, skb
->len
- offset
, 0);
317 ptcp
->check
= csum_tcpudp_magic(iph
->saddr
, iph
->daddr
,
318 skb
->len
- offset
, IPPROTO_TCP
,
320 } else if (iph
->protocol
== IPPROTO_UDP
) {
321 struct udphdr
*pudp
= udp_hdr(skb
);
324 skb
->csum
= skb_checksum(skb
, offset
, skb
->len
- offset
, 0);
325 pudp
->check
= csum_tcpudp_magic(iph
->saddr
, iph
->daddr
,
326 skb
->len
- offset
, IPPROTO_UDP
,
331 static int vnet_rx_one(struct vnet_port
*port
, struct vio_net_desc
*desc
)
333 struct net_device
*dev
= port
->vp
->dev
;
334 unsigned int len
= desc
->size
;
335 unsigned int copy_len
;
341 if (port
->tso
&& port
->tsolen
> port
->rmtu
)
342 maxlen
= port
->tsolen
;
345 if (unlikely(len
< ETH_ZLEN
|| len
> maxlen
)) {
346 dev
->stats
.rx_length_errors
++;
350 skb
= alloc_and_align_skb(dev
, len
);
352 if (unlikely(!skb
)) {
353 dev
->stats
.rx_missed_errors
++;
357 copy_len
= (len
+ VNET_PACKET_SKIP
+ 7U) & ~7U;
358 skb_put(skb
, copy_len
);
359 err
= ldc_copy(port
->vio
.lp
, LDC_COPY_IN
,
360 skb
->data
, copy_len
, 0,
361 desc
->cookies
, desc
->ncookies
);
362 if (unlikely(err
< 0)) {
363 dev
->stats
.rx_frame_errors
++;
367 skb_pull(skb
, VNET_PACKET_SKIP
);
369 skb
->protocol
= eth_type_trans(skb
, dev
);
371 if (vio_version_after_eq(&port
->vio
, 1, 8)) {
372 struct vio_net_dext
*dext
= vio_net_ext(desc
);
374 skb_reset_network_header(skb
);
376 if (dext
->flags
& VNET_PKT_HCK_IPV4_HDRCKSUM
) {
377 if (skb
->protocol
== ETH_P_IP
) {
378 struct iphdr
*iph
= ip_hdr(skb
);
384 if ((dext
->flags
& VNET_PKT_HCK_FULLCKSUM
) &&
385 skb
->ip_summed
== CHECKSUM_NONE
) {
386 if (skb
->protocol
== htons(ETH_P_IP
)) {
387 struct iphdr
*iph
= ip_hdr(skb
);
388 int ihl
= iph
->ihl
* 4;
390 skb_reset_transport_header(skb
);
391 skb_set_transport_header(skb
, ihl
);
395 if (dext
->flags
& VNET_PKT_HCK_IPV4_HDRCKSUM_OK
) {
396 skb
->ip_summed
= CHECKSUM_PARTIAL
;
398 if (dext
->flags
& VNET_PKT_HCK_FULLCKSUM_OK
)
403 skb
->ip_summed
= port
->switch_port
? CHECKSUM_NONE
: CHECKSUM_PARTIAL
;
405 dev
->stats
.rx_packets
++;
406 dev
->stats
.rx_bytes
+= len
;
407 napi_gro_receive(&port
->napi
, skb
);
414 dev
->stats
.rx_dropped
++;
418 static int vnet_send_ack(struct vnet_port
*port
, struct vio_dring_state
*dr
,
419 u32 start
, u32 end
, u8 vio_dring_state
)
421 struct vio_dring_data hdr
= {
423 .type
= VIO_TYPE_DATA
,
424 .stype
= VIO_SUBTYPE_ACK
,
425 .stype_env
= VIO_DRING_DATA
,
426 .sid
= vio_send_sid(&port
->vio
),
428 .dring_ident
= dr
->ident
,
431 .state
= vio_dring_state
,
436 hdr
.seq
= dr
->snd_nxt
;
439 err
= vio_ldc_send(&port
->vio
, &hdr
, sizeof(hdr
));
445 if ((delay
<<= 1) > 128)
447 if (retries
++ > VNET_MAX_RETRIES
) {
448 pr_info("ECONNRESET %x:%x:%x:%x:%x:%x\n",
449 port
->raddr
[0], port
->raddr
[1],
450 port
->raddr
[2], port
->raddr
[3],
451 port
->raddr
[4], port
->raddr
[5]);
454 } while (err
== -EAGAIN
);
456 if (err
<= 0 && vio_dring_state
== VIO_DRING_STOPPED
) {
457 port
->stop_rx_idx
= end
;
458 port
->stop_rx
= true;
460 port
->stop_rx_idx
= 0;
461 port
->stop_rx
= false;
467 static struct vio_net_desc
*get_rx_desc(struct vnet_port
*port
,
468 struct vio_dring_state
*dr
,
471 struct vio_net_desc
*desc
= port
->vio
.desc_buf
;
474 err
= ldc_get_dring_entry(port
->vio
.lp
, desc
, dr
->entry_size
,
475 (index
* dr
->entry_size
),
476 dr
->cookies
, dr
->ncookies
);
483 static int put_rx_desc(struct vnet_port
*port
,
484 struct vio_dring_state
*dr
,
485 struct vio_net_desc
*desc
,
490 err
= ldc_put_dring_entry(port
->vio
.lp
, desc
, dr
->entry_size
,
491 (index
* dr
->entry_size
),
492 dr
->cookies
, dr
->ncookies
);
499 static int vnet_walk_rx_one(struct vnet_port
*port
,
500 struct vio_dring_state
*dr
,
501 u32 index
, int *needs_ack
)
503 struct vio_net_desc
*desc
= get_rx_desc(port
, dr
, index
);
504 struct vio_driver_state
*vio
= &port
->vio
;
507 BUG_ON(desc
== NULL
);
509 return PTR_ERR(desc
);
511 if (desc
->hdr
.state
!= VIO_DESC_READY
)
516 viodbg(DATA
, "vio_walk_rx_one desc[%02x:%02x:%08x:%08x:%llx:%llx]\n",
517 desc
->hdr
.state
, desc
->hdr
.ack
,
518 desc
->size
, desc
->ncookies
,
519 desc
->cookies
[0].cookie_addr
,
520 desc
->cookies
[0].cookie_size
);
522 err
= vnet_rx_one(port
, desc
);
523 if (err
== -ECONNRESET
)
525 trace_vnet_rx_one(port
->vio
._local_sid
, port
->vio
._peer_sid
,
526 index
, desc
->hdr
.ack
);
527 desc
->hdr
.state
= VIO_DESC_DONE
;
528 err
= put_rx_desc(port
, dr
, desc
, index
);
531 *needs_ack
= desc
->hdr
.ack
;
535 static int vnet_walk_rx(struct vnet_port
*port
, struct vio_dring_state
*dr
,
536 u32 start
, u32 end
, int *npkts
, int budget
)
538 struct vio_driver_state
*vio
= &port
->vio
;
539 int ack_start
= -1, ack_end
= -1;
540 bool send_ack
= true;
542 end
= (end
== (u32
) -1) ? vio_dring_prev(dr
, start
)
543 : vio_dring_next(dr
, end
);
545 viodbg(DATA
, "vnet_walk_rx start[%08x] end[%08x]\n", start
, end
);
547 while (start
!= end
) {
548 int ack
= 0, err
= vnet_walk_rx_one(port
, dr
, start
, &ack
);
549 if (err
== -ECONNRESET
)
557 start
= vio_dring_next(dr
, start
);
558 if (ack
&& start
!= end
) {
559 err
= vnet_send_ack(port
, dr
, ack_start
, ack_end
,
561 if (err
== -ECONNRESET
)
565 if ((*npkts
) >= budget
) {
570 if (unlikely(ack_start
== -1))
571 ack_start
= ack_end
= vio_dring_prev(dr
, start
);
573 port
->napi_resume
= false;
574 trace_vnet_tx_send_stopped_ack(port
->vio
._local_sid
,
577 return vnet_send_ack(port
, dr
, ack_start
, ack_end
,
580 trace_vnet_tx_defer_stopped_ack(port
->vio
._local_sid
,
583 port
->napi_resume
= true;
584 port
->napi_stop_idx
= ack_end
;
589 static int vnet_rx(struct vnet_port
*port
, void *msgbuf
, int *npkts
,
592 struct vio_dring_data
*pkt
= msgbuf
;
593 struct vio_dring_state
*dr
= &port
->vio
.drings
[VIO_DRIVER_RX_RING
];
594 struct vio_driver_state
*vio
= &port
->vio
;
596 viodbg(DATA
, "vnet_rx stype_env[%04x] seq[%016llx] rcv_nxt[%016llx]\n",
597 pkt
->tag
.stype_env
, pkt
->seq
, dr
->rcv_nxt
);
599 if (unlikely(pkt
->tag
.stype_env
!= VIO_DRING_DATA
))
601 if (unlikely(pkt
->seq
!= dr
->rcv_nxt
)) {
602 pr_err("RX out of sequence seq[0x%llx] rcv_nxt[0x%llx]\n",
603 pkt
->seq
, dr
->rcv_nxt
);
607 if (!port
->napi_resume
)
610 /* XXX Validate pkt->start_idx and pkt->end_idx XXX */
612 return vnet_walk_rx(port
, dr
, pkt
->start_idx
, pkt
->end_idx
,
616 static int idx_is_pending(struct vio_dring_state
*dr
, u32 end
)
621 while (idx
!= dr
->prod
) {
626 idx
= vio_dring_next(dr
, idx
);
631 static int vnet_ack(struct vnet_port
*port
, void *msgbuf
)
633 struct vio_dring_state
*dr
= &port
->vio
.drings
[VIO_DRIVER_TX_RING
];
634 struct vio_dring_data
*pkt
= msgbuf
;
635 struct net_device
*dev
;
638 struct vio_net_desc
*desc
;
639 struct netdev_queue
*txq
;
641 if (unlikely(pkt
->tag
.stype_env
!= VIO_DRING_DATA
))
648 if (unlikely(!idx_is_pending(dr
, end
))) {
649 netif_tx_unlock(dev
);
653 /* sync for race conditions with vnet_start_xmit() and tell xmit it
654 * is time to send a trigger.
656 trace_vnet_rx_stopped_ack(port
->vio
._local_sid
,
657 port
->vio
._peer_sid
, end
);
658 dr
->cons
= vio_dring_next(dr
, end
);
659 desc
= vio_dring_entry(dr
, dr
->cons
);
660 if (desc
->hdr
.state
== VIO_DESC_READY
&& !port
->start_cons
) {
661 /* vnet_start_xmit() just populated this dring but missed
662 * sending the "start" LDC message to the consumer.
663 * Send a "start" trigger on its behalf.
665 if (__vnet_tx_trigger(port
, dr
->cons
) > 0)
666 port
->start_cons
= false;
668 port
->start_cons
= true;
670 port
->start_cons
= true;
672 netif_tx_unlock(dev
);
674 txq
= netdev_get_tx_queue(dev
, port
->q_index
);
675 if (unlikely(netif_tx_queue_stopped(txq
) &&
676 vnet_tx_dring_avail(dr
) >= VNET_TX_WAKEUP_THRESH(dr
)))
682 static int vnet_nack(struct vnet_port
*port
, void *msgbuf
)
684 /* XXX just reset or similar XXX */
688 static int handle_mcast(struct vnet_port
*port
, void *msgbuf
)
690 struct vio_net_mcast_info
*pkt
= msgbuf
;
692 if (pkt
->tag
.stype
!= VIO_SUBTYPE_ACK
)
693 pr_err("%s: Got unexpected MCAST reply [%02x:%02x:%04x:%08x]\n",
703 /* Got back a STOPPED LDC message on port. If the queue is stopped,
704 * wake it up so that we'll send out another START message at the
707 static void maybe_tx_wakeup(struct vnet_port
*port
)
709 struct netdev_queue
*txq
;
711 txq
= netdev_get_tx_queue(port
->vp
->dev
, port
->q_index
);
712 __netif_tx_lock(txq
, smp_processor_id());
713 if (likely(netif_tx_queue_stopped(txq
))) {
714 struct vio_dring_state
*dr
;
716 dr
= &port
->vio
.drings
[VIO_DRIVER_TX_RING
];
717 netif_tx_wake_queue(txq
);
719 __netif_tx_unlock(txq
);
722 static inline bool port_is_up(struct vnet_port
*vnet
)
724 struct vio_driver_state
*vio
= &vnet
->vio
;
726 return !!(vio
->hs_state
& VIO_HS_COMPLETE
);
729 static int vnet_event_napi(struct vnet_port
*port
, int budget
)
731 struct vio_driver_state
*vio
= &port
->vio
;
734 int event
= (port
->rx_event
& LDC_EVENT_RESET
);
737 if (unlikely(event
== LDC_EVENT_RESET
||
738 event
== LDC_EVENT_UP
)) {
739 vio_link_state_change(vio
, event
);
741 if (event
== LDC_EVENT_RESET
) {
742 vnet_port_reset(port
);
748 /* We may have multiple LDC events in rx_event. Unroll send_events() */
749 event
= (port
->rx_event
& LDC_EVENT_UP
);
750 port
->rx_event
&= ~(LDC_EVENT_RESET
|LDC_EVENT_UP
);
751 if (event
== LDC_EVENT_UP
)
753 event
= port
->rx_event
;
754 if (!(event
& LDC_EVENT_DATA_READY
))
757 /* we dont expect any other bits than RESET, UP, DATA_READY */
758 BUG_ON(event
!= LDC_EVENT_DATA_READY
);
763 struct vio_msg_tag tag
;
767 if (port
->napi_resume
) {
768 struct vio_dring_data
*pkt
=
769 (struct vio_dring_data
*)&msgbuf
;
770 struct vio_dring_state
*dr
=
771 &port
->vio
.drings
[VIO_DRIVER_RX_RING
];
773 pkt
->tag
.type
= VIO_TYPE_DATA
;
774 pkt
->tag
.stype
= VIO_SUBTYPE_INFO
;
775 pkt
->tag
.stype_env
= VIO_DRING_DATA
;
776 pkt
->seq
= dr
->rcv_nxt
;
777 pkt
->start_idx
= vio_dring_next(dr
, port
->napi_stop_idx
);
781 err
= ldc_read(vio
->lp
, &msgbuf
, sizeof(msgbuf
));
782 if (unlikely(err
< 0)) {
783 if (err
== -ECONNRESET
)
789 viodbg(DATA
, "TAG [%02x:%02x:%04x:%08x]\n",
792 msgbuf
.tag
.stype_env
,
794 err
= vio_validate_sid(vio
, &msgbuf
.tag
);
798 if (likely(msgbuf
.tag
.type
== VIO_TYPE_DATA
)) {
799 if (msgbuf
.tag
.stype
== VIO_SUBTYPE_INFO
) {
800 if (!port_is_up(port
)) {
801 /* failures like handshake_failure()
802 * may have cleaned up dring, but
803 * NAPI polling may bring us here.
808 err
= vnet_rx(port
, &msgbuf
, &npkts
, budget
);
813 } else if (msgbuf
.tag
.stype
== VIO_SUBTYPE_ACK
) {
814 err
= vnet_ack(port
, &msgbuf
);
817 } else if (msgbuf
.tag
.stype
== VIO_SUBTYPE_NACK
) {
818 err
= vnet_nack(port
, &msgbuf
);
820 } else if (msgbuf
.tag
.type
== VIO_TYPE_CTRL
) {
821 if (msgbuf
.tag
.stype_env
== VNET_MCAST_INFO
)
822 err
= handle_mcast(port
, &msgbuf
);
824 err
= vio_control_pkt_engine(vio
, &msgbuf
);
828 err
= vnet_handle_unknown(port
, &msgbuf
);
830 if (err
== -ECONNRESET
)
833 if (unlikely(tx_wakeup
&& err
!= -ECONNRESET
))
834 maybe_tx_wakeup(port
);
838 int sunvnet_poll_common(struct napi_struct
*napi
, int budget
)
840 struct vnet_port
*port
= container_of(napi
, struct vnet_port
, napi
);
841 struct vio_driver_state
*vio
= &port
->vio
;
842 int processed
= vnet_event_napi(port
, budget
);
844 if (processed
< budget
) {
846 port
->rx_event
&= ~LDC_EVENT_DATA_READY
;
847 vio_set_intr(vio
->vdev
->rx_ino
, HV_INTR_ENABLED
);
851 EXPORT_SYMBOL_GPL(sunvnet_poll_common
);
853 void sunvnet_event_common(void *arg
, int event
)
855 struct vnet_port
*port
= arg
;
856 struct vio_driver_state
*vio
= &port
->vio
;
858 port
->rx_event
|= event
;
859 vio_set_intr(vio
->vdev
->rx_ino
, HV_INTR_DISABLED
);
860 napi_schedule(&port
->napi
);
863 EXPORT_SYMBOL_GPL(sunvnet_event_common
);
865 static int __vnet_tx_trigger(struct vnet_port
*port
, u32 start
)
867 struct vio_dring_state
*dr
= &port
->vio
.drings
[VIO_DRIVER_TX_RING
];
868 struct vio_dring_data hdr
= {
870 .type
= VIO_TYPE_DATA
,
871 .stype
= VIO_SUBTYPE_INFO
,
872 .stype_env
= VIO_DRING_DATA
,
873 .sid
= vio_send_sid(&port
->vio
),
875 .dring_ident
= dr
->ident
,
883 trace_vnet_tx_pending_stopped_ack(port
->vio
._local_sid
,
885 port
->stop_rx_idx
, -1);
886 err
= vnet_send_ack(port
,
887 &port
->vio
.drings
[VIO_DRIVER_RX_RING
],
888 port
->stop_rx_idx
, -1,
894 hdr
.seq
= dr
->snd_nxt
;
897 err
= vio_ldc_send(&port
->vio
, &hdr
, sizeof(hdr
));
903 if ((delay
<<= 1) > 128)
905 if (retries
++ > VNET_MAX_RETRIES
)
907 } while (err
== -EAGAIN
);
908 trace_vnet_tx_trigger(port
->vio
._local_sid
,
909 port
->vio
._peer_sid
, start
, err
);
914 static struct vnet_port
*__tx_port_find(struct vnet
*vp
, struct sk_buff
*skb
)
916 unsigned int hash
= vnet_hashfn(skb
->data
);
917 struct hlist_head
*hp
= &vp
->port_hash
[hash
];
918 struct vnet_port
*port
;
920 hlist_for_each_entry_rcu(port
, hp
, hash
) {
921 if (!port_is_up(port
))
923 if (ether_addr_equal(port
->raddr
, skb
->data
))
926 list_for_each_entry_rcu(port
, &vp
->port_list
, list
) {
927 if (!port
->switch_port
)
929 if (!port_is_up(port
))
936 static struct sk_buff
*vnet_clean_tx_ring(struct vnet_port
*port
,
939 struct vio_dring_state
*dr
= &port
->vio
.drings
[VIO_DRIVER_TX_RING
];
940 struct sk_buff
*skb
= NULL
;
946 for (i
= 0; i
< VNET_TX_RING_SIZE
; ++i
) {
947 struct vio_net_desc
*d
;
951 txi
= VNET_TX_RING_SIZE
-1;
953 d
= vio_dring_entry(dr
, txi
);
955 if (d
->hdr
.state
== VIO_DESC_READY
) {
959 if (port
->tx_bufs
[txi
].skb
) {
960 if (d
->hdr
.state
!= VIO_DESC_DONE
)
961 pr_notice("invalid ring buffer state %d\n",
963 BUG_ON(port
->tx_bufs
[txi
].skb
->next
);
965 port
->tx_bufs
[txi
].skb
->next
= skb
;
966 skb
= port
->tx_bufs
[txi
].skb
;
967 port
->tx_bufs
[txi
].skb
= NULL
;
969 ldc_unmap(port
->vio
.lp
,
970 port
->tx_bufs
[txi
].cookies
,
971 port
->tx_bufs
[txi
].ncookies
);
972 } else if (d
->hdr
.state
== VIO_DESC_FREE
)
974 d
->hdr
.state
= VIO_DESC_FREE
;
979 static inline void vnet_free_skbs(struct sk_buff
*skb
)
981 struct sk_buff
*next
;
991 void sunvnet_clean_timer_expire_common(unsigned long port0
)
993 struct vnet_port
*port
= (struct vnet_port
*)port0
;
994 struct sk_buff
*freeskbs
;
997 netif_tx_lock(port
->vp
->dev
);
998 freeskbs
= vnet_clean_tx_ring(port
, &pending
);
999 netif_tx_unlock(port
->vp
->dev
);
1001 vnet_free_skbs(freeskbs
);
1004 (void)mod_timer(&port
->clean_timer
,
1005 jiffies
+ VNET_CLEAN_TIMEOUT
);
1007 del_timer(&port
->clean_timer
);
1009 EXPORT_SYMBOL_GPL(sunvnet_clean_timer_expire_common
);
1011 static inline int vnet_skb_map(struct ldc_channel
*lp
, struct sk_buff
*skb
,
1012 struct ldc_trans_cookie
*cookies
, int ncookies
,
1013 unsigned int map_perm
)
1015 int i
, nc
, err
, blen
;
1018 blen
= skb_headlen(skb
);
1019 if (blen
< ETH_ZLEN
)
1021 blen
+= VNET_PACKET_SKIP
;
1022 blen
+= 8 - (blen
& 7);
1024 err
= ldc_map_single(lp
, skb
->data
-VNET_PACKET_SKIP
, blen
, cookies
,
1025 ncookies
, map_perm
);
1030 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
1031 skb_frag_t
*f
= &skb_shinfo(skb
)->frags
[i
];
1034 if (nc
< ncookies
) {
1035 vaddr
= kmap_atomic(skb_frag_page(f
));
1036 blen
= skb_frag_size(f
);
1037 blen
+= 8 - (blen
& 7);
1038 err
= ldc_map_single(lp
, vaddr
+ f
->page_offset
,
1039 blen
, cookies
+ nc
, ncookies
- nc
,
1041 kunmap_atomic(vaddr
);
1047 ldc_unmap(lp
, cookies
, nc
);
1055 static inline struct sk_buff
*vnet_skb_shape(struct sk_buff
*skb
, int ncookies
)
1057 struct sk_buff
*nskb
;
1058 int i
, len
, pad
, docopy
;
1062 if (len
< ETH_ZLEN
) {
1063 pad
+= ETH_ZLEN
- skb
->len
;
1066 len
+= VNET_PACKET_SKIP
;
1067 pad
+= 8 - (len
& 7);
1069 /* make sure we have enough cookies and alignment in every frag */
1070 docopy
= skb_shinfo(skb
)->nr_frags
>= ncookies
;
1071 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
1072 skb_frag_t
*f
= &skb_shinfo(skb
)->frags
[i
];
1074 docopy
|= f
->page_offset
& 7;
1076 if (((unsigned long)skb
->data
& 7) != VNET_PACKET_SKIP
||
1077 skb_tailroom(skb
) < pad
||
1078 skb_headroom(skb
) < VNET_PACKET_SKIP
|| docopy
) {
1079 int start
= 0, offset
;
1082 len
= skb
->len
> ETH_ZLEN
? skb
->len
: ETH_ZLEN
;
1083 nskb
= alloc_and_align_skb(skb
->dev
, len
);
1088 skb_reserve(nskb
, VNET_PACKET_SKIP
);
1090 nskb
->protocol
= skb
->protocol
;
1091 offset
= skb_mac_header(skb
) - skb
->data
;
1092 skb_set_mac_header(nskb
, offset
);
1093 offset
= skb_network_header(skb
) - skb
->data
;
1094 skb_set_network_header(nskb
, offset
);
1095 offset
= skb_transport_header(skb
) - skb
->data
;
1096 skb_set_transport_header(nskb
, offset
);
1099 nskb
->csum_offset
= skb
->csum_offset
;
1100 nskb
->ip_summed
= skb
->ip_summed
;
1102 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
1103 start
= skb_checksum_start_offset(skb
);
1105 struct iphdr
*iph
= ip_hdr(nskb
);
1106 int offset
= start
+ nskb
->csum_offset
;
1108 if (skb_copy_bits(skb
, 0, nskb
->data
, start
)) {
1109 dev_kfree_skb(nskb
);
1113 *(__sum16
*)(skb
->data
+ offset
) = 0;
1114 csum
= skb_copy_and_csum_bits(skb
, start
,
1116 skb
->len
- start
, 0);
1117 if (iph
->protocol
== IPPROTO_TCP
||
1118 iph
->protocol
== IPPROTO_UDP
) {
1119 csum
= csum_tcpudp_magic(iph
->saddr
, iph
->daddr
,
1121 iph
->protocol
, csum
);
1123 *(__sum16
*)(nskb
->data
+ offset
) = csum
;
1125 nskb
->ip_summed
= CHECKSUM_NONE
;
1126 } else if (skb_copy_bits(skb
, 0, nskb
->data
, skb
->len
)) {
1127 dev_kfree_skb(nskb
);
1131 (void)skb_put(nskb
, skb
->len
);
1132 if (skb_is_gso(skb
)) {
1133 skb_shinfo(nskb
)->gso_size
= skb_shinfo(skb
)->gso_size
;
1134 skb_shinfo(nskb
)->gso_type
= skb_shinfo(skb
)->gso_type
;
1136 nskb
->queue_mapping
= skb
->queue_mapping
;
1143 u16
sunvnet_select_queue_common(struct net_device
*dev
, struct sk_buff
*skb
,
1144 void *accel_priv
, select_queue_fallback_t fallback
)
1146 struct vnet
*vp
= netdev_priv(dev
);
1147 struct vnet_port
*port
= __tx_port_find(vp
, skb
);
1151 return port
->q_index
;
1153 EXPORT_SYMBOL_GPL(sunvnet_select_queue_common
);
1155 static int vnet_handle_offloads(struct vnet_port
*port
, struct sk_buff
*skb
)
1157 struct net_device
*dev
= port
->vp
->dev
;
1158 struct vio_dring_state
*dr
= &port
->vio
.drings
[VIO_DRIVER_TX_RING
];
1159 struct sk_buff
*segs
;
1160 int maclen
, datalen
;
1162 int gso_size
, gso_type
, gso_segs
;
1163 int hlen
= skb_transport_header(skb
) - skb_mac_header(skb
);
1164 int proto
= IPPROTO_IP
;
1166 if (skb
->protocol
== htons(ETH_P_IP
))
1167 proto
= ip_hdr(skb
)->protocol
;
1168 else if (skb
->protocol
== htons(ETH_P_IPV6
))
1169 proto
= ipv6_hdr(skb
)->nexthdr
;
1171 if (proto
== IPPROTO_TCP
)
1172 hlen
+= tcp_hdr(skb
)->doff
* 4;
1173 else if (proto
== IPPROTO_UDP
)
1174 hlen
+= sizeof(struct udphdr
);
1176 pr_err("vnet_handle_offloads GSO with unknown transport "
1177 "protocol %d tproto %d\n", skb
->protocol
, proto
);
1178 hlen
= 128; /* XXX */
1180 datalen
= port
->tsolen
- hlen
;
1182 gso_size
= skb_shinfo(skb
)->gso_size
;
1183 gso_type
= skb_shinfo(skb
)->gso_type
;
1184 gso_segs
= skb_shinfo(skb
)->gso_segs
;
1186 if (port
->tso
&& gso_size
< datalen
)
1187 gso_segs
= DIV_ROUND_UP(skb
->len
- hlen
, datalen
);
1189 if (unlikely(vnet_tx_dring_avail(dr
) < gso_segs
)) {
1190 struct netdev_queue
*txq
;
1192 txq
= netdev_get_tx_queue(dev
, port
->q_index
);
1193 netif_tx_stop_queue(txq
);
1194 if (vnet_tx_dring_avail(dr
) < skb_shinfo(skb
)->gso_segs
)
1195 return NETDEV_TX_BUSY
;
1196 netif_tx_wake_queue(txq
);
1199 maclen
= skb_network_header(skb
) - skb_mac_header(skb
);
1200 skb_pull(skb
, maclen
);
1202 if (port
->tso
&& gso_size
< datalen
) {
1203 if (skb_unclone(skb
, GFP_ATOMIC
))
1206 /* segment to TSO size */
1207 skb_shinfo(skb
)->gso_size
= datalen
;
1208 skb_shinfo(skb
)->gso_segs
= gso_segs
;
1210 segs
= skb_gso_segment(skb
, dev
->features
& ~NETIF_F_TSO
);
1214 skb_push(skb
, maclen
);
1215 skb_reset_mac_header(skb
);
1219 struct sk_buff
*curr
= segs
;
1223 if (port
->tso
&& curr
->len
> dev
->mtu
) {
1224 skb_shinfo(curr
)->gso_size
= gso_size
;
1225 skb_shinfo(curr
)->gso_type
= gso_type
;
1226 skb_shinfo(curr
)->gso_segs
=
1227 DIV_ROUND_UP(curr
->len
- hlen
, gso_size
);
1229 skb_shinfo(curr
)->gso_size
= 0;
1231 skb_push(curr
, maclen
);
1232 skb_reset_mac_header(curr
);
1233 memcpy(skb_mac_header(curr
), skb_mac_header(skb
),
1235 curr
->csum_start
= skb_transport_header(curr
) - curr
->head
;
1236 if (ip_hdr(curr
)->protocol
== IPPROTO_TCP
)
1237 curr
->csum_offset
= offsetof(struct tcphdr
, check
);
1238 else if (ip_hdr(curr
)->protocol
== IPPROTO_UDP
)
1239 curr
->csum_offset
= offsetof(struct udphdr
, check
);
1241 if (!(status
& NETDEV_TX_MASK
))
1242 status
= sunvnet_start_xmit_common(curr
, dev
);
1243 if (status
& NETDEV_TX_MASK
)
1244 dev_kfree_skb_any(curr
);
1247 if (!(status
& NETDEV_TX_MASK
))
1248 dev_kfree_skb_any(skb
);
1251 dev
->stats
.tx_dropped
++;
1252 dev_kfree_skb_any(skb
);
1253 return NETDEV_TX_OK
;
1256 int sunvnet_start_xmit_common(struct sk_buff
*skb
, struct net_device
*dev
)
1258 struct vnet
*vp
= netdev_priv(dev
);
1259 struct vnet_port
*port
= NULL
;
1260 struct vio_dring_state
*dr
;
1261 struct vio_net_desc
*d
;
1263 struct sk_buff
*freeskbs
= NULL
;
1265 unsigned pending
= 0;
1266 struct netdev_queue
*txq
;
1269 port
= __tx_port_find(vp
, skb
);
1270 if (unlikely(!port
)) {
1275 if (skb_is_gso(skb
) && skb
->len
> port
->tsolen
) {
1276 err
= vnet_handle_offloads(port
, skb
);
1281 if (!skb_is_gso(skb
) && skb
->len
> port
->rmtu
) {
1282 unsigned long localmtu
= port
->rmtu
- ETH_HLEN
;
1284 if (vio_version_after_eq(&port
->vio
, 1, 3))
1285 localmtu
-= VLAN_HLEN
;
1287 if (skb
->protocol
== htons(ETH_P_IP
)) {
1289 struct rtable
*rt
= NULL
;
1291 memset(&fl4
, 0, sizeof(fl4
));
1292 fl4
.flowi4_oif
= dev
->ifindex
;
1293 fl4
.flowi4_tos
= RT_TOS(ip_hdr(skb
)->tos
);
1294 fl4
.daddr
= ip_hdr(skb
)->daddr
;
1295 fl4
.saddr
= ip_hdr(skb
)->saddr
;
1297 rt
= ip_route_output_key(dev_net(dev
), &fl4
);
1300 skb_dst_set(skb
, &rt
->dst
);
1301 icmp_send(skb
, ICMP_DEST_UNREACH
,
1306 #if IS_ENABLED(CONFIG_IPV6)
1307 else if (skb
->protocol
== htons(ETH_P_IPV6
))
1308 icmpv6_send(skb
, ICMPV6_PKT_TOOBIG
, 0, localmtu
);
1313 skb
= vnet_skb_shape(skb
, 2);
1318 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
1321 dr
= &port
->vio
.drings
[VIO_DRIVER_TX_RING
];
1322 i
= skb_get_queue_mapping(skb
);
1323 txq
= netdev_get_tx_queue(dev
, i
);
1324 if (unlikely(vnet_tx_dring_avail(dr
) < 1)) {
1325 if (!netif_tx_queue_stopped(txq
)) {
1326 netif_tx_stop_queue(txq
);
1328 /* This is a hard error, log it. */
1329 netdev_err(dev
, "BUG! Tx Ring full when queue awake!\n");
1330 dev
->stats
.tx_errors
++;
1333 return NETDEV_TX_BUSY
;
1336 d
= vio_dring_cur(dr
);
1340 freeskbs
= vnet_clean_tx_ring(port
, &pending
);
1342 BUG_ON(port
->tx_bufs
[txi
].skb
);
1348 err
= vnet_skb_map(port
->vio
.lp
, skb
, port
->tx_bufs
[txi
].cookies
, 2,
1349 (LDC_MAP_SHADOW
| LDC_MAP_DIRECT
| LDC_MAP_RW
));
1351 netdev_info(dev
, "tx buffer map error %d\n", err
);
1355 port
->tx_bufs
[txi
].skb
= skb
;
1357 port
->tx_bufs
[txi
].ncookies
= err
;
1359 /* We don't rely on the ACKs to free the skb in vnet_start_xmit(),
1360 * thus it is safe to not set VIO_ACK_ENABLE for each transmission:
1361 * the protocol itself does not require it as long as the peer
1362 * sends a VIO_SUBTYPE_ACK for VIO_DRING_STOPPED.
1364 * An ACK for every packet in the ring is expensive as the
1365 * sending of LDC messages is slow and affects performance.
1367 d
->hdr
.ack
= VIO_ACK_DISABLE
;
1369 d
->ncookies
= port
->tx_bufs
[txi
].ncookies
;
1370 for (i
= 0; i
< d
->ncookies
; i
++)
1371 d
->cookies
[i
] = port
->tx_bufs
[txi
].cookies
[i
];
1372 if (vio_version_after_eq(&port
->vio
, 1, 7)) {
1373 struct vio_net_dext
*dext
= vio_net_ext(d
);
1375 memset(dext
, 0, sizeof(*dext
));
1376 if (skb_is_gso(port
->tx_bufs
[txi
].skb
)) {
1377 dext
->ipv4_lso_mss
= skb_shinfo(port
->tx_bufs
[txi
].skb
)
1379 dext
->flags
|= VNET_PKT_IPV4_LSO
;
1381 if (vio_version_after_eq(&port
->vio
, 1, 8) &&
1382 !port
->switch_port
) {
1383 dext
->flags
|= VNET_PKT_HCK_IPV4_HDRCKSUM_OK
;
1384 dext
->flags
|= VNET_PKT_HCK_FULLCKSUM_OK
;
1388 /* This has to be a non-SMP write barrier because we are writing
1389 * to memory which is shared with the peer LDOM.
1393 d
->hdr
.state
= VIO_DESC_READY
;
1395 /* Exactly one ldc "start" trigger (for dr->cons) needs to be sent
1396 * to notify the consumer that some descriptors are READY.
1397 * After that "start" trigger, no additional triggers are needed until
1398 * a DRING_STOPPED is received from the consumer. The dr->cons field
1399 * (set up by vnet_ack()) has the value of the next dring index
1400 * that has not yet been ack-ed. We send a "start" trigger here
1401 * if, and only if, start_cons is true (reset it afterward). Conversely,
1402 * vnet_ack() should check if the dring corresponding to cons
1403 * is marked READY, but start_cons was false.
1404 * If so, vnet_ack() should send out the missed "start" trigger.
1406 * Note that the dma_wmb() above makes sure the cookies et al. are
1407 * not globally visible before the VIO_DESC_READY, and that the
1408 * stores are ordered correctly by the compiler. The consumer will
1409 * not proceed until the VIO_DESC_READY is visible assuring that
1410 * the consumer does not observe anything related to descriptors
1411 * out of order. The HV trap from the LDC start trigger is the
1412 * producer to consumer announcement that work is available to the
1415 if (!port
->start_cons
) { /* previous trigger suffices */
1416 trace_vnet_skip_tx_trigger(port
->vio
._local_sid
,
1417 port
->vio
._peer_sid
, dr
->cons
);
1418 goto ldc_start_done
;
1421 err
= __vnet_tx_trigger(port
, dr
->cons
);
1422 if (unlikely(err
< 0)) {
1423 netdev_info(dev
, "TX trigger error %d\n", err
);
1424 d
->hdr
.state
= VIO_DESC_FREE
;
1425 skb
= port
->tx_bufs
[txi
].skb
;
1426 port
->tx_bufs
[txi
].skb
= NULL
;
1427 dev
->stats
.tx_carrier_errors
++;
1432 port
->start_cons
= false;
1434 dev
->stats
.tx_packets
++;
1435 dev
->stats
.tx_bytes
+= port
->tx_bufs
[txi
].skb
->len
;
1437 dr
->prod
= (dr
->prod
+ 1) & (VNET_TX_RING_SIZE
- 1);
1438 if (unlikely(vnet_tx_dring_avail(dr
) < 1)) {
1439 netif_tx_stop_queue(txq
);
1440 if (vnet_tx_dring_avail(dr
) > VNET_TX_WAKEUP_THRESH(dr
))
1441 netif_tx_wake_queue(txq
);
1444 (void)mod_timer(&port
->clean_timer
, jiffies
+ VNET_CLEAN_TIMEOUT
);
1447 vnet_free_skbs(freeskbs
);
1449 return NETDEV_TX_OK
;
1453 (void)mod_timer(&port
->clean_timer
,
1454 jiffies
+ VNET_CLEAN_TIMEOUT
);
1456 del_timer(&port
->clean_timer
);
1461 vnet_free_skbs(freeskbs
);
1462 dev
->stats
.tx_dropped
++;
1463 return NETDEV_TX_OK
;
1465 EXPORT_SYMBOL_GPL(sunvnet_start_xmit_common
);
1467 void sunvnet_tx_timeout_common(struct net_device
*dev
)
1469 /* XXX Implement me XXX */
1471 EXPORT_SYMBOL_GPL(sunvnet_tx_timeout_common
);
1473 int sunvnet_open_common(struct net_device
*dev
)
1475 netif_carrier_on(dev
);
1476 netif_tx_start_all_queues(dev
);
1480 EXPORT_SYMBOL_GPL(sunvnet_open_common
);
1482 int sunvnet_close_common(struct net_device
*dev
)
1484 netif_tx_stop_all_queues(dev
);
1485 netif_carrier_off(dev
);
1489 EXPORT_SYMBOL_GPL(sunvnet_close_common
);
1491 static struct vnet_mcast_entry
*__vnet_mc_find(struct vnet
*vp
, u8
*addr
)
1493 struct vnet_mcast_entry
*m
;
1495 for (m
= vp
->mcast_list
; m
; m
= m
->next
) {
1496 if (ether_addr_equal(m
->addr
, addr
))
1502 static void __update_mc_list(struct vnet
*vp
, struct net_device
*dev
)
1504 struct netdev_hw_addr
*ha
;
1506 netdev_for_each_mc_addr(ha
, dev
) {
1507 struct vnet_mcast_entry
*m
;
1509 m
= __vnet_mc_find(vp
, ha
->addr
);
1516 m
= kzalloc(sizeof(*m
), GFP_ATOMIC
);
1519 memcpy(m
->addr
, ha
->addr
, ETH_ALEN
);
1522 m
->next
= vp
->mcast_list
;
1528 static void __send_mc_list(struct vnet
*vp
, struct vnet_port
*port
)
1530 struct vio_net_mcast_info info
;
1531 struct vnet_mcast_entry
*m
, **pp
;
1534 memset(&info
, 0, sizeof(info
));
1536 info
.tag
.type
= VIO_TYPE_CTRL
;
1537 info
.tag
.stype
= VIO_SUBTYPE_INFO
;
1538 info
.tag
.stype_env
= VNET_MCAST_INFO
;
1539 info
.tag
.sid
= vio_send_sid(&port
->vio
);
1543 for (m
= vp
->mcast_list
; m
; m
= m
->next
) {
1547 memcpy(&info
.mcast_addr
[n_addrs
* ETH_ALEN
],
1549 if (++n_addrs
== VNET_NUM_MCAST
) {
1550 info
.count
= n_addrs
;
1552 (void) vio_ldc_send(&port
->vio
, &info
,
1558 info
.count
= n_addrs
;
1559 (void) vio_ldc_send(&port
->vio
, &info
, sizeof(info
));
1565 pp
= &vp
->mcast_list
;
1566 while ((m
= *pp
) != NULL
) {
1573 memcpy(&info
.mcast_addr
[n_addrs
* ETH_ALEN
],
1575 if (++n_addrs
== VNET_NUM_MCAST
) {
1576 info
.count
= n_addrs
;
1577 (void) vio_ldc_send(&port
->vio
, &info
,
1586 info
.count
= n_addrs
;
1587 (void) vio_ldc_send(&port
->vio
, &info
, sizeof(info
));
1591 void sunvnet_set_rx_mode_common(struct net_device
*dev
)
1593 struct vnet
*vp
= netdev_priv(dev
);
1594 struct vnet_port
*port
;
1597 list_for_each_entry_rcu(port
, &vp
->port_list
, list
) {
1599 if (port
->switch_port
) {
1600 __update_mc_list(vp
, dev
);
1601 __send_mc_list(vp
, port
);
1607 EXPORT_SYMBOL_GPL(sunvnet_set_rx_mode_common
);
1609 int sunvnet_change_mtu_common(struct net_device
*dev
, int new_mtu
)
1611 if (new_mtu
< 68 || new_mtu
> 65535)
1617 EXPORT_SYMBOL_GPL(sunvnet_change_mtu_common
);
1619 int sunvnet_set_mac_addr_common(struct net_device
*dev
, void *p
)
1623 EXPORT_SYMBOL_GPL(sunvnet_set_mac_addr_common
);
1625 void sunvnet_port_free_tx_bufs_common(struct vnet_port
*port
)
1627 struct vio_dring_state
*dr
;
1630 dr
= &port
->vio
.drings
[VIO_DRIVER_TX_RING
];
1632 if (dr
->base
== NULL
)
1635 for (i
= 0; i
< VNET_TX_RING_SIZE
; i
++) {
1636 struct vio_net_desc
*d
;
1637 void *skb
= port
->tx_bufs
[i
].skb
;
1642 d
= vio_dring_entry(dr
, i
);
1644 ldc_unmap(port
->vio
.lp
,
1645 port
->tx_bufs
[i
].cookies
,
1646 port
->tx_bufs
[i
].ncookies
);
1648 port
->tx_bufs
[i
].skb
= NULL
;
1649 d
->hdr
.state
= VIO_DESC_FREE
;
1651 ldc_free_exp_dring(port
->vio
.lp
, dr
->base
,
1652 (dr
->entry_size
* dr
->num_entries
),
1653 dr
->cookies
, dr
->ncookies
);
1656 dr
->num_entries
= 0;
1660 EXPORT_SYMBOL_GPL(sunvnet_port_free_tx_bufs_common
);
1662 static void vnet_port_reset(struct vnet_port
*port
)
1664 del_timer(&port
->clean_timer
);
1665 sunvnet_port_free_tx_bufs_common(port
);
1671 static int vnet_port_alloc_tx_ring(struct vnet_port
*port
)
1673 struct vio_dring_state
*dr
;
1674 unsigned long len
, elen
;
1675 int i
, err
, ncookies
;
1678 dr
= &port
->vio
.drings
[VIO_DRIVER_TX_RING
];
1680 elen
= sizeof(struct vio_net_desc
) +
1681 sizeof(struct ldc_trans_cookie
) * 2;
1682 if (vio_version_after_eq(&port
->vio
, 1, 7))
1683 elen
+= sizeof(struct vio_net_dext
);
1684 len
= VNET_TX_RING_SIZE
* elen
;
1686 ncookies
= VIO_MAX_RING_COOKIES
;
1687 dring
= ldc_alloc_exp_dring(port
->vio
.lp
, len
,
1688 dr
->cookies
, &ncookies
,
1692 if (IS_ERR(dring
)) {
1693 err
= PTR_ERR(dring
);
1698 dr
->entry_size
= elen
;
1699 dr
->num_entries
= VNET_TX_RING_SIZE
;
1700 dr
->prod
= dr
->cons
= 0;
1701 port
->start_cons
= true; /* need an initial trigger */
1702 dr
->pending
= VNET_TX_RING_SIZE
;
1703 dr
->ncookies
= ncookies
;
1705 for (i
= 0; i
< VNET_TX_RING_SIZE
; ++i
) {
1706 struct vio_net_desc
*d
;
1708 d
= vio_dring_entry(dr
, i
);
1709 d
->hdr
.state
= VIO_DESC_FREE
;
1714 sunvnet_port_free_tx_bufs_common(port
);
1719 #ifdef CONFIG_NET_POLL_CONTROLLER
1720 void sunvnet_poll_controller_common(struct net_device
*dev
)
1722 struct vnet
*vp
= netdev_priv(dev
);
1723 struct vnet_port
*port
;
1724 unsigned long flags
;
1726 spin_lock_irqsave(&vp
->lock
, flags
);
1727 if (!list_empty(&vp
->port_list
)) {
1728 port
= list_entry(vp
->port_list
.next
, struct vnet_port
, list
);
1729 napi_schedule(&port
->napi
);
1731 spin_unlock_irqrestore(&vp
->lock
, flags
);
1733 EXPORT_SYMBOL_GPL(sunvnet_poll_controller_common
);
1736 void sunvnet_port_add_txq_common(struct vnet_port
*port
)
1738 struct vnet
*vp
= port
->vp
;
1742 n
= n
& (VNET_MAX_TXQS
- 1);
1744 netif_tx_wake_queue(netdev_get_tx_queue(vp
->dev
, port
->q_index
));
1746 EXPORT_SYMBOL_GPL(sunvnet_port_add_txq_common
);
1748 void sunvnet_port_rm_txq_common(struct vnet_port
*port
)
1751 netif_tx_stop_queue(netdev_get_tx_queue(port
->vp
->dev
, port
->q_index
));
1753 EXPORT_SYMBOL_GPL(sunvnet_port_rm_txq_common
);