1 /**********************************************************************
2 * Author: Cavium Networks
4 * Contact: support@caviumnetworks.com
5 * This file is part of the OCTEON SDK
7 * Copyright (c) 2003-2010 Cavium Networks
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
19 * You should have received a copy of the GNU General Public License
20 * along with this file; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
22 * or visit http://www.gnu.org/licenses/.
24 * This file may also be available under a different license from Cavium.
25 * Contact Cavium Networks for more information
26 **********************************************************************/
27 #include <linux/module.h>
28 #include <linux/kernel.h>
29 #include <linux/cache.h>
30 #include <linux/cpumask.h>
31 #include <linux/netdevice.h>
32 #include <linux/etherdevice.h>
34 #include <linux/string.h>
35 #include <linux/prefetch.h>
36 #include <linux/ratelimit.h>
37 #include <linux/smp.h>
38 #include <linux/interrupt.h>
41 #include <linux/xfrm.h>
43 #endif /* CONFIG_XFRM */
45 #include <linux/atomic.h>
47 #include <asm/octeon/octeon.h>
49 #include "ethernet-defines.h"
50 #include "ethernet-mem.h"
51 #include "ethernet-rx.h"
52 #include "octeon-ethernet.h"
53 #include "ethernet-util.h"
55 #include <asm/octeon/cvmx-helper.h>
56 #include <asm/octeon/cvmx-wqe.h>
57 #include <asm/octeon/cvmx-fau.h>
58 #include <asm/octeon/cvmx-pow.h>
59 #include <asm/octeon/cvmx-pip.h>
60 #include <asm/octeon/cvmx-scratch.h>
62 #include <asm/octeon/cvmx-gmxx-defs.h>
64 static struct napi_struct cvm_oct_napi
;
67 * cvm_oct_do_interrupt - interrupt handler.
69 * The interrupt occurs whenever the POW has packets in our group.
72 static irqreturn_t
cvm_oct_do_interrupt(int cpl
, void *dev_id
)
74 /* Disable the IRQ and start napi_poll. */
75 disable_irq_nosync(OCTEON_IRQ_WORKQ0
+ pow_receive_group
);
76 napi_schedule(&cvm_oct_napi
);
82 * cvm_oct_check_rcv_error - process receive errors
83 * @work: Work queue entry pointing to the packet.
85 * Returns Non-zero if the packet can be dropped, zero otherwise.
87 static inline int cvm_oct_check_rcv_error(cvmx_wqe_t
*work
)
89 if ((work
->word2
.snoip
.err_code
== 10) && (work
->len
<= 64)) {
91 * Ignore length errors on min size packets. Some
92 * equipment incorrectly pads packets to 64+4FCS
93 * instead of 60+4FCS. Note these packets still get
94 * counted as frame errors.
97 if (USE_10MBPS_PREAMBLE_WORKAROUND
98 && ((work
->word2
.snoip
.err_code
== 5)
99 || (work
->word2
.snoip
.err_code
== 7))) {
102 * We received a packet with either an alignment error
103 * or a FCS error. This may be signalling that we are
104 * running 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK]
105 * off. If this is the case we need to parse the
106 * packet to determine if we can remove a non spec
107 * preamble and generate a correct packet.
109 int interface
= cvmx_helper_get_interface_num(work
->ipprt
);
110 int index
= cvmx_helper_get_interface_index_num(work
->ipprt
);
111 union cvmx_gmxx_rxx_frm_ctl gmxx_rxx_frm_ctl
;
113 gmxx_rxx_frm_ctl
.u64
=
114 cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index
, interface
));
115 if (gmxx_rxx_frm_ctl
.s
.pre_chk
== 0) {
118 cvmx_phys_to_ptr(work
->packet_ptr
.s
.addr
);
121 while (i
< work
->len
- 1) {
130 printk_ratelimited("Port %d received 0xd5 preamble\n",
133 work
->packet_ptr
.s
.addr
+= i
+ 1;
135 } else if ((*ptr
& 0xf) == 0xd) {
137 printk_ratelimited("Port %d received 0x?d preamble\n",
140 work
->packet_ptr
.s
.addr
+= i
;
142 for (i
= 0; i
< work
->len
; i
++) {
144 ((*ptr
& 0xf0) >> 4) |
145 ((*(ptr
+ 1) & 0xf) << 4);
149 printk_ratelimited("Port %d unknown preamble, packet dropped\n",
152 cvmx_helper_dump_packet(work);
154 cvm_oct_free_work(work
);
159 printk_ratelimited("Port %d receive error code %d, packet dropped\n",
160 work
->ipprt
, work
->word2
.snoip
.err_code
);
161 cvm_oct_free_work(work
);
169 * cvm_oct_napi_poll - the NAPI poll function.
170 * @napi: The NAPI instance, or null if called from cvm_oct_poll_controller
171 * @budget: Maximum number of packets to receive.
173 * Returns the number of packets processed.
175 static int cvm_oct_napi_poll(struct napi_struct
*napi
, int budget
)
177 const int coreid
= cvmx_get_core_num();
178 uint64_t old_group_mask
;
179 uint64_t old_scratch
;
181 int did_work_request
= 0;
182 int packet_not_copied
;
184 /* Prefetch cvm_oct_device since we know we need it soon */
185 prefetch(cvm_oct_device
);
187 if (USE_ASYNC_IOBDMA
) {
188 /* Save scratch in case userspace is using it */
190 old_scratch
= cvmx_scratch_read64(CVMX_SCR_SCRATCH
);
193 /* Only allow work for our group (and preserve priorities) */
194 old_group_mask
= cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid
));
195 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid
),
196 (old_group_mask
& ~0xFFFFull
) | 1 << pow_receive_group
);
198 if (USE_ASYNC_IOBDMA
) {
199 cvmx_pow_work_request_async(CVMX_SCR_SCRATCH
, CVMX_POW_NO_WAIT
);
200 did_work_request
= 1;
203 while (rx_count
< budget
) {
204 struct sk_buff
*skb
= NULL
;
205 struct sk_buff
**pskb
= NULL
;
209 if (USE_ASYNC_IOBDMA
&& did_work_request
)
210 work
= cvmx_pow_work_response_async(CVMX_SCR_SCRATCH
);
212 work
= cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT
);
215 did_work_request
= 0;
217 union cvmx_pow_wq_int wq_int
;
220 wq_int
.s
.iq_dis
= 1 << pow_receive_group
;
221 wq_int
.s
.wq_int
= 1 << pow_receive_group
;
222 cvmx_write_csr(CVMX_POW_WQ_INT
, wq_int
.u64
);
225 pskb
= (struct sk_buff
**)(cvm_oct_get_buffer_ptr(work
->packet_ptr
) -
229 if (USE_ASYNC_IOBDMA
&& rx_count
< (budget
- 1)) {
230 cvmx_pow_work_request_async_nocheck(CVMX_SCR_SCRATCH
,
232 did_work_request
= 1;
236 skb_in_hw
= USE_SKBUFFS_IN_HW
&& work
->word2
.s
.bufs
== 1;
237 if (likely(skb_in_hw
)) {
239 prefetch(&skb
->head
);
242 prefetch(cvm_oct_device
[work
->ipprt
]);
244 /* Immediately throw away all packets with receive errors */
245 if (unlikely(work
->word2
.snoip
.rcv_error
)) {
246 if (cvm_oct_check_rcv_error(work
))
251 * We can only use the zero copy path if skbuffs are
252 * in the FPA pool and the packet fits in a single
255 if (likely(skb_in_hw
)) {
256 skb
->data
= skb
->head
+ work
->packet_ptr
.s
.addr
-
257 cvmx_ptr_to_phys(skb
->head
);
259 skb
->len
= work
->len
;
260 skb_set_tail_pointer(skb
, skb
->len
);
261 packet_not_copied
= 1;
264 * We have to copy the packet. First allocate
267 skb
= dev_alloc_skb(work
->len
);
269 cvm_oct_free_work(work
);
274 * Check if we've received a packet that was
275 * entirely stored in the work entry.
277 if (unlikely(work
->word2
.s
.bufs
== 0)) {
278 uint8_t *ptr
= work
->packet_data
;
280 if (likely(!work
->word2
.s
.not_IP
)) {
282 * The beginning of the packet
283 * moves for IP packets.
285 if (work
->word2
.s
.is_v6
)
290 memcpy(skb_put(skb
, work
->len
), ptr
, work
->len
);
291 /* No packet buffers to free */
293 int segments
= work
->word2
.s
.bufs
;
294 union cvmx_buf_ptr segment_ptr
=
299 union cvmx_buf_ptr next_ptr
=
300 *(union cvmx_buf_ptr
*)cvmx_phys_to_ptr(segment_ptr
.s
.addr
- 8);
303 * Octeon Errata PKI-100: The segment size is
304 * wrong. Until it is fixed, calculate the
305 * segment size based on the packet pool
306 * buffer size. When it is fixed, the
307 * following line should be replaced with this
308 * one: int segment_size =
309 * segment_ptr.s.size;
312 CVMX_FPA_PACKET_POOL_SIZE
-
313 (segment_ptr
.s
.addr
-
314 (((segment_ptr
.s
.addr
>> 7) -
315 segment_ptr
.s
.back
) << 7));
317 * Don't copy more than what
318 * is left in the packet.
320 if (segment_size
> len
)
322 /* Copy the data into the packet */
323 memcpy(skb_put(skb
, segment_size
),
324 cvmx_phys_to_ptr(segment_ptr
.s
.addr
),
327 segment_ptr
= next_ptr
;
330 packet_not_copied
= 0;
333 if (likely((work
->ipprt
< TOTAL_NUMBER_OF_PORTS
) &&
334 cvm_oct_device
[work
->ipprt
])) {
335 struct net_device
*dev
= cvm_oct_device
[work
->ipprt
];
336 struct octeon_ethernet
*priv
= netdev_priv(dev
);
339 * Only accept packets for devices that are
342 if (likely(dev
->flags
& IFF_UP
)) {
343 skb
->protocol
= eth_type_trans(skb
, dev
);
346 if (unlikely(work
->word2
.s
.not_IP
||
347 work
->word2
.s
.IP_exc
||
348 work
->word2
.s
.L4_error
||
349 !work
->word2
.s
.tcp_or_udp
))
350 skb
->ip_summed
= CHECKSUM_NONE
;
352 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
354 /* Increment RX stats for virtual ports */
355 if (work
->ipprt
>= CVMX_PIP_NUM_INPUT_PORTS
) {
358 (atomic64_t
*)&priv
->stats
.rx_packets
);
359 atomic64_add(skb
->len
,
360 (atomic64_t
*)&priv
->stats
.rx_bytes
);
363 (atomic_t
*)&priv
->stats
.rx_packets
);
365 (atomic_t
*)&priv
->stats
.rx_bytes
);
368 netif_receive_skb(skb
);
370 /* Drop any packet received for a device that isn't up */
372 printk_ratelimited("%s: Device not up, packet dropped\n",
377 (atomic64_t
*)&priv
->stats
.rx_dropped
);
380 (atomic_t
*)&priv
->stats
.rx_dropped
);
382 dev_kfree_skb_irq(skb
);
386 * Drop any packet received for a device that
389 printk_ratelimited("Port %d not controlled by Linux, packet dropped\n",
391 dev_kfree_skb_irq(skb
);
394 * Check to see if the skbuff and work share the same
397 if (USE_SKBUFFS_IN_HW
&& likely(packet_not_copied
)) {
399 * This buffer needs to be replaced, increment
400 * the number of buffers we need to free by
403 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE
,
406 cvmx_fpa_free(work
, CVMX_FPA_WQE_POOL
,
409 cvm_oct_free_work(work
);
412 /* Restore the original POW group mask */
413 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid
), old_group_mask
);
414 if (USE_ASYNC_IOBDMA
) {
415 /* Restore the scratch area */
416 cvmx_scratch_write64(CVMX_SCR_SCRATCH
, old_scratch
);
418 cvm_oct_rx_refill_pool(0);
420 if (rx_count
< budget
&& napi
!= NULL
) {
423 enable_irq(OCTEON_IRQ_WORKQ0
+ pow_receive_group
);
428 #ifdef CONFIG_NET_POLL_CONTROLLER
430 * cvm_oct_poll_controller - poll for receive packets
433 * @dev: Device to poll. Unused
435 void cvm_oct_poll_controller(struct net_device
*dev
)
437 cvm_oct_napi_poll(NULL
, 16);
441 void cvm_oct_rx_initialize(void)
444 struct net_device
*dev_for_napi
= NULL
;
445 union cvmx_pow_wq_int_thrx int_thr
;
446 union cvmx_pow_wq_int_pc int_pc
;
448 for (i
= 0; i
< TOTAL_NUMBER_OF_PORTS
; i
++) {
449 if (cvm_oct_device
[i
]) {
450 dev_for_napi
= cvm_oct_device
[i
];
455 if (NULL
== dev_for_napi
)
456 panic("No net_devices were allocated.");
458 netif_napi_add(dev_for_napi
, &cvm_oct_napi
, cvm_oct_napi_poll
,
460 napi_enable(&cvm_oct_napi
);
462 /* Register an IRQ handler to receive POW interrupts */
463 i
= request_irq(OCTEON_IRQ_WORKQ0
+ pow_receive_group
,
464 cvm_oct_do_interrupt
, 0, "Ethernet", cvm_oct_device
);
467 panic("Could not acquire Ethernet IRQ %d\n",
468 OCTEON_IRQ_WORKQ0
+ pow_receive_group
);
470 disable_irq_nosync(OCTEON_IRQ_WORKQ0
+ pow_receive_group
);
474 int_thr
.s
.tc_thr
= 1;
475 /* Enable POW interrupt when our port has at least one packet */
476 cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group
), int_thr
.u64
);
480 cvmx_write_csr(CVMX_POW_WQ_INT_PC
, int_pc
.u64
);
482 /* Schedule NAPI now. This will indirectly enable the interrupt. */
483 napi_schedule(&cvm_oct_napi
);
486 void cvm_oct_rx_shutdown(void)
488 netif_napi_del(&cvm_oct_napi
);