1 /**********************************************************************
2 * Author: Cavium Networks
4 * Contact: support@caviumnetworks.com
5 * This file is part of the OCTEON SDK
7 * Copyright (c) 2003-2010 Cavium Networks
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
19 * You should have received a copy of the GNU General Public License
20 * along with this file; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
22 * or visit http://www.gnu.org/licenses/.
24 * This file may also be available under a different license from Cavium.
25 * Contact Cavium Networks for more information
26 **********************************************************************/
27 #include <linux/module.h>
28 #include <linux/kernel.h>
29 #include <linux/cache.h>
30 #include <linux/cpumask.h>
31 #include <linux/netdevice.h>
32 #include <linux/etherdevice.h>
34 #include <linux/string.h>
35 #include <linux/prefetch.h>
36 #include <linux/ratelimit.h>
37 #include <linux/smp.h>
38 #include <linux/interrupt.h>
41 #include <linux/xfrm.h>
43 #endif /* CONFIG_XFRM */
45 #include <linux/atomic.h>
47 #include <asm/octeon/octeon.h>
49 #include "ethernet-defines.h"
50 #include "ethernet-mem.h"
51 #include "ethernet-rx.h"
52 #include "octeon-ethernet.h"
53 #include "ethernet-util.h"
55 #include <asm/octeon/cvmx-helper.h>
56 #include <asm/octeon/cvmx-wqe.h>
57 #include <asm/octeon/cvmx-fau.h>
58 #include <asm/octeon/cvmx-pow.h>
59 #include <asm/octeon/cvmx-pip.h>
60 #include <asm/octeon/cvmx-scratch.h>
62 #include <asm/octeon/cvmx-gmxx-defs.h>
64 static struct napi_struct cvm_oct_napi
;
67 * cvm_oct_do_interrupt - interrupt handler.
69 * The interrupt occurs whenever the POW has packets in our group.
72 static irqreturn_t
cvm_oct_do_interrupt(int cpl
, void *dev_id
)
74 /* Disable the IRQ and start napi_poll. */
75 disable_irq_nosync(OCTEON_IRQ_WORKQ0
+ pow_receive_group
);
76 napi_schedule(&cvm_oct_napi
);
82 * cvm_oct_check_rcv_error - process receive errors
83 * @work: Work queue entry pointing to the packet.
85 * Returns Non-zero if the packet can be dropped, zero otherwise.
87 static inline int cvm_oct_check_rcv_error(cvmx_wqe_t
*work
)
89 if ((work
->word2
.snoip
.err_code
== 10) && (work
->len
<= 64)) {
91 * Ignore length errors on min size packets. Some
92 * equipment incorrectly pads packets to 64+4FCS
93 * instead of 60+4FCS. Note these packets still get
94 * counted as frame errors.
96 } else if (work
->word2
.snoip
.err_code
== 5 ||
97 work
->word2
.snoip
.err_code
== 7) {
99 * We received a packet with either an alignment error
100 * or a FCS error. This may be signalling that we are
101 * running 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK]
102 * off. If this is the case we need to parse the
103 * packet to determine if we can remove a non spec
104 * preamble and generate a correct packet.
106 int interface
= cvmx_helper_get_interface_num(work
->ipprt
);
107 int index
= cvmx_helper_get_interface_index_num(work
->ipprt
);
108 union cvmx_gmxx_rxx_frm_ctl gmxx_rxx_frm_ctl
;
110 gmxx_rxx_frm_ctl
.u64
=
111 cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index
, interface
));
112 if (gmxx_rxx_frm_ctl
.s
.pre_chk
== 0) {
115 cvmx_phys_to_ptr(work
->packet_ptr
.s
.addr
);
118 while (i
< work
->len
- 1) {
127 printk_ratelimited("Port %d received 0xd5 preamble\n",
130 work
->packet_ptr
.s
.addr
+= i
+ 1;
132 } else if ((*ptr
& 0xf) == 0xd) {
134 printk_ratelimited("Port %d received 0x?d preamble\n",
137 work
->packet_ptr
.s
.addr
+= i
;
139 for (i
= 0; i
< work
->len
; i
++) {
141 ((*ptr
& 0xf0) >> 4) |
142 ((*(ptr
+ 1) & 0xf) << 4);
146 printk_ratelimited("Port %d unknown preamble, packet dropped\n",
149 cvmx_helper_dump_packet(work);
151 cvm_oct_free_work(work
);
156 printk_ratelimited("Port %d receive error code %d, packet dropped\n",
157 work
->ipprt
, work
->word2
.snoip
.err_code
);
158 cvm_oct_free_work(work
);
166 * cvm_oct_napi_poll - the NAPI poll function.
167 * @napi: The NAPI instance, or null if called from cvm_oct_poll_controller
168 * @budget: Maximum number of packets to receive.
170 * Returns the number of packets processed.
172 static int cvm_oct_napi_poll(struct napi_struct
*napi
, int budget
)
174 const int coreid
= cvmx_get_core_num();
175 uint64_t old_group_mask
;
176 uint64_t old_scratch
;
178 int did_work_request
= 0;
179 int packet_not_copied
;
181 /* Prefetch cvm_oct_device since we know we need it soon */
182 prefetch(cvm_oct_device
);
184 if (USE_ASYNC_IOBDMA
) {
185 /* Save scratch in case userspace is using it */
187 old_scratch
= cvmx_scratch_read64(CVMX_SCR_SCRATCH
);
190 /* Only allow work for our group (and preserve priorities) */
191 old_group_mask
= cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid
));
192 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid
),
193 (old_group_mask
& ~0xFFFFull
) | 1 << pow_receive_group
);
195 if (USE_ASYNC_IOBDMA
) {
196 cvmx_pow_work_request_async(CVMX_SCR_SCRATCH
, CVMX_POW_NO_WAIT
);
197 did_work_request
= 1;
200 while (rx_count
< budget
) {
201 struct sk_buff
*skb
= NULL
;
202 struct sk_buff
**pskb
= NULL
;
206 if (USE_ASYNC_IOBDMA
&& did_work_request
)
207 work
= cvmx_pow_work_response_async(CVMX_SCR_SCRATCH
);
209 work
= cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT
);
212 did_work_request
= 0;
214 union cvmx_pow_wq_int wq_int
;
217 wq_int
.s
.iq_dis
= 1 << pow_receive_group
;
218 wq_int
.s
.wq_int
= 1 << pow_receive_group
;
219 cvmx_write_csr(CVMX_POW_WQ_INT
, wq_int
.u64
);
222 pskb
= (struct sk_buff
**)(cvm_oct_get_buffer_ptr(work
->packet_ptr
) -
226 if (USE_ASYNC_IOBDMA
&& rx_count
< (budget
- 1)) {
227 cvmx_pow_work_request_async_nocheck(CVMX_SCR_SCRATCH
,
229 did_work_request
= 1;
233 skb_in_hw
= work
->word2
.s
.bufs
== 1;
234 if (likely(skb_in_hw
)) {
236 prefetch(&skb
->head
);
239 prefetch(cvm_oct_device
[work
->ipprt
]);
241 /* Immediately throw away all packets with receive errors */
242 if (unlikely(work
->word2
.snoip
.rcv_error
)) {
243 if (cvm_oct_check_rcv_error(work
))
248 * We can only use the zero copy path if skbuffs are
249 * in the FPA pool and the packet fits in a single
252 if (likely(skb_in_hw
)) {
253 skb
->data
= skb
->head
+ work
->packet_ptr
.s
.addr
-
254 cvmx_ptr_to_phys(skb
->head
);
256 skb
->len
= work
->len
;
257 skb_set_tail_pointer(skb
, skb
->len
);
258 packet_not_copied
= 1;
261 * We have to copy the packet. First allocate
264 skb
= dev_alloc_skb(work
->len
);
266 cvm_oct_free_work(work
);
271 * Check if we've received a packet that was
272 * entirely stored in the work entry.
274 if (unlikely(work
->word2
.s
.bufs
== 0)) {
275 uint8_t *ptr
= work
->packet_data
;
277 if (likely(!work
->word2
.s
.not_IP
)) {
279 * The beginning of the packet
280 * moves for IP packets.
282 if (work
->word2
.s
.is_v6
)
287 memcpy(skb_put(skb
, work
->len
), ptr
, work
->len
);
288 /* No packet buffers to free */
290 int segments
= work
->word2
.s
.bufs
;
291 union cvmx_buf_ptr segment_ptr
=
296 union cvmx_buf_ptr next_ptr
=
297 *(union cvmx_buf_ptr
*)cvmx_phys_to_ptr(segment_ptr
.s
.addr
- 8);
300 * Octeon Errata PKI-100: The segment size is
301 * wrong. Until it is fixed, calculate the
302 * segment size based on the packet pool
303 * buffer size. When it is fixed, the
304 * following line should be replaced with this
305 * one: int segment_size =
306 * segment_ptr.s.size;
309 CVMX_FPA_PACKET_POOL_SIZE
-
310 (segment_ptr
.s
.addr
-
311 (((segment_ptr
.s
.addr
>> 7) -
312 segment_ptr
.s
.back
) << 7));
314 * Don't copy more than what
315 * is left in the packet.
317 if (segment_size
> len
)
319 /* Copy the data into the packet */
320 memcpy(skb_put(skb
, segment_size
),
321 cvmx_phys_to_ptr(segment_ptr
.s
.addr
),
324 segment_ptr
= next_ptr
;
327 packet_not_copied
= 0;
330 if (likely((work
->ipprt
< TOTAL_NUMBER_OF_PORTS
) &&
331 cvm_oct_device
[work
->ipprt
])) {
332 struct net_device
*dev
= cvm_oct_device
[work
->ipprt
];
333 struct octeon_ethernet
*priv
= netdev_priv(dev
);
336 * Only accept packets for devices that are
339 if (likely(dev
->flags
& IFF_UP
)) {
340 skb
->protocol
= eth_type_trans(skb
, dev
);
343 if (unlikely(work
->word2
.s
.not_IP
||
344 work
->word2
.s
.IP_exc
||
345 work
->word2
.s
.L4_error
||
346 !work
->word2
.s
.tcp_or_udp
))
347 skb
->ip_summed
= CHECKSUM_NONE
;
349 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
351 /* Increment RX stats for virtual ports */
352 if (work
->ipprt
>= CVMX_PIP_NUM_INPUT_PORTS
) {
355 (atomic64_t
*)&priv
->stats
.rx_packets
);
356 atomic64_add(skb
->len
,
357 (atomic64_t
*)&priv
->stats
.rx_bytes
);
360 (atomic_t
*)&priv
->stats
.rx_packets
);
362 (atomic_t
*)&priv
->stats
.rx_bytes
);
365 netif_receive_skb(skb
);
367 /* Drop any packet received for a device that isn't up */
369 printk_ratelimited("%s: Device not up, packet dropped\n",
374 (atomic64_t
*)&priv
->stats
.rx_dropped
);
377 (atomic_t
*)&priv
->stats
.rx_dropped
);
379 dev_kfree_skb_irq(skb
);
383 * Drop any packet received for a device that
386 printk_ratelimited("Port %d not controlled by Linux, packet dropped\n",
388 dev_kfree_skb_irq(skb
);
391 * Check to see if the skbuff and work share the same
394 if (likely(packet_not_copied
)) {
396 * This buffer needs to be replaced, increment
397 * the number of buffers we need to free by
400 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE
,
403 cvmx_fpa_free(work
, CVMX_FPA_WQE_POOL
,
406 cvm_oct_free_work(work
);
409 /* Restore the original POW group mask */
410 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid
), old_group_mask
);
411 if (USE_ASYNC_IOBDMA
) {
412 /* Restore the scratch area */
413 cvmx_scratch_write64(CVMX_SCR_SCRATCH
, old_scratch
);
415 cvm_oct_rx_refill_pool(0);
417 if (rx_count
< budget
&& napi
!= NULL
) {
420 enable_irq(OCTEON_IRQ_WORKQ0
+ pow_receive_group
);
425 #ifdef CONFIG_NET_POLL_CONTROLLER
427 * cvm_oct_poll_controller - poll for receive packets
430 * @dev: Device to poll. Unused
432 void cvm_oct_poll_controller(struct net_device
*dev
)
434 cvm_oct_napi_poll(NULL
, 16);
438 void cvm_oct_rx_initialize(void)
441 struct net_device
*dev_for_napi
= NULL
;
442 union cvmx_pow_wq_int_thrx int_thr
;
443 union cvmx_pow_wq_int_pc int_pc
;
445 for (i
= 0; i
< TOTAL_NUMBER_OF_PORTS
; i
++) {
446 if (cvm_oct_device
[i
]) {
447 dev_for_napi
= cvm_oct_device
[i
];
452 if (NULL
== dev_for_napi
)
453 panic("No net_devices were allocated.");
455 netif_napi_add(dev_for_napi
, &cvm_oct_napi
, cvm_oct_napi_poll
,
457 napi_enable(&cvm_oct_napi
);
459 /* Register an IRQ handler to receive POW interrupts */
460 i
= request_irq(OCTEON_IRQ_WORKQ0
+ pow_receive_group
,
461 cvm_oct_do_interrupt
, 0, "Ethernet", cvm_oct_device
);
464 panic("Could not acquire Ethernet IRQ %d\n",
465 OCTEON_IRQ_WORKQ0
+ pow_receive_group
);
467 disable_irq_nosync(OCTEON_IRQ_WORKQ0
+ pow_receive_group
);
471 int_thr
.s
.tc_thr
= 1;
472 /* Enable POW interrupt when our port has at least one packet */
473 cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group
), int_thr
.u64
);
477 cvmx_write_csr(CVMX_POW_WQ_INT_PC
, int_pc
.u64
);
479 /* Schedule NAPI now. This will indirectly enable the interrupt. */
480 napi_schedule(&cvm_oct_napi
);
483 void cvm_oct_rx_shutdown(void)
485 netif_napi_del(&cvm_oct_napi
);