projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
openvswitch: Fix FLOW_BUFSIZE definition.
[deliverable/linux.git] / drivers / net / ethernet / broadcom / bnx2x / bnx2x_cmn.c
1 /* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
3 * Copyright (c) 2007-2012 Broadcom Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20 #include <linux/etherdevice.h>
21 #include <linux/if_vlan.h>
22 #include <linux/interrupt.h>
23 #include <linux/ip.h>
24 #include <net/ipv6.h>
25 #include <net/ip6_checksum.h>
26 #include <linux/prefetch.h>
27 #include "bnx2x_cmn.h"
28 #include "bnx2x_init.h"
29 #include "bnx2x_sp.h"
30
31
32
33 /**
34 * bnx2x_move_fp - move content of the fastpath structure.
35 *
36 * @bp: driver handle
37 * @from: source FP index
38 * @to: destination FP index
39 *
40 * Makes sure the contents of the bp->fp[to].napi is kept
41 * intact. This is done by first copying the napi struct from
42 * the target to the source, and then mem copying the entire
43 * source onto the target. Update txdata pointers and related
44 * content.
45 */
46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
47 {
48 struct bnx2x_fastpath *from_fp = &bp->fp[from];
49 struct bnx2x_fastpath *to_fp = &bp->fp[to];
50 struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
51 struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
52 struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
53 struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
54 int old_max_eth_txqs, new_max_eth_txqs;
55 int old_txdata_index = 0, new_txdata_index = 0;
56
57 /* Copy the NAPI object as it has been already initialized */
58 from_fp->napi = to_fp->napi;
59
60 /* Move bnx2x_fastpath contents */
61 memcpy(to_fp, from_fp, sizeof(*to_fp));
62 to_fp->index = to;
63
64 /* move sp_objs contents as well, as their indices match fp ones */
65 memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
66
67 /* move fp_stats contents as well, as their indices match fp ones */
68 memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
69
70 /* Update txdata pointers in fp and move txdata content accordingly:
71 * Each fp consumes 'max_cos' txdata structures, so the index should be
72 * decremented by max_cos x delta.
73 */
74
75 old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
76 new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
77 (bp)->max_cos;
78 if (from == FCOE_IDX(bp)) {
79 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
80 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
81 }
82
83 memcpy(&bp->bnx2x_txq[old_txdata_index],
84 &bp->bnx2x_txq[new_txdata_index],
85 sizeof(struct bnx2x_fp_txdata));
86 to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
87 }
88
89 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
90
91 /* free skb in the packet ring at pos idx
92 * return idx of last bd freed
93 */
94 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
95 u16 idx, unsigned int *pkts_compl,
96 unsigned int *bytes_compl)
97 {
98 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
99 struct eth_tx_start_bd *tx_start_bd;
100 struct eth_tx_bd *tx_data_bd;
101 struct sk_buff *skb = tx_buf->skb;
102 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
103 int nbd;
104
105 /* prefetch skb end pointer to speedup dev_kfree_skb() */
106 prefetch(&skb->end);
107
108 DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n",
109 txdata->txq_index, idx, tx_buf, skb);
110
111 /* unmap first bd */
112 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
113 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
114 BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
115
116
117 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
118 #ifdef BNX2X_STOP_ON_ERROR
119 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
120 BNX2X_ERR("BAD nbd!\n");
121 bnx2x_panic();
122 }
123 #endif
124 new_cons = nbd + tx_buf->first_bd;
125
126 /* Get the next bd */
127 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
128
129 /* Skip a parse bd... */
130 --nbd;
131 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
132
133 /* ...and the TSO split header bd since they have no mapping */
134 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
135 --nbd;
136 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
137 }
138
139 /* now free frags */
140 while (nbd > 0) {
141
142 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
143 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
144 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
145 if (--nbd)
146 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
147 }
148
149 /* release skb */
150 WARN_ON(!skb);
151 if (likely(skb)) {
152 (*pkts_compl)++;
153 (*bytes_compl) += skb->len;
154 }
155
156 dev_kfree_skb_any(skb);
157 tx_buf->first_bd = 0;
158 tx_buf->skb = NULL;
159
160 return new_cons;
161 }
162
163 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
164 {
165 struct netdev_queue *txq;
166 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
167 unsigned int pkts_compl = 0, bytes_compl = 0;
168
169 #ifdef BNX2X_STOP_ON_ERROR
170 if (unlikely(bp->panic))
171 return -1;
172 #endif
173
174 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
175 hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
176 sw_cons = txdata->tx_pkt_cons;
177
178 while (sw_cons != hw_cons) {
179 u16 pkt_cons;
180
181 pkt_cons = TX_BD(sw_cons);
182
183 DP(NETIF_MSG_TX_DONE,
184 "queue[%d]: hw_cons %u sw_cons %u pkt_cons %u\n",
185 txdata->txq_index, hw_cons, sw_cons, pkt_cons);
186
187 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
188 &pkts_compl, &bytes_compl);
189
190 sw_cons++;
191 }
192
193 netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
194
195 txdata->tx_pkt_cons = sw_cons;
196 txdata->tx_bd_cons = bd_cons;
197
198 /* Need to make the tx_bd_cons update visible to start_xmit()
199 * before checking for netif_tx_queue_stopped(). Without the
200 * memory barrier, there is a small possibility that
201 * start_xmit() will miss it and cause the queue to be stopped
202 * forever.
203 * On the other hand we need an rmb() here to ensure the proper
204 * ordering of bit testing in the following
205 * netif_tx_queue_stopped(txq) call.
206 */
207 smp_mb();
208
209 if (unlikely(netif_tx_queue_stopped(txq))) {
210 /* Taking tx_lock() is needed to prevent reenabling the queue
211 * while it's empty. This could have happen if rx_action() gets
212 * suspended in bnx2x_tx_int() after the condition before
213 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
214 *
215 * stops the queue->sees fresh tx_bd_cons->releases the queue->
216 * sends some packets consuming the whole queue again->
217 * stops the queue
218 */
219
220 __netif_tx_lock(txq, smp_processor_id());
221
222 if ((netif_tx_queue_stopped(txq)) &&
223 (bp->state == BNX2X_STATE_OPEN) &&
224 (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
225 netif_tx_wake_queue(txq);
226
227 __netif_tx_unlock(txq);
228 }
229 return 0;
230 }
231
232 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
233 u16 idx)
234 {
235 u16 last_max = fp->last_max_sge;
236
237 if (SUB_S16(idx, last_max) > 0)
238 fp->last_max_sge = idx;
239 }
240
241 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
242 u16 sge_len,
243 struct eth_end_agg_rx_cqe *cqe)
244 {
245 struct bnx2x *bp = fp->bp;
246 u16 last_max, last_elem, first_elem;
247 u16 delta = 0;
248 u16 i;
249
250 if (!sge_len)
251 return;
252
253 /* First mark all used pages */
254 for (i = 0; i < sge_len; i++)
255 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
256 RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
257
258 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
259 sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
260
261 /* Here we assume that the last SGE index is the biggest */
262 prefetch((void *)(fp->sge_mask));
263 bnx2x_update_last_max_sge(fp,
264 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
265
266 last_max = RX_SGE(fp->last_max_sge);
267 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
268 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
269
270 /* If ring is not full */
271 if (last_elem + 1 != first_elem)
272 last_elem++;
273
274 /* Now update the prod */
275 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
276 if (likely(fp->sge_mask[i]))
277 break;
278
279 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
280 delta += BIT_VEC64_ELEM_SZ;
281 }
282
283 if (delta > 0) {
284 fp->rx_sge_prod += delta;
285 /* clear page-end entries */
286 bnx2x_clear_sge_mask_next_elems(fp);
287 }
288
289 DP(NETIF_MSG_RX_STATUS,
290 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
291 fp->last_max_sge, fp->rx_sge_prod);
292 }
293
294 /* Set Toeplitz hash value in the skb using the value from the
295 * CQE (calculated by HW).
296 */
297 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
298 const struct eth_fast_path_rx_cqe *cqe,
299 bool *l4_rxhash)
300 {
301 /* Set Toeplitz hash from CQE */
302 if ((bp->dev->features & NETIF_F_RXHASH) &&
303 (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
304 enum eth_rss_hash_type htype;
305
306 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
307 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
308 (htype == TCP_IPV6_HASH_TYPE);
309 return le32_to_cpu(cqe->rss_hash_result);
310 }
311 *l4_rxhash = false;
312 return 0;
313 }
314
315 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
316 u16 cons, u16 prod,
317 struct eth_fast_path_rx_cqe *cqe)
318 {
319 struct bnx2x *bp = fp->bp;
320 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
321 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
322 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
323 dma_addr_t mapping;
324 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
325 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
326
327 /* print error if current state != stop */
328 if (tpa_info->tpa_state != BNX2X_TPA_STOP)
329 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
330
331 /* Try to map an empty data buffer from the aggregation info */
332 mapping = dma_map_single(&bp->pdev->dev,
333 first_buf->data + NET_SKB_PAD,
334 fp->rx_buf_size, DMA_FROM_DEVICE);
335 /*
336 * ...if it fails - move the skb from the consumer to the producer
337 * and set the current aggregation state as ERROR to drop it
338 * when TPA_STOP arrives.
339 */
340
341 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
342 /* Move the BD from the consumer to the producer */
343 bnx2x_reuse_rx_data(fp, cons, prod);
344 tpa_info->tpa_state = BNX2X_TPA_ERROR;
345 return;
346 }
347
348 /* move empty data from pool to prod */
349 prod_rx_buf->data = first_buf->data;
350 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
351 /* point prod_bd to new data */
352 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
353 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
354
355 /* move partial skb from cons to pool (don't unmap yet) */
356 *first_buf = *cons_rx_buf;
357
358 /* mark bin state as START */
359 tpa_info->parsing_flags =
360 le16_to_cpu(cqe->pars_flags.flags);
361 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
362 tpa_info->tpa_state = BNX2X_TPA_START;
363 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
364 tpa_info->placement_offset = cqe->placement_offset;
365 tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
366 if (fp->mode == TPA_MODE_GRO) {
367 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
368 tpa_info->full_page =
369 SGE_PAGE_SIZE * PAGES_PER_SGE / gro_size * gro_size;
370 tpa_info->gro_size = gro_size;
371 }
372
373 #ifdef BNX2X_STOP_ON_ERROR
374 fp->tpa_queue_used |= (1 << queue);
375 #ifdef _ASM_GENERIC_INT_L64_H
376 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
377 #else
378 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
379 #endif
380 fp->tpa_queue_used);
381 #endif
382 }
383
384 /* Timestamp option length allowed for TPA aggregation:
385 *
386 * nop nop kind length echo val
387 */
388 #define TPA_TSTAMP_OPT_LEN 12
389 /**
390 * bnx2x_set_lro_mss - calculate the approximate value of the MSS
391 *
392 * @bp: driver handle
393 * @parsing_flags: parsing flags from the START CQE
394 * @len_on_bd: total length of the first packet for the
395 * aggregation.
396 *
397 * Approximate value of the MSS for this aggregation calculated using
398 * the first packet of it.
399 */
400 static u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
401 u16 len_on_bd)
402 {
403 /*
404 * TPA arrgregation won't have either IP options or TCP options
405 * other than timestamp or IPv6 extension headers.
406 */
407 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
408
409 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
410 PRS_FLAG_OVERETH_IPV6)
411 hdrs_len += sizeof(struct ipv6hdr);
412 else /* IPv4 */
413 hdrs_len += sizeof(struct iphdr);
414
415
416 /* Check if there was a TCP timestamp, if there is it's will
417 * always be 12 bytes length: nop nop kind length echo val.
418 *
419 * Otherwise FW would close the aggregation.
420 */
421 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
422 hdrs_len += TPA_TSTAMP_OPT_LEN;
423
424 return len_on_bd - hdrs_len;
425 }
426
427 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
428 struct bnx2x_fastpath *fp, u16 index)
429 {
430 struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
431 struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
432 struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
433 dma_addr_t mapping;
434
435 if (unlikely(page == NULL)) {
436 BNX2X_ERR("Can't alloc sge\n");
437 return -ENOMEM;
438 }
439
440 mapping = dma_map_page(&bp->pdev->dev, page, 0,
441 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
442 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
443 __free_pages(page, PAGES_PER_SGE_SHIFT);
444 BNX2X_ERR("Can't map sge\n");
445 return -ENOMEM;
446 }
447
448 sw_buf->page = page;
449 dma_unmap_addr_set(sw_buf, mapping, mapping);
450
451 sge->addr_hi = cpu_to_le32(U64_HI(mapping));
452 sge->addr_lo = cpu_to_le32(U64_LO(mapping));
453
454 return 0;
455 }
456
457 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
458 struct bnx2x_agg_info *tpa_info,
459 u16 pages,
460 struct sk_buff *skb,
461 struct eth_end_agg_rx_cqe *cqe,
462 u16 cqe_idx)
463 {
464 struct sw_rx_page *rx_pg, old_rx_pg;
465 u32 i, frag_len, frag_size;
466 int err, j, frag_id = 0;
467 u16 len_on_bd = tpa_info->len_on_bd;
468 u16 full_page = 0, gro_size = 0;
469
470 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
471
472 if (fp->mode == TPA_MODE_GRO) {
473 gro_size = tpa_info->gro_size;
474 full_page = tpa_info->full_page;
475 }
476
477 /* This is needed in order to enable forwarding support */
478 if (frag_size) {
479 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
480 tpa_info->parsing_flags, len_on_bd);
481
482 /* set for GRO */
483 if (fp->mode == TPA_MODE_GRO)
484 skb_shinfo(skb)->gso_type =
485 (GET_FLAG(tpa_info->parsing_flags,
486 PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
487 PRS_FLAG_OVERETH_IPV6) ?
488 SKB_GSO_TCPV6 : SKB_GSO_TCPV4;
489 }
490
491
492 #ifdef BNX2X_STOP_ON_ERROR
493 if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
494 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
495 pages, cqe_idx);
496 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
497 bnx2x_panic();
498 return -EINVAL;
499 }
500 #endif
501
502 /* Run through the SGL and compose the fragmented skb */
503 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
504 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
505
506 /* FW gives the indices of the SGE as if the ring is an array
507 (meaning that "next" element will consume 2 indices) */
508 if (fp->mode == TPA_MODE_GRO)
509 frag_len = min_t(u32, frag_size, (u32)full_page);
510 else /* LRO */
511 frag_len = min_t(u32, frag_size,
512 (u32)(SGE_PAGE_SIZE * PAGES_PER_SGE));
513
514 rx_pg = &fp->rx_page_ring[sge_idx];
515 old_rx_pg = *rx_pg;
516
517 /* If we fail to allocate a substitute page, we simply stop
518 where we are and drop the whole packet */
519 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
520 if (unlikely(err)) {
521 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
522 return err;
523 }
524
525 /* Unmap the page as we r going to pass it to the stack */
526 dma_unmap_page(&bp->pdev->dev,
527 dma_unmap_addr(&old_rx_pg, mapping),
528 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
529 /* Add one frag and update the appropriate fields in the skb */
530 if (fp->mode == TPA_MODE_LRO)
531 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
532 else { /* GRO */
533 int rem;
534 int offset = 0;
535 for (rem = frag_len; rem > 0; rem -= gro_size) {
536 int len = rem > gro_size ? gro_size : rem;
537 skb_fill_page_desc(skb, frag_id++,
538 old_rx_pg.page, offset, len);
539 if (offset)
540 get_page(old_rx_pg.page);
541 offset += len;
542 }
543 }
544
545 skb->data_len += frag_len;
546 skb->truesize += SGE_PAGE_SIZE * PAGES_PER_SGE;
547 skb->len += frag_len;
548
549 frag_size -= frag_len;
550 }
551
552 return 0;
553 }
554
555 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
556 struct bnx2x_agg_info *tpa_info,
557 u16 pages,
558 struct eth_end_agg_rx_cqe *cqe,
559 u16 cqe_idx)
560 {
561 struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
562 u8 pad = tpa_info->placement_offset;
563 u16 len = tpa_info->len_on_bd;
564 struct sk_buff *skb = NULL;
565 u8 *new_data, *data = rx_buf->data;
566 u8 old_tpa_state = tpa_info->tpa_state;
567
568 tpa_info->tpa_state = BNX2X_TPA_STOP;
569
570 /* If we there was an error during the handling of the TPA_START -
571 * drop this aggregation.
572 */
573 if (old_tpa_state == BNX2X_TPA_ERROR)
574 goto drop;
575
576 /* Try to allocate the new data */
577 new_data = kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
578
579 /* Unmap skb in the pool anyway, as we are going to change
580 pool entry status to BNX2X_TPA_STOP even if new skb allocation
581 fails. */
582 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
583 fp->rx_buf_size, DMA_FROM_DEVICE);
584 if (likely(new_data))
585 skb = build_skb(data, 0);
586
587 if (likely(skb)) {
588 #ifdef BNX2X_STOP_ON_ERROR
589 if (pad + len > fp->rx_buf_size) {
590 BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n",
591 pad, len, fp->rx_buf_size);
592 bnx2x_panic();
593 return;
594 }
595 #endif
596
597 skb_reserve(skb, pad + NET_SKB_PAD);
598 skb_put(skb, len);
599 skb->rxhash = tpa_info->rxhash;
600 skb->l4_rxhash = tpa_info->l4_rxhash;
601
602 skb->protocol = eth_type_trans(skb, bp->dev);
603 skb->ip_summed = CHECKSUM_UNNECESSARY;
604
605 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
606 skb, cqe, cqe_idx)) {
607 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
608 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
609 napi_gro_receive(&fp->napi, skb);
610 } else {
611 DP(NETIF_MSG_RX_STATUS,
612 "Failed to allocate new pages - dropping packet!\n");
613 dev_kfree_skb_any(skb);
614 }
615
616
617 /* put new data in bin */
618 rx_buf->data = new_data;
619
620 return;
621 }
622 kfree(new_data);
623 drop:
624 /* drop the packet and keep the buffer in the bin */
625 DP(NETIF_MSG_RX_STATUS,
626 "Failed to allocate or map a new skb - dropping packet!\n");
627 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
628 }
629
630 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
631 struct bnx2x_fastpath *fp, u16 index)
632 {
633 u8 *data;
634 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
635 struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
636 dma_addr_t mapping;
637
638 data = kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
639 if (unlikely(data == NULL))
640 return -ENOMEM;
641
642 mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
643 fp->rx_buf_size,
644 DMA_FROM_DEVICE);
645 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
646 kfree(data);
647 BNX2X_ERR("Can't map rx data\n");
648 return -ENOMEM;
649 }
650
651 rx_buf->data = data;
652 dma_unmap_addr_set(rx_buf, mapping, mapping);
653
654 rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
655 rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
656
657 return 0;
658 }
659
660 static
661 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
662 struct bnx2x_fastpath *fp,
663 struct bnx2x_eth_q_stats *qstats)
664 {
665 /* Do nothing if no IP/L4 csum validation was done */
666
667 if (cqe->fast_path_cqe.status_flags &
668 (ETH_FAST_PATH_RX_CQE_IP_XSUM_NO_VALIDATION_FLG |
669 ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG))
670 return;
671
672 /* If both IP/L4 validation were done, check if an error was found. */
673
674 if (cqe->fast_path_cqe.type_error_flags &
675 (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
676 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
677 qstats->hw_csum_err++;
678 else
679 skb->ip_summed = CHECKSUM_UNNECESSARY;
680 }
681
682 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
683 {
684 struct bnx2x *bp = fp->bp;
685 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
686 u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
687 int rx_pkt = 0;
688
689 #ifdef BNX2X_STOP_ON_ERROR
690 if (unlikely(bp->panic))
691 return 0;
692 #endif
693
694 /* CQ "next element" is of the size of the regular element,
695 that's why it's ok here */
696 hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
697 if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
698 hw_comp_cons++;
699
700 bd_cons = fp->rx_bd_cons;
701 bd_prod = fp->rx_bd_prod;
702 bd_prod_fw = bd_prod;
703 sw_comp_cons = fp->rx_comp_cons;
704 sw_comp_prod = fp->rx_comp_prod;
705
706 /* Memory barrier necessary as speculative reads of the rx
707 * buffer can be ahead of the index in the status block
708 */
709 rmb();
710
711 DP(NETIF_MSG_RX_STATUS,
712 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n",
713 fp->index, hw_comp_cons, sw_comp_cons);
714
715 while (sw_comp_cons != hw_comp_cons) {
716 struct sw_rx_bd *rx_buf = NULL;
717 struct sk_buff *skb;
718 union eth_rx_cqe *cqe;
719 struct eth_fast_path_rx_cqe *cqe_fp;
720 u8 cqe_fp_flags;
721 enum eth_rx_cqe_type cqe_fp_type;
722 u16 len, pad, queue;
723 u8 *data;
724 bool l4_rxhash;
725
726 #ifdef BNX2X_STOP_ON_ERROR
727 if (unlikely(bp->panic))
728 return 0;
729 #endif
730
731 comp_ring_cons = RCQ_BD(sw_comp_cons);
732 bd_prod = RX_BD(bd_prod);
733 bd_cons = RX_BD(bd_cons);
734
735 cqe = &fp->rx_comp_ring[comp_ring_cons];
736 cqe_fp = &cqe->fast_path_cqe;
737 cqe_fp_flags = cqe_fp->type_error_flags;
738 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
739
740 DP(NETIF_MSG_RX_STATUS,
741 "CQE type %x err %x status %x queue %x vlan %x len %u\n",
742 CQE_TYPE(cqe_fp_flags),
743 cqe_fp_flags, cqe_fp->status_flags,
744 le32_to_cpu(cqe_fp->rss_hash_result),
745 le16_to_cpu(cqe_fp->vlan_tag),
746 le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
747
748 /* is this a slowpath msg? */
749 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
750 bnx2x_sp_event(fp, cqe);
751 goto next_cqe;
752 }
753
754 rx_buf = &fp->rx_buf_ring[bd_cons];
755 data = rx_buf->data;
756
757 if (!CQE_TYPE_FAST(cqe_fp_type)) {
758 struct bnx2x_agg_info *tpa_info;
759 u16 frag_size, pages;
760 #ifdef BNX2X_STOP_ON_ERROR
761 /* sanity check */
762 if (fp->disable_tpa &&
763 (CQE_TYPE_START(cqe_fp_type) ||
764 CQE_TYPE_STOP(cqe_fp_type)))
765 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
766 CQE_TYPE(cqe_fp_type));
767 #endif
768
769 if (CQE_TYPE_START(cqe_fp_type)) {
770 u16 queue = cqe_fp->queue_index;
771 DP(NETIF_MSG_RX_STATUS,
772 "calling tpa_start on queue %d\n",
773 queue);
774
775 bnx2x_tpa_start(fp, queue,
776 bd_cons, bd_prod,
777 cqe_fp);
778
779 goto next_rx;
780
781 }
782 queue = cqe->end_agg_cqe.queue_index;
783 tpa_info = &fp->tpa_info[queue];
784 DP(NETIF_MSG_RX_STATUS,
785 "calling tpa_stop on queue %d\n",
786 queue);
787
788 frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
789 tpa_info->len_on_bd;
790
791 if (fp->mode == TPA_MODE_GRO)
792 pages = (frag_size + tpa_info->full_page - 1) /
793 tpa_info->full_page;
794 else
795 pages = SGE_PAGE_ALIGN(frag_size) >>
796 SGE_PAGE_SHIFT;
797
798 bnx2x_tpa_stop(bp, fp, tpa_info, pages,
799 &cqe->end_agg_cqe, comp_ring_cons);
800 #ifdef BNX2X_STOP_ON_ERROR
801 if (bp->panic)
802 return 0;
803 #endif
804
805 bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
806 goto next_cqe;
807 }
808 /* non TPA */
809 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
810 pad = cqe_fp->placement_offset;
811 dma_sync_single_for_cpu(&bp->pdev->dev,
812 dma_unmap_addr(rx_buf, mapping),
813 pad + RX_COPY_THRESH,
814 DMA_FROM_DEVICE);
815 pad += NET_SKB_PAD;
816 prefetch(data + pad); /* speedup eth_type_trans() */
817 /* is this an error packet? */
818 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
819 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
820 "ERROR flags %x rx packet %u\n",
821 cqe_fp_flags, sw_comp_cons);
822 bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
823 goto reuse_rx;
824 }
825
826 /* Since we don't have a jumbo ring
827 * copy small packets if mtu > 1500
828 */
829 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
830 (len <= RX_COPY_THRESH)) {
831 skb = netdev_alloc_skb_ip_align(bp->dev, len);
832 if (skb == NULL) {
833 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
834 "ERROR packet dropped because of alloc failure\n");
835 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
836 goto reuse_rx;
837 }
838 memcpy(skb->data, data + pad, len);
839 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
840 } else {
841 if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
842 dma_unmap_single(&bp->pdev->dev,
843 dma_unmap_addr(rx_buf, mapping),
844 fp->rx_buf_size,
845 DMA_FROM_DEVICE);
846 skb = build_skb(data, 0);
847 if (unlikely(!skb)) {
848 kfree(data);
849 bnx2x_fp_qstats(bp, fp)->
850 rx_skb_alloc_failed++;
851 goto next_rx;
852 }
853 skb_reserve(skb, pad);
854 } else {
855 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
856 "ERROR packet dropped because of alloc failure\n");
857 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
858 reuse_rx:
859 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
860 goto next_rx;
861 }
862 }
863
864 skb_put(skb, len);
865 skb->protocol = eth_type_trans(skb, bp->dev);
866
867 /* Set Toeplitz hash for a none-LRO skb */
868 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
869 skb->l4_rxhash = l4_rxhash;
870
871 skb_checksum_none_assert(skb);
872
873 if (bp->dev->features & NETIF_F_RXCSUM)
874 bnx2x_csum_validate(skb, cqe, fp,
875 bnx2x_fp_qstats(bp, fp));
876
877 skb_record_rx_queue(skb, fp->rx_queue);
878
879 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
880 PARSING_FLAGS_VLAN)
881 __vlan_hwaccel_put_tag(skb,
882 le16_to_cpu(cqe_fp->vlan_tag));
883 napi_gro_receive(&fp->napi, skb);
884
885
886 next_rx:
887 rx_buf->data = NULL;
888
889 bd_cons = NEXT_RX_IDX(bd_cons);
890 bd_prod = NEXT_RX_IDX(bd_prod);
891 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
892 rx_pkt++;
893 next_cqe:
894 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
895 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
896
897 if (rx_pkt == budget)
898 break;
899 } /* while */
900
901 fp->rx_bd_cons = bd_cons;
902 fp->rx_bd_prod = bd_prod_fw;
903 fp->rx_comp_cons = sw_comp_cons;
904 fp->rx_comp_prod = sw_comp_prod;
905
906 /* Update producers */
907 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
908 fp->rx_sge_prod);
909
910 fp->rx_pkt += rx_pkt;
911 fp->rx_calls++;
912
913 return rx_pkt;
914 }
915
916 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
917 {
918 struct bnx2x_fastpath *fp = fp_cookie;
919 struct bnx2x *bp = fp->bp;
920 u8 cos;
921
922 DP(NETIF_MSG_INTR,
923 "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
924 fp->index, fp->fw_sb_id, fp->igu_sb_id);
925 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
926
927 #ifdef BNX2X_STOP_ON_ERROR
928 if (unlikely(bp->panic))
929 return IRQ_HANDLED;
930 #endif
931
932 /* Handle Rx and Tx according to MSI-X vector */
933 prefetch(fp->rx_cons_sb);
934
935 for_each_cos_in_tx_queue(fp, cos)
936 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
937
938 prefetch(&fp->sb_running_index[SM_RX_ID]);
939 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
940
941 return IRQ_HANDLED;
942 }
943
944 /* HW Lock for shared dual port PHYs */
945 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
946 {
947 mutex_lock(&bp->port.phy_mutex);
948
949 if (bp->port.need_hw_lock)
950 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
951 }
952
953 void bnx2x_release_phy_lock(struct bnx2x *bp)
954 {
955 if (bp->port.need_hw_lock)
956 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
957
958 mutex_unlock(&bp->port.phy_mutex);
959 }
960
961 /* calculates MF speed according to current linespeed and MF configuration */
962 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
963 {
964 u16 line_speed = bp->link_vars.line_speed;
965 if (IS_MF(bp)) {
966 u16 maxCfg = bnx2x_extract_max_cfg(bp,
967 bp->mf_config[BP_VN(bp)]);
968
969 /* Calculate the current MAX line speed limit for the MF
970 * devices
971 */
972 if (IS_MF_SI(bp))
973 line_speed = (line_speed * maxCfg) / 100;
974 else { /* SD mode */
975 u16 vn_max_rate = maxCfg * 100;
976
977 if (vn_max_rate < line_speed)
978 line_speed = vn_max_rate;
979 }
980 }
981
982 return line_speed;
983 }
984
985 /**
986 * bnx2x_fill_report_data - fill link report data to report
987 *
988 * @bp: driver handle
989 * @data: link state to update
990 *
991 * It uses a none-atomic bit operations because is called under the mutex.
992 */
993 static void bnx2x_fill_report_data(struct bnx2x *bp,
994 struct bnx2x_link_report_data *data)
995 {
996 u16 line_speed = bnx2x_get_mf_speed(bp);
997
998 memset(data, 0, sizeof(*data));
999
1000 /* Fill the report data: efective line speed */
1001 data->line_speed = line_speed;
1002
1003 /* Link is down */
1004 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1005 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1006 &data->link_report_flags);
1007
1008 /* Full DUPLEX */
1009 if (bp->link_vars.duplex == DUPLEX_FULL)
1010 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1011
1012 /* Rx Flow Control is ON */
1013 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1014 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1015
1016 /* Tx Flow Control is ON */
1017 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1018 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1019 }
1020
1021 /**
1022 * bnx2x_link_report - report link status to OS.
1023 *
1024 * @bp: driver handle
1025 *
1026 * Calls the __bnx2x_link_report() under the same locking scheme
1027 * as a link/PHY state managing code to ensure a consistent link
1028 * reporting.
1029 */
1030
1031 void bnx2x_link_report(struct bnx2x *bp)
1032 {
1033 bnx2x_acquire_phy_lock(bp);
1034 __bnx2x_link_report(bp);
1035 bnx2x_release_phy_lock(bp);
1036 }
1037
1038 /**
1039 * __bnx2x_link_report - report link status to OS.
1040 *
1041 * @bp: driver handle
1042 *
1043 * None atomic inmlementation.
1044 * Should be called under the phy_lock.
1045 */
1046 void __bnx2x_link_report(struct bnx2x *bp)
1047 {
1048 struct bnx2x_link_report_data cur_data;
1049
1050 /* reread mf_cfg */
1051 if (!CHIP_IS_E1(bp))
1052 bnx2x_read_mf_cfg(bp);
1053
1054 /* Read the current link report info */
1055 bnx2x_fill_report_data(bp, &cur_data);
1056
1057 /* Don't report link down or exactly the same link status twice */
1058 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1059 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1060 &bp->last_reported_link.link_report_flags) &&
1061 test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1062 &cur_data.link_report_flags)))
1063 return;
1064
1065 bp->link_cnt++;
1066
1067 /* We are going to report a new link parameters now -
1068 * remember the current data for the next time.
1069 */
1070 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1071
1072 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1073 &cur_data.link_report_flags)) {
1074 netif_carrier_off(bp->dev);
1075 netdev_err(bp->dev, "NIC Link is Down\n");
1076 return;
1077 } else {
1078 const char *duplex;
1079 const char *flow;
1080
1081 netif_carrier_on(bp->dev);
1082
1083 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1084 &cur_data.link_report_flags))
1085 duplex = "full";
1086 else
1087 duplex = "half";
1088
1089 /* Handle the FC at the end so that only these flags would be
1090 * possibly set. This way we may easily check if there is no FC
1091 * enabled.
1092 */
1093 if (cur_data.link_report_flags) {
1094 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1095 &cur_data.link_report_flags)) {
1096 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1097 &cur_data.link_report_flags))
1098 flow = "ON - receive & transmit";
1099 else
1100 flow = "ON - receive";
1101 } else {
1102 flow = "ON - transmit";
1103 }
1104 } else {
1105 flow = "none";
1106 }
1107 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1108 cur_data.line_speed, duplex, flow);
1109 }
1110 }
1111
1112 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1113 {
1114 int i;
1115
1116 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1117 struct eth_rx_sge *sge;
1118
1119 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1120 sge->addr_hi =
1121 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1122 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1123
1124 sge->addr_lo =
1125 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1126 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1127 }
1128 }
1129
1130 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1131 struct bnx2x_fastpath *fp, int last)
1132 {
1133 int i;
1134
1135 for (i = 0; i < last; i++) {
1136 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1137 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1138 u8 *data = first_buf->data;
1139
1140 if (data == NULL) {
1141 DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1142 continue;
1143 }
1144 if (tpa_info->tpa_state == BNX2X_TPA_START)
1145 dma_unmap_single(&bp->pdev->dev,
1146 dma_unmap_addr(first_buf, mapping),
1147 fp->rx_buf_size, DMA_FROM_DEVICE);
1148 kfree(data);
1149 first_buf->data = NULL;
1150 }
1151 }
1152
1153 void bnx2x_init_rx_rings(struct bnx2x *bp)
1154 {
1155 int func = BP_FUNC(bp);
1156 u16 ring_prod;
1157 int i, j;
1158
1159 /* Allocate TPA resources */
1160 for_each_rx_queue(bp, j) {
1161 struct bnx2x_fastpath *fp = &bp->fp[j];
1162
1163 DP(NETIF_MSG_IFUP,
1164 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1165
1166 if (!fp->disable_tpa) {
1167 /* Fill the per-aggregtion pool */
1168 for (i = 0; i < MAX_AGG_QS(bp); i++) {
1169 struct bnx2x_agg_info *tpa_info =
1170 &fp->tpa_info[i];
1171 struct sw_rx_bd *first_buf =
1172 &tpa_info->first_buf;
1173
1174 first_buf->data = kmalloc(fp->rx_buf_size + NET_SKB_PAD,
1175 GFP_ATOMIC);
1176 if (!first_buf->data) {
1177 BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1178 j);
1179 bnx2x_free_tpa_pool(bp, fp, i);
1180 fp->disable_tpa = 1;
1181 break;
1182 }
1183 dma_unmap_addr_set(first_buf, mapping, 0);
1184 tpa_info->tpa_state = BNX2X_TPA_STOP;
1185 }
1186
1187 /* "next page" elements initialization */
1188 bnx2x_set_next_page_sgl(fp);
1189
1190 /* set SGEs bit mask */
1191 bnx2x_init_sge_ring_bit_mask(fp);
1192
1193 /* Allocate SGEs and initialize the ring elements */
1194 for (i = 0, ring_prod = 0;
1195 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1196
1197 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1198 BNX2X_ERR("was only able to allocate %d rx sges\n",
1199 i);
1200 BNX2X_ERR("disabling TPA for queue[%d]\n",
1201 j);
1202 /* Cleanup already allocated elements */
1203 bnx2x_free_rx_sge_range(bp, fp,
1204 ring_prod);
1205 bnx2x_free_tpa_pool(bp, fp,
1206 MAX_AGG_QS(bp));
1207 fp->disable_tpa = 1;
1208 ring_prod = 0;
1209 break;
1210 }
1211 ring_prod = NEXT_SGE_IDX(ring_prod);
1212 }
1213
1214 fp->rx_sge_prod = ring_prod;
1215 }
1216 }
1217
1218 for_each_rx_queue(bp, j) {
1219 struct bnx2x_fastpath *fp = &bp->fp[j];
1220
1221 fp->rx_bd_cons = 0;
1222
1223 /* Activate BD ring */
1224 /* Warning!
1225 * this will generate an interrupt (to the TSTORM)
1226 * must only be done after chip is initialized
1227 */
1228 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1229 fp->rx_sge_prod);
1230
1231 if (j != 0)
1232 continue;
1233
1234 if (CHIP_IS_E1(bp)) {
1235 REG_WR(bp, BAR_USTRORM_INTMEM +
1236 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1237 U64_LO(fp->rx_comp_mapping));
1238 REG_WR(bp, BAR_USTRORM_INTMEM +
1239 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1240 U64_HI(fp->rx_comp_mapping));
1241 }
1242 }
1243 }
1244
1245 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1246 {
1247 int i;
1248 u8 cos;
1249
1250 for_each_tx_queue(bp, i) {
1251 struct bnx2x_fastpath *fp = &bp->fp[i];
1252 for_each_cos_in_tx_queue(fp, cos) {
1253 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1254 unsigned pkts_compl = 0, bytes_compl = 0;
1255
1256 u16 sw_prod = txdata->tx_pkt_prod;
1257 u16 sw_cons = txdata->tx_pkt_cons;
1258
1259 while (sw_cons != sw_prod) {
1260 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1261 &pkts_compl, &bytes_compl);
1262 sw_cons++;
1263 }
1264 netdev_tx_reset_queue(
1265 netdev_get_tx_queue(bp->dev,
1266 txdata->txq_index));
1267 }
1268 }
1269 }
1270
1271 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1272 {
1273 struct bnx2x *bp = fp->bp;
1274 int i;
1275
1276 /* ring wasn't allocated */
1277 if (fp->rx_buf_ring == NULL)
1278 return;
1279
1280 for (i = 0; i < NUM_RX_BD; i++) {
1281 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1282 u8 *data = rx_buf->data;
1283
1284 if (data == NULL)
1285 continue;
1286 dma_unmap_single(&bp->pdev->dev,
1287 dma_unmap_addr(rx_buf, mapping),
1288 fp->rx_buf_size, DMA_FROM_DEVICE);
1289
1290 rx_buf->data = NULL;
1291 kfree(data);
1292 }
1293 }
1294
1295 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1296 {
1297 int j;
1298
1299 for_each_rx_queue(bp, j) {
1300 struct bnx2x_fastpath *fp = &bp->fp[j];
1301
1302 bnx2x_free_rx_bds(fp);
1303
1304 if (!fp->disable_tpa)
1305 bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1306 }
1307 }
1308
1309 void bnx2x_free_skbs(struct bnx2x *bp)
1310 {
1311 bnx2x_free_tx_skbs(bp);
1312 bnx2x_free_rx_skbs(bp);
1313 }
1314
1315 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1316 {
1317 /* load old values */
1318 u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1319
1320 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1321 /* leave all but MAX value */
1322 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1323
1324 /* set new MAX value */
1325 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1326 & FUNC_MF_CFG_MAX_BW_MASK;
1327
1328 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1329 }
1330 }
1331
1332 /**
1333 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1334 *
1335 * @bp: driver handle
1336 * @nvecs: number of vectors to be released
1337 */
1338 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1339 {
1340 int i, offset = 0;
1341
1342 if (nvecs == offset)
1343 return;
1344 free_irq(bp->msix_table[offset].vector, bp->dev);
1345 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1346 bp->msix_table[offset].vector);
1347 offset++;
1348 #ifdef BCM_CNIC
1349 if (nvecs == offset)
1350 return;
1351 offset++;
1352 #endif
1353
1354 for_each_eth_queue(bp, i) {
1355 if (nvecs == offset)
1356 return;
1357 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1358 i, bp->msix_table[offset].vector);
1359
1360 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1361 }
1362 }
1363
1364 void bnx2x_free_irq(struct bnx2x *bp)
1365 {
1366 if (bp->flags & USING_MSIX_FLAG &&
1367 !(bp->flags & USING_SINGLE_MSIX_FLAG))
1368 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1369 CNIC_PRESENT + 1);
1370 else
1371 free_irq(bp->dev->irq, bp->dev);
1372 }
1373
1374 int bnx2x_enable_msix(struct bnx2x *bp)
1375 {
1376 int msix_vec = 0, i, rc, req_cnt;
1377
1378 bp->msix_table[msix_vec].entry = msix_vec;
1379 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1380 bp->msix_table[0].entry);
1381 msix_vec++;
1382
1383 #ifdef BCM_CNIC
1384 bp->msix_table[msix_vec].entry = msix_vec;
1385 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1386 bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1387 msix_vec++;
1388 #endif
1389 /* We need separate vectors for ETH queues only (not FCoE) */
1390 for_each_eth_queue(bp, i) {
1391 bp->msix_table[msix_vec].entry = msix_vec;
1392 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1393 msix_vec, msix_vec, i);
1394 msix_vec++;
1395 }
1396
1397 req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_PRESENT + 1;
1398
1399 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1400
1401 /*
1402 * reconfigure number of tx/rx queues according to available
1403 * MSI-X vectors
1404 */
1405 if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1406 /* how less vectors we will have? */
1407 int diff = req_cnt - rc;
1408
1409 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1410
1411 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1412
1413 if (rc) {
1414 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1415 goto no_msix;
1416 }
1417 /*
1418 * decrease number of queues by number of unallocated entries
1419 */
1420 bp->num_queues -= diff;
1421
1422 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1423 bp->num_queues);
1424 } else if (rc > 0) {
1425 /* Get by with single vector */
1426 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1427 if (rc) {
1428 BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1429 rc);
1430 goto no_msix;
1431 }
1432
1433 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1434 bp->flags |= USING_SINGLE_MSIX_FLAG;
1435
1436 } else if (rc < 0) {
1437 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1438 goto no_msix;
1439 }
1440
1441 bp->flags |= USING_MSIX_FLAG;
1442
1443 return 0;
1444
1445 no_msix:
1446 /* fall to INTx if not enough memory */
1447 if (rc == -ENOMEM)
1448 bp->flags |= DISABLE_MSI_FLAG;
1449
1450 return rc;
1451 }
1452
1453 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1454 {
1455 int i, rc, offset = 0;
1456
1457 rc = request_irq(bp->msix_table[offset++].vector,
1458 bnx2x_msix_sp_int, 0,
1459 bp->dev->name, bp->dev);
1460 if (rc) {
1461 BNX2X_ERR("request sp irq failed\n");
1462 return -EBUSY;
1463 }
1464
1465 #ifdef BCM_CNIC
1466 offset++;
1467 #endif
1468 for_each_eth_queue(bp, i) {
1469 struct bnx2x_fastpath *fp = &bp->fp[i];
1470 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1471 bp->dev->name, i);
1472
1473 rc = request_irq(bp->msix_table[offset].vector,
1474 bnx2x_msix_fp_int, 0, fp->name, fp);
1475 if (rc) {
1476 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i,
1477 bp->msix_table[offset].vector, rc);
1478 bnx2x_free_msix_irqs(bp, offset);
1479 return -EBUSY;
1480 }
1481
1482 offset++;
1483 }
1484
1485 i = BNX2X_NUM_ETH_QUEUES(bp);
1486 offset = 1 + CNIC_PRESENT;
1487 netdev_info(bp->dev, "using MSI-X IRQs: sp %d fp[%d] %d ... fp[%d] %d\n",
1488 bp->msix_table[0].vector,
1489 0, bp->msix_table[offset].vector,
1490 i - 1, bp->msix_table[offset + i - 1].vector);
1491
1492 return 0;
1493 }
1494
1495 int bnx2x_enable_msi(struct bnx2x *bp)
1496 {
1497 int rc;
1498
1499 rc = pci_enable_msi(bp->pdev);
1500 if (rc) {
1501 BNX2X_DEV_INFO("MSI is not attainable\n");
1502 return -1;
1503 }
1504 bp->flags |= USING_MSI_FLAG;
1505
1506 return 0;
1507 }
1508
1509 static int bnx2x_req_irq(struct bnx2x *bp)
1510 {
1511 unsigned long flags;
1512 unsigned int irq;
1513
1514 if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1515 flags = 0;
1516 else
1517 flags = IRQF_SHARED;
1518
1519 if (bp->flags & USING_MSIX_FLAG)
1520 irq = bp->msix_table[0].vector;
1521 else
1522 irq = bp->pdev->irq;
1523
1524 return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1525 }
1526
1527 static int bnx2x_setup_irqs(struct bnx2x *bp)
1528 {
1529 int rc = 0;
1530 if (bp->flags & USING_MSIX_FLAG &&
1531 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1532 rc = bnx2x_req_msix_irqs(bp);
1533 if (rc)
1534 return rc;
1535 } else {
1536 bnx2x_ack_int(bp);
1537 rc = bnx2x_req_irq(bp);
1538 if (rc) {
1539 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1540 return rc;
1541 }
1542 if (bp->flags & USING_MSI_FLAG) {
1543 bp->dev->irq = bp->pdev->irq;
1544 netdev_info(bp->dev, "using MSI IRQ %d\n",
1545 bp->dev->irq);
1546 }
1547 if (bp->flags & USING_MSIX_FLAG) {
1548 bp->dev->irq = bp->msix_table[0].vector;
1549 netdev_info(bp->dev, "using MSIX IRQ %d\n",
1550 bp->dev->irq);
1551 }
1552 }
1553
1554 return 0;
1555 }
1556
1557 static void bnx2x_napi_enable(struct bnx2x *bp)
1558 {
1559 int i;
1560
1561 for_each_rx_queue(bp, i)
1562 napi_enable(&bnx2x_fp(bp, i, napi));
1563 }
1564
1565 static void bnx2x_napi_disable(struct bnx2x *bp)
1566 {
1567 int i;
1568
1569 for_each_rx_queue(bp, i)
1570 napi_disable(&bnx2x_fp(bp, i, napi));
1571 }
1572
1573 void bnx2x_netif_start(struct bnx2x *bp)
1574 {
1575 if (netif_running(bp->dev)) {
1576 bnx2x_napi_enable(bp);
1577 bnx2x_int_enable(bp);
1578 if (bp->state == BNX2X_STATE_OPEN)
1579 netif_tx_wake_all_queues(bp->dev);
1580 }
1581 }
1582
1583 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1584 {
1585 bnx2x_int_disable_sync(bp, disable_hw);
1586 bnx2x_napi_disable(bp);
1587 }
1588
1589 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1590 {
1591 struct bnx2x *bp = netdev_priv(dev);
1592
1593 #ifdef BCM_CNIC
1594 if (!NO_FCOE(bp)) {
1595 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1596 u16 ether_type = ntohs(hdr->h_proto);
1597
1598 /* Skip VLAN tag if present */
1599 if (ether_type == ETH_P_8021Q) {
1600 struct vlan_ethhdr *vhdr =
1601 (struct vlan_ethhdr *)skb->data;
1602
1603 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1604 }
1605
1606 /* If ethertype is FCoE or FIP - use FCoE ring */
1607 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1608 return bnx2x_fcoe_tx(bp, txq_index);
1609 }
1610 #endif
1611 /* select a non-FCoE queue */
1612 return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1613 }
1614
1615
1616 void bnx2x_set_num_queues(struct bnx2x *bp)
1617 {
1618 /* RSS queues */
1619 bp->num_queues = bnx2x_calc_num_queues(bp);
1620
1621 #ifdef BCM_CNIC
1622 /* override in STORAGE SD modes */
1623 if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1624 bp->num_queues = 1;
1625 #endif
1626 /* Add special queues */
1627 bp->num_queues += NON_ETH_CONTEXT_USE;
1628
1629 BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1630 }
1631
1632 /**
1633 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1634 *
1635 * @bp: Driver handle
1636 *
1637 * We currently support for at most 16 Tx queues for each CoS thus we will
1638 * allocate a multiple of 16 for ETH L2 rings according to the value of the
1639 * bp->max_cos.
1640 *
1641 * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1642 * index after all ETH L2 indices.
1643 *
1644 * If the actual number of Tx queues (for each CoS) is less than 16 then there
1645 * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1646 * 16..31,...) with indicies that are not coupled with any real Tx queue.
1647 *
1648 * The proper configuration of skb->queue_mapping is handled by
1649 * bnx2x_select_queue() and __skb_tx_hash().
1650 *
1651 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1652 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1653 */
1654 static int bnx2x_set_real_num_queues(struct bnx2x *bp)
1655 {
1656 int rc, tx, rx;
1657
1658 tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1659 rx = BNX2X_NUM_QUEUES(bp) - NON_ETH_CONTEXT_USE;
1660
1661 /* account for fcoe queue */
1662 #ifdef BCM_CNIC
1663 if (!NO_FCOE(bp)) {
1664 rx += FCOE_PRESENT;
1665 tx += FCOE_PRESENT;
1666 }
1667 #endif
1668
1669 rc = netif_set_real_num_tx_queues(bp->dev, tx);
1670 if (rc) {
1671 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1672 return rc;
1673 }
1674 rc = netif_set_real_num_rx_queues(bp->dev, rx);
1675 if (rc) {
1676 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1677 return rc;
1678 }
1679
1680 DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1681 tx, rx);
1682
1683 return rc;
1684 }
1685
1686 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1687 {
1688 int i;
1689
1690 for_each_queue(bp, i) {
1691 struct bnx2x_fastpath *fp = &bp->fp[i];
1692 u32 mtu;
1693
1694 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1695 if (IS_FCOE_IDX(i))
1696 /*
1697 * Although there are no IP frames expected to arrive to
1698 * this ring we still want to add an
1699 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1700 * overrun attack.
1701 */
1702 mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1703 else
1704 mtu = bp->dev->mtu;
1705 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1706 IP_HEADER_ALIGNMENT_PADDING +
1707 ETH_OVREHEAD +
1708 mtu +
1709 BNX2X_FW_RX_ALIGN_END;
1710 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1711 }
1712 }
1713
1714 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1715 {
1716 int i;
1717 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1718
1719 /* Prepare the initial contents fo the indirection table if RSS is
1720 * enabled
1721 */
1722 for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1723 bp->rss_conf_obj.ind_table[i] =
1724 bp->fp->cl_id +
1725 ethtool_rxfh_indir_default(i, num_eth_queues);
1726
1727 /*
1728 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1729 * per-port, so if explicit configuration is needed , do it only
1730 * for a PMF.
1731 *
1732 * For 57712 and newer on the other hand it's a per-function
1733 * configuration.
1734 */
1735 return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1736 }
1737
1738 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1739 bool config_hash)
1740 {
1741 struct bnx2x_config_rss_params params = {NULL};
1742 int i;
1743
1744 /* Although RSS is meaningless when there is a single HW queue we
1745 * still need it enabled in order to have HW Rx hash generated.
1746 *
1747 * if (!is_eth_multi(bp))
1748 * bp->multi_mode = ETH_RSS_MODE_DISABLED;
1749 */
1750
1751 params.rss_obj = rss_obj;
1752
1753 __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1754
1755 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1756
1757 /* RSS configuration */
1758 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1759 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1760 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1761 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1762 if (rss_obj->udp_rss_v4)
1763 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1764 if (rss_obj->udp_rss_v6)
1765 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
1766
1767 /* Hash bits */
1768 params.rss_result_mask = MULTI_MASK;
1769
1770 memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
1771
1772 if (config_hash) {
1773 /* RSS keys */
1774 for (i = 0; i < sizeof(params.rss_key) / 4; i++)
1775 params.rss_key[i] = random32();
1776
1777 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1778 }
1779
1780 return bnx2x_config_rss(bp, &params);
1781 }
1782
1783 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1784 {
1785 struct bnx2x_func_state_params func_params = {NULL};
1786
1787 /* Prepare parameters for function state transitions */
1788 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1789
1790 func_params.f_obj = &bp->func_obj;
1791 func_params.cmd = BNX2X_F_CMD_HW_INIT;
1792
1793 func_params.params.hw_init.load_phase = load_code;
1794
1795 return bnx2x_func_state_change(bp, &func_params);
1796 }
1797
1798 /*
1799 * Cleans the object that have internal lists without sending
1800 * ramrods. Should be run when interrutps are disabled.
1801 */
1802 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1803 {
1804 int rc;
1805 unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1806 struct bnx2x_mcast_ramrod_params rparam = {NULL};
1807 struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
1808
1809 /***************** Cleanup MACs' object first *************************/
1810
1811 /* Wait for completion of requested */
1812 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1813 /* Perform a dry cleanup */
1814 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1815
1816 /* Clean ETH primary MAC */
1817 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1818 rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
1819 &ramrod_flags);
1820 if (rc != 0)
1821 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1822
1823 /* Cleanup UC list */
1824 vlan_mac_flags = 0;
1825 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1826 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1827 &ramrod_flags);
1828 if (rc != 0)
1829 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1830
1831 /***************** Now clean mcast object *****************************/
1832 rparam.mcast_obj = &bp->mcast_obj;
1833 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1834
1835 /* Add a DEL command... */
1836 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1837 if (rc < 0)
1838 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
1839 rc);
1840
1841 /* ...and wait until all pending commands are cleared */
1842 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1843 while (rc != 0) {
1844 if (rc < 0) {
1845 BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1846 rc);
1847 return;
1848 }
1849
1850 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1851 }
1852 }
1853
1854 #ifndef BNX2X_STOP_ON_ERROR
1855 #define LOAD_ERROR_EXIT(bp, label) \
1856 do { \
1857 (bp)->state = BNX2X_STATE_ERROR; \
1858 goto label; \
1859 } while (0)
1860 #else
1861 #define LOAD_ERROR_EXIT(bp, label) \
1862 do { \
1863 (bp)->state = BNX2X_STATE_ERROR; \
1864 (bp)->panic = 1; \
1865 return -EBUSY; \
1866 } while (0)
1867 #endif
1868
1869 bool bnx2x_test_firmware_version(struct bnx2x *bp, bool is_err)
1870 {
1871 /* build FW version dword */
1872 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
1873 (BCM_5710_FW_MINOR_VERSION << 8) +
1874 (BCM_5710_FW_REVISION_VERSION << 16) +
1875 (BCM_5710_FW_ENGINEERING_VERSION << 24);
1876
1877 /* read loaded FW from chip */
1878 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
1879
1880 DP(NETIF_MSG_IFUP, "loaded fw %x, my fw %x\n", loaded_fw, my_fw);
1881
1882 if (loaded_fw != my_fw) {
1883 if (is_err)
1884 BNX2X_ERR("bnx2x with FW %x was already loaded, which mismatches my %x FW. aborting\n",
1885 loaded_fw, my_fw);
1886 return false;
1887 }
1888
1889 return true;
1890 }
1891
1892 /**
1893 * bnx2x_bz_fp - zero content of the fastpath structure.
1894 *
1895 * @bp: driver handle
1896 * @index: fastpath index to be zeroed
1897 *
1898 * Makes sure the contents of the bp->fp[index].napi is kept
1899 * intact.
1900 */
1901 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
1902 {
1903 struct bnx2x_fastpath *fp = &bp->fp[index];
1904 struct bnx2x_fp_stats *fp_stats = &bp->fp_stats[index];
1905
1906 int cos;
1907 struct napi_struct orig_napi = fp->napi;
1908 struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
1909 /* bzero bnx2x_fastpath contents */
1910 if (bp->stats_init) {
1911 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
1912 memset(fp, 0, sizeof(*fp));
1913 } else {
1914 /* Keep Queue statistics */
1915 struct bnx2x_eth_q_stats *tmp_eth_q_stats;
1916 struct bnx2x_eth_q_stats_old *tmp_eth_q_stats_old;
1917
1918 tmp_eth_q_stats = kzalloc(sizeof(struct bnx2x_eth_q_stats),
1919 GFP_KERNEL);
1920 if (tmp_eth_q_stats)
1921 memcpy(tmp_eth_q_stats, &fp_stats->eth_q_stats,
1922 sizeof(struct bnx2x_eth_q_stats));
1923
1924 tmp_eth_q_stats_old =
1925 kzalloc(sizeof(struct bnx2x_eth_q_stats_old),
1926 GFP_KERNEL);
1927 if (tmp_eth_q_stats_old)
1928 memcpy(tmp_eth_q_stats_old, &fp_stats->eth_q_stats_old,
1929 sizeof(struct bnx2x_eth_q_stats_old));
1930
1931 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
1932 memset(fp, 0, sizeof(*fp));
1933
1934 if (tmp_eth_q_stats) {
1935 memcpy(&fp_stats->eth_q_stats, tmp_eth_q_stats,
1936 sizeof(struct bnx2x_eth_q_stats));
1937 kfree(tmp_eth_q_stats);
1938 }
1939
1940 if (tmp_eth_q_stats_old) {
1941 memcpy(&fp_stats->eth_q_stats_old, tmp_eth_q_stats_old,
1942 sizeof(struct bnx2x_eth_q_stats_old));
1943 kfree(tmp_eth_q_stats_old);
1944 }
1945
1946 }
1947
1948 /* Restore the NAPI object as it has been already initialized */
1949 fp->napi = orig_napi;
1950 fp->tpa_info = orig_tpa_info;
1951 fp->bp = bp;
1952 fp->index = index;
1953 if (IS_ETH_FP(fp))
1954 fp->max_cos = bp->max_cos;
1955 else
1956 /* Special queues support only one CoS */
1957 fp->max_cos = 1;
1958
1959 /* Init txdata pointers */
1960 #ifdef BCM_CNIC
1961 if (IS_FCOE_FP(fp))
1962 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
1963 #endif
1964 if (IS_ETH_FP(fp))
1965 for_each_cos_in_tx_queue(fp, cos)
1966 fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
1967 BNX2X_NUM_ETH_QUEUES(bp) + index];
1968
1969 /*
1970 * set the tpa flag for each queue. The tpa flag determines the queue
1971 * minimal size so it must be set prior to queue memory allocation
1972 */
1973 fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
1974 (bp->flags & GRO_ENABLE_FLAG &&
1975 bnx2x_mtu_allows_gro(bp->dev->mtu)));
1976 if (bp->flags & TPA_ENABLE_FLAG)
1977 fp->mode = TPA_MODE_LRO;
1978 else if (bp->flags & GRO_ENABLE_FLAG)
1979 fp->mode = TPA_MODE_GRO;
1980
1981 #ifdef BCM_CNIC
1982 /* We don't want TPA on an FCoE L2 ring */
1983 if (IS_FCOE_FP(fp))
1984 fp->disable_tpa = 1;
1985 #endif
1986 }
1987
1988
1989 /* must be called with rtnl_lock */
1990 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1991 {
1992 int port = BP_PORT(bp);
1993 u32 load_code;
1994 int i, rc;
1995
1996 #ifdef BNX2X_STOP_ON_ERROR
1997 if (unlikely(bp->panic)) {
1998 BNX2X_ERR("Can't load NIC when there is panic\n");
1999 return -EPERM;
2000 }
2001 #endif
2002
2003 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2004
2005 /* Set the initial link reported state to link down */
2006 bnx2x_acquire_phy_lock(bp);
2007 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2008 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2009 &bp->last_reported_link.link_report_flags);
2010 bnx2x_release_phy_lock(bp);
2011
2012 /* must be called before memory allocation and HW init */
2013 bnx2x_ilt_set_info(bp);
2014
2015 /*
2016 * Zero fastpath structures preserving invariants like napi, which are
2017 * allocated only once, fp index, max_cos, bp pointer.
2018 * Also set fp->disable_tpa and txdata_ptr.
2019 */
2020 DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2021 for_each_queue(bp, i)
2022 bnx2x_bz_fp(bp, i);
2023 memset(bp->bnx2x_txq, 0, bp->bnx2x_txq_size *
2024 sizeof(struct bnx2x_fp_txdata));
2025
2026
2027 /* Set the receive queues buffer size */
2028 bnx2x_set_rx_buf_size(bp);
2029
2030 if (bnx2x_alloc_mem(bp))
2031 return -ENOMEM;
2032
2033 /* As long as bnx2x_alloc_mem() may possibly update
2034 * bp->num_queues, bnx2x_set_real_num_queues() should always
2035 * come after it.
2036 */
2037 rc = bnx2x_set_real_num_queues(bp);
2038 if (rc) {
2039 BNX2X_ERR("Unable to set real_num_queues\n");
2040 LOAD_ERROR_EXIT(bp, load_error0);
2041 }
2042
2043 /* configure multi cos mappings in kernel.
2044 * this configuration may be overriden by a multi class queue discipline
2045 * or by a dcbx negotiation result.
2046 */
2047 bnx2x_setup_tc(bp->dev, bp->max_cos);
2048
2049 bnx2x_napi_enable(bp);
2050
2051 /* set pf load just before approaching the MCP */
2052 bnx2x_set_pf_load(bp);
2053
2054 /* Send LOAD_REQUEST command to MCP
2055 * Returns the type of LOAD command:
2056 * if it is the first port to be initialized
2057 * common blocks should be initialized, otherwise - not
2058 */
2059 if (!BP_NOMCP(bp)) {
2060 /* init fw_seq */
2061 bp->fw_seq =
2062 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2063 DRV_MSG_SEQ_NUMBER_MASK);
2064 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2065
2066 /* Get current FW pulse sequence */
2067 bp->fw_drv_pulse_wr_seq =
2068 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2069 DRV_PULSE_SEQ_MASK);
2070 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2071
2072 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
2073 if (!load_code) {
2074 BNX2X_ERR("MCP response failure, aborting\n");
2075 rc = -EBUSY;
2076 LOAD_ERROR_EXIT(bp, load_error1);
2077 }
2078 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2079 BNX2X_ERR("Driver load refused\n");
2080 rc = -EBUSY; /* other port in diagnostic mode */
2081 LOAD_ERROR_EXIT(bp, load_error1);
2082 }
2083 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2084 load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2085 /* abort nic load if version mismatch */
2086 if (!bnx2x_test_firmware_version(bp, true)) {
2087 rc = -EBUSY;
2088 LOAD_ERROR_EXIT(bp, load_error2);
2089 }
2090 }
2091
2092 } else {
2093 int path = BP_PATH(bp);
2094
2095 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n",
2096 path, load_count[path][0], load_count[path][1],
2097 load_count[path][2]);
2098 load_count[path][0]++;
2099 load_count[path][1 + port]++;
2100 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n",
2101 path, load_count[path][0], load_count[path][1],
2102 load_count[path][2]);
2103 if (load_count[path][0] == 1)
2104 load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
2105 else if (load_count[path][1 + port] == 1)
2106 load_code = FW_MSG_CODE_DRV_LOAD_PORT;
2107 else
2108 load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
2109 }
2110
2111 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2112 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2113 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2114 bp->port.pmf = 1;
2115 /*
2116 * We need the barrier to ensure the ordering between the
2117 * writing to bp->port.pmf here and reading it from the
2118 * bnx2x_periodic_task().
2119 */
2120 smp_mb();
2121 } else
2122 bp->port.pmf = 0;
2123
2124 DP(NETIF_MSG_IFUP, "pmf %d\n", bp->port.pmf);
2125
2126 /* Init Function state controlling object */
2127 bnx2x__init_func_obj(bp);
2128
2129 /* Initialize HW */
2130 rc = bnx2x_init_hw(bp, load_code);
2131 if (rc) {
2132 BNX2X_ERR("HW init failed, aborting\n");
2133 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2134 LOAD_ERROR_EXIT(bp, load_error2);
2135 }
2136
2137 /* Connect to IRQs */
2138 rc = bnx2x_setup_irqs(bp);
2139 if (rc) {
2140 BNX2X_ERR("IRQs setup failed\n");
2141 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2142 LOAD_ERROR_EXIT(bp, load_error2);
2143 }
2144
2145 /* Setup NIC internals and enable interrupts */
2146 bnx2x_nic_init(bp, load_code);
2147
2148 /* Init per-function objects */
2149 bnx2x_init_bp_objs(bp);
2150
2151 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2152 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2153 (bp->common.shmem2_base)) {
2154 if (SHMEM2_HAS(bp, dcc_support))
2155 SHMEM2_WR(bp, dcc_support,
2156 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2157 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2158 if (SHMEM2_HAS(bp, afex_driver_support))
2159 SHMEM2_WR(bp, afex_driver_support,
2160 SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2161 }
2162
2163 /* Set AFEX default VLAN tag to an invalid value */
2164 bp->afex_def_vlan_tag = -1;
2165
2166 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2167 rc = bnx2x_func_start(bp);
2168 if (rc) {
2169 BNX2X_ERR("Function start failed!\n");
2170 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2171 LOAD_ERROR_EXIT(bp, load_error3);
2172 }
2173
2174 /* Send LOAD_DONE command to MCP */
2175 if (!BP_NOMCP(bp)) {
2176 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2177 if (!load_code) {
2178 BNX2X_ERR("MCP response failure, aborting\n");
2179 rc = -EBUSY;
2180 LOAD_ERROR_EXIT(bp, load_error3);
2181 }
2182 }
2183
2184 rc = bnx2x_setup_leading(bp);
2185 if (rc) {
2186 BNX2X_ERR("Setup leading failed!\n");
2187 LOAD_ERROR_EXIT(bp, load_error3);
2188 }
2189
2190 #ifdef BCM_CNIC
2191 /* Enable Timer scan */
2192 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2193 #endif
2194
2195 for_each_nondefault_queue(bp, i) {
2196 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2197 if (rc) {
2198 BNX2X_ERR("Queue setup failed\n");
2199 LOAD_ERROR_EXIT(bp, load_error4);
2200 }
2201 }
2202
2203 rc = bnx2x_init_rss_pf(bp);
2204 if (rc) {
2205 BNX2X_ERR("PF RSS init failed\n");
2206 LOAD_ERROR_EXIT(bp, load_error4);
2207 }
2208
2209 /* Now when Clients are configured we are ready to work */
2210 bp->state = BNX2X_STATE_OPEN;
2211
2212 /* Configure a ucast MAC */
2213 rc = bnx2x_set_eth_mac(bp, true);
2214 if (rc) {
2215 BNX2X_ERR("Setting Ethernet MAC failed\n");
2216 LOAD_ERROR_EXIT(bp, load_error4);
2217 }
2218
2219 if (bp->pending_max) {
2220 bnx2x_update_max_mf_config(bp, bp->pending_max);
2221 bp->pending_max = 0;
2222 }
2223
2224 if (bp->port.pmf)
2225 bnx2x_initial_phy_init(bp, load_mode);
2226
2227 /* Start fast path */
2228
2229 /* Initialize Rx filter. */
2230 netif_addr_lock_bh(bp->dev);
2231 bnx2x_set_rx_mode(bp->dev);
2232 netif_addr_unlock_bh(bp->dev);
2233
2234 /* Start the Tx */
2235 switch (load_mode) {
2236 case LOAD_NORMAL:
2237 /* Tx queue should be only reenabled */
2238 netif_tx_wake_all_queues(bp->dev);
2239 break;
2240
2241 case LOAD_OPEN:
2242 netif_tx_start_all_queues(bp->dev);
2243 smp_mb__after_clear_bit();
2244 break;
2245
2246 case LOAD_DIAG:
2247 case LOAD_LOOPBACK_EXT:
2248 bp->state = BNX2X_STATE_DIAG;
2249 break;
2250
2251 default:
2252 break;
2253 }
2254
2255 if (bp->port.pmf)
2256 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_DCB_CONFIGURED, 0);
2257 else
2258 bnx2x__link_status_update(bp);
2259
2260 /* start the timer */
2261 mod_timer(&bp->timer, jiffies + bp->current_interval);
2262
2263 #ifdef BCM_CNIC
2264 /* re-read iscsi info */
2265 bnx2x_get_iscsi_info(bp);
2266 bnx2x_setup_cnic_irq_info(bp);
2267 bnx2x_setup_cnic_info(bp);
2268 if (bp->state == BNX2X_STATE_OPEN)
2269 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2270 #endif
2271
2272 /* mark driver is loaded in shmem2 */
2273 if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2274 u32 val;
2275 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2276 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2277 val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2278 DRV_FLAGS_CAPABILITIES_LOADED_L2);
2279 }
2280
2281 /* Wait for all pending SP commands to complete */
2282 if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2283 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2284 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2285 return -EBUSY;
2286 }
2287
2288 /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2289 if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2290 bnx2x_dcbx_init(bp, false);
2291
2292 return 0;
2293
2294 #ifndef BNX2X_STOP_ON_ERROR
2295 load_error4:
2296 #ifdef BCM_CNIC
2297 /* Disable Timer scan */
2298 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2299 #endif
2300 load_error3:
2301 bnx2x_int_disable_sync(bp, 1);
2302
2303 /* Clean queueable objects */
2304 bnx2x_squeeze_objects(bp);
2305
2306 /* Free SKBs, SGEs, TPA pool and driver internals */
2307 bnx2x_free_skbs(bp);
2308 for_each_rx_queue(bp, i)
2309 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2310
2311 /* Release IRQs */
2312 bnx2x_free_irq(bp);
2313 load_error2:
2314 if (!BP_NOMCP(bp)) {
2315 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2316 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2317 }
2318
2319 bp->port.pmf = 0;
2320 load_error1:
2321 bnx2x_napi_disable(bp);
2322 /* clear pf_load status, as it was already set */
2323 bnx2x_clear_pf_load(bp);
2324 load_error0:
2325 bnx2x_free_mem(bp);
2326
2327 return rc;
2328 #endif /* ! BNX2X_STOP_ON_ERROR */
2329 }
2330
2331 /* must be called with rtnl_lock */
2332 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
2333 {
2334 int i;
2335 bool global = false;
2336
2337 /* mark driver is unloaded in shmem2 */
2338 if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2339 u32 val;
2340 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2341 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2342 val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2343 }
2344
2345 if ((bp->state == BNX2X_STATE_CLOSED) ||
2346 (bp->state == BNX2X_STATE_ERROR)) {
2347 /* We can get here if the driver has been unloaded
2348 * during parity error recovery and is either waiting for a
2349 * leader to complete or for other functions to unload and
2350 * then ifdown has been issued. In this case we want to
2351 * unload and let other functions to complete a recovery
2352 * process.
2353 */
2354 bp->recovery_state = BNX2X_RECOVERY_DONE;
2355 bp->is_leader = 0;
2356 bnx2x_release_leader_lock(bp);
2357 smp_mb();
2358
2359 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2360 BNX2X_ERR("Can't unload in closed or error state\n");
2361 return -EINVAL;
2362 }
2363
2364 /*
2365 * It's important to set the bp->state to the value different from
2366 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2367 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2368 */
2369 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2370 smp_mb();
2371
2372 /* Stop Tx */
2373 bnx2x_tx_disable(bp);
2374 netdev_reset_tc(bp->dev);
2375
2376 #ifdef BCM_CNIC
2377 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2378 #endif
2379
2380 bp->rx_mode = BNX2X_RX_MODE_NONE;
2381
2382 del_timer_sync(&bp->timer);
2383
2384 /* Set ALWAYS_ALIVE bit in shmem */
2385 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2386
2387 bnx2x_drv_pulse(bp);
2388
2389 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2390 bnx2x_save_statistics(bp);
2391
2392 /* Cleanup the chip if needed */
2393 if (unload_mode != UNLOAD_RECOVERY)
2394 bnx2x_chip_cleanup(bp, unload_mode);
2395 else {
2396 /* Send the UNLOAD_REQUEST to the MCP */
2397 bnx2x_send_unload_req(bp, unload_mode);
2398
2399 /*
2400 * Prevent transactions to host from the functions on the
2401 * engine that doesn't reset global blocks in case of global
2402 * attention once gloabl blocks are reset and gates are opened
2403 * (the engine which leader will perform the recovery
2404 * last).
2405 */
2406 if (!CHIP_IS_E1x(bp))
2407 bnx2x_pf_disable(bp);
2408
2409 /* Disable HW interrupts, NAPI */
2410 bnx2x_netif_stop(bp, 1);
2411
2412 /* Release IRQs */
2413 bnx2x_free_irq(bp);
2414
2415 /* Report UNLOAD_DONE to MCP */
2416 bnx2x_send_unload_done(bp);
2417 }
2418
2419 /*
2420 * At this stage no more interrupts will arrive so we may safly clean
2421 * the queueable objects here in case they failed to get cleaned so far.
2422 */
2423 bnx2x_squeeze_objects(bp);
2424
2425 /* There should be no more pending SP commands at this stage */
2426 bp->sp_state = 0;
2427
2428 bp->port.pmf = 0;
2429
2430 /* Free SKBs, SGEs, TPA pool and driver internals */
2431 bnx2x_free_skbs(bp);
2432 for_each_rx_queue(bp, i)
2433 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2434
2435 bnx2x_free_mem(bp);
2436
2437 bp->state = BNX2X_STATE_CLOSED;
2438
2439 /* Check if there are pending parity attentions. If there are - set
2440 * RECOVERY_IN_PROGRESS.
2441 */
2442 if (bnx2x_chk_parity_attn(bp, &global, false)) {
2443 bnx2x_set_reset_in_progress(bp);
2444
2445 /* Set RESET_IS_GLOBAL if needed */
2446 if (global)
2447 bnx2x_set_reset_global(bp);
2448 }
2449
2450
2451 /* The last driver must disable a "close the gate" if there is no
2452 * parity attention or "process kill" pending.
2453 */
2454 if (!bnx2x_clear_pf_load(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2455 bnx2x_disable_close_the_gate(bp);
2456
2457 return 0;
2458 }
2459
2460 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2461 {
2462 u16 pmcsr;
2463
2464 /* If there is no power capability, silently succeed */
2465 if (!bp->pm_cap) {
2466 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2467 return 0;
2468 }
2469
2470 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2471
2472 switch (state) {
2473 case PCI_D0:
2474 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2475 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2476 PCI_PM_CTRL_PME_STATUS));
2477
2478 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2479 /* delay required during transition out of D3hot */
2480 msleep(20);
2481 break;
2482
2483 case PCI_D3hot:
2484 /* If there are other clients above don't
2485 shut down the power */
2486 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2487 return 0;
2488 /* Don't shut down the power for emulation and FPGA */
2489 if (CHIP_REV_IS_SLOW(bp))
2490 return 0;
2491
2492 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2493 pmcsr |= 3;
2494
2495 if (bp->wol)
2496 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2497
2498 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2499 pmcsr);
2500
2501 /* No more memory access after this point until
2502 * device is brought back to D0.
2503 */
2504 break;
2505
2506 default:
2507 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
2508 return -EINVAL;
2509 }
2510 return 0;
2511 }
2512
2513 /*
2514 * net_device service functions
2515 */
2516 int bnx2x_poll(struct napi_struct *napi, int budget)
2517 {
2518 int work_done = 0;
2519 u8 cos;
2520 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2521 napi);
2522 struct bnx2x *bp = fp->bp;
2523
2524 while (1) {
2525 #ifdef BNX2X_STOP_ON_ERROR
2526 if (unlikely(bp->panic)) {
2527 napi_complete(napi);
2528 return 0;
2529 }
2530 #endif
2531
2532 for_each_cos_in_tx_queue(fp, cos)
2533 if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
2534 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
2535
2536
2537 if (bnx2x_has_rx_work(fp)) {
2538 work_done += bnx2x_rx_int(fp, budget - work_done);
2539
2540 /* must not complete if we consumed full budget */
2541 if (work_done >= budget)
2542 break;
2543 }
2544
2545 /* Fall out from the NAPI loop if needed */
2546 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2547 #ifdef BCM_CNIC
2548 /* No need to update SB for FCoE L2 ring as long as
2549 * it's connected to the default SB and the SB
2550 * has been updated when NAPI was scheduled.
2551 */
2552 if (IS_FCOE_FP(fp)) {
2553 napi_complete(napi);
2554 break;
2555 }
2556 #endif
2557
2558 bnx2x_update_fpsb_idx(fp);
2559 /* bnx2x_has_rx_work() reads the status block,
2560 * thus we need to ensure that status block indices
2561 * have been actually read (bnx2x_update_fpsb_idx)
2562 * prior to this check (bnx2x_has_rx_work) so that
2563 * we won't write the "newer" value of the status block
2564 * to IGU (if there was a DMA right after
2565 * bnx2x_has_rx_work and if there is no rmb, the memory
2566 * reading (bnx2x_update_fpsb_idx) may be postponed
2567 * to right before bnx2x_ack_sb). In this case there
2568 * will never be another interrupt until there is
2569 * another update of the status block, while there
2570 * is still unhandled work.
2571 */
2572 rmb();
2573
2574 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2575 napi_complete(napi);
2576 /* Re-enable interrupts */
2577 DP(NETIF_MSG_RX_STATUS,
2578 "Update index to %d\n", fp->fp_hc_idx);
2579 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2580 le16_to_cpu(fp->fp_hc_idx),
2581 IGU_INT_ENABLE, 1);
2582 break;
2583 }
2584 }
2585 }
2586
2587 return work_done;
2588 }
2589
2590 /* we split the first BD into headers and data BDs
2591 * to ease the pain of our fellow microcode engineers
2592 * we use one mapping for both BDs
2593 */
2594 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2595 struct bnx2x_fp_txdata *txdata,
2596 struct sw_tx_bd *tx_buf,
2597 struct eth_tx_start_bd **tx_bd, u16 hlen,
2598 u16 bd_prod, int nbd)
2599 {
2600 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2601 struct eth_tx_bd *d_tx_bd;
2602 dma_addr_t mapping;
2603 int old_len = le16_to_cpu(h_tx_bd->nbytes);
2604
2605 /* first fix first BD */
2606 h_tx_bd->nbd = cpu_to_le16(nbd);
2607 h_tx_bd->nbytes = cpu_to_le16(hlen);
2608
2609 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x) nbd %d\n",
2610 h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo, h_tx_bd->nbd);
2611
2612 /* now get a new data BD
2613 * (after the pbd) and fill it */
2614 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2615 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2616
2617 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2618 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2619
2620 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2621 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2622 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2623
2624 /* this marks the BD as one that has no individual mapping */
2625 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2626
2627 DP(NETIF_MSG_TX_QUEUED,
2628 "TSO split data size is %d (%x:%x)\n",
2629 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2630
2631 /* update tx_bd */
2632 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2633
2634 return bd_prod;
2635 }
2636
2637 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2638 {
2639 if (fix > 0)
2640 csum = (u16) ~csum_fold(csum_sub(csum,
2641 csum_partial(t_header - fix, fix, 0)));
2642
2643 else if (fix < 0)
2644 csum = (u16) ~csum_fold(csum_add(csum,
2645 csum_partial(t_header, -fix, 0)));
2646
2647 return swab16(csum);
2648 }
2649
2650 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2651 {
2652 u32 rc;
2653
2654 if (skb->ip_summed != CHECKSUM_PARTIAL)
2655 rc = XMIT_PLAIN;
2656
2657 else {
2658 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2659 rc = XMIT_CSUM_V6;
2660 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2661 rc |= XMIT_CSUM_TCP;
2662
2663 } else {
2664 rc = XMIT_CSUM_V4;
2665 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2666 rc |= XMIT_CSUM_TCP;
2667 }
2668 }
2669
2670 if (skb_is_gso_v6(skb))
2671 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2672 else if (skb_is_gso(skb))
2673 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2674
2675 return rc;
2676 }
2677
2678 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2679 /* check if packet requires linearization (packet is too fragmented)
2680 no need to check fragmentation if page size > 8K (there will be no
2681 violation to FW restrictions) */
2682 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2683 u32 xmit_type)
2684 {
2685 int to_copy = 0;
2686 int hlen = 0;
2687 int first_bd_sz = 0;
2688
2689 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2690 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2691
2692 if (xmit_type & XMIT_GSO) {
2693 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2694 /* Check if LSO packet needs to be copied:
2695 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2696 int wnd_size = MAX_FETCH_BD - 3;
2697 /* Number of windows to check */
2698 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2699 int wnd_idx = 0;
2700 int frag_idx = 0;
2701 u32 wnd_sum = 0;
2702
2703 /* Headers length */
2704 hlen = (int)(skb_transport_header(skb) - skb->data) +
2705 tcp_hdrlen(skb);
2706
2707 /* Amount of data (w/o headers) on linear part of SKB*/
2708 first_bd_sz = skb_headlen(skb) - hlen;
2709
2710 wnd_sum = first_bd_sz;
2711
2712 /* Calculate the first sum - it's special */
2713 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2714 wnd_sum +=
2715 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
2716
2717 /* If there was data on linear skb data - check it */
2718 if (first_bd_sz > 0) {
2719 if (unlikely(wnd_sum < lso_mss)) {
2720 to_copy = 1;
2721 goto exit_lbl;
2722 }
2723
2724 wnd_sum -= first_bd_sz;
2725 }
2726
2727 /* Others are easier: run through the frag list and
2728 check all windows */
2729 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2730 wnd_sum +=
2731 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
2732
2733 if (unlikely(wnd_sum < lso_mss)) {
2734 to_copy = 1;
2735 break;
2736 }
2737 wnd_sum -=
2738 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
2739 }
2740 } else {
2741 /* in non-LSO too fragmented packet should always
2742 be linearized */
2743 to_copy = 1;
2744 }
2745 }
2746
2747 exit_lbl:
2748 if (unlikely(to_copy))
2749 DP(NETIF_MSG_TX_QUEUED,
2750 "Linearization IS REQUIRED for %s packet. num_frags %d hlen %d first_bd_sz %d\n",
2751 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2752 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2753
2754 return to_copy;
2755 }
2756 #endif
2757
2758 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2759 u32 xmit_type)
2760 {
2761 *parsing_data |= (skb_shinfo(skb)->gso_size <<
2762 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2763 ETH_TX_PARSE_BD_E2_LSO_MSS;
2764 if ((xmit_type & XMIT_GSO_V6) &&
2765 (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2766 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2767 }
2768
2769 /**
2770 * bnx2x_set_pbd_gso - update PBD in GSO case.
2771 *
2772 * @skb: packet skb
2773 * @pbd: parse BD
2774 * @xmit_type: xmit flags
2775 */
2776 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2777 struct eth_tx_parse_bd_e1x *pbd,
2778 u32 xmit_type)
2779 {
2780 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2781 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2782 pbd->tcp_flags = pbd_tcp_flags(skb);
2783
2784 if (xmit_type & XMIT_GSO_V4) {
2785 pbd->ip_id = swab16(ip_hdr(skb)->id);
2786 pbd->tcp_pseudo_csum =
2787 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2788 ip_hdr(skb)->daddr,
2789 0, IPPROTO_TCP, 0));
2790
2791 } else
2792 pbd->tcp_pseudo_csum =
2793 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2794 &ipv6_hdr(skb)->daddr,
2795 0, IPPROTO_TCP, 0));
2796
2797 pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2798 }
2799
2800 /**
2801 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
2802 *
2803 * @bp: driver handle
2804 * @skb: packet skb
2805 * @parsing_data: data to be updated
2806 * @xmit_type: xmit flags
2807 *
2808 * 57712 related
2809 */
2810 static inline u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2811 u32 *parsing_data, u32 xmit_type)
2812 {
2813 *parsing_data |=
2814 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
2815 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2816 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
2817
2818 if (xmit_type & XMIT_CSUM_TCP) {
2819 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
2820 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2821 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
2822
2823 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2824 } else
2825 /* We support checksum offload for TCP and UDP only.
2826 * No need to pass the UDP header length - it's a constant.
2827 */
2828 return skb_transport_header(skb) +
2829 sizeof(struct udphdr) - skb->data;
2830 }
2831
2832 static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2833 struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
2834 {
2835 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2836
2837 if (xmit_type & XMIT_CSUM_V4)
2838 tx_start_bd->bd_flags.as_bitfield |=
2839 ETH_TX_BD_FLAGS_IP_CSUM;
2840 else
2841 tx_start_bd->bd_flags.as_bitfield |=
2842 ETH_TX_BD_FLAGS_IPV6;
2843
2844 if (!(xmit_type & XMIT_CSUM_TCP))
2845 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
2846 }
2847
2848 /**
2849 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
2850 *
2851 * @bp: driver handle
2852 * @skb: packet skb
2853 * @pbd: parse BD to be updated
2854 * @xmit_type: xmit flags
2855 */
2856 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2857 struct eth_tx_parse_bd_e1x *pbd,
2858 u32 xmit_type)
2859 {
2860 u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
2861
2862 /* for now NS flag is not used in Linux */
2863 pbd->global_data =
2864 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2865 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2866
2867 pbd->ip_hlen_w = (skb_transport_header(skb) -
2868 skb_network_header(skb)) >> 1;
2869
2870 hlen += pbd->ip_hlen_w;
2871
2872 /* We support checksum offload for TCP and UDP only */
2873 if (xmit_type & XMIT_CSUM_TCP)
2874 hlen += tcp_hdrlen(skb) / 2;
2875 else
2876 hlen += sizeof(struct udphdr) / 2;
2877
2878 pbd->total_hlen_w = cpu_to_le16(hlen);
2879 hlen = hlen*2;
2880
2881 if (xmit_type & XMIT_CSUM_TCP) {
2882 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2883
2884 } else {
2885 s8 fix = SKB_CS_OFF(skb); /* signed! */
2886
2887 DP(NETIF_MSG_TX_QUEUED,
2888 "hlen %d fix %d csum before fix %x\n",
2889 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2890
2891 /* HW bug: fixup the CSUM */
2892 pbd->tcp_pseudo_csum =
2893 bnx2x_csum_fix(skb_transport_header(skb),
2894 SKB_CS(skb), fix);
2895
2896 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2897 pbd->tcp_pseudo_csum);
2898 }
2899
2900 return hlen;
2901 }
2902
2903 /* called with netif_tx_lock
2904 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2905 * netif_wake_queue()
2906 */
2907 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2908 {
2909 struct bnx2x *bp = netdev_priv(dev);
2910
2911 struct netdev_queue *txq;
2912 struct bnx2x_fp_txdata *txdata;
2913 struct sw_tx_bd *tx_buf;
2914 struct eth_tx_start_bd *tx_start_bd, *first_bd;
2915 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2916 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
2917 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2918 u32 pbd_e2_parsing_data = 0;
2919 u16 pkt_prod, bd_prod;
2920 int nbd, txq_index;
2921 dma_addr_t mapping;
2922 u32 xmit_type = bnx2x_xmit_type(bp, skb);
2923 int i;
2924 u8 hlen = 0;
2925 __le16 pkt_size = 0;
2926 struct ethhdr *eth;
2927 u8 mac_type = UNICAST_ADDRESS;
2928
2929 #ifdef BNX2X_STOP_ON_ERROR
2930 if (unlikely(bp->panic))
2931 return NETDEV_TX_BUSY;
2932 #endif
2933
2934 txq_index = skb_get_queue_mapping(skb);
2935 txq = netdev_get_tx_queue(dev, txq_index);
2936
2937 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + FCOE_PRESENT);
2938
2939 txdata = &bp->bnx2x_txq[txq_index];
2940
2941 /* enable this debug print to view the transmission queue being used
2942 DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
2943 txq_index, fp_index, txdata_index); */
2944
2945 /* enable this debug print to view the tranmission details
2946 DP(NETIF_MSG_TX_QUEUED,
2947 "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
2948 txdata->cid, fp_index, txdata_index, txdata, fp); */
2949
2950 if (unlikely(bnx2x_tx_avail(bp, txdata) <
2951 skb_shinfo(skb)->nr_frags +
2952 BDS_PER_TX_PKT +
2953 NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
2954 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
2955 netif_tx_stop_queue(txq);
2956 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2957 return NETDEV_TX_BUSY;
2958 }
2959
2960 DP(NETIF_MSG_TX_QUEUED,
2961 "queue[%d]: SKB: summed %x protocol %x protocol(%x,%x) gso type %x xmit_type %x\n",
2962 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2963 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2964
2965 eth = (struct ethhdr *)skb->data;
2966
2967 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2968 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2969 if (is_broadcast_ether_addr(eth->h_dest))
2970 mac_type = BROADCAST_ADDRESS;
2971 else
2972 mac_type = MULTICAST_ADDRESS;
2973 }
2974
2975 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2976 /* First, check if we need to linearize the skb (due to FW
2977 restrictions). No need to check fragmentation if page size > 8K
2978 (there will be no violation to FW restrictions) */
2979 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2980 /* Statistics of linearization */
2981 bp->lin_cnt++;
2982 if (skb_linearize(skb) != 0) {
2983 DP(NETIF_MSG_TX_QUEUED,
2984 "SKB linearization failed - silently dropping this SKB\n");
2985 dev_kfree_skb_any(skb);
2986 return NETDEV_TX_OK;
2987 }
2988 }
2989 #endif
2990 /* Map skb linear data for DMA */
2991 mapping = dma_map_single(&bp->pdev->dev, skb->data,
2992 skb_headlen(skb), DMA_TO_DEVICE);
2993 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2994 DP(NETIF_MSG_TX_QUEUED,
2995 "SKB mapping failed - silently dropping this SKB\n");
2996 dev_kfree_skb_any(skb);
2997 return NETDEV_TX_OK;
2998 }
2999 /*
3000 Please read carefully. First we use one BD which we mark as start,
3001 then we have a parsing info BD (used for TSO or xsum),
3002 and only then we have the rest of the TSO BDs.
3003 (don't forget to mark the last one as last,
3004 and to unmap only AFTER you write to the BD ...)
3005 And above all, all pdb sizes are in words - NOT DWORDS!
3006 */
3007
3008 /* get current pkt produced now - advance it just before sending packet
3009 * since mapping of pages may fail and cause packet to be dropped
3010 */
3011 pkt_prod = txdata->tx_pkt_prod;
3012 bd_prod = TX_BD(txdata->tx_bd_prod);
3013
3014 /* get a tx_buf and first BD
3015 * tx_start_bd may be changed during SPLIT,
3016 * but first_bd will always stay first
3017 */
3018 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3019 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3020 first_bd = tx_start_bd;
3021
3022 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3023 SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
3024 mac_type);
3025
3026 /* header nbd */
3027 SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
3028
3029 /* remember the first BD of the packet */
3030 tx_buf->first_bd = txdata->tx_bd_prod;
3031 tx_buf->skb = skb;
3032 tx_buf->flags = 0;
3033
3034 DP(NETIF_MSG_TX_QUEUED,
3035 "sending pkt %u @%p next_idx %u bd %u @%p\n",
3036 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3037
3038 if (vlan_tx_tag_present(skb)) {
3039 tx_start_bd->vlan_or_ethertype =
3040 cpu_to_le16(vlan_tx_tag_get(skb));
3041 tx_start_bd->bd_flags.as_bitfield |=
3042 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3043 } else
3044 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3045
3046 /* turn on parsing and get a BD */
3047 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3048
3049 if (xmit_type & XMIT_CSUM)
3050 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3051
3052 if (!CHIP_IS_E1x(bp)) {
3053 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3054 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3055 /* Set PBD in checksum offload case */
3056 if (xmit_type & XMIT_CSUM)
3057 hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3058 &pbd_e2_parsing_data,
3059 xmit_type);
3060 if (IS_MF_SI(bp)) {
3061 /*
3062 * fill in the MAC addresses in the PBD - for local
3063 * switching
3064 */
3065 bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
3066 &pbd_e2->src_mac_addr_mid,
3067 &pbd_e2->src_mac_addr_lo,
3068 eth->h_source);
3069 bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
3070 &pbd_e2->dst_mac_addr_mid,
3071 &pbd_e2->dst_mac_addr_lo,
3072 eth->h_dest);
3073 }
3074 } else {
3075 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3076 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3077 /* Set PBD in checksum offload case */
3078 if (xmit_type & XMIT_CSUM)
3079 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3080
3081 }
3082
3083 /* Setup the data pointer of the first BD of the packet */
3084 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3085 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3086 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3087 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3088 pkt_size = tx_start_bd->nbytes;
3089
3090 DP(NETIF_MSG_TX_QUEUED,
3091 "first bd @%p addr (%x:%x) nbd %d nbytes %d flags %x vlan %x\n",
3092 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3093 le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
3094 tx_start_bd->bd_flags.as_bitfield,
3095 le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3096
3097 if (xmit_type & XMIT_GSO) {
3098
3099 DP(NETIF_MSG_TX_QUEUED,
3100 "TSO packet len %d hlen %d total len %d tso size %d\n",
3101 skb->len, hlen, skb_headlen(skb),
3102 skb_shinfo(skb)->gso_size);
3103
3104 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3105
3106 if (unlikely(skb_headlen(skb) > hlen))
3107 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3108 &tx_start_bd, hlen,
3109 bd_prod, ++nbd);
3110 if (!CHIP_IS_E1x(bp))
3111 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3112 xmit_type);
3113 else
3114 bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
3115 }
3116
3117 /* Set the PBD's parsing_data field if not zero
3118 * (for the chips newer than 57711).
3119 */
3120 if (pbd_e2_parsing_data)
3121 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3122
3123 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3124
3125 /* Handle fragmented skb */
3126 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3127 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3128
3129 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3130 skb_frag_size(frag), DMA_TO_DEVICE);
3131 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3132 unsigned int pkts_compl = 0, bytes_compl = 0;
3133
3134 DP(NETIF_MSG_TX_QUEUED,
3135 "Unable to map page - dropping packet...\n");
3136
3137 /* we need unmap all buffers already mapped
3138 * for this SKB;
3139 * first_bd->nbd need to be properly updated
3140 * before call to bnx2x_free_tx_pkt
3141 */
3142 first_bd->nbd = cpu_to_le16(nbd);
3143 bnx2x_free_tx_pkt(bp, txdata,
3144 TX_BD(txdata->tx_pkt_prod),
3145 &pkts_compl, &bytes_compl);
3146 return NETDEV_TX_OK;
3147 }
3148
3149 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3150 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3151 if (total_pkt_bd == NULL)
3152 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3153
3154 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3155 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3156 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3157 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3158 nbd++;
3159
3160 DP(NETIF_MSG_TX_QUEUED,
3161 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
3162 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3163 le16_to_cpu(tx_data_bd->nbytes));
3164 }
3165
3166 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3167
3168 /* update with actual num BDs */
3169 first_bd->nbd = cpu_to_le16(nbd);
3170
3171 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3172
3173 /* now send a tx doorbell, counting the next BD
3174 * if the packet contains or ends with it
3175 */
3176 if (TX_BD_POFF(bd_prod) < nbd)
3177 nbd++;
3178
3179 /* total_pkt_bytes should be set on the first data BD if
3180 * it's not an LSO packet and there is more than one
3181 * data BD. In this case pkt_size is limited by an MTU value.
3182 * However we prefer to set it for an LSO packet (while we don't
3183 * have to) in order to save some CPU cycles in a none-LSO
3184 * case, when we much more care about them.
3185 */
3186 if (total_pkt_bd != NULL)
3187 total_pkt_bd->total_pkt_bytes = pkt_size;
3188
3189 if (pbd_e1x)
3190 DP(NETIF_MSG_TX_QUEUED,
3191 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u tcp_flags %x xsum %x seq %u hlen %u\n",
3192 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3193 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3194 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3195 le16_to_cpu(pbd_e1x->total_hlen_w));
3196 if (pbd_e2)
3197 DP(NETIF_MSG_TX_QUEUED,
3198 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
3199 pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
3200 pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
3201 pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
3202 pbd_e2->parsing_data);
3203 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
3204
3205 netdev_tx_sent_queue(txq, skb->len);
3206
3207 skb_tx_timestamp(skb);
3208
3209 txdata->tx_pkt_prod++;
3210 /*
3211 * Make sure that the BD data is updated before updating the producer
3212 * since FW might read the BD right after the producer is updated.
3213 * This is only applicable for weak-ordered memory model archs such
3214 * as IA-64. The following barrier is also mandatory since FW will
3215 * assumes packets must have BDs.
3216 */
3217 wmb();
3218
3219 txdata->tx_db.data.prod += nbd;
3220 barrier();
3221
3222 DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
3223
3224 mmiowb();
3225
3226 txdata->tx_bd_prod += nbd;
3227
3228 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
3229 netif_tx_stop_queue(txq);
3230
3231 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
3232 * ordering of set_bit() in netif_tx_stop_queue() and read of
3233 * fp->bd_tx_cons */
3234 smp_mb();
3235
3236 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3237 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
3238 netif_tx_wake_queue(txq);
3239 }
3240 txdata->tx_pkt++;
3241
3242 return NETDEV_TX_OK;
3243 }
3244
3245 /**
3246 * bnx2x_setup_tc - routine to configure net_device for multi tc
3247 *
3248 * @netdev: net device to configure
3249 * @tc: number of traffic classes to enable
3250 *
3251 * callback connected to the ndo_setup_tc function pointer
3252 */
3253 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3254 {
3255 int cos, prio, count, offset;
3256 struct bnx2x *bp = netdev_priv(dev);
3257
3258 /* setup tc must be called under rtnl lock */
3259 ASSERT_RTNL();
3260
3261 /* no traffic classes requested. aborting */
3262 if (!num_tc) {
3263 netdev_reset_tc(dev);
3264 return 0;
3265 }
3266
3267 /* requested to support too many traffic classes */
3268 if (num_tc > bp->max_cos) {
3269 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3270 num_tc, bp->max_cos);
3271 return -EINVAL;
3272 }
3273
3274 /* declare amount of supported traffic classes */
3275 if (netdev_set_num_tc(dev, num_tc)) {
3276 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
3277 return -EINVAL;
3278 }
3279
3280 /* configure priority to traffic class mapping */
3281 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
3282 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
3283 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3284 "mapping priority %d to tc %d\n",
3285 prio, bp->prio_to_cos[prio]);
3286 }
3287
3288
3289 /* Use this configuration to diffrentiate tc0 from other COSes
3290 This can be used for ets or pfc, and save the effort of setting
3291 up a multio class queue disc or negotiating DCBX with a switch
3292 netdev_set_prio_tc_map(dev, 0, 0);
3293 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
3294 for (prio = 1; prio < 16; prio++) {
3295 netdev_set_prio_tc_map(dev, prio, 1);
3296 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
3297 } */
3298
3299 /* configure traffic class to transmission queue mapping */
3300 for (cos = 0; cos < bp->max_cos; cos++) {
3301 count = BNX2X_NUM_ETH_QUEUES(bp);
3302 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
3303 netdev_set_tc_queue(dev, cos, count, offset);
3304 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3305 "mapping tc %d to offset %d count %d\n",
3306 cos, offset, count);
3307 }
3308
3309 return 0;
3310 }
3311
3312 /* called with rtnl_lock */
3313 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
3314 {
3315 struct sockaddr *addr = p;
3316 struct bnx2x *bp = netdev_priv(dev);
3317 int rc = 0;
3318
3319 if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
3320 BNX2X_ERR("Requested MAC address is not valid\n");
3321 return -EINVAL;
3322 }
3323
3324 #ifdef BCM_CNIC
3325 if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
3326 !is_zero_ether_addr(addr->sa_data)) {
3327 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
3328 return -EINVAL;
3329 }
3330 #endif
3331
3332 if (netif_running(dev)) {
3333 rc = bnx2x_set_eth_mac(bp, false);
3334 if (rc)
3335 return rc;
3336 }
3337
3338 dev->addr_assign_type &= ~NET_ADDR_RANDOM;
3339 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3340
3341 if (netif_running(dev))
3342 rc = bnx2x_set_eth_mac(bp, true);
3343
3344 return rc;
3345 }
3346
3347 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
3348 {
3349 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
3350 struct bnx2x_fastpath *fp = &bp->fp[fp_index];
3351 u8 cos;
3352
3353 /* Common */
3354 #ifdef BCM_CNIC
3355 if (IS_FCOE_IDX(fp_index)) {
3356 memset(sb, 0, sizeof(union host_hc_status_block));
3357 fp->status_blk_mapping = 0;
3358
3359 } else {
3360 #endif
3361 /* status blocks */
3362 if (!CHIP_IS_E1x(bp))
3363 BNX2X_PCI_FREE(sb->e2_sb,
3364 bnx2x_fp(bp, fp_index,
3365 status_blk_mapping),
3366 sizeof(struct host_hc_status_block_e2));
3367 else
3368 BNX2X_PCI_FREE(sb->e1x_sb,
3369 bnx2x_fp(bp, fp_index,
3370 status_blk_mapping),
3371 sizeof(struct host_hc_status_block_e1x));
3372 #ifdef BCM_CNIC
3373 }
3374 #endif
3375 /* Rx */
3376 if (!skip_rx_queue(bp, fp_index)) {
3377 bnx2x_free_rx_bds(fp);
3378
3379 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3380 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
3381 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
3382 bnx2x_fp(bp, fp_index, rx_desc_mapping),
3383 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3384
3385 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
3386 bnx2x_fp(bp, fp_index, rx_comp_mapping),
3387 sizeof(struct eth_fast_path_rx_cqe) *
3388 NUM_RCQ_BD);
3389
3390 /* SGE ring */
3391 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
3392 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
3393 bnx2x_fp(bp, fp_index, rx_sge_mapping),
3394 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3395 }
3396
3397 /* Tx */
3398 if (!skip_tx_queue(bp, fp_index)) {
3399 /* fastpath tx rings: tx_buf tx_desc */
3400 for_each_cos_in_tx_queue(fp, cos) {
3401 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
3402
3403 DP(NETIF_MSG_IFDOWN,
3404 "freeing tx memory of fp %d cos %d cid %d\n",
3405 fp_index, cos, txdata->cid);
3406
3407 BNX2X_FREE(txdata->tx_buf_ring);
3408 BNX2X_PCI_FREE(txdata->tx_desc_ring,
3409 txdata->tx_desc_mapping,
3410 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3411 }
3412 }
3413 /* end of fastpath */
3414 }
3415
3416 void bnx2x_free_fp_mem(struct bnx2x *bp)
3417 {
3418 int i;
3419 for_each_queue(bp, i)
3420 bnx2x_free_fp_mem_at(bp, i);
3421 }
3422
3423 static void set_sb_shortcuts(struct bnx2x *bp, int index)
3424 {
3425 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
3426 if (!CHIP_IS_E1x(bp)) {
3427 bnx2x_fp(bp, index, sb_index_values) =
3428 (__le16 *)status_blk.e2_sb->sb.index_values;
3429 bnx2x_fp(bp, index, sb_running_index) =
3430 (__le16 *)status_blk.e2_sb->sb.running_index;
3431 } else {
3432 bnx2x_fp(bp, index, sb_index_values) =
3433 (__le16 *)status_blk.e1x_sb->sb.index_values;
3434 bnx2x_fp(bp, index, sb_running_index) =
3435 (__le16 *)status_blk.e1x_sb->sb.running_index;
3436 }
3437 }
3438
3439 /* Returns the number of actually allocated BDs */
3440 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
3441 int rx_ring_size)
3442 {
3443 struct bnx2x *bp = fp->bp;
3444 u16 ring_prod, cqe_ring_prod;
3445 int i, failure_cnt = 0;
3446
3447 fp->rx_comp_cons = 0;
3448 cqe_ring_prod = ring_prod = 0;
3449
3450 /* This routine is called only during fo init so
3451 * fp->eth_q_stats.rx_skb_alloc_failed = 0
3452 */
3453 for (i = 0; i < rx_ring_size; i++) {
3454 if (bnx2x_alloc_rx_data(bp, fp, ring_prod) < 0) {
3455 failure_cnt++;
3456 continue;
3457 }
3458 ring_prod = NEXT_RX_IDX(ring_prod);
3459 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
3460 WARN_ON(ring_prod <= (i - failure_cnt));
3461 }
3462
3463 if (failure_cnt)
3464 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
3465 i - failure_cnt, fp->index);
3466
3467 fp->rx_bd_prod = ring_prod;
3468 /* Limit the CQE producer by the CQE ring size */
3469 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
3470 cqe_ring_prod);
3471 fp->rx_pkt = fp->rx_calls = 0;
3472
3473 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
3474
3475 return i - failure_cnt;
3476 }
3477
3478 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
3479 {
3480 int i;
3481
3482 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
3483 struct eth_rx_cqe_next_page *nextpg;
3484
3485 nextpg = (struct eth_rx_cqe_next_page *)
3486 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
3487 nextpg->addr_hi =
3488 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
3489 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
3490 nextpg->addr_lo =
3491 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
3492 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
3493 }
3494 }
3495
3496 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
3497 {
3498 union host_hc_status_block *sb;
3499 struct bnx2x_fastpath *fp = &bp->fp[index];
3500 int ring_size = 0;
3501 u8 cos;
3502 int rx_ring_size = 0;
3503
3504 #ifdef BCM_CNIC
3505 if (!bp->rx_ring_size &&
3506 (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
3507 rx_ring_size = MIN_RX_SIZE_NONTPA;
3508 bp->rx_ring_size = rx_ring_size;
3509 } else
3510 #endif
3511 if (!bp->rx_ring_size) {
3512 u32 cfg = SHMEM_RD(bp,
3513 dev_info.port_hw_config[BP_PORT(bp)].default_cfg);
3514
3515 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
3516
3517 /* Dercease ring size for 1G functions */
3518 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
3519 PORT_HW_CFG_NET_SERDES_IF_SGMII)
3520 rx_ring_size /= 10;
3521
3522 /* allocate at least number of buffers required by FW */
3523 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
3524 MIN_RX_SIZE_TPA, rx_ring_size);
3525
3526 bp->rx_ring_size = rx_ring_size;
3527 } else /* if rx_ring_size specified - use it */
3528 rx_ring_size = bp->rx_ring_size;
3529
3530 /* Common */
3531 sb = &bnx2x_fp(bp, index, status_blk);
3532 #ifdef BCM_CNIC
3533 if (!IS_FCOE_IDX(index)) {
3534 #endif
3535 /* status blocks */
3536 if (!CHIP_IS_E1x(bp))
3537 BNX2X_PCI_ALLOC(sb->e2_sb,
3538 &bnx2x_fp(bp, index, status_blk_mapping),
3539 sizeof(struct host_hc_status_block_e2));
3540 else
3541 BNX2X_PCI_ALLOC(sb->e1x_sb,
3542 &bnx2x_fp(bp, index, status_blk_mapping),
3543 sizeof(struct host_hc_status_block_e1x));
3544 #ifdef BCM_CNIC
3545 }
3546 #endif
3547
3548 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
3549 * set shortcuts for it.
3550 */
3551 if (!IS_FCOE_IDX(index))
3552 set_sb_shortcuts(bp, index);
3553
3554 /* Tx */
3555 if (!skip_tx_queue(bp, index)) {
3556 /* fastpath tx rings: tx_buf tx_desc */
3557 for_each_cos_in_tx_queue(fp, cos) {
3558 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
3559
3560 DP(NETIF_MSG_IFUP,
3561 "allocating tx memory of fp %d cos %d\n",
3562 index, cos);
3563
3564 BNX2X_ALLOC(txdata->tx_buf_ring,
3565 sizeof(struct sw_tx_bd) * NUM_TX_BD);
3566 BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
3567 &txdata->tx_desc_mapping,
3568 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3569 }
3570 }
3571
3572 /* Rx */
3573 if (!skip_rx_queue(bp, index)) {
3574 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3575 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
3576 sizeof(struct sw_rx_bd) * NUM_RX_BD);
3577 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
3578 &bnx2x_fp(bp, index, rx_desc_mapping),
3579 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3580
3581 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
3582 &bnx2x_fp(bp, index, rx_comp_mapping),
3583 sizeof(struct eth_fast_path_rx_cqe) *
3584 NUM_RCQ_BD);
3585
3586 /* SGE ring */
3587 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
3588 sizeof(struct sw_rx_page) * NUM_RX_SGE);
3589 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
3590 &bnx2x_fp(bp, index, rx_sge_mapping),
3591 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3592 /* RX BD ring */
3593 bnx2x_set_next_page_rx_bd(fp);
3594
3595 /* CQ ring */
3596 bnx2x_set_next_page_rx_cq(fp);
3597
3598 /* BDs */
3599 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
3600 if (ring_size < rx_ring_size)
3601 goto alloc_mem_err;
3602 }
3603
3604 return 0;
3605
3606 /* handles low memory cases */
3607 alloc_mem_err:
3608 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
3609 index, ring_size);
3610 /* FW will drop all packets if queue is not big enough,
3611 * In these cases we disable the queue
3612 * Min size is different for OOO, TPA and non-TPA queues
3613 */
3614 if (ring_size < (fp->disable_tpa ?
3615 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
3616 /* release memory allocated for this queue */
3617 bnx2x_free_fp_mem_at(bp, index);
3618 return -ENOMEM;
3619 }
3620 return 0;
3621 }
3622
3623 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
3624 {
3625 int i;
3626
3627 /**
3628 * 1. Allocate FP for leading - fatal if error
3629 * 2. {CNIC} Allocate FCoE FP - fatal if error
3630 * 3. {CNIC} Allocate OOO + FWD - disable OOO if error
3631 * 4. Allocate RSS - fix number of queues if error
3632 */
3633
3634 /* leading */
3635 if (bnx2x_alloc_fp_mem_at(bp, 0))
3636 return -ENOMEM;
3637
3638 #ifdef BCM_CNIC
3639 if (!NO_FCOE(bp))
3640 /* FCoE */
3641 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
3642 /* we will fail load process instead of mark
3643 * NO_FCOE_FLAG
3644 */
3645 return -ENOMEM;
3646 #endif
3647
3648 /* RSS */
3649 for_each_nondefault_eth_queue(bp, i)
3650 if (bnx2x_alloc_fp_mem_at(bp, i))
3651 break;
3652
3653 /* handle memory failures */
3654 if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
3655 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
3656
3657 WARN_ON(delta < 0);
3658 #ifdef BCM_CNIC
3659 /**
3660 * move non eth FPs next to last eth FP
3661 * must be done in that order
3662 * FCOE_IDX < FWD_IDX < OOO_IDX
3663 */
3664
3665 /* move FCoE fp even NO_FCOE_FLAG is on */
3666 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
3667 #endif
3668 bp->num_queues -= delta;
3669 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
3670 bp->num_queues + delta, bp->num_queues);
3671 }
3672
3673 return 0;
3674 }
3675
3676 void bnx2x_free_mem_bp(struct bnx2x *bp)
3677 {
3678 kfree(bp->fp->tpa_info);
3679 kfree(bp->fp);
3680 kfree(bp->sp_objs);
3681 kfree(bp->fp_stats);
3682 kfree(bp->bnx2x_txq);
3683 kfree(bp->msix_table);
3684 kfree(bp->ilt);
3685 }
3686
3687 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
3688 {
3689 struct bnx2x_fastpath *fp;
3690 struct msix_entry *tbl;
3691 struct bnx2x_ilt *ilt;
3692 int msix_table_size = 0;
3693 int fp_array_size;
3694 int i;
3695
3696 /*
3697 * The biggest MSI-X table we might need is as a maximum number of fast
3698 * path IGU SBs plus default SB (for PF).
3699 */
3700 msix_table_size = bp->igu_sb_cnt + 1;
3701
3702 /* fp array: RSS plus CNIC related L2 queues */
3703 fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + NON_ETH_CONTEXT_USE;
3704 BNX2X_DEV_INFO("fp_array_size %d", fp_array_size);
3705
3706 fp = kcalloc(fp_array_size, sizeof(*fp), GFP_KERNEL);
3707 if (!fp)
3708 goto alloc_err;
3709 for (i = 0; i < fp_array_size; i++) {
3710 fp[i].tpa_info =
3711 kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
3712 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
3713 if (!(fp[i].tpa_info))
3714 goto alloc_err;
3715 }
3716
3717 bp->fp = fp;
3718
3719 /* allocate sp objs */
3720 bp->sp_objs = kcalloc(fp_array_size, sizeof(struct bnx2x_sp_objs),
3721 GFP_KERNEL);
3722 if (!bp->sp_objs)
3723 goto alloc_err;
3724
3725 /* allocate fp_stats */
3726 bp->fp_stats = kcalloc(fp_array_size, sizeof(struct bnx2x_fp_stats),
3727 GFP_KERNEL);
3728 if (!bp->fp_stats)
3729 goto alloc_err;
3730
3731 /* Allocate memory for the transmission queues array */
3732 bp->bnx2x_txq_size = BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS;
3733 #ifdef BCM_CNIC
3734 bp->bnx2x_txq_size++;
3735 #endif
3736 bp->bnx2x_txq = kcalloc(bp->bnx2x_txq_size,
3737 sizeof(struct bnx2x_fp_txdata), GFP_KERNEL);
3738 if (!bp->bnx2x_txq)
3739 goto alloc_err;
3740
3741 /* msix table */
3742 tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
3743 if (!tbl)
3744 goto alloc_err;
3745 bp->msix_table = tbl;
3746
3747 /* ilt */
3748 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
3749 if (!ilt)
3750 goto alloc_err;
3751 bp->ilt = ilt;
3752
3753 return 0;
3754 alloc_err:
3755 bnx2x_free_mem_bp(bp);
3756 return -ENOMEM;
3757
3758 }
3759
3760 int bnx2x_reload_if_running(struct net_device *dev)
3761 {
3762 struct bnx2x *bp = netdev_priv(dev);
3763
3764 if (unlikely(!netif_running(dev)))
3765 return 0;
3766
3767 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
3768 return bnx2x_nic_load(bp, LOAD_NORMAL);
3769 }
3770
3771 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
3772 {
3773 u32 sel_phy_idx = 0;
3774 if (bp->link_params.num_phys <= 1)
3775 return INT_PHY;
3776
3777 if (bp->link_vars.link_up) {
3778 sel_phy_idx = EXT_PHY1;
3779 /* In case link is SERDES, check if the EXT_PHY2 is the one */
3780 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
3781 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
3782 sel_phy_idx = EXT_PHY2;
3783 } else {
3784
3785 switch (bnx2x_phy_selection(&bp->link_params)) {
3786 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
3787 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
3788 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
3789 sel_phy_idx = EXT_PHY1;
3790 break;
3791 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
3792 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
3793 sel_phy_idx = EXT_PHY2;
3794 break;
3795 }
3796 }
3797
3798 return sel_phy_idx;
3799
3800 }
3801 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
3802 {
3803 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
3804 /*
3805 * The selected actived PHY is always after swapping (in case PHY
3806 * swapping is enabled). So when swapping is enabled, we need to reverse
3807 * the configuration
3808 */
3809
3810 if (bp->link_params.multi_phy_config &
3811 PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
3812 if (sel_phy_idx == EXT_PHY1)
3813 sel_phy_idx = EXT_PHY2;
3814 else if (sel_phy_idx == EXT_PHY2)
3815 sel_phy_idx = EXT_PHY1;
3816 }
3817 return LINK_CONFIG_IDX(sel_phy_idx);
3818 }
3819
3820 #if defined(NETDEV_FCOE_WWNN) && defined(BCM_CNIC)
3821 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
3822 {
3823 struct bnx2x *bp = netdev_priv(dev);
3824 struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
3825
3826 switch (type) {
3827 case NETDEV_FCOE_WWNN:
3828 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
3829 cp->fcoe_wwn_node_name_lo);
3830 break;
3831 case NETDEV_FCOE_WWPN:
3832 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
3833 cp->fcoe_wwn_port_name_lo);
3834 break;
3835 default:
3836 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
3837 return -EINVAL;
3838 }
3839
3840 return 0;
3841 }
3842 #endif
3843
3844 /* called with rtnl_lock */
3845 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
3846 {
3847 struct bnx2x *bp = netdev_priv(dev);
3848
3849 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3850 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
3851 return -EAGAIN;
3852 }
3853
3854 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
3855 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
3856 BNX2X_ERR("Can't support requested MTU size\n");
3857 return -EINVAL;
3858 }
3859
3860 /* This does not race with packet allocation
3861 * because the actual alloc size is
3862 * only updated as part of load
3863 */
3864 dev->mtu = new_mtu;
3865
3866 return bnx2x_reload_if_running(dev);
3867 }
3868
3869 netdev_features_t bnx2x_fix_features(struct net_device *dev,
3870 netdev_features_t features)
3871 {
3872 struct bnx2x *bp = netdev_priv(dev);
3873
3874 /* TPA requires Rx CSUM offloading */
3875 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
3876 features &= ~NETIF_F_LRO;
3877 features &= ~NETIF_F_GRO;
3878 }
3879
3880 return features;
3881 }
3882
3883 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
3884 {
3885 struct bnx2x *bp = netdev_priv(dev);
3886 u32 flags = bp->flags;
3887 bool bnx2x_reload = false;
3888
3889 if (features & NETIF_F_LRO)
3890 flags |= TPA_ENABLE_FLAG;
3891 else
3892 flags &= ~TPA_ENABLE_FLAG;
3893
3894 if (features & NETIF_F_GRO)
3895 flags |= GRO_ENABLE_FLAG;
3896 else
3897 flags &= ~GRO_ENABLE_FLAG;
3898
3899 if (features & NETIF_F_LOOPBACK) {
3900 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
3901 bp->link_params.loopback_mode = LOOPBACK_BMAC;
3902 bnx2x_reload = true;
3903 }
3904 } else {
3905 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
3906 bp->link_params.loopback_mode = LOOPBACK_NONE;
3907 bnx2x_reload = true;
3908 }
3909 }
3910
3911 if (flags ^ bp->flags) {
3912 bp->flags = flags;
3913 bnx2x_reload = true;
3914 }
3915
3916 if (bnx2x_reload) {
3917 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
3918 return bnx2x_reload_if_running(dev);
3919 /* else: bnx2x_nic_load() will be called at end of recovery */
3920 }
3921
3922 return 0;
3923 }
3924
3925 void bnx2x_tx_timeout(struct net_device *dev)
3926 {
3927 struct bnx2x *bp = netdev_priv(dev);
3928
3929 #ifdef BNX2X_STOP_ON_ERROR
3930 if (!bp->panic)
3931 bnx2x_panic();
3932 #endif
3933
3934 smp_mb__before_clear_bit();
3935 set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
3936 smp_mb__after_clear_bit();
3937
3938 /* This allows the netif to be shutdown gracefully before resetting */
3939 schedule_delayed_work(&bp->sp_rtnl_task, 0);
3940 }
3941
3942 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
3943 {
3944 struct net_device *dev = pci_get_drvdata(pdev);
3945 struct bnx2x *bp;
3946
3947 if (!dev) {
3948 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3949 return -ENODEV;
3950 }
3951 bp = netdev_priv(dev);
3952
3953 rtnl_lock();
3954
3955 pci_save_state(pdev);
3956
3957 if (!netif_running(dev)) {
3958 rtnl_unlock();
3959 return 0;
3960 }
3961
3962 netif_device_detach(dev);
3963
3964 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
3965
3966 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
3967
3968 rtnl_unlock();
3969
3970 return 0;
3971 }
3972
3973 int bnx2x_resume(struct pci_dev *pdev)
3974 {
3975 struct net_device *dev = pci_get_drvdata(pdev);
3976 struct bnx2x *bp;
3977 int rc;
3978
3979 if (!dev) {
3980 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3981 return -ENODEV;
3982 }
3983 bp = netdev_priv(dev);
3984
3985 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3986 BNX2X_ERR("Handling parity error recovery. Try again later\n");
3987 return -EAGAIN;
3988 }
3989
3990 rtnl_lock();
3991
3992 pci_restore_state(pdev);
3993
3994 if (!netif_running(dev)) {
3995 rtnl_unlock();
3996 return 0;
3997 }
3998
3999 bnx2x_set_power_state(bp, PCI_D0);
4000 netif_device_attach(dev);
4001
4002 rc = bnx2x_nic_load(bp, LOAD_OPEN);
4003
4004 rtnl_unlock();
4005
4006 return rc;
4007 }
4008
4009
4010 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4011 u32 cid)
4012 {
4013 /* ustorm cxt validation */
4014 cxt->ustorm_ag_context.cdu_usage =
4015 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4016 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4017 /* xcontext validation */
4018 cxt->xstorm_ag_context.cdu_reserved =
4019 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4020 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4021 }
4022
4023 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4024 u8 fw_sb_id, u8 sb_index,
4025 u8 ticks)
4026 {
4027
4028 u32 addr = BAR_CSTRORM_INTMEM +
4029 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4030 REG_WR8(bp, addr, ticks);
4031 DP(NETIF_MSG_IFUP,
4032 "port %x fw_sb_id %d sb_index %d ticks %d\n",
4033 port, fw_sb_id, sb_index, ticks);
4034 }
4035
4036 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4037 u16 fw_sb_id, u8 sb_index,
4038 u8 disable)
4039 {
4040 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4041 u32 addr = BAR_CSTRORM_INTMEM +
4042 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4043 u16 flags = REG_RD16(bp, addr);
4044 /* clear and set */
4045 flags &= ~HC_INDEX_DATA_HC_ENABLED;
4046 flags |= enable_flag;
4047 REG_WR16(bp, addr, flags);
4048 DP(NETIF_MSG_IFUP,
4049 "port %x fw_sb_id %d sb_index %d disable %d\n",
4050 port, fw_sb_id, sb_index, disable);
4051 }
4052
4053 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4054 u8 sb_index, u8 disable, u16 usec)
4055 {
4056 int port = BP_PORT(bp);
4057 u8 ticks = usec / BNX2X_BTR;
4058
4059 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4060
4061 disable = disable ? 1 : (usec ? 0 : 1);
4062 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4063 }
Linux kernel - Deliverables
This page took 0.162022 seconds and 5 git commands to generate.