projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: can: remove custom hex_to_bin()
[deliverable/linux.git] / drivers / net / bnx2x / bnx2x_cmn.c
CommitLineData
9f6c9258
DK		 1/* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
5de92408 3 * Copyright (c) 2007-2011 Broadcom Corporation
9f6c9258
DK		 4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
9f6c9258 18#include <linux/etherdevice.h>
9bcc0893 19#include <linux/if_vlan.h>
a6b7a407 20#include <linux/interrupt.h>
9f6c9258 21#include <linux/ip.h>
f2e0899f 22#include <net/ipv6.h>
7f3e01fe 23#include <net/ip6_checksum.h>
6891dd25 24#include <linux/firmware.h>
c0cba59e 25#include <linux/prefetch.h>
9f6c9258 26#include "bnx2x_cmn.h"
523224a3 27#include "bnx2x_init.h"
042181f5 28#include "bnx2x_sp.h"
523224a3 29
619c5cb6 30
9f6c9258 31
b3b83c3f
DK		 32/**
33 * bnx2x_bz_fp - zero content of the fastpath structure.
34 *
35 * @bp: driver handle
36 * @index: fastpath index to be zeroed
37 *
38 * Makes sure the contents of the bp->fp[index].napi is kept
39 * intact.
40 */
41static inline void bnx2x_bz_fp(struct bnx2x *bp, int index)
42{
43 struct bnx2x_fastpath *fp = &bp->fp[index];
44 struct napi_struct orig_napi = fp->napi;
45 /* bzero bnx2x_fastpath contents */
46 memset(fp, 0, sizeof(*fp));
47
48 /* Restore the NAPI object as it has been already initialized */
49 fp->napi = orig_napi;
6383c0b3
AE		 50
51 fp->bp = bp;
52 fp->index = index;
53 if (IS_ETH_FP(fp))
54 fp->max_cos = bp->max_cos;
55 else
56 /* Special queues support only one CoS */
57 fp->max_cos = 1;
58
59 /*
60 * set the tpa flag for each queue. The tpa flag determines the queue
61 * minimal size so it must be set prior to queue memory allocation
62 */
63 fp->disable_tpa = ((bp->flags & TPA_ENABLE_FLAG) == 0);
64
65#ifdef BCM_CNIC
66 /* We don't want TPA on FCoE, FWD and OOO L2 rings */
67 bnx2x_fcoe(bp, disable_tpa) = 1;
68#endif
b3b83c3f
DK		 69}
70
71/**
72 * bnx2x_move_fp - move content of the fastpath structure.
73 *
74 * @bp: driver handle
75 * @from: source FP index
76 * @to: destination FP index
77 *
78 * Makes sure the contents of the bp->fp[to].napi is kept
79 * intact.
80 */
81static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
82{
83 struct bnx2x_fastpath *from_fp = &bp->fp[from];
84 struct bnx2x_fastpath *to_fp = &bp->fp[to];
85 struct napi_struct orig_napi = to_fp->napi;
86 /* Move bnx2x_fastpath contents */
87 memcpy(to_fp, from_fp, sizeof(*to_fp));
88 to_fp->index = to;
89
90 /* Restore the NAPI object as it has been already initialized */
91 to_fp->napi = orig_napi;
92}
93
619c5cb6
VZ		 94int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
95
9f6c9258
DK		 96/* free skb in the packet ring at pos idx
97 * return idx of last bd freed
98 */
6383c0b3 99static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
9f6c9258
DK		 100 u16 idx)
101{
6383c0b3 102 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
9f6c9258
DK		 103 struct eth_tx_start_bd *tx_start_bd;
104 struct eth_tx_bd *tx_data_bd;
105 struct sk_buff *skb = tx_buf->skb;
106 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
107 int nbd;
108
109 /* prefetch skb end pointer to speedup dev_kfree_skb() */
110 prefetch(&skb->end);
111
619c5cb6 112 DP(BNX2X_MSG_FP, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n",
6383c0b3 113 txdata->txq_index, idx, tx_buf, skb);
9f6c9258
DK		 114
115 /* unmap first bd */
116 DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
6383c0b3 117 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
9f6c9258 118 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
4bca60f4 119 BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
9f6c9258 120
619c5cb6 121
9f6c9258
DK		 122 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
123#ifdef BNX2X_STOP_ON_ERROR
124 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
125 BNX2X_ERR("BAD nbd!\n");
126 bnx2x_panic();
127 }
128#endif
129 new_cons = nbd + tx_buf->first_bd;
130
131 /* Get the next bd */
132 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
133
134 /* Skip a parse bd... */
135 --nbd;
136 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
137
138 /* ...and the TSO split header bd since they have no mapping */
139 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
140 --nbd;
141 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
142 }
143
144 /* now free frags */
145 while (nbd > 0) {
146
147 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
6383c0b3 148 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
9f6c9258
DK		 149 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
150 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
151 if (--nbd)
152 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
153 }
154
155 /* release skb */
156 WARN_ON(!skb);
40955532 157 dev_kfree_skb_any(skb);
9f6c9258
DK		 158 tx_buf->first_bd = 0;
159 tx_buf->skb = NULL;
160
161 return new_cons;
162}
163
6383c0b3 164int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
9f6c9258 165{
9f6c9258 166 struct netdev_queue *txq;
6383c0b3 167 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
9f6c9258
DK		 168
169#ifdef BNX2X_STOP_ON_ERROR
170 if (unlikely(bp->panic))
171 return -1;
172#endif
173
6383c0b3
AE		 174 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
175 hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
176 sw_cons = txdata->tx_pkt_cons;
9f6c9258
DK		 177
178 while (sw_cons != hw_cons) {
179 u16 pkt_cons;
180
181 pkt_cons = TX_BD(sw_cons);
182
f2e0899f
DK		 183 DP(NETIF_MSG_TX_DONE, "queue[%d]: hw_cons %u sw_cons %u "
184 " pkt_cons %u\n",
6383c0b3 185 txdata->txq_index, hw_cons, sw_cons, pkt_cons);
9f6c9258 186
6383c0b3 187 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons);
9f6c9258
DK		 188 sw_cons++;
189 }
190
6383c0b3
AE		 191 txdata->tx_pkt_cons = sw_cons;
192 txdata->tx_bd_cons = bd_cons;
9f6c9258
DK		 193
194 /* Need to make the tx_bd_cons update visible to start_xmit()
195 * before checking for netif_tx_queue_stopped(). Without the
196 * memory barrier, there is a small possibility that
197 * start_xmit() will miss it and cause the queue to be stopped
198 * forever.
619c5cb6
VZ		 199 * On the other hand we need an rmb() here to ensure the proper
200 * ordering of bit testing in the following
201 * netif_tx_queue_stopped(txq) call.
9f6c9258
DK		 202 */
203 smp_mb();
204
9f6c9258
DK		 205 if (unlikely(netif_tx_queue_stopped(txq))) {
206 /* Taking tx_lock() is needed to prevent reenabling the queue
207 * while it's empty. This could have happen if rx_action() gets
208 * suspended in bnx2x_tx_int() after the condition before
209 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
210 *
211 * stops the queue->sees fresh tx_bd_cons->releases the queue->
212 * sends some packets consuming the whole queue again->
213 * stops the queue
214 */
215
216 __netif_tx_lock(txq, smp_processor_id());
217
218 if ((netif_tx_queue_stopped(txq)) &&
219 (bp->state == BNX2X_STATE_OPEN) &&
6383c0b3 220 (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3))
9f6c9258
DK		 221 netif_tx_wake_queue(txq);
222
223 __netif_tx_unlock(txq);
224 }
225 return 0;
226}
227
228static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
229 u16 idx)
230{
231 u16 last_max = fp->last_max_sge;
232
233 if (SUB_S16(idx, last_max) > 0)
234 fp->last_max_sge = idx;
235}
236
237static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
238 struct eth_fast_path_rx_cqe *fp_cqe)
239{
240 struct bnx2x *bp = fp->bp;
241 u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
242 le16_to_cpu(fp_cqe->len_on_bd)) >>
243 SGE_PAGE_SHIFT;
244 u16 last_max, last_elem, first_elem;
245 u16 delta = 0;
246 u16 i;
247
248 if (!sge_len)
249 return;
250
251 /* First mark all used pages */
252 for (i = 0; i < sge_len; i++)
619c5cb6 253 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
523224a3 254 RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[i])));
9f6c9258
DK		 255
256 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
523224a3 257 sge_len - 1, le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
9f6c9258
DK		 258
259 /* Here we assume that the last SGE index is the biggest */
260 prefetch((void *)(fp->sge_mask));
523224a3
DK		 261 bnx2x_update_last_max_sge(fp,
262 le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
9f6c9258
DK		 263
264 last_max = RX_SGE(fp->last_max_sge);
619c5cb6
VZ		 265 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
266 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
9f6c9258
DK		 267
268 /* If ring is not full */
269 if (last_elem + 1 != first_elem)
270 last_elem++;
271
272 /* Now update the prod */
273 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
274 if (likely(fp->sge_mask[i]))
275 break;
276
619c5cb6
VZ		 277 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
278 delta += BIT_VEC64_ELEM_SZ;
9f6c9258
DK		 279 }
280
281 if (delta > 0) {
282 fp->rx_sge_prod += delta;
283 /* clear page-end entries */
284 bnx2x_clear_sge_mask_next_elems(fp);
285 }
286
287 DP(NETIF_MSG_RX_STATUS,
288 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
289 fp->last_max_sge, fp->rx_sge_prod);
290}
291
292static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
619c5cb6
VZ		 293 struct sk_buff *skb, u16 cons, u16 prod,
294 struct eth_fast_path_rx_cqe *cqe)
9f6c9258
DK		 295{
296 struct bnx2x *bp = fp->bp;
297 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
298 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
299 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
300 dma_addr_t mapping;
619c5cb6
VZ		 301 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
302 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
9f6c9258 303
619c5cb6
VZ		 304 /* print error if current state != stop */
305 if (tpa_info->tpa_state != BNX2X_TPA_STOP)
9f6c9258
DK		 306 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
307
619c5cb6
VZ		 308 /* Try to map an empty skb from the aggregation info */
309 mapping = dma_map_single(&bp->pdev->dev,
310 first_buf->skb->data,
311 fp->rx_buf_size, DMA_FROM_DEVICE);
312 /*
313 * ...if it fails - move the skb from the consumer to the producer
314 * and set the current aggregation state as ERROR to drop it
315 * when TPA_STOP arrives.
316 */
317
318 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
319 /* Move the BD from the consumer to the producer */
320 bnx2x_reuse_rx_skb(fp, cons, prod);
321 tpa_info->tpa_state = BNX2X_TPA_ERROR;
322 return;
323 }
9f6c9258 324
619c5cb6
VZ		 325 /* move empty skb from pool to prod */
326 prod_rx_buf->skb = first_buf->skb;
327 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
9f6c9258
DK		 328 /* point prod_bd to new skb */
329 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
330 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
331
619c5cb6
VZ		 332 /* move partial skb from cons to pool (don't unmap yet) */
333 *first_buf = *cons_rx_buf;
334
335 /* mark bin state as START */
336 tpa_info->parsing_flags =
337 le16_to_cpu(cqe->pars_flags.flags);
338 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
339 tpa_info->tpa_state = BNX2X_TPA_START;
340 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
341 tpa_info->placement_offset = cqe->placement_offset;
342
9f6c9258
DK		 343#ifdef BNX2X_STOP_ON_ERROR
344 fp->tpa_queue_used |= (1 << queue);
345#ifdef _ASM_GENERIC_INT_L64_H
346 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
347#else
348 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
349#endif
350 fp->tpa_queue_used);
351#endif
352}
353
e4e3c02a
VZ		 354/* Timestamp option length allowed for TPA aggregation:
355 *
356 * nop nop kind length echo val
357 */
358#define TPA_TSTAMP_OPT_LEN 12
359/**
e8920674 360 * bnx2x_set_lro_mss - calculate the approximate value of the MSS
e4e3c02a 361 *
e8920674
DK		 362 * @bp: driver handle
363 * @parsing_flags: parsing flags from the START CQE
364 * @len_on_bd: total length of the first packet for the
365 * aggregation.
366 *
367 * Approximate value of the MSS for this aggregation calculated using
368 * the first packet of it.
e4e3c02a
VZ		 369 */
370static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
371 u16 len_on_bd)
372{
619c5cb6
VZ		 373 /*
374 * TPA arrgregation won't have either IP options or TCP options
375 * other than timestamp or IPv6 extension headers.
e4e3c02a 376 */
619c5cb6
VZ		 377 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
378
379 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
380 PRS_FLAG_OVERETH_IPV6)
381 hdrs_len += sizeof(struct ipv6hdr);
382 else /* IPv4 */
383 hdrs_len += sizeof(struct iphdr);
e4e3c02a
VZ		 384
385
386 /* Check if there was a TCP timestamp, if there is it's will
387 * always be 12 bytes length: nop nop kind length echo val.
388 *
389 * Otherwise FW would close the aggregation.
390 */
391 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
392 hdrs_len += TPA_TSTAMP_OPT_LEN;
393
394 return len_on_bd - hdrs_len;
395}
396
9f6c9258 397static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
619c5cb6
VZ		 398 u16 queue, struct sk_buff *skb,
399 struct eth_end_agg_rx_cqe *cqe,
400 u16 cqe_idx)
9f6c9258
DK		 401{
402 struct sw_rx_page *rx_pg, old_rx_pg;
9f6c9258
DK		 403 u32 i, frag_len, frag_size, pages;
404 int err;
405 int j;
619c5cb6
VZ		 406 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
407 u16 len_on_bd = tpa_info->len_on_bd;
9f6c9258 408
619c5cb6 409 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
9f6c9258
DK		 410 pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
411
412 /* This is needed in order to enable forwarding support */
413 if (frag_size)
619c5cb6
VZ		 414 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
415 tpa_info->parsing_flags, len_on_bd);
9f6c9258
DK		 416
417#ifdef BNX2X_STOP_ON_ERROR
418 if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
419 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
420 pages, cqe_idx);
619c5cb6 421 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
9f6c9258
DK		 422 bnx2x_panic();
423 return -EINVAL;
424 }
425#endif
426
427 /* Run through the SGL and compose the fragmented skb */
428 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
619c5cb6 429 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
9f6c9258
DK		 430
431 /* FW gives the indices of the SGE as if the ring is an array
432 (meaning that "next" element will consume 2 indices) */
433 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
434 rx_pg = &fp->rx_page_ring[sge_idx];
435 old_rx_pg = *rx_pg;
436
437 /* If we fail to allocate a substitute page, we simply stop
438 where we are and drop the whole packet */
439 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
440 if (unlikely(err)) {
441 fp->eth_q_stats.rx_skb_alloc_failed++;
442 return err;
443 }
444
445 /* Unmap the page as we r going to pass it to the stack */
446 dma_unmap_page(&bp->pdev->dev,
447 dma_unmap_addr(&old_rx_pg, mapping),
448 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
449
450 /* Add one frag and update the appropriate fields in the skb */
451 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
452
453 skb->data_len += frag_len;
454 skb->truesize += frag_len;
455 skb->len += frag_len;
456
457 frag_size -= frag_len;
458 }
459
460 return 0;
461}
462
463static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
619c5cb6 464 u16 queue, struct eth_end_agg_rx_cqe *cqe,
9f6c9258
DK		 465 u16 cqe_idx)
466{
619c5cb6
VZ		 467 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
468 struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
469 u8 pad = tpa_info->placement_offset;
470 u16 len = tpa_info->len_on_bd;
9f6c9258
DK		 471 struct sk_buff *skb = rx_buf->skb;
472 /* alloc new skb */
619c5cb6
VZ		 473 struct sk_buff *new_skb;
474 u8 old_tpa_state = tpa_info->tpa_state;
475
476 tpa_info->tpa_state = BNX2X_TPA_STOP;
477
478 /* If we there was an error during the handling of the TPA_START -
479 * drop this aggregation.
480 */
481 if (old_tpa_state == BNX2X_TPA_ERROR)
482 goto drop;
483
484 /* Try to allocate the new skb */
485 new_skb = netdev_alloc_skb(bp->dev, fp->rx_buf_size);
9f6c9258
DK		 486
487 /* Unmap skb in the pool anyway, as we are going to change
488 pool entry status to BNX2X_TPA_STOP even if new skb allocation
489 fails. */
490 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
a8c94b91 491 fp->rx_buf_size, DMA_FROM_DEVICE);
9f6c9258
DK		 492
493 if (likely(new_skb)) {
9f6c9258 494 prefetch(skb);
217de5aa 495 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
9f6c9258
DK		 496
497#ifdef BNX2X_STOP_ON_ERROR
a8c94b91 498 if (pad + len > fp->rx_buf_size) {
9f6c9258
DK		 499 BNX2X_ERR("skb_put is about to fail... "
500 "pad %d len %d rx_buf_size %d\n",
a8c94b91 501 pad, len, fp->rx_buf_size);
9f6c9258
DK		 502 bnx2x_panic();
503 return;
504 }
505#endif
506
507 skb_reserve(skb, pad);
508 skb_put(skb, len);
509
510 skb->protocol = eth_type_trans(skb, bp->dev);
511 skb->ip_summed = CHECKSUM_UNNECESSARY;
512
619c5cb6
VZ		 513 if (!bnx2x_fill_frag_skb(bp, fp, queue, skb, cqe, cqe_idx)) {
514 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
515 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
9bcc0893 516 napi_gro_receive(&fp->napi, skb);
9f6c9258
DK		 517 } else {
518 DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
519 " - dropping packet!\n");
40955532 520 dev_kfree_skb_any(skb);
9f6c9258
DK		 521 }
522
523
524 /* put new skb in bin */
619c5cb6 525 rx_buf->skb = new_skb;
9f6c9258 526
619c5cb6 527 return;
9f6c9258
DK		 528 }
529
619c5cb6
VZ		 530drop:
531 /* drop the packet and keep the buffer in the bin */
532 DP(NETIF_MSG_RX_STATUS,
533 "Failed to allocate or map a new skb - dropping packet!\n");
534 fp->eth_q_stats.rx_skb_alloc_failed++;
9f6c9258
DK		 535}
536
537/* Set Toeplitz hash value in the skb using the value from the
538 * CQE (calculated by HW).
539 */
540static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
541 struct sk_buff *skb)
542{
543 /* Set Toeplitz hash from CQE */
544 if ((bp->dev->features & NETIF_F_RXHASH) &&
545 (cqe->fast_path_cqe.status_flags &
546 ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
547 skb->rxhash =
548 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
549}
550
551int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
552{
553 struct bnx2x *bp = fp->bp;
554 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
555 u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
556 int rx_pkt = 0;
557
558#ifdef BNX2X_STOP_ON_ERROR
559 if (unlikely(bp->panic))
560 return 0;
561#endif
562
563 /* CQ "next element" is of the size of the regular element,
564 that's why it's ok here */
565 hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
566 if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
567 hw_comp_cons++;
568
569 bd_cons = fp->rx_bd_cons;
570 bd_prod = fp->rx_bd_prod;
571 bd_prod_fw = bd_prod;
572 sw_comp_cons = fp->rx_comp_cons;
573 sw_comp_prod = fp->rx_comp_prod;
574
575 /* Memory barrier necessary as speculative reads of the rx
576 * buffer can be ahead of the index in the status block
577 */
578 rmb();
579
580 DP(NETIF_MSG_RX_STATUS,
581 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n",
582 fp->index, hw_comp_cons, sw_comp_cons);
583
584 while (sw_comp_cons != hw_comp_cons) {
585 struct sw_rx_bd *rx_buf = NULL;
586 struct sk_buff *skb;
587 union eth_rx_cqe *cqe;
619c5cb6 588 struct eth_fast_path_rx_cqe *cqe_fp;
9f6c9258 589 u8 cqe_fp_flags;
619c5cb6 590 enum eth_rx_cqe_type cqe_fp_type;
9f6c9258
DK		 591 u16 len, pad;
592
619c5cb6
VZ		 593#ifdef BNX2X_STOP_ON_ERROR
594 if (unlikely(bp->panic))
595 return 0;
596#endif
597
9f6c9258
DK		 598 comp_ring_cons = RCQ_BD(sw_comp_cons);
599 bd_prod = RX_BD(bd_prod);
600 bd_cons = RX_BD(bd_cons);
601
602 /* Prefetch the page containing the BD descriptor
603 at producer's index. It will be needed when new skb is
604 allocated */
605 prefetch((void *)(PAGE_ALIGN((unsigned long)
606 (&fp->rx_desc_ring[bd_prod])) -
607 PAGE_SIZE + 1));
608
609 cqe = &fp->rx_comp_ring[comp_ring_cons];
619c5cb6
VZ		 610 cqe_fp = &cqe->fast_path_cqe;
611 cqe_fp_flags = cqe_fp->type_error_flags;
612 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
9f6c9258
DK		 613
614 DP(NETIF_MSG_RX_STATUS, "CQE type %x err %x status %x"
615 " queue %x vlan %x len %u\n", CQE_TYPE(cqe_fp_flags),
619c5cb6
VZ		 616 cqe_fp_flags, cqe_fp->status_flags,
617 le32_to_cpu(cqe_fp->rss_hash_result),
618 le16_to_cpu(cqe_fp->vlan_tag), le16_to_cpu(cqe_fp->pkt_len));
9f6c9258
DK		 619
620 /* is this a slowpath msg? */
619c5cb6 621 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
9f6c9258
DK		 622 bnx2x_sp_event(fp, cqe);
623 goto next_cqe;
624
625 /* this is an rx packet */
626 } else {
627 rx_buf = &fp->rx_buf_ring[bd_cons];
628 skb = rx_buf->skb;
629 prefetch(skb);
9f6c9258 630
619c5cb6
VZ		 631 if (!CQE_TYPE_FAST(cqe_fp_type)) {
632#ifdef BNX2X_STOP_ON_ERROR
633 /* sanity check */
634 if (fp->disable_tpa &&
635 (CQE_TYPE_START(cqe_fp_type) ||
636 CQE_TYPE_STOP(cqe_fp_type)))
637 BNX2X_ERR("START/STOP packet while "
638 "disable_tpa type %x\n",
639 CQE_TYPE(cqe_fp_type));
640#endif
9f6c9258 641
619c5cb6
VZ		 642 if (CQE_TYPE_START(cqe_fp_type)) {
643 u16 queue = cqe_fp->queue_index;
9f6c9258
DK		 644 DP(NETIF_MSG_RX_STATUS,
645 "calling tpa_start on queue %d\n",
646 queue);
647
648 bnx2x_tpa_start(fp, queue, skb,
619c5cb6
VZ		 649 bd_cons, bd_prod,
650 cqe_fp);
9f6c9258 651
619c5cb6 652 /* Set Toeplitz hash for LRO skb */
9f6c9258
DK		 653 bnx2x_set_skb_rxhash(bp, cqe, skb);
654
655 goto next_rx;
619c5cb6
VZ		 656
657 } else {
658 u16 queue =
659 cqe->end_agg_cqe.queue_index;
9f6c9258
DK		 660 DP(NETIF_MSG_RX_STATUS,
661 "calling tpa_stop on queue %d\n",
662 queue);
663
619c5cb6
VZ		 664 bnx2x_tpa_stop(bp, fp, queue,
665 &cqe->end_agg_cqe,
666 comp_ring_cons);
9f6c9258
DK		 667#ifdef BNX2X_STOP_ON_ERROR
668 if (bp->panic)
669 return 0;
670#endif
671
619c5cb6 672 bnx2x_update_sge_prod(fp, cqe_fp);
9f6c9258
DK		 673 goto next_cqe;
674 }
675 }
619c5cb6
VZ		 676 /* non TPA */
677 len = le16_to_cpu(cqe_fp->pkt_len);
678 pad = cqe_fp->placement_offset;
9f6c9258
DK		 679 dma_sync_single_for_device(&bp->pdev->dev,
680 dma_unmap_addr(rx_buf, mapping),
619c5cb6
VZ		 681 pad + RX_COPY_THRESH,
682 DMA_FROM_DEVICE);
217de5aa 683 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
9f6c9258
DK		 684
685 /* is this an error packet? */
686 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
687 DP(NETIF_MSG_RX_ERR,
688 "ERROR flags %x rx packet %u\n",
689 cqe_fp_flags, sw_comp_cons);
690 fp->eth_q_stats.rx_err_discard_pkt++;
691 goto reuse_rx;
692 }
693
694 /* Since we don't have a jumbo ring
695 * copy small packets if mtu > 1500
696 */
697 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
698 (len <= RX_COPY_THRESH)) {
699 struct sk_buff *new_skb;
700
619c5cb6 701 new_skb = netdev_alloc_skb(bp->dev, len + pad);
9f6c9258
DK		 702 if (new_skb == NULL) {
703 DP(NETIF_MSG_RX_ERR,
704 "ERROR packet dropped "
705 "because of alloc failure\n");
706 fp->eth_q_stats.rx_skb_alloc_failed++;
707 goto reuse_rx;
708 }
709
710 /* aligned copy */
711 skb_copy_from_linear_data_offset(skb, pad,
712 new_skb->data + pad, len);
713 skb_reserve(new_skb, pad);
714 skb_put(new_skb, len);
715
749a8503 716 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
9f6c9258
DK		 717
718 skb = new_skb;
719
720 } else
721 if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
722 dma_unmap_single(&bp->pdev->dev,
723 dma_unmap_addr(rx_buf, mapping),
a8c94b91 724 fp->rx_buf_size,
9f6c9258
DK		 725 DMA_FROM_DEVICE);
726 skb_reserve(skb, pad);
727 skb_put(skb, len);
728
729 } else {
730 DP(NETIF_MSG_RX_ERR,
731 "ERROR packet dropped because "
732 "of alloc failure\n");
733 fp->eth_q_stats.rx_skb_alloc_failed++;
734reuse_rx:
749a8503 735 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
9f6c9258
DK		 736 goto next_rx;
737 }
738
739 skb->protocol = eth_type_trans(skb, bp->dev);
740
741 /* Set Toeplitz hash for a none-LRO skb */
742 bnx2x_set_skb_rxhash(bp, cqe, skb);
743
bc8acf2c 744 skb_checksum_none_assert(skb);
f85582f8 745
66371c44 746 if (bp->dev->features & NETIF_F_RXCSUM) {
619c5cb6 747
9f6c9258
DK		 748 if (likely(BNX2X_RX_CSUM_OK(cqe)))
749 skb->ip_summed = CHECKSUM_UNNECESSARY;
750 else
751 fp->eth_q_stats.hw_csum_err++;
752 }
753 }
754
755 skb_record_rx_queue(skb, fp->index);
756
619c5cb6
VZ		 757 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
758 PARSING_FLAGS_VLAN)
9bcc0893 759 __vlan_hwaccel_put_tag(skb,
619c5cb6 760 le16_to_cpu(cqe_fp->vlan_tag));
9bcc0893 761 napi_gro_receive(&fp->napi, skb);
9f6c9258
DK		 762
763
764next_rx:
765 rx_buf->skb = NULL;
766
767 bd_cons = NEXT_RX_IDX(bd_cons);
768 bd_prod = NEXT_RX_IDX(bd_prod);
769 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
770 rx_pkt++;
771next_cqe:
772 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
773 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
774
775 if (rx_pkt == budget)
776 break;
777 } /* while */
778
779 fp->rx_bd_cons = bd_cons;
780 fp->rx_bd_prod = bd_prod_fw;
781 fp->rx_comp_cons = sw_comp_cons;
782 fp->rx_comp_prod = sw_comp_prod;
783
784 /* Update producers */
785 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
786 fp->rx_sge_prod);
787
788 fp->rx_pkt += rx_pkt;
789 fp->rx_calls++;
790
791 return rx_pkt;
792}
793
794static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
795{
796 struct bnx2x_fastpath *fp = fp_cookie;
797 struct bnx2x *bp = fp->bp;
6383c0b3 798 u8 cos;
9f6c9258 799
523224a3
DK		 800 DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB "
801 "[fp %d fw_sd %d igusb %d]\n",
802 fp->index, fp->fw_sb_id, fp->igu_sb_id);
803 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
9f6c9258
DK		 804
805#ifdef BNX2X_STOP_ON_ERROR
806 if (unlikely(bp->panic))
807 return IRQ_HANDLED;
808#endif
809
810 /* Handle Rx and Tx according to MSI-X vector */
811 prefetch(fp->rx_cons_sb);
6383c0b3
AE		 812
813 for_each_cos_in_tx_queue(fp, cos)
814 prefetch(fp->txdata[cos].tx_cons_sb);
815
523224a3 816 prefetch(&fp->sb_running_index[SM_RX_ID]);
9f6c9258
DK		 817 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
818
819 return IRQ_HANDLED;
820}
821
9f6c9258
DK		 822/* HW Lock for shared dual port PHYs */
823void bnx2x_acquire_phy_lock(struct bnx2x *bp)
824{
825 mutex_lock(&bp->port.phy_mutex);
826
827 if (bp->port.need_hw_lock)
828 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
829}
830
831void bnx2x_release_phy_lock(struct bnx2x *bp)
832{
833 if (bp->port.need_hw_lock)
834 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
835
836 mutex_unlock(&bp->port.phy_mutex);
837}
838
0793f83f
DK		 839/* calculates MF speed according to current linespeed and MF configuration */
840u16 bnx2x_get_mf_speed(struct bnx2x *bp)
841{
842 u16 line_speed = bp->link_vars.line_speed;
843 if (IS_MF(bp)) {
faa6fcbb
DK		 844 u16 maxCfg = bnx2x_extract_max_cfg(bp,
845 bp->mf_config[BP_VN(bp)]);
846
847 /* Calculate the current MAX line speed limit for the MF
848 * devices
0793f83f 849 */
faa6fcbb
DK		 850 if (IS_MF_SI(bp))
851 line_speed = (line_speed * maxCfg) / 100;
852 else { /* SD mode */
0793f83f
DK		 853 u16 vn_max_rate = maxCfg * 100;
854
855 if (vn_max_rate < line_speed)
856 line_speed = vn_max_rate;
faa6fcbb 857 }
0793f83f
DK		 858 }
859
860 return line_speed;
861}
862
2ae17f66
VZ		 863/**
864 * bnx2x_fill_report_data - fill link report data to report
865 *
866 * @bp: driver handle
867 * @data: link state to update
868 *
869 * It uses a none-atomic bit operations because is called under the mutex.
870 */
871static inline void bnx2x_fill_report_data(struct bnx2x *bp,
872 struct bnx2x_link_report_data *data)
873{
874 u16 line_speed = bnx2x_get_mf_speed(bp);
875
876 memset(data, 0, sizeof(*data));
877
878 /* Fill the report data: efective line speed */
879 data->line_speed = line_speed;
880
881 /* Link is down */
882 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
883 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
884 &data->link_report_flags);
885
886 /* Full DUPLEX */
887 if (bp->link_vars.duplex == DUPLEX_FULL)
888 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
889
890 /* Rx Flow Control is ON */
891 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
892 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
893
894 /* Tx Flow Control is ON */
895 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
896 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
897}
898
899/**
900 * bnx2x_link_report - report link status to OS.
901 *
902 * @bp: driver handle
903 *
904 * Calls the __bnx2x_link_report() under the same locking scheme
905 * as a link/PHY state managing code to ensure a consistent link
906 * reporting.
907 */
908
9f6c9258
DK		 909void bnx2x_link_report(struct bnx2x *bp)
910{
2ae17f66
VZ		 911 bnx2x_acquire_phy_lock(bp);
912 __bnx2x_link_report(bp);
913 bnx2x_release_phy_lock(bp);
914}
9f6c9258 915
2ae17f66
VZ		 916/**
917 * __bnx2x_link_report - report link status to OS.
918 *
919 * @bp: driver handle
920 *
921 * None atomic inmlementation.
922 * Should be called under the phy_lock.
923 */
924void __bnx2x_link_report(struct bnx2x *bp)
925{
926 struct bnx2x_link_report_data cur_data;
9f6c9258 927
2ae17f66
VZ		 928 /* reread mf_cfg */
929 if (!CHIP_IS_E1(bp))
930 bnx2x_read_mf_cfg(bp);
931
932 /* Read the current link report info */
933 bnx2x_fill_report_data(bp, &cur_data);
934
935 /* Don't report link down or exactly the same link status twice */
936 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
937 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
938 &bp->last_reported_link.link_report_flags) &&
939 test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
940 &cur_data.link_report_flags)))
941 return;
942
943 bp->link_cnt++;
9f6c9258 944
2ae17f66
VZ		 945 /* We are going to report a new link parameters now -
946 * remember the current data for the next time.
947 */
948 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
9f6c9258 949
2ae17f66
VZ		 950 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
951 &cur_data.link_report_flags)) {
952 netif_carrier_off(bp->dev);
953 netdev_err(bp->dev, "NIC Link is Down\n");
954 return;
955 } else {
956 netif_carrier_on(bp->dev);
957 netdev_info(bp->dev, "NIC Link is Up, ");
958 pr_cont("%d Mbps ", cur_data.line_speed);
9f6c9258 959
2ae17f66
VZ		 960 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
961 &cur_data.link_report_flags))
9f6c9258
DK		 962 pr_cont("full duplex");
963 else
964 pr_cont("half duplex");
965
2ae17f66
VZ		 966 /* Handle the FC at the end so that only these flags would be
967 * possibly set. This way we may easily check if there is no FC
968 * enabled.
969 */
970 if (cur_data.link_report_flags) {
971 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
972 &cur_data.link_report_flags)) {
9f6c9258 973 pr_cont(", receive ");
2ae17f66
VZ		 974 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
975 &cur_data.link_report_flags))
9f6c9258
DK		 976 pr_cont("& transmit ");
977 } else {
978 pr_cont(", transmit ");
979 }
980 pr_cont("flow control ON");
981 }
982 pr_cont("\n");
9f6c9258
DK		 983 }
984}
985
986void bnx2x_init_rx_rings(struct bnx2x *bp)
987{
988 int func = BP_FUNC(bp);
989 int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
619c5cb6 990 ETH_MAX_AGGREGATION_QUEUES_E1H_E2;
523224a3 991 u16 ring_prod;
9f6c9258 992 int i, j;
25141580 993
b3b83c3f 994 /* Allocate TPA resources */
ec6ba945 995 for_each_rx_queue(bp, j) {
523224a3 996 struct bnx2x_fastpath *fp = &bp->fp[j];
9f6c9258 997
a8c94b91
VZ		 998 DP(NETIF_MSG_IFUP,
999 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1000
523224a3 1001 if (!fp->disable_tpa) {
619c5cb6 1002 /* Fill the per-aggregtion pool */
9f6c9258 1003 for (i = 0; i < max_agg_queues; i++) {
619c5cb6
VZ		 1004 struct bnx2x_agg_info *tpa_info =
1005 &fp->tpa_info[i];
1006 struct sw_rx_bd *first_buf =
1007 &tpa_info->first_buf;
1008
1009 first_buf->skb = netdev_alloc_skb(bp->dev,
1010 fp->rx_buf_size);
1011 if (!first_buf->skb) {
9f6c9258
DK		 1012 BNX2X_ERR("Failed to allocate TPA "
1013 "skb pool for queue[%d] - "
1014 "disabling TPA on this "
1015 "queue!\n", j);
1016 bnx2x_free_tpa_pool(bp, fp, i);
1017 fp->disable_tpa = 1;
1018 break;
1019 }
619c5cb6
VZ		 1020 dma_unmap_addr_set(first_buf, mapping, 0);
1021 tpa_info->tpa_state = BNX2X_TPA_STOP;
9f6c9258 1022 }
523224a3
DK		 1023
1024 /* "next page" elements initialization */
1025 bnx2x_set_next_page_sgl(fp);
1026
1027 /* set SGEs bit mask */
1028 bnx2x_init_sge_ring_bit_mask(fp);
1029
1030 /* Allocate SGEs and initialize the ring elements */
1031 for (i = 0, ring_prod = 0;
1032 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1033
1034 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1035 BNX2X_ERR("was only able to allocate "
1036 "%d rx sges\n", i);
619c5cb6
VZ		 1037 BNX2X_ERR("disabling TPA for "
1038 "queue[%d]\n", j);
523224a3 1039 /* Cleanup already allocated elements */
619c5cb6
VZ		 1040 bnx2x_free_rx_sge_range(bp, fp,
1041 ring_prod);
1042 bnx2x_free_tpa_pool(bp, fp,
1043 max_agg_queues);
523224a3
DK		 1044 fp->disable_tpa = 1;
1045 ring_prod = 0;
1046 break;
1047 }
1048 ring_prod = NEXT_SGE_IDX(ring_prod);
1049 }
1050
1051 fp->rx_sge_prod = ring_prod;
9f6c9258
DK		 1052 }
1053 }
1054
ec6ba945 1055 for_each_rx_queue(bp, j) {
9f6c9258
DK		 1056 struct bnx2x_fastpath *fp = &bp->fp[j];
1057
1058 fp->rx_bd_cons = 0;
9f6c9258 1059
b3b83c3f
DK		 1060 /* Activate BD ring */
1061 /* Warning!
1062 * this will generate an interrupt (to the TSTORM)
1063 * must only be done after chip is initialized
1064 */
1065 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1066 fp->rx_sge_prod);
9f6c9258 1067
9f6c9258
DK		 1068 if (j != 0)
1069 continue;
1070
619c5cb6 1071 if (CHIP_IS_E1(bp)) {
f2e0899f
DK		 1072 REG_WR(bp, BAR_USTRORM_INTMEM +
1073 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1074 U64_LO(fp->rx_comp_mapping));
1075 REG_WR(bp, BAR_USTRORM_INTMEM +
1076 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1077 U64_HI(fp->rx_comp_mapping));
1078 }
9f6c9258
DK		 1079 }
1080}
f85582f8 1081
9f6c9258
DK		 1082static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1083{
1084 int i;
6383c0b3 1085 u8 cos;
9f6c9258 1086
ec6ba945 1087 for_each_tx_queue(bp, i) {
9f6c9258 1088 struct bnx2x_fastpath *fp = &bp->fp[i];
6383c0b3
AE		 1089 for_each_cos_in_tx_queue(fp, cos) {
1090 struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
9f6c9258 1091
6383c0b3
AE		 1092 u16 bd_cons = txdata->tx_bd_cons;
1093 u16 sw_prod = txdata->tx_pkt_prod;
1094 u16 sw_cons = txdata->tx_pkt_cons;
9f6c9258 1095
6383c0b3
AE		 1096 while (sw_cons != sw_prod) {
1097 bd_cons = bnx2x_free_tx_pkt(bp, txdata,
1098 TX_BD(sw_cons));
1099 sw_cons++;
1100 }
9f6c9258
DK		 1101 }
1102 }
1103}
1104
b3b83c3f
DK		 1105static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1106{
1107 struct bnx2x *bp = fp->bp;
1108 int i;
1109
1110 /* ring wasn't allocated */
1111 if (fp->rx_buf_ring == NULL)
1112 return;
1113
1114 for (i = 0; i < NUM_RX_BD; i++) {
1115 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1116 struct sk_buff *skb = rx_buf->skb;
1117
1118 if (skb == NULL)
1119 continue;
b3b83c3f
DK		 1120 dma_unmap_single(&bp->pdev->dev,
1121 dma_unmap_addr(rx_buf, mapping),
1122 fp->rx_buf_size, DMA_FROM_DEVICE);
1123
1124 rx_buf->skb = NULL;
1125 dev_kfree_skb(skb);
1126 }
1127}
1128
9f6c9258
DK		 1129static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1130{
b3b83c3f 1131 int j;
9f6c9258 1132
ec6ba945 1133 for_each_rx_queue(bp, j) {
9f6c9258
DK		 1134 struct bnx2x_fastpath *fp = &bp->fp[j];
1135
b3b83c3f 1136 bnx2x_free_rx_bds(fp);
9f6c9258 1137
9f6c9258
DK		 1138 if (!fp->disable_tpa)
1139 bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
1140 ETH_MAX_AGGREGATION_QUEUES_E1 :
619c5cb6 1141 ETH_MAX_AGGREGATION_QUEUES_E1H_E2);
9f6c9258
DK		 1142 }
1143}
1144
1145void bnx2x_free_skbs(struct bnx2x *bp)
1146{
1147 bnx2x_free_tx_skbs(bp);
1148 bnx2x_free_rx_skbs(bp);
1149}
1150
e3835b99
DK		 1151void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1152{
1153 /* load old values */
1154 u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1155
1156 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1157 /* leave all but MAX value */
1158 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1159
1160 /* set new MAX value */
1161 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1162 & FUNC_MF_CFG_MAX_BW_MASK;
1163
1164 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1165 }
1166}
1167
ca92429f
DK		 1168/**
1169 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1170 *
1171 * @bp: driver handle
1172 * @nvecs: number of vectors to be released
1173 */
1174static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
9f6c9258 1175{
ca92429f 1176 int i, offset = 0;
9f6c9258 1177
ca92429f
DK		 1178 if (nvecs == offset)
1179 return;
1180 free_irq(bp->msix_table[offset].vector, bp->dev);
9f6c9258 1181 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
ca92429f
DK		 1182 bp->msix_table[offset].vector);
1183 offset++;
9f6c9258 1184#ifdef BCM_CNIC
ca92429f
DK		 1185 if (nvecs == offset)
1186 return;
9f6c9258
DK		 1187 offset++;
1188#endif
ca92429f 1189
ec6ba945 1190 for_each_eth_queue(bp, i) {
ca92429f
DK		 1191 if (nvecs == offset)
1192 return;
1193 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d "
1194 "irq\n", i, bp->msix_table[offset].vector);
9f6c9258 1195
ca92429f 1196 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
9f6c9258
DK		 1197 }
1198}
1199
d6214d7a 1200void bnx2x_free_irq(struct bnx2x *bp)
9f6c9258 1201{
d6214d7a 1202 if (bp->flags & USING_MSIX_FLAG)
ca92429f 1203 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
6383c0b3 1204 CNIC_PRESENT + 1);
d6214d7a
DK		 1205 else if (bp->flags & USING_MSI_FLAG)
1206 free_irq(bp->pdev->irq, bp->dev);
1207 else
9f6c9258
DK		 1208 free_irq(bp->pdev->irq, bp->dev);
1209}
1210
d6214d7a 1211int bnx2x_enable_msix(struct bnx2x *bp)
9f6c9258 1212{
d6214d7a 1213 int msix_vec = 0, i, rc, req_cnt;
9f6c9258 1214
d6214d7a
DK		 1215 bp->msix_table[msix_vec].entry = msix_vec;
1216 DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n",
1217 bp->msix_table[0].entry);
1218 msix_vec++;
9f6c9258
DK		 1219
1220#ifdef BCM_CNIC
d6214d7a
DK		 1221 bp->msix_table[msix_vec].entry = msix_vec;
1222 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d (CNIC)\n",
1223 bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1224 msix_vec++;
9f6c9258 1225#endif
6383c0b3 1226 /* We need separate vectors for ETH queues only (not FCoE) */
ec6ba945 1227 for_each_eth_queue(bp, i) {
d6214d7a 1228 bp->msix_table[msix_vec].entry = msix_vec;
9f6c9258 1229 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
d6214d7a
DK		 1230 "(fastpath #%u)\n", msix_vec, msix_vec, i);
1231 msix_vec++;
9f6c9258
DK		 1232 }
1233
6383c0b3 1234 req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_PRESENT + 1;
d6214d7a
DK		 1235
1236 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
9f6c9258
DK		 1237
1238 /*
1239 * reconfigure number of tx/rx queues according to available
1240 * MSI-X vectors
1241 */
1242 if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
d6214d7a
DK		 1243 /* how less vectors we will have? */
1244 int diff = req_cnt - rc;
9f6c9258
DK		 1245
1246 DP(NETIF_MSG_IFUP,
1247 "Trying to use less MSI-X vectors: %d\n", rc);
1248
1249 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1250
1251 if (rc) {
1252 DP(NETIF_MSG_IFUP,
1253 "MSI-X is not attainable rc %d\n", rc);
1254 return rc;
1255 }
d6214d7a
DK		 1256 /*
1257 * decrease number of queues by number of unallocated entries
1258 */
1259 bp->num_queues -= diff;
9f6c9258
DK		 1260
1261 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1262 bp->num_queues);
1263 } else if (rc) {
d6214d7a
DK		 1264 /* fall to INTx if not enough memory */
1265 if (rc == -ENOMEM)
1266 bp->flags |= DISABLE_MSI_FLAG;
9f6c9258
DK		 1267 DP(NETIF_MSG_IFUP, "MSI-X is not attainable rc %d\n", rc);
1268 return rc;
1269 }
1270
1271 bp->flags |= USING_MSIX_FLAG;
1272
1273 return 0;
1274}
1275
1276static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1277{
ca92429f 1278 int i, rc, offset = 0;
9f6c9258 1279
ca92429f
DK		 1280 rc = request_irq(bp->msix_table[offset++].vector,
1281 bnx2x_msix_sp_int, 0,
9f6c9258
DK		 1282 bp->dev->name, bp->dev);
1283 if (rc) {
1284 BNX2X_ERR("request sp irq failed\n");
1285 return -EBUSY;
1286 }
1287
1288#ifdef BCM_CNIC
1289 offset++;
1290#endif
ec6ba945 1291 for_each_eth_queue(bp, i) {
9f6c9258
DK		 1292 struct bnx2x_fastpath *fp = &bp->fp[i];
1293 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1294 bp->dev->name, i);
1295
d6214d7a 1296 rc = request_irq(bp->msix_table[offset].vector,
9f6c9258
DK		 1297 bnx2x_msix_fp_int, 0, fp->name, fp);
1298 if (rc) {
ca92429f
DK		 1299 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i,
1300 bp->msix_table[offset].vector, rc);
1301 bnx2x_free_msix_irqs(bp, offset);
9f6c9258
DK		 1302 return -EBUSY;
1303 }
1304
d6214d7a 1305 offset++;
9f6c9258
DK		 1306 }
1307
ec6ba945 1308 i = BNX2X_NUM_ETH_QUEUES(bp);
6383c0b3 1309 offset = 1 + CNIC_PRESENT;
9f6c9258
DK		 1310 netdev_info(bp->dev, "using MSI-X IRQs: sp %d fp[%d] %d"
1311 " ... fp[%d] %d\n",
1312 bp->msix_table[0].vector,
1313 0, bp->msix_table[offset].vector,
1314 i - 1, bp->msix_table[offset + i - 1].vector);
1315
1316 return 0;
1317}
1318
d6214d7a 1319int bnx2x_enable_msi(struct bnx2x *bp)
9f6c9258
DK		 1320{
1321 int rc;
1322
1323 rc = pci_enable_msi(bp->pdev);
1324 if (rc) {
1325 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1326 return -1;
1327 }
1328 bp->flags |= USING_MSI_FLAG;
1329
1330 return 0;
1331}
1332
1333static int bnx2x_req_irq(struct bnx2x *bp)
1334{
1335 unsigned long flags;
1336 int rc;
1337
1338 if (bp->flags & USING_MSI_FLAG)
1339 flags = 0;
1340 else
1341 flags = IRQF_SHARED;
1342
1343 rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1344 bp->dev->name, bp->dev);
9f6c9258
DK		 1345 return rc;
1346}
1347
619c5cb6
VZ		 1348static inline int bnx2x_setup_irqs(struct bnx2x *bp)
1349{
1350 int rc = 0;
1351 if (bp->flags & USING_MSIX_FLAG) {
1352 rc = bnx2x_req_msix_irqs(bp);
1353 if (rc)
1354 return rc;
1355 } else {
1356 bnx2x_ack_int(bp);
1357 rc = bnx2x_req_irq(bp);
1358 if (rc) {
1359 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1360 return rc;
1361 }
1362 if (bp->flags & USING_MSI_FLAG) {
1363 bp->dev->irq = bp->pdev->irq;
1364 netdev_info(bp->dev, "using MSI IRQ %d\n",
1365 bp->pdev->irq);
1366 }
1367 }
1368
1369 return 0;
1370}
1371
1372static inline void bnx2x_napi_enable(struct bnx2x *bp)
9f6c9258
DK		 1373{
1374 int i;
1375
619c5cb6 1376 for_each_rx_queue(bp, i)
9f6c9258
DK		 1377 napi_enable(&bnx2x_fp(bp, i, napi));
1378}
1379
619c5cb6 1380static inline void bnx2x_napi_disable(struct bnx2x *bp)
9f6c9258
DK		 1381{
1382 int i;
1383
619c5cb6 1384 for_each_rx_queue(bp, i)
9f6c9258
DK		 1385 napi_disable(&bnx2x_fp(bp, i, napi));
1386}
1387
1388void bnx2x_netif_start(struct bnx2x *bp)
1389{
4b7ed897
DK		 1390 if (netif_running(bp->dev)) {
1391 bnx2x_napi_enable(bp);
1392 bnx2x_int_enable(bp);
1393 if (bp->state == BNX2X_STATE_OPEN)
1394 netif_tx_wake_all_queues(bp->dev);
9f6c9258
DK		 1395 }
1396}
1397
1398void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1399{
1400 bnx2x_int_disable_sync(bp, disable_hw);
1401 bnx2x_napi_disable(bp);
9f6c9258 1402}
9f6c9258 1403
8307fa3e
VZ		 1404u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1405{
8307fa3e 1406 struct bnx2x *bp = netdev_priv(dev);
faa28314 1407#ifdef BCM_CNIC
8307fa3e
VZ		 1408 if (NO_FCOE(bp))
1409 return skb_tx_hash(dev, skb);
1410 else {
1411 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1412 u16 ether_type = ntohs(hdr->h_proto);
1413
1414 /* Skip VLAN tag if present */
1415 if (ether_type == ETH_P_8021Q) {
1416 struct vlan_ethhdr *vhdr =
1417 (struct vlan_ethhdr *)skb->data;
1418
1419 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1420 }
1421
1422 /* If ethertype is FCoE or FIP - use FCoE ring */
1423 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
6383c0b3 1424 return bnx2x_fcoe_tx(bp, txq_index);
8307fa3e
VZ		 1425 }
1426#endif
1427 /* Select a none-FCoE queue: if FCoE is enabled, exclude FCoE L2 ring
1428 */
6383c0b3 1429 return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
8307fa3e
VZ		 1430}
1431
d6214d7a
DK		 1432void bnx2x_set_num_queues(struct bnx2x *bp)
1433{
1434 switch (bp->multi_mode) {
1435 case ETH_RSS_MODE_DISABLED:
9f6c9258 1436 bp->num_queues = 1;
d6214d7a
DK		 1437 break;
1438 case ETH_RSS_MODE_REGULAR:
1439 bp->num_queues = bnx2x_calc_num_queues(bp);
9f6c9258 1440 break;
f85582f8 1441
9f6c9258 1442 default:
d6214d7a 1443 bp->num_queues = 1;
9f6c9258
DK		 1444 break;
1445 }
ec6ba945
VZ		 1446
1447 /* Add special queues */
6383c0b3 1448 bp->num_queues += NON_ETH_CONTEXT_USE;
ec6ba945
VZ		 1449}
1450
ec6ba945
VZ		 1451static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
1452{
6383c0b3 1453 int rc, tx, rx;
ec6ba945 1454
6383c0b3
AE		 1455 tx = MAX_TXQS_PER_COS * bp->max_cos;
1456 rx = BNX2X_NUM_ETH_QUEUES(bp);
ec6ba945 1457
6383c0b3
AE		 1458/* account for fcoe queue */
1459#ifdef BCM_CNIC
1460 if (!NO_FCOE(bp)) {
1461 rx += FCOE_PRESENT;
1462 tx += FCOE_PRESENT;
1463 }
ec6ba945 1464#endif
6383c0b3
AE		 1465
1466 rc = netif_set_real_num_tx_queues(bp->dev, tx);
1467 if (rc) {
1468 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1469 return rc;
1470 }
1471 rc = netif_set_real_num_rx_queues(bp->dev, rx);
1472 if (rc) {
1473 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1474 return rc;
1475 }
1476
1477 DP(NETIF_MSG_DRV, "Setting real num queues to (tx, rx) (%d, %d)\n",
1478 tx, rx);
1479
ec6ba945
VZ		 1480 return rc;
1481}
1482
a8c94b91
VZ		 1483static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1484{
1485 int i;
1486
1487 for_each_queue(bp, i) {
1488 struct bnx2x_fastpath *fp = &bp->fp[i];
1489
1490 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1491 if (IS_FCOE_IDX(i))
1492 /*
1493 * Although there are no IP frames expected to arrive to
1494 * this ring we still want to add an
1495 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1496 * overrun attack.
1497 */
1498 fp->rx_buf_size =
1499 BNX2X_FCOE_MINI_JUMBO_MTU + ETH_OVREHEAD +
619c5cb6 1500 BNX2X_FW_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
a8c94b91
VZ		 1501 else
1502 fp->rx_buf_size =
619c5cb6
VZ		 1503 bp->dev->mtu + ETH_OVREHEAD +
1504 BNX2X_FW_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
a8c94b91
VZ		 1505 }
1506}
1507
619c5cb6
VZ		 1508static inline int bnx2x_init_rss_pf(struct bnx2x *bp)
1509{
1510 int i;
1511 u8 ind_table[T_ETH_INDIRECTION_TABLE_SIZE] = {0};
1512 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1513
1514 /*
1515 * Prepare the inital contents fo the indirection table if RSS is
1516 * enabled
1517 */
1518 if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
1519 for (i = 0; i < sizeof(ind_table); i++)
1520 ind_table[i] =
1521 bp->fp->cl_id + (i % num_eth_queues);
1522 }
1523
1524 /*
1525 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1526 * per-port, so if explicit configuration is needed , do it only
1527 * for a PMF.
1528 *
1529 * For 57712 and newer on the other hand it's a per-function
1530 * configuration.
1531 */
1532 return bnx2x_config_rss_pf(bp, ind_table,
1533 bp->port.pmf || !CHIP_IS_E1x(bp));
1534}
1535
1536int bnx2x_config_rss_pf(struct bnx2x *bp, u8 *ind_table, bool config_hash)
1537{
1538 struct bnx2x_config_rss_params params = {0};
1539 int i;
1540
1541 /* Although RSS is meaningless when there is a single HW queue we
1542 * still need it enabled in order to have HW Rx hash generated.
1543 *
1544 * if (!is_eth_multi(bp))
1545 * bp->multi_mode = ETH_RSS_MODE_DISABLED;
1546 */
1547
1548 params.rss_obj = &bp->rss_conf_obj;
1549
1550 __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1551
1552 /* RSS mode */
1553 switch (bp->multi_mode) {
1554 case ETH_RSS_MODE_DISABLED:
1555 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
1556 break;
1557 case ETH_RSS_MODE_REGULAR:
1558 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1559 break;
1560 case ETH_RSS_MODE_VLAN_PRI:
1561 __set_bit(BNX2X_RSS_MODE_VLAN_PRI, &params.rss_flags);
1562 break;
1563 case ETH_RSS_MODE_E1HOV_PRI:
1564 __set_bit(BNX2X_RSS_MODE_E1HOV_PRI, &params.rss_flags);
1565 break;
1566 case ETH_RSS_MODE_IP_DSCP:
1567 __set_bit(BNX2X_RSS_MODE_IP_DSCP, &params.rss_flags);
1568 break;
1569 default:
1570 BNX2X_ERR("Unknown multi_mode: %d\n", bp->multi_mode);
1571 return -EINVAL;
1572 }
1573
1574 /* If RSS is enabled */
1575 if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
1576 /* RSS configuration */
1577 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1578 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1579 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1580 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1581
1582 /* Hash bits */
1583 params.rss_result_mask = MULTI_MASK;
1584
1585 memcpy(params.ind_table, ind_table, sizeof(params.ind_table));
1586
1587 if (config_hash) {
1588 /* RSS keys */
1589 for (i = 0; i < sizeof(params.rss_key) / 4; i++)
1590 params.rss_key[i] = random32();
1591
1592 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1593 }
1594 }
1595
1596 return bnx2x_config_rss(bp, &params);
1597}
1598
1599static inline int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1600{
1601 struct bnx2x_func_state_params func_params = {0};
1602
1603 /* Prepare parameters for function state transitions */
1604 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1605
1606 func_params.f_obj = &bp->func_obj;
1607 func_params.cmd = BNX2X_F_CMD_HW_INIT;
1608
1609 func_params.params.hw_init.load_phase = load_code;
1610
1611 return bnx2x_func_state_change(bp, &func_params);
1612}
1613
1614/*
1615 * Cleans the object that have internal lists without sending
1616 * ramrods. Should be run when interrutps are disabled.
1617 */
1618static void bnx2x_squeeze_objects(struct bnx2x *bp)
1619{
1620 int rc;
1621 unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1622 struct bnx2x_mcast_ramrod_params rparam = {0};
1623 struct bnx2x_vlan_mac_obj *mac_obj = &bp->fp->mac_obj;
1624
1625 /***************** Cleanup MACs' object first *************************/
1626
1627 /* Wait for completion of requested */
1628 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1629 /* Perform a dry cleanup */
1630 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1631
1632 /* Clean ETH primary MAC */
1633 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1634 rc = mac_obj->delete_all(bp, &bp->fp->mac_obj, &vlan_mac_flags,
1635 &ramrod_flags);
1636 if (rc != 0)
1637 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1638
1639 /* Cleanup UC list */
1640 vlan_mac_flags = 0;
1641 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1642 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1643 &ramrod_flags);
1644 if (rc != 0)
1645 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1646
1647 /***************** Now clean mcast object *****************************/
1648 rparam.mcast_obj = &bp->mcast_obj;
1649 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1650
1651 /* Add a DEL command... */
1652 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1653 if (rc < 0)
1654 BNX2X_ERR("Failed to add a new DEL command to a multi-cast "
1655 "object: %d\n", rc);
1656
1657 /* ...and wait until all pending commands are cleared */
1658 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1659 while (rc != 0) {
1660 if (rc < 0) {
1661 BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1662 rc);
1663 return;
1664 }
1665
1666 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1667 }
1668}
1669
1670#ifndef BNX2X_STOP_ON_ERROR
1671#define LOAD_ERROR_EXIT(bp, label) \
1672 do { \
1673 (bp)->state = BNX2X_STATE_ERROR; \
1674 goto label; \
1675 } while (0)
1676#else
1677#define LOAD_ERROR_EXIT(bp, label) \
1678 do { \
1679 (bp)->state = BNX2X_STATE_ERROR; \
1680 (bp)->panic = 1; \
1681 return -EBUSY; \
1682 } while (0)
1683#endif
1684
9f6c9258
DK		 1685/* must be called with rtnl_lock */
1686int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1687{
619c5cb6 1688 int port = BP_PORT(bp);
9f6c9258
DK		 1689 u32 load_code;
1690 int i, rc;
1691
1692#ifdef BNX2X_STOP_ON_ERROR
1693 if (unlikely(bp->panic))
1694 return -EPERM;
1695#endif
1696
1697 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1698
2ae17f66
VZ		 1699 /* Set the initial link reported state to link down */
1700 bnx2x_acquire_phy_lock(bp);
1701 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
1702 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1703 &bp->last_reported_link.link_report_flags);
1704 bnx2x_release_phy_lock(bp);
1705
523224a3
DK		 1706 /* must be called before memory allocation and HW init */
1707 bnx2x_ilt_set_info(bp);
1708
6383c0b3
AE		 1709 /*
1710 * Zero fastpath structures preserving invariants like napi, which are
1711 * allocated only once, fp index, max_cos, bp pointer.
1712 * Also set fp->disable_tpa.
b3b83c3f
DK		 1713 */
1714 for_each_queue(bp, i)
1715 bnx2x_bz_fp(bp, i);
1716
6383c0b3 1717
a8c94b91
VZ		 1718 /* Set the receive queues buffer size */
1719 bnx2x_set_rx_buf_size(bp);
1720
d6214d7a 1721 if (bnx2x_alloc_mem(bp))
9f6c9258 1722 return -ENOMEM;
d6214d7a 1723
b3b83c3f
DK		 1724 /* As long as bnx2x_alloc_mem() may possibly update
1725 * bp->num_queues, bnx2x_set_real_num_queues() should always
1726 * come after it.
1727 */
ec6ba945 1728 rc = bnx2x_set_real_num_queues(bp);
d6214d7a 1729 if (rc) {
ec6ba945 1730 BNX2X_ERR("Unable to set real_num_queues\n");
619c5cb6 1731 LOAD_ERROR_EXIT(bp, load_error0);
9f6c9258
DK		 1732 }
1733
6383c0b3
AE		 1734 /* configure multi cos mappings in kernel.
1735 * this configuration may be overriden by a multi class queue discipline
1736 * or by a dcbx negotiation result.
1737 */
1738 bnx2x_setup_tc(bp->dev, bp->max_cos);
1739
9f6c9258
DK		 1740 bnx2x_napi_enable(bp);
1741
9f6c9258 1742 /* Send LOAD_REQUEST command to MCP
619c5cb6
VZ		 1743 * Returns the type of LOAD command:
1744 * if it is the first port to be initialized
1745 * common blocks should be initialized, otherwise - not
1746 */
9f6c9258 1747 if (!BP_NOMCP(bp)) {
a22f0788 1748 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
9f6c9258
DK		 1749 if (!load_code) {
1750 BNX2X_ERR("MCP response failure, aborting\n");
1751 rc = -EBUSY;
619c5cb6 1752 LOAD_ERROR_EXIT(bp, load_error1);
9f6c9258
DK		 1753 }
1754 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1755 rc = -EBUSY; /* other port in diagnostic mode */
619c5cb6 1756 LOAD_ERROR_EXIT(bp, load_error1);
9f6c9258
DK		 1757 }
1758
1759 } else {
f2e0899f 1760 int path = BP_PATH(bp);
9f6c9258 1761
f2e0899f
DK		 1762 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n",
1763 path, load_count[path][0], load_count[path][1],
1764 load_count[path][2]);
1765 load_count[path][0]++;
1766 load_count[path][1 + port]++;
1767 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n",
1768 path, load_count[path][0], load_count[path][1],
1769 load_count[path][2]);
1770 if (load_count[path][0] == 1)
9f6c9258 1771 load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
f2e0899f 1772 else if (load_count[path][1 + port] == 1)
9f6c9258
DK		 1773 load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1774 else
1775 load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1776 }
1777
1778 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
f2e0899f 1779 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
3deb8167 1780 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
9f6c9258 1781 bp->port.pmf = 1;
3deb8167
YR		 1782 /*
1783 * We need the barrier to ensure the ordering between the
1784 * writing to bp->port.pmf here and reading it from the
1785 * bnx2x_periodic_task().
1786 */
1787 smp_mb();
1788 queue_delayed_work(bnx2x_wq, &bp->period_task, 0);
1789 } else
9f6c9258 1790 bp->port.pmf = 0;
6383c0b3 1791
9f6c9258
DK		 1792 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1793
619c5cb6
VZ		 1794 /* Init Function state controlling object */
1795 bnx2x__init_func_obj(bp);
1796
9f6c9258
DK		 1797 /* Initialize HW */
1798 rc = bnx2x_init_hw(bp, load_code);
1799 if (rc) {
1800 BNX2X_ERR("HW init failed, aborting\n");
a22f0788 1801 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
619c5cb6 1802 LOAD_ERROR_EXIT(bp, load_error2);
9f6c9258
DK		 1803 }
1804
d6214d7a
DK		 1805 /* Connect to IRQs */
1806 rc = bnx2x_setup_irqs(bp);
523224a3
DK		 1807 if (rc) {
1808 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
619c5cb6 1809 LOAD_ERROR_EXIT(bp, load_error2);
523224a3
DK		 1810 }
1811
9f6c9258
DK		 1812 /* Setup NIC internals and enable interrupts */
1813 bnx2x_nic_init(bp, load_code);
1814
619c5cb6
VZ		 1815 /* Init per-function objects */
1816 bnx2x_init_bp_objs(bp);
1817
f2e0899f
DK		 1818 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1819 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
619c5cb6
VZ		 1820 (bp->common.shmem2_base)) {
1821 if (SHMEM2_HAS(bp, dcc_support))
1822 SHMEM2_WR(bp, dcc_support,
1823 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1824 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1825 }
1826
1827 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1828 rc = bnx2x_func_start(bp);
1829 if (rc) {
1830 BNX2X_ERR("Function start failed!\n");
1831 LOAD_ERROR_EXIT(bp, load_error3);
1832 }
9f6c9258
DK		 1833
1834 /* Send LOAD_DONE command to MCP */
1835 if (!BP_NOMCP(bp)) {
a22f0788 1836 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
9f6c9258
DK		 1837 if (!load_code) {
1838 BNX2X_ERR("MCP response failure, aborting\n");
1839 rc = -EBUSY;
619c5cb6 1840 LOAD_ERROR_EXIT(bp, load_error3);
9f6c9258
DK		 1841 }
1842 }
1843
619c5cb6 1844 rc = bnx2x_setup_leading(bp);
9f6c9258
DK		 1845 if (rc) {
1846 BNX2X_ERR("Setup leading failed!\n");
619c5cb6 1847 LOAD_ERROR_EXIT(bp, load_error3);
f2e0899f 1848 }
9f6c9258 1849
9f6c9258 1850#ifdef BCM_CNIC
523224a3 1851 /* Enable Timer scan */
619c5cb6 1852 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
9f6c9258 1853#endif
f85582f8 1854
523224a3 1855 for_each_nondefault_queue(bp, i) {
619c5cb6 1856 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
523224a3 1857 if (rc)
619c5cb6 1858 LOAD_ERROR_EXIT(bp, load_error4);
523224a3
DK		 1859 }
1860
619c5cb6
VZ		 1861 rc = bnx2x_init_rss_pf(bp);
1862 if (rc)
1863 LOAD_ERROR_EXIT(bp, load_error4);
1864
523224a3
DK		 1865 /* Now when Clients are configured we are ready to work */
1866 bp->state = BNX2X_STATE_OPEN;
1867
619c5cb6
VZ		 1868 /* Configure a ucast MAC */
1869 rc = bnx2x_set_eth_mac(bp, true);
1870 if (rc)
1871 LOAD_ERROR_EXIT(bp, load_error4);
6e30dd4e 1872
e3835b99
DK		 1873 if (bp->pending_max) {
1874 bnx2x_update_max_mf_config(bp, bp->pending_max);
1875 bp->pending_max = 0;
1876 }
1877
9f6c9258
DK		 1878 if (bp->port.pmf)
1879 bnx2x_initial_phy_init(bp, load_mode);
1880
619c5cb6
VZ		 1881 /* Start fast path */
1882
1883 /* Initialize Rx filter. */
1884 netif_addr_lock_bh(bp->dev);
6e30dd4e 1885 bnx2x_set_rx_mode(bp->dev);
619c5cb6 1886 netif_addr_unlock_bh(bp->dev);
6e30dd4e 1887
619c5cb6 1888 /* Start the Tx */
9f6c9258
DK		 1889 switch (load_mode) {
1890 case LOAD_NORMAL:
523224a3
DK		 1891 /* Tx queue should be only reenabled */
1892 netif_tx_wake_all_queues(bp->dev);
9f6c9258
DK		 1893 break;
1894
1895 case LOAD_OPEN:
1896 netif_tx_start_all_queues(bp->dev);
523224a3 1897 smp_mb__after_clear_bit();
9f6c9258
DK		 1898 break;
1899
1900 case LOAD_DIAG:
9f6c9258
DK		 1901 bp->state = BNX2X_STATE_DIAG;
1902 break;
1903
1904 default:
1905 break;
1906 }
1907
1908 if (!bp->port.pmf)
1909 bnx2x__link_status_update(bp);
1910
1911 /* start the timer */
1912 mod_timer(&bp->timer, jiffies + bp->current_interval);
1913
1914#ifdef BCM_CNIC
1915 bnx2x_setup_cnic_irq_info(bp);
1916 if (bp->state == BNX2X_STATE_OPEN)
1917 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1918#endif
1919 bnx2x_inc_load_cnt(bp);
1920
619c5cb6
VZ		 1921 /* Wait for all pending SP commands to complete */
1922 if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
1923 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
1924 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
1925 return -EBUSY;
1926 }
6891dd25 1927
619c5cb6 1928 bnx2x_dcbx_init(bp);
9f6c9258
DK		 1929 return 0;
1930
619c5cb6 1931#ifndef BNX2X_STOP_ON_ERROR
9f6c9258 1932load_error4:
619c5cb6 1933#ifdef BCM_CNIC
9f6c9258 1934 /* Disable Timer scan */
619c5cb6 1935 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
9f6c9258
DK		 1936#endif
1937load_error3:
1938 bnx2x_int_disable_sync(bp, 1);
d6214d7a 1939
619c5cb6
VZ		 1940 /* Clean queueable objects */
1941 bnx2x_squeeze_objects(bp);
1942
9f6c9258
DK		 1943 /* Free SKBs, SGEs, TPA pool and driver internals */
1944 bnx2x_free_skbs(bp);
ec6ba945 1945 for_each_rx_queue(bp, i)
9f6c9258 1946 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
d6214d7a 1947
9f6c9258 1948 /* Release IRQs */
d6214d7a
DK		 1949 bnx2x_free_irq(bp);
1950load_error2:
1951 if (!BP_NOMCP(bp)) {
1952 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1953 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1954 }
1955
1956 bp->port.pmf = 0;
9f6c9258
DK		 1957load_error1:
1958 bnx2x_napi_disable(bp);
d6214d7a 1959load_error0:
9f6c9258
DK		 1960 bnx2x_free_mem(bp);
1961
1962 return rc;
619c5cb6 1963#endif /* ! BNX2X_STOP_ON_ERROR */
9f6c9258
DK		 1964}
1965
1966/* must be called with rtnl_lock */
1967int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1968{
1969 int i;
c9ee9206
VZ		 1970 bool global = false;
1971
1972 if ((bp->state == BNX2X_STATE_CLOSED) ||
1973 (bp->state == BNX2X_STATE_ERROR)) {
1974 /* We can get here if the driver has been unloaded
1975 * during parity error recovery and is either waiting for a
1976 * leader to complete or for other functions to unload and
1977 * then ifdown has been issued. In this case we want to
1978 * unload and let other functions to complete a recovery
1979 * process.
1980 */
9f6c9258
DK		 1981 bp->recovery_state = BNX2X_RECOVERY_DONE;
1982 bp->is_leader = 0;
c9ee9206
VZ		 1983 bnx2x_release_leader_lock(bp);
1984 smp_mb();
1985
1986 DP(NETIF_MSG_HW, "Releasing a leadership...\n");
9f6c9258
DK		 1987
1988 return -EINVAL;
1989 }
1990
1991#ifdef BCM_CNIC
1992 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1993#endif
1994 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
619c5cb6 1995 smp_mb();
9f6c9258 1996
9f6c9258 1997 bp->rx_mode = BNX2X_RX_MODE_NONE;
9f6c9258 1998
f2e0899f
DK		 1999 /* Stop Tx */
2000 bnx2x_tx_disable(bp);
f85582f8 2001
9f6c9258 2002 del_timer_sync(&bp->timer);
f85582f8 2003
619c5cb6
VZ		 2004 /* Set ALWAYS_ALIVE bit in shmem */
2005 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2006
2007 bnx2x_drv_pulse(bp);
9f6c9258 2008
f85582f8 2009 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
9f6c9258
DK		 2010
2011 /* Cleanup the chip if needed */
2012 if (unload_mode != UNLOAD_RECOVERY)
2013 bnx2x_chip_cleanup(bp, unload_mode);
523224a3 2014 else {
c9ee9206
VZ		 2015 /* Send the UNLOAD_REQUEST to the MCP */
2016 bnx2x_send_unload_req(bp, unload_mode);
2017
2018 /*
2019 * Prevent transactions to host from the functions on the
2020 * engine that doesn't reset global blocks in case of global
2021 * attention once gloabl blocks are reset and gates are opened
2022 * (the engine which leader will perform the recovery
2023 * last).
2024 */
2025 if (!CHIP_IS_E1x(bp))
2026 bnx2x_pf_disable(bp);
2027
2028 /* Disable HW interrupts, NAPI */
523224a3
DK		 2029 bnx2x_netif_stop(bp, 1);
2030
2031 /* Release IRQs */
d6214d7a 2032 bnx2x_free_irq(bp);
c9ee9206
VZ		 2033
2034 /* Report UNLOAD_DONE to MCP */
2035 bnx2x_send_unload_done(bp);
523224a3 2036 }
9f6c9258 2037
619c5cb6
VZ		 2038 /*
2039 * At this stage no more interrupts will arrive so we may safly clean
2040 * the queueable objects here in case they failed to get cleaned so far.
2041 */
2042 bnx2x_squeeze_objects(bp);
2043
9f6c9258
DK		 2044 bp->port.pmf = 0;
2045
2046 /* Free SKBs, SGEs, TPA pool and driver internals */
2047 bnx2x_free_skbs(bp);
ec6ba945 2048 for_each_rx_queue(bp, i)
9f6c9258 2049 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
d6214d7a 2050
9f6c9258
DK		 2051 bnx2x_free_mem(bp);
2052
2053 bp->state = BNX2X_STATE_CLOSED;
2054
c9ee9206
VZ		 2055 /* Check if there are pending parity attentions. If there are - set
2056 * RECOVERY_IN_PROGRESS.
2057 */
2058 if (bnx2x_chk_parity_attn(bp, &global, false)) {
2059 bnx2x_set_reset_in_progress(bp);
2060
2061 /* Set RESET_IS_GLOBAL if needed */
2062 if (global)
2063 bnx2x_set_reset_global(bp);
2064 }
2065
2066
9f6c9258
DK		 2067 /* The last driver must disable a "close the gate" if there is no
2068 * parity attention or "process kill" pending.
2069 */
c9ee9206 2070 if (!bnx2x_dec_load_cnt(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
9f6c9258
DK		 2071 bnx2x_disable_close_the_gate(bp);
2072
9f6c9258
DK		 2073 return 0;
2074}
f85582f8 2075
9f6c9258
DK		 2076int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2077{
2078 u16 pmcsr;
2079
adf5f6a1
DK		 2080 /* If there is no power capability, silently succeed */
2081 if (!bp->pm_cap) {
2082 DP(NETIF_MSG_HW, "No power capability. Breaking.\n");
2083 return 0;
2084 }
2085
9f6c9258
DK		 2086 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2087
2088 switch (state) {
2089 case PCI_D0:
2090 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2091 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2092 PCI_PM_CTRL_PME_STATUS));
2093
2094 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2095 /* delay required during transition out of D3hot */
2096 msleep(20);
2097 break;
2098
2099 case PCI_D3hot:
2100 /* If there are other clients above don't
2101 shut down the power */
2102 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2103 return 0;
2104 /* Don't shut down the power for emulation and FPGA */
2105 if (CHIP_REV_IS_SLOW(bp))
2106 return 0;
2107
2108 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2109 pmcsr |= 3;
2110
2111 if (bp->wol)
2112 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2113
2114 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2115 pmcsr);
2116
2117 /* No more memory access after this point until
2118 * device is brought back to D0.
2119 */
2120 break;
2121
2122 default:
2123 return -EINVAL;
2124 }
2125 return 0;
2126}
2127
9f6c9258
DK		 2128/*
2129 * net_device service functions
2130 */
d6214d7a 2131int bnx2x_poll(struct napi_struct *napi, int budget)
9f6c9258
DK		 2132{
2133 int work_done = 0;
6383c0b3 2134 u8 cos;
9f6c9258
DK		 2135 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2136 napi);
2137 struct bnx2x *bp = fp->bp;
2138
2139 while (1) {
2140#ifdef BNX2X_STOP_ON_ERROR
2141 if (unlikely(bp->panic)) {
2142 napi_complete(napi);
2143 return 0;
2144 }
2145#endif
2146
6383c0b3
AE		 2147 for_each_cos_in_tx_queue(fp, cos)
2148 if (bnx2x_tx_queue_has_work(&fp->txdata[cos]))
2149 bnx2x_tx_int(bp, &fp->txdata[cos]);
2150
9f6c9258
DK		 2151
2152 if (bnx2x_has_rx_work(fp)) {
2153 work_done += bnx2x_rx_int(fp, budget - work_done);
2154
2155 /* must not complete if we consumed full budget */
2156 if (work_done >= budget)
2157 break;
2158 }
2159
2160 /* Fall out from the NAPI loop if needed */
2161 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
ec6ba945
VZ		 2162#ifdef BCM_CNIC
2163 /* No need to update SB for FCoE L2 ring as long as
2164 * it's connected to the default SB and the SB
2165 * has been updated when NAPI was scheduled.
2166 */
2167 if (IS_FCOE_FP(fp)) {
2168 napi_complete(napi);
2169 break;
2170 }
2171#endif
2172
9f6c9258 2173 bnx2x_update_fpsb_idx(fp);
f85582f8
DK		 2174 /* bnx2x_has_rx_work() reads the status block,
2175 * thus we need to ensure that status block indices
2176 * have been actually read (bnx2x_update_fpsb_idx)
2177 * prior to this check (bnx2x_has_rx_work) so that
2178 * we won't write the "newer" value of the status block
2179 * to IGU (if there was a DMA right after
2180 * bnx2x_has_rx_work and if there is no rmb, the memory
2181 * reading (bnx2x_update_fpsb_idx) may be postponed
2182 * to right before bnx2x_ack_sb). In this case there
2183 * will never be another interrupt until there is
2184 * another update of the status block, while there
2185 * is still unhandled work.
2186 */
9f6c9258
DK		 2187 rmb();
2188
2189 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2190 napi_complete(napi);
2191 /* Re-enable interrupts */
523224a3
DK		 2192 DP(NETIF_MSG_HW,
2193 "Update index to %d\n", fp->fp_hc_idx);
2194 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2195 le16_to_cpu(fp->fp_hc_idx),
9f6c9258
DK		 2196 IGU_INT_ENABLE, 1);
2197 break;
2198 }
2199 }
2200 }
2201
2202 return work_done;
2203}
2204
9f6c9258
DK		 2205/* we split the first BD into headers and data BDs
2206 * to ease the pain of our fellow microcode engineers
2207 * we use one mapping for both BDs
2208 * So far this has only been observed to happen
2209 * in Other Operating Systems(TM)
2210 */
2211static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
6383c0b3 2212 struct bnx2x_fp_txdata *txdata,
9f6c9258
DK		 2213 struct sw_tx_bd *tx_buf,
2214 struct eth_tx_start_bd **tx_bd, u16 hlen,
2215 u16 bd_prod, int nbd)
2216{
2217 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2218 struct eth_tx_bd *d_tx_bd;
2219 dma_addr_t mapping;
2220 int old_len = le16_to_cpu(h_tx_bd->nbytes);
2221
2222 /* first fix first BD */
2223 h_tx_bd->nbd = cpu_to_le16(nbd);
2224 h_tx_bd->nbytes = cpu_to_le16(hlen);
2225
2226 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
2227 "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
2228 h_tx_bd->addr_lo, h_tx_bd->nbd);
2229
2230 /* now get a new data BD
2231 * (after the pbd) and fill it */
2232 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
6383c0b3 2233 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258
DK		 2234
2235 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2236 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2237
2238 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2239 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2240 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2241
2242 /* this marks the BD as one that has no individual mapping */
2243 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2244
2245 DP(NETIF_MSG_TX_QUEUED,
2246 "TSO split data size is %d (%x:%x)\n",
2247 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2248
2249 /* update tx_bd */
2250 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2251
2252 return bd_prod;
2253}
2254
2255static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2256{
2257 if (fix > 0)
2258 csum = (u16) ~csum_fold(csum_sub(csum,
2259 csum_partial(t_header - fix, fix, 0)));
2260
2261 else if (fix < 0)
2262 csum = (u16) ~csum_fold(csum_add(csum,
2263 csum_partial(t_header, -fix, 0)));
2264
2265 return swab16(csum);
2266}
2267
2268static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2269{
2270 u32 rc;
2271
2272 if (skb->ip_summed != CHECKSUM_PARTIAL)
2273 rc = XMIT_PLAIN;
2274
2275 else {
d0d9d8ef 2276 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
9f6c9258
DK		 2277 rc = XMIT_CSUM_V6;
2278 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2279 rc |= XMIT_CSUM_TCP;
2280
2281 } else {
2282 rc = XMIT_CSUM_V4;
2283 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2284 rc |= XMIT_CSUM_TCP;
2285 }
2286 }
2287
5892b9e9
VZ		 2288 if (skb_is_gso_v6(skb))
2289 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2290 else if (skb_is_gso(skb))
2291 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
9f6c9258
DK		 2292
2293 return rc;
2294}
2295
2296#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2297/* check if packet requires linearization (packet is too fragmented)
2298 no need to check fragmentation if page size > 8K (there will be no
2299 violation to FW restrictions) */
2300static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2301 u32 xmit_type)
2302{
2303 int to_copy = 0;
2304 int hlen = 0;
2305 int first_bd_sz = 0;
2306
2307 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2308 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2309
2310 if (xmit_type & XMIT_GSO) {
2311 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2312 /* Check if LSO packet needs to be copied:
2313 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2314 int wnd_size = MAX_FETCH_BD - 3;
2315 /* Number of windows to check */
2316 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2317 int wnd_idx = 0;
2318 int frag_idx = 0;
2319 u32 wnd_sum = 0;
2320
2321 /* Headers length */
2322 hlen = (int)(skb_transport_header(skb) - skb->data) +
2323 tcp_hdrlen(skb);
2324
2325 /* Amount of data (w/o headers) on linear part of SKB*/
2326 first_bd_sz = skb_headlen(skb) - hlen;
2327
2328 wnd_sum = first_bd_sz;
2329
2330 /* Calculate the first sum - it's special */
2331 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2332 wnd_sum +=
2333 skb_shinfo(skb)->frags[frag_idx].size;
2334
2335 /* If there was data on linear skb data - check it */
2336 if (first_bd_sz > 0) {
2337 if (unlikely(wnd_sum < lso_mss)) {
2338 to_copy = 1;
2339 goto exit_lbl;
2340 }
2341
2342 wnd_sum -= first_bd_sz;
2343 }
2344
2345 /* Others are easier: run through the frag list and
2346 check all windows */
2347 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2348 wnd_sum +=
2349 skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
2350
2351 if (unlikely(wnd_sum < lso_mss)) {
2352 to_copy = 1;
2353 break;
2354 }
2355 wnd_sum -=
2356 skb_shinfo(skb)->frags[wnd_idx].size;
2357 }
2358 } else {
2359 /* in non-LSO too fragmented packet should always
2360 be linearized */
2361 to_copy = 1;
2362 }
2363 }
2364
2365exit_lbl:
2366 if (unlikely(to_copy))
2367 DP(NETIF_MSG_TX_QUEUED,
2368 "Linearization IS REQUIRED for %s packet. "
2369 "num_frags %d hlen %d first_bd_sz %d\n",
2370 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2371 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2372
2373 return to_copy;
2374}
2375#endif
2376
2297a2da
VZ		 2377static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2378 u32 xmit_type)
f2e0899f 2379{
2297a2da
VZ		 2380 *parsing_data |= (skb_shinfo(skb)->gso_size <<
2381 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2382 ETH_TX_PARSE_BD_E2_LSO_MSS;
f2e0899f
DK		 2383 if ((xmit_type & XMIT_GSO_V6) &&
2384 (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2297a2da 2385 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
f2e0899f
DK		 2386}
2387
2388/**
e8920674 2389 * bnx2x_set_pbd_gso - update PBD in GSO case.
f2e0899f 2390 *
e8920674
DK		 2391 * @skb: packet skb
2392 * @pbd: parse BD
2393 * @xmit_type: xmit flags
f2e0899f
DK		 2394 */
2395static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2396 struct eth_tx_parse_bd_e1x *pbd,
2397 u32 xmit_type)
2398{
2399 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2400 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2401 pbd->tcp_flags = pbd_tcp_flags(skb);
2402
2403 if (xmit_type & XMIT_GSO_V4) {
2404 pbd->ip_id = swab16(ip_hdr(skb)->id);
2405 pbd->tcp_pseudo_csum =
2406 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2407 ip_hdr(skb)->daddr,
2408 0, IPPROTO_TCP, 0));
2409
2410 } else
2411 pbd->tcp_pseudo_csum =
2412 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2413 &ipv6_hdr(skb)->daddr,
2414 0, IPPROTO_TCP, 0));
2415
2416 pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2417}
f85582f8 2418
f2e0899f 2419/**
e8920674 2420 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
f2e0899f 2421 *
e8920674
DK		 2422 * @bp: driver handle
2423 * @skb: packet skb
2424 * @parsing_data: data to be updated
2425 * @xmit_type: xmit flags
f2e0899f 2426 *
e8920674 2427 * 57712 related
f2e0899f
DK		 2428 */
2429static inline u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2297a2da 2430 u32 *parsing_data, u32 xmit_type)
f2e0899f 2431{
e39aece7
VZ		 2432 *parsing_data |=
2433 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
2434 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2435 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
f2e0899f 2436
e39aece7
VZ		 2437 if (xmit_type & XMIT_CSUM_TCP) {
2438 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
2439 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2440 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
f2e0899f 2441
e39aece7
VZ		 2442 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2443 } else
2444 /* We support checksum offload for TCP and UDP only.
2445 * No need to pass the UDP header length - it's a constant.
2446 */
2447 return skb_transport_header(skb) +
2448 sizeof(struct udphdr) - skb->data;
f2e0899f
DK		 2449}
2450
93ef5c02
DK		 2451static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2452 struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
2453{
93ef5c02
DK		 2454 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2455
2456 if (xmit_type & XMIT_CSUM_V4)
2457 tx_start_bd->bd_flags.as_bitfield |=
2458 ETH_TX_BD_FLAGS_IP_CSUM;
2459 else
2460 tx_start_bd->bd_flags.as_bitfield |=
2461 ETH_TX_BD_FLAGS_IPV6;
2462
2463 if (!(xmit_type & XMIT_CSUM_TCP))
2464 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
93ef5c02
DK		 2465}
2466
f2e0899f 2467/**
e8920674 2468 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
f2e0899f 2469 *
e8920674
DK		 2470 * @bp: driver handle
2471 * @skb: packet skb
2472 * @pbd: parse BD to be updated
2473 * @xmit_type: xmit flags
f2e0899f
DK		 2474 */
2475static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2476 struct eth_tx_parse_bd_e1x *pbd,
2477 u32 xmit_type)
2478{
e39aece7 2479 u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
f2e0899f
DK		 2480
2481 /* for now NS flag is not used in Linux */
2482 pbd->global_data =
2483 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2484 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2485
2486 pbd->ip_hlen_w = (skb_transport_header(skb) -
e39aece7 2487 skb_network_header(skb)) >> 1;
f2e0899f 2488
e39aece7
VZ		 2489 hlen += pbd->ip_hlen_w;
2490
2491 /* We support checksum offload for TCP and UDP only */
2492 if (xmit_type & XMIT_CSUM_TCP)
2493 hlen += tcp_hdrlen(skb) / 2;
2494 else
2495 hlen += sizeof(struct udphdr) / 2;
f2e0899f
DK		 2496
2497 pbd->total_hlen_w = cpu_to_le16(hlen);
2498 hlen = hlen*2;
2499
2500 if (xmit_type & XMIT_CSUM_TCP) {
2501 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2502
2503 } else {
2504 s8 fix = SKB_CS_OFF(skb); /* signed! */
2505
2506 DP(NETIF_MSG_TX_QUEUED,
2507 "hlen %d fix %d csum before fix %x\n",
2508 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2509
2510 /* HW bug: fixup the CSUM */
2511 pbd->tcp_pseudo_csum =
2512 bnx2x_csum_fix(skb_transport_header(skb),
2513 SKB_CS(skb), fix);
2514
2515 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2516 pbd->tcp_pseudo_csum);
2517 }
2518
2519 return hlen;
2520}
f85582f8 2521
9f6c9258
DK		 2522/* called with netif_tx_lock
2523 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2524 * netif_wake_queue()
2525 */
2526netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2527{
2528 struct bnx2x *bp = netdev_priv(dev);
6383c0b3 2529
9f6c9258
DK		 2530 struct bnx2x_fastpath *fp;
2531 struct netdev_queue *txq;
6383c0b3 2532 struct bnx2x_fp_txdata *txdata;
9f6c9258 2533 struct sw_tx_bd *tx_buf;
619c5cb6 2534 struct eth_tx_start_bd *tx_start_bd, *first_bd;
9f6c9258 2535 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
523224a3 2536 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
f2e0899f 2537 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2297a2da 2538 u32 pbd_e2_parsing_data = 0;
9f6c9258 2539 u16 pkt_prod, bd_prod;
6383c0b3 2540 int nbd, txq_index, fp_index, txdata_index;
9f6c9258
DK		 2541 dma_addr_t mapping;
2542 u32 xmit_type = bnx2x_xmit_type(bp, skb);
2543 int i;
2544 u8 hlen = 0;
2545 __le16 pkt_size = 0;
2546 struct ethhdr *eth;
2547 u8 mac_type = UNICAST_ADDRESS;
2548
2549#ifdef BNX2X_STOP_ON_ERROR
2550 if (unlikely(bp->panic))
2551 return NETDEV_TX_BUSY;
2552#endif
2553
6383c0b3
AE		 2554 txq_index = skb_get_queue_mapping(skb);
2555 txq = netdev_get_tx_queue(dev, txq_index);
2556
2557 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + FCOE_PRESENT);
2558
2559 /* decode the fastpath index and the cos index from the txq */
2560 fp_index = TXQ_TO_FP(txq_index);
2561 txdata_index = TXQ_TO_COS(txq_index);
2562
2563#ifdef BCM_CNIC
2564 /*
2565 * Override the above for the FCoE queue:
2566 * - FCoE fp entry is right after the ETH entries.
2567 * - FCoE L2 queue uses bp->txdata[0] only.
2568 */
2569 if (unlikely(!NO_FCOE(bp) && (txq_index ==
2570 bnx2x_fcoe_tx(bp, txq_index)))) {
2571 fp_index = FCOE_IDX;
2572 txdata_index = 0;
2573 }
2574#endif
2575
2576 /* enable this debug print to view the transmission queue being used
2577 DP(BNX2X_MSG_FP, "indices: txq %d, fp %d, txdata %d",
2578 txq_index, fp_index, txdata_index); */
9f6c9258 2579
6383c0b3 2580 /* locate the fastpath and the txdata */
9f6c9258 2581 fp = &bp->fp[fp_index];
6383c0b3
AE		 2582 txdata = &fp->txdata[txdata_index];
2583
2584 /* enable this debug print to view the tranmission details
2585 DP(BNX2X_MSG_FP,"transmitting packet cid %d fp index %d txdata_index %d"
2586 " tx_data ptr %p fp pointer %p",
2587 txdata->cid, fp_index, txdata_index, txdata, fp); */
9f6c9258 2588
6383c0b3
AE		 2589 if (unlikely(bnx2x_tx_avail(bp, txdata) <
2590 (skb_shinfo(skb)->nr_frags + 3))) {
9f6c9258
DK		 2591 fp->eth_q_stats.driver_xoff++;
2592 netif_tx_stop_queue(txq);
2593 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2594 return NETDEV_TX_BUSY;
2595 }
2596
f2e0899f
DK		 2597 DP(NETIF_MSG_TX_QUEUED, "queue[%d]: SKB: summed %x protocol %x "
2598 "protocol(%x,%x) gso type %x xmit_type %x\n",
6383c0b3 2599 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
9f6c9258
DK		 2600 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2601
2602 eth = (struct ethhdr *)skb->data;
2603
2604 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2605 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2606 if (is_broadcast_ether_addr(eth->h_dest))
2607 mac_type = BROADCAST_ADDRESS;
2608 else
2609 mac_type = MULTICAST_ADDRESS;
2610 }
2611
2612#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2613 /* First, check if we need to linearize the skb (due to FW
2614 restrictions). No need to check fragmentation if page size > 8K
2615 (there will be no violation to FW restrictions) */
2616 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2617 /* Statistics of linearization */
2618 bp->lin_cnt++;
2619 if (skb_linearize(skb) != 0) {
2620 DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
2621 "silently dropping this SKB\n");
2622 dev_kfree_skb_any(skb);
2623 return NETDEV_TX_OK;
2624 }
2625 }
2626#endif
619c5cb6
VZ		 2627 /* Map skb linear data for DMA */
2628 mapping = dma_map_single(&bp->pdev->dev, skb->data,
2629 skb_headlen(skb), DMA_TO_DEVICE);
2630 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2631 DP(NETIF_MSG_TX_QUEUED, "SKB mapping failed - "
2632 "silently dropping this SKB\n");
2633 dev_kfree_skb_any(skb);
2634 return NETDEV_TX_OK;
2635 }
9f6c9258
DK		 2636 /*
2637 Please read carefully. First we use one BD which we mark as start,
2638 then we have a parsing info BD (used for TSO or xsum),
2639 and only then we have the rest of the TSO BDs.
2640 (don't forget to mark the last one as last,
2641 and to unmap only AFTER you write to the BD ...)
2642 And above all, all pdb sizes are in words - NOT DWORDS!
2643 */
2644
619c5cb6
VZ		 2645 /* get current pkt produced now - advance it just before sending packet
2646 * since mapping of pages may fail and cause packet to be dropped
2647 */
6383c0b3
AE		 2648 pkt_prod = txdata->tx_pkt_prod;
2649 bd_prod = TX_BD(txdata->tx_bd_prod);
9f6c9258 2650
619c5cb6
VZ		 2651 /* get a tx_buf and first BD
2652 * tx_start_bd may be changed during SPLIT,
2653 * but first_bd will always stay first
2654 */
6383c0b3
AE		 2655 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
2656 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
619c5cb6 2657 first_bd = tx_start_bd;
9f6c9258
DK		 2658
2659 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
f85582f8
DK		 2660 SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2661 mac_type);
2662
9f6c9258 2663 /* header nbd */
f85582f8 2664 SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
9f6c9258
DK		 2665
2666 /* remember the first BD of the packet */
6383c0b3 2667 tx_buf->first_bd = txdata->tx_bd_prod;
9f6c9258
DK		 2668 tx_buf->skb = skb;
2669 tx_buf->flags = 0;
2670
2671 DP(NETIF_MSG_TX_QUEUED,
2672 "sending pkt %u @%p next_idx %u bd %u @%p\n",
6383c0b3 2673 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
9f6c9258 2674
eab6d18d 2675 if (vlan_tx_tag_present(skb)) {
523224a3
DK		 2676 tx_start_bd->vlan_or_ethertype =
2677 cpu_to_le16(vlan_tx_tag_get(skb));
2678 tx_start_bd->bd_flags.as_bitfield |=
2679 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
9f6c9258 2680 } else
523224a3 2681 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
9f6c9258
DK		 2682
2683 /* turn on parsing and get a BD */
2684 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
9f6c9258 2685
93ef5c02
DK		 2686 if (xmit_type & XMIT_CSUM)
2687 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
9f6c9258 2688
619c5cb6 2689 if (!CHIP_IS_E1x(bp)) {
6383c0b3 2690 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
f2e0899f
DK		 2691 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2692 /* Set PBD in checksum offload case */
2693 if (xmit_type & XMIT_CSUM)
2297a2da
VZ		 2694 hlen = bnx2x_set_pbd_csum_e2(bp, skb,
2695 &pbd_e2_parsing_data,
2696 xmit_type);
619c5cb6
VZ		 2697 if (IS_MF_SI(bp)) {
2698 /*
2699 * fill in the MAC addresses in the PBD - for local
2700 * switching
2701 */
2702 bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
2703 &pbd_e2->src_mac_addr_mid,
2704 &pbd_e2->src_mac_addr_lo,
2705 eth->h_source);
2706 bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
2707 &pbd_e2->dst_mac_addr_mid,
2708 &pbd_e2->dst_mac_addr_lo,
2709 eth->h_dest);
2710 }
f2e0899f 2711 } else {
6383c0b3 2712 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
f2e0899f
DK		 2713 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2714 /* Set PBD in checksum offload case */
2715 if (xmit_type & XMIT_CSUM)
2716 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
9f6c9258 2717
9f6c9258
DK		 2718 }
2719
f85582f8 2720 /* Setup the data pointer of the first BD of the packet */
9f6c9258
DK		 2721 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2722 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
619c5cb6 2723 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
9f6c9258
DK		 2724 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2725 pkt_size = tx_start_bd->nbytes;
2726
2727 DP(NETIF_MSG_TX_QUEUED, "first bd @%p addr (%x:%x) nbd %d"
2728 " nbytes %d flags %x vlan %x\n",
2729 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2730 le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
523224a3
DK		 2731 tx_start_bd->bd_flags.as_bitfield,
2732 le16_to_cpu(tx_start_bd->vlan_or_ethertype));
9f6c9258
DK		 2733
2734 if (xmit_type & XMIT_GSO) {
2735
2736 DP(NETIF_MSG_TX_QUEUED,
2737 "TSO packet len %d hlen %d total len %d tso size %d\n",
2738 skb->len, hlen, skb_headlen(skb),
2739 skb_shinfo(skb)->gso_size);
2740
2741 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2742
2743 if (unlikely(skb_headlen(skb) > hlen))
6383c0b3
AE		 2744 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
2745 &tx_start_bd, hlen,
2746 bd_prod, ++nbd);
619c5cb6 2747 if (!CHIP_IS_E1x(bp))
2297a2da
VZ		 2748 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
2749 xmit_type);
f2e0899f
DK		 2750 else
2751 bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
9f6c9258 2752 }
2297a2da
VZ		 2753
2754 /* Set the PBD's parsing_data field if not zero
2755 * (for the chips newer than 57711).
2756 */
2757 if (pbd_e2_parsing_data)
2758 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
2759
9f6c9258
DK		 2760 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2761
f85582f8 2762 /* Handle fragmented skb */
9f6c9258
DK		 2763 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2764 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2765
619c5cb6
VZ		 2766 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2767 frag->page_offset, frag->size,
2768 DMA_TO_DEVICE);
2769 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2770
2771 DP(NETIF_MSG_TX_QUEUED, "Unable to map page - "
2772 "dropping packet...\n");
2773
2774 /* we need unmap all buffers already mapped
2775 * for this SKB;
2776 * first_bd->nbd need to be properly updated
2777 * before call to bnx2x_free_tx_pkt
2778 */
2779 first_bd->nbd = cpu_to_le16(nbd);
6383c0b3
AE		 2780 bnx2x_free_tx_pkt(bp, txdata,
2781 TX_BD(txdata->tx_pkt_prod));
619c5cb6
VZ		 2782 return NETDEV_TX_OK;
2783 }
2784
9f6c9258 2785 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
6383c0b3 2786 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258 2787 if (total_pkt_bd == NULL)
6383c0b3 2788 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
9f6c9258 2789
9f6c9258
DK		 2790 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2791 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2792 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2793 le16_add_cpu(&pkt_size, frag->size);
619c5cb6 2794 nbd++;
9f6c9258
DK		 2795
2796 DP(NETIF_MSG_TX_QUEUED,
2797 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
2798 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2799 le16_to_cpu(tx_data_bd->nbytes));
2800 }
2801
2802 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2803
619c5cb6
VZ		 2804 /* update with actual num BDs */
2805 first_bd->nbd = cpu_to_le16(nbd);
2806
9f6c9258
DK		 2807 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2808
2809 /* now send a tx doorbell, counting the next BD
2810 * if the packet contains or ends with it
2811 */
2812 if (TX_BD_POFF(bd_prod) < nbd)
2813 nbd++;
2814
619c5cb6
VZ		 2815 /* total_pkt_bytes should be set on the first data BD if
2816 * it's not an LSO packet and there is more than one
2817 * data BD. In this case pkt_size is limited by an MTU value.
2818 * However we prefer to set it for an LSO packet (while we don't
2819 * have to) in order to save some CPU cycles in a none-LSO
2820 * case, when we much more care about them.
2821 */
9f6c9258
DK		 2822 if (total_pkt_bd != NULL)
2823 total_pkt_bd->total_pkt_bytes = pkt_size;
2824
523224a3 2825 if (pbd_e1x)
9f6c9258 2826 DP(NETIF_MSG_TX_QUEUED,
523224a3 2827 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u"
9f6c9258 2828 " tcp_flags %x xsum %x seq %u hlen %u\n",
523224a3
DK		 2829 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2830 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2831 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2832 le16_to_cpu(pbd_e1x->total_hlen_w));
f2e0899f
DK		 2833 if (pbd_e2)
2834 DP(NETIF_MSG_TX_QUEUED,
2835 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
2836 pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2837 pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2838 pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2839 pbd_e2->parsing_data);
9f6c9258
DK		 2840 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
2841
6383c0b3 2842 txdata->tx_pkt_prod++;
9f6c9258
DK		 2843 /*
2844 * Make sure that the BD data is updated before updating the producer
2845 * since FW might read the BD right after the producer is updated.
2846 * This is only applicable for weak-ordered memory model archs such
2847 * as IA-64. The following barrier is also mandatory since FW will
2848 * assumes packets must have BDs.
2849 */
2850 wmb();
2851
6383c0b3 2852 txdata->tx_db.data.prod += nbd;
9f6c9258 2853 barrier();
f85582f8 2854
6383c0b3 2855 DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
9f6c9258
DK		 2856
2857 mmiowb();
2858
6383c0b3 2859 txdata->tx_bd_prod += nbd;
9f6c9258 2860
6383c0b3 2861 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_SKB_FRAGS + 3)) {
9f6c9258
DK		 2862 netif_tx_stop_queue(txq);
2863
2864 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2865 * ordering of set_bit() in netif_tx_stop_queue() and read of
2866 * fp->bd_tx_cons */
2867 smp_mb();
2868
2869 fp->eth_q_stats.driver_xoff++;
6383c0b3 2870 if (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3)
9f6c9258
DK		 2871 netif_tx_wake_queue(txq);
2872 }
6383c0b3 2873 txdata->tx_pkt++;
9f6c9258
DK		 2874
2875 return NETDEV_TX_OK;
2876}
f85582f8 2877
6383c0b3
AE		 2878/**
2879 * bnx2x_setup_tc - routine to configure net_device for multi tc
2880 *
2881 * @netdev: net device to configure
2882 * @tc: number of traffic classes to enable
2883 *
2884 * callback connected to the ndo_setup_tc function pointer
2885 */
2886int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
2887{
2888 int cos, prio, count, offset;
2889 struct bnx2x *bp = netdev_priv(dev);
2890
2891 /* setup tc must be called under rtnl lock */
2892 ASSERT_RTNL();
2893
2894 /* no traffic classes requested. aborting */
2895 if (!num_tc) {
2896 netdev_reset_tc(dev);
2897 return 0;
2898 }
2899
2900 /* requested to support too many traffic classes */
2901 if (num_tc > bp->max_cos) {
2902 DP(NETIF_MSG_TX_ERR, "support for too many traffic classes"
2903 " requested: %d. max supported is %d",
2904 num_tc, bp->max_cos);
2905 return -EINVAL;
2906 }
2907
2908 /* declare amount of supported traffic classes */
2909 if (netdev_set_num_tc(dev, num_tc)) {
2910 DP(NETIF_MSG_TX_ERR, "failed to declare %d traffic classes",
2911 num_tc);
2912 return -EINVAL;
2913 }
2914
2915 /* configure priority to traffic class mapping */
2916 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
2917 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
2918 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d",
2919 prio, bp->prio_to_cos[prio]);
2920 }
2921
2922
2923 /* Use this configuration to diffrentiate tc0 from other COSes
2924 This can be used for ets or pfc, and save the effort of setting
2925 up a multio class queue disc or negotiating DCBX with a switch
2926 netdev_set_prio_tc_map(dev, 0, 0);
2927 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d", 0, 0);
2928 for (prio = 1; prio < 16; prio++) {
2929 netdev_set_prio_tc_map(dev, prio, 1);
2930 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d", prio, 1);
2931 } */
2932
2933 /* configure traffic class to transmission queue mapping */
2934 for (cos = 0; cos < bp->max_cos; cos++) {
2935 count = BNX2X_NUM_ETH_QUEUES(bp);
2936 offset = cos * MAX_TXQS_PER_COS;
2937 netdev_set_tc_queue(dev, cos, count, offset);
2938 DP(BNX2X_MSG_SP, "mapping tc %d to offset %d count %d",
2939 cos, offset, count);
2940 }
2941
2942 return 0;
2943}
2944
9f6c9258
DK		 2945/* called with rtnl_lock */
2946int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2947{
2948 struct sockaddr *addr = p;
2949 struct bnx2x *bp = netdev_priv(dev);
619c5cb6 2950 int rc = 0;
9f6c9258
DK		 2951
2952 if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2953 return -EINVAL;
2954
619c5cb6
VZ		 2955 if (netif_running(dev)) {
2956 rc = bnx2x_set_eth_mac(bp, false);
2957 if (rc)
2958 return rc;
2959 }
2960
9f6c9258 2961 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
619c5cb6 2962
523224a3 2963 if (netif_running(dev))
619c5cb6 2964 rc = bnx2x_set_eth_mac(bp, true);
9f6c9258 2965
619c5cb6 2966 return rc;
9f6c9258
DK		 2967}
2968
b3b83c3f
DK		 2969static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
2970{
2971 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
2972 struct bnx2x_fastpath *fp = &bp->fp[fp_index];
6383c0b3 2973 u8 cos;
b3b83c3f
DK		 2974
2975 /* Common */
2976#ifdef BCM_CNIC
2977 if (IS_FCOE_IDX(fp_index)) {
2978 memset(sb, 0, sizeof(union host_hc_status_block));
2979 fp->status_blk_mapping = 0;
2980
2981 } else {
2982#endif
2983 /* status blocks */
619c5cb6 2984 if (!CHIP_IS_E1x(bp))
b3b83c3f
DK		 2985 BNX2X_PCI_FREE(sb->e2_sb,
2986 bnx2x_fp(bp, fp_index,
2987 status_blk_mapping),
2988 sizeof(struct host_hc_status_block_e2));
2989 else
2990 BNX2X_PCI_FREE(sb->e1x_sb,
2991 bnx2x_fp(bp, fp_index,
2992 status_blk_mapping),
2993 sizeof(struct host_hc_status_block_e1x));
2994#ifdef BCM_CNIC
2995 }
2996#endif
2997 /* Rx */
2998 if (!skip_rx_queue(bp, fp_index)) {
2999 bnx2x_free_rx_bds(fp);
3000
3001 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3002 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
3003 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
3004 bnx2x_fp(bp, fp_index, rx_desc_mapping),
3005 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3006
3007 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
3008 bnx2x_fp(bp, fp_index, rx_comp_mapping),
3009 sizeof(struct eth_fast_path_rx_cqe) *
3010 NUM_RCQ_BD);
3011
3012 /* SGE ring */
3013 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
3014 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
3015 bnx2x_fp(bp, fp_index, rx_sge_mapping),
3016 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3017 }
3018
3019 /* Tx */
3020 if (!skip_tx_queue(bp, fp_index)) {
3021 /* fastpath tx rings: tx_buf tx_desc */
6383c0b3
AE		 3022 for_each_cos_in_tx_queue(fp, cos) {
3023 struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3024
3025 DP(BNX2X_MSG_SP,
3026 "freeing tx memory of fp %d cos %d cid %d",
3027 fp_index, cos, txdata->cid);
3028
3029 BNX2X_FREE(txdata->tx_buf_ring);
3030 BNX2X_PCI_FREE(txdata->tx_desc_ring,
3031 txdata->tx_desc_mapping,
3032 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3033 }
b3b83c3f
DK		 3034 }
3035 /* end of fastpath */
3036}
3037
3038void bnx2x_free_fp_mem(struct bnx2x *bp)
3039{
3040 int i;
3041 for_each_queue(bp, i)
3042 bnx2x_free_fp_mem_at(bp, i);
3043}
3044
3045static inline void set_sb_shortcuts(struct bnx2x *bp, int index)
3046{
3047 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
619c5cb6 3048 if (!CHIP_IS_E1x(bp)) {
b3b83c3f
DK		 3049 bnx2x_fp(bp, index, sb_index_values) =
3050 (__le16 *)status_blk.e2_sb->sb.index_values;
3051 bnx2x_fp(bp, index, sb_running_index) =
3052 (__le16 *)status_blk.e2_sb->sb.running_index;
3053 } else {
3054 bnx2x_fp(bp, index, sb_index_values) =
3055 (__le16 *)status_blk.e1x_sb->sb.index_values;
3056 bnx2x_fp(bp, index, sb_running_index) =
3057 (__le16 *)status_blk.e1x_sb->sb.running_index;
3058 }
3059}
3060
3061static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
3062{
3063 union host_hc_status_block *sb;
3064 struct bnx2x_fastpath *fp = &bp->fp[index];
3065 int ring_size = 0;
6383c0b3 3066 u8 cos;
b3b83c3f
DK		 3067
3068 /* if rx_ring_size specified - use it */
3069 int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
6383c0b3 3070 MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
b3b83c3f
DK		 3071
3072 /* allocate at least number of buffers required by FW */
6383c0b3 3073 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
b3b83c3f
DK		 3074 MIN_RX_SIZE_TPA,
3075 rx_ring_size);
3076
b3b83c3f
DK		 3077 /* Common */
3078 sb = &bnx2x_fp(bp, index, status_blk);
3079#ifdef BCM_CNIC
3080 if (!IS_FCOE_IDX(index)) {
3081#endif
3082 /* status blocks */
619c5cb6 3083 if (!CHIP_IS_E1x(bp))
b3b83c3f
DK		 3084 BNX2X_PCI_ALLOC(sb->e2_sb,
3085 &bnx2x_fp(bp, index, status_blk_mapping),
3086 sizeof(struct host_hc_status_block_e2));
3087 else
3088 BNX2X_PCI_ALLOC(sb->e1x_sb,
3089 &bnx2x_fp(bp, index, status_blk_mapping),
3090 sizeof(struct host_hc_status_block_e1x));
3091#ifdef BCM_CNIC
3092 }
3093#endif
8eef2af1
DK		 3094
3095 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
3096 * set shortcuts for it.
3097 */
3098 if (!IS_FCOE_IDX(index))
3099 set_sb_shortcuts(bp, index);
b3b83c3f
DK		 3100
3101 /* Tx */
3102 if (!skip_tx_queue(bp, index)) {
3103 /* fastpath tx rings: tx_buf tx_desc */
6383c0b3
AE		 3104 for_each_cos_in_tx_queue(fp, cos) {
3105 struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3106
3107 DP(BNX2X_MSG_SP, "allocating tx memory of "
3108 "fp %d cos %d",
3109 index, cos);
3110
3111 BNX2X_ALLOC(txdata->tx_buf_ring,
b3b83c3f 3112 sizeof(struct sw_tx_bd) * NUM_TX_BD);
6383c0b3
AE		 3113 BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
3114 &txdata->tx_desc_mapping,
b3b83c3f 3115 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
6383c0b3 3116 }
b3b83c3f
DK		 3117 }
3118
3119 /* Rx */
3120 if (!skip_rx_queue(bp, index)) {
3121 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3122 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
3123 sizeof(struct sw_rx_bd) * NUM_RX_BD);
3124 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
3125 &bnx2x_fp(bp, index, rx_desc_mapping),
3126 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3127
3128 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
3129 &bnx2x_fp(bp, index, rx_comp_mapping),
3130 sizeof(struct eth_fast_path_rx_cqe) *
3131 NUM_RCQ_BD);
3132
3133 /* SGE ring */
3134 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
3135 sizeof(struct sw_rx_page) * NUM_RX_SGE);
3136 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
3137 &bnx2x_fp(bp, index, rx_sge_mapping),
3138 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3139 /* RX BD ring */
3140 bnx2x_set_next_page_rx_bd(fp);
3141
3142 /* CQ ring */
3143 bnx2x_set_next_page_rx_cq(fp);
3144
3145 /* BDs */
3146 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
3147 if (ring_size < rx_ring_size)
3148 goto alloc_mem_err;
3149 }
3150
3151 return 0;
3152
3153/* handles low memory cases */
3154alloc_mem_err:
3155 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
3156 index, ring_size);
3157 /* FW will drop all packets if queue is not big enough,
3158 * In these cases we disable the queue
6383c0b3 3159 * Min size is different for OOO, TPA and non-TPA queues
b3b83c3f
DK		 3160 */
3161 if (ring_size < (fp->disable_tpa ?
eb722d7a 3162 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
b3b83c3f
DK		 3163 /* release memory allocated for this queue */
3164 bnx2x_free_fp_mem_at(bp, index);
3165 return -ENOMEM;
3166 }
3167 return 0;
3168}
3169
3170int bnx2x_alloc_fp_mem(struct bnx2x *bp)
3171{
3172 int i;
3173
3174 /**
3175 * 1. Allocate FP for leading - fatal if error
3176 * 2. {CNIC} Allocate FCoE FP - fatal if error
6383c0b3
AE		 3177 * 3. {CNIC} Allocate OOO + FWD - disable OOO if error
3178 * 4. Allocate RSS - fix number of queues if error
b3b83c3f
DK		 3179 */
3180
3181 /* leading */
3182 if (bnx2x_alloc_fp_mem_at(bp, 0))
3183 return -ENOMEM;
6383c0b3 3184
b3b83c3f 3185#ifdef BCM_CNIC
8eef2af1
DK		 3186 if (!NO_FCOE(bp))
3187 /* FCoE */
3188 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX))
3189 /* we will fail load process instead of mark
3190 * NO_FCOE_FLAG
3191 */
3192 return -ENOMEM;
b3b83c3f 3193#endif
6383c0b3 3194
b3b83c3f
DK		 3195 /* RSS */
3196 for_each_nondefault_eth_queue(bp, i)
3197 if (bnx2x_alloc_fp_mem_at(bp, i))
3198 break;
3199
3200 /* handle memory failures */
3201 if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
3202 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
3203
3204 WARN_ON(delta < 0);
3205#ifdef BCM_CNIC
3206 /**
3207 * move non eth FPs next to last eth FP
3208 * must be done in that order
3209 * FCOE_IDX < FWD_IDX < OOO_IDX
3210 */
3211
6383c0b3 3212 /* move FCoE fp even NO_FCOE_FLAG is on */
b3b83c3f
DK		 3213 bnx2x_move_fp(bp, FCOE_IDX, FCOE_IDX - delta);
3214#endif
3215 bp->num_queues -= delta;
3216 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
3217 bp->num_queues + delta, bp->num_queues);
3218 }
3219
3220 return 0;
3221}
d6214d7a 3222
523224a3
DK		 3223void bnx2x_free_mem_bp(struct bnx2x *bp)
3224{
3225 kfree(bp->fp);
3226 kfree(bp->msix_table);
3227 kfree(bp->ilt);
3228}
3229
3230int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
3231{
3232 struct bnx2x_fastpath *fp;
3233 struct msix_entry *tbl;
3234 struct bnx2x_ilt *ilt;
6383c0b3
AE		 3235 int msix_table_size = 0;
3236
3237 /*
3238 * The biggest MSI-X table we might need is as a maximum number of fast
3239 * path IGU SBs plus default SB (for PF).
3240 */
3241 msix_table_size = bp->igu_sb_cnt + 1;
523224a3 3242
6383c0b3
AE		 3243 /* fp array: RSS plus CNIC related L2 queues */
3244 fp = kzalloc((BNX2X_MAX_RSS_COUNT(bp) + NON_ETH_CONTEXT_USE) *
3245 sizeof(*fp), GFP_KERNEL);
523224a3
DK		 3246 if (!fp)
3247 goto alloc_err;
3248 bp->fp = fp;
3249
3250 /* msix table */
6383c0b3 3251 tbl = kzalloc(msix_table_size * sizeof(*tbl), GFP_KERNEL);
523224a3
DK		 3252 if (!tbl)
3253 goto alloc_err;
3254 bp->msix_table = tbl;
3255
3256 /* ilt */
3257 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
3258 if (!ilt)
3259 goto alloc_err;
3260 bp->ilt = ilt;
3261
3262 return 0;
3263alloc_err:
3264 bnx2x_free_mem_bp(bp);
3265 return -ENOMEM;
3266
3267}
3268
a9fccec7 3269int bnx2x_reload_if_running(struct net_device *dev)
66371c44
MM		 3270{
3271 struct bnx2x *bp = netdev_priv(dev);
3272
3273 if (unlikely(!netif_running(dev)))
3274 return 0;
3275
3276 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
3277 return bnx2x_nic_load(bp, LOAD_NORMAL);
3278}
3279
1ac9e428
YR		 3280int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
3281{
3282 u32 sel_phy_idx = 0;
3283 if (bp->link_params.num_phys <= 1)
3284 return INT_PHY;
3285
3286 if (bp->link_vars.link_up) {
3287 sel_phy_idx = EXT_PHY1;
3288 /* In case link is SERDES, check if the EXT_PHY2 is the one */
3289 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
3290 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
3291 sel_phy_idx = EXT_PHY2;
3292 } else {
3293
3294 switch (bnx2x_phy_selection(&bp->link_params)) {
3295 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
3296 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
3297 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
3298 sel_phy_idx = EXT_PHY1;
3299 break;
3300 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
3301 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
3302 sel_phy_idx = EXT_PHY2;
3303 break;
3304 }
3305 }
3306
3307 return sel_phy_idx;
3308
3309}
3310int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
3311{
3312 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
3313 /*
3314 * The selected actived PHY is always after swapping (in case PHY
3315 * swapping is enabled). So when swapping is enabled, we need to reverse
3316 * the configuration
3317 */
3318
3319 if (bp->link_params.multi_phy_config &
3320 PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
3321 if (sel_phy_idx == EXT_PHY1)
3322 sel_phy_idx = EXT_PHY2;
3323 else if (sel_phy_idx == EXT_PHY2)
3324 sel_phy_idx = EXT_PHY1;
3325 }
3326 return LINK_CONFIG_IDX(sel_phy_idx);
3327}
3328
9f6c9258
DK		 3329/* called with rtnl_lock */
3330int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
3331{
3332 struct bnx2x *bp = netdev_priv(dev);
9f6c9258
DK		 3333
3334 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3335 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
3336 return -EAGAIN;
3337 }
3338
3339 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
3340 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
3341 return -EINVAL;
3342
3343 /* This does not race with packet allocation
3344 * because the actual alloc size is
3345 * only updated as part of load
3346 */
3347 dev->mtu = new_mtu;
3348
66371c44
MM		 3349 return bnx2x_reload_if_running(dev);
3350}
3351
3352u32 bnx2x_fix_features(struct net_device *dev, u32 features)
3353{
3354 struct bnx2x *bp = netdev_priv(dev);
3355
3356 /* TPA requires Rx CSUM offloading */
3357 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa)
3358 features &= ~NETIF_F_LRO;
3359
3360 return features;
3361}
3362
3363int bnx2x_set_features(struct net_device *dev, u32 features)
3364{
3365 struct bnx2x *bp = netdev_priv(dev);
3366 u32 flags = bp->flags;
538dd2e3 3367 bool bnx2x_reload = false;
66371c44
MM		 3368
3369 if (features & NETIF_F_LRO)
3370 flags |= TPA_ENABLE_FLAG;
3371 else
3372 flags &= ~TPA_ENABLE_FLAG;
3373
538dd2e3
MB		 3374 if (features & NETIF_F_LOOPBACK) {
3375 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
3376 bp->link_params.loopback_mode = LOOPBACK_BMAC;
3377 bnx2x_reload = true;
3378 }
3379 } else {
3380 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
3381 bp->link_params.loopback_mode = LOOPBACK_NONE;
3382 bnx2x_reload = true;
3383 }
3384 }
3385
66371c44
MM		 3386 if (flags ^ bp->flags) {
3387 bp->flags = flags;
538dd2e3
MB		 3388 bnx2x_reload = true;
3389 }
66371c44 3390
538dd2e3 3391 if (bnx2x_reload) {
66371c44
MM		 3392 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
3393 return bnx2x_reload_if_running(dev);
3394 /* else: bnx2x_nic_load() will be called at end of recovery */
9f6c9258
DK		 3395 }
3396
66371c44 3397 return 0;
9f6c9258
DK		 3398}
3399
3400void bnx2x_tx_timeout(struct net_device *dev)
3401{
3402 struct bnx2x *bp = netdev_priv(dev);
3403
3404#ifdef BNX2X_STOP_ON_ERROR
3405 if (!bp->panic)
3406 bnx2x_panic();
3407#endif
7be08a72
AE		 3408
3409 smp_mb__before_clear_bit();
3410 set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
3411 smp_mb__after_clear_bit();
3412
9f6c9258 3413 /* This allows the netif to be shutdown gracefully before resetting */
7be08a72 3414 schedule_delayed_work(&bp->sp_rtnl_task, 0);
9f6c9258
DK		 3415}
3416
9f6c9258
DK		 3417int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
3418{
3419 struct net_device *dev = pci_get_drvdata(pdev);
3420 struct bnx2x *bp;
3421
3422 if (!dev) {
3423 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3424 return -ENODEV;
3425 }
3426 bp = netdev_priv(dev);
3427
3428 rtnl_lock();
3429
3430 pci_save_state(pdev);
3431
3432 if (!netif_running(dev)) {
3433 rtnl_unlock();
3434 return 0;
3435 }
3436
3437 netif_device_detach(dev);
3438
3439 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
3440
3441 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
3442
3443 rtnl_unlock();
3444
3445 return 0;
3446}
3447
3448int bnx2x_resume(struct pci_dev *pdev)
3449{
3450 struct net_device *dev = pci_get_drvdata(pdev);
3451 struct bnx2x *bp;
3452 int rc;
3453
3454 if (!dev) {
3455 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3456 return -ENODEV;
3457 }
3458 bp = netdev_priv(dev);
3459
3460 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3461 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
3462 return -EAGAIN;
3463 }
3464
3465 rtnl_lock();
3466
3467 pci_restore_state(pdev);
3468
3469 if (!netif_running(dev)) {
3470 rtnl_unlock();
3471 return 0;
3472 }
3473
3474 bnx2x_set_power_state(bp, PCI_D0);
3475 netif_device_attach(dev);
3476
f2e0899f
DK		 3477 /* Since the chip was reset, clear the FW sequence number */
3478 bp->fw_seq = 0;
9f6c9258
DK		 3479 rc = bnx2x_nic_load(bp, LOAD_OPEN);
3480
3481 rtnl_unlock();
3482
3483 return rc;
3484}
619c5cb6
VZ		 3485
3486
3487void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
3488 u32 cid)
3489{
3490 /* ustorm cxt validation */
3491 cxt->ustorm_ag_context.cdu_usage =
3492 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3493 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
3494 /* xcontext validation */
3495 cxt->xstorm_ag_context.cdu_reserved =
3496 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3497 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
3498}
3499
3500static inline void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
3501 u8 fw_sb_id, u8 sb_index,
3502 u8 ticks)
3503{
3504
3505 u32 addr = BAR_CSTRORM_INTMEM +
3506 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
3507 REG_WR8(bp, addr, ticks);
3508 DP(NETIF_MSG_HW, "port %x fw_sb_id %d sb_index %d ticks %d\n",
3509 port, fw_sb_id, sb_index, ticks);
3510}
3511
3512static inline void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
3513 u16 fw_sb_id, u8 sb_index,
3514 u8 disable)
3515{
3516 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
3517 u32 addr = BAR_CSTRORM_INTMEM +
3518 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
3519 u16 flags = REG_RD16(bp, addr);
3520 /* clear and set */
3521 flags &= ~HC_INDEX_DATA_HC_ENABLED;
3522 flags |= enable_flag;
3523 REG_WR16(bp, addr, flags);
3524 DP(NETIF_MSG_HW, "port %x fw_sb_id %d sb_index %d disable %d\n",
3525 port, fw_sb_id, sb_index, disable);
3526}
3527
3528void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
3529 u8 sb_index, u8 disable, u16 usec)
3530{
3531 int port = BP_PORT(bp);
3532 u8 ticks = usec / BNX2X_BTR;
3533
3534 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
3535
3536 disable = disable ? 1 : (usec ? 0 : 1);
3537 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
3538}
Linux kernel - Deliverables
This page took 0.299293 seconds and 5 git commands to generate.