projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bnxt_en: Reduce maximum ring pages if page size is 64K.
[deliverable/linux.git] / drivers / net / ethernet / broadcom / bnxt / bnxt.c
CommitLineData
c0c050c5
MC		 1/* Broadcom NetXtreme-C/E network driver.
2 *
11f15ed3 3 * Copyright (c) 2014-2016 Broadcom Corporation
c0c050c5
MC		 4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 */
9
10#include <linux/module.h>
11
12#include <linux/stringify.h>
13#include <linux/kernel.h>
14#include <linux/timer.h>
15#include <linux/errno.h>
16#include <linux/ioport.h>
17#include <linux/slab.h>
18#include <linux/vmalloc.h>
19#include <linux/interrupt.h>
20#include <linux/pci.h>
21#include <linux/netdevice.h>
22#include <linux/etherdevice.h>
23#include <linux/skbuff.h>
24#include <linux/dma-mapping.h>
25#include <linux/bitops.h>
26#include <linux/io.h>
27#include <linux/irq.h>
28#include <linux/delay.h>
29#include <asm/byteorder.h>
30#include <asm/page.h>
31#include <linux/time.h>
32#include <linux/mii.h>
33#include <linux/if.h>
34#include <linux/if_vlan.h>
35#include <net/ip.h>
36#include <net/tcp.h>
37#include <net/udp.h>
38#include <net/checksum.h>
39#include <net/ip6_checksum.h>
40#if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
41#include <net/vxlan.h>
42#endif
43#ifdef CONFIG_NET_RX_BUSY_POLL
44#include <net/busy_poll.h>
45#endif
46#include <linux/workqueue.h>
47#include <linux/prefetch.h>
48#include <linux/cache.h>
49#include <linux/log2.h>
50#include <linux/aer.h>
51#include <linux/bitmap.h>
52#include <linux/cpu_rmap.h>
53
54#include "bnxt_hsi.h"
55#include "bnxt.h"
56#include "bnxt_sriov.h"
57#include "bnxt_ethtool.h"
58
59#define BNXT_TX_TIMEOUT (5 * HZ)
60
61static const char version[] =
62 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
63
64MODULE_LICENSE("GPL");
65MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
66MODULE_VERSION(DRV_MODULE_VERSION);
67
68#define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
69#define BNXT_RX_DMA_OFFSET NET_SKB_PAD
70#define BNXT_RX_COPY_THRESH 256
71
4419dbe6 72#define BNXT_TX_PUSH_THRESH 164
c0c050c5
MC		 73
74enum board_idx {
fbc9a523 75 BCM57301,
c0c050c5
MC		 76 BCM57302,
77 BCM57304,
fbc9a523 78 BCM57402,
c0c050c5
MC		 79 BCM57404,
80 BCM57406,
81 BCM57304_VF,
82 BCM57404_VF,
83};
84
85/* indexed by enum above */
86static const struct {
87 char *name;
88} board_info[] = {
fbc9a523
DC		 89 { "Broadcom BCM57301 NetXtreme-C Single-port 10Gb Ethernet" },
90 { "Broadcom BCM57302 NetXtreme-C Dual-port 10Gb/25Gb Ethernet" },
c0c050c5 91 { "Broadcom BCM57304 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" },
fbc9a523 92 { "Broadcom BCM57402 NetXtreme-E Dual-port 10Gb Ethernet" },
c0c050c5 93 { "Broadcom BCM57404 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" },
fbc9a523 94 { "Broadcom BCM57406 NetXtreme-E Dual-port 10GBase-T Ethernet" },
c0c050c5
MC		 95 { "Broadcom BCM57304 NetXtreme-C Ethernet Virtual Function" },
96 { "Broadcom BCM57404 NetXtreme-E Ethernet Virtual Function" },
97};
98
99static const struct pci_device_id bnxt_pci_tbl[] = {
fbc9a523 100 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
c0c050c5
MC		 101 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
102 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
fbc9a523 103 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
c0c050c5
MC		 104 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
105 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
106#ifdef CONFIG_BNXT_SRIOV
107 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = BCM57304_VF },
108 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = BCM57404_VF },
109#endif
110 { 0 }
111};
112
113MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
114
115static const u16 bnxt_vf_req_snif[] = {
116 HWRM_FUNC_CFG,
117 HWRM_PORT_PHY_QCFG,
118 HWRM_CFA_L2_FILTER_ALLOC,
119};
120
25be8623
MC		 121static const u16 bnxt_async_events_arr[] = {
122 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
123 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
4bb13abf 124 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
8cbde117 125 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
25be8623
MC		 126};
127
c0c050c5
MC		 128static bool bnxt_vf_pciid(enum board_idx idx)
129{
130 return (idx == BCM57304_VF || idx == BCM57404_VF);
131}
132
133#define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
134#define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
135#define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS)
136
137#define BNXT_CP_DB_REARM(db, raw_cons) \
138 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db)
139
140#define BNXT_CP_DB(db, raw_cons) \
141 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db)
142
143#define BNXT_CP_DB_IRQ_DIS(db) \
144 writel(DB_CP_IRQ_DIS_FLAGS, db)
145
146static inline u32 bnxt_tx_avail(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
147{
148 /* Tell compiler to fetch tx indices from memory. */
149 barrier();
150
151 return bp->tx_ring_size -
152 ((txr->tx_prod - txr->tx_cons) & bp->tx_ring_mask);
153}
154
155static const u16 bnxt_lhint_arr[] = {
156 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
157 TX_BD_FLAGS_LHINT_512_TO_1023,
158 TX_BD_FLAGS_LHINT_1024_TO_2047,
159 TX_BD_FLAGS_LHINT_1024_TO_2047,
160 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
161 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
162 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
163 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
164 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
165 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
166 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
167 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
168 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
169 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
170 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
171 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
172 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
173 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
174 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
175};
176
177static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
178{
179 struct bnxt *bp = netdev_priv(dev);
180 struct tx_bd *txbd;
181 struct tx_bd_ext *txbd1;
182 struct netdev_queue *txq;
183 int i;
184 dma_addr_t mapping;
185 unsigned int length, pad = 0;
186 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
187 u16 prod, last_frag;
188 struct pci_dev *pdev = bp->pdev;
c0c050c5
MC		 189 struct bnxt_tx_ring_info *txr;
190 struct bnxt_sw_tx_bd *tx_buf;
191
192 i = skb_get_queue_mapping(skb);
193 if (unlikely(i >= bp->tx_nr_rings)) {
194 dev_kfree_skb_any(skb);
195 return NETDEV_TX_OK;
196 }
197
b6ab4b01 198 txr = &bp->tx_ring[i];
c0c050c5
MC		 199 txq = netdev_get_tx_queue(dev, i);
200 prod = txr->tx_prod;
201
202 free_size = bnxt_tx_avail(bp, txr);
203 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
204 netif_tx_stop_queue(txq);
205 return NETDEV_TX_BUSY;
206 }
207
208 length = skb->len;
209 len = skb_headlen(skb);
210 last_frag = skb_shinfo(skb)->nr_frags;
211
212 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
213
214 txbd->tx_bd_opaque = prod;
215
216 tx_buf = &txr->tx_buf_ring[prod];
217 tx_buf->skb = skb;
218 tx_buf->nr_frags = last_frag;
219
220 vlan_tag_flags = 0;
221 cfa_action = 0;
222 if (skb_vlan_tag_present(skb)) {
223 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
224 skb_vlan_tag_get(skb);
225 /* Currently supports 8021Q, 8021AD vlan offloads
226 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
227 */
228 if (skb->vlan_proto == htons(ETH_P_8021Q))
229 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
230 }
231
232 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
4419dbe6
MC		 233 struct tx_push_buffer *tx_push_buf = txr->tx_push;
234 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
235 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
236 void *pdata = tx_push_buf->data;
237 u64 *end;
238 int j, push_len;
c0c050c5
MC		 239
240 /* Set COAL_NOW to be ready quickly for the next push */
241 tx_push->tx_bd_len_flags_type =
242 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
243 TX_BD_TYPE_LONG_TX_BD |
244 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
245 TX_BD_FLAGS_COAL_NOW |
246 TX_BD_FLAGS_PACKET_END |
247 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
248
249 if (skb->ip_summed == CHECKSUM_PARTIAL)
250 tx_push1->tx_bd_hsize_lflags =
251 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
252 else
253 tx_push1->tx_bd_hsize_lflags = 0;
254
255 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
256 tx_push1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
257
fbb0fa8b
MC		 258 end = pdata + length;
259 end = PTR_ALIGN(end, 8) - 1;
4419dbe6
MC		 260 *end = 0;
261
c0c050c5
MC		 262 skb_copy_from_linear_data(skb, pdata, len);
263 pdata += len;
264 for (j = 0; j < last_frag; j++) {
265 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
266 void *fptr;
267
268 fptr = skb_frag_address_safe(frag);
269 if (!fptr)
270 goto normal_tx;
271
272 memcpy(pdata, fptr, skb_frag_size(frag));
273 pdata += skb_frag_size(frag);
274 }
275
4419dbe6
MC		 276 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
277 txbd->tx_bd_haddr = txr->data_mapping;
c0c050c5
MC		 278 prod = NEXT_TX(prod);
279 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
280 memcpy(txbd, tx_push1, sizeof(*txbd));
281 prod = NEXT_TX(prod);
4419dbe6 282 tx_push->doorbell =
c0c050c5
MC		 283 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
284 txr->tx_prod = prod;
285
286 netdev_tx_sent_queue(txq, skb->len);
287
4419dbe6
MC		 288 push_len = (length + sizeof(*tx_push) + 7) / 8;
289 if (push_len > 16) {
290 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16);
291 __iowrite64_copy(txr->tx_doorbell + 4, tx_push_buf + 1,
292 push_len - 16);
293 } else {
294 __iowrite64_copy(txr->tx_doorbell, tx_push_buf,
295 push_len);
296 }
c0c050c5
MC		 297
298 tx_buf->is_push = 1;
c0c050c5
MC		 299 goto tx_done;
300 }
301
302normal_tx:
303 if (length < BNXT_MIN_PKT_SIZE) {
304 pad = BNXT_MIN_PKT_SIZE - length;
305 if (skb_pad(skb, pad)) {
306 /* SKB already freed. */
307 tx_buf->skb = NULL;
308 return NETDEV_TX_OK;
309 }
310 length = BNXT_MIN_PKT_SIZE;
311 }
312
313 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
314
315 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
316 dev_kfree_skb_any(skb);
317 tx_buf->skb = NULL;
318 return NETDEV_TX_OK;
319 }
320
321 dma_unmap_addr_set(tx_buf, mapping, mapping);
322 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
323 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
324
325 txbd->tx_bd_haddr = cpu_to_le64(mapping);
326
327 prod = NEXT_TX(prod);
328 txbd1 = (struct tx_bd_ext *)
329 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
330
331 txbd1->tx_bd_hsize_lflags = 0;
332 if (skb_is_gso(skb)) {
333 u32 hdr_len;
334
335 if (skb->encapsulation)
336 hdr_len = skb_inner_network_offset(skb) +
337 skb_inner_network_header_len(skb) +
338 inner_tcp_hdrlen(skb);
339 else
340 hdr_len = skb_transport_offset(skb) +
341 tcp_hdrlen(skb);
342
343 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
344 TX_BD_FLAGS_T_IPID |
345 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
346 length = skb_shinfo(skb)->gso_size;
347 txbd1->tx_bd_mss = cpu_to_le32(length);
348 length += hdr_len;
349 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
350 txbd1->tx_bd_hsize_lflags =
351 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
352 txbd1->tx_bd_mss = 0;
353 }
354
355 length >>= 9;
356 flags |= bnxt_lhint_arr[length];
357 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
358
359 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
360 txbd1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
361 for (i = 0; i < last_frag; i++) {
362 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
363
364 prod = NEXT_TX(prod);
365 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
366
367 len = skb_frag_size(frag);
368 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
369 DMA_TO_DEVICE);
370
371 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
372 goto tx_dma_error;
373
374 tx_buf = &txr->tx_buf_ring[prod];
375 dma_unmap_addr_set(tx_buf, mapping, mapping);
376
377 txbd->tx_bd_haddr = cpu_to_le64(mapping);
378
379 flags = len << TX_BD_LEN_SHIFT;
380 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
381 }
382
383 flags &= ~TX_BD_LEN;
384 txbd->tx_bd_len_flags_type =
385 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
386 TX_BD_FLAGS_PACKET_END);
387
388 netdev_tx_sent_queue(txq, skb->len);
389
390 /* Sync BD data before updating doorbell */
391 wmb();
392
393 prod = NEXT_TX(prod);
394 txr->tx_prod = prod;
395
396 writel(DB_KEY_TX | prod, txr->tx_doorbell);
397 writel(DB_KEY_TX | prod, txr->tx_doorbell);
398
399tx_done:
400
401 mmiowb();
402
403 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
404 netif_tx_stop_queue(txq);
405
406 /* netif_tx_stop_queue() must be done before checking
407 * tx index in bnxt_tx_avail() below, because in
408 * bnxt_tx_int(), we update tx index before checking for
409 * netif_tx_queue_stopped().
410 */
411 smp_mb();
412 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
413 netif_tx_wake_queue(txq);
414 }
415 return NETDEV_TX_OK;
416
417tx_dma_error:
418 last_frag = i;
419
420 /* start back at beginning and unmap skb */
421 prod = txr->tx_prod;
422 tx_buf = &txr->tx_buf_ring[prod];
423 tx_buf->skb = NULL;
424 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
425 skb_headlen(skb), PCI_DMA_TODEVICE);
426 prod = NEXT_TX(prod);
427
428 /* unmap remaining mapped pages */
429 for (i = 0; i < last_frag; i++) {
430 prod = NEXT_TX(prod);
431 tx_buf = &txr->tx_buf_ring[prod];
432 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
433 skb_frag_size(&skb_shinfo(skb)->frags[i]),
434 PCI_DMA_TODEVICE);
435 }
436
437 dev_kfree_skb_any(skb);
438 return NETDEV_TX_OK;
439}
440
441static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
442{
b6ab4b01 443 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
b81a90d3 444 int index = txr - &bp->tx_ring[0];
c0c050c5
MC		 445 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, index);
446 u16 cons = txr->tx_cons;
447 struct pci_dev *pdev = bp->pdev;
448 int i;
449 unsigned int tx_bytes = 0;
450
451 for (i = 0; i < nr_pkts; i++) {
452 struct bnxt_sw_tx_bd *tx_buf;
453 struct sk_buff *skb;
454 int j, last;
455
456 tx_buf = &txr->tx_buf_ring[cons];
457 cons = NEXT_TX(cons);
458 skb = tx_buf->skb;
459 tx_buf->skb = NULL;
460
461 if (tx_buf->is_push) {
462 tx_buf->is_push = 0;
463 goto next_tx_int;
464 }
465
466 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
467 skb_headlen(skb), PCI_DMA_TODEVICE);
468 last = tx_buf->nr_frags;
469
470 for (j = 0; j < last; j++) {
471 cons = NEXT_TX(cons);
472 tx_buf = &txr->tx_buf_ring[cons];
473 dma_unmap_page(
474 &pdev->dev,
475 dma_unmap_addr(tx_buf, mapping),
476 skb_frag_size(&skb_shinfo(skb)->frags[j]),
477 PCI_DMA_TODEVICE);
478 }
479
480next_tx_int:
481 cons = NEXT_TX(cons);
482
483 tx_bytes += skb->len;
484 dev_kfree_skb_any(skb);
485 }
486
487 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
488 txr->tx_cons = cons;
489
490 /* Need to make the tx_cons update visible to bnxt_start_xmit()
491 * before checking for netif_tx_queue_stopped(). Without the
492 * memory barrier, there is a small possibility that bnxt_start_xmit()
493 * will miss it and cause the queue to be stopped forever.
494 */
495 smp_mb();
496
497 if (unlikely(netif_tx_queue_stopped(txq)) &&
498 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
499 __netif_tx_lock(txq, smp_processor_id());
500 if (netif_tx_queue_stopped(txq) &&
501 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
502 txr->dev_state != BNXT_DEV_STATE_CLOSING)
503 netif_tx_wake_queue(txq);
504 __netif_tx_unlock(txq);
505 }
506}
507
508static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
509 gfp_t gfp)
510{
511 u8 *data;
512 struct pci_dev *pdev = bp->pdev;
513
514 data = kmalloc(bp->rx_buf_size, gfp);
515 if (!data)
516 return NULL;
517
518 *mapping = dma_map_single(&pdev->dev, data + BNXT_RX_DMA_OFFSET,
519 bp->rx_buf_use_size, PCI_DMA_FROMDEVICE);
520
521 if (dma_mapping_error(&pdev->dev, *mapping)) {
522 kfree(data);
523 data = NULL;
524 }
525 return data;
526}
527
528static inline int bnxt_alloc_rx_data(struct bnxt *bp,
529 struct bnxt_rx_ring_info *rxr,
530 u16 prod, gfp_t gfp)
531{
532 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
533 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
534 u8 *data;
535 dma_addr_t mapping;
536
537 data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
538 if (!data)
539 return -ENOMEM;
540
541 rx_buf->data = data;
542 dma_unmap_addr_set(rx_buf, mapping, mapping);
543
544 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
545
546 return 0;
547}
548
549static void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons,
550 u8 *data)
551{
552 u16 prod = rxr->rx_prod;
553 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
554 struct rx_bd *cons_bd, *prod_bd;
555
556 prod_rx_buf = &rxr->rx_buf_ring[prod];
557 cons_rx_buf = &rxr->rx_buf_ring[cons];
558
559 prod_rx_buf->data = data;
560
561 dma_unmap_addr_set(prod_rx_buf, mapping,
562 dma_unmap_addr(cons_rx_buf, mapping));
563
564 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
565 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
566
567 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
568}
569
570static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
571{
572 u16 next, max = rxr->rx_agg_bmap_size;
573
574 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
575 if (next >= max)
576 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
577 return next;
578}
579
580static inline int bnxt_alloc_rx_page(struct bnxt *bp,
581 struct bnxt_rx_ring_info *rxr,
582 u16 prod, gfp_t gfp)
583{
584 struct rx_bd *rxbd =
585 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
586 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
587 struct pci_dev *pdev = bp->pdev;
588 struct page *page;
589 dma_addr_t mapping;
590 u16 sw_prod = rxr->rx_sw_agg_prod;
89d0a06c 591 unsigned int offset = 0;
c0c050c5 592
89d0a06c
MC		 593 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
594 page = rxr->rx_page;
595 if (!page) {
596 page = alloc_page(gfp);
597 if (!page)
598 return -ENOMEM;
599 rxr->rx_page = page;
600 rxr->rx_page_offset = 0;
601 }
602 offset = rxr->rx_page_offset;
603 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
604 if (rxr->rx_page_offset == PAGE_SIZE)
605 rxr->rx_page = NULL;
606 else
607 get_page(page);
608 } else {
609 page = alloc_page(gfp);
610 if (!page)
611 return -ENOMEM;
612 }
c0c050c5 613
89d0a06c 614 mapping = dma_map_page(&pdev->dev, page, offset, BNXT_RX_PAGE_SIZE,
c0c050c5
MC		 615 PCI_DMA_FROMDEVICE);
616 if (dma_mapping_error(&pdev->dev, mapping)) {
617 __free_page(page);
618 return -EIO;
619 }
620
621 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
622 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
623
624 __set_bit(sw_prod, rxr->rx_agg_bmap);
625 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
626 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
627
628 rx_agg_buf->page = page;
89d0a06c 629 rx_agg_buf->offset = offset;
c0c050c5
MC		 630 rx_agg_buf->mapping = mapping;
631 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
632 rxbd->rx_bd_opaque = sw_prod;
633 return 0;
634}
635
636static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons,
637 u32 agg_bufs)
638{
639 struct bnxt *bp = bnapi->bp;
640 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
b6ab4b01 641 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 642 u16 prod = rxr->rx_agg_prod;
643 u16 sw_prod = rxr->rx_sw_agg_prod;
644 u32 i;
645
646 for (i = 0; i < agg_bufs; i++) {
647 u16 cons;
648 struct rx_agg_cmp *agg;
649 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
650 struct rx_bd *prod_bd;
651 struct page *page;
652
653 agg = (struct rx_agg_cmp *)
654 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
655 cons = agg->rx_agg_cmp_opaque;
656 __clear_bit(cons, rxr->rx_agg_bmap);
657
658 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
659 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
660
661 __set_bit(sw_prod, rxr->rx_agg_bmap);
662 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
663 cons_rx_buf = &rxr->rx_agg_ring[cons];
664
665 /* It is possible for sw_prod to be equal to cons, so
666 * set cons_rx_buf->page to NULL first.
667 */
668 page = cons_rx_buf->page;
669 cons_rx_buf->page = NULL;
670 prod_rx_buf->page = page;
89d0a06c 671 prod_rx_buf->offset = cons_rx_buf->offset;
c0c050c5
MC		 672
673 prod_rx_buf->mapping = cons_rx_buf->mapping;
674
675 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
676
677 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
678 prod_bd->rx_bd_opaque = sw_prod;
679
680 prod = NEXT_RX_AGG(prod);
681 sw_prod = NEXT_RX_AGG(sw_prod);
682 cp_cons = NEXT_CMP(cp_cons);
683 }
684 rxr->rx_agg_prod = prod;
685 rxr->rx_sw_agg_prod = sw_prod;
686}
687
688static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
689 struct bnxt_rx_ring_info *rxr, u16 cons,
690 u16 prod, u8 *data, dma_addr_t dma_addr,
691 unsigned int len)
692{
693 int err;
694 struct sk_buff *skb;
695
696 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
697 if (unlikely(err)) {
698 bnxt_reuse_rx_data(rxr, cons, data);
699 return NULL;
700 }
701
702 skb = build_skb(data, 0);
703 dma_unmap_single(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
704 PCI_DMA_FROMDEVICE);
705 if (!skb) {
706 kfree(data);
707 return NULL;
708 }
709
710 skb_reserve(skb, BNXT_RX_OFFSET);
711 skb_put(skb, len);
712 return skb;
713}
714
715static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi,
716 struct sk_buff *skb, u16 cp_cons,
717 u32 agg_bufs)
718{
719 struct pci_dev *pdev = bp->pdev;
720 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
b6ab4b01 721 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 722 u16 prod = rxr->rx_agg_prod;
723 u32 i;
724
725 for (i = 0; i < agg_bufs; i++) {
726 u16 cons, frag_len;
727 struct rx_agg_cmp *agg;
728 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
729 struct page *page;
730 dma_addr_t mapping;
731
732 agg = (struct rx_agg_cmp *)
733 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
734 cons = agg->rx_agg_cmp_opaque;
735 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
736 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
737
738 cons_rx_buf = &rxr->rx_agg_ring[cons];
89d0a06c
MC		 739 skb_fill_page_desc(skb, i, cons_rx_buf->page,
740 cons_rx_buf->offset, frag_len);
c0c050c5
MC		 741 __clear_bit(cons, rxr->rx_agg_bmap);
742
743 /* It is possible for bnxt_alloc_rx_page() to allocate
744 * a sw_prod index that equals the cons index, so we
745 * need to clear the cons entry now.
746 */
747 mapping = dma_unmap_addr(cons_rx_buf, mapping);
748 page = cons_rx_buf->page;
749 cons_rx_buf->page = NULL;
750
751 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
752 struct skb_shared_info *shinfo;
753 unsigned int nr_frags;
754
755 shinfo = skb_shinfo(skb);
756 nr_frags = --shinfo->nr_frags;
757 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
758
759 dev_kfree_skb(skb);
760
761 cons_rx_buf->page = page;
762
763 /* Update prod since possibly some pages have been
764 * allocated already.
765 */
766 rxr->rx_agg_prod = prod;
767 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i);
768 return NULL;
769 }
770
2839f28b 771 dma_unmap_page(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
c0c050c5
MC		 772 PCI_DMA_FROMDEVICE);
773
774 skb->data_len += frag_len;
775 skb->len += frag_len;
776 skb->truesize += PAGE_SIZE;
777
778 prod = NEXT_RX_AGG(prod);
779 cp_cons = NEXT_CMP(cp_cons);
780 }
781 rxr->rx_agg_prod = prod;
782 return skb;
783}
784
785static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
786 u8 agg_bufs, u32 *raw_cons)
787{
788 u16 last;
789 struct rx_agg_cmp *agg;
790
791 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
792 last = RING_CMP(*raw_cons);
793 agg = (struct rx_agg_cmp *)
794 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
795 return RX_AGG_CMP_VALID(agg, *raw_cons);
796}
797
798static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
799 unsigned int len,
800 dma_addr_t mapping)
801{
802 struct bnxt *bp = bnapi->bp;
803 struct pci_dev *pdev = bp->pdev;
804 struct sk_buff *skb;
805
806 skb = napi_alloc_skb(&bnapi->napi, len);
807 if (!skb)
808 return NULL;
809
810 dma_sync_single_for_cpu(&pdev->dev, mapping,
811 bp->rx_copy_thresh, PCI_DMA_FROMDEVICE);
812
813 memcpy(skb->data - BNXT_RX_OFFSET, data, len + BNXT_RX_OFFSET);
814
815 dma_sync_single_for_device(&pdev->dev, mapping,
816 bp->rx_copy_thresh,
817 PCI_DMA_FROMDEVICE);
818
819 skb_put(skb, len);
820 return skb;
821}
822
fa7e2812
MC		 823static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi,
824 u32 *raw_cons, void *cmp)
825{
826 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
827 struct rx_cmp *rxcmp = cmp;
828 u32 tmp_raw_cons = *raw_cons;
829 u8 cmp_type, agg_bufs = 0;
830
831 cmp_type = RX_CMP_TYPE(rxcmp);
832
833 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
834 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
835 RX_CMP_AGG_BUFS) >>
836 RX_CMP_AGG_BUFS_SHIFT;
837 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
838 struct rx_tpa_end_cmp *tpa_end = cmp;
839
840 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
841 RX_TPA_END_CMP_AGG_BUFS) >>
842 RX_TPA_END_CMP_AGG_BUFS_SHIFT;
843 }
844
845 if (agg_bufs) {
846 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
847 return -EBUSY;
848 }
849 *raw_cons = tmp_raw_cons;
850 return 0;
851}
852
853static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
854{
855 if (!rxr->bnapi->in_reset) {
856 rxr->bnapi->in_reset = true;
857 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
858 schedule_work(&bp->sp_task);
859 }
860 rxr->rx_next_cons = 0xffff;
861}
862
c0c050c5
MC		 863static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
864 struct rx_tpa_start_cmp *tpa_start,
865 struct rx_tpa_start_cmp_ext *tpa_start1)
866{
867 u8 agg_id = TPA_START_AGG_ID(tpa_start);
868 u16 cons, prod;
869 struct bnxt_tpa_info *tpa_info;
870 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
871 struct rx_bd *prod_bd;
872 dma_addr_t mapping;
873
874 cons = tpa_start->rx_tpa_start_cmp_opaque;
875 prod = rxr->rx_prod;
876 cons_rx_buf = &rxr->rx_buf_ring[cons];
877 prod_rx_buf = &rxr->rx_buf_ring[prod];
878 tpa_info = &rxr->rx_tpa[agg_id];
879
fa7e2812
MC		 880 if (unlikely(cons != rxr->rx_next_cons)) {
881 bnxt_sched_reset(bp, rxr);
882 return;
883 }
884
c0c050c5
MC		 885 prod_rx_buf->data = tpa_info->data;
886
887 mapping = tpa_info->mapping;
888 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
889
890 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
891
892 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
893
894 tpa_info->data = cons_rx_buf->data;
895 cons_rx_buf->data = NULL;
896 tpa_info->mapping = dma_unmap_addr(cons_rx_buf, mapping);
897
898 tpa_info->len =
899 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
900 RX_TPA_START_CMP_LEN_SHIFT;
901 if (likely(TPA_START_HASH_VALID(tpa_start))) {
902 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
903
904 tpa_info->hash_type = PKT_HASH_TYPE_L4;
905 tpa_info->gso_type = SKB_GSO_TCPV4;
906 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
907 if (hash_type == 3)
908 tpa_info->gso_type = SKB_GSO_TCPV6;
909 tpa_info->rss_hash =
910 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
911 } else {
912 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
913 tpa_info->gso_type = 0;
914 if (netif_msg_rx_err(bp))
915 netdev_warn(bp->dev, "TPA packet without valid hash\n");
916 }
917 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
918 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
919
920 rxr->rx_prod = NEXT_RX(prod);
921 cons = NEXT_RX(cons);
376a5b86 922 rxr->rx_next_cons = NEXT_RX(cons);
c0c050c5
MC		 923 cons_rx_buf = &rxr->rx_buf_ring[cons];
924
925 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
926 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
927 cons_rx_buf->data = NULL;
928}
929
930static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi,
931 u16 cp_cons, u32 agg_bufs)
932{
933 if (agg_bufs)
934 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
935}
936
937#define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
938#define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
939
940static inline struct sk_buff *bnxt_gro_skb(struct bnxt_tpa_info *tpa_info,
941 struct rx_tpa_end_cmp *tpa_end,
942 struct rx_tpa_end_cmp_ext *tpa_end1,
943 struct sk_buff *skb)
944{
d1611c3a 945#ifdef CONFIG_INET
c0c050c5
MC		 946 struct tcphdr *th;
947 int payload_off, tcp_opt_len = 0;
948 int len, nw_off;
27e24189 949 u16 segs;
c0c050c5 950
27e24189
MC		 951 segs = TPA_END_TPA_SEGS(tpa_end);
952 if (segs == 1)
953 return skb;
954
955 NAPI_GRO_CB(skb)->count = segs;
c0c050c5
MC		 956 skb_shinfo(skb)->gso_size =
957 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
958 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
959 payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
960 RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
961 RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
962 if (TPA_END_GRO_TS(tpa_end))
963 tcp_opt_len = 12;
964
c0c050c5
MC		 965 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
966 struct iphdr *iph;
967
968 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
969 ETH_HLEN;
970 skb_set_network_header(skb, nw_off);
971 iph = ip_hdr(skb);
972 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
973 len = skb->len - skb_transport_offset(skb);
974 th = tcp_hdr(skb);
975 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
976 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
977 struct ipv6hdr *iph;
978
979 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
980 ETH_HLEN;
981 skb_set_network_header(skb, nw_off);
982 iph = ipv6_hdr(skb);
983 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
984 len = skb->len - skb_transport_offset(skb);
985 th = tcp_hdr(skb);
986 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
987 } else {
988 dev_kfree_skb_any(skb);
989 return NULL;
990 }
991 tcp_gro_complete(skb);
992
993 if (nw_off) { /* tunnel */
994 struct udphdr *uh = NULL;
995
996 if (skb->protocol == htons(ETH_P_IP)) {
997 struct iphdr *iph = (struct iphdr *)skb->data;
998
999 if (iph->protocol == IPPROTO_UDP)
1000 uh = (struct udphdr *)(iph + 1);
1001 } else {
1002 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1003
1004 if (iph->nexthdr == IPPROTO_UDP)
1005 uh = (struct udphdr *)(iph + 1);
1006 }
1007 if (uh) {
1008 if (uh->check)
1009 skb_shinfo(skb)->gso_type |=
1010 SKB_GSO_UDP_TUNNEL_CSUM;
1011 else
1012 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1013 }
1014 }
1015#endif
1016 return skb;
1017}
1018
1019static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1020 struct bnxt_napi *bnapi,
1021 u32 *raw_cons,
1022 struct rx_tpa_end_cmp *tpa_end,
1023 struct rx_tpa_end_cmp_ext *tpa_end1,
1024 bool *agg_event)
1025{
1026 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
b6ab4b01 1027 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 1028 u8 agg_id = TPA_END_AGG_ID(tpa_end);
1029 u8 *data, agg_bufs;
1030 u16 cp_cons = RING_CMP(*raw_cons);
1031 unsigned int len;
1032 struct bnxt_tpa_info *tpa_info;
1033 dma_addr_t mapping;
1034 struct sk_buff *skb;
1035
fa7e2812
MC		 1036 if (unlikely(bnapi->in_reset)) {
1037 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end);
1038
1039 if (rc < 0)
1040 return ERR_PTR(-EBUSY);
1041 return NULL;
1042 }
1043
c0c050c5
MC		 1044 tpa_info = &rxr->rx_tpa[agg_id];
1045 data = tpa_info->data;
1046 prefetch(data);
1047 len = tpa_info->len;
1048 mapping = tpa_info->mapping;
1049
1050 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1051 RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1052
1053 if (agg_bufs) {
1054 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1055 return ERR_PTR(-EBUSY);
1056
1057 *agg_event = true;
1058 cp_cons = NEXT_CMP(cp_cons);
1059 }
1060
1061 if (unlikely(agg_bufs > MAX_SKB_FRAGS)) {
1062 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1063 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1064 agg_bufs, (int)MAX_SKB_FRAGS);
1065 return NULL;
1066 }
1067
1068 if (len <= bp->rx_copy_thresh) {
1069 skb = bnxt_copy_skb(bnapi, data, len, mapping);
1070 if (!skb) {
1071 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1072 return NULL;
1073 }
1074 } else {
1075 u8 *new_data;
1076 dma_addr_t new_mapping;
1077
1078 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1079 if (!new_data) {
1080 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1081 return NULL;
1082 }
1083
1084 tpa_info->data = new_data;
1085 tpa_info->mapping = new_mapping;
1086
1087 skb = build_skb(data, 0);
1088 dma_unmap_single(&bp->pdev->dev, mapping, bp->rx_buf_use_size,
1089 PCI_DMA_FROMDEVICE);
1090
1091 if (!skb) {
1092 kfree(data);
1093 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1094 return NULL;
1095 }
1096 skb_reserve(skb, BNXT_RX_OFFSET);
1097 skb_put(skb, len);
1098 }
1099
1100 if (agg_bufs) {
1101 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1102 if (!skb) {
1103 /* Page reuse already handled by bnxt_rx_pages(). */
1104 return NULL;
1105 }
1106 }
1107 skb->protocol = eth_type_trans(skb, bp->dev);
1108
1109 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1110 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1111
1112 if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) {
1113 netdev_features_t features = skb->dev->features;
1114 u16 vlan_proto = tpa_info->metadata >>
1115 RX_CMP_FLAGS2_METADATA_TPID_SFT;
1116
1117 if (((features & NETIF_F_HW_VLAN_CTAG_RX) &&
1118 vlan_proto == ETH_P_8021Q) ||
1119 ((features & NETIF_F_HW_VLAN_STAG_RX) &&
1120 vlan_proto == ETH_P_8021AD)) {
1121 __vlan_hwaccel_put_tag(skb, htons(vlan_proto),
1122 tpa_info->metadata &
1123 RX_CMP_FLAGS2_METADATA_VID_MASK);
1124 }
1125 }
1126
1127 skb_checksum_none_assert(skb);
1128 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1129 skb->ip_summed = CHECKSUM_UNNECESSARY;
1130 skb->csum_level =
1131 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1132 }
1133
1134 if (TPA_END_GRO(tpa_end))
1135 skb = bnxt_gro_skb(tpa_info, tpa_end, tpa_end1, skb);
1136
1137 return skb;
1138}
1139
1140/* returns the following:
1141 * 1 - 1 packet successfully received
1142 * 0 - successful TPA_START, packet not completed yet
1143 * -EBUSY - completion ring does not have all the agg buffers yet
1144 * -ENOMEM - packet aborted due to out of memory
1145 * -EIO - packet aborted due to hw error indicated in BD
1146 */
1147static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
1148 bool *agg_event)
1149{
1150 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
b6ab4b01 1151 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 1152 struct net_device *dev = bp->dev;
1153 struct rx_cmp *rxcmp;
1154 struct rx_cmp_ext *rxcmp1;
1155 u32 tmp_raw_cons = *raw_cons;
1156 u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1157 struct bnxt_sw_rx_bd *rx_buf;
1158 unsigned int len;
1159 u8 *data, agg_bufs, cmp_type;
1160 dma_addr_t dma_addr;
1161 struct sk_buff *skb;
1162 int rc = 0;
1163
1164 rxcmp = (struct rx_cmp *)
1165 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1166
1167 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1168 cp_cons = RING_CMP(tmp_raw_cons);
1169 rxcmp1 = (struct rx_cmp_ext *)
1170 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1171
1172 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1173 return -EBUSY;
1174
1175 cmp_type = RX_CMP_TYPE(rxcmp);
1176
1177 prod = rxr->rx_prod;
1178
1179 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1180 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1181 (struct rx_tpa_start_cmp_ext *)rxcmp1);
1182
1183 goto next_rx_no_prod;
1184
1185 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1186 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons,
1187 (struct rx_tpa_end_cmp *)rxcmp,
1188 (struct rx_tpa_end_cmp_ext *)rxcmp1,
1189 agg_event);
1190
1191 if (unlikely(IS_ERR(skb)))
1192 return -EBUSY;
1193
1194 rc = -ENOMEM;
1195 if (likely(skb)) {
1196 skb_record_rx_queue(skb, bnapi->index);
1197 skb_mark_napi_id(skb, &bnapi->napi);
1198 if (bnxt_busy_polling(bnapi))
1199 netif_receive_skb(skb);
1200 else
1201 napi_gro_receive(&bnapi->napi, skb);
1202 rc = 1;
1203 }
1204 goto next_rx_no_prod;
1205 }
1206
1207 cons = rxcmp->rx_cmp_opaque;
1208 rx_buf = &rxr->rx_buf_ring[cons];
1209 data = rx_buf->data;
fa7e2812
MC		 1210 if (unlikely(cons != rxr->rx_next_cons)) {
1211 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp);
1212
1213 bnxt_sched_reset(bp, rxr);
1214 return rc1;
1215 }
c0c050c5
MC		 1216 prefetch(data);
1217
1218 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & RX_CMP_AGG_BUFS) >>
1219 RX_CMP_AGG_BUFS_SHIFT;
1220
1221 if (agg_bufs) {
1222 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1223 return -EBUSY;
1224
1225 cp_cons = NEXT_CMP(cp_cons);
1226 *agg_event = true;
1227 }
1228
1229 rx_buf->data = NULL;
1230 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1231 bnxt_reuse_rx_data(rxr, cons, data);
1232 if (agg_bufs)
1233 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1234
1235 rc = -EIO;
1236 goto next_rx;
1237 }
1238
1239 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1240 dma_addr = dma_unmap_addr(rx_buf, mapping);
1241
1242 if (len <= bp->rx_copy_thresh) {
1243 skb = bnxt_copy_skb(bnapi, data, len, dma_addr);
1244 bnxt_reuse_rx_data(rxr, cons, data);
1245 if (!skb) {
1246 rc = -ENOMEM;
1247 goto next_rx;
1248 }
1249 } else {
1250 skb = bnxt_rx_skb(bp, rxr, cons, prod, data, dma_addr, len);
1251 if (!skb) {
1252 rc = -ENOMEM;
1253 goto next_rx;
1254 }
1255 }
1256
1257 if (agg_bufs) {
1258 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1259 if (!skb) {
1260 rc = -ENOMEM;
1261 goto next_rx;
1262 }
1263 }
1264
1265 if (RX_CMP_HASH_VALID(rxcmp)) {
1266 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1267 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1268
1269 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1270 if (hash_type != 1 && hash_type != 3)
1271 type = PKT_HASH_TYPE_L3;
1272 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1273 }
1274
1275 skb->protocol = eth_type_trans(skb, dev);
1276
1277 if (rxcmp1->rx_cmp_flags2 &
1278 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) {
1279 netdev_features_t features = skb->dev->features;
1280 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1281 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1282
1283 if (((features & NETIF_F_HW_VLAN_CTAG_RX) &&
1284 vlan_proto == ETH_P_8021Q) ||
1285 ((features & NETIF_F_HW_VLAN_STAG_RX) &&
1286 vlan_proto == ETH_P_8021AD))
1287 __vlan_hwaccel_put_tag(skb, htons(vlan_proto),
1288 meta_data &
1289 RX_CMP_FLAGS2_METADATA_VID_MASK);
1290 }
1291
1292 skb_checksum_none_assert(skb);
1293 if (RX_CMP_L4_CS_OK(rxcmp1)) {
1294 if (dev->features & NETIF_F_RXCSUM) {
1295 skb->ip_summed = CHECKSUM_UNNECESSARY;
1296 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1297 }
1298 } else {
665e350d
SB		 1299 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1300 if (dev->features & NETIF_F_RXCSUM)
1301 cpr->rx_l4_csum_errors++;
1302 }
c0c050c5
MC		 1303 }
1304
1305 skb_record_rx_queue(skb, bnapi->index);
1306 skb_mark_napi_id(skb, &bnapi->napi);
1307 if (bnxt_busy_polling(bnapi))
1308 netif_receive_skb(skb);
1309 else
1310 napi_gro_receive(&bnapi->napi, skb);
1311 rc = 1;
1312
1313next_rx:
1314 rxr->rx_prod = NEXT_RX(prod);
376a5b86 1315 rxr->rx_next_cons = NEXT_RX(cons);
c0c050c5
MC		 1316
1317next_rx_no_prod:
1318 *raw_cons = tmp_raw_cons;
1319
1320 return rc;
1321}
1322
4bb13abf
MC		 1323#define BNXT_GET_EVENT_PORT(data) \
1324 ((data) & \
1325 HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1326
1327#define BNXT_EVENT_POLICY_MASK \
1328 HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_MASK
1329
1330#define BNXT_EVENT_POLICY_SFT \
1331 HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_SFT
1332
1333#define BNXT_GET_EVENT_POLICY(data) \
1334 (((data) & BNXT_EVENT_POLICY_MASK) >> BNXT_EVENT_POLICY_SFT)
1335
c0c050c5
MC		 1336static int bnxt_async_event_process(struct bnxt *bp,
1337 struct hwrm_async_event_cmpl *cmpl)
1338{
1339 u16 event_id = le16_to_cpu(cmpl->event_id);
1340
1341 /* TODO CHIMP_FW: Define event id's for link change, error etc */
1342 switch (event_id) {
8cbde117
MC		 1343 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1344 u32 data1 = le32_to_cpu(cmpl->event_data1);
1345 struct bnxt_link_info *link_info = &bp->link_info;
1346
1347 if (BNXT_VF(bp))
1348 goto async_event_process_exit;
1349 if (data1 & 0x20000) {
1350 u16 fw_speed = link_info->force_link_speed;
1351 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1352
1353 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1354 speed);
1355 }
1356 /* fall thru */
1357 }
c0c050c5
MC		 1358 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1359 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
19241368
JH		 1360 break;
1361 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1362 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
c0c050c5 1363 break;
4bb13abf
MC		 1364 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1365 u32 data1 = le32_to_cpu(cmpl->event_data1);
1366 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1367
1368 if (BNXT_VF(bp))
1369 break;
1370
1371 if (bp->pf.port_id != port_id)
1372 break;
1373
1374 bp->link_info.last_port_module_event =
1375 BNXT_GET_EVENT_POLICY(data1);
1376
1377 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1378 break;
1379 }
c0c050c5
MC		 1380 default:
1381 netdev_err(bp->dev, "unhandled ASYNC event (id 0x%x)\n",
1382 event_id);
19241368 1383 goto async_event_process_exit;
c0c050c5 1384 }
19241368
JH		 1385 schedule_work(&bp->sp_task);
1386async_event_process_exit:
c0c050c5
MC		 1387 return 0;
1388}
1389
1390static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1391{
1392 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1393 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1394 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1395 (struct hwrm_fwd_req_cmpl *)txcmp;
1396
1397 switch (cmpl_type) {
1398 case CMPL_BASE_TYPE_HWRM_DONE:
1399 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1400 if (seq_id == bp->hwrm_intr_seq_id)
1401 bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID;
1402 else
1403 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1404 break;
1405
1406 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1407 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1408
1409 if ((vf_id < bp->pf.first_vf_id) ||
1410 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1411 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1412 vf_id);
1413 return -EINVAL;
1414 }
1415
1416 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1417 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
1418 schedule_work(&bp->sp_task);
1419 break;
1420
1421 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1422 bnxt_async_event_process(bp,
1423 (struct hwrm_async_event_cmpl *)txcmp);
1424
1425 default:
1426 break;
1427 }
1428
1429 return 0;
1430}
1431
1432static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1433{
1434 struct bnxt_napi *bnapi = dev_instance;
1435 struct bnxt *bp = bnapi->bp;
1436 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1437 u32 cons = RING_CMP(cpr->cp_raw_cons);
1438
1439 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1440 napi_schedule(&bnapi->napi);
1441 return IRQ_HANDLED;
1442}
1443
1444static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1445{
1446 u32 raw_cons = cpr->cp_raw_cons;
1447 u16 cons = RING_CMP(raw_cons);
1448 struct tx_cmp *txcmp;
1449
1450 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1451
1452 return TX_CMP_VALID(txcmp, raw_cons);
1453}
1454
c0c050c5
MC		 1455static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1456{
1457 struct bnxt_napi *bnapi = dev_instance;
1458 struct bnxt *bp = bnapi->bp;
1459 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1460 u32 cons = RING_CMP(cpr->cp_raw_cons);
1461 u32 int_status;
1462
1463 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1464
1465 if (!bnxt_has_work(bp, cpr)) {
11809490 1466 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
c0c050c5
MC		 1467 /* return if erroneous interrupt */
1468 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1469 return IRQ_NONE;
1470 }
1471
1472 /* disable ring IRQ */
1473 BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell);
1474
1475 /* Return here if interrupt is shared and is disabled. */
1476 if (unlikely(atomic_read(&bp->intr_sem) != 0))
1477 return IRQ_HANDLED;
1478
1479 napi_schedule(&bnapi->napi);
1480 return IRQ_HANDLED;
1481}
1482
1483static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
1484{
1485 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1486 u32 raw_cons = cpr->cp_raw_cons;
1487 u32 cons;
1488 int tx_pkts = 0;
1489 int rx_pkts = 0;
1490 bool rx_event = false;
1491 bool agg_event = false;
1492 struct tx_cmp *txcmp;
1493
1494 while (1) {
1495 int rc;
1496
1497 cons = RING_CMP(raw_cons);
1498 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1499
1500 if (!TX_CMP_VALID(txcmp, raw_cons))
1501 break;
1502
67a95e20
MC		 1503 /* The valid test of the entry must be done first before
1504 * reading any further.
1505 */
1506 rmb();
c0c050c5
MC		 1507 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
1508 tx_pkts++;
1509 /* return full budget so NAPI will complete. */
1510 if (unlikely(tx_pkts > bp->tx_wake_thresh))
1511 rx_pkts = budget;
1512 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1513 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &agg_event);
1514 if (likely(rc >= 0))
1515 rx_pkts += rc;
1516 else if (rc == -EBUSY) /* partial completion */
1517 break;
1518 rx_event = true;
1519 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
1520 CMPL_BASE_TYPE_HWRM_DONE) ||
1521 (TX_CMP_TYPE(txcmp) ==
1522 CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
1523 (TX_CMP_TYPE(txcmp) ==
1524 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
1525 bnxt_hwrm_handler(bp, txcmp);
1526 }
1527 raw_cons = NEXT_RAW_CMP(raw_cons);
1528
1529 if (rx_pkts == budget)
1530 break;
1531 }
1532
1533 cpr->cp_raw_cons = raw_cons;
1534 /* ACK completion ring before freeing tx ring and producing new
1535 * buffers in rx/agg rings to prevent overflowing the completion
1536 * ring.
1537 */
1538 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
1539
1540 if (tx_pkts)
1541 bnxt_tx_int(bp, bnapi, tx_pkts);
1542
1543 if (rx_event) {
b6ab4b01 1544 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c0c050c5
MC		 1545
1546 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
1547 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
1548 if (agg_event) {
1549 writel(DB_KEY_RX | rxr->rx_agg_prod,
1550 rxr->rx_agg_doorbell);
1551 writel(DB_KEY_RX | rxr->rx_agg_prod,
1552 rxr->rx_agg_doorbell);
1553 }
1554 }
1555 return rx_pkts;
1556}
1557
1558static int bnxt_poll(struct napi_struct *napi, int budget)
1559{
1560 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1561 struct bnxt *bp = bnapi->bp;
1562 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1563 int work_done = 0;
1564
1565 if (!bnxt_lock_napi(bnapi))
1566 return budget;
1567
1568 while (1) {
1569 work_done += bnxt_poll_work(bp, bnapi, budget - work_done);
1570
1571 if (work_done >= budget)
1572 break;
1573
1574 if (!bnxt_has_work(bp, cpr)) {
1575 napi_complete(napi);
1576 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1577 break;
1578 }
1579 }
1580 mmiowb();
1581 bnxt_unlock_napi(bnapi);
1582 return work_done;
1583}
1584
1585#ifdef CONFIG_NET_RX_BUSY_POLL
1586static int bnxt_busy_poll(struct napi_struct *napi)
1587{
1588 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1589 struct bnxt *bp = bnapi->bp;
1590 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1591 int rx_work, budget = 4;
1592
1593 if (atomic_read(&bp->intr_sem) != 0)
1594 return LL_FLUSH_FAILED;
1595
1596 if (!bnxt_lock_poll(bnapi))
1597 return LL_FLUSH_BUSY;
1598
1599 rx_work = bnxt_poll_work(bp, bnapi, budget);
1600
1601 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1602
1603 bnxt_unlock_poll(bnapi);
1604 return rx_work;
1605}
1606#endif
1607
1608static void bnxt_free_tx_skbs(struct bnxt *bp)
1609{
1610 int i, max_idx;
1611 struct pci_dev *pdev = bp->pdev;
1612
b6ab4b01 1613 if (!bp->tx_ring)
c0c050c5
MC		 1614 return;
1615
1616 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
1617 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 1618 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 1619 int j;
1620
c0c050c5
MC		 1621 for (j = 0; j < max_idx;) {
1622 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
1623 struct sk_buff *skb = tx_buf->skb;
1624 int k, last;
1625
1626 if (!skb) {
1627 j++;
1628 continue;
1629 }
1630
1631 tx_buf->skb = NULL;
1632
1633 if (tx_buf->is_push) {
1634 dev_kfree_skb(skb);
1635 j += 2;
1636 continue;
1637 }
1638
1639 dma_unmap_single(&pdev->dev,
1640 dma_unmap_addr(tx_buf, mapping),
1641 skb_headlen(skb),
1642 PCI_DMA_TODEVICE);
1643
1644 last = tx_buf->nr_frags;
1645 j += 2;
d612a579
MC		 1646 for (k = 0; k < last; k++, j++) {
1647 int ring_idx = j & bp->tx_ring_mask;
c0c050c5
MC		 1648 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
1649
d612a579 1650 tx_buf = &txr->tx_buf_ring[ring_idx];
c0c050c5
MC		 1651 dma_unmap_page(
1652 &pdev->dev,
1653 dma_unmap_addr(tx_buf, mapping),
1654 skb_frag_size(frag), PCI_DMA_TODEVICE);
1655 }
1656 dev_kfree_skb(skb);
1657 }
1658 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
1659 }
1660}
1661
1662static void bnxt_free_rx_skbs(struct bnxt *bp)
1663{
1664 int i, max_idx, max_agg_idx;
1665 struct pci_dev *pdev = bp->pdev;
1666
b6ab4b01 1667 if (!bp->rx_ring)
c0c050c5
MC		 1668 return;
1669
1670 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
1671 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
1672 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 1673 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 1674 int j;
1675
c0c050c5
MC		 1676 if (rxr->rx_tpa) {
1677 for (j = 0; j < MAX_TPA; j++) {
1678 struct bnxt_tpa_info *tpa_info =
1679 &rxr->rx_tpa[j];
1680 u8 *data = tpa_info->data;
1681
1682 if (!data)
1683 continue;
1684
1685 dma_unmap_single(
1686 &pdev->dev,
1687 dma_unmap_addr(tpa_info, mapping),
1688 bp->rx_buf_use_size,
1689 PCI_DMA_FROMDEVICE);
1690
1691 tpa_info->data = NULL;
1692
1693 kfree(data);
1694 }
1695 }
1696
1697 for (j = 0; j < max_idx; j++) {
1698 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
1699 u8 *data = rx_buf->data;
1700
1701 if (!data)
1702 continue;
1703
1704 dma_unmap_single(&pdev->dev,
1705 dma_unmap_addr(rx_buf, mapping),
1706 bp->rx_buf_use_size,
1707 PCI_DMA_FROMDEVICE);
1708
1709 rx_buf->data = NULL;
1710
1711 kfree(data);
1712 }
1713
1714 for (j = 0; j < max_agg_idx; j++) {
1715 struct bnxt_sw_rx_agg_bd *rx_agg_buf =
1716 &rxr->rx_agg_ring[j];
1717 struct page *page = rx_agg_buf->page;
1718
1719 if (!page)
1720 continue;
1721
1722 dma_unmap_page(&pdev->dev,
1723 dma_unmap_addr(rx_agg_buf, mapping),
2839f28b 1724 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE);
c0c050c5
MC		 1725
1726 rx_agg_buf->page = NULL;
1727 __clear_bit(j, rxr->rx_agg_bmap);
1728
1729 __free_page(page);
1730 }
89d0a06c
MC		 1731 if (rxr->rx_page) {
1732 __free_page(rxr->rx_page);
1733 rxr->rx_page = NULL;
1734 }
c0c050c5
MC		 1735 }
1736}
1737
1738static void bnxt_free_skbs(struct bnxt *bp)
1739{
1740 bnxt_free_tx_skbs(bp);
1741 bnxt_free_rx_skbs(bp);
1742}
1743
1744static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
1745{
1746 struct pci_dev *pdev = bp->pdev;
1747 int i;
1748
1749 for (i = 0; i < ring->nr_pages; i++) {
1750 if (!ring->pg_arr[i])
1751 continue;
1752
1753 dma_free_coherent(&pdev->dev, ring->page_size,
1754 ring->pg_arr[i], ring->dma_arr[i]);
1755
1756 ring->pg_arr[i] = NULL;
1757 }
1758 if (ring->pg_tbl) {
1759 dma_free_coherent(&pdev->dev, ring->nr_pages * 8,
1760 ring->pg_tbl, ring->pg_tbl_map);
1761 ring->pg_tbl = NULL;
1762 }
1763 if (ring->vmem_size && *ring->vmem) {
1764 vfree(*ring->vmem);
1765 *ring->vmem = NULL;
1766 }
1767}
1768
1769static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
1770{
1771 int i;
1772 struct pci_dev *pdev = bp->pdev;
1773
1774 if (ring->nr_pages > 1) {
1775 ring->pg_tbl = dma_alloc_coherent(&pdev->dev,
1776 ring->nr_pages * 8,
1777 &ring->pg_tbl_map,
1778 GFP_KERNEL);
1779 if (!ring->pg_tbl)
1780 return -ENOMEM;
1781 }
1782
1783 for (i = 0; i < ring->nr_pages; i++) {
1784 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
1785 ring->page_size,
1786 &ring->dma_arr[i],
1787 GFP_KERNEL);
1788 if (!ring->pg_arr[i])
1789 return -ENOMEM;
1790
1791 if (ring->nr_pages > 1)
1792 ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]);
1793 }
1794
1795 if (ring->vmem_size) {
1796 *ring->vmem = vzalloc(ring->vmem_size);
1797 if (!(*ring->vmem))
1798 return -ENOMEM;
1799 }
1800 return 0;
1801}
1802
1803static void bnxt_free_rx_rings(struct bnxt *bp)
1804{
1805 int i;
1806
b6ab4b01 1807 if (!bp->rx_ring)
c0c050c5
MC		 1808 return;
1809
1810 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 1811 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 1812 struct bnxt_ring_struct *ring;
1813
c0c050c5
MC		 1814 kfree(rxr->rx_tpa);
1815 rxr->rx_tpa = NULL;
1816
1817 kfree(rxr->rx_agg_bmap);
1818 rxr->rx_agg_bmap = NULL;
1819
1820 ring = &rxr->rx_ring_struct;
1821 bnxt_free_ring(bp, ring);
1822
1823 ring = &rxr->rx_agg_ring_struct;
1824 bnxt_free_ring(bp, ring);
1825 }
1826}
1827
1828static int bnxt_alloc_rx_rings(struct bnxt *bp)
1829{
1830 int i, rc, agg_rings = 0, tpa_rings = 0;
1831
b6ab4b01
MC		 1832 if (!bp->rx_ring)
1833 return -ENOMEM;
1834
c0c050c5
MC		 1835 if (bp->flags & BNXT_FLAG_AGG_RINGS)
1836 agg_rings = 1;
1837
1838 if (bp->flags & BNXT_FLAG_TPA)
1839 tpa_rings = 1;
1840
1841 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 1842 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 1843 struct bnxt_ring_struct *ring;
1844
c0c050c5
MC		 1845 ring = &rxr->rx_ring_struct;
1846
1847 rc = bnxt_alloc_ring(bp, ring);
1848 if (rc)
1849 return rc;
1850
1851 if (agg_rings) {
1852 u16 mem_size;
1853
1854 ring = &rxr->rx_agg_ring_struct;
1855 rc = bnxt_alloc_ring(bp, ring);
1856 if (rc)
1857 return rc;
1858
1859 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
1860 mem_size = rxr->rx_agg_bmap_size / 8;
1861 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
1862 if (!rxr->rx_agg_bmap)
1863 return -ENOMEM;
1864
1865 if (tpa_rings) {
1866 rxr->rx_tpa = kcalloc(MAX_TPA,
1867 sizeof(struct bnxt_tpa_info),
1868 GFP_KERNEL);
1869 if (!rxr->rx_tpa)
1870 return -ENOMEM;
1871 }
1872 }
1873 }
1874 return 0;
1875}
1876
1877static void bnxt_free_tx_rings(struct bnxt *bp)
1878{
1879 int i;
1880 struct pci_dev *pdev = bp->pdev;
1881
b6ab4b01 1882 if (!bp->tx_ring)
c0c050c5
MC		 1883 return;
1884
1885 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 1886 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 1887 struct bnxt_ring_struct *ring;
1888
c0c050c5
MC		 1889 if (txr->tx_push) {
1890 dma_free_coherent(&pdev->dev, bp->tx_push_size,
1891 txr->tx_push, txr->tx_push_mapping);
1892 txr->tx_push = NULL;
1893 }
1894
1895 ring = &txr->tx_ring_struct;
1896
1897 bnxt_free_ring(bp, ring);
1898 }
1899}
1900
1901static int bnxt_alloc_tx_rings(struct bnxt *bp)
1902{
1903 int i, j, rc;
1904 struct pci_dev *pdev = bp->pdev;
1905
1906 bp->tx_push_size = 0;
1907 if (bp->tx_push_thresh) {
1908 int push_size;
1909
1910 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
1911 bp->tx_push_thresh);
1912
4419dbe6 1913 if (push_size > 256) {
c0c050c5
MC		 1914 push_size = 0;
1915 bp->tx_push_thresh = 0;
1916 }
1917
1918 bp->tx_push_size = push_size;
1919 }
1920
1921 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 1922 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 1923 struct bnxt_ring_struct *ring;
1924
c0c050c5
MC		 1925 ring = &txr->tx_ring_struct;
1926
1927 rc = bnxt_alloc_ring(bp, ring);
1928 if (rc)
1929 return rc;
1930
1931 if (bp->tx_push_size) {
c0c050c5
MC		 1932 dma_addr_t mapping;
1933
1934 /* One pre-allocated DMA buffer to backup
1935 * TX push operation
1936 */
1937 txr->tx_push = dma_alloc_coherent(&pdev->dev,
1938 bp->tx_push_size,
1939 &txr->tx_push_mapping,
1940 GFP_KERNEL);
1941
1942 if (!txr->tx_push)
1943 return -ENOMEM;
1944
c0c050c5
MC		 1945 mapping = txr->tx_push_mapping +
1946 sizeof(struct tx_push_bd);
4419dbe6 1947 txr->data_mapping = cpu_to_le64(mapping);
c0c050c5 1948
4419dbe6 1949 memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
c0c050c5
MC		 1950 }
1951 ring->queue_id = bp->q_info[j].queue_id;
1952 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
1953 j++;
1954 }
1955 return 0;
1956}
1957
1958static void bnxt_free_cp_rings(struct bnxt *bp)
1959{
1960 int i;
1961
1962 if (!bp->bnapi)
1963 return;
1964
1965 for (i = 0; i < bp->cp_nr_rings; i++) {
1966 struct bnxt_napi *bnapi = bp->bnapi[i];
1967 struct bnxt_cp_ring_info *cpr;
1968 struct bnxt_ring_struct *ring;
1969
1970 if (!bnapi)
1971 continue;
1972
1973 cpr = &bnapi->cp_ring;
1974 ring = &cpr->cp_ring_struct;
1975
1976 bnxt_free_ring(bp, ring);
1977 }
1978}
1979
1980static int bnxt_alloc_cp_rings(struct bnxt *bp)
1981{
1982 int i, rc;
1983
1984 for (i = 0; i < bp->cp_nr_rings; i++) {
1985 struct bnxt_napi *bnapi = bp->bnapi[i];
1986 struct bnxt_cp_ring_info *cpr;
1987 struct bnxt_ring_struct *ring;
1988
1989 if (!bnapi)
1990 continue;
1991
1992 cpr = &bnapi->cp_ring;
1993 ring = &cpr->cp_ring_struct;
1994
1995 rc = bnxt_alloc_ring(bp, ring);
1996 if (rc)
1997 return rc;
1998 }
1999 return 0;
2000}
2001
2002static void bnxt_init_ring_struct(struct bnxt *bp)
2003{
2004 int i;
2005
2006 for (i = 0; i < bp->cp_nr_rings; i++) {
2007 struct bnxt_napi *bnapi = bp->bnapi[i];
2008 struct bnxt_cp_ring_info *cpr;
2009 struct bnxt_rx_ring_info *rxr;
2010 struct bnxt_tx_ring_info *txr;
2011 struct bnxt_ring_struct *ring;
2012
2013 if (!bnapi)
2014 continue;
2015
2016 cpr = &bnapi->cp_ring;
2017 ring = &cpr->cp_ring_struct;
2018 ring->nr_pages = bp->cp_nr_pages;
2019 ring->page_size = HW_CMPD_RING_SIZE;
2020 ring->pg_arr = (void **)cpr->cp_desc_ring;
2021 ring->dma_arr = cpr->cp_desc_mapping;
2022 ring->vmem_size = 0;
2023
b6ab4b01 2024 rxr = bnapi->rx_ring;
3b2b7d9d
MC		 2025 if (!rxr)
2026 goto skip_rx;
2027
c0c050c5
MC		 2028 ring = &rxr->rx_ring_struct;
2029 ring->nr_pages = bp->rx_nr_pages;
2030 ring->page_size = HW_RXBD_RING_SIZE;
2031 ring->pg_arr = (void **)rxr->rx_desc_ring;
2032 ring->dma_arr = rxr->rx_desc_mapping;
2033 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
2034 ring->vmem = (void **)&rxr->rx_buf_ring;
2035
2036 ring = &rxr->rx_agg_ring_struct;
2037 ring->nr_pages = bp->rx_agg_nr_pages;
2038 ring->page_size = HW_RXBD_RING_SIZE;
2039 ring->pg_arr = (void **)rxr->rx_agg_desc_ring;
2040 ring->dma_arr = rxr->rx_agg_desc_mapping;
2041 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
2042 ring->vmem = (void **)&rxr->rx_agg_ring;
2043
3b2b7d9d 2044skip_rx:
b6ab4b01 2045 txr = bnapi->tx_ring;
3b2b7d9d
MC		 2046 if (!txr)
2047 continue;
2048
c0c050c5
MC		 2049 ring = &txr->tx_ring_struct;
2050 ring->nr_pages = bp->tx_nr_pages;
2051 ring->page_size = HW_RXBD_RING_SIZE;
2052 ring->pg_arr = (void **)txr->tx_desc_ring;
2053 ring->dma_arr = txr->tx_desc_mapping;
2054 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
2055 ring->vmem = (void **)&txr->tx_buf_ring;
2056 }
2057}
2058
2059static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
2060{
2061 int i;
2062 u32 prod;
2063 struct rx_bd **rx_buf_ring;
2064
2065 rx_buf_ring = (struct rx_bd **)ring->pg_arr;
2066 for (i = 0, prod = 0; i < ring->nr_pages; i++) {
2067 int j;
2068 struct rx_bd *rxbd;
2069
2070 rxbd = rx_buf_ring[i];
2071 if (!rxbd)
2072 continue;
2073
2074 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
2075 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
2076 rxbd->rx_bd_opaque = prod;
2077 }
2078 }
2079}
2080
2081static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
2082{
2083 struct net_device *dev = bp->dev;
c0c050c5
MC		 2084 struct bnxt_rx_ring_info *rxr;
2085 struct bnxt_ring_struct *ring;
2086 u32 prod, type;
2087 int i;
2088
c0c050c5
MC		 2089 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
2090 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
2091
2092 if (NET_IP_ALIGN == 2)
2093 type |= RX_BD_FLAGS_SOP;
2094
b6ab4b01 2095 rxr = &bp->rx_ring[ring_nr];
c0c050c5
MC		 2096 ring = &rxr->rx_ring_struct;
2097 bnxt_init_rxbd_pages(ring, type);
2098
2099 prod = rxr->rx_prod;
2100 for (i = 0; i < bp->rx_ring_size; i++) {
2101 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
2102 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
2103 ring_nr, i, bp->rx_ring_size);
2104 break;
2105 }
2106 prod = NEXT_RX(prod);
2107 }
2108 rxr->rx_prod = prod;
2109 ring->fw_ring_id = INVALID_HW_RING_ID;
2110
edd0c2cc
MC		 2111 ring = &rxr->rx_agg_ring_struct;
2112 ring->fw_ring_id = INVALID_HW_RING_ID;
2113
c0c050c5
MC		 2114 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
2115 return 0;
2116
2839f28b 2117 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
c0c050c5
MC		 2118 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
2119
2120 bnxt_init_rxbd_pages(ring, type);
2121
2122 prod = rxr->rx_agg_prod;
2123 for (i = 0; i < bp->rx_agg_ring_size; i++) {
2124 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
2125 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
2126 ring_nr, i, bp->rx_ring_size);
2127 break;
2128 }
2129 prod = NEXT_RX_AGG(prod);
2130 }
2131 rxr->rx_agg_prod = prod;
c0c050c5
MC		 2132
2133 if (bp->flags & BNXT_FLAG_TPA) {
2134 if (rxr->rx_tpa) {
2135 u8 *data;
2136 dma_addr_t mapping;
2137
2138 for (i = 0; i < MAX_TPA; i++) {
2139 data = __bnxt_alloc_rx_data(bp, &mapping,
2140 GFP_KERNEL);
2141 if (!data)
2142 return -ENOMEM;
2143
2144 rxr->rx_tpa[i].data = data;
2145 rxr->rx_tpa[i].mapping = mapping;
2146 }
2147 } else {
2148 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
2149 return -ENOMEM;
2150 }
2151 }
2152
2153 return 0;
2154}
2155
2156static int bnxt_init_rx_rings(struct bnxt *bp)
2157{
2158 int i, rc = 0;
2159
2160 for (i = 0; i < bp->rx_nr_rings; i++) {
2161 rc = bnxt_init_one_rx_ring(bp, i);
2162 if (rc)
2163 break;
2164 }
2165
2166 return rc;
2167}
2168
2169static int bnxt_init_tx_rings(struct bnxt *bp)
2170{
2171 u16 i;
2172
2173 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
2174 MAX_SKB_FRAGS + 1);
2175
2176 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 2177 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
c0c050c5
MC		 2178 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
2179
2180 ring->fw_ring_id = INVALID_HW_RING_ID;
2181 }
2182
2183 return 0;
2184}
2185
2186static void bnxt_free_ring_grps(struct bnxt *bp)
2187{
2188 kfree(bp->grp_info);
2189 bp->grp_info = NULL;
2190}
2191
2192static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
2193{
2194 int i;
2195
2196 if (irq_re_init) {
2197 bp->grp_info = kcalloc(bp->cp_nr_rings,
2198 sizeof(struct bnxt_ring_grp_info),
2199 GFP_KERNEL);
2200 if (!bp->grp_info)
2201 return -ENOMEM;
2202 }
2203 for (i = 0; i < bp->cp_nr_rings; i++) {
2204 if (irq_re_init)
2205 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
2206 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
2207 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
2208 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
2209 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
2210 }
2211 return 0;
2212}
2213
2214static void bnxt_free_vnics(struct bnxt *bp)
2215{
2216 kfree(bp->vnic_info);
2217 bp->vnic_info = NULL;
2218 bp->nr_vnics = 0;
2219}
2220
2221static int bnxt_alloc_vnics(struct bnxt *bp)
2222{
2223 int num_vnics = 1;
2224
2225#ifdef CONFIG_RFS_ACCEL
2226 if (bp->flags & BNXT_FLAG_RFS)
2227 num_vnics += bp->rx_nr_rings;
2228#endif
2229
2230 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
2231 GFP_KERNEL);
2232 if (!bp->vnic_info)
2233 return -ENOMEM;
2234
2235 bp->nr_vnics = num_vnics;
2236 return 0;
2237}
2238
2239static void bnxt_init_vnics(struct bnxt *bp)
2240{
2241 int i;
2242
2243 for (i = 0; i < bp->nr_vnics; i++) {
2244 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2245
2246 vnic->fw_vnic_id = INVALID_HW_RING_ID;
2247 vnic->fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
2248 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
2249
2250 if (bp->vnic_info[i].rss_hash_key) {
2251 if (i == 0)
2252 prandom_bytes(vnic->rss_hash_key,
2253 HW_HASH_KEY_SIZE);
2254 else
2255 memcpy(vnic->rss_hash_key,
2256 bp->vnic_info[0].rss_hash_key,
2257 HW_HASH_KEY_SIZE);
2258 }
2259 }
2260}
2261
2262static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
2263{
2264 int pages;
2265
2266 pages = ring_size / desc_per_pg;
2267
2268 if (!pages)
2269 return 1;
2270
2271 pages++;
2272
2273 while (pages & (pages - 1))
2274 pages++;
2275
2276 return pages;
2277}
2278
2279static void bnxt_set_tpa_flags(struct bnxt *bp)
2280{
2281 bp->flags &= ~BNXT_FLAG_TPA;
2282 if (bp->dev->features & NETIF_F_LRO)
2283 bp->flags |= BNXT_FLAG_LRO;
2284 if ((bp->dev->features & NETIF_F_GRO) && (bp->pdev->revision > 0))
2285 bp->flags |= BNXT_FLAG_GRO;
2286}
2287
2288/* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
2289 * be set on entry.
2290 */
2291void bnxt_set_ring_params(struct bnxt *bp)
2292{
2293 u32 ring_size, rx_size, rx_space;
2294 u32 agg_factor = 0, agg_ring_size = 0;
2295
2296 /* 8 for CRC and VLAN */
2297 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
2298
2299 rx_space = rx_size + NET_SKB_PAD +
2300 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2301
2302 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
2303 ring_size = bp->rx_ring_size;
2304 bp->rx_agg_ring_size = 0;
2305 bp->rx_agg_nr_pages = 0;
2306
2307 if (bp->flags & BNXT_FLAG_TPA)
2839f28b 2308 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
c0c050c5
MC		 2309
2310 bp->flags &= ~BNXT_FLAG_JUMBO;
2311 if (rx_space > PAGE_SIZE) {
2312 u32 jumbo_factor;
2313
2314 bp->flags |= BNXT_FLAG_JUMBO;
2315 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
2316 if (jumbo_factor > agg_factor)
2317 agg_factor = jumbo_factor;
2318 }
2319 agg_ring_size = ring_size * agg_factor;
2320
2321 if (agg_ring_size) {
2322 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
2323 RX_DESC_CNT);
2324 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
2325 u32 tmp = agg_ring_size;
2326
2327 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
2328 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
2329 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
2330 tmp, agg_ring_size);
2331 }
2332 bp->rx_agg_ring_size = agg_ring_size;
2333 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
2334 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
2335 rx_space = rx_size + NET_SKB_PAD +
2336 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2337 }
2338
2339 bp->rx_buf_use_size = rx_size;
2340 bp->rx_buf_size = rx_space;
2341
2342 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
2343 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
2344
2345 ring_size = bp->tx_ring_size;
2346 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
2347 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
2348
2349 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
2350 bp->cp_ring_size = ring_size;
2351
2352 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
2353 if (bp->cp_nr_pages > MAX_CP_PAGES) {
2354 bp->cp_nr_pages = MAX_CP_PAGES;
2355 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
2356 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
2357 ring_size, bp->cp_ring_size);
2358 }
2359 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
2360 bp->cp_ring_mask = bp->cp_bit - 1;
2361}
2362
2363static void bnxt_free_vnic_attributes(struct bnxt *bp)
2364{
2365 int i;
2366 struct bnxt_vnic_info *vnic;
2367 struct pci_dev *pdev = bp->pdev;
2368
2369 if (!bp->vnic_info)
2370 return;
2371
2372 for (i = 0; i < bp->nr_vnics; i++) {
2373 vnic = &bp->vnic_info[i];
2374
2375 kfree(vnic->fw_grp_ids);
2376 vnic->fw_grp_ids = NULL;
2377
2378 kfree(vnic->uc_list);
2379 vnic->uc_list = NULL;
2380
2381 if (vnic->mc_list) {
2382 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
2383 vnic->mc_list, vnic->mc_list_mapping);
2384 vnic->mc_list = NULL;
2385 }
2386
2387 if (vnic->rss_table) {
2388 dma_free_coherent(&pdev->dev, PAGE_SIZE,
2389 vnic->rss_table,
2390 vnic->rss_table_dma_addr);
2391 vnic->rss_table = NULL;
2392 }
2393
2394 vnic->rss_hash_key = NULL;
2395 vnic->flags = 0;
2396 }
2397}
2398
2399static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
2400{
2401 int i, rc = 0, size;
2402 struct bnxt_vnic_info *vnic;
2403 struct pci_dev *pdev = bp->pdev;
2404 int max_rings;
2405
2406 for (i = 0; i < bp->nr_vnics; i++) {
2407 vnic = &bp->vnic_info[i];
2408
2409 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
2410 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
2411
2412 if (mem_size > 0) {
2413 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
2414 if (!vnic->uc_list) {
2415 rc = -ENOMEM;
2416 goto out;
2417 }
2418 }
2419 }
2420
2421 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
2422 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
2423 vnic->mc_list =
2424 dma_alloc_coherent(&pdev->dev,
2425 vnic->mc_list_size,
2426 &vnic->mc_list_mapping,
2427 GFP_KERNEL);
2428 if (!vnic->mc_list) {
2429 rc = -ENOMEM;
2430 goto out;
2431 }
2432 }
2433
2434 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
2435 max_rings = bp->rx_nr_rings;
2436 else
2437 max_rings = 1;
2438
2439 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
2440 if (!vnic->fw_grp_ids) {
2441 rc = -ENOMEM;
2442 goto out;
2443 }
2444
2445 /* Allocate rss table and hash key */
2446 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2447 &vnic->rss_table_dma_addr,
2448 GFP_KERNEL);
2449 if (!vnic->rss_table) {
2450 rc = -ENOMEM;
2451 goto out;
2452 }
2453
2454 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
2455
2456 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
2457 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
2458 }
2459 return 0;
2460
2461out:
2462 return rc;
2463}
2464
2465static void bnxt_free_hwrm_resources(struct bnxt *bp)
2466{
2467 struct pci_dev *pdev = bp->pdev;
2468
2469 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
2470 bp->hwrm_cmd_resp_dma_addr);
2471
2472 bp->hwrm_cmd_resp_addr = NULL;
2473 if (bp->hwrm_dbg_resp_addr) {
2474 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE,
2475 bp->hwrm_dbg_resp_addr,
2476 bp->hwrm_dbg_resp_dma_addr);
2477
2478 bp->hwrm_dbg_resp_addr = NULL;
2479 }
2480}
2481
2482static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
2483{
2484 struct pci_dev *pdev = bp->pdev;
2485
2486 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2487 &bp->hwrm_cmd_resp_dma_addr,
2488 GFP_KERNEL);
2489 if (!bp->hwrm_cmd_resp_addr)
2490 return -ENOMEM;
2491 bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev,
2492 HWRM_DBG_REG_BUF_SIZE,
2493 &bp->hwrm_dbg_resp_dma_addr,
2494 GFP_KERNEL);
2495 if (!bp->hwrm_dbg_resp_addr)
2496 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n");
2497
2498 return 0;
2499}
2500
2501static void bnxt_free_stats(struct bnxt *bp)
2502{
2503 u32 size, i;
2504 struct pci_dev *pdev = bp->pdev;
2505
3bdf56c4
MC		 2506 if (bp->hw_rx_port_stats) {
2507 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
2508 bp->hw_rx_port_stats,
2509 bp->hw_rx_port_stats_map);
2510 bp->hw_rx_port_stats = NULL;
2511 bp->flags &= ~BNXT_FLAG_PORT_STATS;
2512 }
2513
c0c050c5
MC		 2514 if (!bp->bnapi)
2515 return;
2516
2517 size = sizeof(struct ctx_hw_stats);
2518
2519 for (i = 0; i < bp->cp_nr_rings; i++) {
2520 struct bnxt_napi *bnapi = bp->bnapi[i];
2521 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2522
2523 if (cpr->hw_stats) {
2524 dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
2525 cpr->hw_stats_map);
2526 cpr->hw_stats = NULL;
2527 }
2528 }
2529}
2530
2531static int bnxt_alloc_stats(struct bnxt *bp)
2532{
2533 u32 size, i;
2534 struct pci_dev *pdev = bp->pdev;
2535
2536 size = sizeof(struct ctx_hw_stats);
2537
2538 for (i = 0; i < bp->cp_nr_rings; i++) {
2539 struct bnxt_napi *bnapi = bp->bnapi[i];
2540 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2541
2542 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
2543 &cpr->hw_stats_map,
2544 GFP_KERNEL);
2545 if (!cpr->hw_stats)
2546 return -ENOMEM;
2547
2548 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
2549 }
3bdf56c4
MC		 2550
2551 if (BNXT_PF(bp)) {
2552 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
2553 sizeof(struct tx_port_stats) + 1024;
2554
2555 bp->hw_rx_port_stats =
2556 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
2557 &bp->hw_rx_port_stats_map,
2558 GFP_KERNEL);
2559 if (!bp->hw_rx_port_stats)
2560 return -ENOMEM;
2561
2562 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) +
2563 512;
2564 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
2565 sizeof(struct rx_port_stats) + 512;
2566 bp->flags |= BNXT_FLAG_PORT_STATS;
2567 }
c0c050c5
MC		 2568 return 0;
2569}
2570
2571static void bnxt_clear_ring_indices(struct bnxt *bp)
2572{
2573 int i;
2574
2575 if (!bp->bnapi)
2576 return;
2577
2578 for (i = 0; i < bp->cp_nr_rings; i++) {
2579 struct bnxt_napi *bnapi = bp->bnapi[i];
2580 struct bnxt_cp_ring_info *cpr;
2581 struct bnxt_rx_ring_info *rxr;
2582 struct bnxt_tx_ring_info *txr;
2583
2584 if (!bnapi)
2585 continue;
2586
2587 cpr = &bnapi->cp_ring;
2588 cpr->cp_raw_cons = 0;
2589
b6ab4b01 2590 txr = bnapi->tx_ring;
3b2b7d9d
MC		 2591 if (txr) {
2592 txr->tx_prod = 0;
2593 txr->tx_cons = 0;
2594 }
c0c050c5 2595
b6ab4b01 2596 rxr = bnapi->rx_ring;
3b2b7d9d
MC		 2597 if (rxr) {
2598 rxr->rx_prod = 0;
2599 rxr->rx_agg_prod = 0;
2600 rxr->rx_sw_agg_prod = 0;
376a5b86 2601 rxr->rx_next_cons = 0;
3b2b7d9d 2602 }
c0c050c5
MC		 2603 }
2604}
2605
2606static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
2607{
2608#ifdef CONFIG_RFS_ACCEL
2609 int i;
2610
2611 /* Under rtnl_lock and all our NAPIs have been disabled. It's
2612 * safe to delete the hash table.
2613 */
2614 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
2615 struct hlist_head *head;
2616 struct hlist_node *tmp;
2617 struct bnxt_ntuple_filter *fltr;
2618
2619 head = &bp->ntp_fltr_hash_tbl[i];
2620 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
2621 hlist_del(&fltr->hash);
2622 kfree(fltr);
2623 }
2624 }
2625 if (irq_reinit) {
2626 kfree(bp->ntp_fltr_bmap);
2627 bp->ntp_fltr_bmap = NULL;
2628 }
2629 bp->ntp_fltr_count = 0;
2630#endif
2631}
2632
2633static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
2634{
2635#ifdef CONFIG_RFS_ACCEL
2636 int i, rc = 0;
2637
2638 if (!(bp->flags & BNXT_FLAG_RFS))
2639 return 0;
2640
2641 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
2642 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
2643
2644 bp->ntp_fltr_count = 0;
2645 bp->ntp_fltr_bmap = kzalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
2646 GFP_KERNEL);
2647
2648 if (!bp->ntp_fltr_bmap)
2649 rc = -ENOMEM;
2650
2651 return rc;
2652#else
2653 return 0;
2654#endif
2655}
2656
2657static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
2658{
2659 bnxt_free_vnic_attributes(bp);
2660 bnxt_free_tx_rings(bp);
2661 bnxt_free_rx_rings(bp);
2662 bnxt_free_cp_rings(bp);
2663 bnxt_free_ntp_fltrs(bp, irq_re_init);
2664 if (irq_re_init) {
2665 bnxt_free_stats(bp);
2666 bnxt_free_ring_grps(bp);
2667 bnxt_free_vnics(bp);
b6ab4b01
MC		 2668 kfree(bp->tx_ring);
2669 bp->tx_ring = NULL;
2670 kfree(bp->rx_ring);
2671 bp->rx_ring = NULL;
c0c050c5
MC		 2672 kfree(bp->bnapi);
2673 bp->bnapi = NULL;
2674 } else {
2675 bnxt_clear_ring_indices(bp);
2676 }
2677}
2678
2679static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
2680{
01657bcd 2681 int i, j, rc, size, arr_size;
c0c050c5
MC		 2682 void *bnapi;
2683
2684 if (irq_re_init) {
2685 /* Allocate bnapi mem pointer array and mem block for
2686 * all queues
2687 */
2688 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
2689 bp->cp_nr_rings);
2690 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
2691 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
2692 if (!bnapi)
2693 return -ENOMEM;
2694
2695 bp->bnapi = bnapi;
2696 bnapi += arr_size;
2697 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
2698 bp->bnapi[i] = bnapi;
2699 bp->bnapi[i]->index = i;
2700 bp->bnapi[i]->bp = bp;
2701 }
2702
b6ab4b01
MC		 2703 bp->rx_ring = kcalloc(bp->rx_nr_rings,
2704 sizeof(struct bnxt_rx_ring_info),
2705 GFP_KERNEL);
2706 if (!bp->rx_ring)
2707 return -ENOMEM;
2708
2709 for (i = 0; i < bp->rx_nr_rings; i++) {
2710 bp->rx_ring[i].bnapi = bp->bnapi[i];
2711 bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
2712 }
2713
2714 bp->tx_ring = kcalloc(bp->tx_nr_rings,
2715 sizeof(struct bnxt_tx_ring_info),
2716 GFP_KERNEL);
2717 if (!bp->tx_ring)
2718 return -ENOMEM;
2719
01657bcd
MC		 2720 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
2721 j = 0;
2722 else
2723 j = bp->rx_nr_rings;
2724
2725 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
2726 bp->tx_ring[i].bnapi = bp->bnapi[j];
2727 bp->bnapi[j]->tx_ring = &bp->tx_ring[i];
b6ab4b01
MC		 2728 }
2729
c0c050c5
MC		 2730 rc = bnxt_alloc_stats(bp);
2731 if (rc)
2732 goto alloc_mem_err;
2733
2734 rc = bnxt_alloc_ntp_fltrs(bp);
2735 if (rc)
2736 goto alloc_mem_err;
2737
2738 rc = bnxt_alloc_vnics(bp);
2739 if (rc)
2740 goto alloc_mem_err;
2741 }
2742
2743 bnxt_init_ring_struct(bp);
2744
2745 rc = bnxt_alloc_rx_rings(bp);
2746 if (rc)
2747 goto alloc_mem_err;
2748
2749 rc = bnxt_alloc_tx_rings(bp);
2750 if (rc)
2751 goto alloc_mem_err;
2752
2753 rc = bnxt_alloc_cp_rings(bp);
2754 if (rc)
2755 goto alloc_mem_err;
2756
2757 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
2758 BNXT_VNIC_UCAST_FLAG;
2759 rc = bnxt_alloc_vnic_attributes(bp);
2760 if (rc)
2761 goto alloc_mem_err;
2762 return 0;
2763
2764alloc_mem_err:
2765 bnxt_free_mem(bp, true);
2766 return rc;
2767}
2768
2769void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
2770 u16 cmpl_ring, u16 target_id)
2771{
a8643e16 2772 struct input *req = request;
c0c050c5 2773
a8643e16
MC		 2774 req->req_type = cpu_to_le16(req_type);
2775 req->cmpl_ring = cpu_to_le16(cmpl_ring);
2776 req->target_id = cpu_to_le16(target_id);
c0c050c5
MC		 2777 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
2778}
2779
fbfbc485
MC		 2780static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
2781 int timeout, bool silent)
c0c050c5
MC		 2782{
2783 int i, intr_process, rc;
a8643e16 2784 struct input *req = msg;
c0c050c5
MC		 2785 u32 *data = msg;
2786 __le32 *resp_len, *valid;
2787 u16 cp_ring_id, len = 0;
2788 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
2789
a8643e16 2790 req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++);
c0c050c5 2791 memset(resp, 0, PAGE_SIZE);
a8643e16 2792 cp_ring_id = le16_to_cpu(req->cmpl_ring);
c0c050c5
MC		 2793 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
2794
2795 /* Write request msg to hwrm channel */
2796 __iowrite32_copy(bp->bar0, data, msg_len / 4);
2797
e6ef2699 2798 for (i = msg_len; i < BNXT_HWRM_MAX_REQ_LEN; i += 4)
d79979a1
MC		 2799 writel(0, bp->bar0 + i);
2800
c0c050c5
MC		 2801 /* currently supports only one outstanding message */
2802 if (intr_process)
a8643e16 2803 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
c0c050c5
MC		 2804
2805 /* Ring channel doorbell */
2806 writel(1, bp->bar0 + 0x100);
2807
ff4fe81d
MC		 2808 if (!timeout)
2809 timeout = DFLT_HWRM_CMD_TIMEOUT;
2810
c0c050c5
MC		 2811 i = 0;
2812 if (intr_process) {
2813 /* Wait until hwrm response cmpl interrupt is processed */
2814 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID &&
2815 i++ < timeout) {
2816 usleep_range(600, 800);
2817 }
2818
2819 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) {
2820 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
a8643e16 2821 le16_to_cpu(req->req_type));
c0c050c5
MC		 2822 return -1;
2823 }
2824 } else {
2825 /* Check if response len is updated */
2826 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET;
2827 for (i = 0; i < timeout; i++) {
2828 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
2829 HWRM_RESP_LEN_SFT;
2830 if (len)
2831 break;
2832 usleep_range(600, 800);
2833 }
2834
2835 if (i >= timeout) {
2836 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
a8643e16
MC		 2837 timeout, le16_to_cpu(req->req_type),
2838 le16_to_cpu(req->seq_id), *resp_len);
c0c050c5
MC		 2839 return -1;
2840 }
2841
2842 /* Last word of resp contains valid bit */
2843 valid = bp->hwrm_cmd_resp_addr + len - 4;
2844 for (i = 0; i < timeout; i++) {
2845 if (le32_to_cpu(*valid) & HWRM_RESP_VALID_MASK)
2846 break;
2847 usleep_range(600, 800);
2848 }
2849
2850 if (i >= timeout) {
2851 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
a8643e16
MC		 2852 timeout, le16_to_cpu(req->req_type),
2853 le16_to_cpu(req->seq_id), len, *valid);
c0c050c5
MC		 2854 return -1;
2855 }
2856 }
2857
2858 rc = le16_to_cpu(resp->error_code);
fbfbc485 2859 if (rc && !silent)
c0c050c5
MC		 2860 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
2861 le16_to_cpu(resp->req_type),
2862 le16_to_cpu(resp->seq_id), rc);
fbfbc485
MC		 2863 return rc;
2864}
2865
2866int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
2867{
2868 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
c0c050c5
MC		 2869}
2870
2871int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
2872{
2873 int rc;
2874
2875 mutex_lock(&bp->hwrm_cmd_lock);
2876 rc = _hwrm_send_message(bp, msg, msg_len, timeout);
2877 mutex_unlock(&bp->hwrm_cmd_lock);
2878 return rc;
2879}
2880
90e20921
MC		 2881int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
2882 int timeout)
2883{
2884 int rc;
2885
2886 mutex_lock(&bp->hwrm_cmd_lock);
2887 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
2888 mutex_unlock(&bp->hwrm_cmd_lock);
2889 return rc;
2890}
2891
c0c050c5
MC		 2892static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
2893{
2894 struct hwrm_func_drv_rgtr_input req = {0};
2895 int i;
25be8623
MC		 2896 DECLARE_BITMAP(async_events_bmap, 256);
2897 u32 *events = (u32 *)async_events_bmap;
c0c050c5
MC		 2898
2899 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
2900
2901 req.enables =
2902 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
2903 FUNC_DRV_RGTR_REQ_ENABLES_VER |
2904 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
2905
25be8623
MC		 2906 memset(async_events_bmap, 0, sizeof(async_events_bmap));
2907 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
2908 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
2909
2910 for (i = 0; i < 8; i++)
2911 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
2912
11f15ed3 2913 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
c0c050c5
MC		 2914 req.ver_maj = DRV_VER_MAJ;
2915 req.ver_min = DRV_VER_MIN;
2916 req.ver_upd = DRV_VER_UPD;
2917
2918 if (BNXT_PF(bp)) {
de68f5de 2919 DECLARE_BITMAP(vf_req_snif_bmap, 256);
c0c050c5
MC		 2920 u32 *data = (u32 *)vf_req_snif_bmap;
2921
de68f5de 2922 memset(vf_req_snif_bmap, 0, sizeof(vf_req_snif_bmap));
c0c050c5
MC		 2923 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++)
2924 __set_bit(bnxt_vf_req_snif[i], vf_req_snif_bmap);
2925
de68f5de
MC		 2926 for (i = 0; i < 8; i++)
2927 req.vf_req_fwd[i] = cpu_to_le32(data[i]);
2928
c0c050c5
MC		 2929 req.enables |=
2930 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
2931 }
2932
2933 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2934}
2935
be58a0da
JH		 2936static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
2937{
2938 struct hwrm_func_drv_unrgtr_input req = {0};
2939
2940 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
2941 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2942}
2943
c0c050c5
MC		 2944static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
2945{
2946 u32 rc = 0;
2947 struct hwrm_tunnel_dst_port_free_input req = {0};
2948
2949 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
2950 req.tunnel_type = tunnel_type;
2951
2952 switch (tunnel_type) {
2953 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
2954 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
2955 break;
2956 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
2957 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
2958 break;
2959 default:
2960 break;
2961 }
2962
2963 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2964 if (rc)
2965 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
2966 rc);
2967 return rc;
2968}
2969
2970static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
2971 u8 tunnel_type)
2972{
2973 u32 rc = 0;
2974 struct hwrm_tunnel_dst_port_alloc_input req = {0};
2975 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
2976
2977 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
2978
2979 req.tunnel_type = tunnel_type;
2980 req.tunnel_dst_port_val = port;
2981
2982 mutex_lock(&bp->hwrm_cmd_lock);
2983 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2984 if (rc) {
2985 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
2986 rc);
2987 goto err_out;
2988 }
2989
2990 if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN)
2991 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
2992
2993 else if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE)
2994 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
2995err_out:
2996 mutex_unlock(&bp->hwrm_cmd_lock);
2997 return rc;
2998}
2999
3000static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
3001{
3002 struct hwrm_cfa_l2_set_rx_mask_input req = {0};
3003 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3004
3005 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
c193554e 3006 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
c0c050c5
MC		 3007
3008 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
3009 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
3010 req.mask = cpu_to_le32(vnic->rx_mask);
3011 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3012}
3013
3014#ifdef CONFIG_RFS_ACCEL
3015static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
3016 struct bnxt_ntuple_filter *fltr)
3017{
3018 struct hwrm_cfa_ntuple_filter_free_input req = {0};
3019
3020 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
3021 req.ntuple_filter_id = fltr->filter_id;
3022 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3023}
3024
3025#define BNXT_NTP_FLTR_FLAGS \
3026 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \
3027 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \
3028 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \
3029 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \
3030 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \
3031 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \
3032 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \
3033 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \
3034 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \
3035 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \
3036 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \
3037 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \
3038 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \
c193554e 3039 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
c0c050c5
MC		 3040
3041static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
3042 struct bnxt_ntuple_filter *fltr)
3043{
3044 int rc = 0;
3045 struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
3046 struct hwrm_cfa_ntuple_filter_alloc_output *resp =
3047 bp->hwrm_cmd_resp_addr;
3048 struct flow_keys *keys = &fltr->fkeys;
3049 struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1];
3050
3051 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
3052 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[0];
3053
3054 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
3055
3056 req.ethertype = htons(ETH_P_IP);
3057 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
c193554e 3058 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
c0c050c5
MC		 3059 req.ip_protocol = keys->basic.ip_proto;
3060
3061 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
3062 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3063 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
3064 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3065
3066 req.src_port = keys->ports.src;
3067 req.src_port_mask = cpu_to_be16(0xffff);
3068 req.dst_port = keys->ports.dst;
3069 req.dst_port_mask = cpu_to_be16(0xffff);
3070
c193554e 3071 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
c0c050c5
MC		 3072 mutex_lock(&bp->hwrm_cmd_lock);
3073 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3074 if (!rc)
3075 fltr->filter_id = resp->ntuple_filter_id;
3076 mutex_unlock(&bp->hwrm_cmd_lock);
3077 return rc;
3078}
3079#endif
3080
3081static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
3082 u8 *mac_addr)
3083{
3084 u32 rc = 0;
3085 struct hwrm_cfa_l2_filter_alloc_input req = {0};
3086 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3087
3088 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
3089 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX |
3090 CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
c193554e 3091 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
c0c050c5
MC		 3092 req.enables =
3093 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
c193554e 3094 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
c0c050c5
MC		 3095 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
3096 memcpy(req.l2_addr, mac_addr, ETH_ALEN);
3097 req.l2_addr_mask[0] = 0xff;
3098 req.l2_addr_mask[1] = 0xff;
3099 req.l2_addr_mask[2] = 0xff;
3100 req.l2_addr_mask[3] = 0xff;
3101 req.l2_addr_mask[4] = 0xff;
3102 req.l2_addr_mask[5] = 0xff;
3103
3104 mutex_lock(&bp->hwrm_cmd_lock);
3105 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3106 if (!rc)
3107 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
3108 resp->l2_filter_id;
3109 mutex_unlock(&bp->hwrm_cmd_lock);
3110 return rc;
3111}
3112
3113static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
3114{
3115 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
3116 int rc = 0;
3117
3118 /* Any associated ntuple filters will also be cleared by firmware. */
3119 mutex_lock(&bp->hwrm_cmd_lock);
3120 for (i = 0; i < num_of_vnics; i++) {
3121 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3122
3123 for (j = 0; j < vnic->uc_filter_count; j++) {
3124 struct hwrm_cfa_l2_filter_free_input req = {0};
3125
3126 bnxt_hwrm_cmd_hdr_init(bp, &req,
3127 HWRM_CFA_L2_FILTER_FREE, -1, -1);
3128
3129 req.l2_filter_id = vnic->fw_l2_filter_id[j];
3130
3131 rc = _hwrm_send_message(bp, &req, sizeof(req),
3132 HWRM_CMD_TIMEOUT);
3133 }
3134 vnic->uc_filter_count = 0;
3135 }
3136 mutex_unlock(&bp->hwrm_cmd_lock);
3137
3138 return rc;
3139}
3140
3141static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
3142{
3143 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3144 struct hwrm_vnic_tpa_cfg_input req = {0};
3145
3146 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
3147
3148 if (tpa_flags) {
3149 u16 mss = bp->dev->mtu - 40;
3150 u32 nsegs, n, segs = 0, flags;
3151
3152 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
3153 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
3154 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
3155 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
3156 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
3157 if (tpa_flags & BNXT_FLAG_GRO)
3158 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
3159
3160 req.flags = cpu_to_le32(flags);
3161
3162 req.enables =
3163 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
c193554e
MC		 3164 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
3165 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
c0c050c5
MC		 3166
3167 /* Number of segs are log2 units, and first packet is not
3168 * included as part of this units.
3169 */
2839f28b
MC		 3170 if (mss <= BNXT_RX_PAGE_SIZE) {
3171 n = BNXT_RX_PAGE_SIZE / mss;
c0c050c5
MC		 3172 nsegs = (MAX_SKB_FRAGS - 1) * n;
3173 } else {
2839f28b
MC		 3174 n = mss / BNXT_RX_PAGE_SIZE;
3175 if (mss & (BNXT_RX_PAGE_SIZE - 1))
c0c050c5
MC		 3176 n++;
3177 nsegs = (MAX_SKB_FRAGS - n) / n;
3178 }
3179
3180 segs = ilog2(nsegs);
3181 req.max_agg_segs = cpu_to_le16(segs);
3182 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX);
c193554e
MC		 3183
3184 req.min_agg_len = cpu_to_le32(512);
c0c050c5
MC		 3185 }
3186 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3187
3188 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3189}
3190
3191static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
3192{
3193 u32 i, j, max_rings;
3194 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3195 struct hwrm_vnic_rss_cfg_input req = {0};
3196
3197 if (vnic->fw_rss_cos_lb_ctx == INVALID_HW_RING_ID)
3198 return 0;
3199
3200 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
3201 if (set_rss) {
3202 vnic->hash_type = BNXT_RSS_HASH_TYPE_FLAG_IPV4 |
3203 BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV4 |
3204 BNXT_RSS_HASH_TYPE_FLAG_IPV6 |
3205 BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV6;
3206
3207 req.hash_type = cpu_to_le32(vnic->hash_type);
3208
3209 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3210 max_rings = bp->rx_nr_rings;
3211 else
3212 max_rings = 1;
3213
3214 /* Fill the RSS indirection table with ring group ids */
3215 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
3216 if (j == max_rings)
3217 j = 0;
3218 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
3219 }
3220
3221 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
3222 req.hash_key_tbl_addr =
3223 cpu_to_le64(vnic->rss_hash_key_dma_addr);
3224 }
3225 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx);
3226 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3227}
3228
3229static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
3230{
3231 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3232 struct hwrm_vnic_plcmodes_cfg_input req = {0};
3233
3234 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
3235 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
3236 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
3237 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
3238 req.enables =
3239 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
3240 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
3241 /* thresholds not implemented in firmware yet */
3242 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
3243 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
3244 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3245 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3246}
3247
3248static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id)
3249{
3250 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
3251
3252 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
3253 req.rss_cos_lb_ctx_id =
3254 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx);
3255
3256 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3257 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
3258}
3259
3260static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
3261{
3262 int i;
3263
3264 for (i = 0; i < bp->nr_vnics; i++) {
3265 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3266
3267 if (vnic->fw_rss_cos_lb_ctx != INVALID_HW_RING_ID)
3268 bnxt_hwrm_vnic_ctx_free_one(bp, i);
3269 }
3270 bp->rsscos_nr_ctxs = 0;
3271}
3272
3273static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id)
3274{
3275 int rc;
3276 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
3277 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
3278 bp->hwrm_cmd_resp_addr;
3279
3280 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
3281 -1);
3282
3283 mutex_lock(&bp->hwrm_cmd_lock);
3284 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3285 if (!rc)
3286 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx =
3287 le16_to_cpu(resp->rss_cos_lb_ctx_id);
3288 mutex_unlock(&bp->hwrm_cmd_lock);
3289
3290 return rc;
3291}
3292
3293static int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
3294{
b81a90d3 3295 unsigned int ring = 0, grp_idx;
c0c050c5
MC		 3296 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3297 struct hwrm_vnic_cfg_input req = {0};
3298
3299 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
3300 /* Only RSS support for now TBD: COS & LB */
3301 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP |
3302 VNIC_CFG_REQ_ENABLES_RSS_RULE);
3303 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx);
3304 req.cos_rule = cpu_to_le16(0xffff);
3305 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
b81a90d3 3306 ring = 0;
c0c050c5 3307 else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
b81a90d3 3308 ring = vnic_id - 1;
c0c050c5 3309
b81a90d3 3310 grp_idx = bp->rx_ring[ring].bnapi->index;
c0c050c5
MC		 3311 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3312 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
3313
3314 req.lb_rule = cpu_to_le16(0xffff);
3315 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
3316 VLAN_HLEN);
3317
3318 if (bp->flags & BNXT_FLAG_STRIP_VLAN)
3319 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
3320
3321 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3322}
3323
3324static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
3325{
3326 u32 rc = 0;
3327
3328 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
3329 struct hwrm_vnic_free_input req = {0};
3330
3331 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
3332 req.vnic_id =
3333 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
3334
3335 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3336 if (rc)
3337 return rc;
3338 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
3339 }
3340 return rc;
3341}
3342
3343static void bnxt_hwrm_vnic_free(struct bnxt *bp)
3344{
3345 u16 i;
3346
3347 for (i = 0; i < bp->nr_vnics; i++)
3348 bnxt_hwrm_vnic_free_one(bp, i);
3349}
3350
b81a90d3
MC		 3351static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
3352 unsigned int start_rx_ring_idx,
3353 unsigned int nr_rings)
c0c050c5 3354{
b81a90d3
MC		 3355 int rc = 0;
3356 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
c0c050c5
MC		 3357 struct hwrm_vnic_alloc_input req = {0};
3358 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3359
3360 /* map ring groups to this vnic */
b81a90d3
MC		 3361 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
3362 grp_idx = bp->rx_ring[i].bnapi->index;
3363 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
c0c050c5 3364 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
b81a90d3 3365 j, nr_rings);
c0c050c5
MC		 3366 break;
3367 }
3368 bp->vnic_info[vnic_id].fw_grp_ids[j] =
b81a90d3 3369 bp->grp_info[grp_idx].fw_grp_id;
c0c050c5
MC		 3370 }
3371
3372 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
3373 if (vnic_id == 0)
3374 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
3375
3376 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
3377
3378 mutex_lock(&bp->hwrm_cmd_lock);
3379 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3380 if (!rc)
3381 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id);
3382 mutex_unlock(&bp->hwrm_cmd_lock);
3383 return rc;
3384}
3385
3386static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
3387{
3388 u16 i;
3389 u32 rc = 0;
3390
3391 mutex_lock(&bp->hwrm_cmd_lock);
3392 for (i = 0; i < bp->rx_nr_rings; i++) {
3393 struct hwrm_ring_grp_alloc_input req = {0};
3394 struct hwrm_ring_grp_alloc_output *resp =
3395 bp->hwrm_cmd_resp_addr;
b81a90d3 3396 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
c0c050c5
MC		 3397
3398 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
3399
b81a90d3
MC		 3400 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
3401 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
3402 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
3403 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
c0c050c5
MC		 3404
3405 rc = _hwrm_send_message(bp, &req, sizeof(req),
3406 HWRM_CMD_TIMEOUT);
3407 if (rc)
3408 break;
3409
b81a90d3
MC		 3410 bp->grp_info[grp_idx].fw_grp_id =
3411 le32_to_cpu(resp->ring_group_id);
c0c050c5
MC		 3412 }
3413 mutex_unlock(&bp->hwrm_cmd_lock);
3414 return rc;
3415}
3416
3417static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
3418{
3419 u16 i;
3420 u32 rc = 0;
3421 struct hwrm_ring_grp_free_input req = {0};
3422
3423 if (!bp->grp_info)
3424 return 0;
3425
3426 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
3427
3428 mutex_lock(&bp->hwrm_cmd_lock);
3429 for (i = 0; i < bp->cp_nr_rings; i++) {
3430 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
3431 continue;
3432 req.ring_group_id =
3433 cpu_to_le32(bp->grp_info[i].fw_grp_id);
3434
3435 rc = _hwrm_send_message(bp, &req, sizeof(req),
3436 HWRM_CMD_TIMEOUT);
3437 if (rc)
3438 break;
3439 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3440 }
3441 mutex_unlock(&bp->hwrm_cmd_lock);
3442 return rc;
3443}
3444
3445static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
3446 struct bnxt_ring_struct *ring,
3447 u32 ring_type, u32 map_index,
3448 u32 stats_ctx_id)
3449{
3450 int rc = 0, err = 0;
3451 struct hwrm_ring_alloc_input req = {0};
3452 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3453 u16 ring_id;
3454
3455 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
3456
3457 req.enables = 0;
3458 if (ring->nr_pages > 1) {
3459 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map);
3460 /* Page size is in log2 units */
3461 req.page_size = BNXT_PAGE_SHIFT;
3462 req.page_tbl_depth = 1;
3463 } else {
3464 req.page_tbl_addr = cpu_to_le64(ring->dma_arr[0]);
3465 }
3466 req.fbo = 0;
3467 /* Association of ring index with doorbell index and MSIX number */
3468 req.logical_id = cpu_to_le16(map_index);
3469
3470 switch (ring_type) {
3471 case HWRM_RING_ALLOC_TX:
3472 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
3473 /* Association of transmit ring with completion ring */
3474 req.cmpl_ring_id =
3475 cpu_to_le16(bp->grp_info[map_index].cp_fw_ring_id);
3476 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
3477 req.stat_ctx_id = cpu_to_le32(stats_ctx_id);
3478 req.queue_id = cpu_to_le16(ring->queue_id);
3479 break;
3480 case HWRM_RING_ALLOC_RX:
3481 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
3482 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
3483 break;
3484 case HWRM_RING_ALLOC_AGG:
3485 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
3486 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
3487 break;
3488 case HWRM_RING_ALLOC_CMPL:
3489 req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL;
3490 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
3491 if (bp->flags & BNXT_FLAG_USING_MSIX)
3492 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
3493 break;
3494 default:
3495 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
3496 ring_type);
3497 return -1;
3498 }
3499
3500 mutex_lock(&bp->hwrm_cmd_lock);
3501 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3502 err = le16_to_cpu(resp->error_code);
3503 ring_id = le16_to_cpu(resp->ring_id);
3504 mutex_unlock(&bp->hwrm_cmd_lock);
3505
3506 if (rc || err) {
3507 switch (ring_type) {
3508 case RING_FREE_REQ_RING_TYPE_CMPL:
3509 netdev_err(bp->dev, "hwrm_ring_alloc cp failed. rc:%x err:%x\n",
3510 rc, err);
3511 return -1;
3512
3513 case RING_FREE_REQ_RING_TYPE_RX:
3514 netdev_err(bp->dev, "hwrm_ring_alloc rx failed. rc:%x err:%x\n",
3515 rc, err);
3516 return -1;
3517
3518 case RING_FREE_REQ_RING_TYPE_TX:
3519 netdev_err(bp->dev, "hwrm_ring_alloc tx failed. rc:%x err:%x\n",
3520 rc, err);
3521 return -1;
3522
3523 default:
3524 netdev_err(bp->dev, "Invalid ring\n");
3525 return -1;
3526 }
3527 }
3528 ring->fw_ring_id = ring_id;
3529 return rc;
3530}
3531
3532static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
3533{
3534 int i, rc = 0;
3535
edd0c2cc
MC		 3536 for (i = 0; i < bp->cp_nr_rings; i++) {
3537 struct bnxt_napi *bnapi = bp->bnapi[i];
3538 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3539 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
c0c050c5 3540
33e52d88 3541 cpr->cp_doorbell = bp->bar1 + i * 0x80;
edd0c2cc
MC		 3542 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, i,
3543 INVALID_STATS_CTX_ID);
3544 if (rc)
3545 goto err_out;
edd0c2cc
MC		 3546 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
3547 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
c0c050c5
MC		 3548 }
3549
edd0c2cc 3550 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 3551 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
edd0c2cc 3552 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
b81a90d3
MC		 3553 u32 map_idx = txr->bnapi->index;
3554 u16 fw_stats_ctx = bp->grp_info[map_idx].fw_stats_ctx;
c0c050c5 3555
b81a90d3
MC		 3556 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX,
3557 map_idx, fw_stats_ctx);
edd0c2cc
MC		 3558 if (rc)
3559 goto err_out;
b81a90d3 3560 txr->tx_doorbell = bp->bar1 + map_idx * 0x80;
c0c050c5
MC		 3561 }
3562
edd0c2cc 3563 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 3564 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 3565 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
b81a90d3 3566 u32 map_idx = rxr->bnapi->index;
c0c050c5 3567
b81a90d3
MC		 3568 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX,
3569 map_idx, INVALID_STATS_CTX_ID);
edd0c2cc
MC		 3570 if (rc)
3571 goto err_out;
b81a90d3 3572 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80;
edd0c2cc 3573 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
b81a90d3 3574 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
c0c050c5
MC		 3575 }
3576
3577 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
3578 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 3579 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
c0c050c5
MC		 3580 struct bnxt_ring_struct *ring =
3581 &rxr->rx_agg_ring_struct;
b81a90d3
MC		 3582 u32 grp_idx = rxr->bnapi->index;
3583 u32 map_idx = grp_idx + bp->rx_nr_rings;
c0c050c5
MC		 3584
3585 rc = hwrm_ring_alloc_send_msg(bp, ring,
3586 HWRM_RING_ALLOC_AGG,
b81a90d3 3587 map_idx,
c0c050c5
MC		 3588 INVALID_STATS_CTX_ID);
3589 if (rc)
3590 goto err_out;
3591
b81a90d3 3592 rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80;
c0c050c5
MC		 3593 writel(DB_KEY_RX | rxr->rx_agg_prod,
3594 rxr->rx_agg_doorbell);
b81a90d3 3595 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
c0c050c5
MC		 3596 }
3597 }
3598err_out:
3599 return rc;
3600}
3601
3602static int hwrm_ring_free_send_msg(struct bnxt *bp,
3603 struct bnxt_ring_struct *ring,
3604 u32 ring_type, int cmpl_ring_id)
3605{
3606 int rc;
3607 struct hwrm_ring_free_input req = {0};
3608 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
3609 u16 error_code;
3610
74608fc9 3611 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
c0c050c5
MC		 3612 req.ring_type = ring_type;
3613 req.ring_id = cpu_to_le16(ring->fw_ring_id);
3614
3615 mutex_lock(&bp->hwrm_cmd_lock);
3616 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3617 error_code = le16_to_cpu(resp->error_code);
3618 mutex_unlock(&bp->hwrm_cmd_lock);
3619
3620 if (rc || error_code) {
3621 switch (ring_type) {
3622 case RING_FREE_REQ_RING_TYPE_CMPL:
3623 netdev_err(bp->dev, "hwrm_ring_free cp failed. rc:%d\n",
3624 rc);
3625 return rc;
3626 case RING_FREE_REQ_RING_TYPE_RX:
3627 netdev_err(bp->dev, "hwrm_ring_free rx failed. rc:%d\n",
3628 rc);
3629 return rc;
3630 case RING_FREE_REQ_RING_TYPE_TX:
3631 netdev_err(bp->dev, "hwrm_ring_free tx failed. rc:%d\n",
3632 rc);
3633 return rc;
3634 default:
3635 netdev_err(bp->dev, "Invalid ring\n");
3636 return -1;
3637 }
3638 }
3639 return 0;
3640}
3641
edd0c2cc 3642static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
c0c050c5 3643{
edd0c2cc 3644 int i;
c0c050c5
MC		 3645
3646 if (!bp->bnapi)
edd0c2cc 3647 return;
c0c050c5 3648
edd0c2cc 3649 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 3650 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
edd0c2cc 3651 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
b81a90d3
MC		 3652 u32 grp_idx = txr->bnapi->index;
3653 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
edd0c2cc
MC		 3654
3655 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3656 hwrm_ring_free_send_msg(bp, ring,
3657 RING_FREE_REQ_RING_TYPE_TX,
3658 close_path ? cmpl_ring_id :
3659 INVALID_HW_RING_ID);
3660 ring->fw_ring_id = INVALID_HW_RING_ID;
c0c050c5
MC		 3661 }
3662 }
3663
edd0c2cc 3664 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 3665 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 3666 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
b81a90d3
MC		 3667 u32 grp_idx = rxr->bnapi->index;
3668 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
edd0c2cc
MC		 3669
3670 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3671 hwrm_ring_free_send_msg(bp, ring,
3672 RING_FREE_REQ_RING_TYPE_RX,
3673 close_path ? cmpl_ring_id :
3674 INVALID_HW_RING_ID);
3675 ring->fw_ring_id = INVALID_HW_RING_ID;
b81a90d3
MC		 3676 bp->grp_info[grp_idx].rx_fw_ring_id =
3677 INVALID_HW_RING_ID;
c0c050c5
MC		 3678 }
3679 }
3680
edd0c2cc 3681 for (i = 0; i < bp->rx_nr_rings; i++) {
b6ab4b01 3682 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
edd0c2cc 3683 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
b81a90d3
MC		 3684 u32 grp_idx = rxr->bnapi->index;
3685 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
edd0c2cc
MC		 3686
3687 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3688 hwrm_ring_free_send_msg(bp, ring,
3689 RING_FREE_REQ_RING_TYPE_RX,
3690 close_path ? cmpl_ring_id :
3691 INVALID_HW_RING_ID);
3692 ring->fw_ring_id = INVALID_HW_RING_ID;
b81a90d3
MC		 3693 bp->grp_info[grp_idx].agg_fw_ring_id =
3694 INVALID_HW_RING_ID;
c0c050c5
MC		 3695 }
3696 }
3697
edd0c2cc
MC		 3698 for (i = 0; i < bp->cp_nr_rings; i++) {
3699 struct bnxt_napi *bnapi = bp->bnapi[i];
3700 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3701 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3702
3703 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3704 hwrm_ring_free_send_msg(bp, ring,
3705 RING_FREE_REQ_RING_TYPE_CMPL,
3706 INVALID_HW_RING_ID);
3707 ring->fw_ring_id = INVALID_HW_RING_ID;
3708 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
c0c050c5
MC		 3709 }
3710 }
c0c050c5
MC		 3711}
3712
bb053f52
MC		 3713static void bnxt_hwrm_set_coal_params(struct bnxt *bp, u32 max_bufs,
3714 u32 buf_tmrs, u16 flags,
3715 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
3716{
3717 req->flags = cpu_to_le16(flags);
3718 req->num_cmpl_dma_aggr = cpu_to_le16((u16)max_bufs);
3719 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(max_bufs >> 16);
3720 req->cmpl_aggr_dma_tmr = cpu_to_le16((u16)buf_tmrs);
3721 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(buf_tmrs >> 16);
3722 /* Minimum time between 2 interrupts set to buf_tmr x 2 */
3723 req->int_lat_tmr_min = cpu_to_le16((u16)buf_tmrs * 2);
3724 req->int_lat_tmr_max = cpu_to_le16((u16)buf_tmrs * 4);
3725 req->num_cmpl_aggr_int = cpu_to_le16((u16)max_bufs * 4);
3726}
3727
c0c050c5
MC		 3728int bnxt_hwrm_set_coal(struct bnxt *bp)
3729{
3730 int i, rc = 0;
dfc9c94a
MC		 3731 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
3732 req_tx = {0}, *req;
c0c050c5
MC		 3733 u16 max_buf, max_buf_irq;
3734 u16 buf_tmr, buf_tmr_irq;
3735 u32 flags;
3736
dfc9c94a
MC		 3737 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
3738 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
3739 bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
3740 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
c0c050c5 3741
dfb5b894
MC		 3742 /* Each rx completion (2 records) should be DMAed immediately.
3743 * DMA 1/4 of the completion buffers at a time.
3744 */
3745 max_buf = min_t(u16, bp->rx_coal_bufs / 4, 2);
c0c050c5
MC		 3746 /* max_buf must not be zero */
3747 max_buf = clamp_t(u16, max_buf, 1, 63);
dfb5b894
MC		 3748 max_buf_irq = clamp_t(u16, bp->rx_coal_bufs_irq, 1, 63);
3749 buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks);
3750 /* buf timer set to 1/4 of interrupt timer */
3751 buf_tmr = max_t(u16, buf_tmr / 4, 1);
3752 buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks_irq);
3753 buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
c0c050c5
MC		 3754
3755 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
3756
3757 /* RING_IDLE generates more IRQs for lower latency. Enable it only
3758 * if coal_ticks is less than 25 us.
3759 */
dfb5b894 3760 if (bp->rx_coal_ticks < 25)
c0c050c5
MC		 3761 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
3762
bb053f52 3763 bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
dfc9c94a
MC		 3764 buf_tmr_irq << 16 | buf_tmr, flags, &req_rx);
3765
3766 /* max_buf must not be zero */
3767 max_buf = clamp_t(u16, bp->tx_coal_bufs, 1, 63);
3768 max_buf_irq = clamp_t(u16, bp->tx_coal_bufs_irq, 1, 63);
3769 buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks);
3770 /* buf timer set to 1/4 of interrupt timer */
3771 buf_tmr = max_t(u16, buf_tmr / 4, 1);
3772 buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks_irq);
3773 buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
3774
3775 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
3776 bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
3777 buf_tmr_irq << 16 | buf_tmr, flags, &req_tx);
c0c050c5
MC		 3778
3779 mutex_lock(&bp->hwrm_cmd_lock);
3780 for (i = 0; i < bp->cp_nr_rings; i++) {
dfc9c94a 3781 struct bnxt_napi *bnapi = bp->bnapi[i];
c0c050c5 3782
dfc9c94a
MC		 3783 req = &req_rx;
3784 if (!bnapi->rx_ring)
3785 req = &req_tx;
3786 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id);
3787
3788 rc = _hwrm_send_message(bp, req, sizeof(*req),
c0c050c5
MC		 3789 HWRM_CMD_TIMEOUT);
3790 if (rc)
3791 break;
3792 }
3793 mutex_unlock(&bp->hwrm_cmd_lock);
3794 return rc;
3795}
3796
3797static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
3798{
3799 int rc = 0, i;
3800 struct hwrm_stat_ctx_free_input req = {0};
3801
3802 if (!bp->bnapi)
3803 return 0;
3804
3805 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
3806
3807 mutex_lock(&bp->hwrm_cmd_lock);
3808 for (i = 0; i < bp->cp_nr_rings; i++) {
3809 struct bnxt_napi *bnapi = bp->bnapi[i];
3810 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3811
3812 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
3813 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
3814
3815 rc = _hwrm_send_message(bp, &req, sizeof(req),
3816 HWRM_CMD_TIMEOUT);
3817 if (rc)
3818 break;
3819
3820 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3821 }
3822 }
3823 mutex_unlock(&bp->hwrm_cmd_lock);
3824 return rc;
3825}
3826
3827static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
3828{
3829 int rc = 0, i;
3830 struct hwrm_stat_ctx_alloc_input req = {0};
3831 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3832
3833 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
3834
3835 req.update_period_ms = cpu_to_le32(1000);
3836
3837 mutex_lock(&bp->hwrm_cmd_lock);
3838 for (i = 0; i < bp->cp_nr_rings; i++) {
3839 struct bnxt_napi *bnapi = bp->bnapi[i];
3840 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3841
3842 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
3843
3844 rc = _hwrm_send_message(bp, &req, sizeof(req),
3845 HWRM_CMD_TIMEOUT);
3846 if (rc)
3847 break;
3848
3849 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
3850
3851 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
3852 }
3853 mutex_unlock(&bp->hwrm_cmd_lock);
3854 return 0;
3855}
3856
4a21b49b 3857int bnxt_hwrm_func_qcaps(struct bnxt *bp)
c0c050c5
MC		 3858{
3859 int rc = 0;
3860 struct hwrm_func_qcaps_input req = {0};
3861 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
3862
3863 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
3864 req.fid = cpu_to_le16(0xffff);
3865
3866 mutex_lock(&bp->hwrm_cmd_lock);
3867 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3868 if (rc)
3869 goto hwrm_func_qcaps_exit;
3870
3871 if (BNXT_PF(bp)) {
3872 struct bnxt_pf_info *pf = &bp->pf;
3873
3874 pf->fw_fid = le16_to_cpu(resp->fid);
3875 pf->port_id = le16_to_cpu(resp->port_id);
11f15ed3 3876 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
bdd4347b 3877 memcpy(bp->dev->dev_addr, pf->mac_addr, ETH_ALEN);
c0c050c5
MC		 3878 pf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
3879 pf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
3880 pf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
c0c050c5 3881 pf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
b72d4a68
MC		 3882 pf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
3883 if (!pf->max_hw_ring_grps)
3884 pf->max_hw_ring_grps = pf->max_tx_rings;
c0c050c5
MC		 3885 pf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
3886 pf->max_vnics = le16_to_cpu(resp->max_vnics);
3887 pf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
3888 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
3889 pf->max_vfs = le16_to_cpu(resp->max_vfs);
3890 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
3891 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
3892 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
3893 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
3894 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
3895 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
3896 } else {
379a80a1 3897#ifdef CONFIG_BNXT_SRIOV
c0c050c5
MC		 3898 struct bnxt_vf_info *vf = &bp->vf;
3899
3900 vf->fw_fid = le16_to_cpu(resp->fid);
11f15ed3 3901 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
bdd4347b
JH		 3902 if (is_valid_ether_addr(vf->mac_addr))
3903 /* overwrite netdev dev_adr with admin VF MAC */
3904 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
3905 else
3906 random_ether_addr(bp->dev->dev_addr);
c0c050c5
MC		 3907
3908 vf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
3909 vf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
3910 vf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
3911 vf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
b72d4a68
MC		 3912 vf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
3913 if (!vf->max_hw_ring_grps)
3914 vf->max_hw_ring_grps = vf->max_tx_rings;
c0c050c5
MC		 3915 vf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
3916 vf->max_vnics = le16_to_cpu(resp->max_vnics);
3917 vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
379a80a1 3918#endif
c0c050c5
MC		 3919 }
3920
3921 bp->tx_push_thresh = 0;
3922 if (resp->flags &
3923 cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED))
3924 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
3925
3926hwrm_func_qcaps_exit:
3927 mutex_unlock(&bp->hwrm_cmd_lock);
3928 return rc;
3929}
3930
3931static int bnxt_hwrm_func_reset(struct bnxt *bp)
3932{
3933 struct hwrm_func_reset_input req = {0};
3934
3935 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
3936 req.enables = 0;
3937
3938 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
3939}
3940
3941static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
3942{
3943 int rc = 0;
3944 struct hwrm_queue_qportcfg_input req = {0};
3945 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
3946 u8 i, *qptr;
3947
3948 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
3949
3950 mutex_lock(&bp->hwrm_cmd_lock);
3951 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3952 if (rc)
3953 goto qportcfg_exit;
3954
3955 if (!resp->max_configurable_queues) {
3956 rc = -EINVAL;
3957 goto qportcfg_exit;
3958 }
3959 bp->max_tc = resp->max_configurable_queues;
3960 if (bp->max_tc > BNXT_MAX_QUEUE)
3961 bp->max_tc = BNXT_MAX_QUEUE;
3962
3963 qptr = &resp->queue_id0;
3964 for (i = 0; i < bp->max_tc; i++) {
3965 bp->q_info[i].queue_id = *qptr++;
3966 bp->q_info[i].queue_profile = *qptr++;
3967 }
3968
3969qportcfg_exit:
3970 mutex_unlock(&bp->hwrm_cmd_lock);
3971 return rc;
3972}
3973
3974static int bnxt_hwrm_ver_get(struct bnxt *bp)
3975{
3976 int rc;
3977 struct hwrm_ver_get_input req = {0};
3978 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
3979
e6ef2699 3980 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
c0c050c5
MC		 3981 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
3982 req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
3983 req.hwrm_intf_min = HWRM_VERSION_MINOR;
3984 req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
3985 mutex_lock(&bp->hwrm_cmd_lock);
3986 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3987 if (rc)
3988 goto hwrm_ver_get_exit;
3989
3990 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
3991
11f15ed3
MC		 3992 bp->hwrm_spec_code = resp->hwrm_intf_maj << 16 |
3993 resp->hwrm_intf_min << 8 | resp->hwrm_intf_upd;
c193554e
MC		 3994 if (resp->hwrm_intf_maj < 1) {
3995 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
c0c050c5 3996 resp->hwrm_intf_maj, resp->hwrm_intf_min,
c193554e
MC		 3997 resp->hwrm_intf_upd);
3998 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
c0c050c5 3999 }
3ebf6f0a 4000 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d/%d.%d.%d",
c0c050c5
MC		 4001 resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld,
4002 resp->hwrm_intf_maj, resp->hwrm_intf_min, resp->hwrm_intf_upd);
4003
ff4fe81d
MC		 4004 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
4005 if (!bp->hwrm_cmd_timeout)
4006 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
4007
e6ef2699
MC		 4008 if (resp->hwrm_intf_maj >= 1)
4009 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
4010
c0c050c5
MC		 4011hwrm_ver_get_exit:
4012 mutex_unlock(&bp->hwrm_cmd_lock);
4013 return rc;
4014}
4015
3bdf56c4
MC		 4016static int bnxt_hwrm_port_qstats(struct bnxt *bp)
4017{
4018 int rc;
4019 struct bnxt_pf_info *pf = &bp->pf;
4020 struct hwrm_port_qstats_input req = {0};
4021
4022 if (!(bp->flags & BNXT_FLAG_PORT_STATS))
4023 return 0;
4024
4025 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
4026 req.port_id = cpu_to_le16(pf->port_id);
4027 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
4028 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
4029 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4030 return rc;
4031}
4032
c0c050c5
MC		 4033static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
4034{
4035 if (bp->vxlan_port_cnt) {
4036 bnxt_hwrm_tunnel_dst_port_free(
4037 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
4038 }
4039 bp->vxlan_port_cnt = 0;
4040 if (bp->nge_port_cnt) {
4041 bnxt_hwrm_tunnel_dst_port_free(
4042 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
4043 }
4044 bp->nge_port_cnt = 0;
4045}
4046
4047static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
4048{
4049 int rc, i;
4050 u32 tpa_flags = 0;
4051
4052 if (set_tpa)
4053 tpa_flags = bp->flags & BNXT_FLAG_TPA;
4054 for (i = 0; i < bp->nr_vnics; i++) {
4055 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
4056 if (rc) {
4057 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
4058 rc, i);
4059 return rc;
4060 }
4061 }
4062 return 0;
4063}
4064
4065static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
4066{
4067 int i;
4068
4069 for (i = 0; i < bp->nr_vnics; i++)
4070 bnxt_hwrm_vnic_set_rss(bp, i, false);
4071}
4072
4073static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
4074 bool irq_re_init)
4075{
4076 if (bp->vnic_info) {
4077 bnxt_hwrm_clear_vnic_filter(bp);
4078 /* clear all RSS setting before free vnic ctx */
4079 bnxt_hwrm_clear_vnic_rss(bp);
4080 bnxt_hwrm_vnic_ctx_free(bp);
4081 /* before free the vnic, undo the vnic tpa settings */
4082 if (bp->flags & BNXT_FLAG_TPA)
4083 bnxt_set_tpa(bp, false);
4084 bnxt_hwrm_vnic_free(bp);
4085 }
4086 bnxt_hwrm_ring_free(bp, close_path);
4087 bnxt_hwrm_ring_grp_free(bp);
4088 if (irq_re_init) {
4089 bnxt_hwrm_stat_ctx_free(bp);
4090 bnxt_hwrm_free_tunnel_ports(bp);
4091 }
4092}
4093
4094static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
4095{
4096 int rc;
4097
4098 /* allocate context for vnic */
4099 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id);
4100 if (rc) {
4101 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
4102 vnic_id, rc);
4103 goto vnic_setup_err;
4104 }
4105 bp->rsscos_nr_ctxs++;
4106
4107 /* configure default vnic, ring grp */
4108 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
4109 if (rc) {
4110 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
4111 vnic_id, rc);
4112 goto vnic_setup_err;
4113 }
4114
4115 /* Enable RSS hashing on vnic */
4116 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
4117 if (rc) {
4118 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
4119 vnic_id, rc);
4120 goto vnic_setup_err;
4121 }
4122
4123 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
4124 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
4125 if (rc) {
4126 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
4127 vnic_id, rc);
4128 }
4129 }
4130
4131vnic_setup_err:
4132 return rc;
4133}
4134
4135static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
4136{
4137#ifdef CONFIG_RFS_ACCEL
4138 int i, rc = 0;
4139
4140 for (i = 0; i < bp->rx_nr_rings; i++) {
4141 u16 vnic_id = i + 1;
4142 u16 ring_id = i;
4143
4144 if (vnic_id >= bp->nr_vnics)
4145 break;
4146
4147 bp->vnic_info[vnic_id].flags |= BNXT_VNIC_RFS_FLAG;
b81a90d3 4148 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
c0c050c5
MC		 4149 if (rc) {
4150 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
4151 vnic_id, rc);
4152 break;
4153 }
4154 rc = bnxt_setup_vnic(bp, vnic_id);
4155 if (rc)
4156 break;
4157 }
4158 return rc;
4159#else
4160 return 0;
4161#endif
4162}
4163
b664f008 4164static int bnxt_cfg_rx_mode(struct bnxt *);
7d2837dd 4165static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
b664f008 4166
c0c050c5
MC		 4167static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
4168{
7d2837dd 4169 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
c0c050c5
MC		 4170 int rc = 0;
4171
4172 if (irq_re_init) {
4173 rc = bnxt_hwrm_stat_ctx_alloc(bp);
4174 if (rc) {
4175 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
4176 rc);
4177 goto err_out;
4178 }
4179 }
4180
4181 rc = bnxt_hwrm_ring_alloc(bp);
4182 if (rc) {
4183 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
4184 goto err_out;
4185 }
4186
4187 rc = bnxt_hwrm_ring_grp_alloc(bp);
4188 if (rc) {
4189 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
4190 goto err_out;
4191 }
4192
4193 /* default vnic 0 */
4194 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, bp->rx_nr_rings);
4195 if (rc) {
4196 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
4197 goto err_out;
4198 }
4199
4200 rc = bnxt_setup_vnic(bp, 0);
4201 if (rc)
4202 goto err_out;
4203
4204 if (bp->flags & BNXT_FLAG_RFS) {
4205 rc = bnxt_alloc_rfs_vnics(bp);
4206 if (rc)
4207 goto err_out;
4208 }
4209
4210 if (bp->flags & BNXT_FLAG_TPA) {
4211 rc = bnxt_set_tpa(bp, true);
4212 if (rc)
4213 goto err_out;
4214 }
4215
4216 if (BNXT_VF(bp))
4217 bnxt_update_vf_mac(bp);
4218
4219 /* Filter for default vnic 0 */
4220 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
4221 if (rc) {
4222 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
4223 goto err_out;
4224 }
7d2837dd 4225 vnic->uc_filter_count = 1;
c0c050c5 4226
7d2837dd 4227 vnic->rx_mask = CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
c0c050c5
MC		 4228
4229 if ((bp->dev->flags & IFF_PROMISC) && BNXT_PF(bp))
7d2837dd
MC		 4230 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
4231
4232 if (bp->dev->flags & IFF_ALLMULTI) {
4233 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
4234 vnic->mc_list_count = 0;
4235 } else {
4236 u32 mask = 0;
4237
4238 bnxt_mc_list_updated(bp, &mask);
4239 vnic->rx_mask |= mask;
4240 }
c0c050c5 4241
b664f008
MC		 4242 rc = bnxt_cfg_rx_mode(bp);
4243 if (rc)
c0c050c5 4244 goto err_out;
c0c050c5
MC		 4245
4246 rc = bnxt_hwrm_set_coal(bp);
4247 if (rc)
4248 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
4249 rc);
4250
4251 return 0;
4252
4253err_out:
4254 bnxt_hwrm_resource_free(bp, 0, true);
4255
4256 return rc;
4257}
4258
4259static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
4260{
4261 bnxt_hwrm_resource_free(bp, 1, irq_re_init);
4262 return 0;
4263}
4264
4265static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
4266{
4267 bnxt_init_rx_rings(bp);
4268 bnxt_init_tx_rings(bp);
4269 bnxt_init_ring_grps(bp, irq_re_init);
4270 bnxt_init_vnics(bp);
4271
4272 return bnxt_init_chip(bp, irq_re_init);
4273}
4274
4275static void bnxt_disable_int(struct bnxt *bp)
4276{
4277 int i;
4278
4279 if (!bp->bnapi)
4280 return;
4281
4282 for (i = 0; i < bp->cp_nr_rings; i++) {
4283 struct bnxt_napi *bnapi = bp->bnapi[i];
4284 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4285
4286 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
4287 }
4288}
4289
4290static void bnxt_enable_int(struct bnxt *bp)
4291{
4292 int i;
4293
4294 atomic_set(&bp->intr_sem, 0);
4295 for (i = 0; i < bp->cp_nr_rings; i++) {
4296 struct bnxt_napi *bnapi = bp->bnapi[i];
4297 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4298
4299 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
4300 }
4301}
4302
4303static int bnxt_set_real_num_queues(struct bnxt *bp)
4304{
4305 int rc;
4306 struct net_device *dev = bp->dev;
4307
4308 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings);
4309 if (rc)
4310 return rc;
4311
4312 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
4313 if (rc)
4314 return rc;
4315
4316#ifdef CONFIG_RFS_ACCEL
45019a18 4317 if (bp->flags & BNXT_FLAG_RFS)
c0c050c5 4318 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
c0c050c5
MC		 4319#endif
4320
4321 return rc;
4322}
4323
6e6c5a57
MC		 4324static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
4325 bool shared)
4326{
4327 int _rx = *rx, _tx = *tx;
4328
4329 if (shared) {
4330 *rx = min_t(int, _rx, max);
4331 *tx = min_t(int, _tx, max);
4332 } else {
4333 if (max < 2)
4334 return -ENOMEM;
4335
4336 while (_rx + _tx > max) {
4337 if (_rx > _tx && _rx > 1)
4338 _rx--;
4339 else if (_tx > 1)
4340 _tx--;
4341 }
4342 *rx = _rx;
4343 *tx = _tx;
4344 }
4345 return 0;
4346}
4347
c0c050c5
MC		 4348static int bnxt_setup_msix(struct bnxt *bp)
4349{
4350 struct msix_entry *msix_ent;
4351 struct net_device *dev = bp->dev;
01657bcd 4352 int i, total_vecs, rc = 0, min = 1;
c0c050c5
MC		 4353 const int len = sizeof(bp->irq_tbl[0].name);
4354
4355 bp->flags &= ~BNXT_FLAG_USING_MSIX;
4356 total_vecs = bp->cp_nr_rings;
4357
4358 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
4359 if (!msix_ent)
4360 return -ENOMEM;
4361
4362 for (i = 0; i < total_vecs; i++) {
4363 msix_ent[i].entry = i;
4364 msix_ent[i].vector = 0;
4365 }
4366
01657bcd
MC		 4367 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
4368 min = 2;
4369
4370 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
c0c050c5
MC		 4371 if (total_vecs < 0) {
4372 rc = -ENODEV;
4373 goto msix_setup_exit;
4374 }
4375
4376 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
4377 if (bp->irq_tbl) {
4378 int tcs;
4379
4380 /* Trim rings based upon num of vectors allocated */
6e6c5a57 4381 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
01657bcd 4382 total_vecs, min == 1);
6e6c5a57
MC		 4383 if (rc)
4384 goto msix_setup_exit;
4385
c0c050c5
MC		 4386 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4387 tcs = netdev_get_num_tc(dev);
4388 if (tcs > 1) {
4389 bp->tx_nr_rings_per_tc = bp->tx_nr_rings / tcs;
4390 if (bp->tx_nr_rings_per_tc == 0) {
4391 netdev_reset_tc(dev);
4392 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4393 } else {
4394 int i, off, count;
4395
4396 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tcs;
4397 for (i = 0; i < tcs; i++) {
4398 count = bp->tx_nr_rings_per_tc;
4399 off = i * count;
4400 netdev_set_tc_queue(dev, i, count, off);
4401 }
4402 }
4403 }
01657bcd 4404 bp->cp_nr_rings = total_vecs;
c0c050c5
MC		 4405
4406 for (i = 0; i < bp->cp_nr_rings; i++) {
01657bcd
MC		 4407 char *attr;
4408
c0c050c5 4409 bp->irq_tbl[i].vector = msix_ent[i].vector;
01657bcd
MC		 4410 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4411 attr = "TxRx";
4412 else if (i < bp->rx_nr_rings)
4413 attr = "rx";
4414 else
4415 attr = "tx";
4416
c0c050c5 4417 snprintf(bp->irq_tbl[i].name, len,
01657bcd 4418 "%s-%s-%d", dev->name, attr, i);
c0c050c5
MC		 4419 bp->irq_tbl[i].handler = bnxt_msix;
4420 }
4421 rc = bnxt_set_real_num_queues(bp);
4422 if (rc)
4423 goto msix_setup_exit;
4424 } else {
4425 rc = -ENOMEM;
4426 goto msix_setup_exit;
4427 }
4428 bp->flags |= BNXT_FLAG_USING_MSIX;
4429 kfree(msix_ent);
4430 return 0;
4431
4432msix_setup_exit:
4433 netdev_err(bp->dev, "bnxt_setup_msix err: %x\n", rc);
4434 pci_disable_msix(bp->pdev);
4435 kfree(msix_ent);
4436 return rc;
4437}
4438
4439static int bnxt_setup_inta(struct bnxt *bp)
4440{
4441 int rc;
4442 const int len = sizeof(bp->irq_tbl[0].name);
4443
4444 if (netdev_get_num_tc(bp->dev))
4445 netdev_reset_tc(bp->dev);
4446
4447 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
4448 if (!bp->irq_tbl) {
4449 rc = -ENOMEM;
4450 return rc;
4451 }
4452 bp->rx_nr_rings = 1;
4453 bp->tx_nr_rings = 1;
4454 bp->cp_nr_rings = 1;
4455 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
01657bcd 4456 bp->flags |= BNXT_FLAG_SHARED_RINGS;
c0c050c5
MC		 4457 bp->irq_tbl[0].vector = bp->pdev->irq;
4458 snprintf(bp->irq_tbl[0].name, len,
4459 "%s-%s-%d", bp->dev->name, "TxRx", 0);
4460 bp->irq_tbl[0].handler = bnxt_inta;
4461 rc = bnxt_set_real_num_queues(bp);
4462 return rc;
4463}
4464
4465static int bnxt_setup_int_mode(struct bnxt *bp)
4466{
4467 int rc = 0;
4468
4469 if (bp->flags & BNXT_FLAG_MSIX_CAP)
4470 rc = bnxt_setup_msix(bp);
4471
1fa72e29 4472 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
c0c050c5
MC		 4473 /* fallback to INTA */
4474 rc = bnxt_setup_inta(bp);
4475 }
4476 return rc;
4477}
4478
4479static void bnxt_free_irq(struct bnxt *bp)
4480{
4481 struct bnxt_irq *irq;
4482 int i;
4483
4484#ifdef CONFIG_RFS_ACCEL
4485 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
4486 bp->dev->rx_cpu_rmap = NULL;
4487#endif
4488 if (!bp->irq_tbl)
4489 return;
4490
4491 for (i = 0; i < bp->cp_nr_rings; i++) {
4492 irq = &bp->irq_tbl[i];
4493 if (irq->requested)
4494 free_irq(irq->vector, bp->bnapi[i]);
4495 irq->requested = 0;
4496 }
4497 if (bp->flags & BNXT_FLAG_USING_MSIX)
4498 pci_disable_msix(bp->pdev);
4499 kfree(bp->irq_tbl);
4500 bp->irq_tbl = NULL;
4501}
4502
4503static int bnxt_request_irq(struct bnxt *bp)
4504{
b81a90d3 4505 int i, j, rc = 0;
c0c050c5
MC		 4506 unsigned long flags = 0;
4507#ifdef CONFIG_RFS_ACCEL
4508 struct cpu_rmap *rmap = bp->dev->rx_cpu_rmap;
4509#endif
4510
4511 if (!(bp->flags & BNXT_FLAG_USING_MSIX))
4512 flags = IRQF_SHARED;
4513
b81a90d3 4514 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
c0c050c5
MC		 4515 struct bnxt_irq *irq = &bp->irq_tbl[i];
4516#ifdef CONFIG_RFS_ACCEL
b81a90d3 4517 if (rmap && bp->bnapi[i]->rx_ring) {
c0c050c5
MC		 4518 rc = irq_cpu_rmap_add(rmap, irq->vector);
4519 if (rc)
4520 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
b81a90d3
MC		 4521 j);
4522 j++;
c0c050c5
MC		 4523 }
4524#endif
4525 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
4526 bp->bnapi[i]);
4527 if (rc)
4528 break;
4529
4530 irq->requested = 1;
4531 }
4532 return rc;
4533}
4534
4535static void bnxt_del_napi(struct bnxt *bp)
4536{
4537 int i;
4538
4539 if (!bp->bnapi)
4540 return;
4541
4542 for (i = 0; i < bp->cp_nr_rings; i++) {
4543 struct bnxt_napi *bnapi = bp->bnapi[i];
4544
4545 napi_hash_del(&bnapi->napi);
4546 netif_napi_del(&bnapi->napi);
4547 }
4548}
4549
4550static void bnxt_init_napi(struct bnxt *bp)
4551{
4552 int i;
4553 struct bnxt_napi *bnapi;
4554
4555 if (bp->flags & BNXT_FLAG_USING_MSIX) {
4556 for (i = 0; i < bp->cp_nr_rings; i++) {
4557 bnapi = bp->bnapi[i];
4558 netif_napi_add(bp->dev, &bnapi->napi,
4559 bnxt_poll, 64);
c0c050c5
MC		 4560 }
4561 } else {
4562 bnapi = bp->bnapi[0];
4563 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
c0c050c5
MC		 4564 }
4565}
4566
4567static void bnxt_disable_napi(struct bnxt *bp)
4568{
4569 int i;
4570
4571 if (!bp->bnapi)
4572 return;
4573
4574 for (i = 0; i < bp->cp_nr_rings; i++) {
4575 napi_disable(&bp->bnapi[i]->napi);
4576 bnxt_disable_poll(bp->bnapi[i]);
4577 }
4578}
4579
4580static void bnxt_enable_napi(struct bnxt *bp)
4581{
4582 int i;
4583
4584 for (i = 0; i < bp->cp_nr_rings; i++) {
fa7e2812 4585 bp->bnapi[i]->in_reset = false;
c0c050c5
MC		 4586 bnxt_enable_poll(bp->bnapi[i]);
4587 napi_enable(&bp->bnapi[i]->napi);
4588 }
4589}
4590
4591static void bnxt_tx_disable(struct bnxt *bp)
4592{
4593 int i;
c0c050c5
MC		 4594 struct bnxt_tx_ring_info *txr;
4595 struct netdev_queue *txq;
4596
b6ab4b01 4597 if (bp->tx_ring) {
c0c050c5 4598 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 4599 txr = &bp->tx_ring[i];
c0c050c5
MC		 4600 txq = netdev_get_tx_queue(bp->dev, i);
4601 __netif_tx_lock(txq, smp_processor_id());
4602 txr->dev_state = BNXT_DEV_STATE_CLOSING;
4603 __netif_tx_unlock(txq);
4604 }
4605 }
4606 /* Stop all TX queues */
4607 netif_tx_disable(bp->dev);
4608 netif_carrier_off(bp->dev);
4609}
4610
4611static void bnxt_tx_enable(struct bnxt *bp)
4612{
4613 int i;
c0c050c5
MC		 4614 struct bnxt_tx_ring_info *txr;
4615 struct netdev_queue *txq;
4616
4617 for (i = 0; i < bp->tx_nr_rings; i++) {
b6ab4b01 4618 txr = &bp->tx_ring[i];
c0c050c5
MC		 4619 txq = netdev_get_tx_queue(bp->dev, i);
4620 txr->dev_state = 0;
4621 }
4622 netif_tx_wake_all_queues(bp->dev);
4623 if (bp->link_info.link_up)
4624 netif_carrier_on(bp->dev);
4625}
4626
4627static void bnxt_report_link(struct bnxt *bp)
4628{
4629 if (bp->link_info.link_up) {
4630 const char *duplex;
4631 const char *flow_ctrl;
4632 u16 speed;
4633
4634 netif_carrier_on(bp->dev);
4635 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
4636 duplex = "full";
4637 else
4638 duplex = "half";
4639 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
4640 flow_ctrl = "ON - receive & transmit";
4641 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
4642 flow_ctrl = "ON - transmit";
4643 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
4644 flow_ctrl = "ON - receive";
4645 else
4646 flow_ctrl = "none";
4647 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
4648 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
4649 speed, duplex, flow_ctrl);
170ce013
MC		 4650 if (bp->flags & BNXT_FLAG_EEE_CAP)
4651 netdev_info(bp->dev, "EEE is %s\n",
4652 bp->eee.eee_active ? "active" :
4653 "not active");
c0c050c5
MC		 4654 } else {
4655 netif_carrier_off(bp->dev);
4656 netdev_err(bp->dev, "NIC Link is Down\n");
4657 }
4658}
4659
170ce013
MC		 4660static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
4661{
4662 int rc = 0;
4663 struct hwrm_port_phy_qcaps_input req = {0};
4664 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4665
4666 if (bp->hwrm_spec_code < 0x10201)
4667 return 0;
4668
4669 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
4670
4671 mutex_lock(&bp->hwrm_cmd_lock);
4672 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4673 if (rc)
4674 goto hwrm_phy_qcaps_exit;
4675
4676 if (resp->eee_supported & PORT_PHY_QCAPS_RESP_EEE_SUPPORTED) {
4677 struct ethtool_eee *eee = &bp->eee;
4678 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
4679
4680 bp->flags |= BNXT_FLAG_EEE_CAP;
4681 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4682 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
4683 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
4684 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
4685 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
4686 }
4687
4688hwrm_phy_qcaps_exit:
4689 mutex_unlock(&bp->hwrm_cmd_lock);
4690 return rc;
4691}
4692
c0c050c5
MC		 4693static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
4694{
4695 int rc = 0;
4696 struct bnxt_link_info *link_info = &bp->link_info;
4697 struct hwrm_port_phy_qcfg_input req = {0};
4698 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4699 u8 link_up = link_info->link_up;
4700
4701 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
4702
4703 mutex_lock(&bp->hwrm_cmd_lock);
4704 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4705 if (rc) {
4706 mutex_unlock(&bp->hwrm_cmd_lock);
4707 return rc;
4708 }
4709
4710 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
4711 link_info->phy_link_status = resp->link;
4712 link_info->duplex = resp->duplex;
4713 link_info->pause = resp->pause;
4714 link_info->auto_mode = resp->auto_mode;
4715 link_info->auto_pause_setting = resp->auto_pause;
3277360e 4716 link_info->lp_pause = resp->link_partner_adv_pause;
c0c050c5 4717 link_info->force_pause_setting = resp->force_pause;
c193554e 4718 link_info->duplex_setting = resp->duplex;
c0c050c5
MC		 4719 if (link_info->phy_link_status == BNXT_LINK_LINK)
4720 link_info->link_speed = le16_to_cpu(resp->link_speed);
4721 else
4722 link_info->link_speed = 0;
4723 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
c0c050c5
MC		 4724 link_info->support_speeds = le16_to_cpu(resp->support_speeds);
4725 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
3277360e
MC		 4726 link_info->lp_auto_link_speeds =
4727 le16_to_cpu(resp->link_partner_adv_speeds);
c0c050c5
MC		 4728 link_info->preemphasis = le32_to_cpu(resp->preemphasis);
4729 link_info->phy_ver[0] = resp->phy_maj;
4730 link_info->phy_ver[1] = resp->phy_min;
4731 link_info->phy_ver[2] = resp->phy_bld;
4732 link_info->media_type = resp->media_type;
03efbec0 4733 link_info->phy_type = resp->phy_type;
11f15ed3 4734 link_info->transceiver = resp->xcvr_pkg_type;
170ce013
MC		 4735 link_info->phy_addr = resp->eee_config_phy_addr &
4736 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
42ee18fe 4737 link_info->module_status = resp->module_status;
170ce013
MC		 4738
4739 if (bp->flags & BNXT_FLAG_EEE_CAP) {
4740 struct ethtool_eee *eee = &bp->eee;
4741 u16 fw_speeds;
4742
4743 eee->eee_active = 0;
4744 if (resp->eee_config_phy_addr &
4745 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
4746 eee->eee_active = 1;
4747 fw_speeds = le16_to_cpu(
4748 resp->link_partner_adv_eee_link_speed_mask);
4749 eee->lp_advertised =
4750 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4751 }
4752
4753 /* Pull initial EEE config */
4754 if (!chng_link_state) {
4755 if (resp->eee_config_phy_addr &
4756 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
4757 eee->eee_enabled = 1;
c0c050c5 4758
170ce013
MC		 4759 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
4760 eee->advertised =
4761 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4762
4763 if (resp->eee_config_phy_addr &
4764 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
4765 __le32 tmr;
4766
4767 eee->tx_lpi_enabled = 1;
4768 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
4769 eee->tx_lpi_timer = le32_to_cpu(tmr) &
4770 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
4771 }
4772 }
4773 }
c0c050c5
MC		 4774 /* TODO: need to add more logic to report VF link */
4775 if (chng_link_state) {
4776 if (link_info->phy_link_status == BNXT_LINK_LINK)
4777 link_info->link_up = 1;
4778 else
4779 link_info->link_up = 0;
4780 if (link_up != link_info->link_up)
4781 bnxt_report_link(bp);
4782 } else {
4783 /* alwasy link down if not require to update link state */
4784 link_info->link_up = 0;
4785 }
4786 mutex_unlock(&bp->hwrm_cmd_lock);
4787 return 0;
4788}
4789
4790static void
4791bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
4792{
4793 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
c9ee9516
MC		 4794 if (bp->hwrm_spec_code >= 0x10201)
4795 req->auto_pause =
4796 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
c0c050c5
MC		 4797 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
4798 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
4799 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
49b5c7a1 4800 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
c0c050c5
MC		 4801 req->enables |=
4802 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
4803 } else {
4804 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
4805 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
4806 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
4807 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
4808 req->enables |=
4809 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
c9ee9516
MC		 4810 if (bp->hwrm_spec_code >= 0x10201) {
4811 req->auto_pause = req->force_pause;
4812 req->enables |= cpu_to_le32(
4813 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
4814 }
c0c050c5
MC		 4815 }
4816}
4817
4818static void bnxt_hwrm_set_link_common(struct bnxt *bp,
4819 struct hwrm_port_phy_cfg_input *req)
4820{
4821 u8 autoneg = bp->link_info.autoneg;
4822 u16 fw_link_speed = bp->link_info.req_link_speed;
4823 u32 advertising = bp->link_info.advertising;
4824
4825 if (autoneg & BNXT_AUTONEG_SPEED) {
4826 req->auto_mode |=
11f15ed3 4827 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
c0c050c5
MC		 4828
4829 req->enables |= cpu_to_le32(
4830 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
4831 req->auto_link_speed_mask = cpu_to_le16(advertising);
4832
4833 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
4834 req->flags |=
4835 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
4836 } else {
4837 req->force_link_speed = cpu_to_le16(fw_link_speed);
4838 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
4839 }
4840
c0c050c5
MC		 4841 /* tell chimp that the setting takes effect immediately */
4842 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
4843}
4844
4845int bnxt_hwrm_set_pause(struct bnxt *bp)
4846{
4847 struct hwrm_port_phy_cfg_input req = {0};
4848 int rc;
4849
4850 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4851 bnxt_hwrm_set_pause_common(bp, &req);
4852
4853 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
4854 bp->link_info.force_link_chng)
4855 bnxt_hwrm_set_link_common(bp, &req);
4856
4857 mutex_lock(&bp->hwrm_cmd_lock);
4858 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4859 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
4860 /* since changing of pause setting doesn't trigger any link
4861 * change event, the driver needs to update the current pause
4862 * result upon successfully return of the phy_cfg command
4863 */
4864 bp->link_info.pause =
4865 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
4866 bp->link_info.auto_pause_setting = 0;
4867 if (!bp->link_info.force_link_chng)
4868 bnxt_report_link(bp);
4869 }
4870 bp->link_info.force_link_chng = false;
4871 mutex_unlock(&bp->hwrm_cmd_lock);
4872 return rc;
4873}
4874
939f7f0c
MC		 4875static void bnxt_hwrm_set_eee(struct bnxt *bp,
4876 struct hwrm_port_phy_cfg_input *req)
4877{
4878 struct ethtool_eee *eee = &bp->eee;
4879
4880 if (eee->eee_enabled) {
4881 u16 eee_speeds;
4882 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
4883
4884 if (eee->tx_lpi_enabled)
4885 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
4886 else
4887 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
4888
4889 req->flags |= cpu_to_le32(flags);
4890 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
4891 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
4892 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
4893 } else {
4894 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
4895 }
4896}
4897
4898int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
c0c050c5
MC		 4899{
4900 struct hwrm_port_phy_cfg_input req = {0};
4901
4902 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4903 if (set_pause)
4904 bnxt_hwrm_set_pause_common(bp, &req);
4905
4906 bnxt_hwrm_set_link_common(bp, &req);
939f7f0c
MC		 4907
4908 if (set_eee)
4909 bnxt_hwrm_set_eee(bp, &req);
c0c050c5
MC		 4910 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4911}
4912
33f7d55f
MC		 4913static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
4914{
4915 struct hwrm_port_phy_cfg_input req = {0};
4916
4917 if (BNXT_VF(bp))
4918 return 0;
4919
4920 if (pci_num_vf(bp->pdev))
4921 return 0;
4922
4923 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4924 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DOWN);
4925 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4926}
4927
939f7f0c
MC		 4928static bool bnxt_eee_config_ok(struct bnxt *bp)
4929{
4930 struct ethtool_eee *eee = &bp->eee;
4931 struct bnxt_link_info *link_info = &bp->link_info;
4932
4933 if (!(bp->flags & BNXT_FLAG_EEE_CAP))
4934 return true;
4935
4936 if (eee->eee_enabled) {
4937 u32 advertising =
4938 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
4939
4940 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
4941 eee->eee_enabled = 0;
4942 return false;
4943 }
4944 if (eee->advertised & ~advertising) {
4945 eee->advertised = advertising & eee->supported;
4946 return false;
4947 }
4948 }
4949 return true;
4950}
4951
c0c050c5
MC		 4952static int bnxt_update_phy_setting(struct bnxt *bp)
4953{
4954 int rc;
4955 bool update_link = false;
4956 bool update_pause = false;
939f7f0c 4957 bool update_eee = false;
c0c050c5
MC		 4958 struct bnxt_link_info *link_info = &bp->link_info;
4959
4960 rc = bnxt_update_link(bp, true);
4961 if (rc) {
4962 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
4963 rc);
4964 return rc;
4965 }
4966 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
c9ee9516
MC		 4967 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
4968 link_info->req_flow_ctrl)
c0c050c5
MC		 4969 update_pause = true;
4970 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
4971 link_info->force_pause_setting != link_info->req_flow_ctrl)
4972 update_pause = true;
c0c050c5
MC		 4973 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
4974 if (BNXT_AUTO_MODE(link_info->auto_mode))
4975 update_link = true;
4976 if (link_info->req_link_speed != link_info->force_link_speed)
4977 update_link = true;
de73018f
MC		 4978 if (link_info->req_duplex != link_info->duplex_setting)
4979 update_link = true;
c0c050c5
MC		 4980 } else {
4981 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
4982 update_link = true;
4983 if (link_info->advertising != link_info->auto_link_speeds)
4984 update_link = true;
c0c050c5
MC		 4985 }
4986
939f7f0c
MC		 4987 if (!bnxt_eee_config_ok(bp))
4988 update_eee = true;
4989
c0c050c5 4990 if (update_link)
939f7f0c 4991 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
c0c050c5
MC		 4992 else if (update_pause)
4993 rc = bnxt_hwrm_set_pause(bp);
4994 if (rc) {
4995 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
4996 rc);
4997 return rc;
4998 }
4999
5000 return rc;
5001}
5002
11809490
JH		 5003/* Common routine to pre-map certain register block to different GRC window.
5004 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
5005 * in PF and 3 windows in VF that can be customized to map in different
5006 * register blocks.
5007 */
5008static void bnxt_preset_reg_win(struct bnxt *bp)
5009{
5010 if (BNXT_PF(bp)) {
5011 /* CAG registers map to GRC window #4 */
5012 writel(BNXT_CAG_REG_BASE,
5013 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
5014 }
5015}
5016
c0c050c5
MC		 5017static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
5018{
5019 int rc = 0;
5020
11809490 5021 bnxt_preset_reg_win(bp);
c0c050c5
MC		 5022 netif_carrier_off(bp->dev);
5023 if (irq_re_init) {
5024 rc = bnxt_setup_int_mode(bp);
5025 if (rc) {
5026 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
5027 rc);
5028 return rc;
5029 }
5030 }
5031 if ((bp->flags & BNXT_FLAG_RFS) &&
5032 !(bp->flags & BNXT_FLAG_USING_MSIX)) {
5033 /* disable RFS if falling back to INTA */
5034 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
5035 bp->flags &= ~BNXT_FLAG_RFS;
5036 }
5037
5038 rc = bnxt_alloc_mem(bp, irq_re_init);
5039 if (rc) {
5040 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
5041 goto open_err_free_mem;
5042 }
5043
5044 if (irq_re_init) {
5045 bnxt_init_napi(bp);
5046 rc = bnxt_request_irq(bp);
5047 if (rc) {
5048 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
5049 goto open_err;
5050 }
5051 }
5052
5053 bnxt_enable_napi(bp);
5054
5055 rc = bnxt_init_nic(bp, irq_re_init);
5056 if (rc) {
5057 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
5058 goto open_err;
5059 }
5060
5061 if (link_re_init) {
5062 rc = bnxt_update_phy_setting(bp);
5063 if (rc)
ba41d46f 5064 netdev_warn(bp->dev, "failed to update phy settings\n");
c0c050c5
MC		 5065 }
5066
5067 if (irq_re_init) {
5068#if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
5069 vxlan_get_rx_port(bp->dev);
5070#endif
5071 if (!bnxt_hwrm_tunnel_dst_port_alloc(
5072 bp, htons(0x17c1),
5073 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE))
5074 bp->nge_port_cnt = 1;
5075 }
5076
caefe526 5077 set_bit(BNXT_STATE_OPEN, &bp->state);
c0c050c5
MC		 5078 bnxt_enable_int(bp);
5079 /* Enable TX queues */
5080 bnxt_tx_enable(bp);
5081 mod_timer(&bp->timer, jiffies + bp->current_interval);
035a1539 5082 bnxt_update_link(bp, true);
c0c050c5
MC		 5083
5084 return 0;
5085
5086open_err:
5087 bnxt_disable_napi(bp);
5088 bnxt_del_napi(bp);
5089
5090open_err_free_mem:
5091 bnxt_free_skbs(bp);
5092 bnxt_free_irq(bp);
5093 bnxt_free_mem(bp, true);
5094 return rc;
5095}
5096
5097/* rtnl_lock held */
5098int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
5099{
5100 int rc = 0;
5101
5102 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
5103 if (rc) {
5104 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
5105 dev_close(bp->dev);
5106 }
5107 return rc;
5108}
5109
5110static int bnxt_open(struct net_device *dev)
5111{
5112 struct bnxt *bp = netdev_priv(dev);
5113 int rc = 0;
5114
5115 rc = bnxt_hwrm_func_reset(bp);
5116 if (rc) {
5117 netdev_err(bp->dev, "hwrm chip reset failure rc: %x\n",
5118 rc);
5119 rc = -1;
5120 return rc;
5121 }
5122 return __bnxt_open_nic(bp, true, true);
5123}
5124
5125static void bnxt_disable_int_sync(struct bnxt *bp)
5126{
5127 int i;
5128
5129 atomic_inc(&bp->intr_sem);
5130 if (!netif_running(bp->dev))
5131 return;
5132
5133 bnxt_disable_int(bp);
5134 for (i = 0; i < bp->cp_nr_rings; i++)
5135 synchronize_irq(bp->irq_tbl[i].vector);
5136}
5137
5138int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
5139{
5140 int rc = 0;
5141
5142#ifdef CONFIG_BNXT_SRIOV
5143 if (bp->sriov_cfg) {
5144 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
5145 !bp->sriov_cfg,
5146 BNXT_SRIOV_CFG_WAIT_TMO);
5147 if (rc)
5148 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
5149 }
5150#endif
5151 /* Change device state to avoid TX queue wake up's */
5152 bnxt_tx_disable(bp);
5153
caefe526 5154 clear_bit(BNXT_STATE_OPEN, &bp->state);
4cebdcec
MC		 5155 smp_mb__after_atomic();
5156 while (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state))
5157 msleep(20);
c0c050c5
MC		 5158
5159 /* Flush rings before disabling interrupts */
5160 bnxt_shutdown_nic(bp, irq_re_init);
5161
5162 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
5163
5164 bnxt_disable_napi(bp);
5165 bnxt_disable_int_sync(bp);
5166 del_timer_sync(&bp->timer);
5167 bnxt_free_skbs(bp);
5168
5169 if (irq_re_init) {
5170 bnxt_free_irq(bp);
5171 bnxt_del_napi(bp);
5172 }
5173 bnxt_free_mem(bp, irq_re_init);
5174 return rc;
5175}
5176
5177static int bnxt_close(struct net_device *dev)
5178{
5179 struct bnxt *bp = netdev_priv(dev);
5180
5181 bnxt_close_nic(bp, true, true);
33f7d55f 5182 bnxt_hwrm_shutdown_link(bp);
c0c050c5
MC		 5183 return 0;
5184}
5185
5186/* rtnl_lock held */
5187static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
5188{
5189 switch (cmd) {
5190 case SIOCGMIIPHY:
5191 /* fallthru */
5192 case SIOCGMIIREG: {
5193 if (!netif_running(dev))
5194 return -EAGAIN;
5195
5196 return 0;
5197 }
5198
5199 case SIOCSMIIREG:
5200 if (!netif_running(dev))
5201 return -EAGAIN;
5202
5203 return 0;
5204
5205 default:
5206 /* do nothing */
5207 break;
5208 }
5209 return -EOPNOTSUPP;
5210}
5211
5212static struct rtnl_link_stats64 *
5213bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
5214{
5215 u32 i;
5216 struct bnxt *bp = netdev_priv(dev);
5217
5218 memset(stats, 0, sizeof(struct rtnl_link_stats64));
5219
5220 if (!bp->bnapi)
5221 return stats;
5222
5223 /* TODO check if we need to synchronize with bnxt_close path */
5224 for (i = 0; i < bp->cp_nr_rings; i++) {
5225 struct bnxt_napi *bnapi = bp->bnapi[i];
5226 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5227 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
5228
5229 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
5230 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
5231 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
5232
5233 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
5234 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
5235 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
5236
5237 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
5238 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
5239 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
5240
5241 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
5242 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
5243 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
5244
5245 stats->rx_missed_errors +=
5246 le64_to_cpu(hw_stats->rx_discard_pkts);
5247
5248 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
5249
c0c050c5
MC		 5250 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
5251 }
5252
9947f83f
MC		 5253 if (bp->flags & BNXT_FLAG_PORT_STATS) {
5254 struct rx_port_stats *rx = bp->hw_rx_port_stats;
5255 struct tx_port_stats *tx = bp->hw_tx_port_stats;
5256
5257 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
5258 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
5259 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
5260 le64_to_cpu(rx->rx_ovrsz_frames) +
5261 le64_to_cpu(rx->rx_runt_frames);
5262 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
5263 le64_to_cpu(rx->rx_jbr_frames);
5264 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
5265 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
5266 stats->tx_errors = le64_to_cpu(tx->tx_err);
5267 }
5268
c0c050c5
MC		 5269 return stats;
5270}
5271
5272static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
5273{
5274 struct net_device *dev = bp->dev;
5275 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5276 struct netdev_hw_addr *ha;
5277 u8 *haddr;
5278 int mc_count = 0;
5279 bool update = false;
5280 int off = 0;
5281
5282 netdev_for_each_mc_addr(ha, dev) {
5283 if (mc_count >= BNXT_MAX_MC_ADDRS) {
5284 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5285 vnic->mc_list_count = 0;
5286 return false;
5287 }
5288 haddr = ha->addr;
5289 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
5290 memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
5291 update = true;
5292 }
5293 off += ETH_ALEN;
5294 mc_count++;
5295 }
5296 if (mc_count)
5297 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
5298
5299 if (mc_count != vnic->mc_list_count) {
5300 vnic->mc_list_count = mc_count;
5301 update = true;
5302 }
5303 return update;
5304}
5305
5306static bool bnxt_uc_list_updated(struct bnxt *bp)
5307{
5308 struct net_device *dev = bp->dev;
5309 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5310 struct netdev_hw_addr *ha;
5311 int off = 0;
5312
5313 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
5314 return true;
5315
5316 netdev_for_each_uc_addr(ha, dev) {
5317 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
5318 return true;
5319
5320 off += ETH_ALEN;
5321 }
5322 return false;
5323}
5324
5325static void bnxt_set_rx_mode(struct net_device *dev)
5326{
5327 struct bnxt *bp = netdev_priv(dev);
5328 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5329 u32 mask = vnic->rx_mask;
5330 bool mc_update = false;
5331 bool uc_update;
5332
5333 if (!netif_running(dev))
5334 return;
5335
5336 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
5337 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
5338 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST);
5339
5340 /* Only allow PF to be in promiscuous mode */
5341 if ((dev->flags & IFF_PROMISC) && BNXT_PF(bp))
5342 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5343
5344 uc_update = bnxt_uc_list_updated(bp);
5345
5346 if (dev->flags & IFF_ALLMULTI) {
5347 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5348 vnic->mc_list_count = 0;
5349 } else {
5350 mc_update = bnxt_mc_list_updated(bp, &mask);
5351 }
5352
5353 if (mask != vnic->rx_mask || uc_update || mc_update) {
5354 vnic->rx_mask = mask;
5355
5356 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
5357 schedule_work(&bp->sp_task);
5358 }
5359}
5360
b664f008 5361static int bnxt_cfg_rx_mode(struct bnxt *bp)
c0c050c5
MC		 5362{
5363 struct net_device *dev = bp->dev;
5364 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5365 struct netdev_hw_addr *ha;
5366 int i, off = 0, rc;
5367 bool uc_update;
5368
5369 netif_addr_lock_bh(dev);
5370 uc_update = bnxt_uc_list_updated(bp);
5371 netif_addr_unlock_bh(dev);
5372
5373 if (!uc_update)
5374 goto skip_uc;
5375
5376 mutex_lock(&bp->hwrm_cmd_lock);
5377 for (i = 1; i < vnic->uc_filter_count; i++) {
5378 struct hwrm_cfa_l2_filter_free_input req = {0};
5379
5380 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
5381 -1);
5382
5383 req.l2_filter_id = vnic->fw_l2_filter_id[i];
5384
5385 rc = _hwrm_send_message(bp, &req, sizeof(req),
5386 HWRM_CMD_TIMEOUT);
5387 }
5388 mutex_unlock(&bp->hwrm_cmd_lock);
5389
5390 vnic->uc_filter_count = 1;
5391
5392 netif_addr_lock_bh(dev);
5393 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
5394 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5395 } else {
5396 netdev_for_each_uc_addr(ha, dev) {
5397 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
5398 off += ETH_ALEN;
5399 vnic->uc_filter_count++;
5400 }
5401 }
5402 netif_addr_unlock_bh(dev);
5403
5404 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
5405 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
5406 if (rc) {
5407 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
5408 rc);
5409 vnic->uc_filter_count = i;
b664f008 5410 return rc;
c0c050c5
MC		 5411 }
5412 }
5413
5414skip_uc:
5415 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
5416 if (rc)
5417 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n",
5418 rc);
b664f008
MC		 5419
5420 return rc;
c0c050c5
MC		 5421}
5422
2bcfa6f6
MC		 5423static bool bnxt_rfs_capable(struct bnxt *bp)
5424{
5425#ifdef CONFIG_RFS_ACCEL
5426 struct bnxt_pf_info *pf = &bp->pf;
5427 int vnics;
5428
5429 if (BNXT_VF(bp) || !(bp->flags & BNXT_FLAG_MSIX_CAP))
5430 return false;
5431
5432 vnics = 1 + bp->rx_nr_rings;
5433 if (vnics > pf->max_rsscos_ctxs || vnics > pf->max_vnics)
5434 return false;
5435
5436 return true;
5437#else
5438 return false;
5439#endif
5440}
5441
c0c050c5
MC		 5442static netdev_features_t bnxt_fix_features(struct net_device *dev,
5443 netdev_features_t features)
5444{
2bcfa6f6
MC		 5445 struct bnxt *bp = netdev_priv(dev);
5446
5447 if (!bnxt_rfs_capable(bp))
5448 features &= ~NETIF_F_NTUPLE;
c0c050c5
MC		 5449 return features;
5450}
5451
5452static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
5453{
5454 struct bnxt *bp = netdev_priv(dev);
5455 u32 flags = bp->flags;
5456 u32 changes;
5457 int rc = 0;
5458 bool re_init = false;
5459 bool update_tpa = false;
5460
5461 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
5462 if ((features & NETIF_F_GRO) && (bp->pdev->revision > 0))
5463 flags |= BNXT_FLAG_GRO;
5464 if (features & NETIF_F_LRO)
5465 flags |= BNXT_FLAG_LRO;
5466
5467 if (features & NETIF_F_HW_VLAN_CTAG_RX)
5468 flags |= BNXT_FLAG_STRIP_VLAN;
5469
5470 if (features & NETIF_F_NTUPLE)
5471 flags |= BNXT_FLAG_RFS;
5472
5473 changes = flags ^ bp->flags;
5474 if (changes & BNXT_FLAG_TPA) {
5475 update_tpa = true;
5476 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
5477 (flags & BNXT_FLAG_TPA) == 0)
5478 re_init = true;
5479 }
5480
5481 if (changes & ~BNXT_FLAG_TPA)
5482 re_init = true;
5483
5484 if (flags != bp->flags) {
5485 u32 old_flags = bp->flags;
5486
5487 bp->flags = flags;
5488
2bcfa6f6 5489 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
c0c050c5
MC		 5490 if (update_tpa)
5491 bnxt_set_ring_params(bp);
5492 return rc;
5493 }
5494
5495 if (re_init) {
5496 bnxt_close_nic(bp, false, false);
5497 if (update_tpa)
5498 bnxt_set_ring_params(bp);
5499
5500 return bnxt_open_nic(bp, false, false);
5501 }
5502 if (update_tpa) {
5503 rc = bnxt_set_tpa(bp,
5504 (flags & BNXT_FLAG_TPA) ?
5505 true : false);
5506 if (rc)
5507 bp->flags = old_flags;
5508 }
5509 }
5510 return rc;
5511}
5512
9f554590
MC		 5513static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
5514{
b6ab4b01 5515 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
9f554590
MC		 5516 int i = bnapi->index;
5517
3b2b7d9d
MC		 5518 if (!txr)
5519 return;
5520
9f554590
MC		 5521 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
5522 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
5523 txr->tx_cons);
5524}
5525
5526static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
5527{
b6ab4b01 5528 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
9f554590
MC		 5529 int i = bnapi->index;
5530
3b2b7d9d
MC		 5531 if (!rxr)
5532 return;
5533
9f554590
MC		 5534 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
5535 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
5536 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
5537 rxr->rx_sw_agg_prod);
5538}
5539
5540static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
5541{
5542 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5543 int i = bnapi->index;
5544
5545 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
5546 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
5547}
5548
c0c050c5
MC		 5549static void bnxt_dbg_dump_states(struct bnxt *bp)
5550{
5551 int i;
5552 struct bnxt_napi *bnapi;
c0c050c5
MC		 5553
5554 for (i = 0; i < bp->cp_nr_rings; i++) {
5555 bnapi = bp->bnapi[i];
c0c050c5 5556 if (netif_msg_drv(bp)) {
9f554590
MC		 5557 bnxt_dump_tx_sw_state(bnapi);
5558 bnxt_dump_rx_sw_state(bnapi);
5559 bnxt_dump_cp_sw_state(bnapi);
c0c050c5
MC		 5560 }
5561 }
5562}
5563
5564static void bnxt_reset_task(struct bnxt *bp)
5565{
5566 bnxt_dbg_dump_states(bp);
028de140
MC		 5567 if (netif_running(bp->dev)) {
5568 bnxt_close_nic(bp, false, false);
5569 bnxt_open_nic(bp, false, false);
5570 }
c0c050c5
MC		 5571}
5572
5573static void bnxt_tx_timeout(struct net_device *dev)
5574{
5575 struct bnxt *bp = netdev_priv(dev);
5576
5577 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n");
5578 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
5579 schedule_work(&bp->sp_task);
5580}
5581
5582#ifdef CONFIG_NET_POLL_CONTROLLER
5583static void bnxt_poll_controller(struct net_device *dev)
5584{
5585 struct bnxt *bp = netdev_priv(dev);
5586 int i;
5587
5588 for (i = 0; i < bp->cp_nr_rings; i++) {
5589 struct bnxt_irq *irq = &bp->irq_tbl[i];
5590
5591 disable_irq(irq->vector);
5592 irq->handler(irq->vector, bp->bnapi[i]);
5593 enable_irq(irq->vector);
5594 }
5595}
5596#endif
5597
5598static void bnxt_timer(unsigned long data)
5599{
5600 struct bnxt *bp = (struct bnxt *)data;
5601 struct net_device *dev = bp->dev;
5602
5603 if (!netif_running(dev))
5604 return;
5605
5606 if (atomic_read(&bp->intr_sem) != 0)
5607 goto bnxt_restart_timer;
5608
3bdf56c4
MC		 5609 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS)) {
5610 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
5611 schedule_work(&bp->sp_task);
5612 }
c0c050c5
MC		 5613bnxt_restart_timer:
5614 mod_timer(&bp->timer, jiffies + bp->current_interval);
5615}
5616
4bb13abf
MC		 5617static void bnxt_port_module_event(struct bnxt *bp)
5618{
5619 struct bnxt_link_info *link_info = &bp->link_info;
5620 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
5621
5622 if (bnxt_update_link(bp, true))
5623 return;
5624
5625 if (link_info->last_port_module_event != 0) {
5626 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
5627 bp->pf.port_id);
5628 if (bp->hwrm_spec_code >= 0x10201) {
5629 netdev_warn(bp->dev, "Module part number %s\n",
5630 resp->phy_vendor_partnumber);
5631 }
5632 }
5633 if (link_info->last_port_module_event == 1)
5634 netdev_warn(bp->dev, "TX is disabled\n");
5635 if (link_info->last_port_module_event == 3)
5636 netdev_warn(bp->dev, "Shutdown SFP+ module\n");
5637}
5638
c0c050c5
MC		 5639static void bnxt_cfg_ntp_filters(struct bnxt *);
5640
5641static void bnxt_sp_task(struct work_struct *work)
5642{
5643 struct bnxt *bp = container_of(work, struct bnxt, sp_task);
5644 int rc;
5645
4cebdcec
MC		 5646 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5647 smp_mb__after_atomic();
5648 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
5649 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
c0c050c5 5650 return;
4cebdcec 5651 }
c0c050c5
MC		 5652
5653 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
5654 bnxt_cfg_rx_mode(bp);
5655
5656 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
5657 bnxt_cfg_ntp_filters(bp);
5658 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
5659 rc = bnxt_update_link(bp, true);
5660 if (rc)
5661 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
5662 rc);
5663 }
5664 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
5665 bnxt_hwrm_exec_fwd_req(bp);
5666 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
5667 bnxt_hwrm_tunnel_dst_port_alloc(
5668 bp, bp->vxlan_port,
5669 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
5670 }
5671 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
5672 bnxt_hwrm_tunnel_dst_port_free(
5673 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
5674 }
028de140
MC		 5675 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) {
5676 /* bnxt_reset_task() calls bnxt_close_nic() which waits
5677 * for BNXT_STATE_IN_SP_TASK to clear.
5678 */
5679 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5680 rtnl_lock();
c0c050c5 5681 bnxt_reset_task(bp);
028de140
MC		 5682 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5683 rtnl_unlock();
5684 }
4cebdcec 5685
4bb13abf
MC		 5686 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event))
5687 bnxt_port_module_event(bp);
5688
3bdf56c4
MC		 5689 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event))
5690 bnxt_hwrm_port_qstats(bp);
5691
4cebdcec
MC		 5692 smp_mb__before_atomic();
5693 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
c0c050c5
MC		 5694}
5695
5696static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
5697{
5698 int rc;
5699 struct bnxt *bp = netdev_priv(dev);
5700
5701 SET_NETDEV_DEV(dev, &pdev->dev);
5702
5703 /* enable device (incl. PCI PM wakeup), and bus-mastering */
5704 rc = pci_enable_device(pdev);
5705 if (rc) {
5706 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
5707 goto init_err;
5708 }
5709
5710 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
5711 dev_err(&pdev->dev,
5712 "Cannot find PCI device base address, aborting\n");
5713 rc = -ENODEV;
5714 goto init_err_disable;
5715 }
5716
5717 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
5718 if (rc) {
5719 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
5720 goto init_err_disable;
5721 }
5722
5723 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
5724 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
5725 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
5726 goto init_err_disable;
5727 }
5728
5729 pci_set_master(pdev);
5730
5731 bp->dev = dev;
5732 bp->pdev = pdev;
5733
5734 bp->bar0 = pci_ioremap_bar(pdev, 0);
5735 if (!bp->bar0) {
5736 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
5737 rc = -ENOMEM;
5738 goto init_err_release;
5739 }
5740
5741 bp->bar1 = pci_ioremap_bar(pdev, 2);
5742 if (!bp->bar1) {
5743 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
5744 rc = -ENOMEM;
5745 goto init_err_release;
5746 }
5747
5748 bp->bar2 = pci_ioremap_bar(pdev, 4);
5749 if (!bp->bar2) {
5750 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
5751 rc = -ENOMEM;
5752 goto init_err_release;
5753 }
5754
6316ea6d
SB		 5755 pci_enable_pcie_error_reporting(pdev);
5756
c0c050c5
MC		 5757 INIT_WORK(&bp->sp_task, bnxt_sp_task);
5758
5759 spin_lock_init(&bp->ntp_fltr_lock);
5760
5761 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
5762 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
5763
dfb5b894 5764 /* tick values in micro seconds */
dfc9c94a
MC		 5765 bp->rx_coal_ticks = 12;
5766 bp->rx_coal_bufs = 30;
dfb5b894
MC		 5767 bp->rx_coal_ticks_irq = 1;
5768 bp->rx_coal_bufs_irq = 2;
c0c050c5 5769
dfc9c94a
MC		 5770 bp->tx_coal_ticks = 25;
5771 bp->tx_coal_bufs = 30;
5772 bp->tx_coal_ticks_irq = 2;
5773 bp->tx_coal_bufs_irq = 2;
5774
c0c050c5
MC		 5775 init_timer(&bp->timer);
5776 bp->timer.data = (unsigned long)bp;
5777 bp->timer.function = bnxt_timer;
5778 bp->current_interval = BNXT_TIMER_INTERVAL;
5779
caefe526 5780 clear_bit(BNXT_STATE_OPEN, &bp->state);
c0c050c5
MC		 5781
5782 return 0;
5783
5784init_err_release:
5785 if (bp->bar2) {
5786 pci_iounmap(pdev, bp->bar2);
5787 bp->bar2 = NULL;
5788 }
5789
5790 if (bp->bar1) {
5791 pci_iounmap(pdev, bp->bar1);
5792 bp->bar1 = NULL;
5793 }
5794
5795 if (bp->bar0) {
5796 pci_iounmap(pdev, bp->bar0);
5797 bp->bar0 = NULL;
5798 }
5799
5800 pci_release_regions(pdev);
5801
5802init_err_disable:
5803 pci_disable_device(pdev);
5804
5805init_err:
5806 return rc;
5807}
5808
5809/* rtnl_lock held */
5810static int bnxt_change_mac_addr(struct net_device *dev, void *p)
5811{
5812 struct sockaddr *addr = p;
1fc2cfd0
JH		 5813 struct bnxt *bp = netdev_priv(dev);
5814 int rc = 0;
c0c050c5
MC		 5815
5816 if (!is_valid_ether_addr(addr->sa_data))
5817 return -EADDRNOTAVAIL;
5818
84c33dd3
MC		 5819 rc = bnxt_approve_mac(bp, addr->sa_data);
5820 if (rc)
5821 return rc;
bdd4347b 5822
1fc2cfd0
JH		 5823 if (ether_addr_equal(addr->sa_data, dev->dev_addr))
5824 return 0;
5825
c0c050c5 5826 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
1fc2cfd0
JH		 5827 if (netif_running(dev)) {
5828 bnxt_close_nic(bp, false, false);
5829 rc = bnxt_open_nic(bp, false, false);
5830 }
c0c050c5 5831
1fc2cfd0 5832 return rc;
c0c050c5
MC		 5833}
5834
5835/* rtnl_lock held */
5836static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
5837{
5838 struct bnxt *bp = netdev_priv(dev);
5839
5840 if (new_mtu < 60 || new_mtu > 9000)
5841 return -EINVAL;
5842
5843 if (netif_running(dev))
5844 bnxt_close_nic(bp, false, false);
5845
5846 dev->mtu = new_mtu;
5847 bnxt_set_ring_params(bp);
5848
5849 if (netif_running(dev))
5850 return bnxt_open_nic(bp, false, false);
5851
5852 return 0;
5853}
5854
16e5cc64
JF		 5855static int bnxt_setup_tc(struct net_device *dev, u32 handle, __be16 proto,
5856 struct tc_to_netdev *ntc)
c0c050c5
MC		 5857{
5858 struct bnxt *bp = netdev_priv(dev);
16e5cc64 5859 u8 tc;
c0c050c5 5860
5eb4dce3 5861 if (ntc->type != TC_SETUP_MQPRIO)
e4c6734e
JF		 5862 return -EINVAL;
5863
16e5cc64
JF		 5864 tc = ntc->tc;
5865
c0c050c5
MC		 5866 if (tc > bp->max_tc) {
5867 netdev_err(dev, "too many traffic classes requested: %d Max supported is %d\n",
5868 tc, bp->max_tc);
5869 return -EINVAL;
5870 }
5871
5872 if (netdev_get_num_tc(dev) == tc)
5873 return 0;
5874
5875 if (tc) {
6e6c5a57 5876 int max_rx_rings, max_tx_rings, rc;
01657bcd
MC		 5877 bool sh = false;
5878
5879 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5880 sh = true;
c0c050c5 5881
01657bcd 5882 rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh);
6e6c5a57 5883 if (rc || bp->tx_nr_rings_per_tc * tc > max_tx_rings)
c0c050c5
MC		 5884 return -ENOMEM;
5885 }
5886
5887 /* Needs to close the device and do hw resource re-allocations */
5888 if (netif_running(bp->dev))
5889 bnxt_close_nic(bp, true, false);
5890
5891 if (tc) {
5892 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
5893 netdev_set_num_tc(dev, tc);
5894 } else {
5895 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
5896 netdev_reset_tc(dev);
5897 }
5898 bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
5899 bp->num_stat_ctxs = bp->cp_nr_rings;
5900
5901 if (netif_running(bp->dev))
5902 return bnxt_open_nic(bp, true, false);
5903
5904 return 0;
5905}
5906
5907#ifdef CONFIG_RFS_ACCEL
5908static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
5909 struct bnxt_ntuple_filter *f2)
5910{
5911 struct flow_keys *keys1 = &f1->fkeys;
5912 struct flow_keys *keys2 = &f2->fkeys;
5913
5914 if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
5915 keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
5916 keys1->ports.ports == keys2->ports.ports &&
5917 keys1->basic.ip_proto == keys2->basic.ip_proto &&
5918 keys1->basic.n_proto == keys2->basic.n_proto &&
5919 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr))
5920 return true;
5921
5922 return false;
5923}
5924
5925static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
5926 u16 rxq_index, u32 flow_id)
5927{
5928 struct bnxt *bp = netdev_priv(dev);
5929 struct bnxt_ntuple_filter *fltr, *new_fltr;
5930 struct flow_keys *fkeys;
5931 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
84e86b98 5932 int rc = 0, idx, bit_id;
c0c050c5
MC		 5933 struct hlist_head *head;
5934
5935 if (skb->encapsulation)
5936 return -EPROTONOSUPPORT;
5937
5938 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
5939 if (!new_fltr)
5940 return -ENOMEM;
5941
5942 fkeys = &new_fltr->fkeys;
5943 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
5944 rc = -EPROTONOSUPPORT;
5945 goto err_free;
5946 }
5947
5948 if ((fkeys->basic.n_proto != htons(ETH_P_IP)) ||
5949 ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
5950 (fkeys->basic.ip_proto != IPPROTO_UDP))) {
5951 rc = -EPROTONOSUPPORT;
5952 goto err_free;
5953 }
5954
5955 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
5956
5957 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
5958 head = &bp->ntp_fltr_hash_tbl[idx];
5959 rcu_read_lock();
5960 hlist_for_each_entry_rcu(fltr, head, hash) {
5961 if (bnxt_fltr_match(fltr, new_fltr)) {
5962 rcu_read_unlock();
5963 rc = 0;
5964 goto err_free;
5965 }
5966 }
5967 rcu_read_unlock();
5968
5969 spin_lock_bh(&bp->ntp_fltr_lock);
84e86b98
MC		 5970 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
5971 BNXT_NTP_FLTR_MAX_FLTR, 0);
5972 if (bit_id < 0) {
c0c050c5
MC		 5973 spin_unlock_bh(&bp->ntp_fltr_lock);
5974 rc = -ENOMEM;
5975 goto err_free;
5976 }
5977
84e86b98 5978 new_fltr->sw_id = (u16)bit_id;
c0c050c5
MC		 5979 new_fltr->flow_id = flow_id;
5980 new_fltr->rxq = rxq_index;
5981 hlist_add_head_rcu(&new_fltr->hash, head);
5982 bp->ntp_fltr_count++;
5983 spin_unlock_bh(&bp->ntp_fltr_lock);
5984
5985 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
5986 schedule_work(&bp->sp_task);
5987
5988 return new_fltr->sw_id;
5989
5990err_free:
5991 kfree(new_fltr);
5992 return rc;
5993}
5994
5995static void bnxt_cfg_ntp_filters(struct bnxt *bp)
5996{
5997 int i;
5998
5999 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
6000 struct hlist_head *head;
6001 struct hlist_node *tmp;
6002 struct bnxt_ntuple_filter *fltr;
6003 int rc;
6004
6005 head = &bp->ntp_fltr_hash_tbl[i];
6006 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
6007 bool del = false;
6008
6009 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
6010 if (rps_may_expire_flow(bp->dev, fltr->rxq,
6011 fltr->flow_id,
6012 fltr->sw_id)) {
6013 bnxt_hwrm_cfa_ntuple_filter_free(bp,
6014 fltr);
6015 del = true;
6016 }
6017 } else {
6018 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
6019 fltr);
6020 if (rc)
6021 del = true;
6022 else
6023 set_bit(BNXT_FLTR_VALID, &fltr->state);
6024 }
6025
6026 if (del) {
6027 spin_lock_bh(&bp->ntp_fltr_lock);
6028 hlist_del_rcu(&fltr->hash);
6029 bp->ntp_fltr_count--;
6030 spin_unlock_bh(&bp->ntp_fltr_lock);
6031 synchronize_rcu();
6032 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
6033 kfree(fltr);
6034 }
6035 }
6036 }
19241368
JH		 6037 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
6038 netdev_info(bp->dev, "Receive PF driver unload event!");
c0c050c5
MC		 6039}
6040
6041#else
6042
6043static void bnxt_cfg_ntp_filters(struct bnxt *bp)
6044{
6045}
6046
6047#endif /* CONFIG_RFS_ACCEL */
6048
6049static void bnxt_add_vxlan_port(struct net_device *dev, sa_family_t sa_family,
6050 __be16 port)
6051{
6052 struct bnxt *bp = netdev_priv(dev);
6053
6054 if (!netif_running(dev))
6055 return;
6056
6057 if (sa_family != AF_INET6 && sa_family != AF_INET)
6058 return;
6059
6060 if (bp->vxlan_port_cnt && bp->vxlan_port != port)
6061 return;
6062
6063 bp->vxlan_port_cnt++;
6064 if (bp->vxlan_port_cnt == 1) {
6065 bp->vxlan_port = port;
6066 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
6067 schedule_work(&bp->sp_task);
6068 }
6069}
6070
6071static void bnxt_del_vxlan_port(struct net_device *dev, sa_family_t sa_family,
6072 __be16 port)
6073{
6074 struct bnxt *bp = netdev_priv(dev);
6075
6076 if (!netif_running(dev))
6077 return;
6078
6079 if (sa_family != AF_INET6 && sa_family != AF_INET)
6080 return;
6081
6082 if (bp->vxlan_port_cnt && bp->vxlan_port == port) {
6083 bp->vxlan_port_cnt--;
6084
6085 if (bp->vxlan_port_cnt == 0) {
6086 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
6087 schedule_work(&bp->sp_task);
6088 }
6089 }
6090}
6091
6092static const struct net_device_ops bnxt_netdev_ops = {
6093 .ndo_open = bnxt_open,
6094 .ndo_start_xmit = bnxt_start_xmit,
6095 .ndo_stop = bnxt_close,
6096 .ndo_get_stats64 = bnxt_get_stats64,
6097 .ndo_set_rx_mode = bnxt_set_rx_mode,
6098 .ndo_do_ioctl = bnxt_ioctl,
6099 .ndo_validate_addr = eth_validate_addr,
6100 .ndo_set_mac_address = bnxt_change_mac_addr,
6101 .ndo_change_mtu = bnxt_change_mtu,
6102 .ndo_fix_features = bnxt_fix_features,
6103 .ndo_set_features = bnxt_set_features,
6104 .ndo_tx_timeout = bnxt_tx_timeout,
6105#ifdef CONFIG_BNXT_SRIOV
6106 .ndo_get_vf_config = bnxt_get_vf_config,
6107 .ndo_set_vf_mac = bnxt_set_vf_mac,
6108 .ndo_set_vf_vlan = bnxt_set_vf_vlan,
6109 .ndo_set_vf_rate = bnxt_set_vf_bw,
6110 .ndo_set_vf_link_state = bnxt_set_vf_link_state,
6111 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk,
6112#endif
6113#ifdef CONFIG_NET_POLL_CONTROLLER
6114 .ndo_poll_controller = bnxt_poll_controller,
6115#endif
6116 .ndo_setup_tc = bnxt_setup_tc,
6117#ifdef CONFIG_RFS_ACCEL
6118 .ndo_rx_flow_steer = bnxt_rx_flow_steer,
6119#endif
6120 .ndo_add_vxlan_port = bnxt_add_vxlan_port,
6121 .ndo_del_vxlan_port = bnxt_del_vxlan_port,
6122#ifdef CONFIG_NET_RX_BUSY_POLL
6123 .ndo_busy_poll = bnxt_busy_poll,
6124#endif
6125};
6126
6127static void bnxt_remove_one(struct pci_dev *pdev)
6128{
6129 struct net_device *dev = pci_get_drvdata(pdev);
6130 struct bnxt *bp = netdev_priv(dev);
6131
6132 if (BNXT_PF(bp))
6133 bnxt_sriov_disable(bp);
6134
6316ea6d 6135 pci_disable_pcie_error_reporting(pdev);
c0c050c5
MC		 6136 unregister_netdev(dev);
6137 cancel_work_sync(&bp->sp_task);
6138 bp->sp_event = 0;
6139
be58a0da 6140 bnxt_hwrm_func_drv_unrgtr(bp);
c0c050c5
MC		 6141 bnxt_free_hwrm_resources(bp);
6142 pci_iounmap(pdev, bp->bar2);
6143 pci_iounmap(pdev, bp->bar1);
6144 pci_iounmap(pdev, bp->bar0);
6145 free_netdev(dev);
6146
6147 pci_release_regions(pdev);
6148 pci_disable_device(pdev);
6149}
6150
6151static int bnxt_probe_phy(struct bnxt *bp)
6152{
6153 int rc = 0;
6154 struct bnxt_link_info *link_info = &bp->link_info;
c0c050c5 6155
170ce013
MC		 6156 rc = bnxt_hwrm_phy_qcaps(bp);
6157 if (rc) {
6158 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
6159 rc);
6160 return rc;
6161 }
6162
c0c050c5
MC		 6163 rc = bnxt_update_link(bp, false);
6164 if (rc) {
6165 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
6166 rc);
6167 return rc;
6168 }
6169
6170 /*initialize the ethool setting copy with NVM settings */
0d8abf02 6171 if (BNXT_AUTO_MODE(link_info->auto_mode)) {
c9ee9516
MC		 6172 link_info->autoneg = BNXT_AUTONEG_SPEED;
6173 if (bp->hwrm_spec_code >= 0x10201) {
6174 if (link_info->auto_pause_setting &
6175 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
6176 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
6177 } else {
6178 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
6179 }
0d8abf02 6180 link_info->advertising = link_info->auto_link_speeds;
0d8abf02
MC		 6181 } else {
6182 link_info->req_link_speed = link_info->force_link_speed;
6183 link_info->req_duplex = link_info->duplex_setting;
c0c050c5 6184 }
c9ee9516
MC		 6185 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
6186 link_info->req_flow_ctrl =
6187 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
6188 else
6189 link_info->req_flow_ctrl = link_info->force_pause_setting;
c0c050c5
MC		 6190 return rc;
6191}
6192
6193static int bnxt_get_max_irq(struct pci_dev *pdev)
6194{
6195 u16 ctrl;
6196
6197 if (!pdev->msix_cap)
6198 return 1;
6199
6200 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
6201 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
6202}
6203
6e6c5a57
MC		 6204static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
6205 int *max_cp)
c0c050c5 6206{
6e6c5a57 6207 int max_ring_grps = 0;
c0c050c5 6208
379a80a1 6209#ifdef CONFIG_BNXT_SRIOV
415b6f19 6210 if (!BNXT_PF(bp)) {
c0c050c5
MC		 6211 *max_tx = bp->vf.max_tx_rings;
6212 *max_rx = bp->vf.max_rx_rings;
6e6c5a57
MC		 6213 *max_cp = min_t(int, bp->vf.max_irqs, bp->vf.max_cp_rings);
6214 *max_cp = min_t(int, *max_cp, bp->vf.max_stat_ctxs);
b72d4a68 6215 max_ring_grps = bp->vf.max_hw_ring_grps;
415b6f19 6216 } else
379a80a1 6217#endif
415b6f19
AB		 6218 {
6219 *max_tx = bp->pf.max_tx_rings;
6220 *max_rx = bp->pf.max_rx_rings;
6221 *max_cp = min_t(int, bp->pf.max_irqs, bp->pf.max_cp_rings);
6222 *max_cp = min_t(int, *max_cp, bp->pf.max_stat_ctxs);
6223 max_ring_grps = bp->pf.max_hw_ring_grps;
c0c050c5 6224 }
415b6f19 6225
c0c050c5
MC		 6226 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6227 *max_rx >>= 1;
b72d4a68 6228 *max_rx = min_t(int, *max_rx, max_ring_grps);
6e6c5a57
MC		 6229}
6230
6231int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
6232{
6233 int rx, tx, cp;
6234
6235 _bnxt_get_max_rings(bp, &rx, &tx, &cp);
6236 if (!rx || !tx || !cp)
6237 return -ENOMEM;
6238
6239 *max_rx = rx;
6240 *max_tx = tx;
6241 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
6242}
6243
6244static int bnxt_set_dflt_rings(struct bnxt *bp)
6245{
6246 int dflt_rings, max_rx_rings, max_tx_rings, rc;
6247 bool sh = true;
6248
6249 if (sh)
6250 bp->flags |= BNXT_FLAG_SHARED_RINGS;
6251 dflt_rings = netif_get_num_default_rss_queues();
6252 rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh);
6253 if (rc)
6254 return rc;
6255 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
6256 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
6257 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
6258 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
6259 bp->tx_nr_rings + bp->rx_nr_rings;
6260 bp->num_stat_ctxs = bp->cp_nr_rings;
6261 return rc;
c0c050c5
MC		 6262}
6263
90c4f788
AK		 6264static void bnxt_parse_log_pcie_link(struct bnxt *bp)
6265{
6266 enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
6267 enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
6268
6269 if (pcie_get_minimum_link(bp->pdev, &speed, &width) ||
6270 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
6271 netdev_info(bp->dev, "Failed to determine PCIe Link Info\n");
6272 else
6273 netdev_info(bp->dev, "PCIe: Speed %s Width x%d\n",
6274 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
6275 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
6276 speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
6277 "Unknown", width);
6278}
6279
c0c050c5
MC		 6280static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
6281{
6282 static int version_printed;
6283 struct net_device *dev;
6284 struct bnxt *bp;
6e6c5a57 6285 int rc, max_irqs;
c0c050c5
MC		 6286
6287 if (version_printed++ == 0)
6288 pr_info("%s", version);
6289
6290 max_irqs = bnxt_get_max_irq(pdev);
6291 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
6292 if (!dev)
6293 return -ENOMEM;
6294
6295 bp = netdev_priv(dev);
6296
6297 if (bnxt_vf_pciid(ent->driver_data))
6298 bp->flags |= BNXT_FLAG_VF;
6299
2bcfa6f6 6300 if (pdev->msix_cap)
c0c050c5 6301 bp->flags |= BNXT_FLAG_MSIX_CAP;
c0c050c5
MC		 6302
6303 rc = bnxt_init_board(pdev, dev);
6304 if (rc < 0)
6305 goto init_err_free;
6306
6307 dev->netdev_ops = &bnxt_netdev_ops;
6308 dev->watchdog_timeo = BNXT_TX_TIMEOUT;
6309 dev->ethtool_ops = &bnxt_ethtool_ops;
6310
6311 pci_set_drvdata(pdev, dev);
6312
6313 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
6314 NETIF_F_TSO | NETIF_F_TSO6 |
6315 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
6316 NETIF_F_GSO_IPIP | NETIF_F_GSO_SIT |
152971ee
AD		 6317 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
6318 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
c0c050c5
MC		 6319 NETIF_F_RXCSUM | NETIF_F_LRO | NETIF_F_GRO;
6320
c0c050c5
MC		 6321 dev->hw_enc_features =
6322 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
6323 NETIF_F_TSO | NETIF_F_TSO6 |
6324 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
152971ee
AD		 6325 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
6326 NETIF_F_GSO_IPIP | NETIF_F_GSO_SIT |
6327 NETIF_F_GSO_PARTIAL;
6328 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
6329 NETIF_F_GSO_GRE_CSUM;
c0c050c5
MC		 6330 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
6331 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
6332 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
6333 dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
6334 dev->priv_flags |= IFF_UNICAST_FLT;
6335
6336#ifdef CONFIG_BNXT_SRIOV
6337 init_waitqueue_head(&bp->sriov_cfg_wait);
6338#endif
6339 rc = bnxt_alloc_hwrm_resources(bp);
6340 if (rc)
6341 goto init_err;
6342
6343 mutex_init(&bp->hwrm_cmd_lock);
6344 bnxt_hwrm_ver_get(bp);
6345
6346 rc = bnxt_hwrm_func_drv_rgtr(bp);
6347 if (rc)
6348 goto init_err;
6349
6350 /* Get the MAX capabilities for this function */
6351 rc = bnxt_hwrm_func_qcaps(bp);
6352 if (rc) {
6353 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
6354 rc);
6355 rc = -1;
6356 goto init_err;
6357 }
6358
6359 rc = bnxt_hwrm_queue_qportcfg(bp);
6360 if (rc) {
6361 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
6362 rc);
6363 rc = -1;
6364 goto init_err;
6365 }
6366
6367 bnxt_set_tpa_flags(bp);
6368 bnxt_set_ring_params(bp);
bdd4347b 6369 if (BNXT_PF(bp))
c0c050c5 6370 bp->pf.max_irqs = max_irqs;
379a80a1 6371#if defined(CONFIG_BNXT_SRIOV)
bdd4347b 6372 else
c0c050c5 6373 bp->vf.max_irqs = max_irqs;
379a80a1 6374#endif
6e6c5a57 6375 bnxt_set_dflt_rings(bp);
c0c050c5 6376
2bcfa6f6
MC		 6377 if (BNXT_PF(bp)) {
6378 dev->hw_features |= NETIF_F_NTUPLE;
6379 if (bnxt_rfs_capable(bp)) {
6380 bp->flags |= BNXT_FLAG_RFS;
6381 dev->features |= NETIF_F_NTUPLE;
6382 }
6383 }
6384
c0c050c5
MC		 6385 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
6386 bp->flags |= BNXT_FLAG_STRIP_VLAN;
6387
6388 rc = bnxt_probe_phy(bp);
6389 if (rc)
6390 goto init_err;
6391
6392 rc = register_netdev(dev);
6393 if (rc)
6394 goto init_err;
6395
6396 netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
6397 board_info[ent->driver_data].name,
6398 (long)pci_resource_start(pdev, 0), dev->dev_addr);
6399
90c4f788
AK		 6400 bnxt_parse_log_pcie_link(bp);
6401
c0c050c5
MC		 6402 return 0;
6403
6404init_err:
6405 pci_iounmap(pdev, bp->bar0);
6406 pci_release_regions(pdev);
6407 pci_disable_device(pdev);
6408
6409init_err_free:
6410 free_netdev(dev);
6411 return rc;
6412}
6413
6316ea6d
SB		 6414/**
6415 * bnxt_io_error_detected - called when PCI error is detected
6416 * @pdev: Pointer to PCI device
6417 * @state: The current pci connection state
6418 *
6419 * This function is called after a PCI bus error affecting
6420 * this device has been detected.
6421 */
6422static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
6423 pci_channel_state_t state)
6424{
6425 struct net_device *netdev = pci_get_drvdata(pdev);
6426
6427 netdev_info(netdev, "PCI I/O error detected\n");
6428
6429 rtnl_lock();
6430 netif_device_detach(netdev);
6431
6432 if (state == pci_channel_io_perm_failure) {
6433 rtnl_unlock();
6434 return PCI_ERS_RESULT_DISCONNECT;
6435 }
6436
6437 if (netif_running(netdev))
6438 bnxt_close(netdev);
6439
6440 pci_disable_device(pdev);
6441 rtnl_unlock();
6442
6443 /* Request a slot slot reset. */
6444 return PCI_ERS_RESULT_NEED_RESET;
6445}
6446
6447/**
6448 * bnxt_io_slot_reset - called after the pci bus has been reset.
6449 * @pdev: Pointer to PCI device
6450 *
6451 * Restart the card from scratch, as if from a cold-boot.
6452 * At this point, the card has exprienced a hard reset,
6453 * followed by fixups by BIOS, and has its config space
6454 * set up identically to what it was at cold boot.
6455 */
6456static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
6457{
6458 struct net_device *netdev = pci_get_drvdata(pdev);
6459 struct bnxt *bp = netdev_priv(netdev);
6460 int err = 0;
6461 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
6462
6463 netdev_info(bp->dev, "PCI Slot Reset\n");
6464
6465 rtnl_lock();
6466
6467 if (pci_enable_device(pdev)) {
6468 dev_err(&pdev->dev,
6469 "Cannot re-enable PCI device after reset.\n");
6470 } else {
6471 pci_set_master(pdev);
6472
6473 if (netif_running(netdev))
6474 err = bnxt_open(netdev);
6475
6476 if (!err)
6477 result = PCI_ERS_RESULT_RECOVERED;
6478 }
6479
6480 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
6481 dev_close(netdev);
6482
6483 rtnl_unlock();
6484
6485 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6486 if (err) {
6487 dev_err(&pdev->dev,
6488 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
6489 err); /* non-fatal, continue */
6490 }
6491
6492 return PCI_ERS_RESULT_RECOVERED;
6493}
6494
6495/**
6496 * bnxt_io_resume - called when traffic can start flowing again.
6497 * @pdev: Pointer to PCI device
6498 *
6499 * This callback is called when the error recovery driver tells
6500 * us that its OK to resume normal operation.
6501 */
6502static void bnxt_io_resume(struct pci_dev *pdev)
6503{
6504 struct net_device *netdev = pci_get_drvdata(pdev);
6505
6506 rtnl_lock();
6507
6508 netif_device_attach(netdev);
6509
6510 rtnl_unlock();
6511}
6512
6513static const struct pci_error_handlers bnxt_err_handler = {
6514 .error_detected = bnxt_io_error_detected,
6515 .slot_reset = bnxt_io_slot_reset,
6516 .resume = bnxt_io_resume
6517};
6518
c0c050c5
MC		 6519static struct pci_driver bnxt_pci_driver = {
6520 .name = DRV_MODULE_NAME,
6521 .id_table = bnxt_pci_tbl,
6522 .probe = bnxt_init_one,
6523 .remove = bnxt_remove_one,
6316ea6d 6524 .err_handler = &bnxt_err_handler,
c0c050c5
MC		 6525#if defined(CONFIG_BNXT_SRIOV)
6526 .sriov_configure = bnxt_sriov_configure,
6527#endif
6528};
6529
6530module_pci_driver(bnxt_pci_driver);
Linux kernel - Deliverables
This page took 0.664414 seconds and 5 git commands to generate.